List of usage examples for org.apache.hadoop.io LongWritable set
public void set(long value)
From source file:DeprecatedBAMBaseRecordReader.java
License:Open Source License
private boolean nextFromOldCurrent(LongWritable key, SAMBaseRecord base) { assert base.getParent() == current; if (!base.gotoNextBase()) return false; key.set((key.get() & ~0xffffffffL) | base.getPos()); return true;//from www . jav a 2 s. c o m }
From source file:DeprecatedBAMRecordReader.java
License:Open Source License
@Override public boolean next(LongWritable key, SAMRecordWritable value) { if (!rr.nextKeyValue()) return false; key.set(rr.getCurrentKey().get()); value.set(rr.getCurrentValue().get()); return true;/*from w w w.j av a2 s . c o m*/ }
From source file:Txt2SeqConverter.java
License:Apache License
public static void main(String[] args) { if (args.length != 2) { //System.out.println("Usage: env HADOOP_CLASSPATH=.:$HADOOP_CLASSPATH hadoop Txt2SeqConverter input output"); System.out.println("Usage: hadoop Txt2SeqConverter input output"); System.exit(1);//from w ww . j a v a2s . c om } FileSystem fs = null; String seqFileName = args[1]; Configuration conf = new Configuration(); try { fs = FileSystem.get(URI.create(seqFileName), conf); } catch (IOException e) { System.out.println("ERROR: " + e.getMessage()); } Path path = new Path(seqFileName); LongWritable key = new LongWritable(); Text value = new Text(); SequenceFile.Writer writer = null; try { //writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class, Text.class, SequenceFile.CompressionType.BLOCK); writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class, Text.class, SequenceFile.CompressionType.BLOCK, new com.hadoop.compression.lzo.LzoCodec()); BufferedReader br = new BufferedReader(new FileReader(args[0])); int transactionID = 0; String transaction = null; while ((transaction = br.readLine()) != null) { key.set(transactionID); value.set(transaction); writer.append(key, value); transactionID++; } } catch (IOException e) { System.out.println("ERROR: " + e.getMessage()); } finally { IOUtils.closeStream(writer); } }
From source file:TestHashMap.java
License:Apache License
@Test public void testHashSetString() throws Exception { final Set<String> hashSet = new HashSet<>(); final Random random = new Random(0xDEADBEEF); int matched = 0; LongWritable num = new LongWritable(); long startTime = System.nanoTime(); for (int i = 0; i < SET_SIZE; i++) { // input data is String String input = Long.toString(random.nextLong()); // disable optimizer if (input.length() > 5) { hashSet.add(input);//from w w w. j av a 2 s . co m } } random.setSeed(0xDEADBEEF); for (int i = 0; i < DATA_SIZE; i++) { // query data is LongWritable num.set(random.nextLong()); if (hashSet.contains(num.toString())) { matched++; } } long endTime = System.nanoTime(); System.out.println(" HashSet<String>"); System.out.println(" Elapsed time: " + (endTime - startTime) / 1000000 + " ms"); System.out.println(" Matched " + matched + " times"); }
From source file:TestHashMap.java
License:Apache License
@Test public void testHashSetLong() throws Exception { final Set<Long> hashSet = new HashSet<>(); final Random random = new Random(0xDEADBEEF); int matched = 0; LongWritable num = new LongWritable(); long startTime = System.nanoTime(); for (int i = 0; i < SET_SIZE; i++) { // input data is String String input = Long.toString(random.nextLong()); // disable optimizer if (input.length() > 5) { hashSet.add(Long.parseLong(input)); }//from w ww . j a v a 2 s .co m } random.setSeed(0xDEADBEEF); for (int i = 0; i < DATA_SIZE; i++) { // query data is LongWritable num.set(random.nextLong()); if (hashSet.contains(num.get())) { matched++; } } long endTime = System.nanoTime(); System.out.println(" HashSet<Long>"); System.out.println(" Elapsed time: " + (endTime - startTime) / 1000000f + " ms"); System.out.println(" Matched " + matched + " times"); }
From source file:Assignment4_P4_MemoryConscious.MovingRatingMemConscious_Combiner.java
public void reduce(IntWritable key, Iterable<SortedMapWritable> values, Context context) throws IOException, InterruptedException { // loop through each hashmap for this movie id for (SortedMapWritable val : values) { // inside each hashmap, loop for every entry for (Map.Entry<WritableComparable, Writable> entry : val.entrySet()) { // check if current entry's key is already present in new hashmap if (result.containsKey(entry.getKey())) { //if yes, extract current value from result hashmap for this key LongWritable existingValue = (LongWritable) result.get(entry.getKey()); // increment existing value by 1 existingValue.set(existingValue.get() + 1); // update result hashmap with new value result.put(entry.getKey(), existingValue); } else { //if not, create new entry with init value 1 result.put(entry.getKey(), new LongWritable(1)); }//from ww w. j a v a 2s .c o m } val.clear(); } context.write(key, result); }
From source file:ca.sparkera.adapters.mapred.MainframeVBRecordReader.java
License:Apache License
@Override public synchronized boolean next(LongWritable key, BytesWritable value) throws IOException { boolean dataRead = reader.nextKeyValue(); if (dataRead) { LongWritable newKey = reader.getCurrentKey(); BytesWritable newValue = reader.getCurrentValue(); key.set(newKey.get()); value.set(newValue);// w w w . j a va 2s .co m } return dataRead; }
From source file:ca.uwaterloo.iss4e.hive.UDAFPercentile.java
License:Apache License
/** * Increment the State object with o as the key, and i as the count. *//*from w w w. j a va 2 s .c o m*/ private static void increment(State s, LongWritable o, long i) { if (s.counts == null) { s.counts = new HashMap<LongWritable, LongWritable>(); } LongWritable count = s.counts.get(o); if (count == null) { // We have to create a new object, because the object o belongs // to the code that creates it and may get its value changed. LongWritable key = new LongWritable(); key.set(o.get()); s.counts.put(key, new LongWritable(i)); } else { count.set(count.get() + i); } }
From source file:com.alexholmes.hadooputils.sort.DelimitedLineRecordReader.java
License:Apache License
/** Read a line. */ public synchronized boolean next(LongWritable key, Text value) throws IOException { while (pos < end) { key.set(pos); int newSize = in.readLine(value, maxLineLength, Math.max((int) Math.min(Integer.MAX_VALUE, end - pos), maxLineLength)); if (newSize == 0) { return false; }//from ww w . ja va2s . c om pos += newSize; if (newSize < maxLineLength) { return true; } } return false; }
From source file:com.alexholmes.hadooputils.sort.LzoDelimitedLineRecordReader.java
License:Apache License
@Override public boolean next(LongWritable key, Text value) throws IOException { //since the lzop codec reads everything in lzo blocks //we can't stop if the pos == end //instead we wait for the next block to be read in when //pos will be > end while (pos <= end) { key.set(pos); int newSize = in.readLine(value); if (newSize == 0) { return false; }//from w w w .j a va 2 s . c o m pos = fileIn.getPos(); return true; } return false; }