List of usage examples for org.apache.hadoop.io IntWritable get
public int get()
From source file:com.linkedin.cubert.io.CompactWritablesDeserializer.java
License:Open Source License
@SuppressWarnings("unchecked") @Override// w w w.jav a 2s. co m public K deserialize(K object) throws IOException { if (in.available() == 0) throw new IOException(); Tuple t = (Tuple) object; if (t == null) { t = TupleFactory.getInstance().newTuple(datatypes.length); } for (int i = 0; i < datatypes.length; i++) { Object field = null; switch (datatypes[i]) { case BOOLEAN: { IntWritable iw = VariableLengthEncoder.decodeInteger(in); if (iw != null) { ((BooleanWritable) writables[i]).set(iw.get() == 1); field = writables[i]; } break; } case BYTE: { IntWritable iw = VariableLengthEncoder.decodeInteger(in); if (iw != null) { ((ByteWritable) writables[i]).set((byte) iw.get()); field = writables[i]; } break; } case DOUBLE: { DoubleWritable dw = VariableLengthEncoder.decodeDouble(in); if (dw != null) { ((DoubleWritable) writables[i]).set(dw.get()); field = writables[i]; } break; } case FLOAT: { FloatWritable fw = VariableLengthEncoder.decodeFloat(in); if (fw != null) { ((FloatWritable) writables[i]).set(fw.get()); field = writables[i]; } break; } case INT: { IntWritable iw = VariableLengthEncoder.decodeInteger(in); if (iw != null) { ((IntWritable) writables[i]).set(iw.get()); field = writables[i]; } break; } case LONG: { LongWritable lw = VariableLengthEncoder.decodeLong(in); if (lw != null) { ((LongWritable) writables[i]).set(lw.get()); field = writables[i]; } break; } case STRING: { IntWritable iw = VariableLengthEncoder.decodeInteger(in); if (iw != null) { int length = iw.get(); if (length > buffer.length) buffer = new byte[2 * buffer.length]; in.read(buffer, 0, length); ((Text) writables[i]).set(new String(buffer, 0, length)); field = writables[i]; } break; } default: throw new RuntimeException("Cannot deserialize column of type " + datatypes[i]); } t.set(i, field); } return (K) t; }
From source file:com.m6d.filecrush.crush.CrushTest.java
License:Apache License
@Test public void bucketing() throws Exception { File in = tmp.newFolder("in"); Counters expectedCounters = new Counters(); List<String> expectedBucketFiles = new ArrayList<String>(); /*//w w w . j a va2 s . co m * Create a hierarchy of directories. Directories are distinguished by a trailing slash in these comments. * * 1/ * 1.1/ * file1 10 bytes * file2 20 bytes * file3 30 bytes * file4 41 bytes * file5 15 bytes * file6 30 bytes * file7 20 bytes * 1.2/ * file1 20 bytes * file2 10 bytes * 1.3/ * 2/ * file1 70 bytes * file2 30 bytes * file3 25 bytes * file4 30 bytes * file5 35 bytes * 2.1/ * file1 10 bytes * 2.2/ * file1 25 bytes * file2 15 bytes * file3 35 bytes * 2.3/ * file1 41 bytes * file2 10 bytes * 2.4/ * 2.4.1/ * file1 100 bytes * file2 30 bytes * 2.4.2/ * file1 20 bytes * file2 20 bytes * file3 10 bytes */ /* * in contains 2 dirs and no files so it is skipped. * * in/ * 1/ * 2/ */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); tmp.newFolder("in/1"); File dir2 = tmp.newFolder("in/2"); /* * in/1 contains three dirs and no files so it is skipped. * * in/ * 1/ * 1.1/ * 1.2/ * 1.3/ */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); File dir1_1 = tmp.newFolder("in/1/1.1"); File dir1_2 = tmp.newFolder("in/1/1.2"); tmp.newFolder("in/1/1.3"); /* * in/2 contains five files and four dirs. * * in/ * 2/ * file1 70 bytes * file2 30 bytes * file3 25 bytes * file4 30 bytes * file5 35 bytes * 2.1/ * 2.2/ * 2.3/ * 2.4/ * * 0 1 2 * file5 35 file2 30 file4 30 * file3 25 * * Buckets 0 and 2 have a single file each so they are ignored. */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_ELIGIBLE, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 5); expectedCounters.incrCounter(MapperCounter.FILES_ELIGIBLE, 2); expectedCounters.incrCounter(MapperCounter.FILES_SKIPPED, 3); File dir2_1 = tmp.newFolder("in/2/2.1"); File dir2_2 = tmp.newFolder("in/2/2.2"); File dir2_3 = tmp.newFolder("in/2/2.3"); tmp.newFolder("in/2/2.4"); createFile(dir2, "file1", 70); createFile(dir2, "file2", 30); createFile(dir2, "file3", 25); createFile(dir2, "file4", 30); createFile(dir2, "file5", 35); expectedBucketFiles .add(format("%s %s", dir2.getAbsolutePath() + "-1", new File(dir2, "file2").getAbsolutePath())); expectedBucketFiles .add(format("%s %s", dir2.getAbsolutePath() + "-1", new File(dir2, "file3").getAbsolutePath())); /* * in/1/1.1 contains seven files and no dirs. * * in/ * 1/ * 1.1/ * file1 10 bytes * file2 20 bytes * file3 30 bytes * file4 41 bytes * file5 15 bytes * file6 30 bytes * file7 20 bytes * * 0 1 2 * file3 30 file6 30 file2 20 * file5 15 file1 10 file7 20 * * file4 is > 50 * 0.8 so it is ignored. */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_ELIGIBLE, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 7); expectedCounters.incrCounter(MapperCounter.FILES_ELIGIBLE, 6); expectedCounters.incrCounter(MapperCounter.FILES_SKIPPED, 1); createFile(dir1_1, "file1", 10); createFile(dir1_1, "file2", 20); createFile(dir1_1, "file3", 30); createFile(dir1_1, "file4", 41); createFile(dir1_1, "file5", 15); createFile(dir1_1, "file6", 30); createFile(dir1_1, "file7", 20); expectedBucketFiles.add( format("%s %s", dir1_1.getAbsolutePath() + "-0", new File(dir1_1, "file3").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir1_1.getAbsolutePath() + "-0", new File(dir1_1, "file5").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir1_1.getAbsolutePath() + "-1", new File(dir1_1, "file6").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir1_1.getAbsolutePath() + "-1", new File(dir1_1, "file1").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir1_1.getAbsolutePath() + "-2", new File(dir1_1, "file2").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir1_1.getAbsolutePath() + "-2", new File(dir1_1, "file7").getAbsolutePath())); /* * in/1/1.2 contains to files. * * in/ * 1/ * 1.2/ * file1 20 bytes * file2 10 bytes */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_ELIGIBLE, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 2); expectedCounters.incrCounter(MapperCounter.FILES_ELIGIBLE, 2); createFile(dir1_2, "file1", 20); createFile(dir1_2, "file2", 10); expectedBucketFiles.add( format("%s %s", dir1_2.getAbsolutePath() + "-0", new File(dir1_2, "file1").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir1_2.getAbsolutePath() + "-0", new File(dir1_2, "file2").getAbsolutePath())); /* * in/1/1.3 is empty. * * in/ * 1/ * 1.3/ */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); tmp.newFolder("in/1/1.3"); /* * in/2/2.1 contains on file. * * in/ * 2/ * 2.1/ * file1 10 bytes * * Single file dirs are ignored. */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 1); expectedCounters.incrCounter(MapperCounter.FILES_SKIPPED, 1); createFile(dir2_1, "file1", 10); /* * in/2/2.2 contains three files. * * in/ * 2/ * 2.2/ * file1 25 bytes * file2 15 bytes * file3 35 bytes * * 0 1 * file3 35 file1 25 * file2 15 * * Bucket 0 with a single file is ignored. */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_ELIGIBLE, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 3); expectedCounters.incrCounter(MapperCounter.FILES_ELIGIBLE, 2); expectedCounters.incrCounter(MapperCounter.FILES_SKIPPED, 1); createFile(dir2_2, "file1", 25); createFile(dir2_2, "file2", 15); createFile(dir2_2, "file3", 35); expectedBucketFiles.add( format("%s %s", dir2_2.getAbsolutePath() + "-1", new File(dir2_2, "file1").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir2_2.getAbsolutePath() + "-1", new File(dir2_2, "file2").getAbsolutePath())); /* * in/2/2.3 contains 2 files. * * in/ * 2/ * 2.3/ * file1 41 bytes * file2 10 bytes * * file1 is too big and leaving file2 as a single file, which is also ignored. */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 2); expectedCounters.incrCounter(MapperCounter.FILES_SKIPPED, 2); createFile(dir2_3, "file1", 41); createFile(dir2_3, "file2", 10); /* * in/2/2.4 contains two sub directories and no files. * * in/ * 2/ * 2.4/ * 2.4.1/ * 2.4.2/ */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); tmp.newFolder("in/2/2.4"); File dir2_4_1 = tmp.newFolder("in/2/2.4/2.4.1"); File dir2_4_2 = tmp.newFolder("in/2/2.4/2.4.2"); /* * in/ * 2/ * 2.4/ * 2.4.1/ * file1 100 bytes * file2 30 bytes */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 2); expectedCounters.incrCounter(MapperCounter.FILES_SKIPPED, 2); createFile(dir2_4_1, "file1", 100); createFile(dir2_4_1, "file2", 30); /* * in/ * 2/ * 2.4/ * 2.4.2/ * file1 20 bytes * file2 20 bytes * file3 10 bytes * 0 * file1 20 * file2 20 * file3 10 */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_ELIGIBLE, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 3); expectedCounters.incrCounter(MapperCounter.FILES_ELIGIBLE, 3); createFile(dir2_4_2, "file1", 20); createFile(dir2_4_2, "file2", 20); createFile(dir2_4_2, "file3", 10); expectedBucketFiles.add(format("%s %s", dir2_4_2.getAbsolutePath() + "-0", new File(dir2_4_2, "file1").getAbsolutePath())); expectedBucketFiles.add(format("%s %s", dir2_4_2.getAbsolutePath() + "-0", new File(dir2_4_2, "file2").getAbsolutePath())); expectedBucketFiles.add(format("%s %s", dir2_4_2.getAbsolutePath() + "-0", new File(dir2_4_2, "file3").getAbsolutePath())); Crush crush = new Crush(); crush.setConf(job); crush.setFileSystem(fileSystem); /* * Call these in the same order that run() does. */ crush.createJobConfAndParseArgs("--compress=none", "--max-file-blocks=1", in.getAbsolutePath(), new File(tmp.getRoot(), "out").getAbsolutePath(), "20101124171730"); crush.writeDirs(); /* * Verify bucket contents. */ List<String> actualBucketFiles = new ArrayList<String>(); Text key = new Text(); Text value = new Text(); Reader reader = new Reader(FileSystem.get(job), crush.getBucketFiles(), job); while (reader.next(key, value)) { actualBucketFiles.add(format("%s\t%s", key, value)); } reader.close(); Collections.sort(expectedBucketFiles); Collections.sort(actualBucketFiles); assertThat(actualBucketFiles, equalTo(expectedBucketFiles)); /* * Verify the partition map. */ Reader partitionMapReader = new Reader(FileSystem.get(job), crush.getPartitionMap(), job); IntWritable partNum = new IntWritable(); Map<String, Integer> actualPartitions = new HashMap<String, Integer>(); while (partitionMapReader.next(key, partNum)) { actualPartitions.put(key.toString(), partNum.get()); } partitionMapReader.close(); /* * These crush files need to allocated into 5 partitions: * * in/2-1 55 bytes * in/1/1.1-0 45 bytes * in/1/1.1-2 40 bytes * in/1/1.1-1 40 bytes * in/1/1.2-0 30 bytes * in/2/2.2-1 40 bytes * in/2/2.4/2.4.2-0 50 bytes * * 0 1 2 3 4 * in/2-1 55 in/2/2.4/2.4.2-0 50 in/1/1.1-0 45 in/1/1.1-2 40 in/1/1.1-1 40 * in/2/2.2-1 40 in/1/1.2-0 39 */ Map<String, Integer> expectedPartitions = new HashMap<String, Integer>(); //TODO: this may not be deterministic due to jvm/hashmap/filesystem expectedPartitions.put(dir2.getAbsolutePath() + "-1", 0); expectedPartitions.put(dir2_4_2.getAbsolutePath() + "-0", 1); expectedPartitions.put(dir1_1.getAbsolutePath() + "-0", 2); expectedPartitions.put(dir1_1.getAbsolutePath() + "-2", 4); expectedPartitions.put(dir2_2.getAbsolutePath() + "-1", 3); expectedPartitions.put(dir1_1.getAbsolutePath() + "-1", 3); expectedPartitions.put(dir1_2.getAbsolutePath() + "-0", 4); assertThat(actualPartitions, equalTo(expectedPartitions)); /* * Verify counters. */ Counters actualCounters = new Counters(); DataInputStream countersStream = FileSystem.get(job).open(crush.getCounters()); actualCounters.readFields(countersStream); countersStream.close(); assertThat(actualCounters, equalTo(expectedCounters)); }
From source file:com.malsolo.hadoop.elephant.guide.MaxTemperatureReducer.java
@Override protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int maxValue = Integer.MIN_VALUE; for (IntWritable value : values) { maxValue = Math.max(maxValue, value.get()); }/*from ww w. j a v a 2s . c o m*/ context.write(key, new IntWritable(maxValue)); }
From source file:com.marcolotz.MRComponents.SerializerConverter.java
License:Creative Commons License
/*** * Reads an Int from the input// ww w. ja v a 2s. co m * @param datainput * @return the int readen * @throws IOException */ public static int readInt(DataInput datainput) throws IOException { IntWritable readenInt = new IntWritable(); readenInt.readFields(datainput); return readenInt.get(); }
From source file:com.mellanox.hadoop.mapred.UdaShuffleConsumerPlugin.java
License:Apache License
@Override // callback from UdaConsumerPluginCallable public MapTaskCompletionEventsUpdate pluginGetMapCompletionEvents(IntWritable fromEventId, int maxEventsToFetch) throws IOException { return udaPlugin.umbilical.getMapCompletionEvents(udaPlugin.reduceTask.getJobID(), fromEventId.get(), maxEventsToFetch, udaPlugin.reduceTask.getTaskID() //, udaPlugin.reduceTask.getJvmContext() - this was for hadoop-1.x );/*from w w w. j av a 2s . c om*/ }
From source file:com.mongodb.hadoop.examples.enron.EnronMailReducer.java
License:Apache License
@Override public void reduce(final MailPair pKey, final Iterable<IntWritable> pValues, final Context pContext) throws IOException, InterruptedException { int sum = 0;//w w w. j a v a2s. co m for (final IntWritable value : pValues) { sum += value.get(); } BSONObject outDoc = BasicDBObjectBuilder.start().add("f", pKey.from).add("t", pKey.to).get(); BSONWritable pkeyOut = new BSONWritable(outDoc); pContext.write(pkeyOut, new IntWritable(sum)); }
From source file:com.mongodb.hadoop.examples.lolmatches.LOLMatchesdReducer.java
License:Apache License
@Override public void reduce(final IntWritable pKey, final Iterable<DoubleWritable> pValues, final Context pContext) throws IOException, InterruptedException { int count = 0; double sum = 0; for (final DoubleWritable value : pValues) { sum += value.get();//from w ww . jav a 2 s . c o m count++; } final double avg = sum / count; if (LOG.isDebugEnabled()) { LOG.debug("Average Kills for hour" + pKey.get() + " was " + avg); } BasicBSONObject output = new BasicBSONObject(); output.put("count", count); output.put("avg", avg); output.put("sum", sum); reduceResult.setDoc(output); pContext.write(pKey, reduceResult); }
From source file:com.mongodb.hadoop.examples.lolmatches.LOLMatchesdReducer.java
License:Apache License
@Override public void reduce(final IntWritable key, final Iterator<DoubleWritable> values, final OutputCollector<IntWritable, BSONWritable> output, final Reporter reporter) throws IOException { int count = 0; double sum = 0; while (values.hasNext()) { sum += values.next().get();/*from w w w . ja v a 2 s. c om*/ count++; } final double avg = sum / count; if (LOG.isDebugEnabled()) { LOG.debug("Average Kills for hour " + key.get() + " was " + avg); } BasicBSONObject bsonObject = new BasicBSONObject(); bsonObject.put("count", count); bsonObject.put("avg", avg); bsonObject.put("sum", sum); reduceResult.setDoc(bsonObject); output.collect(key, reduceResult); }
From source file:com.mongodb.hadoop.examples.treasury.TreasuryYieldReducer.java
License:Apache License
@Override public void reduce(final IntWritable pKey, final Iterable<DoubleWritable> pValues, final Context pContext) throws IOException, InterruptedException { int count = 0; double sum = 0; for (final DoubleWritable value : pValues) { sum += value.get();/*www. j av a 2 s . co m*/ count++; } final double avg = sum / count; LOG.debug("Average 10 Year Treasury for " + pKey.get() + " was " + avg); BasicBSONObject output = new BasicBSONObject(); output.put("count", count); output.put("avg", avg); output.put("sum", sum); pContext.write(pKey, new BSONWritable(output)); }
From source file:com.mongodb.hadoop.examples.treasury.TreasuryYieldReducer.java
License:Apache License
@Override public void reduce(final IntWritable key, final Iterator<DoubleWritable> values, final OutputCollector<IntWritable, BSONWritable> output, final Reporter reporter) throws IOException { int count = 0; double sum = 0; while (values.hasNext()) { sum += values.next().get();/*from w w w .j av a2s .co m*/ count++; } final double avg = sum / count; LOG.debug("Average 10 Year Treasury for " + key.get() + " was " + avg); BasicBSONObject bsonObject = new BasicBSONObject(); bsonObject.put("count", count); bsonObject.put("avg", avg); bsonObject.put("sum", sum); output.collect(key, new BSONWritable(bsonObject)); }