List of usage examples for org.apache.hadoop.io SortedMapWritable entrySet
@Override
public Set<Map.Entry<K, Writable>> entrySet()
From source file:Assignment4_P4_MemoryConscious.MovieRatingMemConscious_Reducer.java
public void reduce(IntWritable key, Iterable<SortedMapWritable> values, Context context) throws IOException, InterruptedException { int running_count = 0; long running_sum = 0; float median = 0; list = new TreeMap<>(); // loop through all ratings in received hashmap for this movieID for (SortedMapWritable val : values) { // iterate through every entry consisting of movieRating, countOfRating for ex (4.5, 10) i.e 10 people rated it 4.5 for (Map.Entry<WritableComparable, Writable> entry : val.entrySet()) { // extract movieRating for ex : 4.5 FloatWritable number = (FloatWritable) entry.getKey(); float movieRating = number.get(); //extract countOfRating for ex : 10 LongWritable counter = (LongWritable) entry.getValue(); long count = counter.get(); // calculate running sum by multiplying movieRating and countRating i.e (4.5 * 10 = 45) running_sum += (movieRating * count); // increment running count for getting average later i.e 10 running_count += count;/* w ww . j ava2 s . c o m*/ // make <count> entries for <movieRating> in new hashmap i.e (4.5, 10) if (list.containsKey(movieRating)) { list.put(movieRating, list.get(movieRating) + count); } else { list.put(movieRating, count); } } } System.out.println("Running count for movieID " + key + " is :- " + running_count); System.out.println("Rating List size for movieID " + key + " is :- " + list.size()); // calculating mean float mean = running_sum / running_count; System.out.println("Mean for movieID " + key + " is :- " + mean); // calculating standard deviation float sumSquare = 0; float stdDev = 0; for (Map.Entry<Float, Long> entry : list.entrySet()) { sumSquare += (entry.getKey() - mean) * (entry.getKey() - mean) * (entry.getValue()); } // finally, std dev stdDev = (float) Math.sqrt(sumSquare / (running_count - 1)); System.out.println("Standard deviation for movieID " + key + " is :- " + stdDev); //.append(median) String outcome = new StringBuilder().append("\t").append(stdDev).append("\t").append(running_sum) .append("\t").append(running_count).toString(); result = new Text(outcome); context.write(key, result); }
From source file:Assignment4_P4_MemoryConscious.MovingRatingMemConscious_Combiner.java
public void reduce(IntWritable key, Iterable<SortedMapWritable> values, Context context) throws IOException, InterruptedException { // loop through each hashmap for this movie id for (SortedMapWritable val : values) { // inside each hashmap, loop for every entry for (Map.Entry<WritableComparable, Writable> entry : val.entrySet()) { // check if current entry's key is already present in new hashmap if (result.containsKey(entry.getKey())) { //if yes, extract current value from result hashmap for this key LongWritable existingValue = (LongWritable) result.get(entry.getKey()); // increment existing value by 1 existingValue.set(existingValue.get() + 1); // update result hashmap with new value result.put(entry.getKey(), existingValue); } else { //if not, create new entry with init value 1 result.put(entry.getKey(), new LongWritable(1)); }/*from www. j a v a 2 s. c o m*/ } val.clear(); } context.write(key, result); }
From source file:com.jfolson.hive.serde.RTypedBytesWritableOutput.java
License:Apache License
public void writeSortedMap(SortedMapWritable smw) throws IOException { out.writeMapHeader(smw.size());//from www . java 2s . c o m for (Map.Entry<WritableComparable, Writable> entry : smw.entrySet()) { write(entry.getKey()); write(entry.getValue()); } }
From source file:io.apigee.lembos.mapreduce.converters.input.SortedMapWritableConverter.java
License:Apache License
/** * Takes in a {@link SortedMapWritable} and returns a {@link Scriptable} map. * * @param scope the JavaScript scope/*from ww w . j a v a 2 s . c om*/ * @param writable the value to convert * * @return the {@link Scriptable} map equivalent */ @Override public Object toJavaScript(final Scriptable scope, final SortedMapWritable writable) { final Map<Object, Object> writableMap = new HashMap<>(); for (final Map.Entry<WritableComparable, Writable> mapEntry : writable.entrySet()) { writableMap.put(ConversionUtils.writableComparableToJS(mapEntry.getKey(), scope), ConversionUtils.writableToJS(mapEntry.getValue(), scope)); } return JavaScriptUtils.asObject(scope, writableMap); }
From source file:org.eobjects.hadoopdatacleaner.datastores.CsvParser.java
License:Open Source License
public static Text toCsvText(SortedMapWritable row) { Text finalText = new Text(); for (@SuppressWarnings("rawtypes") Iterator<Entry<WritableComparable, Writable>> iterator = row.entrySet().iterator(); iterator.hasNext();) { @SuppressWarnings("rawtypes") Entry<WritableComparable, Writable> next = iterator.next(); if (next.getValue() instanceof Text) { Text value = ((Text) next.getValue()); finalText.set(finalText.toString() + value.toString()); } // else do not append anything - the value is null, so empty. if (iterator.hasNext()) finalText.set(finalText.toString() + ";"); else/* w w w. ja va2 s . co m*/ finalText.set(finalText.toString()); } return finalText; }
From source file:org.eobjects.hadoopdatacleaner.datastores.ResultUtils.java
License:Open Source License
public static Result sortedMapWritableToResult(SortedMapWritable row) { List<Cell> cells = new ArrayList<Cell>(); for (@SuppressWarnings("rawtypes") Map.Entry<WritableComparable, Writable> rowEntry : row.entrySet()) { Text columnFamilyAndName = (Text) rowEntry.getKey(); Text columnValue = (Text) rowEntry.getValue(); String[] split = columnFamilyAndName.toString().split(":"); String columnFamily = split[0]; String columnName = split[1]; Cell cell = new KeyValue(Bytes.toBytes(columnValue.toString()), Bytes.toBytes(columnFamily), Bytes.toBytes(columnName), Bytes.toBytes(columnValue.toString())); cells.add(cell);//from w w w. ja va 2s. com } return Result.create(cells); }
From source file:org.eobjects.hadoopdatacleaner.datastores.RowUtils.java
License:Open Source License
public static void printSortedMapWritable(SortedMapWritable row, Logger logger) { logger.info("Row: "); for (@SuppressWarnings("rawtypes") Map.Entry<WritableComparable, Writable> entry : row.entrySet()) { Text columnName = (Text) entry.getKey(); if (entry.getValue() instanceof Text) { Text columnValue = (Text) entry.getValue(); logger.info("\t" + columnName + " = " + columnValue); } else {//from w w w . ja va 2 s . c om logger.info("\t" + columnName + " = " + null); } } }
From source file:org.eobjects.hadoopdatacleaner.datastores.RowUtils.java
License:Open Source License
public static InputRow sortedMapWritableToInputRow(SortedMapWritable rowWritable, Collection<InputColumn<?>> sourceColumns) { MockInputRow inputRow = new MockInputRow(); for (@SuppressWarnings("rawtypes") Map.Entry<WritableComparable, Writable> rowEntry : rowWritable.entrySet()) { Text columnName = (Text) rowEntry.getKey(); if (rowEntry.getValue() instanceof Text) { Text columnValue = (Text) rowEntry.getValue(); for (InputColumn<?> sourceColumn : sourceColumns) { String sourceColumnName = sourceColumn.getName(); if (sourceColumnName.equals(columnName.toString())) { inputRow.put(sourceColumn, columnValue.toString()); break; }//from w w w .j a v a 2 s. co m } } else { for (InputColumn<?> sourceColumn : sourceColumns) { String sourceColumnName = sourceColumn.getName(); if (sourceColumnName.equals(columnName.toString())) { inputRow.put(sourceColumn, null); break; } } } } return inputRow; }
From source file:org.eobjects.hadoopdatacleaner.mapreduce.hbase.HBaseTableMapperTest.java
License:Open Source License
@Test public void testDenmark() throws IOException { ImmutableBytesWritable inputKey = new ImmutableBytesWritable(Bytes.toBytes("Denmark")); List<Cell> cells = new ArrayList<Cell>(); Cell cell = new KeyValue(Bytes.toBytes("Denmark"), Bytes.toBytes("mainFamily"), Bytes.toBytes("country_name"), Bytes.toBytes("Denmark")); cells.add(cell);/*from ww w . jav a 2s. c om*/ cell = new KeyValue(Bytes.toBytes("Denmark"), Bytes.toBytes("mainFamily"), Bytes.toBytes("iso2"), Bytes.toBytes("DK")); cells.add(cell); cell = new KeyValue(Bytes.toBytes("Denmark"), Bytes.toBytes("mainFamily"), Bytes.toBytes("iso3"), Bytes.toBytes("DNK")); cells.add(cell); Result inputResult = Result.create(cells); SortedMapWritable expectedOutput = new SortedMapWritable(); expectedOutput.put(new Text("mainFamily:country_name"), new Text("Denmark")); expectedOutput.put(new Text("mainFamily:iso2"), new Text("DK")); expectedOutput.put(new Text("mainFamily:iso2_iso3"), new Text("DK_DNK")); expectedOutput.put(new Text("mainFamily:iso3"), new Text("DNK")); String expectedAnalyzerKey1 = "Value distribution (mainFamily:country_name)"; String expectedAnalyzerKey2 = "Value distribution (mainFamily:iso2)"; mapDriver.withInput(inputKey, inputResult); List<Pair<Text, SortedMapWritable>> actualOutputs = mapDriver.run(); Assert.assertEquals(actualOutputs.size(), 2); Pair<Text, SortedMapWritable> actualOutput1 = actualOutputs.get(0); Pair<Text, SortedMapWritable> actualOutput2 = actualOutputs.get(1); Assert.assertEquals(expectedAnalyzerKey1, actualOutput1.getFirst().toString()); Assert.assertEquals(expectedAnalyzerKey2, actualOutput2.getFirst().toString()); for (@SuppressWarnings("rawtypes") Map.Entry<WritableComparable, Writable> mapEntry : expectedOutput.entrySet()) { Text expectedColumnName = (Text) mapEntry.getKey(); Text expectedColumnValue = (Text) mapEntry.getValue(); Assert.assertTrue(actualOutput1.getSecond().containsKey(expectedColumnName)); Assert.assertEquals(expectedColumnValue, actualOutput1.getSecond().get(expectedColumnName)); Assert.assertTrue(actualOutput2.getSecond().containsKey(expectedColumnName)); Assert.assertEquals(expectedColumnValue, actualOutput2.getSecond().get(expectedColumnName)); } }
From source file:org.huahinframework.core.io.Key.java
License:Apache License
/** * set new sort/* ww w. ja v a 2 s. c om*/ * @param smw sort */ @SuppressWarnings("rawtypes") public void setSort(SortedMapWritable smw) { for (Entry<WritableComparable, Writable> entry : smw.entrySet()) { KeyDetail kd = (KeyDetail) entry.getKey(); kd.setOrder(++order); writableMap.put(kd, entry.getValue()); } }