List of usage examples for org.apache.hadoop.io LongWritable LongWritable
public LongWritable(long value)
From source file:com.yolodata.tbana.cascading.splunk.SplunkSchemeTest.java
License:Open Source License
private void checkResults(Tuple actual, String row) { String[] rowValues = row.split(","); Tuple expected = new Tuple(new LongWritable(Long.parseLong(rowValues[0])), new Text(rowValues[1])); assertEquals(expected, new Tuple(actual.get(new int[] { 0, 4 }))); }
From source file:com.yolodata.tbana.hadoop.mapred.splunk.recordreader.ParallelRecordReadersTest.java
License:Open Source License
private void addKVToMap(Map expected, int key, String value) { ArrayListTextWritable texts = new ArrayListTextWritable(); texts.add(new Text(value)); expected.put(new LongWritable(key), texts); }
From source file:computation.test.SimpleComputationTest.java
License:MIT License
/** * Test superstep0//w w w . j a v a 2 s. co m */ @Test public void testSuperstep0() throws Exception { Vertex<LongWritable, VertexValuesWritable, EdgeValuesWritable> vertex = new DefaultVertex<LongWritable, VertexValuesWritable, EdgeValuesWritable>(); SimpleComputation computation = new SimpleComputation(); SimpleMasterCompute master = new SimpleMasterCompute(); // computation.setAggregator(mockAggregator); // MockUtils.prepareVertexAndComputation(vertex, new LongWritable(1), // // new VertexValuesWritable(), false, // computation, 0L); vertex.setValue(new VertexValuesWritable()); vertex.addEdge(EdgeFactory.create(new LongWritable(2), new EdgeValuesWritable())); vertex.addEdge(EdgeFactory.create(new LongWritable(3), new EdgeValuesWritable())); computation.compute(vertex, new ArrayList<MessageWritable>()); assertTrue(vertex.isHalted()); //cool should not be called in superstep0 //verify(computation, never()).aggregate("T", new DoubleWritable(-100)); //random positions should be between -50 and 50 assertEquals(0, vertex.getValue().getCoordinate().get().x, 50); assertEquals(0, vertex.getValue().getCoordinate().get().y, 50); }
From source file:computation.test.SimpleComputationTest.java
License:MIT License
/** * Test the messages after finishes superstep1 /*from ww w .j a v a 2 s . c o m*/ */ @Test public void testSuperstep1() throws Exception { //messages: positions of other vertices ArrayList<MessageWritable> messages = new ArrayList<MessageWritable>(); messages.add(new MessageWritable(new LongWritable(2), new CoordinateWritable(), new BooleanWritable())); messages.add(new MessageWritable(new LongWritable(3), new CoordinateWritable(), new BooleanWritable())); Vertex<LongWritable, VertexValuesWritable, EdgeValuesWritable> vertex = new DefaultVertex<LongWritable, VertexValuesWritable, EdgeValuesWritable>(); SimpleComputation computation = new SimpleComputation(); SimpleMasterCompute master = new SimpleMasterCompute(); master.initialize(); MockUtils.MockedEnvironment<LongWritable, VertexValuesWritable, EdgeValuesWritable, MessageWritable> env = MockUtils.prepareVertexAndComputation(vertex, new LongWritable(1L), new VertexValuesWritable(), true, computation, 1L); vertex.setValue(new VertexValuesWritable(new CoordinateWritable(), new DoubleWritable(0))); vertex.addEdge(EdgeFactory.create(new LongWritable(2L), new EdgeValuesWritable())); vertex.addEdge(EdgeFactory.create(new LongWritable(3L), new EdgeValuesWritable())); computation.compute(vertex, messages); assertTrue(vertex.isHalted()); env.verifyMessageSent(new LongWritable(2), new MessageWritable(new LongWritable(1), vertex.getValue().getCoordinate(), new BooleanWritable())); env.verifyMessageSent(new LongWritable(3), new MessageWritable(new LongWritable(1), vertex.getValue().getCoordinate(), new BooleanWritable())); }
From source file:core.advanced.ConnectedComponentsVertex.java
License:Apache License
/** * Propagates the smallest vertex id to all neighbors. Will always choose to * halt and only reactivate if a smaller id has been sent to it. * * @param messages Iterator of messages from the previous superstep. * @throws IOException// w w w. j a v a 2s .c o m */ @Override public void compute(Iterable<LongWritable> messages) throws IOException { long currentComponent = getValue().get(); // First superstep is special, because we can simply look at the neighbors if (getSuperstep() == 0) { for (Edge<LongWritable, NullWritable> edge : getEdges()) { long neighbor = edge.getTargetVertexId().get(); if (neighbor < currentComponent) { currentComponent = neighbor; } } // Only need to send value if it is not the own id if (currentComponent != getValue().get()) { setValue(new LongWritable(currentComponent)); for (Edge<LongWritable, NullWritable> edge : getEdges()) { LongWritable neighbor = edge.getTargetVertexId(); if (neighbor.get() > currentComponent) { sendMessage(neighbor, getValue()); } } } voteToHalt(); return; } boolean changed = false; // did we get a smaller id ? for (LongWritable message : messages) { long candidateComponent = message.get(); if (candidateComponent < currentComponent) { currentComponent = candidateComponent; changed = true; } } // propagate new component id to the neighbors if (changed) { setValue(new LongWritable(currentComponent)); sendMessageToAllEdges(getValue()); } voteToHalt(); }
From source file:corr.util.DummyDataToSeqFile.java
License:Apache License
/** * Export CSV file to SequentialFile.//from ww w . ja v a2s. co m * @param file Name of CSV file. * @throws Exception */ private static void export(String file) throws Exception { String delim = ","; BufferedReader reader = null; SequenceFile.Writer writer = null; try { Path path = toPath(file); Configuration conf = new Configuration(); LocalFileSystem fs = FileSystem.getLocal(conf); writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class, VectorWritable.class); reader = new BufferedReader(new FileReader(file)); String line = null; long counter = 0; while (null != (line = reader.readLine())) { if ("".equals(line)) continue; String[] tokens = line.split(delim); LongWritable key = new LongWritable(counter); VectorWritable val = toVector(tokens); writer.append(key, val); counter++; } } catch (Exception ex) { throw ex; } finally { if (null != reader) { try { reader.close(); } catch (Exception ex) { } } if (null != writer) { try { writer.close(); } catch (Exception ex) { } } } }
From source file:crunch.MaxTemperature.java
License:Apache License
@Test
public void mapWritable() throws IOException {
// vv MapWritableTest
MapWritable src = new MapWritable();
src.put(new IntWritable(1), new Text("cat"));
src.put(new VIntWritable(2), new LongWritable(163));
MapWritable dest = new MapWritable();
WritableUtils.cloneInto(dest, src);
assertThat((Text) dest.get(new IntWritable(1)), is(new Text("cat")));
assertThat((LongWritable) dest.get(new VIntWritable(2)), is(new LongWritable(163)));
// ^^ MapWritableTest
}/* w ww .j av a2 s. co m*/
From source file:de.kp.core.spade.hadoop.ItemsetWritable.java
License:Open Source License
public ItemsetWritable(Itemset itemset) { timestampWritable = new LongWritable(itemset.getTimestamp()); List<Item<?>> items = itemset.getItems(); ArrayList<ItemWritable> itemWritableList = new ArrayList<ItemWritable>(); for (int i = 0; i < items.size(); i++) { itemWritableList.add(new ItemWritable(items.get(i))); }//w ww.ja v a 2 s .co m this.itemlistWritable = new ArrayWritable(ItemWritable.class, itemWritableList.toArray(new Writable[itemWritableList.size()])); }
From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.deduplication.DeDuplicationTextOutputReducer.java
License:Apache License
@Override protected void reduce(Text key, Iterable<DocumentInfo> values, Context context) throws IOException, InterruptedException { List<DocumentInfo> documents = new ArrayList<DocumentInfo>(); //collect the values of each band#_bitString for (DocumentInfo v : values) { // we really need the copy here! DocumentInfo documentInfo = new DocumentInfo(); documentInfo.setDocSimHash(new LongWritable(v.getDocSimHash().get())); documentInfo.setDocLength(new IntWritable(v.getDocLength().get())); documentInfo.setDocID(new Text(v.getDocID().toString())); documentInfo.setDocLanguage(new Text(v.getDocLang().toString())); documents.add(documentInfo);/*from ww w. jav a 2 s . c o m*/ } //choose candidates for similarity check if (documents.size() >= 2) { //sort the list to be able to remove redundancies later Collections.sort(documents, new DocIDComparator()); // set the file name prefix String fileName = documents.get(0).getDocLang().toString(); multipleOutputs.write(NullWritable.get(), documents, fileName); } }
From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.deduplication.DocumentInfo.java
License:Apache License
public void createDocumentInfo(String commaSeparatedInfo) { String[] idLengSimHashOfDoc = commaSeparatedInfo.split(";"); String docID = idLengSimHashOfDoc[0].replaceAll("\\[", "").trim(); this.docID = new Text(docID); int docLength = Integer.valueOf(idLengSimHashOfDoc[1].trim()); this.docLength = new IntWritable(docLength); long docSimHash = Long.valueOf(idLengSimHashOfDoc[2].trim()); this.docSimHash = new LongWritable(docSimHash); String lang = idLengSimHashOfDoc[3].replaceAll("\\]", "").trim(); this.language = new Text(lang); }