List of usage examples for org.apache.hadoop.io LongWritable LongWritable
public LongWritable(long value)
From source file:com.cloudera.impala.IncompatibleUdfTest.java
License:Apache License
public LongWritable evaluate(LongWritable a, List<IntWritable> b) { if (a == null) return null; return new LongWritable(a.get()); }
From source file:com.cloudera.oryx.lambda.batch.ValueToWritableFunctionTest.java
License:Open Source License
@Test public void testFunction() { ValueToWritableFunction<Integer, Long> function = new ValueToWritableFunction<>(Integer.class, Long.class, IntWritable.class, LongWritable.class); Tuple2<Integer, Long> in = new Tuple2<>(3, 4L); Tuple2<Writable, Writable> out = function.call(in); assertEquals(new IntWritable(3), out._1()); assertEquals(new LongWritable(4L), out._2()); }
From source file:com.cloudera.oryx.lambda.batch.ValueWritableConverterTest.java
License:Open Source License
@Test public void testLong() { ValueWritableConverter<Long> converter = new ValueWritableConverter<>(Long.class, LongWritable.class); assertEquals(-1L, converter.fromWritable(new LongWritable(-1L)).longValue()); assertEquals(new LongWritable(1L), converter.toWritable(1L)); }
From source file:com.cloudera.sa.giraph.examples.ktrusses.KTrussVertex.java
License:Apache License
@Override public void compute(Iterable<Message> messages) throws IOException { // Retreive the current phase of processing which is set by the MasterCompute PhaseWritable writablePhase = getAggregatedValue(Constants.PHASE); Phase phase = writablePhase.get();/*from w w w . j av a2 s . co m*/ int k = getContext().getConfiguration().getInt(Constants.K, 3); /* * This phase reduces the graph to k-1 core, which reduces the amount of work to find trusses. */ if (phase == Phase.KCORE) { // Deactivate edges that can't be part of a truss because their degree is too low int degree = getActiveDegree(); if (degree < k - 1) { for (Edge<LongWritable, EdgeState> edge : getEdges()) { if (edge.getValue().isActive()) { EdgeState ev = getEdgeValue(edge.getTargetVertexId()); ev.setActive(false); setEdgeValue(edge.getTargetVertexId(), ev); aggregate(Constants.UPDATES, ONE); } } } } /* * This phase determines the active degree of a node (the number of nodes I connect to * after not considering edges which can't be part of a truss). The degree is used to make triangle * finding more efficient in later phases. */ if (phase == Phase.DEGREE) { // Determine active degree, store it in the node state and send it to all active neighbours int degree = getActiveDegree(); setValue(new VertexState(degree, -1, false)); sendMessageToAllActiveNodes(new Message(getId(), new IntWritable(degree))); } /* * This phase starts the process of finding triangles. Triangles could be found by any of the * three nodes, so to ensure we only find a triangle once, we pick one of the nodes to do the work. * Since we can pick which node does the work, we pick the node with lowest degree (fewest links), * which is efficient. If two nodes have the same degree, we pick the node with the lowest name. * * Triangles are found by asking one of the other nodes whether it connects to a third node. This * is done with a double-for-loop over the edges, considering only active edges. Again, we send the * query to the node that isn't the highest degree. */ if (phase == Phase.QUERY_FOR_CLOSING_EDGES) { //First, we need to process the degree information and store it on our edges. for (Message message : messages) { EdgeState edgeValue = getEdgeValue(message.getSource()); if (edgeValue == null) { edgeValue = new EdgeState(); } int targetDegree = message.getDegree().get(); edgeValue.setTargetDegree(targetDegree); setEdgeValue(message.getSource(), edgeValue); } // Now we find the possible triangles and send a query message to each. int myDegree = getValue().getDegree(); if (myDegree > 1) { for (Edge<LongWritable, EdgeState> neighbourA : getEdges()) { if (neighbourA.getValue().isActive()) { int neighbourADegree = neighbourA.getValue().getTargetDegree(); if (ordering(myDegree, getId().get(), neighbourADegree, neighbourA.getTargetVertexId().get())) { for (Edge<LongWritable, EdgeState> neighbourB : getEdges()) { if (neighbourB.getValue().isActive()) { int neighbourBDegree = neighbourB.getValue().getTargetDegree(); if (ordering(neighbourADegree, neighbourA.getTargetVertexId().get(), neighbourBDegree, neighbourB.getTargetVertexId().get())) { sendMessage(neighbourA.getTargetVertexId(), new Message(getId(), neighbourB.getTargetVertexId())); } } } } } } } } /* * In this phase, we process the queries about closing edges and find triangles if the are present. We then * send details of the triangle (which is only found by one node) to all nodes in that triangle. */ if (phase == Phase.FIND_TRIANGLES) { // Cache my active neighbour's names Set<Long> edges = new HashSet<Long>(); for (MutableEdge<LongWritable, EdgeState> edge : getMutableEdges()) { if (edge.getValue().isActive()) { edges.add(edge.getTargetVertexId().get()); } } // Find triangles and report them to participating nodes. for (Message message : messages) { if (edges.contains(message.getTriadA().get()) && edges.contains(message.getTriadB().get())) { Message triangle = new Message(getId(), message.getTriadA(), message.getTriadB()); sendMessage(getId(), triangle); sendMessage(message.getTriadA(), triangle); sendMessage(message.getTriadB(), triangle); } } } /* * In this phase, the node processes the triangle reports and uses that to build a picture of how * many triangles use each edge - this is known as the "support". We then use that to determine which edges * are part of a truss by having enough support, and deactivating those that don't. Once this phase is complete, * we evaluate (in the MasterCompute) whether or not the graph changed. If it did we iterate again, starting by * determining the active degree, otherwise we continue. * */ if (phase == Phase.TRUSSES) { // Clear out support values from previous iterations for (Edge<LongWritable, EdgeState> edge : getEdges()) { EdgeState ev = getEdgeValue(edge.getTargetVertexId()); ev.setSupport(0); setEdgeValue(edge.getTargetVertexId(), ev); } // Build a picture of the support of each edge for (Message message : messages) { incrementSupport(message.getTriangleA()); incrementSupport(message.getTriangleB()); incrementSupport(message.getTriangleC()); } /* * Go through our edges, marking any that don't have enough support as inactive. We also track how * many edges we deactivated. */ for (Edge<LongWritable, EdgeState> edge : getEdges()) { if (edge.getValue().isActive()) { if (edge.getValue().getSupport() < k - 2) { EdgeState ev = getEdgeValue(edge.getTargetVertexId()); ev.setActive(false); ev.setSupport(0); setEdgeValue(edge.getTargetVertexId(), ev); } } } } /* * This is the first componentisation phase. At this point, we know our neighbour's trussID * because it is initialised to be the same as their ID, so we use that to determine the lowest * local truss ID. Once that is found, we broadcast it in order to let the value propagate. */ if (phase == Phase.COMPONENTISATION_1) { boolean updated = false; boolean isInATruss = false; // Determine whether the current node is in a truss or not. for (Edge<LongWritable, EdgeState> edge : getEdges()) { if (edge.getValue().isActive()) { isInATruss = true; break; } } if (isInATruss) { // Determine lowest Truss ID long lowestId = getId().get(); for (Edge<LongWritable, EdgeState> edge : getEdges()) { if (edge.getValue().isActive()) { lowestId = Math.min(lowestId, edge.getTargetVertexId().get()); } } if (lowestId <= getId().get()) { getValue().setTrussID(lowestId); getValue().setInATruss(true); updated = true; } // Broadcast the lowest truss ID if we were updated with it if (updated) { sendMessageToAllActiveNodes(new Message(new LongWritable(getValue().getTrussID()))); } } } /* * This is the second phase of the componentisation, in which the lowest truss ID received is propagated. * If the truss ID is lowered, we again rebroadcast it and stay in this phase until there are no more updates. */ if (phase == Phase.COMPONENTISATION_2) { // Determine lowest Truss ID long lowestValue = getValue().getTrussID(); for (Message message : messages) { lowestValue = Math.min(lowestValue, message.getTrussID().get()); } boolean updated = false; if (lowestValue < getValue().getTrussID()) { getValue().setTrussID(lowestValue); aggregate(Constants.UPDATES, ONE); updated = true; } // Broadcast the lowest truss ID if we were updated with it if (updated) { sendMessageToAllActiveNodes(new Message(new LongWritable(getValue().getTrussID()))); } } /* * And we're done :) */ if (phase == Phase.OUTPUT) { voteToHalt(); } }
From source file:com.cloudera.sa.giraph.examples.ktrusses.Message.java
License:Apache License
public Message(LongWritable source, IntWritable degree) { this.type = Type.DEGREE_UPDATE; this.source = new LongWritable(source.get()); this.degree = new IntWritable(degree.get()); }
From source file:com.cloudera.sa.giraph.examples.ktrusses.Message.java
License:Apache License
public Message(LongWritable triadA, LongWritable triadB) { this.type = Type.OPEN_TRIAD; this.triadA = new LongWritable(triadA.get()); this.triadB = new LongWritable(triadB.get()); }
From source file:com.cloudera.sa.giraph.examples.ktrusses.Message.java
License:Apache License
public Message(LongWritable triangleA, LongWritable triangleB, LongWritable triangleC) { this.type = Type.TRIANGLE; this.triangleA = new LongWritable(triangleA.get()); this.triangleB = new LongWritable(triangleB.get()); this.triangleC = new LongWritable(triangleC.get()); }
From source file:com.cloudera.sa.giraph.examples.ktrusses.Message.java
License:Apache License
public Message(LongWritable trussID) { this.type = Type.TRUSS_ID; this.trussID = new LongWritable(trussID.get()); }
From source file:com.cloudera.sqoop.TestExport.java
License:Apache License
/** * Create a data file in SequenceFile format that gets exported to the db. * @param fileNum the number of the file (for multi-file export). * @param numRecords how many records to write to the file. * @param className the table class name to instantiate and populate * for each record./*w w w . ja v a 2 s . co m*/ */ private void createSequenceFile(int fileNum, int numRecords, String className) throws IOException { try { // Instantiate the value record object via reflection. Class cls = Class.forName(className, true, Thread.currentThread().getContextClassLoader()); SqoopRecord record = (SqoopRecord) ReflectionUtils.newInstance(cls, new Configuration()); // Create the SequenceFile. Configuration conf = new Configuration(); if (!BaseSqoopTestCase.isOnPhysicalCluster()) { conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); } FileSystem fs = FileSystem.get(conf); Path tablePath = getTablePath(); Path filePath = new Path(tablePath, "part" + fileNum); fs.mkdirs(tablePath); SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, filePath, LongWritable.class, cls); // Now write the data. int startId = fileNum * numRecords; for (int i = 0; i < numRecords; i++) { record.parse(getRecordLine(startId + i)); w.append(new LongWritable(startId + i), record); } w.close(); } catch (ClassNotFoundException cnfe) { throw new IOException(cnfe); } catch (RecordParser.ParseError pe) { throw new IOException(pe); } }
From source file:com.cloudera.traffic.TestTrafficAverager.java
License:Apache License
@Test public void testMapper() throws IOException { String line = "01/01/2012 00:00:00,311831,3,5,S,OR,,118,0,200,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"; mapDriver.withInput(new LongWritable(0), new Text(line)); Text outKey = new Text("311831_" + TimeUtil.toTimeOfWeek("01/01/2012 00:00:00")); AverageWritable outVal = new AverageWritable(); outVal.set(1, 200.0);//w w w. j a v a 2s .c o m mapDriver.withOutput(outKey, outVal); mapDriver.runTest(); }