List of usage examples for org.apache.hadoop.io LongWritable LongWritable
public LongWritable(long value)
From source file:fileformats.AuctionVertexValue.java
License:Apache License
public AuctionVertexValue(boolean r) { row = r;// ww w . j a va2 s. c o m N = 0; benefit = new double[N]; colOwned = new LongWritable(-1); rowOwnedBy = new LongWritable(-1); price = 0; }
From source file:fileformats.AuctionVertexValue.java
License:Apache License
public AuctionVertexValue(int n, boolean r) { row = r;// w ww. ja v a2 s . com N = n; benefit = new double[N]; colOwned = new LongWritable(-1); rowOwnedBy = new LongWritable(-1); price = 0; }
From source file:fileformats.AuctionVertexValue.java
License:Apache License
public AuctionVertexValue(int n, boolean r, double[] b) { row = r;// www. j av a 2 s . c o m N = n; benefit = b; colOwned = new LongWritable(-1); rowOwnedBy = new LongWritable(-1); price = 0; }
From source file:formats.test.TestJsonCustomVertexOutputFormat.java
License:MIT License
/** * Compare the json output by VertexWriter and by direct call of * Gson.toJson()// w ww .j a va2 s. co m * * @param toWrite * Used to populate Vertex in Giraph, and to produce json string * for comparison. * @throws IOException * @throws InterruptedException */ private void testWorker(VertexType toWrite) throws IOException, InterruptedException { TaskAttemptContext tac = mock(TaskAttemptContext.class); when(tac.getConfiguration()).thenReturn(conf); Vertex vertex = mock(Vertex.class); when(vertex.getId()).thenReturn(new LongWritable(toWrite.getId())); Coordinate coordinateToWrite = toWrite.getValues().getCoordinate(); double weightToWrite = toWrite.getValues().getWeight(); when(vertex.getValue()).thenReturn(new VertexValuesWritable(coordinateToWrite, weightToWrite)); List<Edge<LongWritable, EdgeValuesWritable>> edges = Lists .newArrayListWithCapacity(toWrite.getEdges().size()); for (int i = 0; i < toWrite.getEdges().size(); i++) { EdgeType edgeType = toWrite.getEdges().get(i); edges.add(EdgeFactory.create(new LongWritable(edgeType.getTargetId()), new EdgeValuesWritable(edgeType.getWeight()))); } when(vertex.getEdges()).thenReturn(edges); final RecordWriter<Text, Text> tw = mock(RecordWriter.class); JsonCustomVertexWriter writer = new JsonCustomVertexWriter() { @Override protected RecordWriter<Text, Text> createLineRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { return tw; } }; writer.setConf(conf); writer.initialize(tac); writer.writeVertex(vertex); Text expected = new Text(new Gson().toJson(toWrite)); verify(tw).write(expected, null); }
From source file:fr.ens.biologie.genomique.eoulsan.modules.mgmt.hadoop.DistCp.java
License:LGPL
/** * Initialize DFSCopyFileMapper specific job-configuration. * @param conf : The dfs/mapred configuration. * @param jobConf : The handle to the jobConf object to be initialized. * @param args Arguments/*from w ww .j a v a 2s . co m*/ */ private static void setup(final Configuration conf, final JobConf jobConf, final Arguments args) throws IOException { jobConf.set(DST_DIR_LABEL, args.dst.toUri().toString()); // set boolean values final boolean update = args.flags.contains(Options.UPDATE); final boolean overwrite = !update && args.flags.contains(Options.OVERWRITE); jobConf.setBoolean(Options.UPDATE.propertyname, update); jobConf.setBoolean(Options.OVERWRITE.propertyname, overwrite); jobConf.setBoolean(Options.IGNORE_READ_FAILURES.propertyname, args.flags.contains(Options.IGNORE_READ_FAILURES)); jobConf.setBoolean(Options.PRESERVE_STATUS.propertyname, args.flags.contains(Options.PRESERVE_STATUS)); final String randomId = getRandomId(); JobClient jClient = new JobClient(jobConf); Path jobDirectory = new Path(jClient.getSystemDir(), NAME + "_" + randomId); jobConf.set(JOB_DIR_LABEL, jobDirectory.toString()); long maxBytesPerMap = conf.getLong(BYTES_PER_MAP_LABEL, BYTES_PER_MAP); FileSystem dstfs = args.dst.getFileSystem(conf); boolean dstExists = dstfs.exists(args.dst); boolean dstIsDir = false; if (dstExists) { dstIsDir = dstfs.getFileStatus(args.dst).isDir(); } // default logPath Path logPath = args.log; if (logPath == null) { String filename = "_distcp_logs_" + randomId; if (!dstExists || !dstIsDir) { Path parent = args.dst.getParent(); if (null == parent) { // If dst is '/' on S3, it might not exist yet, but dst.getParent() // will return null. In this case, use '/' as its own parent to // prevent // NPE errors below. parent = args.dst; } if (!dstfs.exists(parent)) { dstfs.mkdirs(parent); } logPath = new Path(parent, filename); } else { logPath = new Path(args.dst, filename); } } FileOutputFormat.setOutputPath(jobConf, logPath); // create src list, dst list FileSystem jobfs = jobDirectory.getFileSystem(jobConf); Path srcfilelist = new Path(jobDirectory, "_distcp_src_files"); jobConf.set(SRC_LIST_LABEL, srcfilelist.toString()); SequenceFile.Writer src_writer = SequenceFile.createWriter(jobfs, jobConf, srcfilelist, LongWritable.class, FilePair.class, SequenceFile.CompressionType.NONE); Path dstfilelist = new Path(jobDirectory, "_distcp_dst_files"); SequenceFile.Writer dst_writer = SequenceFile.createWriter(jobfs, jobConf, dstfilelist, Text.class, Text.class, SequenceFile.CompressionType.NONE); Path dstdirlist = new Path(jobDirectory, "_distcp_dst_dirs"); jobConf.set(DST_DIR_LIST_LABEL, dstdirlist.toString()); SequenceFile.Writer dir_writer = SequenceFile.createWriter(jobfs, jobConf, dstdirlist, Text.class, FilePair.class, SequenceFile.CompressionType.NONE); // handle the case where the destination directory doesn't exist // and we've only a single src directory OR we're updating/overwriting // the contents of the destination directory. final boolean special = (args.srcs.size() == 1 && !dstExists) || update || overwrite; int srcCount = 0, cnsyncf = 0, dirsyn = 0; long fileCount = 0L, byteCount = 0L, cbsyncs = 0L; try { for (Iterator<Path> srcItr = args.srcs.iterator(); srcItr.hasNext();) { final Path src = srcItr.next(); FileSystem srcfs = src.getFileSystem(conf); FileStatus srcfilestat = srcfs.getFileStatus(src); Path root = special && srcfilestat.isDir() ? src : src.getParent(); if (srcfilestat.isDir()) { ++srcCount; } Stack<FileStatus> pathstack = new Stack<>(); for (pathstack.push(srcfilestat); !pathstack.empty();) { FileStatus cur = pathstack.pop(); FileStatus[] children = srcfs.listStatus(cur.getPath()); for (int i = 0; i < children.length; i++) { boolean skipfile = false; final FileStatus child = children[i]; final String dst = makeRelative(root, child.getPath()); ++srcCount; if (child.isDir()) { pathstack.push(child); } else { // skip file if the src and the dst files are the same. skipfile = update && sameFile(srcfs, child, dstfs, new Path(args.dst, dst)); // skip file if it exceed file limit or size limit skipfile |= fileCount == args.filelimit || byteCount + child.getLen() > args.sizelimit; if (!skipfile) { ++fileCount; byteCount += child.getLen(); // if (LOG.isTraceEnabled()) { // LOG.trace("adding file " + child.getPath()); // } ++cnsyncf; cbsyncs += child.getLen(); if (cnsyncf > SYNC_FILE_MAX || cbsyncs > maxBytesPerMap) { src_writer.sync(); dst_writer.sync(); cnsyncf = 0; cbsyncs = 0L; } } } if (!skipfile) { src_writer.append(new LongWritable(child.isDir() ? 0 : child.getLen()), new FilePair(child, dst)); } dst_writer.append(new Text(dst), new Text(child.getPath().toString())); } if (cur.isDir()) { String dst = makeRelative(root, cur.getPath()); dir_writer.append(new Text(dst), new FilePair(cur, dst)); if (++dirsyn > SYNC_FILE_MAX) { dirsyn = 0; dir_writer.sync(); } } } } } finally { checkAndClose(src_writer); checkAndClose(dst_writer); checkAndClose(dir_writer); } FileStatus dststatus = null; try { dststatus = dstfs.getFileStatus(args.dst); } catch (FileNotFoundException fnfe) { getLogger().info(args.dst + " does not exist."); } // create dest path dir if copying > 1 file if (dststatus == null) { if (srcCount > 1 && !dstfs.mkdirs(args.dst)) { throw new IOException("Failed to create" + args.dst); } } final Path sorted = new Path(jobDirectory, "_distcp_sorted"); checkDuplication(jobfs, dstfilelist, sorted, conf); if (dststatus != null && args.flags.contains(Options.DELETE)) { deleteNonexisting(dstfs, dststatus, sorted, jobfs, jobDirectory, jobConf, conf); } Path tmpDir = new Path( (dstExists && !dstIsDir) || (!dstExists && srcCount == 1) ? args.dst.getParent() : args.dst, "_distcp_tmp_" + randomId); jobConf.set(TMP_DIR_LABEL, tmpDir.toUri().toString()); // Explicitly create the tmpDir to ensure that it can be cleaned // up by fullyDelete() later. tmpDir.getFileSystem(conf).mkdirs(tmpDir); getLogger().info("srcCount=" + srcCount); jobConf.setInt(SRC_COUNT_LABEL, srcCount); jobConf.setLong(TOTAL_SIZE_LABEL, byteCount); setMapCount(byteCount, jobConf); }
From source file:gr.ntua.ece.cslab.modissense.queries.clients.mr.GeneralHotIntQueryMapper.java
@Override protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException { // super.map(key, value, context); for (Map.Entry<byte[], byte[]> e : value.getFamilyMap("cf".getBytes()).entrySet()) { POI p = new POI(); p.parseBytes(e.getValue());// ww w .j a v a2s.co m context.write(new LongWritable(p.getId()), new HotnessInterestWritable(1, p.getScore())); } }
From source file:graphvis.engine.FruchtermanReingoldGraphVis.java
License:MIT License
/** * Apply the FruchtermanReingold algorithm on every vertex. Divided in to 6 supersteps. * @param vertex the vertex to calculate on * @param messages messages received from previous superstep * @throws IOException/*from w ww.j a va2 s . c o m*/ */ @Override public void compute(Vertex<LongWritable, VertexValueWritable, EdgeValueWritable> vertex, Iterable<MessageWritable> messages) throws IOException { // Super Step 0 if (getSuperstep() % SUPERSTEPS == 0) { // Everybody is awake //Get default aggregator values. double T = ((DoubleWritable) aggregator.getAggregatedValue("T")).get(); double k = ((DoubleWritable) aggregator.getAggregatedValue("k")).get(); // Very first superstep: init in random position if (getSuperstep() == 0) { Random random = new Random(); VectorWritable pos = new VectorWritable((random.nextDouble() - 0.5) * 100.0, (random.nextDouble() - 0.5) * 100.0); VectorWritable disp = new VectorWritable(0.0, 0.0); VertexValueWritable vertexValue = new VertexValueWritable(pos, disp); vertex.setValue(vertexValue); if (vertex.getId().get() == 1) { //Initialize aggregator values. aggregator.aggregate("k", new DoubleWritable(-k + Math.sqrt(AREA / getTotalNumVertices()))); aggregator.aggregate("T", new DoubleWritable(-T + W / 10)); T = W / 10; } } else { // If it's not the very first superstep, let's chill! if (vertex.getId().get() == 1) { //cool aggregator.aggregate("T", new DoubleWritable(-SPEED)); T = T - SPEED; } } // If we're not frozen yet, wake everyone up and send own position to everyone for next superstep if (T > 0 || (T < MIN_DIST && T > -MIN_DIST && vertex.getId().get() == 1)) { LongWritable ownId = vertex.getId(); long intOwnId = ownId.get(); VectorWritable ownPos = new VectorWritable(vertex.getValue().getPos().getX(), vertex.getValue().getPos().getY()); long totalVertices = getTotalNumVertices(); // We assume that vertices are numbered 1..n where n is the number // of vertices for (long i = 1; i <= totalVertices; i++) { // Send position messages to everyone except self if (i != intOwnId) { sendMessage(new LongWritable(i), new MessageWritable(ownId, ownPos)); } } } } else if (getSuperstep() % SUPERSTEPS == 1) { // Everybody is awake // calculate repulsive forces between everyone VertexValueWritable vertexValue = vertex.getValue(); VectorWritable pos = vertexValue.getPos(); // We start with zero displacement VectorWritable disp = new VectorWritable(0.0, 0.0); for (MessageWritable messageWritable : messages) { VectorWritable otherPos = messageWritable.getPos(); VectorWritable delta = pos.subtract(otherPos); double deltaLength = delta.length(); // if dots are in the same place, let's try to separate them if (deltaLength < MIN_DIST) { delta = makeUpDelta(); deltaLength = delta.length(); } // Update displacement disp = disp.add(delta.multiply(fr(deltaLength) / deltaLength)); } // set new disp vertex.setValue(new VertexValueWritable(pos, disp)); // Send position to neighbors VectorWritable ownPos = new VectorWritable(pos.getX(), pos.getY()); LongWritable ownId = vertex.getId(); for (Edge<LongWritable, EdgeValueWritable> edge : vertex.getEdges()) { sendMessage(edge.getTargetVertexId(), new MessageWritable(ownId, ownPos)); } } else if (getSuperstep() % SUPERSTEPS == 2) { // Vertices with in-edges are awake // anyone who received a message, calculate displacement, then // reply and wait, because in the next step they might get more messages // from vertices which are connected by out-edges // param3 true: send a message back with position applyAttractiveForces(vertex, messages, true); // if there are no out-edges, move, otherwise wait until next step and move then if (vertex.getNumEdges() == 0) { move(vertex); } } else if (getSuperstep() % SUPERSTEPS == 3) { // Vertices with out-edges are awake // param3 true: no need to send another message back applyAttractiveForces(vertex, messages, false); // move, those who don't have out-edges have already moved move(vertex); // Wake up vertices with in-edges for (Edge<LongWritable, EdgeValueWritable> edge : vertex.getEdges()) { sendMessage(edge.getTargetVertexId(), new MessageWritable(vertex.getId(), new VectorWritable())); } } else if (getSuperstep() % SUPERSTEPS == 4) { // Vertices with in-edges are awake LongWritable ownId = new LongWritable(vertex.getId().get()); VectorWritable ownPos = new VectorWritable(vertex.getValue().getPos().getX(), vertex.getValue().getPos().getY()); // Send new position back to everyone from whom a msg was rcvd for (MessageWritable messageWritable : messages) { sendMessage(messageWritable.getSrcId(), new MessageWritable(ownId, ownPos)); } } else if (getSuperstep() % SUPERSTEPS == 5) { // Vertices with out-edges are awake // Set neighbor's position in edge value for (MessageWritable messageWritable : messages) { long srcId = messageWritable.getSrcId().get(); VectorWritable pos = new VectorWritable(messageWritable.getPos().getX(), messageWritable.getPos().getY()); for (Edge<LongWritable, EdgeValueWritable> edge : vertex.getEdges()) { long targetId = edge.getTargetVertexId().get(); if (targetId == srcId) { // Keep weight LongWritable weight = new LongWritable(edge.getValue().getWeight().get()); vertex.setEdgeValue(new LongWritable(targetId), new EdgeValueWritable(weight, pos)); } } } // Wake everyone up for the next superstep, including self long totalVertices = getTotalNumVertices(); for (int i = 1; i <= totalVertices; i++) { sendMessage(new LongWritable(i), new MessageWritable()); } } vertex.voteToHalt(); }
From source file:graphvis.test.FruchtermanReingoldGraphVisTest.java
License:MIT License
/** * Test superstep0//w ww.ja v a 2 s.c o m */ @Test public void testSuperstep0() throws Exception { Vertex<LongWritable, VertexValueWritable, EdgeValueWritable> vertex = new DefaultVertex<LongWritable, VertexValueWritable, EdgeValueWritable>(); FruchtermanReingoldGraphVis computation = new FruchtermanReingoldGraphVis(); computation.setAggregator(mockAggregator); MockUtils.prepareVertexAndComputation(vertex, new LongWritable(1), new VertexValueWritable(), false, computation, 0L); vertex.setValue(new VertexValueWritable()); vertex.addEdge(EdgeFactory.create(new LongWritable(2), new EdgeValueWritable())); vertex.addEdge(EdgeFactory.create(new LongWritable(3), new EdgeValueWritable())); computation.compute(vertex, new ArrayList<MessageWritable>()); assertTrue(vertex.isHalted()); //cool should not be called in superstep0 //verify(computation, never()).aggregate("T", new DoubleWritable(-100)); //random positions should be between -50 and 50 assertEquals(0, vertex.getValue().getPos().getX(), 50); assertEquals(0, vertex.getValue().getPos().getY(), 50); }
From source file:graphvis.test.FruchtermanReingoldGraphVisTest.java
License:MIT License
/** * Test the messages after finishes superstep1 *//* w w w . j ava 2 s. c o m*/ @Test public void testSuperstep1() throws Exception { //messages: positions of other vertices ArrayList<MessageWritable> messages = new ArrayList<MessageWritable>(); messages.add(new MessageWritable(new LongWritable(2), new VectorWritable(545, 234))); messages.add(new MessageWritable(new LongWritable(3), new VectorWritable(242, 677))); Vertex<LongWritable, VertexValueWritable, EdgeValueWritable> vertex = new DefaultVertex<LongWritable, VertexValueWritable, EdgeValueWritable>(); FruchtermanReingoldGraphVis computation = new FruchtermanReingoldGraphVis(); computation.setAggregator(mockAggregator); MockUtils.MockedEnvironment<LongWritable, VertexValueWritable, EdgeValueWritable, MessageWritable> env = MockUtils .prepareVertexAndComputation(vertex, new LongWritable(1L), new VertexValueWritable(), true, computation, 1L); vertex.setValue(new VertexValueWritable(new VectorWritable(532, 542), new VectorWritable(322, 445))); vertex.addEdge(EdgeFactory.create(new LongWritable(2L), new EdgeValueWritable())); vertex.addEdge(EdgeFactory.create(new LongWritable(3L), new EdgeValueWritable())); computation.compute(vertex, messages); assertTrue(vertex.isHalted()); env.verifyMessageSent(new LongWritable(2), new MessageWritable(new LongWritable(1), vertex.getValue().getPos())); env.verifyMessageSent(new LongWritable(3), new MessageWritable(new LongWritable(1), vertex.getValue().getPos())); }
From source file:graphvis.test.FruchtermanReingoldGraphVisTest.java
License:MIT License
/** * Test the messages after finishes superstep2 */// w w w . ja va2 s .c om @Test public void testSuperstep2() throws Exception { //messages: positions of other vertices ArrayList<MessageWritable> messages = new ArrayList<MessageWritable>(); messages.add(new MessageWritable(new LongWritable(2), new VectorWritable(545, 234))); messages.add(new MessageWritable(new LongWritable(3), new VectorWritable(242, 677))); Vertex<LongWritable, VertexValueWritable, EdgeValueWritable> vertex = new DefaultVertex<LongWritable, VertexValueWritable, EdgeValueWritable>(); FruchtermanReingoldGraphVis computation = new FruchtermanReingoldGraphVis(); computation.setAggregator(mockAggregator); MockUtils.MockedEnvironment<LongWritable, VertexValueWritable, EdgeValueWritable, MessageWritable> env = MockUtils .prepareVertexAndComputation(vertex, new LongWritable(1L), new VertexValueWritable(), false, computation, 2L); vertex.setValue(new VertexValueWritable(new VectorWritable(10, 20), new VectorWritable(0, 0))); vertex.addEdge(EdgeFactory.create(new LongWritable(2L), new EdgeValueWritable())); vertex.addEdge(EdgeFactory.create(new LongWritable(3L), new EdgeValueWritable())); computation.compute(vertex, messages); //assertTrue(vertex.isHalted()); env.verifyMessageSent(new LongWritable(2L), new MessageWritable(new LongWritable(1), new VectorWritable(10, 20))); env.verifyMessageSent(new LongWritable(3L), new MessageWritable(new LongWritable(1), new VectorWritable(10, 20))); }