List of usage examples for org.apache.hadoop.io LongWritable set
public void set(long value)
From source file:nthu.scopelab.stsqr.test.QRredScheJob.java
License:Apache License
public static void main(String[] args) throws Exception { String inputpath = getArgument("-input", args); checkArgument(inputpath);/*from w ww. j a v a2 s. c o m*/ String master = getArgument("-master", args); checkArgument(master); String outputpath = getArgument("-output", args); if (outputpath == null) outputpath = "output"; String redsche_str = getArgument("-reduceSchedule", args); if (redsche_str == null) redsche_str = "1"; String exememory = getArgument("-em", args); if (exememory == null) exememory = "512m"; String matsize_str = getArgument("-matsize", args); checkArgument(matsize_str); String matnum_str = getArgument("-matnum", args); checkArgument(matnum_str); String sparkHome = System.getenv("SPARK_HOME"); SparkConf sconf = new SparkConf().setMaster(master).setAppName("QRredScheJob") .setSparkHome(System.getenv("SPARK_HOME")) .setJars(new String[] { sparkHome + "/oproject/target/simple-project-1.0.jar" }) .set("spark.executor.memory", exememory); JavaSparkContext ctx = new JavaSparkContext(sconf); // 1. prepartionJob: turn matrix from text(string) to hadoop sequencefile(sLMatrixWritable) Thread.sleep(2000); int matsize = Integer.valueOf(matsize_str); int matnum = Integer.valueOf(matnum_str); if (matnum % 100 != 0) { System.out.println( "The argument is incorrect.\n -matnum <number of matrix> that number must be divisible by 100."); return; } //1. Create synthesized R matrix int num = matnum / 100; JavaPairRDD<Long, Tuple2<Long, sLMatrixWritable>> Rrdd = ctx.textFile(inputpath).repartition(100) .mapPartitions(new createRMat(matsize, num)); ; long numRmat = Rrdd.cache().count(); //2. Compute TSQR but not build Q long start, end, s1, e1; start = new Date().getTime(); String[] splitStr = redsche_str.split(","); int[] redSche = new int[splitStr.length]; int itrNum = redSche.length; for (int i = 0; i < itrNum; i++) redSche[i] = Integer.valueOf(splitStr[i]); JavaPairRDD<Long, Tuple2<Long, sLMatrixWritable>> pRrdd = Rrdd; Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); String Qspath = outputpath + "/Qs"; fs.delete(new Path(Qspath), true); for (int i = 0; i < itrNum - 1; i++) { //gradually reduce number of partitions which is argument of "repartition" function pRrdd = pRrdd.repartition(redSche[i]).mapPartitions(new QRJob.mergedQR(i + 1, Qspath)); } s1 = new Date().getTime(); List<Tuple2<Long, Tuple2<Long, sLMatrixWritable>>> RrddList = pRrdd.collect(); e1 = new Date().getTime(); cmDenseMatrix R; if (RrddList.size() > 1) { //merge Rf and then do QR factorization to get Qs and Rs int columns = RrddList.get(0)._2._2.matNumColumns(); int rows = columns * RrddList.size(); cmDenseMatrix mR = new cmDenseMatrix(rows, columns); Long[] keyIdArray = new Long[RrddList.size()]; long mergedId; int count = 0; s1 = new Date().getTime(); for (Tuple2<Long, Tuple2<Long, sLMatrixWritable>> Rkvpair : RrddList) { keyIdArray[count] = Rkvpair._2._1; cmDenseMatrix Rv = Rkvpair._2._2.getDense(); //R matrix is square, its row size equal to column size for (int i = 0; i < columns; i++) for (int j = 0; j < columns; j++) { mR.set(i + count * columns, j, Rv.get(i, j)); } count++; } e1 = new Date().getTime(); mergedId = keyIdArray[0]; s1 = new Date().getTime(); QRF qrf2 = QRF.factorize(mR); e1 = new Date().getTime(); cmDenseMatrix Qsmat = qrf2.getQ(); cmDenseMatrix finalR = qrf2.getR(); R = finalR; //split Qs and write the last Qs file SequenceFile.Writer swriter = new SequenceFile.Writer(fs, conf, new Path(Qspath + "/" + itrNum + "/" + mergedId), LongWritable.class, sMatrixWritable.class); LongWritable okey = new LongWritable(); sMatrixWritable ovalue = new sMatrixWritable(); long tt1, tt2, writingTime = 0; for (int i = 0; i < keyIdArray.length; i++) { cmDenseMatrix Qsvalue = new cmDenseMatrix(columns, columns); for (int j = 0; j < columns; j++) for (int k = 0; k < columns; k++) { Qsvalue.set(j, k, Qsmat.get(j + columns * i, k)); } //previous Q mergedId and last Q uId has same value, the last Id of Q use uId and previous Q use mergedId for key. okey.set(keyIdArray[i]); ovalue.set(Qsvalue); tt1 = new Date().getTime(); swriter.append(okey, ovalue); tt2 = new Date().getTime(); writingTime += tt2 - tt1; } System.out.println("Time of Writing :" + writingTime); swriter.close(); } // if end: RrddList.size > 1 end = new Date().getTime(); long exectime = end - start; System.out.println("Running Time: " + exectime); System.exit(0); }
From source file:org.apache.avro.mapred.TestSequenceFileReader.java
License:Apache License
@BeforeClass public static void testWriteSequenceFile() throws IOException { FILE.delete();//from w ww .j a va 2 s . co m Configuration c = new Configuration(); URI uri = FILE.toURI(); SequenceFile.Writer writer = new SequenceFile.Writer(FileSystem.get(uri, c), c, new Path(uri.toString()), LongWritable.class, Text.class); final LongWritable key = new LongWritable(); final Text val = new Text(); for (int i = 0; i < COUNT; ++i) { key.set(i); val.set(Integer.toString(i)); writer.append(key, val); } writer.close(); }
From source file:org.apache.camel.component.hdfs.HdfsConsumerTest.java
License:Apache License
@Test public void testReadLong() throws Exception { if (!canTest()) { return;// w w w . j a v a 2s .com } final Path file = new Path(new File("target/test/test-camel-long").getAbsolutePath()); Configuration conf = new Configuration(); FileSystem fs1 = FileSystem.get(file.toUri(), conf); SequenceFile.Writer writer = createWriter(fs1, conf, file, NullWritable.class, LongWritable.class); NullWritable keyWritable = NullWritable.get(); LongWritable valueWritable = new LongWritable(); long value = 31415926535L; valueWritable.set(value); writer.append(keyWritable, valueWritable); writer.sync(); writer.close(); MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class); resultEndpoint.expectedMessageCount(1); context.addRoutes(new RouteBuilder() { public void configure() { from("hdfs:///" + file.toUri() + "?fileSystemType=LOCAL&fileType=SEQUENCE_FILE&initialDelay=0") .to("mock:result"); } }); context.start(); resultEndpoint.assertIsSatisfied(); }
From source file:org.apache.camel.component.hdfs2.HdfsConsumerTest.java
License:Apache License
@Test public void testReadLong() throws Exception { if (!canTest()) { return;/* w w w . j a v a2s .com*/ } final Path file = new Path(new File("target/test/test-camel-long").getAbsolutePath()); Configuration conf = new Configuration(); SequenceFile.Writer writer = createWriter(conf, file, NullWritable.class, LongWritable.class); NullWritable keyWritable = NullWritable.get(); LongWritable valueWritable = new LongWritable(); long value = 31415926535L; valueWritable.set(value); writer.append(keyWritable, valueWritable); writer.sync(); writer.close(); MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class); resultEndpoint.expectedMessageCount(1); context.addRoutes(new RouteBuilder() { public void configure() { from("hdfs2:///" + file.toUri() + "?fileSystemType=LOCAL&fileType=SEQUENCE_FILE&initialDelay=0") .to("mock:result"); } }); context.start(); resultEndpoint.assertIsSatisfied(); }
From source file:org.apache.flink.test.hadoop.mapred.HadoopIOFormatsITCase.java
License:Apache License
@Override protected void preSubmit() throws Exception { resultPath = new String[] { getTempDirPath("result0"), getTempDirPath("result1") }; File sequenceFile = createAndRegisterTempFile("seqFile"); sequenceFileInPath = sequenceFile.toURI().toString(); // Create a sequence file org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(); FileSystem fs = FileSystem.get(URI.create(sequenceFile.getAbsolutePath()), conf); Path path = new Path(sequenceFile.getAbsolutePath()); // ------------------ Long / Text Key Value pair: ------------ int kvCount = 4; LongWritable key = new LongWritable(); Text value = new Text(); SequenceFile.Writer writer = null; try {//from www. j a v a 2s . c om writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass()); for (int i = 0; i < kvCount; i++) { if (i == 1) { // write key = 0 a bit more often. for (int a = 0; a < 15; a++) { key.set(i); value.set(i + " - somestring"); writer.append(key, value); } } key.set(i); value.set(i + " - somestring"); writer.append(key, value); } } finally { IOUtils.closeStream(writer); } // ------------------ Long / Text Key Value pair: ------------ File sequenceFileNull = createAndRegisterTempFile("seqFileNullKey"); sequenceFileInPathNull = sequenceFileNull.toURI().toString(); path = new Path(sequenceFileInPathNull); LongWritable value1 = new LongWritable(); SequenceFile.Writer writer1 = null; try { writer1 = SequenceFile.createWriter(fs, conf, path, NullWritable.class, value1.getClass()); for (int i = 0; i < kvCount; i++) { value1.set(i); writer1.append(NullWritable.get(), value1); } } finally { IOUtils.closeStream(writer1); } }
From source file:org.apache.giraph.block_app.framework.BlockExecutionTest.java
License:Apache License
@Test public void testReducing() { TestGraph<LongWritable, LongWritable, NullWritable> graph = createTestGraph(); final LongWritable value = new LongWritable(); LocalBlockRunner.runBlock(graph, new Piece<WritableComparable, Writable, Writable, NoMessage, Object>() { private ReducerHandle<LongWritable, LongWritable> numVertices; @Override// w w w . j ava 2 s . co m public void registerReducers(CreateReducersApi reduceApi, Object executionStage) { numVertices = reduceApi.createLocalReducer(SumReduce.LONG); } @Override public VertexSender<WritableComparable, Writable, Writable> getVertexSender( BlockWorkerSendApi<WritableComparable, Writable, Writable, NoMessage> workerApi, Object executionStage) { return new InnerVertexSender() { @Override public void vertexSend(Vertex<WritableComparable, Writable, Writable> vertex) { numVertices.reduce(new LongWritable(1)); } }; } @Override public void masterCompute(BlockMasterApi masterApi, Object executionStage) { value.set(numVertices.getReducedValue(masterApi).get()); } }, new Object()); Assert.assertEquals(4, value.get()); }
From source file:org.apache.giraph.block_app.framework.piece.global_comm.ReduceUtilsObject.java
License:Apache License
public void reduceLong(ReducerHandle<LongWritable, ?> reduceHandle, long value) { LongWritable tmp = reusableLong; tmp.set(value); reduceHandle.reduce(tmp);/*ww w.jav a 2s. c om*/ }
From source file:org.apache.giraph.block_app.library.prepare_graph.UndirectedConnectedComponents.java
License:Apache License
/** * Calculates sizes of all components by aggregating on master, and allows * each vertex to consume its size. Differs from CalculateComponentSizesPiece * in that aggregation happens on master, instead of message sends to the * component_id.//from w w w .j a v a 2 s .co m */ public static <V extends Writable> Block calculateConnectedComponentSizes( SupplierFromVertex<LongWritable, V, Writable, LongWritable> getComponent, ConsumerWithVertex<LongWritable, V, Writable, LongWritable> sizeConsumer) { Pair<LongWritable, LongWritable> componentToReducePair = Pair.of(new LongWritable(), new LongWritable(1)); LongWritable reusableLong = new LongWritable(); return Pieces.reduceAndBroadcast("CalcConnectedComponentSizes", new BasicMapReduce<>(LongTypeOps.INSTANCE, LongTypeOps.INSTANCE, SumReduce.LONG), (Vertex<LongWritable, V, Writable> vertex) -> { componentToReducePair.getLeft().set(getComponent.get(vertex).get()); return componentToReducePair; }, (vertex, componentSizes) -> { long compSize = componentSizes.get(getComponent.get(vertex)).get(); reusableLong.set(compSize); sizeConsumer.apply(vertex, reusableLong); }); }
From source file:org.apache.giraph.block_app.library.prepare_graph.UndirectedConnectedComponents.java
License:Apache License
/** * Given a graph with already calculated connected components - calculates * ID of the largest one.//ww w . j a va2s .com */ public static <V extends Writable> Block calculateLargestConnectedComponent( SupplierFromVertex<LongWritable, V, Writable, LongWritable> getComponent, LongWritable largestComponent) { return calculateLargestConnectedComponentStats(getComponent, (t) -> largestComponent.set(t.getLeft().get())); }
From source file:org.apache.giraph.block_app.library.prepare_graph.UndirectedConnectedComponents.java
License:Apache License
/** * Given a graph with already calculated connected components - calculates * size of the largest one.//from ww w. ja v a 2 s. c o m */ public static <V extends Writable> Block calculateLargestConnectedComponentSize( SupplierFromVertex<LongWritable, V, Writable, LongWritable> getComponent, LongWritable largestComponentSize) { return calculateLargestConnectedComponentStats(getComponent, (t) -> largestComponentSize.set(t.getRight().get())); }