List of usage examples for org.apache.hadoop.io LongWritable LongWritable
public LongWritable(long value)
From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFTrainHybridBSP.java
License:Apache License
public static List<Preference<Long, Long>> convertInputData(Configuration conf, FileSystem fs, Path in, Path preferencesIn, String inputFile, String separator, int maxTestPrefs) throws IOException { List<Preference<Long, Long>> test_prefs = new ArrayList<Preference<Long, Long>>(); // Delete input files if already exist if (fs.exists(in)) { fs.delete(in, true);/*ww w . j a va 2 s .c om*/ } if (fs.exists(preferencesIn)) { fs.delete(preferencesIn, true); } final SequenceFile.Writer prefWriter = SequenceFile.createWriter(fs, conf, preferencesIn, LongWritable.class, PipesVectorWritable.class, CompressionType.NONE); BufferedReader br = new BufferedReader(new FileReader(inputFile)); String line; while ((line = br.readLine()) != null) { String[] values = line.split(separator); long userId = Long.parseLong(values[0]); long itemId = Long.parseLong(values[1]); double rating = Double.parseDouble(values[2]); // System.out.println("userId: " + userId + " itemId: " + itemId // + " rating: " + rating); double vector[] = new double[2]; vector[0] = itemId; vector[1] = rating; prefWriter.append(new LongWritable(userId), new PipesVectorWritable(new DenseDoubleVector(vector))); // Add test preferences maxTestPrefs--; if (maxTestPrefs > 0) { test_prefs.add(new Preference<Long, Long>(userId, itemId, rating)); } } br.close(); prefWriter.close(); return test_prefs; }
From source file:at.illecker.hama.hybrid.examples.piestimator.PiEstimatorHybridBSP.java
License:Apache License
@Override public void bsp(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer) throws IOException, SyncException, InterruptedException { long startTime = 0; if (m_timeMeasurement) { startTime = System.currentTimeMillis(); }/*from www . j a v a2 s . c o m*/ long seed = System.currentTimeMillis(); LinearCongruentialRandomGenerator lcrg = new LinearCongruentialRandomGenerator(seed); long hits = 0; for (long i = 0; i < m_iterationsPerCPUTask; i++) { double x = 2.0 * lcrg.nextDouble() - 1.0; double y = 2.0 * lcrg.nextDouble() - 1.0; if ((x * x + y * y) <= 1.0) { hits++; } } long syncTime = 0; if (m_timeMeasurement) { syncTime = System.currentTimeMillis(); } // Send result to MasterTask peer.send(m_masterTask, new LongWritable(hits)); peer.sync(); if (m_timeMeasurement) { long stopTime = System.currentTimeMillis(); LOG.info("# bspGpuTime: " + ((syncTime - startTime) / 1000.0) + " sec"); LOG.info("# syncTime: " + ((stopTime - syncTime) / 1000.0) + " sec"); if (m_isDebuggingEnabled) { m_logger.writeChars("PiEstimatorHybrid,bspTime: " + ((syncTime - startTime) / 1000.0) + " sec\n"); m_logger.writeChars("PiEstimatorHybrid,syncTime: " + ((stopTime - syncTime) / 1000.0) + " sec\n\n"); } } }
From source file:at.illecker.hama.hybrid.examples.piestimator.PiEstimatorHybridBSP.java
License:Apache License
@Override public void bspGpu(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer, Rootbeer rootbeer) throws IOException, SyncException, InterruptedException { long startTime = 0; if (m_timeMeasurement) { startTime = System.currentTimeMillis(); }/* w w w. ja v a 2 s .c o m*/ ResultList resultList = new ResultList(); PiEstimatorKernel kernel = new PiEstimatorKernel(m_iterationsPerGPUThread, System.currentTimeMillis(), m_blockSize, resultList); // Run GPU Kernels Context context = rootbeer.createDefaultContext(); Stopwatch watch = new Stopwatch(); watch.start(); rootbeer.run(kernel, new ThreadConfig(m_blockSize, m_gridSize, m_blockSize * m_gridSize), context); watch.stop(); // Get GPU results long totalHits = 0; for (Result result : resultList.getList()) { if (result == null) { // break at end of list break; } totalHits += result.hits; } // DEBUG if (m_isDebuggingEnabled) { m_logger.writeChars("BSP=PiEstimatorHybrid,Iterations=" + m_iterations + ",GPUTime=" + watch.elapsedTimeMillis() + "ms\n"); List<StatsRow> stats = context.getStats(); for (StatsRow row : stats) { m_logger.writeChars(" StatsRow:\n"); m_logger.writeChars(" serial time: " + row.getSerializationTime() + "\n"); m_logger.writeChars(" exec time: " + row.getExecutionTime() + "\n"); m_logger.writeChars(" deserial time: " + row.getDeserializationTime() + "\n"); m_logger.writeChars(" num blocks: " + row.getNumBlocks() + "\n"); m_logger.writeChars(" num threads: " + row.getNumThreads() + "\n"); } m_logger.writeChars("totalHits: " + totalHits + "\n"); m_logger.writeChars("iterationsPerGPUThread: " + m_iterationsPerGPUThread + "\n"); m_logger.writeChars( "totalIterationsOnGPU: " + m_iterationsPerGPUThread * m_blockSize * m_gridSize + "\n"); } long syncTime = 0; if (m_timeMeasurement) { syncTime = System.currentTimeMillis(); } // Send result to MasterTask peer.send(m_masterTask, new LongWritable(totalHits)); peer.sync(); if (m_timeMeasurement) { long stopTime = System.currentTimeMillis(); LOG.info("# bspGpuTime: " + ((syncTime - startTime) / 1000.0) + " sec"); LOG.info("# syncTime: " + ((stopTime - syncTime) / 1000.0) + " sec"); if (m_isDebuggingEnabled) { m_logger.writeChars( "PiEstimatorHybrid,bspGpuTime: " + ((syncTime - startTime) / 1000.0) + " sec\n"); m_logger.writeChars("PiEstimatorHybrid,syncTime: " + ((stopTime - syncTime) / 1000.0) + " sec\n\n"); } } }
From source file:at.illecker.hama.rootbeer.examples.piestimator.cpu.PiEstimatorCpuBSP.java
License:Apache License
@Override public void bsp(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer) throws IOException, SyncException, InterruptedException { long seed = System.currentTimeMillis(); LinearCongruentialRandomGenerator m_lcg = new LinearCongruentialRandomGenerator(seed); long hits = 0; for (long i = 0; i < m_calculationsPerBspTask; i++) { double x = 2.0 * m_lcg.nextDouble() - 1.0; double y = 2.0 * m_lcg.nextDouble() - 1.0; if ((x * x + y * y) <= 1.0) { hits++;// w w w .java2 s . c om } } // Send result to MasterTask peer.send(m_masterTask, new LongWritable(hits)); peer.sync(); }
From source file:at.illecker.hama.rootbeer.examples.piestimator.gpu.PiEstimatorGpuBSP.java
License:Apache License
@Override public void bsp(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer) throws IOException, SyncException, InterruptedException { PiEstimatorKernel kernel = new PiEstimatorKernel(m_calculationsPerThread, System.currentTimeMillis()); // Run GPU Kernels Rootbeer rootbeer = new Rootbeer(); Context context = rootbeer.createDefaultContext(); Stopwatch watch = new Stopwatch(); watch.start();/*from w ww.j a va 2 s . c om*/ rootbeer.run(kernel, new ThreadConfig(m_blockSize, m_gridSize, m_blockSize * m_gridSize), context); watch.stop(); // Get GPU results long totalHits = 0; Result[] resultList = kernel.resultList.getList(); for (Result result : resultList) { if (result == null) { // break at end of list break; } totalHits += result.hits; } // DEBUG if (m_isDebuggingEnabled) { // Write log to dfs BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("BSP=PiEstimatorGpuBSP,Iterations=" + m_iterations + ",GPUTime=" + watch.elapsedTimeMillis() + "ms\n"); List<StatsRow> stats = context.getStats(); for (StatsRow row : stats) { outStream.writeChars(" StatsRow:\n"); outStream.writeChars(" serial time: " + row.getSerializationTime() + "\n"); outStream.writeChars(" exec time: " + row.getExecutionTime() + "\n"); outStream.writeChars(" deserial time: " + row.getDeserializationTime() + "\n"); outStream.writeChars(" num blocks: " + row.getNumBlocks() + "\n"); outStream.writeChars(" num threads: " + row.getNumThreads() + "\n"); } outStream.writeChars("totalHits: " + totalHits + "\n"); outStream.writeChars("calculationsPerThread: " + m_calculationsPerThread + "\n"); outStream.writeChars("calculationsTotal: " + m_calculationsPerThread * m_blockSize * m_gridSize + "\n"); outStream.close(); } // Send result to MasterTask peer.send(m_masterTask, new LongWritable(totalHits)); peer.sync(); }
From source file:backup.store.ExternalExtendedBlockSort.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Path dir = new Path("file:///home/apm/Development/git-projects/hdfs-backup/hdfs-backup-core/tmp"); dir.getFileSystem(conf).delete(dir, true); long start = System.nanoTime(); try (ExternalExtendedBlockSort<LongWritable> sort = new ExternalExtendedBlockSort<>(conf, dir, LongWritable.class)) { Random random = new Random(); for (int bp = 0; bp < 1; bp++) { String bpid = UUID.randomUUID().toString(); for (int i = 0; i < 10000000; i++) { // for (int i = 0; i < 10; i++) { long genstamp = random.nextInt(20000); long blockId = random.nextLong(); ExtendedBlock extendedBlock = new ExtendedBlock(bpid, blockId, random.nextInt(Integer.MAX_VALUE), genstamp); sort.add(extendedBlock, new LongWritable(blockId)); }//from www.ja va 2 s . c o m } System.out.println("finished"); sort.finished(); System.out.println("interate"); for (String blockPoolId : sort.getBlockPoolIds()) { ExtendedBlockEnum<LongWritable> blockEnum = sort.getBlockEnum(blockPoolId); ExtendedBlock block; long l = 0; while ((block = blockEnum.next()) != null) { // System.out.println(block); long blockId = block.getBlockId(); l += blockId; LongWritable currentValue = blockEnum.currentValue(); if (currentValue.get() != blockId) { System.err.println("Error " + blockId); } } System.out.println(l); } } long end = System.nanoTime(); System.out.println("Time [" + (end - start) / 1000000.0 + " ms]"); }
From source file:be.uantwerpen.adrem.bigfim.AprioriPhaseMapperTest.java
License:Apache License
@Test public void phase_1_With_Input() throws Exception { AprioriPhaseMapper.Context ctx = createMock(Mapper.Context.class); ctx.write(new Text(""), new Text("1 5")); ctx.write(new Text(""), new Text("2 3")); ctx.write(new Text(""), new Text("3 6")); ctx.write(new Text(""), new Text("4 5")); ctx.write(new Text(""), new Text("5 4")); EasyMock.replay(ctx);//from w ww . j a v a2 s . com AprioriPhaseMapper mapper = createMapper(1, create_Count_Trie_Empty()); for (int i = 0; i < data.length; i++) { mapper.map(new LongWritable(i), new Text(data[i]), ctx); } mapper.cleanup(ctx); EasyMock.verify(ctx); }
From source file:be.uantwerpen.adrem.bigfim.AprioriPhaseMapperTest.java
License:Apache License
@Test public void phase_2_With_Input() throws Exception { AprioriPhaseMapper.Context ctx = createMock(Mapper.Context.class); ctx.write(new Text("1"), new Text("2 1")); ctx.write(new Text("1"), new Text("3 4")); ctx.write(new Text("1"), new Text("4 3")); ctx.write(new Text("2"), new Text("3 2")); ctx.write(new Text("2"), new Text("5 1")); ctx.write(new Text("3"), new Text("4 5")); ctx.write(new Text("4"), new Text("5 2")); EasyMock.replay(ctx);// www . j a v a 2s.co m AprioriPhaseMapper mapper = createMapper(2, create_Count_Trie_Not_Empty()); for (int i = 0; i < data.length; i++) { mapper.map(new LongWritable(i), new Text(data[i]), ctx); } mapper.cleanup(ctx); EasyMock.verify(ctx); }
From source file:be.uantwerpen.adrem.bigfim.AprioriPhaseMapperTest.java
License:Apache License
@Test public void phase_2_With_Input_Empty_Count_Trie() throws Exception { AprioriPhaseMapper.Context ctx = createMock(Mapper.Context.class); EasyMock.replay(ctx);/*from w w w.j av a 2s. c o m*/ AprioriPhaseMapper mapper = createMapper(2, create_Count_Trie_Empty()); for (int i = 0; i < data.length; i++) { mapper.map(new LongWritable(i), new Text(data[i]), ctx); } mapper.cleanup(ctx); EasyMock.verify(ctx); }
From source file:be.uantwerpen.adrem.bigfim.ComputeTidListMapperTest.java
License:Apache License
@Test public void phase_1_With_Input() throws Exception { ComputeTidListMapper.Context ctx = createMock(Mapper.Context.class); ctx.write(new Text(""), newIAW(-1, 1, 0, 2, 3, 5, 7)); ctx.write(new Text(""), newIAW(-1, 2, 0, 1, 6)); ctx.write(new Text(""), newIAW(-1, 3, 0, 1, 2, 4, 5, 7)); ctx.write(new Text(""), newIAW(-1, 4, 0, 1, 4, 5, 7)); ctx.write(new Text(""), newIAW(-1, 5, 2, 4, 5, 6)); EasyMock.replay(ctx);/* ww w. j a v a 2 s . c om*/ ComputeTidListMapper mapper = new ComputeTidListMapper(); setField(mapper, "delimiter", " "); for (int i = 0; i < data.length; i++) { mapper.map(new LongWritable(i), new Text(data[i]), ctx); } mapper.cleanup(ctx); EasyMock.verify(ctx); }