Example usage for org.apache.hadoop.io NullWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io NullWritable get.

Prototype

public static NullWritable get() 

Source Link

Document

Returns the single instance of this class.

Usage

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

private void recalculateAssignmentsAndWrite(
        BSPPeer<PipesVectorWritable, NullWritable, IntWritable, PipesVectorWritable, CenterMessage> peer)
        throws IOException {

    IntWritable keyWrite = new IntWritable();
    for (DoubleVector v : m_cache) {
        final int lowestDistantCenter = getNearestCenter(v);
        keyWrite.set(lowestDistantCenter);
        peer.write(keyWrite, new PipesVectorWritable(v));
    }/*from  w w  w . jav  a  2 s. c o  m*/

    // just on the first task write the centers to filesystem to prevent
    // collisions
    if (peer.getPeerName().equals(peer.getPeerName(0))) {
        String pathString = m_conf.get(CONF_CENTER_OUT_PATH);
        if (pathString != null) {
            final SequenceFile.Writer dataWriter = SequenceFile.createWriter(FileSystem.get(m_conf), m_conf,
                    new Path(pathString), PipesVectorWritable.class, NullWritable.class, CompressionType.NONE);
            final NullWritable value = NullWritable.get();

            for (DoubleVector center : m_centers_cpu) {
                dataWriter.append(new PipesVectorWritable(center), value);
            }
            dataWriter.close();
        }
    }
}

---

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

/********************************* GPU *********************************/
@Override// w ww. j  a v  a 2 s  .  co m
public void setupGpu(
        BSPPeer<PipesVectorWritable, NullWritable, IntWritable, PipesVectorWritable, CenterMessage> peer)
        throws IOException, SyncException, InterruptedException {

    this.m_conf = peer.getConfiguration();
    this.m_timeMeasurement = m_conf.getBoolean(CONF_TIME, false);
    this.m_isDebuggingEnabled = m_conf.getBoolean(CONF_DEBUG, false);
    this.m_maxIterations = m_conf.getInt(CONF_MAX_ITERATIONS, -1);
    this.m_blockSize = Integer.parseInt(this.m_conf.get(CONF_BLOCKSIZE));
    this.m_gridSize = Integer.parseInt(this.m_conf.get(CONF_GRIDSIZE));

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(m_conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) m_conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    long startTime = 0;
    if (m_timeMeasurement) {
        startTime = System.currentTimeMillis();
    }

    // Init center vectors
    Path centroids = new Path(m_conf.get(CONF_CENTER_IN_PATH));
    FileSystem fs = FileSystem.get(m_conf);

    final List<double[]> centers = new ArrayList<double[]>();
    SequenceFile.Reader reader = null;
    try {
        reader = new SequenceFile.Reader(fs, centroids, m_conf);
        PipesVectorWritable key = new PipesVectorWritable();
        NullWritable value = NullWritable.get();
        while (reader.next(key, value)) {
            centers.add(key.getVector().toArray());
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        if (reader != null) {
            reader.close();
        }
    }

    Preconditions.checkArgument(centers.size() > 0, "Centers file must contain at least a single center!");

    // build centers_gpu double[][]
    this.m_centers_gpu = new double[centers.size()][centers.get(0).length];
    for (int i = 0; i < centers.size(); i++) {
        double[] vector = centers.get(i);
        for (int j = 0; j < vector.length; j++) {
            this.m_centers_gpu[i][j] = vector[j];
        }
    }

    long stopTime = 0;
    if (m_timeMeasurement) {
        stopTime = System.currentTimeMillis();
        LOG.info("# setupGpuTime: " + ((stopTime - startTime) / 1000.0) + " sec");
        if (m_isDebuggingEnabled) {
            m_logger.writeChars(
                    "PiEstimatorHybrid,setupGpuTime: " + ((stopTime - startTime) / 1000.0) + " sec\n");
        }
    }
}

---

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

@Override
public void bspGpu(
        BSPPeer<PipesVectorWritable, NullWritable, IntWritable, PipesVectorWritable, CenterMessage> peer,
        Rootbeer rootbeer) throws IOException, SyncException, InterruptedException {

    long startTime = 0;
    if (m_timeMeasurement) {
        startTime = System.currentTimeMillis();
    }/*from   w  w  w  . j  av a  2s  .  c o m*/

    // Fetch inputs
    final List<DoubleVector> inputs = new ArrayList<DoubleVector>();
    final PipesVectorWritable key = new PipesVectorWritable();
    final NullWritable nullValue = NullWritable.get();
    while (peer.readNext(key, nullValue)) {
        inputs.add(key.getVector());
    }
    // Convert inputs to double[][]
    double[][] inputsArr = new double[inputs.size()][inputs.get(0).getLength()];
    for (int i = 0; i < inputs.size(); i++) {
        double[] vector = inputs.get(i).toArray();
        for (int j = 0; j < vector.length; j++) {
            inputsArr[i][j] = vector[j];
        }
    }

    // Logging
    if (m_isDebuggingEnabled) {
        m_logger.writeChars("KMeansHybrid.bspGpu executed on GPU!\n");
        m_logger.writeChars(
                "KMeansHybrid.bspGpu blockSize: " + m_blockSize + " gridSize: " + m_gridSize + "\n");
        m_logger.writeChars("KMeansHybrid.bspGpu inputSize: " + inputs.size() + "\n");
    }

    KMeansHybridKernel kernel = new KMeansHybridKernel(inputsArr, m_centers_gpu,
            m_conf.getInt(CONF_MAX_ITERATIONS, 0), peer.getAllPeerNames());

    // Run GPU Kernels
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    watch.start();
    rootbeer.run(kernel, new ThreadConfig(m_blockSize, m_gridSize, m_blockSize * m_gridSize), context);
    watch.stop();

    // Output inputs with corresponding new center id
    for (int i = 0; i < inputs.size(); i++) {
        peer.write(new IntWritable(kernel.m_input_centers[i]), new PipesVectorWritable(inputs.get(i)));
    }

    // Output new Centers only on first task
    // to prevent collisions
    if (peer.getPeerName().equals(peer.getPeerName(0))) {
        String pathString = m_conf.get(CONF_CENTER_OUT_PATH);
        if (pathString != null) {
            final SequenceFile.Writer dataWriter = SequenceFile.createWriter(FileSystem.get(m_conf), m_conf,
                    new Path(pathString), PipesVectorWritable.class, NullWritable.class, CompressionType.NONE);

            for (int i = 0; i < kernel.m_centers.length; i++) {
                dataWriter.append(new PipesVectorWritable(new DenseDoubleVector(kernel.m_centers[i])),
                        nullValue);
            }
            dataWriter.close();
        }
    }

    long stopTime = System.currentTimeMillis();
    if (m_timeMeasurement) {
        LOG.info("# bspGpuTime: " + ((stopTime - startTime) / 1000.0) + " sec");

        if (m_isDebuggingEnabled) {
            m_logger.writeChars(
                    "PiEstimatorHybrid,bspGpuTime: " + ((stopTime - startTime) / 1000.0) + " sec\n");
        }
    }

    // Logging
    if (m_isDebuggingEnabled) {
        List<StatsRow> stats = context.getStats();
        for (StatsRow row : stats) {
            m_logger.writeChars("  StatsRow:\n");
            m_logger.writeChars("    serial time: " + row.getSerializationTime() + "\n");
            m_logger.writeChars("    exec time: " + row.getExecutionTime() + "\n");
            m_logger.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
            m_logger.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
            m_logger.writeChars("    num threads: " + row.getNumThreads() + "\n");
            m_logger.writeChars("GPUTime: " + watch.elapsedTimeMillis() + " ms" + "\n");
        }

        m_logger.close();
    }
}

---

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

public static void main(String[] args) throws Exception {

    // Defaults//  ww  w .j  av a 2 s  .c om
    int numBspTask = 1;
    int numGpuBspTask = 1;
    int blockSize = BLOCK_SIZE;
    int gridSize = GRID_SIZE;
    long n = 10; // input vectors
    int k = 3; // start vectors
    int vectorDimension = 2;
    int maxIteration = 10;
    boolean useTestExampleInput = false;
    boolean isDebugging = false;
    boolean timeMeasurement = false;
    int GPUPercentage = 80;

    Configuration conf = new HamaConfiguration();
    FileSystem fs = FileSystem.get(conf);

    // Set numBspTask to maxTasks
    // BSPJobClient jobClient = new BSPJobClient(conf);
    // ClusterStatus cluster = jobClient.getClusterStatus(true);
    // numBspTask = cluster.getMaxTasks();

    if (args.length > 0) {
        if (args.length == 12) {
            numBspTask = Integer.parseInt(args[0]);
            numGpuBspTask = Integer.parseInt(args[1]);
            blockSize = Integer.parseInt(args[2]);
            gridSize = Integer.parseInt(args[3]);
            n = Long.parseLong(args[4]);
            k = Integer.parseInt(args[5]);
            vectorDimension = Integer.parseInt(args[6]);
            maxIteration = Integer.parseInt(args[7]);
            useTestExampleInput = Boolean.parseBoolean(args[8]);
            GPUPercentage = Integer.parseInt(args[9]);
            isDebugging = Boolean.parseBoolean(args[10]);
            timeMeasurement = Boolean.parseBoolean(args[11]);

        } else {
            System.out.println("Wrong argument size!");
            System.out.println("    Argument1=numBspTask");
            System.out.println("    Argument2=numGpuBspTask");
            System.out.println("    Argument3=blockSize");
            System.out.println("    Argument4=gridSize");
            System.out.println("    Argument5=n | Number of input vectors (" + n + ")");
            System.out.println("    Argument6=k | Number of start vectors (" + k + ")");
            System.out.println(
                    "    Argument7=vectorDimension | Dimension of each vector (" + vectorDimension + ")");
            System.out.println(
                    "    Argument8=maxIterations | Number of maximal iterations (" + maxIteration + ")");
            System.out.println("    Argument9=testExample | Use testExample input (true|false=default)");
            System.out.println("    Argument10=GPUPercentage (percentage of input)");
            System.out.println("    Argument11=isDebugging (true|false=defaul)");
            System.out.println("    Argument12=timeMeasurement (true|false=defaul)");
            return;
        }
    }

    // Set config variables
    conf.setBoolean(CONF_DEBUG, isDebugging);
    conf.setBoolean("hama.pipes.logging", false);
    conf.setBoolean(CONF_TIME, timeMeasurement);

    // Set CPU tasks
    conf.setInt("bsp.peers.num", numBspTask);
    // Set GPU tasks
    conf.setInt("bsp.peers.gpu.num", numGpuBspTask);
    // Set GPU blockSize and gridSize
    conf.set(CONF_BLOCKSIZE, "" + blockSize);
    conf.set(CONF_GRIDSIZE, "" + gridSize);
    // Set maxIterations for KMeans
    conf.setInt(CONF_MAX_ITERATIONS, maxIteration);
    // Set n for KMeans
    conf.setLong(CONF_N, n);
    // Set GPU workload
    conf.setInt(CONF_GPU_PERCENTAGE, GPUPercentage);

    LOG.info("NumBspTask: " + conf.getInt("bsp.peers.num", 0));
    LOG.info("NumGpuBspTask: " + conf.getInt("bsp.peers.gpu.num", 0));
    LOG.info("bsp.tasks.maximum: " + conf.get("bsp.tasks.maximum"));
    LOG.info("GPUPercentage: " + conf.get(CONF_GPU_PERCENTAGE));
    LOG.info("BlockSize: " + conf.get(CONF_BLOCKSIZE));
    LOG.info("GridSize: " + conf.get(CONF_GRIDSIZE));
    LOG.info("isDebugging: " + conf.get(CONF_DEBUG));
    LOG.info("timeMeasurement: " + conf.get(CONF_TIME));
    LOG.info("useTestExampleInput: " + useTestExampleInput);
    LOG.info("inputPath: " + CONF_INPUT_DIR);
    LOG.info("centersPath: " + CONF_CENTER_DIR);
    LOG.info("outputPath: " + CONF_OUTPUT_DIR);
    LOG.info("n: " + n);
    LOG.info("k: " + k);
    LOG.info("vectorDimension: " + vectorDimension);
    LOG.info("maxIteration: " + maxIteration);

    Path centerIn = new Path(CONF_CENTER_DIR, "center_in.seq");
    Path centerOut = new Path(CONF_CENTER_DIR, "center_out.seq");
    conf.set(CONF_CENTER_IN_PATH, centerIn.toString());
    conf.set(CONF_CENTER_OUT_PATH, centerOut.toString());

    // prepare Input
    if (useTestExampleInput) {
        // prepareTestInput(conf, fs, input, centerIn);
        prepareInputData(conf, fs, CONF_INPUT_DIR, centerIn, numBspTask, numGpuBspTask, n, k, vectorDimension,
                null, GPUPercentage);
    } else {
        prepareInputData(conf, fs, CONF_INPUT_DIR, centerIn, numBspTask, numGpuBspTask, n, k, vectorDimension,
                new Random(3337L), GPUPercentage);
    }

    BSPJob job = createKMeansHybridBSPConf(conf, CONF_INPUT_DIR, CONF_OUTPUT_DIR);

    long startTime = System.currentTimeMillis();
    if (job.waitForCompletion(true)) {
        LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");

        if (isDebugging) {
            printFile(conf, fs, centerOut, new PipesVectorWritable(), NullWritable.get());
            printOutput(conf, fs, ".log", new IntWritable(), new PipesVectorWritable());
        }

        if (k < 50) {
            printFile(conf, fs, centerOut, new PipesVectorWritable(), NullWritable.get());
        }
    }
}

---

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

/**
 * prepareInputData/* w  w w  . j  a va  2  s . c o  m*/
 * 
 */
public static void prepareInputData(Configuration conf, FileSystem fs, Path in, Path centerIn, int numBspTask,
        int numGPUBspTask, long n, int k, int vectorDimension, Random rand, int GPUPercentage)
        throws IOException {

    // Delete input files if already exist
    if (fs.exists(in)) {
        fs.delete(in, true);
    }
    if (fs.exists(centerIn)) {
        fs.delete(centerIn, true);
    }

    final NullWritable nullValue = NullWritable.get();
    final SequenceFile.Writer centerWriter = SequenceFile.createWriter(fs, conf, centerIn,
            PipesVectorWritable.class, NullWritable.class, CompressionType.NONE);

    // Compute work distributions
    int cpuTaskNum = numBspTask - numGPUBspTask;
    long inputVectorsPerGPUTask = 0;
    long inputVectorsPerCPU = 0;
    long inputVectorsPerCPUTask = 0;
    if ((numGPUBspTask > 0) && (GPUPercentage > 0) && (GPUPercentage <= 100)) {
        inputVectorsPerGPUTask = (n * GPUPercentage) / 100;
        inputVectorsPerCPU = n - inputVectorsPerGPUTask;
    } else {
        inputVectorsPerCPU = n;
    }
    if (cpuTaskNum > 0) {
        inputVectorsPerCPUTask = inputVectorsPerCPU / cpuTaskNum;
    }

    // long interval = totalNumberOfPoints / numBspTask;
    long centers = 0;

    for (int part = 0; part < numBspTask; part++) {
        Path partIn = new Path(in, "part" + part + ".seq");
        final SequenceFile.Writer dataWriter = SequenceFile.createWriter(fs, conf, partIn,
                PipesVectorWritable.class, NullWritable.class, CompressionType.NONE);

        long interval = 0;
        if (part > cpuTaskNum) {
            interval = inputVectorsPerGPUTask;
        } else {
            interval = inputVectorsPerCPUTask;
        }
        long start = interval * part;
        long end = start + interval - 1;
        if ((numBspTask - 1) == part) {
            end = n; // set to totalNumberOfPoints
        }
        LOG.info("Partition " + part + ": from " + start + " to " + end);

        for (long i = start; i <= end; i++) {
            double[] arr = new double[vectorDimension];
            for (int j = 0; j < vectorDimension; j++) {
                if (rand != null) {
                    arr[j] = rand.nextInt((int) n);
                } else {
                    arr[j] = i;
                }
            }
            PipesVectorWritable vector = new PipesVectorWritable(new DenseDoubleVector(arr));

            // LOG.info("input[" + i + "]: " + Arrays.toString(arr));
            dataWriter.append(vector, nullValue);

            if (k > centers) {
                // LOG.info("center[" + i + "]: " + Arrays.toString(arr));
                centerWriter.append(vector, nullValue);
                centers++;
            } else {
                centerWriter.close();
            }

        }
        dataWriter.close();
    }

}

---

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

/**
 * Create testExample vectors and centers as input from
 * http://www.maplesoft.com/support/help/Maple/view.aspx?path=NAG/g03efc
 * /*from  w  w w . j ava  2  s  .c om*/
 * n := 20: vectorDimension := 5: k := 3: maxIterations := 10:
 * 
 * x := Matrix([ [77.3, 13, 9.699999999999999, 1.5, 6.4], [82.5, 10, 7.5, 1.5,
 * 6.5], [66.90000000000001, 20.6, 12.5, 2.3, 7], [47.2, 33.8, 19, 2.8, 5.8],
 * [65.3, 20.5, 14.2, 1.9, 6.9], [83.3, 10, 6.7, 2.2, 7], [81.59999999999999,
 * 12.7, 5.7, 2.9, 6.7], [47.8, 36.5, 15.7, 2.3, 7.2], [48.6, 37.1, 14.3, 2.1,
 * 7.2], [61.6, 25.5, 12.9, 1.9, 7.3], [58.6, 26.5, 14.9, 2.4, 6.7], [69.3,
 * 22.3, 8.4, 4, 7], [61.8, 30.8, 7.4, 2.7, 6.4], [67.7, 25.3, 7, 4.8, 7.3],
 * [57.2, 31.2, 11.6, 2.4, 6.5], [67.2, 22.7, 10.1, 3.3, 6.2], [59.2, 31.2,
 * 9.6, 2.4, 6], [80.2, 13.2, 6.6, 2, 5.8], [82.2, 11.1, 6.7, 2.2, 7.2],
 * [69.7, 20.7, 9.6, 3.1, 5.9]], datatype=float[8], order='C_order'):
 * 
 * cmeans := Matrix( [[82.5, 10, 7.5, 1.5, 6.5], [47.8, 36.5, 15.7, 2.3, 7.2],
 * [67.2, 22.7, 10.1, 3.3, 6.2]], datatype=float[8], order='C_order'):
 * 
 * 
 * Results
 * 
 * cmeans := Matrix([ [81.1833333333333371, 11.6666666666666661,
 * 7.1499999999999947, 2.0500000000000027, 6.6000000000000052],
 * [47.8666666666666671, 35.8000000000000043, 16.3333333333333321,
 * 2.3999999999999992, 6.7333333333333340], [64.0454545454545610,
 * 25.2090909090909037, 10.7454545454545425, 2.83636363636363642,
 * 6.65454545454545521]]):
 * 
 * inc := Vector([0, 0, 2, 1, 2, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
 * 2]):
 * 
 * nic := Vector([6, 3, 11]):
 * 
 * css := Vector([46.5716666666666583, 20.3800000000000097,
 * 468.896363636363503]):
 * 
 */
public static void prepareTestInput(Configuration conf, FileSystem fs, Path in, Path centerIn)
        throws IOException {

    // Delete input files if already exist
    if (fs.exists(in)) {
        fs.delete(in, true);
    }
    if (fs.exists(centerIn)) {
        fs.delete(centerIn, true);
    }

    double[][] input = { { 77.3, 13, 9.699999999999999, 1.5, 6.4 }, { 82.5, 10, 7.5, 1.5, 6.5 },
            { 66.90000000000001, 20.6, 12.5, 2.3, 7 }, { 47.2, 33.8, 19, 2.8, 5.8 },
            { 65.3, 20.5, 14.2, 1.9, 6.9 }, { 83.3, 10, 6.7, 2.2, 7 },
            { 81.59999999999999, 12.7, 5.7, 2.9, 6.7 }, { 47.8, 36.5, 15.7, 2.3, 7.2 },
            { 48.6, 37.1, 14.3, 2.1, 7.2 }, { 61.6, 25.5, 12.9, 1.9, 7.3 }, { 58.6, 26.5, 14.9, 2.4, 6.7 },
            { 69.3, 22.3, 8.4, 4, 7 }, { 61.8, 30.8, 7.4, 2.7, 6.4 }, { 67.7, 25.3, 7, 4.8, 7.3 },
            { 57.2, 31.2, 11.6, 2.4, 6.5 }, { 67.2, 22.7, 10.1, 3.3, 6.2 }, { 59.2, 31.2, 9.6, 2.4, 6 },
            { 80.2, 13.2, 6.6, 2, 5.8 }, { 82.2, 11.1, 6.7, 2.2, 7.2 }, { 69.7, 20.7, 9.6, 3.1, 5.9 } };
    double[][] centers = { { 82.5, 10, 7.5, 1.5, 6.5 }, { 47.8, 36.5, 15.7, 2.3, 7.2 },
            { 67.2, 22.7, 10.1, 3.3, 6.2 } };

    final NullWritable nullValue = NullWritable.get();

    // Write inputs
    LOG.info("inputs: ");
    final SequenceFile.Writer dataWriter = SequenceFile.createWriter(fs, conf, in, PipesVectorWritable.class,
            NullWritable.class, CompressionType.NONE);

    for (int i = 0; i < input.length; i++) {
        dataWriter.append(new PipesVectorWritable(new DenseDoubleVector(input[i])), nullValue);
        LOG.info("input[" + i + "]: " + Arrays.toString(input[i]));
    }

    dataWriter.close();

    // Write centers
    LOG.info("centers: ");
    final SequenceFile.Writer centerWriter = SequenceFile.createWriter(fs, conf, centerIn,
            PipesVectorWritable.class, NullWritable.class, CompressionType.NONE);

    for (int i = 0; i < centers.length; i++) {
        centerWriter.append(new PipesVectorWritable(new DenseDoubleVector(centers[i])), nullValue);
        LOG.info("center[" + i + "]: " + Arrays.toString(centers[i]));
    }

    centerWriter.close();
}

---

From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java

License:Apache License

static void printOutput(BSPJob job, Path path) throws IOException {
    FileSystem fs = path.getFileSystem(job.getConfiguration());
    FileStatus[] files = fs.listStatus(path);
    for (int i = 0; i < files.length; i++) {
        if (files[i].getLen() > 0) {
            System.out.println("File " + files[i].getPath());
            SequenceFile.Reader reader = null;
            try {
                reader = new SequenceFile.Reader(fs, files[i].getPath(), job.getConfiguration());

                NullWritable key = NullWritable.get();
                IntWritable value = new IntWritable();
                while (reader.next(key, value)) {
                    System.out.println("key: '" + key + "' value: '" + value.get() + "'\n");
                }//  w w  w .j av a 2s.  co m
            } catch (IOException e) {
                FSDataInputStream in = fs.open(files[i].getPath());
                IOUtils.copyBytes(in, System.out, job.getConfiguration(), false);
                in.close();
            } finally {
                if (reader != null) {
                    reader.close();
                }
            }
        }
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}

---

From source file:backup.namenode.NameNodeBackupBlockCheckProcessor.java

License:Apache License

private ExternalExtendedBlockSort<NullWritable> fetchBlocksFromBackupStore(BackupReportWriter writer)
        throws Exception {
    Path sortDir = getLocalSort(BACKUPSTORE_SORT);
    ExternalExtendedBlockSort<NullWritable> backupBlocks = new ExternalExtendedBlockSort<NullWritable>(conf,
            sortDir, NullWritable.class);
    ExtendedBlockEnum<Void> extendedBlocks = backupStore.getExtendedBlocks();
    ExtendedBlock block;// ww w . j  av a2  s.  c  o m
    NullWritable value = NullWritable.get();
    long st = System.nanoTime();
    while ((block = extendedBlocks.next()) != null) {
        if (st + TimeUnit.SECONDS.toNanos(10) < System.nanoTime()) {
            writer.statusBlockMetaDataFetchFromBackStore(block);
            st = System.nanoTime();
        }
        writer.statusExtendedBlocksFromBackStore(block);
        backupBlocks.add(block, value);
    }
    return backupBlocks;
}

---

From source file:boa.aggregators.Aggregator.java

License:Apache License

@SuppressWarnings("unchecked")
protected void collect(final String data, final String metadata) throws IOException, InterruptedException {
    if (this.combining)
        this.getContext().write(this.getKey(), new EmitValue(data, metadata));
    else if (metadata != null)
        this.getContext().write(new Text(this.getKey() + " = " + data + " weight " + metadata),
                NullWritable.get());
    else//from   w  ww.  j a  v  a 2  s .c  om
        this.getContext().write(new Text(this.getKey() + " = " + data), NullWritable.get());
}

---

From source file:boa.aggregators.GraphAggregator.java

License:Apache License

/** {@inheritDoc} */
@Override/*from   w  w  w .ja  va 2s.c o  m*/
@SuppressWarnings("unchecked")
protected void collect(final String data, final String metadata) throws IOException, InterruptedException {
    if (this.isCombining()) {
        this.getContext().write(this.getKey(), new EmitValue(data, metadata));
        return;
    }
    this.getContext().write(new Text(format(this.getKey().getIndex(), data, metadata)), NullWritable.get());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »