Example usage for org.apache.hadoop.io IntWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io IntWritable get.

Prototype

public int get() 

Source Link

Document

Return the value of this IntWritable.

Usage

From source file:Assignment5_P3_PartitionPattern.Partition_IPAddress_By_Month_Partitioner.java

@Override
public int getPartition(IntWritable key, Text value, int numOfPartitions) {
    return (key.get() - minMonth);
}

---

From source file:at.illecker.hama.hybrid.examples.hellohybrid.HelloHybridBSP.java

License:Apache License

@Override
public void bsp(BSPPeer<IntWritable, NullWritable, IntWritable, NullWritable, NullWritable> peer)
        throws IOException, SyncException, InterruptedException {

    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs/*from  w ww  .  j ava2 s .c  om*/
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("HelloHybrid.bsp executed on CPU!\n");

    ArrayList<Integer> summation = new ArrayList<Integer>();

    // test input
    IntWritable key = new IntWritable();
    NullWritable nullValue = NullWritable.get();

    while (peer.readNext(key, nullValue)) {
        outStream.writeChars("input: key: '" + key.get() + "'\n");
        summation.add(key.get());
    }

    // test sequenceFileReader
    Path example = new Path(peer.getConfiguration().get(CONF_EXAMPLE_PATH));
    SequenceFile.Reader reader = null;
    try {
        reader = new SequenceFile.Reader(fs, example, peer.getConfiguration());

        int i = 0;
        while (reader.next(key, nullValue)) {
            outStream.writeChars("sequenceFileReader: key: '" + key.get() + "'\n");
            if (i < summation.size()) {
                summation.set(i, summation.get(i) + key.get());
            }
            i++;
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        if (reader != null) {
            reader.close();
        }
    }

    // test output
    for (Integer i : summation) {
        key.set(i);
        outStream.writeChars("output: key: '" + key.get() + "'\n");
        peer.write(key, nullValue);
    }

    // test getAllPeerNames
    outStream.writeChars("getAllPeerNames: '" + Arrays.toString(peer.getAllPeerNames()) + "'\n");

    // test String.split
    String splitString = "boo:and:foo";
    String[] splits;

    outStream.writeChars("splitString: '" + splitString + "'\n");

    splits = splitString.split(":");
    outStream.writeChars("split(\":\") len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    splits = splitString.split(":", 2);
    outStream.writeChars(
            "split(\":\",2) len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    splits = splitString.split(":", 5);
    outStream.writeChars(
            "split(\":\",5) len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    splits = splitString.split(":", -2);
    outStream.writeChars(
            "split(\":\",-2) len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    splits = splitString.split(";");
    outStream.writeChars("split(\";\") len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    outStream.close();
}

---

From source file:at.illecker.hama.hybrid.examples.hellohybrid.HelloHybridBSP.java

License:Apache License

static void printOutput(BSPJob job, Path path) throws IOException {
    FileSystem fs = path.getFileSystem(job.getConfiguration());
    FileStatus[] files = fs.listStatus(path);
    for (int i = 0; i < files.length; i++) {
        if (files[i].getLen() > 0) {
            System.out.println("File " + files[i].getPath());
            SequenceFile.Reader reader = null;
            try {
                reader = new SequenceFile.Reader(fs, files[i].getPath(), job.getConfiguration());

                IntWritable key = new IntWritable();
                NullWritable value = NullWritable.get();
                while (reader.next(key, value)) {
                    System.out.println("key: '" + key.get() + "' value: '" + value + "'\n");
                }//  w ww  . j av  a  2 s.  c o m
            } catch (IOException e) {
                FSDataInputStream in = fs.open(files[i].getPath());
                IOUtils.copyBytes(in, System.out, job.getConfiguration(), false);
                in.close();
            } finally {
                if (reader != null) {
                    reader.close();
                }
            }
        }
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}

---

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication.MatrixMultiplicationHybridBSP.java

License:Apache License

/********************************* CPU *********************************/
@Override/* w  ww.  j  av  a  2  s  .  c  om*/
public void setup(
        BSPPeer<IntWritable, PipesVectorWritable, IntWritable, PipesVectorWritable, MatrixRowMessage> peer)
        throws IOException {

    HamaConfiguration conf = peer.getConfiguration();
    this.m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false);

    // Choose one as a master, who sorts the matrix rows at the end
    // m_masterTask = peer.getPeerName(peer.getNumPeers() / 2);

    // TODO
    // task must be 0 otherwise write out does NOT work!
    this.m_masterTask = peer.getPeerName(0);

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    // Load transposed Matrix B
    SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf),
            new Path(conf.get(CONF_MATRIX_MULT_B_PATH)), conf);

    IntWritable bKey = new IntWritable();
    PipesVectorWritable bVector = new PipesVectorWritable();

    // for each col of matrix B (cause by transposed B)
    while (reader.next(bKey, bVector)) {
        m_bColumns.add(new KeyValuePair<Integer, DoubleVector>(bKey.get(), bVector.getVector()));
        if (m_isDebuggingEnabled) {
            m_logger.writeChars("setup,read,transposedMatrixB,key=" + bKey.get() + ",value="
                    + bVector.getVector().toString() + "\n");
        }
    }
    reader.close();
}

---

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication.MatrixMultiplicationHybridBSP.java

License:Apache License

@Override
public void bsp(
        BSPPeer<IntWritable, PipesVectorWritable, IntWritable, PipesVectorWritable, MatrixRowMessage> peer)
        throws IOException, SyncException, InterruptedException {

    IntWritable aKey = new IntWritable();
    PipesVectorWritable aVector = new PipesVectorWritable();
    // while for each row of matrix A
    while (peer.readNext(aKey, aVector)) {

        // Logging
        if (m_isDebuggingEnabled) {
            m_logger.writeChars("bsp,input,key=" + aKey + ",value=" + aVector.getVector().toString() + "\n");
        }/*from  w w  w  .  jav  a 2s .  c om*/

        DenseDoubleVector outVector = null;
        // for each col of matrix B (cause by transposed B)
        for (KeyValuePair<Integer, DoubleVector> bVectorRow : m_bColumns) {

            if (outVector == null) {
                outVector = new DenseDoubleVector(bVectorRow.getValue().getDimension());
            }

            double dot = aVector.getVector().dot(bVectorRow.getValue());

            outVector.set(bVectorRow.getKey(), dot);
        }

        peer.send(m_masterTask, new MatrixRowMessage(aKey.get(), outVector));

        if (m_isDebuggingEnabled) {
            m_logger.writeChars("bsp,send,key=" + aKey.get() + ",value=" + outVector.toString() + "\n");
            m_logger.flush();
        }
    }

    peer.sync();

    // MasterTask accumulates result
    if (peer.getPeerName().equals(m_masterTask)) {

        MatrixRowMessage currentMatrixRowMessage = null;

        // Collect messages
        while ((currentMatrixRowMessage = peer.getCurrentMessage()) != null) {

            int rowIndex = currentMatrixRowMessage.getRowIndex();
            DoubleVector rowValues = currentMatrixRowMessage.getRowValues();

            if (m_isDebuggingEnabled) {
                m_logger.writeChars("bsp,write,key=" + rowIndex + ",value=" + rowValues.toString() + "\n");
            }
            peer.write(new IntWritable(rowIndex), new PipesVectorWritable(rowValues));
        }
    }
}

---

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBSP.java

License:Apache License

/********************************* CPU *********************************/
@Override//from  ww w.jav a  2  s. co m
public void setup(BSPPeer<IntWritable, VectorWritable, IntWritable, VectorWritable, MatrixRowMessage> peer)
        throws IOException {

    HamaConfiguration conf = peer.getConfiguration();
    this.m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false);

    // used by GPU only
    m_tileWidth = conf.getInt(CONF_TILE_WIDTH, 32);

    // Choose one as a master, who sorts the matrix rows at the end
    // this.m_masterTask = peer.getPeerName(peer.getNumPeers() / 2);
    // TODO
    // task must be 0 otherwise write out does NOT work!
    this.m_masterTask = peer.getPeerName(0);

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    // Load transposed Matrix B
    SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf),
            new Path(conf.get(CONF_MATRIX_B_PATH)), conf);

    IntWritable transposedMatrixBRowId = new IntWritable();
    VectorWritable transposedMatrixBRow = new VectorWritable();

    // for each col of matrix B (cause by transposed B)
    while (reader.next(transposedMatrixBRowId, transposedMatrixBRow)) {
        m_transposedMatrixB.add(new KeyValuePair<Integer, DoubleVector>(transposedMatrixBRowId.get(),
                transposedMatrixBRow.getVector()));
        if (m_isDebuggingEnabled) {
            m_logger.writeChars("setup,read,transposedMatrixB,key=" + transposedMatrixBRowId.get() + ",value="
                    + transposedMatrixBRow.getVector().toString() + "\n");
        }
    }
    reader.close();
}

---

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBSP.java

License:Apache License

@Override
public void bsp(BSPPeer<IntWritable, VectorWritable, IntWritable, VectorWritable, MatrixRowMessage> peer)
        throws IOException, SyncException, InterruptedException {

    IntWritable matrixARowId = new IntWritable();
    VectorWritable matrixARow = new VectorWritable();
    // while for each row of matrix A
    while (peer.readNext(matrixARowId, matrixARow)) {

        // Logging
        if (m_isDebuggingEnabled) {
            m_logger.writeChars(//from w  w  w  .jav  a  2  s .c o  m
                    "bsp,input,key=" + matrixARowId + ",value=" + matrixARow.getVector().toString() + "\n");
        }

        DenseDoubleVector outVector = null;
        // for each column of matrix B (cause by transposed matrix B)
        for (KeyValuePair<Integer, DoubleVector> bVectorRow : m_transposedMatrixB) {
            if (outVector == null) { // init outVector only once
                outVector = new DenseDoubleVector(m_transposedMatrixB.size());
            }
            double dot = matrixARow.getVector().dot(bVectorRow.getValue());

            outVector.set(bVectorRow.getKey(), dot);
        }

        peer.send(m_masterTask, new MatrixRowMessage(matrixARowId.get(), outVector));

        if (m_isDebuggingEnabled) {
            m_logger.writeChars("bsp,send,key=" + matrixARowId.get() + ",value=" + outVector.toString() + "\n");
            m_logger.flush();
        }
    }

    peer.sync();

    // the master task writes out the incoming messages
    if (peer.getPeerName().equals(m_masterTask)) {

        MatrixRowMessage currentMatrixRowMessage = null;

        // Collect messages
        while ((currentMatrixRowMessage = peer.getCurrentMessage()) != null) {

            int rowIndex = currentMatrixRowMessage.getRowIndex();
            DoubleVector rowValues = currentMatrixRowMessage.getRowValues();

            if (m_isDebuggingEnabled) {
                m_logger.writeChars("bsp,write,key=" + rowIndex + ",value=" + rowValues.toString() + "\n");
            }
            peer.write(new IntWritable(rowIndex), new VectorWritable(rowValues));
        }
    }
}

---

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBSP.java

License:Apache License

@Override
public void bspGpu(BSPPeer<IntWritable, VectorWritable, IntWritable, VectorWritable, MatrixRowMessage> peer,
        Rootbeer rootbeer) throws IOException, SyncException, InterruptedException {

    // Fetch inputs
    List<KeyValuePair<Integer, DoubleVector>> matrixA = new ArrayList<KeyValuePair<Integer, DoubleVector>>();
    final IntWritable matrixARowId = new IntWritable();
    final VectorWritable matrixARow = new VectorWritable();
    while (peer.readNext(matrixARowId, matrixARow)) {
        matrixA.add(new KeyValuePair<Integer, DoubleVector>(matrixARowId.get(), matrixARow.getVector()));
        if (m_isDebuggingEnabled) {
            m_logger.writeChars("bspGpu,matrixA,key=" + matrixARowId.get() + ",value="
                    + matrixARow.getVector().toString() + "\n");
        }/*  w  ww .j a va  2s .  co  m*/
    }

    // Convert data for GPU
    // n - rows of matrix A
    int n = matrixA.size();
    // m - cols of matrix A and rows of matrix B
    int m = matrixA.get(0).getValue().getDimension();
    // l - cols of matrix B
    int l = m_transposedMatrixB.size();

    double[] transposedmatrixA = new double[m * n];
    double[] matrixB = new double[m * l];
    double[] matrixC = new double[n * l];

    // Convert matrixA rows to double[]
    int i = 0;
    for (KeyValuePair<Integer, DoubleVector> row : matrixA) {
        for (int j = 0; j < m; j++) {
            // store row column wise to get a transposed matrix A
            transposedmatrixA[(j * n) + i] = row.getValue().get(j);
        }
        i++;
    }
    // Convert transposedMatrixB rows to double[]
    i = 0;
    for (KeyValuePair<Integer, DoubleVector> row : m_transposedMatrixB) {
        for (int j = 0; j < m; j++) {
            // store row column wise to get a normal matrix B
            matrixB[(j * l) + i] = row.getValue().get(j);
        }
        i++;
    }

    // DEBUG
    if (m_isDebuggingEnabled) {
        m_logger.writeChars("transposedmatrixA: \n");
        printMatrix(transposedmatrixA, m, n);
        m_logger.writeChars("\nmatrixB:\n");
        printMatrix(matrixB, m, l);
        m_logger.writeChars("\n");
    }

    int subMatrixSize = m_tileWidth * m_tileWidth;
    int numberOfSubMatrices = divup(n * l, subMatrixSize);
    int gridSize = numberOfSubMatrices;
    int blockSize = subMatrixSize;

    // int subMatrixSize = tileWidth * tileWidth;
    // rows of A and cols of B per block
    int subMatricesPerThread = divup(m, m_tileWidth);

    if (m_isDebuggingEnabled) {
        m_logger.writeChars("bspGpu,tileWidth: " + m_tileWidth + "\n");
        m_logger.writeChars("bspGpu,gridSize: " + gridSize + "\n");
        m_logger.writeChars("bspGpu,blockSize: " + blockSize + "\n");
        m_logger.writeChars("bspGpu,n: " + n + "\n");
        m_logger.writeChars("bspGpu,m: " + m + "\n");
        m_logger.writeChars("bspGpu,l: " + l + "\n");
        m_logger.writeChars("bspGpu,subMatricesPerThread: " + subMatricesPerThread + "\n");
    }

    MatrixMultiplicationHybridKernel kernel = new MatrixMultiplicationHybridKernel(transposedmatrixA, matrixB,
            matrixC, n, m, l, gridSize, blockSize, m_tileWidth, subMatricesPerThread);

    // Run GPU kernel
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    context.init((long) 1024 * 1024 * 1024); // 1GB
    watch.start();
    rootbeer.run(kernel, new ThreadConfig(blockSize, gridSize, blockSize * gridSize), context);
    watch.stop();

    if (m_isDebuggingEnabled) {
        List<StatsRow> stats = context.getStats();
        for (StatsRow row : stats) {
            m_logger.writeChars("  StatsRow:\n");
            m_logger.writeChars("    serial time: " + row.getSerializationTime() + "\n");
            m_logger.writeChars("    exec time: " + row.getExecutionTime() + "\n");
            m_logger.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
            m_logger.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
            m_logger.writeChars("    num threads: " + row.getNumThreads() + "\n");
        }
        m_logger.writeChars("bspGpu,GPUTime=" + watch.elapsedTimeMillis() + "ms\n");
        m_logger.flush();
    }

    // Send results of GPU kernels
    DenseDoubleVector resultRow = new DenseDoubleVector(l);
    for (int x = 0; x < n; x++) {
        for (int y = 0; y < l; y++) {
            // submit in col-wise order
            resultRow.set(y, matrixC[(x * l) + y]);
        }

        peer.send(m_masterTask, new MatrixRowMessage(x, resultRow));

        if (m_isDebuggingEnabled) {
            m_logger.writeChars("bspGpu,send,key=" + x + ",value=" + resultRow.toString() + "\n");
            m_logger.flush();
        }
    }

    // Global barrier synchronization
    peer.sync();

    // the master task writes out the incoming messages
    if (peer.getPeerName().equals(m_masterTask)) {

        MatrixRowMessage currentMatrixRowMessage = null;

        // Collect messages
        while ((currentMatrixRowMessage = peer.getCurrentMessage()) != null) {

            int rowIndex = currentMatrixRowMessage.getRowIndex();
            DoubleVector rowValues = currentMatrixRowMessage.getRowValues();

            if (m_isDebuggingEnabled) {
                m_logger.writeChars("bspGpu,write,key=" + rowIndex + ",value=" + rowValues.toString() + "\n");
            }
            peer.write(new IntWritable(rowIndex), new VectorWritable(rowValues));
        }
    }
}

---

From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java

License:Apache License

@Override
public void bsp(BSPPeer<IntWritable, IntWritable, NullWritable, IntWritable, NullWritable> peer)
        throws IOException, SyncException, InterruptedException {

    long startTime = System.currentTimeMillis();

    // test input
    int[] input = new int[CONF_BLOCK_SIZE * CONF_GRID_SIZE];
    IntWritable key = new IntWritable();
    IntWritable value = new IntWritable();
    while (peer.readNext(key, value)) {
        input[key.get()] = value.get();
    }//from   ww w. j a v  a2 s  .c o m

    String peerName = peer.getPeerName();
    String[] allPeerNames = peer.getAllPeerNames();

    long stopTime = System.currentTimeMillis();

    // Debug output
    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("TestRootbeerHybridBSP.bsp executed on CPU!\n");
    outStream.writeChars("TestRootbeerHybridBSP,CPUTime=" + (stopTime - startTime) + " ms\n");
    outStream.writeChars("TestRootbeerHybridBSP,CPUTime=" + ((stopTime - startTime) / 1000.0) + " seconds\n");
    outStream.writeChars("TestRootbeerHybridBSP,peerName: '" + peerName + "'\n");
    outStream.writeChars("TestRootbeerHybridBSP,getAllPeerNames: '" + Arrays.toString(allPeerNames) + "'\n");
    // outStream.writeChars("TestRootbeerHybridBSP,input: '"
    // + Arrays.toString(input) + "'\n");

    // Verify input
    peer.reopenInput();
    while (peer.readNext(key, value)) {
        Assert.assertEquals(value.get(), input[key.get()]);
    }

    outStream.writeChars("TestRootbeerHybridBSP.bsp: input verified!'\n");
    outStream.close();
}

---

From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java

License:Apache License

@Override
public void bspGpu(BSPPeer<IntWritable, IntWritable, NullWritable, IntWritable, NullWritable> peer,
        Rootbeer rootbeer) throws IOException, SyncException, InterruptedException {

    TestRootbeerKernel kernel = new TestRootbeerKernel(CONF_BLOCK_SIZE * CONF_GRID_SIZE);

    // Run GPU Kernels
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    watch.start();//from www.ja v  a 2s  .  c o m
    rootbeer.run(kernel, new ThreadConfig(CONF_BLOCK_SIZE, CONF_GRID_SIZE, CONF_BLOCK_SIZE * CONF_GRID_SIZE),
            context);
    watch.stop();

    // Debug output
    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("TestRootbeerHybridBSP.bspGpu executed on GPU!\n");
    List<StatsRow> stats = context.getStats();
    for (StatsRow row : stats) {
        outStream.writeChars("  StatsRow:\n");
        outStream.writeChars("    serial time: " + row.getSerializationTime() + "\n");
        outStream.writeChars("    exec time: " + row.getExecutionTime() + "\n");
        outStream.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
        outStream.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
        outStream.writeChars("    num threads: " + row.getNumThreads() + "\n");
    }

    outStream.writeChars("TestRootbeerHybridBSP,GPUTime=" + watch.elapsedTimeMillis() + " ms\n");
    outStream
            .writeChars("TestRootbeerHybridBSP,GPUTime=" + (watch.elapsedTimeMillis() / 1000.0) + " seconds\n");
    outStream.writeChars("TestRootbeerHybridBSP,peerName: '" + kernel.peerName + "'\n");
    outStream.writeChars(
            "TestRootbeerHybridBSP,getAllPeerNames: '" + Arrays.toString(kernel.allPeerNames) + "'\n");
    // outStream.writeChars("TestRootbeerHybridBSP,input: '"
    // + Arrays.toString(kernel.input) + "'\n");

    // Verify input
    peer.reopenInput();
    IntWritable key = new IntWritable();
    IntWritable value = new IntWritable();
    while (peer.readNext(key, value)) {
        Assert.assertEquals(value.get(), kernel.input[key.get()]);
    }

    outStream.writeChars("TestRootbeerHybridBSP.bspGpu: input verified!'\n");
    outStream.close();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »