Example usage for org.apache.hadoop.io IntWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io IntWritable get.

Prototype

public int get() 

Source Link

Document

Return the value of this IntWritable.

Usage

From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java

License:Apache License

static void printOutput(BSPJob job, Path path) throws IOException {
    FileSystem fs = path.getFileSystem(job.getConfiguration());
    FileStatus[] files = fs.listStatus(path);
    for (int i = 0; i < files.length; i++) {
        if (files[i].getLen() > 0) {
            System.out.println("File " + files[i].getPath());
            SequenceFile.Reader reader = null;
            try {
                reader = new SequenceFile.Reader(fs, files[i].getPath(), job.getConfiguration());

                NullWritable key = NullWritable.get();
                IntWritable value = new IntWritable();
                while (reader.next(key, value)) {
                    System.out.println("key: '" + key + "' value: '" + value.get() + "'\n");
                }/* w  ww  .  ja  v  a  2  s .  c o  m*/
            } catch (IOException e) {
                FSDataInputStream in = fs.open(files[i].getPath());
                IOUtils.copyBytes(in, System.out, job.getConfiguration(), false);
                in.close();
            } finally {
                if (reader != null) {
                    reader.close();
                }
            }
        }
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}

---

From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.cpu.MatrixMultiplicationBSPCpu.java

License:Apache License

@Override
public void setup(
        BSPPeer<IntWritable, PipesVectorWritable, IntWritable, PipesVectorWritable, MatrixRowMessage> peer)
        throws IOException {

    Configuration conf = peer.getConfiguration();
    m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false);

    // Choose one as a master, who sorts the matrix rows at the end
    // m_masterTask = peer.getPeerName(peer.getNumPeers() / 2);
    // TODO//from ww w  .ja v  a  2  s. c  o m
    // task must be 0 otherwise write out does NOT work!
    m_masterTask = peer.getPeerName(0);

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    // Receive transposed Matrix B
    SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf),
            new Path(conf.get(CONF_MATRIX_MULT_B_PATH)), conf);

    IntWritable bKey = new IntWritable();
    PipesVectorWritable bVector = new PipesVectorWritable();
    // for each col of matrix B (cause by transposed B)
    while (reader.next(bKey, bVector)) {
        m_bColumns.add(new KeyValuePair<Integer, DoubleVector>(bKey.get(), bVector.getVector()));
        if (m_isDebuggingEnabled) {
            m_logger.writeChars("setup,read,transposedMatrixB,key=" + bKey.get() + ",value="
                    + bVector.getVector().toString() + "\n");
        }
    }
    reader.close();
}

---

From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.gpu.MatrixMultiplicationBSPGpu.java

License:Apache License

@Override
public void setup(BSPPeer<IntWritable, VectorWritable, IntWritable, VectorWritable, NullWritable> peer)
        throws IOException {

    Configuration conf = peer.getConfiguration();
    m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false);

    // Choose one as a master, who sorts the matrix rows at the end
    // m_masterTask = peer.getPeerName(peer.getNumPeers() / 2);
    // TODO/*  www . j a v  a2s . com*/
    // task must be 0 otherwise write out does NOT work!
    m_masterTask = peer.getPeerName(0);

    this.m_blockSize = Integer.parseInt(peer.getConfiguration().get(CONF_BLOCKSIZE));

    this.m_gridSize = Integer.parseInt(peer.getConfiguration().get(CONF_GRIDSIZE));

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    // Load matrixB
    SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf),
            new Path(conf.get(CONF_MATRIX_MULT_B_PATH)), conf);

    List<DoubleVector> matrixB = new ArrayList<DoubleVector>();
    IntWritable bKey = new IntWritable();
    VectorWritable bVector = new VectorWritable();
    // for each row of matrix B
    while (reader.next(bKey, bVector)) {
        matrixB.add(bVector.getVector());

        if (m_isDebuggingEnabled) {
            m_logger.writeChars("bsp,setup,MatrixB (" + bKey.get() + "," + bVector.getVector() + ")\n");
        }
    }
    reader.close();

    // Convert matrixB to double array for GPU kernels
    m_matrixBArr = toArray(matrixB);

    if (m_isDebuggingEnabled) {
        for (int i = 0; i < m_matrixBArr.length; i++) {
            m_logger.writeChars("bsp,setup,MatrixBArr (" + i + "," + Arrays.toString(m_matrixBArr[i]) + ")\n");
        }
    }

    // threadSliceSize defines how much multipliers
    // of column B has to be multiplied with column A
    m_threadSliceSize = divup(m_matrixBArr.length, m_blockSize);

    // blockSliceSize defines the column slice amount
    // columns of B per blockIters
    m_blockSliceSize = divup(m_matrixBArr[0].length, m_gridSize);

    if (m_isDebuggingEnabled) {
        m_logger.writeChars("bsp,setup,blockSize=" + m_blockSize + ",gridSize=" + m_gridSize
                + ",threadSliceSize=" + m_threadSliceSize + ",blockSliceSize=" + m_blockSliceSize + "\n");
    }
}

---

From source file:be.uantwerpen.adrem.disteclat.PrefixComputerMapper.java

License:Apache License

/**
 * Reads the singleton items with their tid lists from the specified file.
 * //from  ww  w  .j ava2  s  . c o m
 * @param conf
 * @param path
 * @return
 * @throws IOException
 * @throws URISyntaxException
 */
private static List<Item> readTidLists(Configuration conf, Path path) throws IOException, URISyntaxException {
    SequenceFile.Reader r = new SequenceFile.Reader(FileSystem.get(new URI("file:///"), conf), path, conf);

    List<Item> items = newArrayList();

    IntWritable key = new IntWritable();
    IntMatrixWritable value = new IntMatrixWritable();

    while (r.next(key, value)) {
        final int[][] tids = value.toIntMatrix();
        int support = 0;

        for (int[] partTids : tids) {
            if (partTids != null) {
                support += partTids.length;
            }
        }

        items.add(new Item(key.get(), support, tids));
    }
    r.close();

    return items;
}

---

From source file:bookExamples.ch4.algorithms.ConnectedComponentsVertex.java

License:Apache License

/**
 * Propagates the smallest vertex id to all neighbors. Will always choose to
 * halt and only reactivate if a smaller id has been sent to it.
 *
 * @param vertex Vertex// w w w . j a  v  a 2 s .com
 * @param messages Iterator of messages from the previous superstep.
 * @throws IOException
 */
@Override
public void compute(Vertex<IntWritable, IntWritable, NullWritable> vertex, Iterable<IntWritable> messages)
        throws IOException {
    int currentComponent = vertex.getValue().get();

    // First superstep is special, because we can simply look at the neighbors
    if (getSuperstep() == 0) {
        for (Edge<IntWritable, NullWritable> edge : vertex.getEdges()) {
            int neighbor = edge.getTargetVertexId().get();
            if (neighbor < currentComponent) {
                currentComponent = neighbor;
            }
        }
        // Only need to send value if it is not the own id
        if (currentComponent != vertex.getValue().get()) {
            vertex.setValue(new IntWritable(currentComponent));
            for (Edge<IntWritable, NullWritable> edge : vertex.getEdges()) {
                IntWritable neighbor = edge.getTargetVertexId();
                if (neighbor.get() > currentComponent) {
                    sendMessage(neighbor, vertex.getValue());
                }
            }
        }

        vertex.voteToHalt();
        return;
    }

    boolean changed = false;
    // did we get a smaller id ?
    for (IntWritable message : messages) {
        int candidateComponent = message.get();
        if (candidateComponent < currentComponent) {
            currentComponent = candidateComponent;
            changed = true;
        }
    }

    // propagate new component id to the neighbors
    if (changed) {
        vertex.setValue(new IntWritable(currentComponent));
        sendMessageToAllEdges(vertex, vertex.getValue());
    }
    vertex.voteToHalt();
}

---

From source file:boostingPL.boosting.BoostingPLFactory.java

License:Open Source License

public static Classifier createBoostingPL(String name, Configuration conf, Path path) throws IOException {
    FileSystem hdfs = FileSystem.get(conf);
    @SuppressWarnings("deprecation")
    SequenceFile.Reader in = new SequenceFile.Reader(hdfs, path, conf);

    IntWritable key = new IntWritable();
    ArrayList<ArrayList<ClassifierWritable>> classifiersW = new ArrayList<ArrayList<ClassifierWritable>>();
    ArrayList<ClassifierWritable> ws = null;
    while (in.next(key)) {
        // key is in order
        if (key.get() + 1 > classifiersW.size()) {
            ws = new ArrayList<ClassifierWritable>();
            classifiersW.add(ws);//from  w w w  . j a  va2  s. c o m
        }
        ClassifierWritable value = new ClassifierWritable();
        in.getCurrentValue(value);
        ws.add(value);
    }
    in.close();

    System.out.println("Number of Worker:" + classifiersW.size());
    System.out.println("Number of Iteration:" + classifiersW.get(0).size());
    System.out.println();

    double[][] corWeights = new double[classifiersW.size()][classifiersW.get(0).size()];
    Classifier[][] classifiers = new Classifier[classifiersW.size()][classifiersW.get(0).size()];

    for (int i = 0; i < classifiersW.size(); i++) {
        for (int j = 0; j < classifiersW.get(i).size(); j++) {
            ClassifierWritable c = classifiersW.get(i).get(j);
            classifiers[i][j] = c.getClassifier();
            corWeights[i][j] += c.getCorWeight();
        }
    }

    return createBoostingPL(name, classifiers, corWeights);
}

---

From source file:br.com.lassal.mrunit.example.mapreduce.SMSCDRReducer.java

protected void reduce(Text key, Iterable<IntWritable> values, Context context)
        throws java.io.IOException, InterruptedException {
    int sum = 0;/* w  w w .  ja  v  a 2 s  .co m*/
    for (IntWritable value : values) {
        sum += value.get();
    }
    context.write(key, new IntWritable(sum));
}

---

From source file:brickhouse.udf.sketch.ConvertToSketchUDF.java

License:Apache License

@Override
public ObjectInspector initialize(ObjectInspector[] arg0) throws UDFArgumentException {
    if (arg0.length > 2) {
        throw new UDFArgumentException(
                "convert_to_sketch takes an array of strings, and an optional sketch set size.");
    }//  www.j a  v a 2 s  .  c  o  m
    if (arg0[0].getCategory() != Category.LIST) {
        throw new UDFArgumentException(
                "convert_to_sketch takes an array of strings, and an optional sketch set size.");
    }
    listInspector = (ListObjectInspector) arg0[0];
    if (listInspector.getListElementObjectInspector().getCategory() != Category.PRIMITIVE
            || ((PrimitiveObjectInspector) listInspector.getListElementObjectInspector())
                    .getPrimitiveCategory() != PrimitiveCategory.STRING) {
        throw new UDFArgumentException(
                "convert_to_sketch takes an array of strings, and an optional sketch set size.");
    }
    listElemInspector = (StringObjectInspector) listInspector.getListElementObjectInspector();

    if (arg0.length > 1) {
        if (!(arg0[1] instanceof IntObjectInspector) || !(arg0[1] instanceof ConstantObjectInspector)) {
            throw new UDFArgumentException("sketch set size must be a constant int value.");
        }
        IntWritable sizeInt = (IntWritable) ((ConstantObjectInspector) arg0[1]).getWritableConstantValue();
        sketchSetSize = sizeInt.get();
    }

    retInspector = ObjectInspectorFactory
            .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    return retInspector;
}

---

From source file:cc.slda.AnnotateDocuments.java

License:Apache License

public static Map<Integer, String> importParameter(SequenceFile.Reader sequenceFileReader) throws IOException {
    Map<Integer, String> hashMap = new HashMap<Integer, String>();

    IntWritable intWritable = new IntWritable();
    Text text = new Text();
    while (sequenceFileReader.next(intWritable, text)) {
        hashMap.put(intWritable.get(), text.toString());
    }/*ww  w  . jav a 2s  .c  o m*/

    return hashMap;
}

---

From source file:cc.slda.DisplayTopic.java

License:Apache License

@SuppressWarnings("unchecked")
public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(Settings.HELP_OPTION, false, "print the help message");
    options.addOption(OptionBuilder.withArgName(Settings.PATH_INDICATOR).hasArg()
            .withDescription("input beta file").create(Settings.INPUT_OPTION));
    options.addOption(OptionBuilder.withArgName(Settings.PATH_INDICATOR).hasArg()
            .withDescription("term index file").create(ParseCorpus.INDEX));
    options.addOption(OptionBuilder.withArgName(Settings.INTEGER_INDICATOR).hasArg()
            .withDescription("display top terms only (default - 10)").create(TOP_DISPLAY_OPTION));

    String betaString = null;//ww w.  j  a va 2  s.  co m
    String indexString = null;
    int topDisplay = TOP_DISPLAY;

    CommandLineParser parser = new GnuParser();
    HelpFormatter formatter = new HelpFormatter();
    try {
        CommandLine line = parser.parse(options, args);

        if (line.hasOption(Settings.HELP_OPTION)) {
            formatter.printHelp(ParseCorpus.class.getName(), options);
            System.exit(0);
        }

        if (line.hasOption(Settings.INPUT_OPTION)) {
            betaString = line.getOptionValue(Settings.INPUT_OPTION);
        } else {
            throw new ParseException("Parsing failed due to " + Settings.INPUT_OPTION + " not initialized...");
        }

        if (line.hasOption(ParseCorpus.INDEX)) {
            indexString = line.getOptionValue(ParseCorpus.INDEX);
        } else {
            throw new ParseException("Parsing failed due to " + ParseCorpus.INDEX + " not initialized...");
        }

        if (line.hasOption(TOP_DISPLAY_OPTION)) {
            topDisplay = Integer.parseInt(line.getOptionValue(TOP_DISPLAY_OPTION));
        }
    } catch (ParseException pe) {
        System.err.println(pe.getMessage());
        formatter.printHelp(ParseCorpus.class.getName(), options);
        System.exit(0);
    } catch (NumberFormatException nfe) {
        System.err.println(nfe.getMessage());
        System.exit(0);
    }

    JobConf conf = new JobConf(DisplayTopic.class);
    FileSystem fs = FileSystem.get(conf);

    Path indexPath = new Path(indexString);
    Preconditions.checkArgument(fs.exists(indexPath) && fs.isFile(indexPath), "Invalid index path...");

    Path betaPath = new Path(betaString);
    Preconditions.checkArgument(fs.exists(betaPath) && fs.isFile(betaPath), "Invalid beta path...");

    SequenceFile.Reader sequenceFileReader = null;
    try {
        IntWritable intWritable = new IntWritable();
        Text text = new Text();
        Map<Integer, String> termIndex = new HashMap<Integer, String>();
        sequenceFileReader = new SequenceFile.Reader(fs, indexPath, conf);
        while (sequenceFileReader.next(intWritable, text)) {
            termIndex.put(intWritable.get(), text.toString());
        }

        PairOfIntFloat pairOfIntFloat = new PairOfIntFloat();
        // HMapIFW hmap = new HMapIFW();
        HMapIDW hmap = new HMapIDW();
        TreeMap<Double, Integer> treeMap = new TreeMap<Double, Integer>();
        sequenceFileReader = new SequenceFile.Reader(fs, betaPath, conf);
        while (sequenceFileReader.next(pairOfIntFloat, hmap)) {
            treeMap.clear();

            System.out.println("==============================");
            System.out.println(
                    "Top ranked " + topDisplay + " terms for Topic " + pairOfIntFloat.getLeftElement());
            System.out.println("==============================");

            Iterator<Integer> itr1 = hmap.keySet().iterator();
            int temp1 = 0;
            while (itr1.hasNext()) {
                temp1 = itr1.next();
                treeMap.put(-hmap.get(temp1), temp1);
                if (treeMap.size() > topDisplay) {
                    treeMap.remove(treeMap.lastKey());
                }
            }

            Iterator<Double> itr2 = treeMap.keySet().iterator();
            double temp2 = 0;
            while (itr2.hasNext()) {
                temp2 = itr2.next();
                if (termIndex.containsKey(treeMap.get(temp2))) {
                    System.out.println(termIndex.get(treeMap.get(temp2)) + "\t\t" + -temp2);
                } else {
                    System.out.println("How embarrassing! Term index not found...");
                }
            }
        }
    } finally {
        IOUtils.closeStream(sequenceFileReader);
    }

    return 0;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »