Example usage for org.apache.hadoop.io LongWritable toString

List of usage examples for org.apache.hadoop.io LongWritable toString

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. LongWritable
  6. toString

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable toString.

Prototype

@Override
    public String toString()

Source Link

Usage

From source file:net.hubs1.mahout.cluster.CRLFMapper.java

License:Apache License

@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    log.info("key=[" + key.toString() + "],value=[" + value.toString() + "]");
    context.write(new Text(value), new Text(value));
}

From source file:nl.gridline.zieook.inx.czp.CZPMap.java

License:Apache License

@Override
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    // process the input:
    String object = value.toString();

    LOG.info("line #" + key.get());

    CollectionItem item = null;//from   ww w  .j  ava2s  .  com
    try {
        item = czpParser.getCZPfromXML(object);
    } catch (JAXBException e) {
        LOG.error("failed to parse collection item: " + key.toString(), e);
        LOG.error("object:\n" + object + "\n");
    } catch (ParseException e) {
        LOG.error("failed to parse key" + key.toString(), e);
        LOG.error("object:\n" + object + "\n");
    }

    LOG.info("writing: " + item.getId());
    if (item != null) {
        item.setCp(cp);

        Long itemid = item.getId();
        if (itemid != null) {
            // create a HBase PUT object:
            Put put = HBasePUTFactory.getCollectionItemCompl(collection, itemid.longValue(), object, item);
            // reuse the key:

            // write the result:
            context.write(key, put);
        } else {
            LOG.error("collection item has no identifier: " + item.toString() + "' ");
        }

    }
    context.progress();
}

From source file:nl.tudelft.graphalytics.mapreducev2.conversion.DirectedVertexReducer.java

License:Apache License

@Override
protected void reduce(LongWritable key, Iterable<EdgeData> values, Context context)
        throws IOException, InterruptedException {
    // Fill separate buffers for incoming and outgoing edges
    StringBuffer sbIn = new StringBuffer();
    StringBuffer sbOut = new StringBuffer();

    // Loop through the messages and add them to the buffers
    boolean foundIn = false, foundOut = false;
    for (EdgeData edge : values) {
        if (edge.getTargetId() == key.get()) {
            // Ignore, this self-edge was added to force this vertex's existence
        } else if (edge.isOutgoing()) {
            if (foundOut)
                sbOut.append(',');
            sbOut.append(edge.getTargetId());
            foundOut = true;// w ww . ja  v a  2s  . c o m
        } else {
            if (foundIn)
                sbIn.append(',');
            sbIn.append(edge.getTargetId());
            foundIn = true;
        }
    }

    // Combine the vertex ID and neighbour lists using Marcin's format
    StringBuffer out = new StringBuffer(key.toString());
    out.append("\t#").append(sbIn.toString()).append("\t@").append(sbOut.toString());
    if (!foundOut)
        out.append('\t');

    // Output the constructed line
    outValue.set(out.toString());
    context.write(NullWritable.get(), outValue);
}

From source file:nl.tudelft.graphalytics.mapreducev2.evo.DirectedForestFireModelMap.java

License:Apache License

public void map(LongWritable key, Text value, OutputCollector<LongWritable, Text> output, Reporter reporter)
        throws IOException {
    DirectedNode node = new DirectedNode();
    node.readFields(value.toString());/*from   w w  w  .  j  a  v a  2s. com*/

    if (this.isFirst) { // INIT_JOB
        this.isFirst = false;
        // create N new vertices
        for (int i = 0; i < this.newVerticesPerSlot; i++) {
            long newID = this.taskID * this.newVerticesPerSlot + i + this.maxID;
            DirectedNode newVertex = new DirectedNode(String.valueOf(newID), new Vector<Edge>(),
                    new Vector<Edge>());

            this.newVertices.add(new LongWritable(newID)); // same as in Giraph can connect only to worker ambassadors

            oKey.set(newID);
            oVal.set(newVertex.toText());
            output.collect(oKey, oVal);
        }
    } else if (this.ambassadors.containsKey(new LongWritable(Long.parseLong(node.getId())))) { //update vertex
        Vector<Edge> edges = node.getInEdges();

        for (LongWritable id : this.ambassadors.get(new LongWritable(Long.parseLong(node.getId()))))
            edges.add(new Edge(node.getId(), id.toString()));
        node.setInEdges(edges);
    } else if (Long.parseLong(node.getId()) < this.maxID) { // check if potential ambassador n send to new vertex
        Set<LongWritable> edges = new HashSet<LongWritable>();
        for (Edge out : node.getOutEdges())
            edges.add(new LongWritable(Long.parseLong(out.getDest())));
        for (Edge in : node.getInEdges())
            edges.add(new LongWritable(Long.parseLong(in.getSrc())));

        for (LongWritable neighbour : edges) {
            if (ambassadors.containsKey(neighbour)) {
                // send my id to new vertices
                List<LongWritable> newVertices = this.ambassadors.get(neighbour);
                for (LongWritable id : newVertices)
                    output.collect(id, new Text(node.getId()));
            }
        }
    }

    // Init step -> pass all worker verticesIDs to all newVertices from this worker
    if (this.isInit) {
        oVal.set(node.getId());
        for (LongWritable id : this.newVertices) {
            oKey.set(id.get());
            output.collect(oKey, oVal);
        }
    }

    // pass node
    oKey.set(Long.parseLong(node.getId()));
    oVal.set(node.toText());
    output.collect(oKey, oVal);
}

From source file:nl.tudelft.graphalytics.mapreducev2.evo.UndirectedForestFireModelMap.java

License:Apache License

public void map(LongWritable key, Text value, OutputCollector<LongWritable, Text> output, Reporter reporter)
        throws IOException {
    UndirectedNode node = new UndirectedNode();
    node.readFields(value.toString());//from  w  w  w . ja va 2s .  c  o m

    if (this.isFirst) { // INIT_JOB
        this.isFirst = false;
        // create N new vertices
        for (int i = 0; i < this.newVerticesPerSlot; i++) {
            long newID = this.taskID * this.newVerticesPerSlot + i + this.maxID;
            UndirectedNode newVertex = new UndirectedNode(String.valueOf(newID), new Vector<Edge>());

            this.newVertices.add(new LongWritable(newID)); // same as in Giraph can connect only to worker ambassadors

            oKey.set(newID);
            oVal.set(newVertex.toText());
            output.collect(oKey, oVal);
        }
    } else if (this.ambassadors.containsKey(new LongWritable(Long.parseLong(node.getId())))) { //update vertex
        Vector<Edge> edges = node.getEdges();

        for (LongWritable id : this.ambassadors.get(new LongWritable(Long.parseLong(node.getId()))))
            edges.add(new Edge(node.getId(), id.toString()));
        node.setEdges(edges);
    } else if (Long.parseLong(node.getId()) < this.maxID) { // check if potential ambassador n send to new vertex
        for (Edge edge : node.getEdges()) {
            long neighbour = Long.parseLong(edge.getDest());
            if (ambassadors.containsKey(new LongWritable(neighbour))) {
                // send my id to new vertices
                List<LongWritable> newVertices = this.ambassadors.get(new LongWritable(neighbour));
                for (LongWritable id : newVertices)
                    output.collect(id, new Text(node.getId()));
            }
        }
    }

    // Init step -> pass all worker verticesIDs to all newVertices from this worker
    if (this.isInit) {
        oVal.set(node.getId());
        for (LongWritable id : this.newVertices) {
            oKey.set(id.get());
            output.collect(oKey, oVal);
        }
    }

    // pass node
    oKey.set(Long.parseLong(node.getId()));
    oVal.set(node.toText());
    output.collect(oKey, oVal);
}

From source file:org.apache.beam.sdk.io.hadoop.format.HadoopFormatIOReadTest.java

License:Apache License

/**
 * This test validates functionality of {@link HadoopFormatIO.Read#validateTransform()
 * Read.validateTransform()} function when myKeyTranslate's (simple function provided by user for
 * key translation) input type is not same as Hadoop InputFormat's keyClass(Which is property set
 * in configuration as "key.class").//from ww w . j  a v  a2  s .com
 */
@Test
public void testReadValidationFailsWithWrongInputTypeKeyTranslationFunction() {
    SimpleFunction<LongWritable, String> myKeyTranslateWithWrongInputType = new SimpleFunction<LongWritable, String>() {
        @Override
        public String apply(LongWritable input) {
            return input.toString();
        }
    };
    HadoopFormatIO.Read<String, Employee> read = HadoopFormatIO.<String, Employee>read()
            .withConfiguration(serConf.get()).withKeyTranslation(myKeyTranslateWithWrongInputType);
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage(String.format(
            "Key translation's input type is not same as hadoop InputFormat : %s key " + "class : %s",
            serConf.get().getClass("mapreduce.job.inputformat.class", InputFormat.class),
            serConf.get().getClass("key.class", Object.class)));
    read.validateTransform();
}

From source file:org.apache.beam.sdk.io.hadoop.format.HadoopFormatIOReadTest.java

License:Apache License

/**
 * This test validates functionality of {@link HadoopFormatIO.Read#validateTransform()
 * Read.validateTransform()} function when myValueTranslate's (simple function provided by user
 * for value translation) input type is not same as Hadoop InputFormat's valueClass(Which is
 * property set in configuration as "value.class").
 *//*from   ww  w.j a  va 2 s  .  c  o  m*/
@Test
public void testReadValidationFailsWithWrongInputTypeValueTranslationFunction() {
    SimpleFunction<LongWritable, String> myValueTranslateWithWrongInputType = new SimpleFunction<LongWritable, String>() {
        @Override
        public String apply(LongWritable input) {
            return input.toString();
        }
    };
    HadoopFormatIO.Read<Text, String> read = HadoopFormatIO.<Text, String>read()
            .withConfiguration(serConf.get()).withValueTranslation(myValueTranslateWithWrongInputType);
    String expectedMessage = String.format(
            "Value translation's input type is not same as hadoop InputFormat :  " + "%s value class : %s",
            serConf.get().getClass("mapreduce.job.inputformat.class", InputFormat.class),
            serConf.get().getClass("value.class", Object.class));
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage(expectedMessage);
    read.validateTransform();
}

From source file:org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIOTest.java

License:Apache License

/**
 * This test validates functionality of {@link HadoopInputFormatIO.Read#validate()
 * Read.validate()} function when myKeyTranslate's (simple function provided by user for key
 * translation) input type is not same as Hadoop InputFormat's keyClass(Which is property set in
 * configuration as "key.class").//from  w w  w  . jav  a2  s  .  c  o m
 */
@Test
public void testReadValidationFailsWithWrongInputTypeKeyTranslationFunction() {
    SimpleFunction<LongWritable, String> myKeyTranslateWithWrongInputType = new SimpleFunction<LongWritable, String>() {
        @Override
        public String apply(LongWritable input) {
            return input.toString();
        }
    };
    HadoopInputFormatIO.Read<String, Employee> read = HadoopInputFormatIO.<String, Employee>read()
            .withConfiguration(serConf.getHadoopConfiguration())
            .withKeyTranslation(myKeyTranslateWithWrongInputType);
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage(String.format(
            "Key translation's input type is not same as hadoop InputFormat : %s key " + "class : %s",
            serConf.getHadoopConfiguration().getClass("mapreduce.job.inputformat.class", InputFormat.class),
            serConf.getHadoopConfiguration().getClass("key.class", Object.class)));
    read.validate(input);
}

From source file:org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIOTest.java

License:Apache License

/**
 * This test validates functionality of {@link HadoopInputFormatIO.Read#validate()
 * Read.validate()} function when myValueTranslate's (simple function provided by user for value
 * translation) input type is not same as Hadoop InputFormat's valueClass(Which is property set in
 * configuration as "value.class")./*  www .ja v a 2 s.  co  m*/
 */
@Test
public void testReadValidationFailsWithWrongInputTypeValueTranslationFunction() {
    SimpleFunction<LongWritable, String> myValueTranslateWithWrongInputType = new SimpleFunction<LongWritable, String>() {
        @Override
        public String apply(LongWritable input) {
            return input.toString();
        }
    };
    HadoopInputFormatIO.Read<Text, String> read = HadoopInputFormatIO.<Text, String>read()
            .withConfiguration(serConf.getHadoopConfiguration())
            .withValueTranslation(myValueTranslateWithWrongInputType);
    String expectedMessage = String.format(
            "Value translation's input type is not same as hadoop InputFormat :  " + "%s value class : %s",
            serConf.getHadoopConfiguration().getClass("mapreduce.job.inputformat.class", InputFormat.class),
            serConf.getHadoopConfiguration().getClass("value.class", Object.class));
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage(expectedMessage);
    read.validate(input);
}

From source file:org.apache.giraph.examples.GlobalClusteringCoefficientVertex.java

License:Apache License

@Override
public void compute(Iterator<LongArrayWritable> msgIterator) {
    DoubleSumAggregator trianglesum = (DoubleSumAggregator) getAggregator("trianglesum");
    DoubleSumAggregator triples = (DoubleSumAggregator) getAggregator("triples");
    if (getSuperstep() == 0) {
        // Each vertex this is connected to

        List<LongWritable> verticesl = new ArrayList<LongWritable>();

        // This is so we know which vertex the messages came from
        verticesl.add(getVertexId());/*w  ww  .j a  v  a 2s .c o  m*/

        // Find all connected vertices with ID less than current vertex
        for (LongWritable targetVertexId : this) {
            if (targetVertexId.get() < getVertexId().get()) {
                verticesl.add(targetVertexId);
            }
        }

        // Need to send list to other vertices, must convert to arraywritable
        LongWritable[] verticesa = verticesl.toArray(new LongWritable[0]);
        LongArrayWritable vertices = new LongArrayWritable(verticesa);

        // Sends list of smaller ID vertices to bigger ID vertices
        for (LongWritable targetVertexId : this) {
            if (targetVertexId.get() > getVertexId().get()) {
                sendMsg(targetVertexId, vertices);
            }
        }
    } else if (getSuperstep() == 1) {
        while (msgIterator.hasNext()) {
            LongArrayWritable law = msgIterator.next();
            Writable[] vertices = law.get();
            LongWritable source = (LongWritable) vertices[0];

            for (int i = 1; i < vertices.length; i++) {
                if (hasEdge((LongWritable) vertices[i])) {
                    double num = getVertexValue().get();
                    setVertexValue(new DoubleWritable(1.0 + num));

                    LongWritable[] one = new LongWritable[] { new LongWritable(1) };
                    LongArrayWritable inc = new LongArrayWritable(one);

                    sendMsg(source, inc);
                    sendMsg(((LongWritable) vertices[i]), inc);

                    triangles.add(source.toString());
                    triangles.add(vertices[i].toString());
                }
            }
        }
    } else if (getSuperstep() == 2) {
        while (msgIterator.hasNext()) {
            LongArrayWritable law = msgIterator.next();
            Writable[] msg = law.get();
            LongWritable value = (LongWritable) msg[0];

            double num = getVertexValue().get();
            setVertexValue(new DoubleWritable(num + value.get()));
        }

        trianglesum.aggregate(getVertexValue());

        double sum = 0.0;
        for (LongWritable source : this) {
            for (LongWritable target : this) {
                if (source.get() > target.get()) {
                    sum++;
                }
            }
        }

        triples.aggregate(new DoubleWritable(sum));
    } else {
        setVertexValue(new DoubleWritable(
                trianglesum.getAggregatedValue().get() / triples.getAggregatedValue().get()));

        voteToHalt();
    }
}