Example usage for org.apache.hadoop.io ByteWritable ByteWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io ByteWritable ByteWritable.

Prototype

public ByteWritable(byte value) 

Source Link

Usage

From source file:jadoop.HadoopGridTaskRunner.java

License:Open Source License

/**
 * The map function that executes the command line task and outputs the
 * return code, standard output and standard error.
 * /*from  w ww  .  j  av a  2s .  com*/
 * The value provided as the input to this method will be a key and an array
 * of Text objects. The key matches the key provided when the HadoopGridTask
 * was added to the HadoopGridJob. The entries in the array are as follows:
 * <UL>
 * <LI>[0] - capStdOut: [true or false] If true, then all output to standard
 * output by the execution of the command should be captured and returned as
 * discussed below. If false, then standard output is not captured. This
 * will be the first element of the ArrayWritable.
 * <LI>[1] - capStdErr: [true or false] If true, then all output to standard
 * error by the execution of the command should be captured and returned as
 * discussed below. If false, then standard error is not captured. This will
 * be the second element in ArrayWritable.
 * <LI>[2] - timeout: The amount of time (in ms) that the task has to
 * execute. If the command does not complete before the timeout it is
 * terminated.
 * <LI>[3] - command: The command to be executed. Any arguments to the
 * command will be contained in subsequent entries.
 * <LI>[4]... - arguments to the command. These would be the individual
 * command line arguments if typed in at the command prompt.
 * </UL>
 * 
 * <p>
 * For example: if standard output of an execution of the cal command for
 * June 2015 were to be captured the array entries would be:
 * <UL>
 * <LI>[0] - true
 * <LI>[1] - false
 * <LI>[2] - 1000
 * <LI>[3] - cal
 * <LI>[4] - 6
 * <LI>[5] - 2015
 * </UL>
 * 
 * <p>
 * Entries 0-2 are used as flags, the remaining entries are converted to an
 * array of Strings and used as the argument in a call to
 * Runtime.getRuntime().exec() to run the command.
 * 
 * <p>
 * The key generated for the Mapper result will be the same key passed into
 * the mapper. The results generated for the Mapper will be a MapWritable
 * object with the following key/value pairs:
 * <UL>
 * <LI>EV,value : the value is a IntWritable containing the exit value
 * generated by the process created by the call to the Runtime.exec method.
 * <LI>TO,value : the value will be a BooleanWriteable indicating if the
 * task timed out (true) or not (false).
 * <LI>SO,value : the value is a Text containing the output written to
 * standard output by the executed program.
 * <LI>SE,value : the value is a Text containing the output written to
 * standard error by the executed program
 * </UL>
 * 
 * @param key
 *            a key that identifies this task. This will match the key
 *            provided in the HadoopGridTask object.
 * @param value
 *            the flags and command line as described above.
 * @param context
 *            a Hadoop Context object provided by the Hadoop system.
 * 
 * @throws InterruptedException
 *             if there is a problem writing the task result to the context.
 * @throws IOException
 *             if there is a problem writing the task result to the context.
 */
public void map(Text key, TextArrayWritable value, Context context) throws IOException, InterruptedException {

    String[] mapVal = value.toStrings();

    boolean capStdOutput = Boolean.parseBoolean(mapVal[0]);
    boolean capStdErr = Boolean.parseBoolean(mapVal[1]);
    long timeout = Long.parseLong(mapVal[2]);

    // Build the command.
    String[] cmdInput = new String[mapVal.length - 3];
    for (int i = 3; i < mapVal.length; i++) {
        cmdInput[i - 3] = mapVal[i];
    }

    StringBuffer stdOutputStr = new StringBuffer();
    StringBuffer errOutputStr = new StringBuffer();
    byte exitValue = 0;
    boolean timedout = false;

    try {
        // Executes the command.
        Process p = Runtime.getRuntime().exec(cmdInput);

        long start = System.currentTimeMillis();
        long cur = System.currentTimeMillis();
        boolean done = false;
        while (!timedout && !done) {
            Thread.sleep(PROCSSESS_POLL_DELAY);

            /*
             * Check if the process has finished. If it has, the exit value
             * will come back, if not it throws an exception.
             */
            try {
                exitValue = (byte) p.exitValue();
                done = true;
            } catch (IllegalThreadStateException e) {
                // process not done yet, keep going...
            }

            cur = System.currentTimeMillis();
            long elapsedTime = (cur - start);
            timedout = (elapsedTime >= timeout);

            // Keep long running tasks alive with hadoop.
            context.setStatus("Running for: " + elapsedTime + " ms.");
        }

        // Capture standard output generated by the command.
        if (capStdOutput) {
            BufferedReader stdOutputPrg = new BufferedReader(new InputStreamReader(p.getInputStream()));
            while (stdOutputPrg.ready()) {
                stdOutputStr.append(stdOutputPrg.readLine());
                stdOutputStr.append("\n");
            }
        }

        // Capture standard error generated by the command
        if (capStdErr) {
            BufferedReader stdErrPrg = new BufferedReader(new InputStreamReader(p.getErrorStream()));
            while (stdErrPrg.ready()) {
                errOutputStr.append(stdErrPrg.readLine());
                errOutputStr.append("\n");
            }
        }

        if (timedout) {
            p.destroy(); // kill the process.
            exitValue = -1;
        }
    } catch (Exception e) {
        /*
         * If an exception occurs put the message and stack trace on the end
         * of the standard error returned.
         */
        errOutputStr.append("\n" + e.getMessage() + "\n" + e.getStackTrace());
        exitValue = -1;
    }

    // Put the results into the context that is returned from this mapper.
    Text evKey = new Text("EV");
    Text toKey = new Text("TO");
    Text soKey = new Text("SO");
    Text seKey = new Text("SE");

    ByteWritable bwExitVal = new ByteWritable(exitValue);
    BooleanWritable bwTimeout = new BooleanWritable(timedout);
    Text tStdOutputStr = new Text(stdOutputStr.toString().trim());
    Text tErrOutputStr = new Text(errOutputStr.toString().trim());

    MapWritable mw = new MapWritable();
    mw.put(evKey, bwExitVal);
    mw.put(toKey, bwTimeout);
    mw.put(soKey, tStdOutputStr);
    mw.put(seKey, tErrOutputStr);

    context.write(key, mw);
}

---

From source file:net.sf.katta.util.WritableType.java

License:Apache License

/**
 * Convert a java primitive type wrapper (like String, Integer, Float, etc...)
 * to the corresponding hadoop {@link WritableComparable}.
 *///  w  w  w.jav  a 2 s . c  om
public WritableComparable convertComparable(Object comparable) {
    switch (this) {
    case TEXT:
        return new Text((String) comparable);
    case BYTE:
        return new ByteWritable(((Byte) comparable).byteValue());
    case INT:
        return new IntWritable(((Integer) comparable).intValue());
    case LONG:
        return new LongWritable((((Long) comparable).longValue()));
    case FLOAT:
        return new FloatWritable(((Float) comparable).floatValue());
    case DOUBLE:
        return new DoubleWritable(((Double) comparable).doubleValue());
    }
    throw getUnhandledTypeException();
}

---

From source file:org.apache.avro.hadoop.io.TestAvroDatumConverterFactory.java

License:Apache License

@Test
public void testConvertByteWritable() {
    AvroDatumConverter<ByteWritable, GenericFixed> converter = mFactory.create(ByteWritable.class);
    assertEquals(42, converter.convert(new ByteWritable((byte) 42)).bytes()[0]);
}

---

From source file:org.apache.flink.hadoopcompatibility.mapred.record.datatypes.DefaultFlinkTypeConverter.java

License:Apache License

@SuppressWarnings("unchecked")
private <T> T convert(Record flinkType, int pos, Class<T> hadoopType) {
    if (hadoopType == LongWritable.class) {
        return (T) new LongWritable((flinkType.getField(pos, LongValue.class)).getValue());
    }//from w ww.  ja  v a 2s  . co  m
    if (hadoopType == org.apache.hadoop.io.Text.class) {
        return (T) new Text((flinkType.getField(pos, StringValue.class)).getValue());
    }
    if (hadoopType == org.apache.hadoop.io.IntWritable.class) {
        return (T) new IntWritable((flinkType.getField(pos, IntValue.class)).getValue());
    }
    if (hadoopType == org.apache.hadoop.io.FloatWritable.class) {
        return (T) new FloatWritable((flinkType.getField(pos, FloatValue.class)).getValue());
    }
    if (hadoopType == org.apache.hadoop.io.DoubleWritable.class) {
        return (T) new DoubleWritable((flinkType.getField(pos, DoubleValue.class)).getValue());
    }
    if (hadoopType == org.apache.hadoop.io.BooleanWritable.class) {
        return (T) new BooleanWritable((flinkType.getField(pos, BooleanValue.class)).getValue());
    }
    if (hadoopType == org.apache.hadoop.io.ByteWritable.class) {
        return (T) new ByteWritable((flinkType.getField(pos, ByteValue.class)).getValue());
    }

    throw new RuntimeException(
            "Unable to convert Flink type (" + flinkType.getClass().getCanonicalName() + ") to Hadoop.");
}

---

From source file:org.apache.giraph.examples.AddCouples.java

License:Apache License

/**
 * Apply the rule "CreateCouple" to a given match.
 * @param vertex The base vertex./*from   www. j  a v a 2 s . c om*/
 * @param match The match object.
 * @param matchIndex Match index.
 * @return true if the rule was applied.
 * @throws IOException On I/O errors.
 */
protected boolean applyCreateCouple(Vertex<VertexId, ByteWritable, ByteWritable> vertex, Match match,
        long matchIndex) throws IOException {
    VertexId cur0 = match.getVertexId(0);
    VertexId cur1 = match.getVertexId(1);
    VertexId new0 = VertexId.randomVertexId();
    addVertexRequest(new0, new ByteWritable(TYPE_COUPLE));
    VertexId src0 = new0;
    VertexId trg0 = cur0;
    Edge<VertexId, ByteWritable> edge0 = EdgeFactory.create(trg0, new ByteWritable(TYPE_COUPLE_P1));
    addEdgeRequest(src0, edge0);
    VertexId src1 = new0;
    VertexId trg1 = cur1;
    Edge<VertexId, ByteWritable> edge1 = EdgeFactory.create(trg1, new ByteWritable(TYPE_COUPLE_P2));
    addEdgeRequest(src1, edge1);
    return true;
}

---

From source file:org.apache.giraph.examples.AddCouples.java

License:Apache License

/**
 * Apply the rule "CreateOccurrence" to a given match.
 * @param vertex The base vertex./*from  w w  w  .j  a v a2s.c o m*/
 * @param match The match object.
 * @param matchIndex Match index.
 * @return true if the rule was applied.
 * @throws IOException On I/O errors.
 */
protected boolean applyCreateOccurrence(Vertex<VertexId, ByteWritable, ByteWritable> vertex, Match match,
        long matchIndex) throws IOException {
    VertexId cur0 = match.getVertexId(0);
    VertexId cur1 = match.getVertexId(1);
    VertexId src0 = cur0;
    VertexId trg0 = cur1;
    Edge<VertexId, ByteWritable> edge0 = EdgeFactory.create(trg0, new ByteWritable(TYPE_COUPLE_OCCURS_IN));
    addEdgeRequest(src0, edge0);
    return true;
}

---

From source file:org.apache.giraph.examples.RequireOne.java

License:Apache License

/**
 * Apply the rule "RequireOne" to a given match.
 * @param vertex The base vertex.//w  w w  .j ava  2s .  com
 * @param match The match object.
 * @param matchIndex Match index.
 * @return true if the rule was applied.
 * @throws IOException On I/O errors.
 */
protected boolean applyRequireOne(Vertex<VertexId, ByteWritable, ByteWritable> vertex, Match match,
        long matchIndex) throws IOException {
    VertexId cur0 = match.getVertexId(0);
    LOG.info("Vertex " + vertex.getId() + " applying rule RequireOne with match " + match);
    VertexId new0 = deriveVertexId(vertex.getId(), (int) matchIndex, 0);
    addVertexRequest(new0, new ByteWritable(TYPE_VERTEX));
    VertexId src0 = cur0;
    VertexId trg0 = new0;
    Edge<VertexId, ByteWritable> edge0 = EdgeFactory.create(trg0,
            new ByteWritable(TYPE_VERTEX_CONTAINER_VERTICES));
    addEdgeRequest(src0, edge0);
    return true;
}

---

From source file:org.apache.giraph.examples.RequireTwo.java

License:Apache License

/**
 * Apply the rule "RequireTwo" to a given match.
 * @param vertex The base vertex.//from  w  w  w .  j  a  v  a2s  .co  m
 * @param match The match object.
 * @param matchIndex Match index.
 * @return true if the rule was applied.
 * @throws IOException On I/O errors.
 */
protected boolean applyRequireTwo(Vertex<VertexId, ByteWritable, ByteWritable> vertex, Match match,
        long matchIndex) throws IOException {
    VertexId cur0 = match.getVertexId(0);
    LOG.info("Vertex " + vertex.getId() + " applying rule RequireTwo with match " + match);
    VertexId new0 = deriveVertexId(vertex.getId(), (int) matchIndex, 0);
    addVertexRequest(new0, new ByteWritable(TYPE_VERTEX));
    VertexId src0 = cur0;
    VertexId trg0 = new0;
    Edge<VertexId, ByteWritable> edge0 = EdgeFactory.create(trg0,
            new ByteWritable(TYPE_VERTEX_CONTAINER_VERTICES));
    addEdgeRequest(src0, edge0);
    return true;
}

---

From source file:org.apache.giraph.examples.Sierpinski.java

License:Apache License

/**
 * Apply the rule "Sierpinski" to a given match.
 * @param vertex The base vertex.//from  w w  w  .j  av  a2s .  c  o m
 * @param match The match object.
 * @param matchIndex Match index.
 * @return true if the rule was applied.
 * @throws IOException On I/O errors.
 */
protected boolean applySierpinski(Vertex<VertexId, ByteWritable, ByteWritable> vertex, Match match,
        long matchIndex) throws IOException {
    VertexId cur0 = match.getVertexId(0);
    VertexId cur1 = match.getVertexId(1);
    VertexId cur2 = match.getVertexId(2);
    LOG.info("Vertex " + vertex.getId() + " applying rule Sierpinski with match " + match);
    removeEdgesRequest(cur0, cur1);
    removeEdgesRequest(cur0, cur2);
    removeEdgesRequest(cur1, cur2);
    VertexId new0 = deriveVertexId(vertex.getId(), (int) matchIndex, 0);
    addVertexRequest(new0, new ByteWritable(TYPE_VERTEX));
    VertexId new1 = deriveVertexId(vertex.getId(), (int) matchIndex, 1);
    addVertexRequest(new1, new ByteWritable(TYPE_VERTEX));
    VertexId new2 = deriveVertexId(vertex.getId(), (int) matchIndex, 2);
    addVertexRequest(new2, new ByteWritable(TYPE_VERTEX));
    VertexId src0 = cur0;
    VertexId trg0 = new0;
    Edge<VertexId, ByteWritable> edge0 = EdgeFactory.create(trg0, new ByteWritable(TYPE_VERTEX_LEFT));
    addEdgeRequest(src0, edge0);
    VertexId src1 = cur0;
    VertexId trg1 = new1;
    Edge<VertexId, ByteWritable> edge1 = EdgeFactory.create(trg1, new ByteWritable(TYPE_VERTEX_RIGHT));
    addEdgeRequest(src1, edge1);
    VertexId src2 = cur1;
    VertexId trg2 = new2;
    Edge<VertexId, ByteWritable> edge2 = EdgeFactory.create(trg2, new ByteWritable(TYPE_VERTEX_CONN));
    addEdgeRequest(src2, edge2);
    VertexId src3 = new0;
    VertexId trg3 = cur1;
    Edge<VertexId, ByteWritable> edge3 = EdgeFactory.create(trg3, new ByteWritable(TYPE_VERTEX_LEFT));
    addEdgeRequest(src3, edge3);
    VertexId src4 = new0;
    VertexId trg4 = new1;
    Edge<VertexId, ByteWritable> edge4 = EdgeFactory.create(trg4, new ByteWritable(TYPE_VERTEX_CONN));
    addEdgeRequest(src4, edge4);
    VertexId src5 = new0;
    VertexId trg5 = new2;
    Edge<VertexId, ByteWritable> edge5 = EdgeFactory.create(trg5, new ByteWritable(TYPE_VERTEX_RIGHT));
    addEdgeRequest(src5, edge5);
    VertexId src6 = new1;
    VertexId trg6 = new2;
    Edge<VertexId, ByteWritable> edge6 = EdgeFactory.create(trg6, new ByteWritable(TYPE_VERTEX_LEFT));
    addEdgeRequest(src6, edge6);
    VertexId src7 = new1;
    VertexId trg7 = cur2;
    Edge<VertexId, ByteWritable> edge7 = EdgeFactory.create(trg7, new ByteWritable(TYPE_VERTEX_RIGHT));
    addEdgeRequest(src7, edge7);
    VertexId src8 = new2;
    VertexId trg8 = cur2;
    Edge<VertexId, ByteWritable> edge8 = EdgeFactory.create(trg8, new ByteWritable(TYPE_VERTEX_CONN));
    addEdgeRequest(src8, edge8);
    return true;
}

---

From source file:org.apache.giraph.examples.SierpinskiMain1.java

License:Apache License

/**
 * Apply the rule "Sierpinski" to a given match.
 * @param vertex The base vertex./*from w w  w .  ja v a 2s .  co m*/
 * @param match The match object.
 * @param matchIndex Match index.
 * @return true if the rule was applied.
 * @throws IOException On I/O errors.
 */
protected boolean applySierpinski(Vertex<VertexId, ByteWritable, ByteWritable> vertex, Match match,
        long matchIndex) throws IOException {
    VertexId cur0 = match.getVertexId(0);
    VertexId cur1 = match.getVertexId(1);
    VertexId cur2 = match.getVertexId(2);
    removeEdgesRequest(cur0, cur1);
    removeEdgesRequest(cur0, cur2);
    removeEdgesRequest(cur1, cur2);
    VertexId new0 = VertexId.randomVertexId();
    addVertexRequest(new0, new ByteWritable(TYPE_VERTEX));
    VertexId new1 = VertexId.randomVertexId();
    addVertexRequest(new1, new ByteWritable(TYPE_VERTEX));
    VertexId new2 = VertexId.randomVertexId();
    addVertexRequest(new2, new ByteWritable(TYPE_VERTEX));
    VertexId src0 = cur0;
    VertexId trg0 = new0;
    Edge<VertexId, ByteWritable> edge0 = EdgeFactory.create(trg0, new ByteWritable(TYPE_VERTEX_LEFT));
    addEdgeRequest(src0, edge0);
    VertexId src1 = cur0;
    VertexId trg1 = new1;
    Edge<VertexId, ByteWritable> edge1 = EdgeFactory.create(trg1, new ByteWritable(TYPE_VERTEX_RIGHT));
    addEdgeRequest(src1, edge1);
    VertexId src2 = cur1;
    VertexId trg2 = new2;
    Edge<VertexId, ByteWritable> edge2 = EdgeFactory.create(trg2, new ByteWritable(TYPE_VERTEX_CONN));
    addEdgeRequest(src2, edge2);
    VertexId src3 = new0;
    VertexId trg3 = cur1;
    Edge<VertexId, ByteWritable> edge3 = EdgeFactory.create(trg3, new ByteWritable(TYPE_VERTEX_LEFT));
    addEdgeRequest(src3, edge3);
    VertexId src4 = new0;
    VertexId trg4 = new1;
    Edge<VertexId, ByteWritable> edge4 = EdgeFactory.create(trg4, new ByteWritable(TYPE_VERTEX_CONN));
    addEdgeRequest(src4, edge4);
    VertexId src5 = new0;
    VertexId trg5 = new2;
    Edge<VertexId, ByteWritable> edge5 = EdgeFactory.create(trg5, new ByteWritable(TYPE_VERTEX_RIGHT));
    addEdgeRequest(src5, edge5);
    VertexId src6 = new1;
    VertexId trg6 = new2;
    Edge<VertexId, ByteWritable> edge6 = EdgeFactory.create(trg6, new ByteWritable(TYPE_VERTEX_LEFT));
    addEdgeRequest(src6, edge6);
    VertexId src7 = new1;
    VertexId trg7 = cur2;
    Edge<VertexId, ByteWritable> edge7 = EdgeFactory.create(trg7, new ByteWritable(TYPE_VERTEX_RIGHT));
    addEdgeRequest(src7, edge7);
    VertexId src8 = new2;
    VertexId trg8 = cur2;
    Edge<VertexId, ByteWritable> edge8 = EdgeFactory.create(trg8, new ByteWritable(TYPE_VERTEX_CONN));
    addEdgeRequest(src8, edge8);
    return true;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »