Example usage for org.apache.hadoop.io Text writeString

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text writeString.

Prototype

public static int writeString(DataOutput out, String s) throws IOException

Source Link

Document

Write a UTF8 encoded string to out

Usage

From source file:it.crs4.pydoop.mapreduce.pipes.CommonStub.java

License:Apache License

protected void initSoket() throws Exception {
    int port = Integer.parseInt(System.getenv("mapreduce.pipes.command.port"));

    System.err.println("port:" + port);

    java.net.InetAddress address = java.net.InetAddress.getLocalHost();

    socket = new Socket(address.getHostName(), port);
    InputStream input = socket.getInputStream();
    OutputStream output = socket.getOutputStream();

    System.err.println("ready to read");
    // try to read
    dataInput = new DataInputStream(input);

    WritableUtils.readVInt(dataInput);/*from www  . j a  v a 2  s.  co m*/

    String str = Text.readString(dataInput);

    Text.readString(dataInput);

    dataOut = new DataOutputStream(output);
    WritableUtils.writeVInt(dataOut, 57);
    String s = createDigest("password".getBytes(), str);

    Text.writeString(dataOut, s);

    // start
    WritableUtils.readVInt(dataInput);
    int cuttentAnswer = WritableUtils.readVInt(dataInput);
    System.out.println("CURRENT_PROTOCOL_VERSION:" + cuttentAnswer);

    // get configuration
    // should be MessageType.SET_JOB_CONF.code
    WritableUtils.readVInt(dataInput);

    // array length

    int j = WritableUtils.readVInt(dataInput);
    for (int i = 0; i < j; i++) {
        Text.readString(dataInput);
        i++;
        Text.readString(dataInput);
    }
}

From source file:it.crs4.pydoop.mapreduce.pipes.PipeApplicationStub.java

License:Apache License

public void binaryProtocolStub() {
    try {//from  ww w.ja  v a 2  s  . c o  m

        System.err.println("binaryProtocolStub()");
        initSoket();
        System.err.println("done with socket()");

        // output code
        WritableUtils.writeVInt(dataOut, 50);
        IntWritable wt = new IntWritable();
        wt.set(123);
        writeObject(wt, dataOut);
        writeObject(new Text("value"), dataOut);

        //  PARTITIONED_OUTPUT
        WritableUtils.writeVInt(dataOut, 51);
        WritableUtils.writeVInt(dataOut, 0);
        writeObject(wt, dataOut);
        writeObject(new Text("value"), dataOut);

        // STATUS
        WritableUtils.writeVInt(dataOut, 52);
        Text.writeString(dataOut, "PROGRESS");
        dataOut.flush();

        // progress
        WritableUtils.writeVInt(dataOut, 53);
        dataOut.writeFloat(0.55f);
        // register counter
        WritableUtils.writeVInt(dataOut, 55);
        // id
        WritableUtils.writeVInt(dataOut, 0);
        Text.writeString(dataOut, "group");
        Text.writeString(dataOut, "name");
        // increment counter
        WritableUtils.writeVInt(dataOut, 56);
        WritableUtils.writeVInt(dataOut, 0);

        WritableUtils.writeVLong(dataOut, 2);

        // map item
        int intValue = WritableUtils.readVInt(dataInput);
        System.out.println("intValue:" + intValue);
        IntWritable iw = new IntWritable();
        readObject(iw, dataInput);
        System.out.println("key:" + iw.get());
        Text txt = new Text();
        readObject(txt, dataInput);
        System.out.println("value:" + txt.toString());

        // done
        // end of session
        WritableUtils.writeVInt(dataOut, 54);

        System.out.println("finish");
        dataOut.flush();
        dataOut.close();

    } catch (Exception x) {
        x.printStackTrace();
    } finally {
        closeSoket();
    }
}

From source file:it.crs4.pydoop.pipes.BinaryProtocol.java

License:Apache License

public void setJobConf(JobConf job) throws IOException {
    WritableUtils.writeVInt(stream, MessageType.SET_JOB_CONF.code);
    List<String> list = new ArrayList<String>();
    for (Map.Entry<String, String> itm : job) {
        list.add(itm.getKey());/*from   w w w  . j  a v  a2 s  .  c  o m*/
        list.add(itm.getValue());
    }
    WritableUtils.writeVInt(stream, list.size());
    for (String entry : list) {
        Text.writeString(stream, entry);
    }
}

From source file:it.crs4.seal.common.SequenceId.java

License:Open Source License

@Override
public void write(DataOutput out) throws IOException {
    Text.writeString(out, location);
    out.writeByte(read);
}

From source file:kogiri.mapreduce.common.kmermatch.KmerMatchInputSplit.java

License:Open Source License

@Override
public void write(DataOutput out) throws IOException {
    out.writeInt(this.kmerIndexPath.length);
    for (Path indexPath : this.kmerIndexPath) {
        Text.writeString(out, indexPath.toString());
    }//  w  w w . ja va2  s.  c  o  m
    this.partition.write(out);
}

From source file:kogiri.mapreduce.preprocess.common.kmerhistogram.KmerRangePartition.java

License:Open Source License

public void write(DataOutput out) throws IOException {
    out.writeInt(this.kmerSize);
    out.writeInt(this.numPartitions);
    out.writeInt(this.partitionIndex);
    Text.writeString(out, this.partitionSize.toString());
    Text.writeString(out, this.partitionBegin.toString());
    Text.writeString(out, this.parititionEnd.toString());
}

From source file:kogiri.mapreduce.preprocess.common.kmerindex.KmerIndexSplit.java

License:Open Source License

@Override
public void write(DataOutput out) throws IOException {
    out.writeInt(this.indexPaths.length);
    for (Path indexPath : this.indexPaths) {
        Text.writeString(out, indexPath.toString());
    }//from   ww w .j  a  v a  2s.c o  m

    out.writeInt(this.locations.length);
    for (String host : this.locations) {
        Text.writeString(out, host);
    }
}

From source file:ml.grafos.okapi.graphs.betweeness.ShortestPathData.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    Text.writeString(out, source);
    Text.writeString(out, from);/*from ww  w  .  jav  a 2  s  . co m*/
    out.writeInt(distance);

    if (shortestPathSources != null) {
        out.writeInt(shortestPathSources.size());
    } else {
        out.writeInt(0);
    }

    for (String source : shortestPathSources) {
        Text.writeString(out, source);
    }
}

From source file:ml.grafos.okapi.graphs.betweeness.ShortestPathList.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    out.writeInt(distance);//from   w  w w  . j  a v a  2 s  .  co  m
    out.writeInt(this.predecessors.size());
    for (String entry : predecessors) {
        Text.writeString(out, entry);
    }

}

From source file:ml.shifu.guagua.yarn.GuaguaSplitWriter.java

License:Apache License

@SuppressWarnings("unchecked")
private static <T extends InputSplit> SplitMetaInfo[] writeNewSplits(Configuration conf, T[] array,
        FSDataOutputStream out) throws IOException, InterruptedException {

    SplitMetaInfo[] info = new SplitMetaInfo[array.length];
    if (array.length != 0) {
        SerializationFactory factory = new SerializationFactory(conf);
        int i = 0;
        long offset = out.getPos();
        for (T split : array) {
            long prevCount = out.getPos();
            Text.writeString(out, split.getClass().getName());
            Serializer<T> serializer = factory.getSerializer((Class<T>) split.getClass());
            serializer.open(out);//from  w w w . j a  va  2s.co  m
            serializer.serialize(split);
            long currCount = out.getPos();
            String[] locations = split.getLocations();
            final int max_loc = conf.getInt(MAX_SPLIT_LOCATIONS, 10);
            if (locations.length > max_loc) {
                LOG.warn("Max block location exceeded for split: " + split + " splitsize: " + locations.length
                        + " maxsize: " + max_loc);
                locations = Arrays.copyOf(locations, max_loc);
            }
            info[i++] = new JobSplit.SplitMetaInfo(locations, offset, split.getLength());
            offset += currCount - prevCount;
        }
    }
    return info;
}