Example usage for org.apache.hadoop.mapred InputSplit write

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred InputSplit write.

Prototype

void write(DataOutput out) throws IOException;

Source Link

Document

Serialize the fields of this object to out.

Usage

From source file:com.ibm.jaql.lang.expr.io.InputSplitsFn.java

License:Apache License

@Override
public JsonIterator iter(Context context) throws Exception {
    JsonValue iod = exprs[0].eval(context);

    Adapter adapter = JaqlUtil.getAdapterStore().input.getAdapter(iod);
    if (!(adapter instanceof HadoopInputAdapter)) {
        throw new ClassCastException("i/o descriptor must be for an input format");
    }//from  w w  w  .  j a  v  a 2  s .  c o  m
    HadoopInputAdapter hia = (HadoopInputAdapter) adapter;
    JobConf conf = new JobConf(); // TODO: allow configuration
    hia.setParallel(conf); // right thing to do?
    hia.configure(conf); // right thing to do?
    int numSplits = conf.getNumMapTasks(); // TODO: allow override
    final InputSplit[] splits = hia.getSplits(conf, numSplits);
    final MutableJsonString className = new MutableJsonString();
    final MutableJsonBinary rawSplit = new MutableJsonBinary();
    final BufferedJsonRecord rec = new BufferedJsonRecord(3);
    final BufferedJsonArray locArr = new BufferedJsonArray();
    rec.add(CLASS_TAG, className);
    rec.add(SPLIT_TAG, rawSplit);
    rec.add(LOCATIONS_TAG, locArr);

    return new JsonIterator(rec) {
        DataOutputBuffer out = new DataOutputBuffer();
        int i = 0;

        @Override
        public boolean moveNext() throws Exception {
            if (i >= splits.length) {
                return false;
            }
            InputSplit split = splits[i++];
            className.setCopy(split.getClass().getName());
            out.reset();
            split.write(out);
            rawSplit.setCopy(out.getData(), out.getLength());
            locArr.clear();
            String[] locs = split.getLocations();
            if (locs != null) {
                for (String loc : locs) {
                    locArr.add(new JsonString(loc));
                }
            }
            return true;
        }
    };
}

---

From source file:com.yolodata.tbana.hadoop.mapred.splunk.split.InputSplitTest.java

License:Open Source License

protected void testSerialization(InputSplit split, InputSplit emptySplit) throws IOException {

    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    DataOutputStream stream = new DataOutputStream(outputStream);

    split.write(stream);

    byte[] output = outputStream.toByteArray();

    DataInputStream inputStream = new DataInputStream(new ByteArrayInputStream(output));

    emptySplit.readFields(inputStream);// w  ww  .  jav  a  2s. co  m

    assertEquals(split, emptySplit);

}

---

From source file:org.apache.drill.exec.store.hive.HiveInputReader.java

License:Apache License

public static String serializeInputSplit(InputSplit split) throws IOException {
    ByteArrayDataOutput byteArrayOutputStream = ByteStreams.newDataOutput();
    split.write(byteArrayOutputStream);
    return Base64.encodeBase64String(byteArrayOutputStream.toByteArray());
}

---

From source file:org.apache.drill.exec.store.hive.HiveScan.java

License:Apache License

public static String serializeInputSplit(final InputSplit split) throws IOException {
    final ByteArrayDataOutput byteArrayOutputStream = ByteStreams.newDataOutput();
    split.write(byteArrayOutputStream);
    final String encoded = Base64.encodeBase64String(byteArrayOutputStream.toByteArray());
    logger.debug("Encoded split string for split {} : {}", split, encoded);
    return encoded;
}

---

From source file:org.apache.tez.mapreduce.hadoop.MRInputHelpers.java

License:Apache License

@InterfaceStability.Evolving
public static MRRuntimeProtos.MRSplitProto createSplitProto(org.apache.hadoop.mapred.InputSplit oldSplit)
        throws IOException {
    MRRuntimeProtos.MRSplitProto.Builder builder = MRRuntimeProtos.MRSplitProto.newBuilder();

    builder.setSplitClassName(oldSplit.getClass().getName());

    ByteString.Output os = ByteString.newOutput(SPLIT_SERIALIZED_LENGTH_ESTIMATE);
    oldSplit.write(new DataOutputStream(os));
    ByteString splitBs = os.toByteString();
    builder.setSplitBytes(splitBs);//  www . j  a  v a2  s  .  com

    return builder.build();
}

---

From source file:org.apache.tez.mapreduce.processor.MapUtils.java

License:Apache License

private static void writeSplitFiles(FileSystem fs, JobConf conf, InputSplit split) throws IOException {
    Path jobSplitFile = new Path(conf.get(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR,
            MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR_DEFAULT), MRJobConfig.JOB_SPLIT);
    LOG.info("Writing split to: " + jobSplitFile);
    FSDataOutputStream out = FileSystem.create(fs, jobSplitFile, new FsPermission(JOB_FILE_PERMISSION));

    long offset = out.getPos();
    Text.writeString(out, split.getClass().getName());
    split.write(out);
    out.close();//from w  w w  .j av  a2s  . co m

    String[] locations = split.getLocations();

    SplitMetaInfo info = null;
    info = new JobSplit.SplitMetaInfo(locations, offset, split.getLength());

    Path jobSplitMetaInfoFile = new Path(conf.get(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR),
            MRJobConfig.JOB_SPLIT_METAINFO);

    FSDataOutputStream outMeta = FileSystem.create(fs, jobSplitMetaInfoFile,
            new FsPermission(JOB_FILE_PERMISSION));
    outMeta.write(SplitMetaInfoReaderTez.META_SPLIT_FILE_HEADER);
    WritableUtils.writeVInt(outMeta, SplitMetaInfoReaderTez.META_SPLIT_VERSION);
    WritableUtils.writeVInt(outMeta, 1); // Only 1 split meta info being written
    info.write(outMeta);
    outMeta.close();
}

---