Example usage for org.apache.hadoop.mapreduce.lib.input FileSplit FileSplit

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.input FileSplit FileSplit.

Prototype

public FileSplit() 

Source Link

Usage

From source file:it.crs4.pydoop.mapreduce.pipes.PipeApplicationRunnableStub.java

License:Apache License

public void binaryProtocolStub() {
    try {/*from  w  w w .  ja v  a 2 s .  c  om*/

        initSoket();
        System.out.println("start OK");

        // RUN_MAP.code
        // should be 3

        int answer = WritableUtils.readVInt(dataInput);
        System.out.println("RunMap:" + answer);
        FileSplit split = new FileSplit();
        readObject(split, dataInput);

        WritableUtils.readVInt(dataInput);
        WritableUtils.readVInt(dataInput);
        // end runMap
        // get InputTypes
        WritableUtils.readVInt(dataInput);
        String inText = Text.readString(dataInput);
        System.out.println("Key class:" + inText);
        inText = Text.readString(dataInput);
        System.out.println("Value class:" + inText);

        @SuppressWarnings("unused")
        int inCode = 0;

        // read all data from sender and write to output
        while ((inCode = WritableUtils.readVInt(dataInput)) == 4) {
            FloatWritable key = new FloatWritable();
            NullWritable value = NullWritable.get();
            readObject(key, dataInput);
            System.out.println("value:" + key.get());
            readObject(value, dataInput);
        }

        WritableUtils.writeVInt(dataOut, 54);

        dataOut.flush();
        dataOut.close();

    } catch (Exception x) {
        x.printStackTrace();
    } finally {
        closeSoket();
    }

}

---

From source file:it.crs4.pydoop.mapreduce.pipes.TestPipesNonJavaInputFormat.java

License:Apache License

/**
 *  test PipesNonJavaInputFormat//w  ww .j  av  a2  s  .com
  */

@Test
public void testFormat() throws IOException, InterruptedException {
    JobID jobId = new JobID("201408272347", 0);
    TaskID taskId = new TaskID(jobId, TaskType.MAP, 0);
    TaskAttemptID taskAttemptid = new TaskAttemptID(taskId, 0);

    Job job = new Job(new Configuration());
    job.setJobID(jobId);
    Configuration conf = job.getConfiguration();

    TaskAttemptContextImpl tcontext = new TaskAttemptContextImpl(conf, taskAttemptid);

    PipesNonJavaInputFormat input_format = new PipesNonJavaInputFormat();

    DummyRecordReader reader = (DummyRecordReader) input_format.createRecordReader(new FileSplit(), tcontext);
    assertEquals(0.0f, reader.getProgress(), 0.001);

    // input and output files
    File input1 = new File(workSpace + File.separator + "input1");
    if (!input1.getParentFile().exists()) {
        Assert.assertTrue(input1.getParentFile().mkdirs());
    }

    if (!input1.exists()) {
        Assert.assertTrue(input1.createNewFile());
    }

    File input2 = new File(workSpace + File.separator + "input2");
    if (!input2.exists()) {
        Assert.assertTrue(input2.createNewFile());
    }

    // THIS fill fail without hdfs support.
    // // set data for splits
    // conf.set(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR,
    //          StringUtils.escapeString(input1.getAbsolutePath()) + ","
    //          + StringUtils.escapeString(input2.getAbsolutePath()));
    // List<InputSplit> splits = input_format.getSplits(job);
    // assertTrue(splits.size() >= 2);

    PipesNonJavaInputFormat.PipesDummyRecordReader dummyRecordReader = new PipesNonJavaInputFormat.PipesDummyRecordReader(
            new FileSplit(), tcontext);
    // empty dummyRecordReader
    assertEquals(0.0, dummyRecordReader.getProgress(), 0.001);
    // test method next
    assertTrue(dummyRecordReader.next(new FloatWritable(2.0f), NullWritable.get()));
    assertEquals(2.0, dummyRecordReader.getProgress(), 0.001);
    dummyRecordReader.close();
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopV2Splitter.java

License:Apache License

/**
 * @param clsName Input split class name.
 * @param in Input stream./*from w w w  .j av  a2  s.  com*/
 * @param hosts Optional hosts.
 * @return File block or {@code null} if it is not a {@link FileSplit} instance.
 * @throws IgniteCheckedException If failed.
 */
public static HadoopFileBlock readFileBlock(String clsName, DataInput in, @Nullable String[] hosts)
        throws IgniteCheckedException {
    if (!FileSplit.class.getName().equals(clsName))
        return null;

    FileSplit split = new FileSplit();

    try {
        split.readFields(in);
    } catch (IOException e) {
        throw new IgniteCheckedException(e);
    }

    if (hosts == null)
        hosts = EMPTY_HOSTS;

    return new HadoopFileBlock(hosts, split.getPath().toUri(), split.getStart(), split.getLength());
}

---

From source file:org.apache.ignite.internal.processors.hadoop.v2.GridHadoopV2Splitter.java

License:Apache License

/**
 * @param clsName Input split class name.
 * @param in Input stream./*w w  w  . ja v  a2  s .  co m*/
 * @param hosts Optional hosts.
 * @return File block or {@code null} if it is not a {@link FileSplit} instance.
 * @throws IgniteCheckedException If failed.
 */
public static GridHadoopFileBlock readFileBlock(String clsName, DataInput in, @Nullable String[] hosts)
        throws IgniteCheckedException {
    if (!FileSplit.class.getName().equals(clsName))
        return null;

    FileSplit split = new FileSplit();

    try {
        split.readFields(in);
    } catch (IOException e) {
        throw new IgniteCheckedException(e);
    }

    if (hosts == null)
        hosts = EMPTY_HOSTS;

    return new GridHadoopFileBlock(hosts, split.getPath().toUri(), split.getStart(), split.getLength());
}

---

From source file:org.gridgain.grid.kernal.processors.hadoop.v2.GridHadoopV2Splitter.java

License:Open Source License

/**
 * @param clsName Input split class name.
 * @param in Input stream./* ww w .j  a v  a  2s .c o m*/
 * @param hosts Optional hosts.
 * @return File block or {@code null} if it is not a {@link FileSplit} instance.
 * @throws GridException If failed.
 */
public static GridHadoopFileBlock readFileBlock(String clsName, DataInput in, @Nullable String[] hosts)
        throws GridException {
    if (!FileSplit.class.getName().equals(clsName))
        return null;

    FileSplit split = new FileSplit();

    try {
        split.readFields(in);
    } catch (IOException e) {
        throw new GridException(e);
    }

    if (hosts == null)
        hosts = EMPTY_HOSTS;

    return new GridHadoopFileBlock(hosts, split.getPath().toUri(), split.getStart(), split.getLength());
}

---