Example usage for org.apache.hadoop.io Text Text

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text Text.

Prototype

public Text()

Source Link

Usage

From source file:com.asakusafw.lang.compiler.extension.testdriver.mock.MockTextDefinition.java

License:Apache License

@Override
public Text toObject(DataModelReflection reflection) {
    Text text = new Text();
    String string = (String) reflection.getValue(VALUE);
    if (string != null) {
        text.set(string);//from  w w  w  .j a  v a2 s  .c o m
    }
    return text;
}

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test for input./*from   w  w  w .j a v a2  s.com*/
 * @throws Exception if failed
 */
@Test
public void input() throws Exception {
    final int count = 10000;
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class,
            Text.class)) {
        LongWritable k = new LongWritable();
        Text v = new Text();
        for (int i = 0; i < count; i++) {
            k.set(i);
            v.set("Hello, world at " + i);
            writer.append(k, v);
        }
    }

    try (ModelInput<StringOption> in = format.createInput(StringOption.class, fs, path, 0,
            fs.getFileStatus(path).getLen(), new Counter())) {
        StringOption value = new StringOption();
        for (int i = 0; i < count; i++) {
            String answer = "Hello, world at " + i;
            assertThat(answer, in.readTo(value), is(true));
            assertThat(value.getAsString(), is(answer));
        }
        assertThat("eof", in.readTo(value), is(false));
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test for input./* w w w  .ja  v  a 2 s  . c  o m*/
 * @throws Exception if failed
 */
@Test
public void input_fragment() throws Exception {
    final int count = 30000;
    Random rand = new Random();
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class,
            Text.class)) {
        LongWritable k = new LongWritable();
        Text v = new Text();
        for (int i = 0; i < count; i++) {
            k.set(i);
            v.set("Hello, world at " + i);
            writer.append(k, v);
        }
    }

    long fileLen = fs.getFileStatus(path).getLen();
    StringOption value = new StringOption();
    for (int attempt = 0; attempt < 5; attempt++) {
        int index = 0;
        long offset = 0;
        while (offset < fileLen) {
            long length = SequenceFile.SYNC_INTERVAL * (rand.nextInt(10) + 2);
            length = Math.min(length, fileLen - offset);
            try (ModelInput<StringOption> in = format.createInput(StringOption.class, fs, path, offset, length,
                    new Counter())) {
                while (in.readTo(value)) {
                    String answer = "Hello, world at " + index;
                    assertThat(value.getAsString(), is(answer));
                    index++;
                }
                assertThat("eof", in.readTo(value), is(false));
            }
            offset += length;
        }
        assertThat(index, is(count));
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test method for output.//from w w  w. ja v  a2 s . c  om
 * @throws Exception if failed
 */
@SuppressWarnings("deprecation")
@Test
public void output() throws Exception {
    final int count = 10000;
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    try (ModelOutput<StringOption> out = format.createOutput(StringOption.class, fs, path, new Counter())) {
        StringOption value = new StringOption();
        for (int i = 0; i < count; i++) {
            value.modify("Hello, world at " + i);
            out.write(value);
        }
    }
    try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf)) {
        LongWritable k = new LongWritable();
        Text v = new Text();
        for (int i = 0; i < count; i++) {
            String answer = "Hello, world at " + i;
            assertThat(answer, reader.next(k, v), is(true));
            assertThat(answer, k.get(), is(1L));
            assertThat(answer, v.toString(), is(answer));
        }
        assertThat("eof", reader.next(k), is(false));
    }
}

From source file:com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest.java

License:Apache License

/**
 * Reads a sequence file./*from  w  ww  . j a  v a 2s . co  m*/
 * @throws Exception if failed
 */
@Test
public void read() throws Exception {
    Path path = new Path("testing");

    Text key = new Text();
    Text value = new Text();
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, key.getClass(),
            value.getClass())) {
        key.set("Hello");
        value.set("World");
        writer.append(key, value);
    }
    key.clear();
    value.clear();

    FileStatus status = fs.getFileStatus(path);
    try (InputStream in = new FileInputStream(fs.pathToFile(path));
            SequenceFile.Reader reader = SequenceFileUtil.openReader(in, status, conf)) {
        assertThat(reader.next(key, value), is(true));
        assertThat(key.toString(), is("Hello"));
        assertThat(value.toString(), is("World"));
        assertThat(reader.next(key, value), is(false));
    }
}

From source file:com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest.java

License:Apache License

/**
 * Reads a sequence file.//from   w ww .  j  av a 2s.  com
 * @throws Exception if failed
 */
@Test
public void read_new() throws Exception {
    Path path = new Path("testing");

    Text key = new Text();
    Text value = new Text();
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, key.getClass(),
            value.getClass())) {
        key.set("Hello");
        value.set("World");
        writer.append(key, value);
    }
    key.clear();
    value.clear();

    FileStatus status = fs.getFileStatus(path);
    try (InputStream in = new FileInputStream(fs.pathToFile(path));
            SequenceFile.Reader reader = SequenceFileUtil.openReader(in, status.getLen(), conf)) {
        assertThat(reader.next(key, value), is(true));
        assertThat(key.toString(), is("Hello"));
        assertThat(value.toString(), is("World"));
        assertThat(reader.next(key, value), is(false));
    }
}

From source file:com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest.java

License:Apache License

/**
 * Creates a sequence file.// ww w  .  j  av  a  2s.  c  om
 * @throws Exception if failed
 */
@Test
public void write() throws Exception {
    Path path = new Path("testing");

    Text key = new Text();
    Text value = new Text();
    try (OutputStream out = new FileOutputStream(fs.pathToFile(path));
            SequenceFile.Writer writer = SequenceFileUtil.openWriter(new BufferedOutputStream(out), conf,
                    key.getClass(), value.getClass(), null)) {
        key.set("Hello");
        value.set("World");
        writer.append(key, value);
    }
    key.clear();
    value.clear();

    try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf)) {
        assertThat(reader.next(key, value), is(true));
        assertThat(key.toString(), is("Hello"));
        assertThat(value.toString(), is("World"));
        assertThat(reader.next(key, value), is(false));
    }
}

From source file:com.asakusafw.testdriver.directio.DirectFileInputPreparatorTest.java

License:Apache License

private void put(ModelOutput<Text> output, String... contents) throws IOException {
    try {/*  w ww. j a  va2s. co  m*/
        Text text = new Text();
        for (String line : contents) {
            text.set(line);
            output.write(text);
        }
    } finally {
        output.close();
    }
}