Example usage for org.apache.hadoop.io Text Text

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text Text.

Prototype

public Text(byte[] utf8)

Source Link

Document

Construct from a byte array.

Usage

From source file:com.asakusafw.runtime.stage.input.TemporaryInputFormatTest.java

License:Apache License

private FileStatus write(Configuration conf, int count) throws IOException {
    Path path = new Path(folder.newFile().toURI());
    try (ModelOutput<Text> output = TemporaryStorage.openOutput(conf, Text.class, path)) {
        Text buffer = new Text("Hello, world!");
        for (int i = 0; i < count; i++) {
            output.write(buffer);//from   w  ww. j a  va2 s  .  c o  m
        }
    }
    return path.getFileSystem(conf).getFileStatus(path);
}

From source file:com.asakusafw.runtime.stage.temporary.TemporaryFileTest.java

License:Apache License

private void doIo(int count) throws IOException {
    // eagerly initializes snappy
    Snappy.getNativeLibraryVersion();/*from   w  w w.  j a  v a 2  s.  c  om*/

    File file = folder.newFile();
    Text value = new Text("Hello, world!");

    long t0 = System.currentTimeMillis();
    try (ModelOutput<Text> out = new TemporaryFileOutput<>(new BufferedOutputStream(new FileOutputStream(file)),
            Text.class.getName(), 530 * 1024, 512 * 1024)) {
        for (int i = 0; i < count; i++) {
            out.write(value);
        }
    }
    long t1 = System.currentTimeMillis();
    System.out.println(
            MessageFormat.format("WRITE: time={0}, rows={1}, size={2}", t1 - t0, count, file.length()));

    try (TemporaryFileInput<Text> in = new TemporaryFileInput<>(
            new BufferedInputStream(new FileInputStream(file)), 0)) {
        Text result = new Text();
        assertThat(in.getDataTypeName(), is(Text.class.getName()));
        for (int i = 0; i < count; i++) {
            assertTrue(in.readTo(result));
            if (i == 0) {
                assertThat(result, is(value));
            }
        }
        assertThat(in.readTo(result), is(false));
    }

    long t2 = System.currentTimeMillis();
    System.out
            .println(MessageFormat.format("READ: time={0}, rows={1}, size={2}", t2 - t1, count, file.length()));
}

From source file:com.asakusafw.runtime.stage.temporary.TemporaryFileTest.java

License:Apache License

private void doIo_w_TemporaryStorage(int count) throws IOException {
    File file = folder.newFile();
    Text value = new Text("Hello, world!");

    long t0 = System.currentTimeMillis();
    try (ModelOutput<Text> out = TemporaryStorage.openOutput(new Configuration(), Text.class,
            new Path(file.toURI()))) {
        for (int i = 0; i < count; i++) {
            out.write(value);/*from ww w.j ava 2  s. co  m*/
        }
    }
    long t1 = System.currentTimeMillis();
    System.out.println(
            MessageFormat.format("WRITE: time={0}, rows={1}, size={2}", t1 - t0, count, file.length()));

    try (ModelInput<Text> in = TemporaryStorage.openInput(new Configuration(), Text.class,
            new Path(file.toURI()))) {
        Text result = new Text();
        for (int i = 0; i < count; i++) {
            assertTrue(in.readTo(result));
            if (i == 0) {
                assertThat(result, is(value));
            }
        }
        assertThat(in.readTo(result), is(false));
    }

    long t2 = System.currentTimeMillis();
    System.out
            .println(MessageFormat.format("READ: time={0}, rows={1}, size={2}", t2 - t1, count, file.length()));
}

From source file:com.asakusafw.runtime.value.StringOptionTest.java

License:Apache License

/**
 * test for set w/ text.//from ww w .j a  va 2  s  . c o m
 */
@Test
public void text() {
    StringOption option = new StringOption();
    option.modify(new Text("Hello, world!"));
    assertThat(option.isNull(), is(false));
    assertThat(option.get().toString(), is("Hello, world!"));
}

From source file:com.asakusafw.runtime.value.StringOptionTest.java

License:Apache License

/**
 * test for or w/ text./*w  w w . ja v a 2s  .  c o  m*/
 */
@Test
public void textOr() {
    StringOption option = new StringOption();
    assertThat(option.or(new Text("Hello")).toString(), is("Hello"));

    assertThat(option.isNull(), is(true));
    option.modify(new Text("World"));
    assertThat(option.or(new Text("Other")).toString(), is("World"));
}

From source file:com.asakusafw.runtime.value.StringOptionTest.java

License:Apache License

/**
 * test for Japanese characters.//  w w w  .j ava 2s. c  o  m
 */
@Test
public void japanese() {
    StringOption option = new StringOption();
    option.modify(new Text(HELLO_JP));
    assertThat(option.getAsString(), is(HELLO_JP));
}

From source file:com.asakusafw.testdriver.file.FileExporterRetrieverTest.java

License:Apache License

/**
 * minimum test.//from  www . j  a  v  a  2  s  . c  o  m
 * @throws Exception if test was failed
 */
@Test
public void simple() throws Exception {
    MockFileExporter exporter = new MockFileExporter(Text.class, TextOutputFormat.class,
            "target/testing/hello");
    FileExporterRetriever retriever = new FileExporterRetriever(factory);

    putTextRaw("target/testing/hello", "Hello, world!\nThis is a test.\n".getBytes("UTF-8"));

    MockTextDefinition definition = new MockTextDefinition();
    try (DataModelSource result = retriever.createSource(definition, exporter, EMPTY)) {
        DataModelReflection ref;
        ref = result.next();
        assertThat(ref, is(not(nullValue())));
        assertThat(definition.toObject(ref), is(new Text("Hello, world!")));

        ref = result.next();
        assertThat(ref, is(not(nullValue())));
        assertThat(definition.toObject(ref), is(new Text("This is a test.")));

        ref = result.next();
        assertThat(ref, is(nullValue()));
    }
}

From source file:com.asakusafw.testdriver.file.FileExporterRetrieverTest.java

License:Apache License

/**
 * using sequence file.// w w w .j  a v a 2s  . c  om
 * @throws Exception if test was failed
 */
@Test
public void sequenceFile() throws Exception {
    MockFileExporter exporter = new MockFileExporter(Text.class, SequenceFileOutputFormat.class,
            "target/testing/hello");
    FileExporterRetriever retriever = new FileExporterRetriever(factory);

    putTextSequenceFile("target/testing/hello", "Hello, world!", "This is a test.");

    MockTextDefinition definition = new MockTextDefinition();
    try (DataModelSource result = retriever.createSource(definition, exporter, EMPTY)) {
        DataModelReflection ref;
        ref = result.next();
        assertThat(ref, is(not(nullValue())));
        assertThat(definition.toObject(ref), is(new Text("Hello, world!")));

        ref = result.next();
        assertThat(ref, is(not(nullValue())));
        assertThat(definition.toObject(ref), is(new Text("This is a test.")));

        ref = result.next();
        assertThat(ref, is(nullValue()));
    }
}

From source file:com.asakusafw.testdriver.file.FileExporterRetrieverTest.java

License:Apache License

private void putTextSequenceFile(String path, String... lines) throws IOException {
    try (SequenceFile.Writer writer = SequenceFile.createWriter(factory.newInstance(),
            SequenceFile.Writer.file(fileSystem.makeQualified(new Path(path))),
            SequenceFile.Writer.keyClass(NullWritable.class), SequenceFile.Writer.valueClass(Text.class))) {
        for (String s : lines) {
            writer.append(NullWritable.get(), new Text(s));
        }// w w  w  .  j a  v  a 2 s . co  m
    }
}

From source file:com.asakusafw.testdriver.file.FileImporterPreparatorTest.java

License:Apache License

/**
 * minimum.// w w w.j av a  2  s.c o  m
 * @throws Exception if test was failed
 */
@Test
public void simple() throws Exception {
    FileImporterPreparator target = new FileImporterPreparator(factory);
    try (ModelOutput<Text> open = target.createOutput(new MockTextDefinition(),
            new MockFileImporter(Text.class, TextInputFormat.class, "target/testing/input"), EMPTY)) {
        open.write(new Text("Hello, world!"));
    }
    try (Scanner scanner = new Scanner(loadResult("target/testing/input"), "UTF-8")) {
        assertThat(scanner.hasNextLine(), is(true));
        assertThat(scanner.nextLine(), is("Hello, world!"));
        assertThat(scanner.hasNextLine(), is(false));
    }
}