Example usage for org.apache.hadoop.io Text Text

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text Text.

Prototype

public Text()

Source Link

Usage

From source file:com.alexholmes.hadooputils.sort.SortRecordReader.java

License:Apache License

@Override
public Text createKey() {
    return new Text();
}

From source file:com.alibaba.jstorm.hdfs.bolt.format.DefaultSequenceFormat.java

License:Apache License

@Override
public Writable value(Tuple tuple) {
    if (this.value == null) {
        this.value = new Text();
    }/*from w  ww.j a  v  a  2s  .  c o m*/
    this.value.set(tuple.getStringByField(this.valueField));
    return this.value;
}

From source file:com.alibaba.jstorm.hdfs.trident.format.DefaultSequenceFormat.java

License:Apache License

@Override
public Writable value(TridentTuple tuple) {
    if (this.value == null) {
        this.value = new Text();
    }/*from   ww w .  j a  v a  2 s.  c  o m*/
    this.value.set(tuple.getStringByField(this.valueField));
    return this.value;
}

From source file:com.aliyun.fs.oss.common.OssRecordReader.java

License:Apache License

public OssRecordReader(Configuration job, FileSplit split, FileSystem fs, byte[] recordDelimiter)
        throws IOException {
    this.maxLineLength = job.getInt(org.apache.hadoop.mapreduce.lib.input.LineRecordReader.MAX_LINE_LENGTH,
            Integer.MAX_VALUE);// w  w  w  . ja  v a 2  s . co m
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    fileIn = fs.open(file);
    if (isCompressedInput()) {
        decompressor = CodecPool.getDecompressor(codec);
        if (codec instanceof SplittableCompressionCodec) {
            final SplitCompressionInputStream cIn = ((SplittableCompressionCodec) codec).createInputStream(
                    fileIn, decompressor, start, end, SplittableCompressionCodec.READ_MODE.BYBLOCK);
            in = new LineReader(cIn, job, recordDelimiter);
            start = cIn.getAdjustedStart();
            end = cIn.getAdjustedEnd();
            filePosition = cIn; // take pos from compressed stream
        } else {
            in = new LineReader(codec.createInputStream(fileIn, decompressor), job, recordDelimiter);
            filePosition = fileIn;
        }
    } else {
        fileIn.seek(start);
        in = new LineReader(fileIn, job, recordDelimiter);
        filePosition = fileIn;
    }
    // If this is not the first split, we always throw away first record
    // because we always (except the last split) read one extra line in
    // next() method.
    if (start != 0) {
        start += in.readLine(new Text(), 0, maxBytesToConsume(start));
    }
    this.pos = start;
}

From source file:com.asakusafw.dag.compiler.internalio.InternalOutputPrepareGeneratorTest.java

License:Apache License

private static List<String> collect(File folder) throws IOException {
    List<String> results = new ArrayList<>();
    for (File file : folder.listFiles(f -> f.getName().startsWith("part-"))) {
        try (ModelInput<Text> in = LocalInternalInputTaskInfo.open(file)) {
            Text buf = new Text();
            while (in.readTo(buf)) {
                results.add(buf.toString());
            }//  ww w.j  av  a  2 s  .c  o m
        }
    }
    return results;
}

From source file:com.asakusafw.dag.runtime.internalio.HadoopInternalInputTaskInfoTest.java

License:Apache License

private static void put(FileSystem fs, Path path, String... values) throws IOException {
    try (ModelOutput<Text> out = InternalOutputHandler.create(fs.create(path), Text.class)) {
        Text buf = new Text();
        for (String value : values) {
            buf.set(value);//w w  w.  j  av  a  2 s .  co m
            out.write(buf);
        }
    }
}

From source file:com.asakusafw.dag.runtime.internalio.InternalInputAdapterTest.java

License:Apache License

private static File put(File file, String... lines) throws IOException {
    Lang.let(file.getParentFile(), f -> Assume.assumeTrue(f.mkdirs() || f.isDirectory()));
    try (ModelOutput<Text> out = InternalOutputHandler.create(new FileOutputStream(file), Text.class)) {
        Text buf = new Text();
        for (String line : lines) {
            buf.set(line);//ww w  . j  a v  a2 s.  c  o  m
            out.write(buf);
        }
    }
    return file;
}

From source file:com.asakusafw.dag.runtime.internalio.LocalInternalInputTaskInfoTest.java

License:Apache License

private static void put(File file, String... values) throws IOException {
    try (ModelOutput<Text> out = InternalOutputHandler.create(new FileOutputStream(file), Text.class)) {
        Text buf = new Text();
        for (String value : values) {
            buf.set(value);/*from ww  w  .  ja v  a2s. com*/
            out.write(buf);
        }
    }
}

From source file:com.asakusafw.example.direct.seqfile.jobflow.format.CategorySummarySeqFileFormat.java

License:Apache License

@Override
protected Text createKeyObject() {
    return new Text();
}

From source file:com.asakusafw.example.direct.seqfile.jobflow.format.StoreInfoSeqFileFormat.java

License:Apache License

@Override
protected Text createValueObject() {
    return new Text();
}