Example usage for org.apache.hadoop.io Writable write

Introduction

In this page you can find the example usage for org.apache.hadoop.io Writable write.

Prototype

void write(DataOutput out) throws IOException;

Source Link

Document

Serialize the fields of this object to out.

Usage

From source file:org.terrier.structures.collections.FSArrayFile.java

License:Mozilla Public License

/** writeFSArrayFile
 * /*from ww  w . j a v  a2s. co m*/
 * @param filename
 * @return fixed size array file
 * @throws IOException
 */
public static ArrayFileWriter writeFSArrayFile(String filename) throws IOException {
    final DataOutputStream dos = new DataOutputStream(Files.writeFileStream(filename));
    return new ArrayFileWriter() {

        public void write(Writable w) throws IOException {
            w.write(dos);
        }

        public void close() throws IOException {
            dos.close();
        }
    };
}

From source file:org.terrier.structures.collections.FSOrderedMapFile.java

License:Mozilla Public License

/** returns a utility class which can be used to write a FSOrderedMapFile. 
 * Input data MUST be sorted by key. */
public static MapFileWriter mapFileWrite(final String filename) throws IOException {
    return new MapFileWriter() {
        DataOutputStream out = new DataOutputStream(Files.writeFileStream(filename));

        public void write(WritableComparable key, Writable value) throws IOException {
            //System.err.println("writing key "+ key.toString());
            key.write(out);/*from ww  w .j ava  2s.c  o  m*/
            //System.err.println("writing value "+ value.toString());
            value.write(out);
        }

        public void close() throws IOException {
            out.close();
        }
    };
}

From source file:org.terrier.structures.indexing.singlepass.hadoop.TestPositingAwareSplit.java

License:Mozilla Public License

private byte[] toBytes(Writable w) throws Exception {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    w.write(dos);
    return baos.toByteArray();
}

From source file:org.terrier.structures.TestBasicLexiconEntry.java

License:Mozilla Public License

static byte[] getBytes(Writable w) throws Exception {
    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(buffer);
    w.write(dos);
    return buffer.toByteArray();
}

From source file:org.uv.himongo.io.BSONWritable.java

License:Apache License

/**
 * Used by child copy constructors.//from  w  w w .j a v  a  2 s. c om
 *
 * @param other
 */
protected synchronized void copy(Writable other) {
    if (other != null) {
        try {
            DataOutputBuffer out = new DataOutputBuffer();
            other.write(out);
            DataInputBuffer in = new DataInputBuffer();
            in.reset(out.getData(), out.getLength());
            readFields(in);

        } catch (IOException e) {
            throw new IllegalArgumentException("map cannot be copied: " + e.getMessage());
        }

    } else {
        throw new IllegalArgumentException("source map cannot be null");
    }
}

From source file:uk.ac.cam.eng.extraction.hadoop.util.SimpleHFileOutputFormat.java

License:Apache License

@Override
public RecordWriter<Text, TargetFeatureList> getRecordWriter(TaskAttemptContext job) throws IOException {

    final Configuration conf = job.getConfiguration();
    Path file = getDefaultWorkFile(job, ".hfile");
    FileSystem fs = file.getFileSystem(conf);
    HFile.WriterFactory writerFactory = HFile.getWriterFactory(conf);
    final HFile.Writer writer = writerFactory.createWriter(fs, file, 64 * 1024, "gz", null);
    final CacheConfig cacheConfig = new CacheConfig(conf);
    return new RecordWriter<Text, TargetFeatureList>() {

        private ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();

        private DataOutputStream out = new DataOutputStream(bytesOut);

        BloomFilterWriter bloomFilterWriter = BloomFilterFactory.createBloomAtWrite(conf, cacheConfig,
                BloomType.ROW, -1, writer);

        private byte[] createBytes(Writable obj) throws IOException {
            bytesOut.reset();// w  w  w  .  j  a v  a 2  s  .co  m
            obj.write(out);
            return bytesOut.toByteArray();
        }

        @Override
        public void write(Text key, TargetFeatureList value) throws IOException {
            byte[] keyBytes = createBytes(key);
            byte[] valueBytes = createBytes(value);
            writer.append(keyBytes, valueBytes);
            bloomFilterWriter.add(keyBytes, 0, keyBytes.length);
        }

        @Override
        public void close(TaskAttemptContext context) throws IOException {
            writer.addBloomFilter(bloomFilterWriter);
            writer.close();
        }
    };
}

From source file:utils.SerializationUtils.java

License:Apache License

private byte[] serializeWritable(Writable writable) {
    try {//from   w  ww .  j a v a2  s. c om
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        DataOutputStream dataOut = new DataOutputStream(out);
        writable.write(dataOut);
        dataOut.close();
        return out.toByteArray();
    } catch (Exception ex) {
        Throwables.propagate(ex);
    }
    return null;
}