Example usage for org.apache.hadoop.io WritableComparable write

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableComparable write.

Prototype

void write(DataOutput out) throws IOException;

Source Link

Document

Serialize the fields of this object to out.

Usage

From source file:com.ebay.erl.mobius.core.model.WriteImpl.java

License:Apache License

@SuppressWarnings("unchecked")
@Override/*  ww w. ja v a  2 s  .c  o m*/
protected Void on_writable() throws IOException {
    WritableComparable w = (WritableComparable) value;
    out.writeUTF(w.getClass().getCanonicalName());
    w.write(out);
    return null;
}

---

From source file:edu.uci.ics.pregelix.dataflow.VertexFileScanOperatorDescriptor.java

License:Apache License

public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
        IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
        throws HyracksDataException {
    final List<FileSplit> splits = splitsFactory.getSplits();

    return new AbstractUnaryOutputSourceOperatorNodePushable() {
        private ClassLoader ctxCL;
        private ContextFactory ctxFactory = new ContextFactory();

        @Override/*from  w  ww  .ja v  a2 s  . c  om*/
        public void initialize() throws HyracksDataException {
            ctxCL = Thread.currentThread().getContextClassLoader();
            try {
                Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
                Configuration conf = confFactory.createConfiguration();
                writer.open();
                for (int i = 0; i < scheduledLocations.length; i++) {
                    if (scheduledLocations[i]
                            .equals(ctx.getJobletContext().getApplicationContext().getNodeId())) {
                        /**
                         * pick one from the FileSplit queue
                         */
                        synchronized (executed) {
                            if (!executed[i]) {
                                executed[i] = true;
                            } else {
                                continue;
                            }
                        }
                        loadVertices(ctx, conf, i);
                    }
                }
                writer.close();
            } catch (Exception e) {
                throw new HyracksDataException(e);
            } finally {
                Thread.currentThread().setContextClassLoader(ctxCL);
            }
        }

        /**
         * Load the vertices
         * 
         * @parameter IHyracks ctx
         * @throws IOException
         * @throws IllegalAccessException
         * @throws InstantiationException
         * @throws ClassNotFoundException
         * @throws InterruptedException
         */
        @SuppressWarnings("unchecked")
        private void loadVertices(final IHyracksTaskContext ctx, Configuration conf, int splitId)
                throws IOException, ClassNotFoundException, InterruptedException, InstantiationException,
                IllegalAccessException {
            ByteBuffer frame = ctx.allocateFrame();
            FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
            appender.reset(frame, true);

            VertexInputFormat vertexInputFormat = BspUtils.createVertexInputFormat(conf);
            InputSplit split = splits.get(splitId);
            TaskAttemptContext mapperContext = ctxFactory.createContext(conf, splitId);

            VertexReader vertexReader = vertexInputFormat.createVertexReader(split, mapperContext);
            vertexReader.initialize(split, mapperContext);
            Vertex readerVertex = (Vertex) BspUtils.createVertex(conf);
            ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldSize);
            DataOutput dos = tb.getDataOutput();

            /**
             * set context
             */
            Vertex.setContext(mapperContext);

            /**
             * empty vertex value
             */
            Writable emptyVertexValue = (Writable) BspUtils.createVertexValue(conf);

            while (vertexReader.nextVertex()) {
                readerVertex = vertexReader.getCurrentVertex();
                tb.reset();
                if (readerVertex.getVertexId() == null) {
                    throw new IllegalArgumentException("loadVertices: Vertex reader returned a vertex "
                            + "without an id!  - " + readerVertex);
                }
                if (readerVertex.getVertexValue() == null) {
                    readerVertex.setVertexValue(emptyVertexValue);
                }
                WritableComparable vertexId = readerVertex.getVertexId();
                vertexId.write(dos);
                tb.addFieldEndOffset();

                readerVertex.write(dos);
                tb.addFieldEndOffset();

                if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
                    if (appender.getTupleCount() <= 0)
                        throw new IllegalStateException("zero tuples in a frame!");
                    FrameUtils.flushFrame(frame, writer);
                    appender.reset(frame, true);
                    if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
                        throw new IllegalStateException();
                    }
                }
            }

            vertexReader.close();
            if (appender.getTupleCount() > 0) {
                FrameUtils.flushFrame(frame, writer);
            }
            System.gc();
        }
    };
}

---

From source file:edu.uci.ics.pregelix.runtime.converter.ReadConverterFactory.java

License:Apache License

@SuppressWarnings("rawtypes")
@Override//from   w  w  w .  ja va  2s. c o  m
public IReadConverter getReadConverter(IHyracksTaskContext ctx, int partitionId) throws HyracksDataException {
    final Configuration conf = confFactory.createConfiguration();
    // Set context properly
    ContextFactory ctxFactory = new ContextFactory();
    TaskAttemptContext mapperContext = ctxFactory.createContext(conf, partitionId);
    mapperContext.getConfiguration().setClassLoader(ctx.getJobletContext().getClassLoader());
    conf.setClassLoader(ctx.getJobletContext().getClassLoader());
    IterationUtils.setJobContext(BspUtils.getJobId(conf), ctx, mapperContext);
    Vertex.taskContext = mapperContext;

    final Vertex vertex = BspUtils.createVertex(conf);
    vertex.setVertexContext(IterationUtils.getVertexContext(BspUtils.getJobId(conf), ctx));

    final VertexInputConverter inputConverter = BspUtils.createVertexInputConverter(conf);

    return new IReadConverter() {

        @Override
        public void open(ARecordType recordType) throws HyracksDataException {
            inputConverter.open(recordType);
        }

        @Override
        public void convert(ARecordVisitablePointable recordPointable, ArrayTupleBuilder outputTb)
                throws HyracksDataException {
            try {
                // Converts an input AsterixDB record into an vertex object.
                vertex.reset();
                inputConverter.convert(recordPointable, vertex);

                // Outputs a tuple of <vertexId, vertex>.
                outputTb.reset();
                WritableComparable vertexId = vertex.getVertexId();
                DataOutput dos = outputTb.getDataOutput();
                vertexId.write(dos);
                outputTb.addFieldEndOffset();
                vertex.write(dos);
                outputTb.addFieldEndOffset();
            } catch (Exception e) {
                throw new HyracksDataException(e);
            }
        }

        @Override
        public void close() throws HyracksDataException {
            inputConverter.close();
        }

    };
}

---

From source file:org.apache.orc.mapred.OrcStruct.java

License:Apache License

@Override
public void write(DataOutput output) throws IOException {
    for (WritableComparable field : fields) {
        output.writeBoolean(field != null);
        if (field != null) {
            field.write(output);
        }/* w w w .j ava 2s .  c o m*/
    }
}

---

From source file:org.godhuli.rhipe.RHMRHelper.java

License:Apache License

public void write(WritableComparable c) throws IOException {
    c.write(clientOut_);
}

---

From source file:org.terrier.structures.collections.FSOrderedMapFile.java

License:Mozilla Public License

/** returns a utility class which can be used to write a FSOrderedMapFile. 
 * Input data MUST be sorted by key. */
public static MapFileWriter mapFileWrite(final String filename) throws IOException {
    return new MapFileWriter() {
        DataOutputStream out = new DataOutputStream(Files.writeFileStream(filename));

        public void write(WritableComparable key, Writable value) throws IOException {
            //System.err.println("writing key "+ key.toString());
            key.write(out);
            //System.err.println("writing value "+ value.toString());
            value.write(out);//from  w ww . j  a v a2  s .c o m
        }

        public void close() throws IOException {
            out.close();
        }
    };
}

---

From source file:tl.lin.data.WritableComparatorTestHarness.java

License:Apache License

@SuppressWarnings("rawtypes")
public static int compare(WritableComparator comparator, WritableComparable obj1, WritableComparable obj2) {

    byte[] bytes1 = null, bytes2 = null;

    try {//from   w  w w.j a va  2  s  .  c  o m
        ByteArrayOutputStream bytesOut1 = new ByteArrayOutputStream();
        DataOutputStream dataOut1 = new DataOutputStream(bytesOut1);
        obj1.write(dataOut1);
        bytes1 = bytesOut1.toByteArray();

        ByteArrayOutputStream bytesOut2 = new ByteArrayOutputStream();
        DataOutputStream dataOut2 = new DataOutputStream(bytesOut2);
        obj2.write(dataOut2);
        bytes2 = bytesOut2.toByteArray();

    } catch (IOException e) {
        e.printStackTrace();
    }

    return comparator.compare(bytes1, 0, bytes1.length, bytes2, 0, bytes2.length);
}

---

From source file:uk.ac.cam.eng.extraction.hadoop.merge.MergeComparator.java

License:Apache License

@Override
@SuppressWarnings("rawtypes")
public int compare(WritableComparable a, WritableComparable b) {
    try {//from w ww .j  av a 2  s  .  co m
        buffera.reset();
        a.write(buffera);
        bufferb.reset();
        b.write(bufferb);
        return compareBytes(buffera.getData(), 0, buffera.getLength(), bufferb.getData(), 0,
                bufferb.getLength());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

---