Example usage for org.apache.hadoop.io ByteWritable ByteWritable

List of usage examples for org.apache.hadoop.io ByteWritable ByteWritable

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. ByteWritable
  6. ByteWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io ByteWritable ByteWritable.

Prototype

public ByteWritable()

Source Link

Usage

From source file:com.cloudera.recordservice.mr.RecordServiceRecord.java

License:Apache License

/**
 * Returns the corresponding Writable object for this column type.
 */// www .j a v  a 2s . c om
public Writable getWritableInstance(com.cloudera.recordservice.core.Schema.Type type) {
    switch (type) {
    case BOOLEAN:
        return new BooleanWritable();
    case TINYINT:
        return new ByteWritable();
    case SMALLINT:
        return new ShortWritable();
    case INT:
        return new IntWritable();
    case BIGINT:
        return new LongWritable();
    case FLOAT:
        return new FloatWritable();
    case DOUBLE:
        return new DoubleWritable();
    case VARCHAR:
    case CHAR:
    case STRING:
        return new Text();
    case TIMESTAMP_NANOS:
        return new TimestampNanosWritable();
    case DECIMAL:
        return new DecimalWritable();
    default:
        throw new UnsupportedOperationException("Unexpected type: " + toString());
    }
}

From source file:com.dasasian.chok.util.WritableType.java

License:Apache License

public WritableComparable newWritableComparable() {
    switch (this) {
    case TEXT://  w w w . ja va 2  s.  c  o m
        return new Text();
    case BYTE:
        return new ByteWritable();
    case INT:
        return new IntWritable();
    case LONG:
        return new LongWritable();
    case FLOAT:
        return new FloatWritable();
    case DOUBLE:
        return new DoubleWritable();
    }
    throw getUnhandledTypeException();
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.CSVReblockMapper.java

License:Open Source License

@Override
@SuppressWarnings("deprecation")
public void configure(JobConf job) {
    super.configure(job);
    //get the number colums per block

    //load the offset mapping
    byte matrixIndex = representativeMatrixes.get(0);
    try {/*from  w  w w . j  a va 2 s  .c  o  m*/
        FileSystem fs = FileSystem.get(job);
        Path thisPath = new Path(job.get("map.input.file")).makeQualified(fs);
        String filename = thisPath.toString();
        Path headerPath = new Path(job.getStrings(CSVReblockMR.SMALLEST_FILE_NAME_PER_INPUT)[matrixIndex])
                .makeQualified(fs);
        if (headerPath.toString().equals(filename))
            headerFile = true;

        ByteWritable key = new ByteWritable();
        OffsetCount value = new OffsetCount();
        Path p = new Path(job.get(CSVReblockMR.ROWID_FILE_NAME));
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, job);
        while (reader.next(key, value)) {
            if (key.get() == matrixIndex && filename.equals(value.filename))
                offsetMap.put(value.fileOffset, value.count);
        }
        reader.close();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }

    CSVReblockInstruction ins = csv_reblock_instructions.get(0).get(0);
    _delim = ins.delim;
    ignoreFirstLine = ins.hasHeader;

    idxRow = new IndexedBlockRow();
    int maxBclen = 0;

    for (ArrayList<CSVReblockInstruction> insv : csv_reblock_instructions)
        for (CSVReblockInstruction in : insv) {
            if (maxBclen < in.bclen)
                maxBclen = in.bclen;
        }

    //always dense since common csv usecase
    idxRow.getRow().data.reset(1, maxBclen, false);
}

From source file:com.ibm.bi.dml.runtime.transform.ApplyTfBBMapper.java

License:Open Source License

@Override
public void configure(JobConf job) {
    super.configure(job);
    try {/*from   w  w  w . j  av  a2 s .  c o  m*/
        _partFileWithHeader = TfUtils.isPartFileWithHeader(job);
        tfmapper = new TfUtils(job);
        tfmapper.loadTfMetadata(job, true);

        // Load relevant information for CSV Reblock
        ByteWritable key = new ByteWritable();
        OffsetCount value = new OffsetCount();
        Path p = new Path(job.get(CSVReblockMR.ROWID_FILE_NAME));

        FileSystem fs = FileSystem.get(job);
        Path thisPath = new Path(job.get("map.input.file")).makeQualified(fs);
        String thisfile = thisPath.toString();

        SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, job);
        while (reader.next(key, value)) {
            // "key" needn't be checked since the offset file has information about a single CSV input (the raw data file)
            if (thisfile.equals(value.filename))
                offsetMap.put(value.fileOffset, value.count);
        }
        reader.close();

        idxRow = new CSVReblockMapper.IndexedBlockRow();
        int maxBclen = 0;

        for (ArrayList<CSVReblockInstruction> insv : csv_reblock_instructions)
            for (CSVReblockInstruction in : insv) {
                if (maxBclen < in.bclen)
                    maxBclen = in.bclen;
            }

        //always dense since common csv usecase
        idxRow.getRow().data.reset(1, maxBclen, false);

    } catch (IOException e) {
        throw new RuntimeException(e);
    } catch (JSONException e) {
        throw new RuntimeException(e);
    }

}

From source file:com.ibm.bi.dml.runtime.transform.TfUtils.java

License:Open Source License

/**
 * Function to generate custom file names (transform-part-.....) for
 * mappers' output for ApplyTfCSV job. The idea is to find the index 
 * of (thisfile, fileoffset) in the list of all offsets from the 
 * counters/offsets file, which was generated from either GenTfMtdMR
 * or AssignRowIDMR job./*from   w  w  w.j a  v a  2  s  . c o m*/
 * 
 */
public String getPartFileID(JobConf job, long offset) throws IOException {
    Reader reader = initOffsetsReader(job);

    ByteWritable key = new ByteWritable();
    OffsetCount value = new OffsetCount();
    String thisFile = TfUtils.getPartFileName(job);

    int id = 0;
    while (reader.next(key, value)) {
        if (thisFile.equals(value.filename) && value.fileOffset == offset)
            break;
        id++;
    }
    reader.close();

    String sid = Integer.toString(id);
    char[] carr = new char[5 - sid.length()];
    Arrays.fill(carr, '0');
    String ret = (new String(carr)).concat(sid);

    return ret;
}

From source file:com.linkedin.cubert.io.CompactWritablesDeserializer.java

License:Open Source License

private static final WritableComparable<?> createWritable(DataType type) {
    switch (type) {
    case BOOLEAN:
        return new BooleanWritable();
    case BYTE:/*ww  w .  ja  va  2  s  . c om*/
        return new ByteWritable();
    case INT:
        return new IntWritable();
    case LONG:
        return new LongWritable();
    case FLOAT:
        return new FloatWritable();
    case DOUBLE:
        return new DoubleWritable();
    case STRING:
        return new Text();
    default:
        return null;
    }
}

From source file:edu.uci.ics.hivesterix.serde.lazy.LazyByte.java

License:Apache License

public LazyByte(LazyByteObjectInspector oi) {
    super(oi);
    data = new ByteWritable();
}

From source file:org.apache.camel.component.hdfs.HdfsConsumerTest.java

License:Apache License

@Test
public void testReadByte() throws Exception {
    if (!canTest()) {
        return;//w w  w.j av  a  2  s.  c o m
    }

    final Path file = new Path(new File("target/test/test-camel-byte").getAbsolutePath());
    Configuration conf = new Configuration();
    FileSystem fs1 = FileSystem.get(file.toUri(), conf);
    SequenceFile.Writer writer = createWriter(fs1, conf, file, NullWritable.class, ByteWritable.class);
    NullWritable keyWritable = NullWritable.get();
    ByteWritable valueWritable = new ByteWritable();
    byte value = 3;
    valueWritable.set(value);
    writer.append(keyWritable, valueWritable);
    writer.sync();
    writer.close();

    MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class);
    resultEndpoint.expectedMessageCount(1);
    resultEndpoint.message(0).body(byte.class).isEqualTo(3);

    context.addRoutes(new RouteBuilder() {
        public void configure() {
            from("hdfs:///" + file.toUri() + "?fileSystemType=LOCAL&fileType=SEQUENCE_FILE&initialDelay=0")
                    .to("mock:result");
        }
    });
    context.start();

    resultEndpoint.assertIsSatisfied();
}

From source file:org.apache.camel.component.hdfs2.HdfsConsumerTest.java

License:Apache License

@Test
public void testReadByte() throws Exception {
    if (!canTest()) {
        return;/* w w  w .j a v a2  s.co m*/
    }

    final Path file = new Path(new File("target/test/test-camel-byte").getAbsolutePath());
    Configuration conf = new Configuration();
    SequenceFile.Writer writer = createWriter(conf, file, NullWritable.class, ByteWritable.class);
    NullWritable keyWritable = NullWritable.get();
    ByteWritable valueWritable = new ByteWritable();
    byte value = 3;
    valueWritable.set(value);
    writer.append(keyWritable, valueWritable);
    writer.sync();
    writer.close();

    MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class);
    resultEndpoint.expectedMessageCount(1);
    resultEndpoint.message(0).body(byte.class).isEqualTo(3);

    context.addRoutes(new RouteBuilder() {
        public void configure() {
            from("hdfs2:///" + file.toUri() + "?fileSystemType=LOCAL&fileType=SEQUENCE_FILE&initialDelay=0")
                    .to("mock:result");
        }
    });
    context.start();

    resultEndpoint.assertIsSatisfied();
}

From source file:org.apache.giraph.block_app.library.algo.BreadthFirstSearch.java

License:Apache License

/**
 * Default block for computing breadth-first search distances given functions
 * isVertexInSeedSet, getDistance and setDistance. This BFS computation block
 * computes only the shortest distance to seed vertices and does not compute
 * the closest seed./*w  ww.  ja  va  2  s . c  o m*/
 */
public static <I extends WritableComparable, V extends Writable> Block bfs(
        SupplierFromVertex<I, V, Writable, Boolean> isVertexInSeedSet,
        SupplierFromVertex<I, V, Writable, IntWritable> getDistance,
        ConsumerWithVertex<I, V, Writable, IntWritable> setDistance) {
    ObjectTransfer<Boolean> converged = new ObjectTransfer<>();
    ObjectTransfer<ByteWritable> vertexUpdatedDistance = new ObjectTransfer<>();

    IntWritable reusableInt = new IntWritable();
    ByteWritable emptyByteWritable = new ByteWritable();
    SupplierFromVertex<I, V, Writable, ByteWritable> initializeVertex = (vertex) -> {
        if (isVertexInSeedSet.get(vertex)) {
            reusableInt.set(0);
            setDistance.apply(vertex, reusableInt);
            return emptyByteWritable;
        } else {
            reusableInt.set(-1);
            setDistance.apply(vertex, reusableInt);
            return null;
        }
    };

    IntWritable notReachableVertex = new IntWritable(-1);
    TripleFunction<Vertex<I, V, Writable>, IntWritable, Iterator<ByteWritable>, ByteWritable> traverseVertex = (
            vertex, distance, messageIter) -> {
        if (getDistance.get(vertex).compareTo(notReachableVertex) == 0
                || getDistance.get(vertex).compareTo(distance) > 0) {
            setDistance.apply(vertex, distance);
            return emptyByteWritable;
        } else {
            return null;
        }
    };

    Class<ByteWritable> messageClass = ByteWritable.class;
    return new SequenceBlock(createInitializePiece(vertexUpdatedDistance, initializeVertex),
            RepeatUntilBlock.unlimited(createPropagateConnectedComponentsPiece(messageClass,
                    vertexUpdatedDistance, vertexUpdatedDistance, converged, traverseVertex,
                    VertexSuppliers.vertexEdgesSupplier()), converged));
}