Example usage for org.apache.hadoop.io LongWritable LongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable LongWritable.

Prototype

public LongWritable(long value) 

Source Link

Usage

From source file:com.ibm.bi.dml.runtime.instructions.spark.functions.ConvertStringToLongTextPair.java

License:Open Source License

@Override
public Tuple2<LongWritable, Text> call(String arg0) throws Exception {
    return new Tuple2<LongWritable, Text>(new LongWritable(1), new Text(arg0));
}

---

From source file:com.ibm.bi.dml.runtime.instructions.spark.functions.CopyTextInputFunction.java

License:Open Source License

@Override
public Tuple2<LongWritable, Text> call(Tuple2<LongWritable, Text> arg0) throws Exception {
    LongWritable lw = new LongWritable(arg0._1().get());
    Text txt = new Text(arg0._2());
    return new Tuple2<LongWritable, Text>(lw, txt);
}

---

From source file:com.ibm.jaql.lang.expr.system.RJaqlInterface.java

License:Apache License

/**
 * This method provides the functionality of saving simple R objects into HDFS in one of
 * the formats supported by Jaql so that it can be directly read into Jaql.
 * @param localPath/*from  w  ww .ja  va2 s  . c  o m*/
 * @param hdfsPath
 * @param schemaString
 * @param format
 * @param header
 * @param vector
 * @return
 */
public boolean jaqlSave(String localPath, String hdfsPath, String schemaString, String format, boolean header,
        boolean vector) {
    if (format.equalsIgnoreCase(FORMAT_DELIM)) {
        LOG.info("Format: " + FORMAT_DELIM + ", saving to HDFS loc: " + hdfsPath);
        return RUtil.saveToHDFS(localPath, hdfsPath);
    }
    try {
        JobConf conf = new JobConf();
        int DEFAULT_BUFFER_SIZE = 64 * 1024;
        int bufferSize = conf.getInt("io.file.buffer.size", DEFAULT_BUFFER_SIZE);
        BufferedReader reader = new BufferedReader(new FileReader(localPath), bufferSize);
        LongWritable key = new LongWritable(0);
        long count = 0;
        Text value = new Text();
        BufferedJsonRecord options = new BufferedJsonRecord(2);
        BufferedJsonArray headerArray = null;
        if (header) {
            String headerString = reader.readLine();
            String[] headers = splitPattern.split(headerString);
            headerArray = new BufferedJsonArray(headers.length);
            for (int i = 0; i < headers.length; i++) {
                headerArray.set(i, new JsonString(StringUtils.strip(headers[i], "\"")));
            }
            count++;
        }

        Schema schema = null;
        if (schemaString != null) {
            schema = SchemaFactory.parse(schemaString);
        }

        if (headerArray != null) {
            RecordSchema recordSchema = (RecordSchema) schema;

            // construct new matching schema
            List<Field> fields = new LinkedList<Field>();
            for (JsonValue fieldName : headerArray) {
                Field field;
                if (recordSchema == null) {
                    field = new Field((JsonString) fieldName, SchemaFactory.stringSchema(), false);
                } else {
                    field = recordSchema.getField((JsonString) fieldName);
                    if (field == null)
                        throw new NullPointerException("header field not in schema: " + fieldName);
                    // FIXME: schema fields that are not in the header are currently consider OK
                }
                fields.add(field);
            }

            // and set it
            schema = new RecordSchema(fields, null);
        }
        if (schema != null)
            options.add(DelOptionParser.SCHEMA_NAME, new JsonSchema(schema));
        KeyValueImport<LongWritable, Text> converter = null;
        if (vector) {
            converter = new FromLinesConverter();
        } else {
            converter = new FromDelConverter();
        }
        LOG.info("Initializing Converter with options: " + options);
        converter.init(options);
        Schema tmpSchema = converter.getSchema();
        tmpSchema = SchemaTransformation.removeNullability(tmpSchema);
        if (!tmpSchema.is(JsonType.ARRAY, JsonType.RECORD, JsonType.BOOLEAN, JsonType.DECFLOAT, JsonType.DOUBLE,
                JsonType.LONG, JsonType.STRING).always()) {
            throw new IOException("Unrecognized schema type: " + schema.getSchemaType());
        }
        JsonValue outValue = converter.createTarget();
        JsonHolder outKeyHolder;
        JsonHolder outValueHolder;
        if (format.equalsIgnoreCase(FORMAT_DEFAULT)) {
            HadoopSerializationDefault.register(conf);
            outKeyHolder = new JsonHolderDefault();
            outValueHolder = new JsonHolderDefault(outValue);
            LOG.info("Registered serializer for Default format.");
        } else if (format.equalsIgnoreCase(FORMAT_TEMP)) {
            // TODO: There should be a better way of doing this. HadoopSerializationTemp
            // now does it in an ugly way.
            BufferedJsonRecord tmpOptions = new BufferedJsonRecord();
            BufferedJsonRecord outOptions = new BufferedJsonRecord();
            outOptions.add(new JsonString("schema"), new JsonSchema(schema));
            tmpOptions.add(new JsonString("options"), outOptions);
            conf.set(ConfSetter.CONFOUTOPTIONS_NAME, tmpOptions.toString());
            HadoopSerializationTemp.register(conf);
            outKeyHolder = new JsonHolderTempKey(null);
            outValueHolder = new JsonHolderTempValue();
            LOG.info("Registered serializer for HadoopTemp format.");
        } else {
            throw new IOException("Unrecognized serialization format requested: " + format);
        }
        FileSystem fs = FileSystem.get(conf);
        Path outputPath = new Path(hdfsPath);
        Writer writer = SequenceFile.createWriter(fs, conf, outputPath, outKeyHolder.getClass(),
                outValueHolder.getClass());
        String line;
        while ((line = reader.readLine()) != null) {
            key.set(count++);
            value.set(line);
            outValue = converter.convert(key, value, outValue);
            outValueHolder.value = outValue;
            writer.append(outKeyHolder, outValueHolder);
        }
        LOG.info("Transferred " + count + " line(s).");
        reader.close();
        writer.close();
    } catch (IOException e) {
        LOG.info("Error in saving object.", e);
        return false;
    }
    return true;
}

---

From source file:com.ikanow.aleph2.search_service.elasticsearch.utils.TestJsonNodeWritableUtils.java

License:Apache License

@Test
public void test_transform() {
    final ObjectMapper mapper = BeanTemplateUtils.configureMapper(Optional.empty());
    new JsonNodeWritableUtils(); //coverage!

    assertEquals(NullNode.instance, JsonNodeWritableUtils.transform("banana", JsonNodeFactory.instance));
    assertEquals(null, JsonNodeWritableUtils.transform(null, JsonNodeFactory.instance));
    assertEquals(NullNode.instance,/*from w ww . ja  v a  2  s .  c  o  m*/
            JsonNodeWritableUtils.transform(NullWritable.get(), JsonNodeFactory.instance));
    assertEquals(mapper.convertValue(true, JsonNode.class),
            JsonNodeWritableUtils.transform(new BooleanWritable(true), JsonNodeFactory.instance));
    assertEquals(mapper.convertValue("test", JsonNode.class),
            JsonNodeWritableUtils.transform(new Text("test"), JsonNodeFactory.instance));
    assertEquals(mapper.convertValue(new byte[] { (byte) 0xFF }, JsonNode.class),
            JsonNodeWritableUtils.transform(new ByteWritable((byte) 0xFF), JsonNodeFactory.instance));
    assertEquals(mapper.convertValue(4, JsonNode.class),
            JsonNodeWritableUtils.transform(new IntWritable(4), JsonNodeFactory.instance));
    assertEquals(mapper.convertValue(4, JsonNode.class),
            JsonNodeWritableUtils.transform(new VIntWritable(4), JsonNodeFactory.instance));
    assertEquals(mapper.convertValue(4L, JsonNode.class),
            JsonNodeWritableUtils.transform(new LongWritable(4), JsonNodeFactory.instance));
    assertEquals(mapper.convertValue(4L, JsonNode.class),
            JsonNodeWritableUtils.transform(new VLongWritable(4), JsonNodeFactory.instance));
    assertEquals(mapper.convertValue(new byte[] { (byte) 0xFF, (byte) 0xFE }, JsonNode.class),
            JsonNodeWritableUtils.transform(new BytesWritable(new byte[] { (byte) 0xFF, (byte) 0xFE }),
                    JsonNodeFactory.instance));
    assertEquals(mapper.convertValue(4.0, JsonNode.class),
            JsonNodeWritableUtils.transform(new DoubleWritable(4), JsonNodeFactory.instance));
    //(had real trouble creating a float node!)
    assertEquals(JsonNodeFactory.instance.numberNode(Float.valueOf((float) 4.0)),
            JsonNodeWritableUtils.transform(new FloatWritable(4), JsonNodeFactory.instance));

    // will test object writable and array writable below      
}

---

From source file:com.j.distributed.sorter.SorterMapper.java

@Override
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
    String[] data = value.toString().split("\t");
    context.write(new LongWritable(Long.parseLong(data[1])), new Text(data[0]));
}

---

From source file:com.jeffy.hive.ID.java

License:Apache License

public LongWritable evaluate() {
    return new LongWritable(System.nanoTime());
}

---

From source file:com.jeffy.hive.SequenceUDF.java

License:Apache License

public LongWritable evaluate(int prefix) {
    return new LongWritable(Long.parseLong(prefix + "" + seq.getAndIncrement()));
}

---

From source file:com.justgiving.raven.kissmetrics.jsonenricher.KissmetricsJsonToEnrichedJsonMapperTest.java

License:Open Source License

@Test
public void mapper_validRecord_IncrementsIncludedCounter() throws IOException {

    mapDriver.withInput(new LongWritable(1), new Text(jsonRowbuilder.toString()));

    jsonRowbuilder.setValue("event_timedate", "2014-04-15 16:57:33").setValue("filename", "somefile")
            .setValue("event", jsonRowbuilder.getValue("_n")).setValue("user_email", "justgiving@gmail.com")
            .setValue("user_km_id", "3lwlxqlulqe24q/jl4aqlibrtte=").setValue("km_timestamp", "1397577453");
    String expectedJSON = jsonRowbuilder.toString();

    mapDriver.run();/*from w w w  .j ava  2s  .  c  om*/
    Assert.assertEquals(1, getCounter(KissmetricsConstants.TRACKING_COUNTER.VALID_JSON_ROW));
}

---

From source file:com.justgiving.raven.kissmetrics.jsonenricher.KissmetricsJsonToEnrichedJsonMapperTest.java

License:Open Source License

@Test
public void mapper_validRecord_IncrementsIncludedCounterTwice() throws IOException {
    mapDriver.withInput(new LongWritable(1), new Text(jsonRowbuilder.toString()));

    jsonRowbuilder.setValue("event_timedate", "2014-04-15 16:57:33").setValue("filename", "somefile")
            .setValue("event", jsonRowbuilder.getValue("_n")).setValue("user_email", "justgiving@gmail.com")
            .setValue("user_km_id", "3lwlxqlulqe24q/jl4aqlibrtte=").setValue("km_timestamp", "1397577453");
    String secondJSON = jsonRowbuilder.toString();

    mapDriver.withInput(new LongWritable(1), new Text(secondJSON));
    mapDriver.run();//from   w w  w.  j a  va  2  s .  co m
    Assert.assertEquals(2, getCounter(KissmetricsConstants.TRACKING_COUNTER.VALID_JSON_ROW));
}

---

From source file:com.justgiving.raven.kissmetrics.jsonenricher.KissmetricsJsonToEnrichedJsonMapperTest.java

License:Open Source License

@Test
public void mapper_invalidRecord_IncrementsInvalidJsonRowCounter() throws IOException {
    mapDriver.withInput(new LongWritable(1), new Text("{not valid json"));
    mapDriver.run();//from  ww w  .j a v  a2s.c  om
    Assert.assertEquals(1, getCounter(KissmetricsConstants.TRACKING_COUNTER.INVALID_JSON_ROW));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »