Example usage for org.apache.hadoop.io LongWritable LongWritable

List of usage examples for org.apache.hadoop.io LongWritable LongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable LongWritable.

Prototype

public LongWritable(long value) 

Source Link

Usage

From source file:com.tfm.utad.reducerdata.ReducerDataPigMapper.java

@Override
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
    String[] values = value.toString().replaceAll("\n", "").split(ReducerConstants.SPLIT_SEMICOLON);
    Double latitude = Double.valueOf(values[0]);
    Double longitude = Double.valueOf(values[1]);
    String userid = values[2];/* www  . j a  va 2  s .c  om*/
    Date date;
    try {
        date = sdf.parse(values[3]);
    } catch (ParseException ex) {
        LOG.error("Error parsing date: " + ex.toString());
        date = null;
    }
    String activity = values[4];
    if (isValidKey(latitude, longitude, date)) {
        Text text = (Text) key;
        Long id = Long.valueOf(text.toString());
        context.write(new LongWritable(id), new ReducerPigKey(id, latitude, longitude, userid, date, activity));
    } else {
        LOG.error("Invalid values in line: " + value.toString());
        context.getCounter(ReducerDataEnum.MALFORMED_DATA).increment(1);
    }
}

From source file:com.tfm.utad.reducerdata.ReducerDataPigMapperReducerTest.java

@Test
public void testMapper() throws IOException, ParseException {
    String key = "123456";
    mapDriver.withInput(new Text(key),
            new Text("40.48989;-3.65754;User189;2014-12-06 17:43:21;20141206-34567-189"));
    Date date = sdf.parse("2014-12-06 17:43:21");
    mapDriver.withOutput(new LongWritable(new Long(key)), new ReducerPigKey(new Long(key),
            Double.valueOf("40.48989"), Double.valueOf("-3.65754"), "User189", date, "20141206-34567-189"));
    mapDriver.runTest();/*from w  ww.  j  a v a  2  s.c  o m*/
}

From source file:com.tfm.utad.reducerdata.ReducerDataPigMapperReducerTest.java

@Test
public void testReducer() throws IOException, ParseException {
    List<ReducerPigKey> values = new ArrayList<>();
    Date date = sdf.parse("2014-12-06 17:43:21");
    ReducerPigKey pigKey = new ReducerPigKey(new Long("123456"), Double.valueOf("40.48989"),
            Double.valueOf("-3.65754"), "User189", date, "20141206-34567-189");
    values.add(pigKey);/*w w  w  .ja v a 2s . c o  m*/
    reduceDriver.withInput(new LongWritable(new Long("123456")), values);
    reduceDriver.withOutput(new Text(pigKey.toString()), NullWritable.get());
    reduceDriver.runTest();
}

From source file:com.tfm.utad.reducerdata.ReducerDataVerticaMapper.java

@Override
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
    String[] values = value.toString().replaceAll("\n", "").split(ReducerConstants.SPLIT_SEMICOLON);
    Double latitude = Double.valueOf(values[0]);
    Double longitude = Double.valueOf(values[1]);
    String user = values[2];/*  ww w . j a v a  2s.c  o m*/
    Date date;
    try {
        date = sdf.parse(values[3]);
    } catch (ParseException ex) {
        LOG.error("Error parsing date: " + ex.toString());
        date = null;
    }
    String activity = values[4];
    if (isValidKey(latitude, longitude, user)) {
        Text text = (Text) key;
        Long id = Long.valueOf(text.toString());
        Long userid = Long.valueOf(user.substring(4));
        context.write(new Text(user + activity),
                new ReducerVerticaValue(new LongWritable(id), new Text(user), date, new Text(activity),
                        new DoubleWritable(latitude), new DoubleWritable(longitude), new LongWritable(userid)));
    } else {
        LOG.error("Invalid values in line: " + value.toString());
        context.getCounter(ReducerDataEnum.MALFORMED_DATA).increment(1);
    }
}

From source file:com.tfm.utad.reducerdata.ReducerDataVerticaMapperReducerTest.java

@Test
public void testMapper() throws IOException, ParseException {
    String key = "123456";
    Date date = sdf.parse("2014-12-06 17:43:21");
    mapDriver.withInput(new Text(key),
            new Text("40.48989;-3.65754;User189;2014-12-06 17:43:21;20141206-34567-189"));
    mapDriver.withOutput(new Text("User189" + "20141206-34567-189"),
            new ReducerVerticaValue(new LongWritable((long) 123456), new Text("User189"), date,
                    new Text("20141206-34567-189"), new DoubleWritable(Double.valueOf("40.48989")),
                    new DoubleWritable(Double.valueOf("-3.65754")), new LongWritable(new Long("189"))));
    mapDriver.runTest();/*  www  . ja  va  2s  .co m*/
}

From source file:com.tfm.utad.reducerdata.ReducerDataVerticaMapperReducerTest.java

@Test
public void testReducer() throws IOException, ParseException {
    List<ReducerVerticaValue> values = new ArrayList<>();
    Date date = sdf.parse("2014-12-06 17:43:21");
    ReducerVerticaValue verticaValue = new ReducerVerticaValue(new LongWritable((long) 123456),
            new Text("User189"), date, new Text("20141206-34567-189"),
            new DoubleWritable(Double.valueOf("40.48989")), new DoubleWritable(Double.valueOf("-3.65754")),
            new LongWritable(new Long("189")));
    values.add(verticaValue);//from   w ww .j  a  v a 2s.  c o  m
    reduceDriver.withInput(new Text("User189" + "20141206-34567-189"), values);
    reduceDriver.withOutput(new Text(values.get(0).toString()), NullWritable.get());
    reduceDriver.runTest();
}

From source file:com.twitter.elephanttwin.io.ListLongWritable.java

License:Apache License

public void add(LongWritable value) {
    LongWritable a = new LongWritable(value.get());
    list.add(a);
}

From source file:com.twitter.elephanttwin.lucene.indexing.HadoopSplitIndexingMapper.java

License:Apache License

private void emitDocument(Context context) throws IOException, InterruptedException {
    // TODO: I'm still not getting the diff between the offset and split start. Shouldn't we be increasing the offset in map?
    doc.setOffset(split.getStart());/*w w w.j a  v  a2 s . co  m*/
    doc.setCnt(cnt);
    context.write(new LongWritable(split.getStart()), doc);
}

From source file:com.twitter.hraven.FlowKey.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    super.write(out);
    new LongWritable(this.runId).write(out);
}

From source file:com.twitter.hraven.JobId.java

License:Apache License

@Override
public void write(DataOutput out) throws IOException {
    new LongWritable(this.jobEpoch).write(out);
    new LongWritable(this.jobSequence).write(out);
}