Example usage for org.apache.hadoop.io LongWritable LongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable LongWritable.

Prototype

public LongWritable(long value) 

Source Link

Usage

From source file:com.uber.hoodie.hadoop.realtime.AbstractRealtimeRecordReader.java

License:Apache License

/**
 * Convert the projected read from delta record into an array writable
 *//*w w  w  .j  a v  a  2 s.  c  o  m*/
public static Writable avroToArrayWritable(Object value, Schema schema) {

    if (value == null) {
        return null;
    }

    switch (schema.getType()) {
    case STRING:
        return new Text(value.toString());
    case BYTES:
        return new BytesWritable((byte[]) value);
    case INT:
        return new IntWritable((Integer) value);
    case LONG:
        return new LongWritable((Long) value);
    case FLOAT:
        return new FloatWritable((Float) value);
    case DOUBLE:
        return new DoubleWritable((Double) value);
    case BOOLEAN:
        return new BooleanWritable((Boolean) value);
    case NULL:
        return null;
    case RECORD:
        GenericRecord record = (GenericRecord) value;
        Writable[] recordValues = new Writable[schema.getFields().size()];
        int recordValueIndex = 0;
        for (Schema.Field field : schema.getFields()) {
            recordValues[recordValueIndex++] = avroToArrayWritable(record.get(field.name()), field.schema());
        }
        return new ArrayWritable(Writable.class, recordValues);
    case ENUM:
        return new Text(value.toString());
    case ARRAY:
        GenericArray arrayValue = (GenericArray) value;
        Writable[] arrayValues = new Writable[arrayValue.size()];
        int arrayValueIndex = 0;
        for (Object obj : arrayValue) {
            arrayValues[arrayValueIndex++] = avroToArrayWritable(obj, schema.getElementType());
        }
        // Hive 1.x will fail here, it requires values2 to be wrapped into another ArrayWritable
        return new ArrayWritable(Writable.class, arrayValues);
    case MAP:
        Map mapValue = (Map) value;
        Writable[] mapValues = new Writable[mapValue.size()];
        int mapValueIndex = 0;
        for (Object entry : mapValue.entrySet()) {
            Map.Entry mapEntry = (Map.Entry) entry;
            Writable[] nestedMapValues = new Writable[2];
            nestedMapValues[0] = new Text(mapEntry.getKey().toString());
            nestedMapValues[1] = avroToArrayWritable(mapEntry.getValue(), schema.getValueType());
            mapValues[mapValueIndex++] = new ArrayWritable(Writable.class, nestedMapValues);
        }
        // Hive 1.x will fail here, it requires values3 to be wrapped into another ArrayWritable
        return new ArrayWritable(Writable.class, mapValues);
    case UNION:
        List<Schema> types = schema.getTypes();
        if (types.size() != 2) {
            throw new IllegalArgumentException("Only support union with 2 fields");
        }
        Schema s1 = types.get(0);
        Schema s2 = types.get(1);
        if (s1.getType() == Schema.Type.NULL) {
            return avroToArrayWritable(value, s2);
        } else if (s2.getType() == Schema.Type.NULL) {
            return avroToArrayWritable(value, s1);
        } else {
            throw new IllegalArgumentException("Only support union with null");
        }
    case FIXED:
        return new BytesWritable(((GenericFixed) value).bytes());
    default:
        return null;
    }
}

---

From source file:com.xiaomi.linden.hadoop.indexing.LindenMapredTest.java

License:Apache License

@Test
public void TestMapper() throws IOException {
    try {//from  w  w  w .ja  va  2s  . co m
        String propertiesFilePath = LindenMapredTest.class.getClassLoader().getResource("linden.properties")
                .getFile();
        Files.copy(new File(propertiesFilePath).toPath(), Paths.get("lindenProperties"),
                StandardCopyOption.REPLACE_EXISTING);
        String schemaFilePath = LindenMapredTest.class.getClassLoader().getResource("schema.xml").getFile();
        Files.copy(new File(schemaFilePath).toPath(), Paths.get("lindenSchema"),
                StandardCopyOption.REPLACE_EXISTING);
        String json = "{\"id\":0,\"groupid\":\"0\",\"tags\":\"hybrid,leather,moon-roof,reliable\",\"category\":\"compact\",\"mileage\":14900,\"price\":7500,\"contents\":\"yellow compact hybrid leather moon-roof reliable u.s.a. florida tampa asian acura 1.6el \",\"color\":\"yellow\",\"year\":1994,\"makemodel\":\"asian/acura/1.6el\",\"city\":\"u.s.a./florida/tampa\"}";
        mDriver.withInput(new LongWritable(1L), new Text(json.getBytes()));
        mDriver.run();
    } catch (Exception e) {
        e.printStackTrace();
        Assert.assertTrue(false);
    } finally {
        FileUtils.deleteQuietly(Paths.get("lindenProperties").toFile());
        FileUtils.deleteQuietly(Paths.get("lindenSchema").toFile());
    }
}

---

From source file:com.yahoo.glimmer.indexing.preprocessor.TuplesToResourcesMapperTest.java

License:Open Source License

@Test
public void literalObjectText() throws IOException, InterruptedException {
    context.checking(new Expectations() {
        {/*  w ww .j  a  v  a 2 s  . c o  m*/
            allowing(mrContext).getInputSplit();
            will(returnValue(inputSplit));

            one(mrContext).write(with(new TextMatcher("http://www.example.org/terms/name")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://www.example.org/staffid/85740")),
                    with(new TextMatcher("<http://www.example.org/terms/name> \"Smith\" .")));
            one(mrContext).write(with(new TextMatcher("http://www.example.org/terms/name")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://www.example.org/staffid/85741")),
                    with(new TextMatcher("<http://www.example.org/terms/name> \"\n Johnson \t\"@en .")));
            one(mrContext).write(with(new TextMatcher("http://www.example.org/terms/name")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://www.example.org/staffid/85742")),
                    with(new TextMatcher("<http://www.example.org/terms/name> \"Ray\"@ .")));
        }
    });
    TuplesToResourcesMapper mapper = new TuplesToResourcesMapper();

    mapper.map(new LongWritable(5l),
            new Text("<http://www.example.org/staffid/85740> <http://www.example.org/terms/name> \"Smith\" ."),
            mrContext);
    mapper.map(new LongWritable(6l), new Text(
            "<http://www.example.org/staffid/85741> <http://www.example.org/terms/name> \"\n Johnson \t\"@en ."),
            mrContext);
    mapper.map(new LongWritable(7l),
            new Text("<http://www.example.org/staffid/85742> <http://www.example.org/terms/name> \"Ray\"@ ."),
            mrContext);
    context.assertIsSatisfied();
}

---

From source file:com.yahoo.glimmer.indexing.preprocessor.TuplesToResourcesMapperTest.java

License:Open Source License

@Test
public void resourceObjectTest() throws IOException, InterruptedException {
    context.checking(new Expectations() {
        {//from w  w w  . ja  va2s.c om
            allowing(mrContext).getInputSplit();
            will(returnValue(inputSplit));

            one(mrContext).write(with(new TextMatcher("http://purl.org/dc/elements/1.1/creator")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://www.example.org/staffid/85740")),
                    with(new TextMatcher("OBJECT")));
            one(mrContext).write(with(new TextMatcher("http://context/")), with(new TextMatcher("CONTEXT")));
            one(mrContext).write(with(new TextMatcher("http://www.example.org/index.html")),
                    with(new TextMatcher(
                            "<http://purl.org/dc/elements/1.1/creator> <http://www.example.org/staffid/85740> <http://context/> .")));
        }
    });
    TuplesToResourcesMapper mapper = new TuplesToResourcesMapper();

    mapper.map(new LongWritable(5l), new Text(
            "<http://www.example.org/index.html> <http://purl.org/dc/elements/1.1/creator> <http://www.example.org/staffid/85740> <http://context/> ."),
            mrContext);
    context.assertIsSatisfied();
}

---

From source file:com.yahoo.glimmer.indexing.preprocessor.TuplesToResourcesMapperTest.java

License:Open Source License

@Test
public void noContextsObjectTest() throws IOException, InterruptedException {
    context.checking(new Expectations() {
        {//ww w. j av  a2 s .c  o  m
            allowing(mrContext).getInputSplit();
            will(returnValue(inputSplit));

            one(mrContext).write(with(new TextMatcher("http://purl.org/dc/elements/1.1/creator")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://www.example.org/staffid/85740")),
                    with(new TextMatcher("OBJECT")));
            one(mrContext).write(with(new TextMatcher("http://www.example.org/index.html")),
                    with(new TextMatcher(
                            "<http://purl.org/dc/elements/1.1/creator> <http://www.example.org/staffid/85740> .")));
        }
    });
    TuplesToResourcesMapper mapper = new TuplesToResourcesMapper();
    mapper.setIncludeContexts(false);

    mapper.map(new LongWritable(5l), new Text(
            "<http://www.example.org/index.html> <http://purl.org/dc/elements/1.1/creator> <http://www.example.org/staffid/85740> <http://context/> ."),
            mrContext);
    context.assertIsSatisfied();
}

---

From source file:com.yahoo.glimmer.indexing.preprocessor.TuplesToResourcesMapperTest.java

License:Open Source License

@Ignore
@Test/*from  ww  w. j a  v a  2s.  com*/
public void qualifiedIntNxp122Test() throws IOException, InterruptedException {
    context.checking(new Expectations() {
        {
            allowing(mrContext).getInputSplit();
            will(returnValue(inputSplit));

            one(mrContext).getCounter(Counters.NX_PARSER_EXCEPTION);
            will(returnValue(nxParserExceptionCounter));
            one(nxParserExceptionCounter).increment(1l);
            one(mrContext).write(with(new TextMatcher("http://www.example.org/terms/age")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://www.example.org/staffid/85740")),
                    with(new TextMatcher("<http://www.example.org/terms/age> \"27\" .")));
        }
    });
    TuplesToResourcesMapper mapper = new TuplesToResourcesMapper();
    mapper.map(new LongWritable(5l), new Text(
            "<http://www.example.org/staffid/85740>  <http://www.example.org/terms/age> \"27\"^^<http://www.w3.org/2001/XMLSchema#integer> ."),
            mrContext);
    context.assertIsSatisfied();
}

---

From source file:com.yahoo.glimmer.indexing.preprocessor.TuplesToResourcesMapperTest.java

License:Open Source License

@Test
public void qualifiedIntNxp123Test() throws IOException, InterruptedException {
    context.checking(new Expectations() {
        {/*from  w ww  .j ava  2s. c  o  m*/
            allowing(mrContext).getInputSplit();
            will(returnValue(inputSplit));

            one(mrContext).write(with(new TextMatcher("http://www.example.org/terms/age")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://www.example.org/staffid/85740")),
                    with(new TextMatcher(
                            "<http://www.example.org/terms/age> \"27\"^^<http://www.w3.org/2001/XMLSchema#integer> .")));
        }
    });
    TuplesToResourcesMapper mapper = new TuplesToResourcesMapper();
    mapper.map(new LongWritable(5l), new Text(
            "<http://www.example.org/staffid/85740>  <http://www.example.org/terms/age> \"27\"^^<http://www.w3.org/2001/XMLSchema#integer> ."),
            mrContext);
    context.assertIsSatisfied();
}

---

From source file:com.yahoo.glimmer.indexing.preprocessor.TuplesToResourcesMapperTest.java

License:Open Source License

@Test
public void bNodeTest() throws IOException, InterruptedException {
    context.checking(new Expectations() {
        {/*from   w w w .j  a  va2 s .  co m*/
            allowing(mrContext).getInputSplit();
            will(returnValue(inputSplit));

            one(mrContext).write(with(new TextMatcher("http://www.example.org/terms/place")),
                    with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("NodeABC")), with(new TextMatcher("OBJECT")));
            one(mrContext).write(with(new TextMatcher("nodeXYZ")),
                    with(new TextMatcher("<http://www.example.org/terms/place> _:NodeABC .")));
        }
    });
    TuplesToResourcesMapper mapper = new TuplesToResourcesMapper();
    mapper.map(new LongWritable(5l), new Text("_:nodeXYZ  <http://www.example.org/terms/place> _:NodeABC ."),
            mrContext);
    context.assertIsSatisfied();
}

---

From source file:com.yahoo.glimmer.indexing.preprocessor.TuplesToResourcesMapperTest.java

License:Open Source License

@Test
public void filterSubjectOrObjectTest() throws IOException, InterruptedException {
    context.checking(new Expectations() {
        {/*  ww w. j  a  v  a  2s .c  o  m*/
            allowing(mrContext).getInputSplit();
            will(returnValue(inputSplit));

            one(mrContext).write(with(new TextMatcher("http://p2")), with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://s3")), with(new TextMatcher("OBJECT")));
            one(mrContext).write(with(new TextMatcher("http://context/")), with(new TextMatcher("CONTEXT")));
            one(mrContext).write(with(new TextMatcher("http://s1")),
                    with(new TextMatcher("<http://p2> <http://s3> <http://context/> .")));

            one(mrContext).write(with(new TextMatcher("http://p3")), with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://s3")), with(new TextMatcher("OBJECT")));
            one(mrContext).write(with(new TextMatcher("http://context/")), with(new TextMatcher("CONTEXT")));
            one(mrContext).write(with(new TextMatcher("http://s2")),
                    with(new TextMatcher("<http://p3> <http://s3> <http://context/> .")));

            one(mrContext).write(with(new TextMatcher("http://p5")), with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://context/")), with(new TextMatcher("CONTEXT")));
            one(mrContext).write(with(new TextMatcher("http://s3")),
                    with(new TextMatcher("<http://p5> \"o5\" <http://context/> .")));
        }
    });
    TuplesToResourcesMapper mapper = new TuplesToResourcesMapper();

    RegexTupleFilter regexTupleFilter = new RegexTupleFilter();
    regexTupleFilter.setSubjectRegex("^<http://s3");
    regexTupleFilter.setObjectRegex("s3>$");
    // Use OR
    regexTupleFilter.setAndNotOrConjunction(false);

    // We should get the 2nd, 3rd and 5th tuple only.
    mapper.setFilter(regexTupleFilter);

    mapper.map(new LongWritable(15l), new Text("<http://s1> <http://p1> <http://s2> <http://context/> ."),
            mrContext);
    mapper.map(new LongWritable(16l), new Text("<http://s1> <http://p2> <http://s3> <http://context/> ."),
            mrContext);
    mapper.map(new LongWritable(17l), new Text("<http://s2> <http://p3> <http://s3> <http://context/> ."),
            mrContext);
    mapper.map(new LongWritable(18l), new Text("<http://s2> <http://p4> \"o4\" <http://context/> ."),
            mrContext);
    mapper.map(new LongWritable(19l), new Text("<http://s3> <http://p5> \"o5\" <http://context/> ."),
            mrContext);

    context.assertIsSatisfied();
}

---

From source file:com.yahoo.glimmer.indexing.preprocessor.TuplesToResourcesMapperTest.java

License:Open Source License

@Test
public void filterSubjectAndObjectTest() throws IOException, InterruptedException {
    context.checking(new Expectations() {
        {/*  w  w  w  .j  av a2s . c o m*/
            allowing(mrContext).getInputSplit();
            will(returnValue(inputSplit));

            one(mrContext).write(with(new TextMatcher("http://p2")), with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://s3")), with(new TextMatcher("OBJECT")));
            one(mrContext).write(with(new TextMatcher("http://context/")), with(new TextMatcher("CONTEXT")));
            one(mrContext).write(with(new TextMatcher("http://s1")),
                    with(new TextMatcher("<http://p2> <http://s3> <http://context/> .")));

            one(mrContext).write(with(new TextMatcher("http://p5")), with(new TextMatcher("PREDICATE")));
            one(mrContext).write(with(new TextMatcher("http://context/")), with(new TextMatcher("CONTEXT")));
            one(mrContext).write(with(new TextMatcher("http://s3")),
                    with(new TextMatcher("<http://p5> \"o5\" <http://context/> .")));
        }
    });
    TuplesToResourcesMapper mapper = new TuplesToResourcesMapper();

    RegexTupleFilter filter = new RegexTupleFilter();
    // We should get the 2nd and 5th tuple only.
    filter.setSubjectRegex("s1|s3");
    filter.setObjectRegex("(s3|o5)");
    // Use AND
    filter.setAndNotOrConjunction(true);

    mapper.setFilter(filter);

    mapper.map(new LongWritable(25l), new Text("<http://s1> <http://p1> <http://s2> <http://context/> ."),
            mrContext);
    mapper.map(new LongWritable(26l), new Text("<http://s1> <http://p2> <http://s3> <http://context/> ."),
            mrContext);
    mapper.map(new LongWritable(27l), new Text("<http://s2> <http://p3> <http://s3> <http://context/> ."),
            mrContext);
    mapper.map(new LongWritable(28l), new Text("<http://s2> <http://p4> \"o4\" <http://context/> ."),
            mrContext);
    mapper.map(new LongWritable(29l), new Text("<http://s3> <http://p5> \"o5\" <http://context/> ."),
            mrContext);

    context.assertIsSatisfied();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »