Example usage for org.apache.hadoop.io NullWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io NullWritable get.

Prototype

public static NullWritable get() 

Source Link

Document

Returns the single instance of this class.

Usage

From source file:org.apache.phoenix.pig.hadoop.PhoenixRecordReader.java

License:Apache License

@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
    if (key == null) {
        key = NullWritable.get();
    }//from w  w  w  .j a v  a2 s. c  om
    if (value == null) {
        value = new PhoenixRecord();
    }
    Preconditions.checkNotNull(this.resultSet);
    try {
        if (!resultSet.next()) {
            return false;
        }
        value.read(resultSet, columnInfos.size());
        return true;
    } catch (SQLException e) {
        LOG.error(String.format(" Error [%s] occurred while iterating over the resultset. ", e.getMessage()));
        Throwables.propagate(e);
    }
    return false;
}

---

From source file:org.apache.pig.builtin.TrevniStorage.java

License:Apache License

@Override
public InputFormat<NullWritable, GenericData.Record> getInputFormat() throws IOException {

    class TrevniStorageInputFormat extends PigFileInputFormat<NullWritable, GenericData.Record> {

        @Override//w w w  .j a  va 2s .  com
        protected boolean isSplitable(JobContext jc, Path p) {
            return false;
        }

        @Override
        protected List<FileStatus> listStatus(final JobContext job) throws IOException {
            List<FileStatus> results = Lists.newArrayList();
            job.getConfiguration().setBoolean(MRConfiguration.INPUT_DIR_RECURSIVE, true);
            for (FileStatus file : super.listStatus(job)) {
                if (Utils.VISIBLE_FILES.accept(file.getPath())) {
                    results.add(file);
                }
            }
            return results;
        }

        @Override
        public RecordReader<NullWritable, GenericData.Record> createRecordReader(final InputSplit is,
                final TaskAttemptContext tc) throws IOException, InterruptedException {
            RecordReader<NullWritable, GenericData.Record> rr = new RecordReader<NullWritable, GenericData.Record>() {

                private FileSplit fsplit;
                private AvroColumnReader.Params params;
                private AvroColumnReader<GenericData.Record> reader;
                private float rows;
                private long row = 0;
                private GenericData.Record currentRecord = null;

                @Override
                public void close() throws IOException {
                    reader.close();
                }

                @Override
                public NullWritable getCurrentKey() throws IOException, InterruptedException {
                    return NullWritable.get();
                }

                @Override
                public Record getCurrentValue() throws IOException, InterruptedException {
                    return currentRecord;
                }

                @Override
                public float getProgress() throws IOException, InterruptedException {
                    return row / rows;
                }

                @Override
                public void initialize(final InputSplit isplit, final TaskAttemptContext tac)
                        throws IOException, InterruptedException {
                    fsplit = (FileSplit) isplit;
                    params = new AvroColumnReader.Params(
                            new HadoopInput(fsplit.getPath(), tac.getConfiguration()));
                    Schema inputSchema = getInputAvroSchema();
                    params.setSchema(inputSchema);
                    reader = new AvroColumnReader<GenericData.Record>(params);
                    rows = reader.getRowCount();
                }

                @Override
                public boolean nextKeyValue() throws IOException, InterruptedException {
                    if (reader.hasNext()) {
                        currentRecord = reader.next();
                        row++;
                        return true;
                    } else {
                        return false;
                    }
                }
            };

            // rr.initialize(is, tc);
            tc.setStatus(is.toString());
            return rr;
        }

    }

    return new TrevniStorageInputFormat();

}

---

From source file:org.apache.rya.reasoning.mr.DuplicateEliminationTest.java

License:Apache License

@Test
public void testFileMapperOutput() throws Exception {
    new MapDriver<Fact, NullWritable, Fact, Derivation>()
            .withMapper(new DuplicateElimination.DuplicateFileMapper()).withInput(X_SUB_Y, NullWritable.get())
            .withOutput(X_SUB_Y, X_SUB_Y.getDerivation()).runTest();
}

---

From source file:org.apache.rya.reasoning.mr.DuplicateEliminationTest.java

License:Apache License

@Test
public void testInconsistencyMapperOutput() throws Exception {
    Fact empty = new Fact();
    empty.setDerivation(X_DISJOINT);/* w  w w.j  a  v  a2s  .  co m*/
    new MapDriver<Derivation, NullWritable, Fact, Derivation>()
            .withMapper(new DuplicateElimination.InconsistencyMapper())
            .withInput(X_DISJOINT, NullWritable.get()).withOutput(empty, X_DISJOINT).runTest();
}

---

From source file:org.apache.rya.reasoning.mr.DuplicateEliminationTest.java

License:Apache License

@Test
public void testRetainSimplest() throws Exception {
    List<Derivation> facts = new LinkedList<>();
    facts.add(Y_SUPER_X_INV.getDerivation());
    facts.add(Y_SUPER_X.getDerivation());
    Fact unset = Y_SUPER_X.clone();//  w w  w.  j  a v a  2  s.com
    unset.unsetDerivation();
    ReduceDriver<Fact, Derivation, Fact, NullWritable> driver = new ReduceDriver<>();
    driver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, 1);
    driver.withReducer(new DuplicateElimination.DuplicateEliminationReducer()).withInput(unset, facts)
            .withMultiOutput(MRReasoningUtils.INTERMEDIATE_OUT, Y_SUPER_X, NullWritable.get()).runTest();
}

---

From source file:org.apache.rya.reasoning.mr.DuplicateEliminationTest.java

License:Apache License

@Test
public void testInconsistencyReduce() throws Exception {
    List<Derivation> facts = new LinkedList<>();
    facts.add(X_DISJOINT.clone());/*from  ww  w.  j  a  v  a2  s .c om*/
    facts.add(X_DISJOINT.clone());
    ReduceDriver<Fact, Derivation, Fact, NullWritable> driver = new ReduceDriver<>();
    driver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, 1);
    driver.withReducer(new DuplicateElimination.DuplicateEliminationReducer()).withInput(Fact.NONE, facts)
            .withMultiOutput(MRReasoningUtils.INCONSISTENT_OUT, X_DISJOINT, NullWritable.get()).runTest();
}

---

From source file:org.apache.rya.reasoning.mr.ForwardChainTest.java

License:Apache License

@Test
public void testFileMapperOutput() throws Exception {
    ResourceWritable rw1 = new ResourceWritable();
    ResourceWritable rw2 = new ResourceWritable();
    rw1.set(TestUtils.uri("x"));
    rw2.set(TestUtils.uri("y"));
    new MapDriver<Fact, NullWritable, ResourceWritable, Fact>().withMapper(new ForwardChain.FileMapper(schema))
            .withInput(X_SUB_Y, NullWritable.get()).withOutput(rw1, X_SUB_Y).withOutput(rw2, X_SUB_Y).runTest();
}

---

From source file:org.apache.rya.reasoning.mr.ForwardChainTest.java

License:Apache License

@Test
public void testReducerInference() throws Exception {
    schema.processTriple(INV.getTriple());
    schema.closure();/*w  w  w. j a v a2 s.c  om*/
    ResourceWritable rw = new ResourceWritable();
    rw.set(TestUtils.uri("y"));
    List<Fact> facts = new LinkedList<>();
    facts.add(X_SUB_Y);
    new ReduceDriver<ResourceWritable, Fact, Fact, NullWritable>()
            .withReducer(new ForwardChain.ReasoningReducer(schema)).withInput(rw, facts)
            .withMultiOutput(MRReasoningUtils.INTERMEDIATE_OUT, Y_SUPER_X, NullWritable.get()).runTest();
}

---

From source file:org.apache.rya.reasoning.mr.ForwardChainTest.java

License:Apache License

@Test
public void testReducerJoin() throws Exception {
    ResourceWritable rw = new ResourceWritable();
    rw.set(TestUtils.uri("y"));
    List<Fact> facts = new LinkedList<>();
    facts.add(X_SUB_Y);/*from w  w w.jav  a 2s.  c  om*/
    facts.add(Y_SUB_Z);
    ReduceDriver<ResourceWritable, Fact, Fact, NullWritable> driver = new ReduceDriver<>();
    driver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, 1);
    driver.withReducer(new ForwardChain.ReasoningReducer(schema)).withInput(rw, facts)
            .withMultiOutput(MRReasoningUtils.INTERMEDIATE_OUT, X_SUB_Z, NullWritable.get()).runTest();
}

---

From source file:org.apache.rya.reasoning.mr.ForwardChainTest.java

License:Apache License

/**
 * MultipleOutputs support is minimal, so we have to check each map/reduce
 * step explicitly// ww  w . ja v a  2s. c om
 */
@Test
public void testTransitiveChain() throws Exception {
    int max = 8;
    int n = 4;
    URI prop = TestUtils.uri("subOrganizationOf");
    Map<Integer, Map<Integer, Pair<Fact, NullWritable>>> connections = new HashMap<>();
    for (int i = 0; i <= max; i++) {
        connections.put(i, new HashMap<Integer, Pair<Fact, NullWritable>>());
    }
    // Initial input: make a chain from org0 to org8
    for (int i = 0; i < max; i++) {
        URI orgI = TestUtils.uri("org" + i);
        URI orgJ = TestUtils.uri("org" + (i + 1));
        Fact triple = new Fact(orgI, prop, orgJ);
        connections.get(i).put(i + 1, new Pair<>(triple, NullWritable.get()));
    }
    for (int i = 1; i <= n; i++) {
        // Map:
        MapDriver<Fact, NullWritable, ResourceWritable, Fact> mDriver = new MapDriver<>();
        mDriver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, i);
        mDriver.setMapper(new ForwardChain.FileMapper(schema));
        for (int j : connections.keySet()) {
            for (int k : connections.get(j).keySet()) {
                mDriver.addInput(connections.get(j).get(k));
            }
        }
        List<Pair<ResourceWritable, Fact>> mapped = mDriver.run();
        // Convert data for reduce phase:
        ReduceFeeder<ResourceWritable, Fact> feeder = new ReduceFeeder<>(mDriver.getConfiguration());
        List<KeyValueReuseList<ResourceWritable, Fact>> intermediate = feeder.sortAndGroup(mapped,
                new ResourceWritable.SecondaryComparator(), new ResourceWritable.PrimaryComparator());
        // Reduce, and compare to expected output:
        ReduceDriver<ResourceWritable, Fact, Fact, NullWritable> rDriver = new ReduceDriver<>();
        rDriver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, i);
        rDriver.setReducer(new ForwardChain.ReasoningReducer(schema));
        rDriver.addAllElements(intermediate);
        int maxSpan = (int) Math.pow(2, i);
        int minSpan = (maxSpan / 2) + 1;
        // For each j, build all paths starting with j:
        for (int j = 0; j < max; j++) {
            // This includes any path of length k for appropriate k:
            for (int k = minSpan; k <= maxSpan && j + k <= max; k++) {
                int middle = j + minSpan - 1;
                URI left = TestUtils.uri("org" + j);
                URI right = TestUtils.uri("org" + (j + k));
                Fact triple = new Fact(left, prop, right, i, OwlRule.PRP_TRP, TestUtils.uri("org" + middle));
                triple.addSource(connections.get(j).get(middle).getFirst());
                triple.addSource(connections.get(middle).get(j + k).getFirst());
                Pair<Fact, NullWritable> expected = new Pair<>(triple, NullWritable.get());
                connections.get(j).put(j + k, expected);
                rDriver.addMultiOutput("intermediate", expected);
            }
        }
        rDriver.runTest();
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »