Example usage for org.apache.hadoop.io NullWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io NullWritable get.

Prototype

public static NullWritable get() 

Source Link

Document

Returns the single instance of this class.

Usage

From source file:org.apache.mahout.cf.taste.hadoop.similarity.item.CountUsersReducer.java

License:Apache License

@Override
protected void reduce(CountUsersKeyWritable key, Iterable<VarLongWritable> userIDs, Context context)
        throws IOException, InterruptedException {

    long lastSeenUserID = Long.MIN_VALUE;
    int numberOfUsers = 0;

    for (VarLongWritable writable : userIDs) {
        long currentUserID = writable.get();
        if (currentUserID > lastSeenUserID) {
            lastSeenUserID = currentUserID;
            numberOfUsers++;/*from w  w w  .java 2 s. co m*/
        }
    }
    context.write(new VarIntWritable(numberOfUsers), NullWritable.get());
}

---

From source file:org.apache.mahout.cf.taste.hadoop.similarity.item.ItemSimilarityTest.java

License:Apache License

/**
 * tests {@link CountUsersReducer}/*  w w w  .  java2s. com*/
 */
public void testCountUsersReducer() throws Exception {
    Reducer<CountUsersKeyWritable, VarLongWritable, VarIntWritable, NullWritable>.Context context = EasyMock
            .createMock(Reducer.Context.class);
    context.write(new VarIntWritable(3), NullWritable.get());
    EasyMock.replay(context);

    List<VarLongWritable> userIDs = Arrays.asList(new VarLongWritable(1L), new VarLongWritable(1L),
            new VarLongWritable(3L), new VarLongWritable(5L), new VarLongWritable(5L), new VarLongWritable(5L));

    new CountUsersReducer().reduce(null, userIDs, context);
    EasyMock.verify(context);
}

---

From source file:org.apache.mahout.clustering.spectral.common.TestMatrixDiagonalizeJob.java

License:Apache License

@Test
public void testMatrixDiagonalizeMapper() throws Exception {
    MatrixDiagonalizeMapper mapper = new MatrixDiagonalizeMapper();
    Configuration conf = new Configuration();
    conf.setInt(EigencutsKeys.AFFINITY_DIMENSIONS, RAW_DIMENSIONS);

    // set up the dummy writers
    DummyRecordWriter<NullWritable, IntDoublePairWritable> writer = new DummyRecordWriter<NullWritable, IntDoublePairWritable>();
    Mapper<IntWritable, VectorWritable, NullWritable, IntDoublePairWritable>.Context context = DummyRecordWriter
            .build(mapper, conf, writer);

    // perform the mapping
    for (int i = 0; i < RAW_DIMENSIONS; i++) {
        RandomAccessSparseVector toAdd = new RandomAccessSparseVector(RAW_DIMENSIONS);
        toAdd.assign(RAW[i]);//from   w  w  w  . ja v  a 2s  .  c  o  m
        mapper.map(new IntWritable(i), new VectorWritable(toAdd), context);
    }

    // check the number of the results
    assertEquals("Number of map results", RAW_DIMENSIONS, writer.getValue(NullWritable.get()).size());
}

---

From source file:org.apache.mahout.clustering.spectral.common.TestMatrixDiagonalizeJob.java

License:Apache License

@Test
public void testMatrixDiagonalizeReducer() throws Exception {
    MatrixDiagonalizeMapper mapper = new MatrixDiagonalizeMapper();
    Configuration conf = new Configuration();
    conf.setInt(EigencutsKeys.AFFINITY_DIMENSIONS, RAW_DIMENSIONS);

    // set up the dummy writers
    DummyRecordWriter<NullWritable, IntDoublePairWritable> mapWriter = new DummyRecordWriter<NullWritable, IntDoublePairWritable>();
    Mapper<IntWritable, VectorWritable, NullWritable, IntDoublePairWritable>.Context mapContext = DummyRecordWriter
            .build(mapper, conf, mapWriter);

    // perform the mapping
    for (int i = 0; i < RAW_DIMENSIONS; i++) {
        RandomAccessSparseVector toAdd = new RandomAccessSparseVector(RAW_DIMENSIONS);
        toAdd.assign(RAW[i]);// w w w  .j  a  v  a 2  s.  com
        mapper.map(new IntWritable(i), new VectorWritable(toAdd), mapContext);
    }

    // now perform the reduction
    MatrixDiagonalizeReducer reducer = new MatrixDiagonalizeReducer();
    DummyRecordWriter<NullWritable, VectorWritable> redWriter = new DummyRecordWriter<NullWritable, VectorWritable>();
    Reducer<NullWritable, IntDoublePairWritable, NullWritable, VectorWritable>.Context redContext = DummyRecordWriter
            .build(reducer, conf, redWriter, NullWritable.class, IntDoublePairWritable.class);

    // only need one reduction
    reducer.reduce(NullWritable.get(), mapWriter.getValue(NullWritable.get()), redContext);

    // first, make sure there's only one result
    List<VectorWritable> list = redWriter.getValue(NullWritable.get());
    assertEquals("Only a single resulting vector", 1, list.size());
    Vector v = list.get(0).get();
    for (int i = 0; i < v.size(); i++) {
        assertEquals("Element sum is correct", rowSum(RAW[i]), v.get(i), 0.01);
    }
}

---

From source file:org.apache.mahout.clustering.spectral.TestMatrixDiagonalizeJob.java

License:Apache License

@Test
public void testMatrixDiagonalizeMapper() throws Exception {
    MatrixDiagonalizeMapper mapper = new MatrixDiagonalizeMapper();
    Configuration conf = getConfiguration();
    conf.setInt(Keys.AFFINITY_DIMENSIONS, RAW_DIMENSIONS);

    // set up the dummy writers
    DummyRecordWriter<NullWritable, IntDoublePairWritable> writer = new DummyRecordWriter<NullWritable, IntDoublePairWritable>();
    Mapper<IntWritable, VectorWritable, NullWritable, IntDoublePairWritable>.Context context = DummyRecordWriter
            .build(mapper, conf, writer);

    // perform the mapping
    for (int i = 0; i < RAW_DIMENSIONS; i++) {
        RandomAccessSparseVector toAdd = new RandomAccessSparseVector(RAW_DIMENSIONS);
        toAdd.assign(RAW[i]);/*from   w w w .  j a v a2 s  . c  o m*/
        mapper.map(new IntWritable(i), new VectorWritable(toAdd), context);
    }

    // check the number of the results
    assertEquals("Number of map results", RAW_DIMENSIONS, writer.getValue(NullWritable.get()).size());
}

---

From source file:org.apache.mahout.clustering.spectral.TestMatrixDiagonalizeJob.java

License:Apache License

@Test
public void testMatrixDiagonalizeReducer() throws Exception {
    MatrixDiagonalizeMapper mapper = new MatrixDiagonalizeMapper();
    Configuration conf = getConfiguration();
    conf.setInt(Keys.AFFINITY_DIMENSIONS, RAW_DIMENSIONS);

    // set up the dummy writers
    DummyRecordWriter<NullWritable, IntDoublePairWritable> mapWriter = new DummyRecordWriter<NullWritable, IntDoublePairWritable>();
    Mapper<IntWritable, VectorWritable, NullWritable, IntDoublePairWritable>.Context mapContext = DummyRecordWriter
            .build(mapper, conf, mapWriter);

    // perform the mapping
    for (int i = 0; i < RAW_DIMENSIONS; i++) {
        RandomAccessSparseVector toAdd = new RandomAccessSparseVector(RAW_DIMENSIONS);
        toAdd.assign(RAW[i]);//from   w  ww . j av  a2 s.c  om
        mapper.map(new IntWritable(i), new VectorWritable(toAdd), mapContext);
    }

    // now perform the reduction
    MatrixDiagonalizeReducer reducer = new MatrixDiagonalizeReducer();
    DummyRecordWriter<NullWritable, VectorWritable> redWriter = new DummyRecordWriter<NullWritable, VectorWritable>();
    Reducer<NullWritable, IntDoublePairWritable, NullWritable, VectorWritable>.Context redContext = DummyRecordWriter
            .build(reducer, conf, redWriter, NullWritable.class, IntDoublePairWritable.class);

    // only need one reduction
    reducer.reduce(NullWritable.get(), mapWriter.getValue(NullWritable.get()), redContext);

    // first, make sure there's only one result
    List<VectorWritable> list = redWriter.getValue(NullWritable.get());
    assertEquals("Only a single resulting vector", 1, list.size());
    Vector v = list.get(0).get();
    for (int i = 0; i < v.size(); i++) {
        assertEquals("Element sum is correct", rowSum(RAW[i]), v.get(i), 0.01);
    }
}

---

From source file:org.apache.mahout.graph.common.AugmentGraphWithDegreesJobTest.java

License:Apache License

@Test
public void testJoinDegrees() throws Exception {
    Reducer.Context ctx = EasyMock.createMock(Reducer.Context.class);
    Vertex first = new Vertex(1);
    Vertex second = new Vertex(3);

    ctx.write(new UndirectedEdgeWithDegrees(new VertexWithDegree(first, 1), new VertexWithDegree(second, 3)),
            NullWritable.get());

    EasyMock.replay(ctx);// w w  w . ja v  a 2 s  .  c o  m

    new AugmentGraphWithDegreesJob.JoinDegreesReducer().reduce(new UndirectedEdge(first, second),
            Arrays.asList(new VertexWithDegree(first, 1), new VertexWithDegree(second, 3)), ctx);

    EasyMock.verify(ctx);
}

---

From source file:org.apache.mahout.graph.common.AugmentGraphWithDegreesJobTest.java

License:Apache License

@Test
public void toyIntegrationTest() throws Exception {
    File inputFile = getTestTempFile("edges.seq");
    File outputDir = getTestTempDir("output");
    outputDir.delete();/* w  ww .ja v a2s. c o  m*/
    File tempDir = getTestTempDir("tmp");

    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf,
            new Path(inputFile.getAbsolutePath().toString()), UndirectedEdge.class, NullWritable.class);

    try {
        writer.append(new UndirectedEdge(0, 1), NullWritable.get());
        writer.append(new UndirectedEdge(0, 2), NullWritable.get());
        writer.append(new UndirectedEdge(0, 3), NullWritable.get());
        writer.append(new UndirectedEdge(0, 4), NullWritable.get());
        writer.append(new UndirectedEdge(0, 5), NullWritable.get());
        writer.append(new UndirectedEdge(0, 6), NullWritable.get());
        writer.append(new UndirectedEdge(0, 7), NullWritable.get());
        writer.append(new UndirectedEdge(1, 2), NullWritable.get());
        writer.append(new UndirectedEdge(1, 3), NullWritable.get());
        writer.append(new UndirectedEdge(2, 3), NullWritable.get());
        writer.append(new UndirectedEdge(4, 5), NullWritable.get());
        writer.append(new UndirectedEdge(4, 7), NullWritable.get());
    } finally {
        Closeables.closeQuietly(writer);
    }

    AugmentGraphWithDegreesJob augmentGraphWithDegreesJob = new AugmentGraphWithDegreesJob();
    augmentGraphWithDegreesJob.setConf(conf);
    augmentGraphWithDegreesJob.run(new String[] { "--input", inputFile.getAbsolutePath(), "--output",
            outputDir.getAbsolutePath(), "--tempDir", tempDir.getAbsolutePath() });

    Set<UndirectedEdgeWithDegrees> edges = Sets.newHashSet();
    for (Pair<UndirectedEdgeWithDegrees, NullWritable> result : new SequenceFileIterable<UndirectedEdgeWithDegrees, NullWritable>(
            new Path(outputDir.getAbsolutePath() + "/part-r-00000"), false, conf)) {
        edges.add(result.getFirst());
    }

    assertEquals(12, edges.size());
    assertTrue(edges.contains(new UndirectedEdgeWithDegrees(0, 7, 1, 3)));
    assertTrue(edges.contains(new UndirectedEdgeWithDegrees(0, 7, 2, 3)));
    assertTrue(edges.contains(new UndirectedEdgeWithDegrees(0, 7, 3, 3)));
    assertTrue(edges.contains(new UndirectedEdgeWithDegrees(0, 7, 4, 3)));
    assertTrue(edges.contains(new UndirectedEdgeWithDegrees(0, 7, 5, 2)));
    assertTrue(edges.contains(new UndirectedEdgeWithDegrees(0, 7, 6, 1)));
    assertTrue(edges.contains(new UndirectedEdgeWithDegrees(0, 7, 7, 2)));
    assertTrue(edges.contains(new UndirectedEdgeWithDegrees(1, 3, 2, 3)));
    assertTrue(edges.contains(new UndirectedEdgeWithDegrees(1, 3, 3, 3)));
    assertTrue(edges.contains(new UndirectedEdgeWithDegrees(2, 3, 3, 3)));
    assertTrue(edges.contains(new UndirectedEdgeWithDegrees(4, 3, 5, 2)));
    assertTrue(edges.contains(new UndirectedEdgeWithDegrees(4, 3, 7, 2)));
}

---

From source file:org.apache.mahout.graph.common.DegreeDistributionJobTest.java

License:Apache License

@Test
public void toyIntegrationTest() throws Exception {

    File inputFile = getTestTempFile("edges.seq");
    File outputDir = getTestTempDir("output");
    outputDir.delete();/*from w  w w  .  ja  v a  2  s  .  c o  m*/
    File tempDir = getTestTempDir("tmp");

    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, new Path(inputFile.getAbsolutePath()),
            UndirectedEdge.class, NullWritable.class);

    try {
        writer.append(new UndirectedEdge(0, 1), NullWritable.get());
        writer.append(new UndirectedEdge(0, 2), NullWritable.get());
        writer.append(new UndirectedEdge(0, 3), NullWritable.get());
        writer.append(new UndirectedEdge(0, 4), NullWritable.get());
        writer.append(new UndirectedEdge(0, 5), NullWritable.get());
        writer.append(new UndirectedEdge(0, 6), NullWritable.get());
        writer.append(new UndirectedEdge(0, 7), NullWritable.get());
        writer.append(new UndirectedEdge(1, 2), NullWritable.get());
        writer.append(new UndirectedEdge(1, 3), NullWritable.get());
        writer.append(new UndirectedEdge(2, 3), NullWritable.get());
        writer.append(new UndirectedEdge(4, 5), NullWritable.get());
        writer.append(new UndirectedEdge(4, 7), NullWritable.get());
    } finally {
        Closeables.closeQuietly(writer);
    }

    DegreeDistributionJob degreeDistributionJob = new DegreeDistributionJob();
    degreeDistributionJob.setConf(conf);
    degreeDistributionJob.run(new String[] { "--input", inputFile.getAbsolutePath(), "--output",
            outputDir.getAbsolutePath(), "--tempDir", tempDir.getAbsolutePath() });

    Map<Integer, Integer> degreeDistribution = Maps.newHashMap();
    for (CharSequence line : new FileLineIterable(new File(outputDir, "part-r-00000"))) {
        String[] tokens = Iterables.toArray(TAB.split(line), String.class);
        degreeDistribution.put(Integer.parseInt(tokens[0]), Integer.parseInt(tokens[1]));
    }

    assertEquals(4, degreeDistribution.size());
    assertEquals(1, degreeDistribution.get(1).intValue());
    assertEquals(2, degreeDistribution.get(2).intValue());
    assertEquals(4, degreeDistribution.get(3).intValue());
    assertEquals(1, degreeDistribution.get(7).intValue());
}

---

From source file:org.apache.mahout.graph.common.LocalClusteringCoefficientJobTest.java

License:Apache License

@Test
public void toyIntegrationTest() throws Exception {

    File edgesFile = getTestTempFile("edges.seq");
    File trianglesFile = getTestTempFile("triangles.seq");
    File outputDir = getTestTempDir("output");
    outputDir.delete();//  ww w .  j  av  a2 s . c  om
    File tempDir = getTestTempDir("tmp");

    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    SequenceFile.Writer edgesWriter = new SequenceFile.Writer(fs, conf, new Path(edgesFile.getAbsolutePath()),
            UndirectedEdge.class, NullWritable.class);
    try {
        edgesWriter.append(new UndirectedEdge(0, 1), NullWritable.get());
        edgesWriter.append(new UndirectedEdge(0, 2), NullWritable.get());
        edgesWriter.append(new UndirectedEdge(0, 3), NullWritable.get());
        edgesWriter.append(new UndirectedEdge(0, 4), NullWritable.get());
        edgesWriter.append(new UndirectedEdge(0, 5), NullWritable.get());
        edgesWriter.append(new UndirectedEdge(0, 6), NullWritable.get());
        edgesWriter.append(new UndirectedEdge(0, 7), NullWritable.get());
        edgesWriter.append(new UndirectedEdge(1, 2), NullWritable.get());
        edgesWriter.append(new UndirectedEdge(1, 3), NullWritable.get());
        edgesWriter.append(new UndirectedEdge(2, 3), NullWritable.get());
        edgesWriter.append(new UndirectedEdge(4, 5), NullWritable.get());
        edgesWriter.append(new UndirectedEdge(4, 7), NullWritable.get());
    } finally {
        Closeables.closeQuietly(edgesWriter);
    }

    SequenceFile.Writer trianglesWriter = new SequenceFile.Writer(fs, conf,
            new Path(trianglesFile.getAbsolutePath()), Triangle.class, NullWritable.class);
    try {
        trianglesWriter.append(new Triangle(0, 1, 2), NullWritable.get());
        trianglesWriter.append(new Triangle(0, 1, 3), NullWritable.get());
        trianglesWriter.append(new Triangle(0, 2, 3), NullWritable.get());
        trianglesWriter.append(new Triangle(0, 4, 5), NullWritable.get());
        trianglesWriter.append(new Triangle(0, 4, 7), NullWritable.get());
        trianglesWriter.append(new Triangle(1, 2, 3), NullWritable.get());
    } finally {
        Closeables.closeQuietly(trianglesWriter);
    }

    LocalClusteringCoefficientJob clusteringCoefficientJob = new LocalClusteringCoefficientJob();
    clusteringCoefficientJob.setConf(conf);
    clusteringCoefficientJob.run(new String[] { "--edges", edgesFile.getAbsolutePath(), "--triangles",
            trianglesFile.getAbsolutePath(), "--output", outputDir.getAbsolutePath(), "--tempDir",
            tempDir.getAbsolutePath() });

    Map<Long, Double> localClusteringCoefficients = Maps.newHashMap();
    for (CharSequence line : new FileLineIterable(new File(outputDir, "part-r-00000"))) {
        String[] tokens = Iterables.toArray(TAB.split(line), String.class);
        localClusteringCoefficients.put(Long.parseLong(tokens[0]), Double.parseDouble(tokens[1]));
    }

    assertEquals(8, localClusteringCoefficients.size());
    assertEquals(0.119047, localClusteringCoefficients.get(0L), EPSILON);
    assertEquals(0.5, localClusteringCoefficients.get(1L), EPSILON);
    assertEquals(0.5, localClusteringCoefficients.get(2L), EPSILON);
    assertEquals(0.5, localClusteringCoefficients.get(3L), EPSILON);
    assertEquals(0.333333, localClusteringCoefficients.get(4L), EPSILON);
    assertEquals(0.5, localClusteringCoefficients.get(5L), EPSILON);
    assertEquals(0.0, localClusteringCoefficients.get(6L), EPSILON);
    assertEquals(0.5, localClusteringCoefficients.get(7L), EPSILON);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »