Example usage for org.apache.hadoop.io NullWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io NullWritable get.

Prototype

public static NullWritable get() 

Source Link

Document

Returns the single instance of this class.

Usage

From source file:Analysis.A8_Top_10_Most_Popular_Tracks.Top_10_Most_Popular_Tracks_Reducer.java

@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
    for (Map.Entry<Integer, String> entry : top10.entrySet()) {

        //Integer key = entry.getKey();
        String value = entry.getValue().substring(0, 1).toUpperCase() + entry.getValue().substring(1);

        // print top 10 tracks
        context.write(NullWritable.get(), new Text(value));
    }/*from   ww w.java  2 s  .c o  m*/
}

---

From source file:Assignment5_P2_DistinctIPAddress.DistinctIPAddress_Mapper.java

public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
    String[] ipInfo = value.toString().split("-");
    ipaddr = new Text(ipInfo[0].trim());
    context.write(ipaddr, NullWritable.get());
}

---

From source file:Assignment5_P2_DistinctIPAddress.DistinctIPAddress_Reducer.java

public void reduce(Text key, Iterable<NullWritable> values, Context context)
        throws IOException, InterruptedException {
    context.write(key, NullWritable.get());
}

---

From source file:Assignment5_P3_PartitionPattern.Partition_IPAddress_By_Month_Reducer.java

public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
    for (Text value : values) {
        context.write(value, NullWritable.get());
    }//from   www.  j ava 2  s  .c  om
}

---

From source file:Assignment5_P4_BinningPattern.Binning_IPAddress_By_Day_Mapper.java

public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
    String[] ipInfo = value.toString().split("-");
    String timing = ipInfo[2].trim();

    String createDate = timing.substring(1, timing.length()).trim();//System.out.println("Date found : '" + createDate + "'");

    Calendar cal = Calendar.getInstance();
    try {//  w w w .  j a va2s  . co  m
        cal.setTime(fmt.parse(createDate)); //System.out.println("Month evaluated by format : " + fmt.parse(createDate)); //System.out.println("Month evaluated by cal : " + cal.get(Calendar.MONTH));
        createHour.set(cal.get(Calendar.HOUR_OF_DAY));
        multipleOutputs.write("textualBins", NullWritable.get(), value, createHour + "-hour");
        multipleOutputs.write("massaBins", NullWritable.get(), value, createHour + "-trial-by-witnessing");
    } catch (ParseException e) {
        e.printStackTrace();
    }
}

---

From source file:at.illecker.hama.hybrid.examples.hellohybrid.HelloHybridBSP.java

License:Apache License

@Override
public void bsp(BSPPeer<IntWritable, NullWritable, IntWritable, NullWritable, NullWritable> peer)
        throws IOException, SyncException, InterruptedException {

    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs//from  ww  w .j a  v a 2  s .c om
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("HelloHybrid.bsp executed on CPU!\n");

    ArrayList<Integer> summation = new ArrayList<Integer>();

    // test input
    IntWritable key = new IntWritable();
    NullWritable nullValue = NullWritable.get();

    while (peer.readNext(key, nullValue)) {
        outStream.writeChars("input: key: '" + key.get() + "'\n");
        summation.add(key.get());
    }

    // test sequenceFileReader
    Path example = new Path(peer.getConfiguration().get(CONF_EXAMPLE_PATH));
    SequenceFile.Reader reader = null;
    try {
        reader = new SequenceFile.Reader(fs, example, peer.getConfiguration());

        int i = 0;
        while (reader.next(key, nullValue)) {
            outStream.writeChars("sequenceFileReader: key: '" + key.get() + "'\n");
            if (i < summation.size()) {
                summation.set(i, summation.get(i) + key.get());
            }
            i++;
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        if (reader != null) {
            reader.close();
        }
    }

    // test output
    for (Integer i : summation) {
        key.set(i);
        outStream.writeChars("output: key: '" + key.get() + "'\n");
        peer.write(key, nullValue);
    }

    // test getAllPeerNames
    outStream.writeChars("getAllPeerNames: '" + Arrays.toString(peer.getAllPeerNames()) + "'\n");

    // test String.split
    String splitString = "boo:and:foo";
    String[] splits;

    outStream.writeChars("splitString: '" + splitString + "'\n");

    splits = splitString.split(":");
    outStream.writeChars("split(\":\") len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    splits = splitString.split(":", 2);
    outStream.writeChars(
            "split(\":\",2) len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    splits = splitString.split(":", 5);
    outStream.writeChars(
            "split(\":\",5) len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    splits = splitString.split(":", -2);
    outStream.writeChars(
            "split(\":\",-2) len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    splits = splitString.split(";");
    outStream.writeChars("split(\";\") len: " + splits.length + " values: '" + Arrays.toString(splits) + "'\n");

    outStream.close();
}

---

From source file:at.illecker.hama.hybrid.examples.hellohybrid.HelloHybridBSP.java

License:Apache License

private static void prepareInput(Configuration conf, Path inputPath, Path exampleFile, int n)
        throws IOException {
    FileSystem fs = inputPath.getFileSystem(conf);

    // Create input file writers depending on bspTaskNum
    int bspTaskNum = conf.getInt("bsp.peers.num", 1);
    SequenceFile.Writer[] inputWriters = new SequenceFile.Writer[bspTaskNum];
    for (int i = 0; i < bspTaskNum; i++) {
        Path inputFile = new Path(inputPath, "input" + i + ".seq");
        LOG.info("inputFile: " + inputFile.toString());
        inputWriters[i] = SequenceFile.createWriter(fs, conf, inputFile, IntWritable.class, NullWritable.class,
                CompressionType.NONE);/*  w w  w  . j  a va 2 s .  co m*/
    }

    // Create example file writer
    SequenceFile.Writer exampleWriter = SequenceFile.createWriter(fs, conf, exampleFile, IntWritable.class,
            NullWritable.class, CompressionType.NONE);

    // Write random values to input files and example
    IntWritable inputKey = new IntWritable();
    NullWritable nullValue = NullWritable.get();
    Random r = new Random();
    for (long i = 0; i < n; i++) {
        inputKey.set(r.nextInt(n));
        for (int j = 0; j < inputWriters.length; j++) {
            inputWriters[j].append(inputKey, nullValue);
        }
        inputKey.set(r.nextInt(n));
        exampleWriter.append(inputKey, nullValue);
    }

    // Close file writers
    for (int j = 0; j < inputWriters.length; j++) {
        inputWriters[j].close();
    }
    exampleWriter.close();
}

---

From source file:at.illecker.hama.hybrid.examples.hellohybrid.HelloHybridBSP.java

License:Apache License

static void printOutput(BSPJob job, Path path) throws IOException {
    FileSystem fs = path.getFileSystem(job.getConfiguration());
    FileStatus[] files = fs.listStatus(path);
    for (int i = 0; i < files.length; i++) {
        if (files[i].getLen() > 0) {
            System.out.println("File " + files[i].getPath());
            SequenceFile.Reader reader = null;
            try {
                reader = new SequenceFile.Reader(fs, files[i].getPath(), job.getConfiguration());

                IntWritable key = new IntWritable();
                NullWritable value = NullWritable.get();
                while (reader.next(key, value)) {
                    System.out.println("key: '" + key.get() + "' value: '" + value + "'\n");
                }// w w  w  .  ja va2  s . c  om
            } catch (IOException e) {
                FSDataInputStream in = fs.open(files[i].getPath());
                IOUtils.copyBytes(in, System.out, job.getConfiguration(), false);
                in.close();
            } finally {
                if (reader != null) {
                    reader.close();
                }
            }
        }
    }
    // fs.delete(FileOutputFormat.getOutputPath(job), true);
}

---

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

/********************************* CPU *********************************/
@Override/*from w  ww  .  j av a 2 s.  c om*/
public void setup(
        BSPPeer<PipesVectorWritable, NullWritable, IntWritable, PipesVectorWritable, CenterMessage> peer)
        throws IOException {

    this.m_conf = peer.getConfiguration();
    this.m_timeMeasurement = m_conf.getBoolean(CONF_TIME, false);
    this.m_isDebuggingEnabled = m_conf.getBoolean(CONF_DEBUG, false);
    this.m_maxIterations = m_conf.getInt(CONF_MAX_ITERATIONS, -1);

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(m_conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) m_conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    long startTime = 0;
    if (m_timeMeasurement) {
        startTime = System.currentTimeMillis();
    }

    // Init center vectors
    Path centroids = new Path(m_conf.get(CONF_CENTER_IN_PATH));
    FileSystem fs = FileSystem.get(m_conf);

    final ArrayList<DoubleVector> centers = new ArrayList<DoubleVector>();
    SequenceFile.Reader reader = null;
    try {
        reader = new SequenceFile.Reader(fs, centroids, m_conf);
        PipesVectorWritable key = new PipesVectorWritable();
        NullWritable value = NullWritable.get();
        while (reader.next(key, value)) {
            DoubleVector center = key.getVector();
            centers.add(center);
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        if (reader != null) {
            reader.close();
        }
    }

    Preconditions.checkArgument(centers.size() > 0, "Centers file must contain at least a single center!");

    this.m_centers_cpu = centers.toArray(new DoubleVector[centers.size()]);

    long stopTime = 0;
    if (m_timeMeasurement) {
        stopTime = System.currentTimeMillis();
        LOG.info("# setupTime: " + ((stopTime - startTime) / 1000.0) + " sec");
        if (m_isDebuggingEnabled) {
            m_logger.writeChars("PiEstimatorHybrid,setupTime: " + ((stopTime - startTime) / 1000.0) + " sec\n");
        }
    }
}

---

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

private void assignCenters(
        BSPPeer<PipesVectorWritable, NullWritable, IntWritable, PipesVectorWritable, CenterMessage> peer)
        throws IOException {

    // each task has all the centers, if a center has been updated it
    // needs to be broadcasted.
    final DoubleVector[] newCenterArray = new DoubleVector[m_centers_cpu.length];
    final int[] summationCount = new int[m_centers_cpu.length];

    // if our cache is empty, we have to read it from disk first
    if (m_cache.isEmpty()) {
        final PipesVectorWritable key = new PipesVectorWritable();
        final NullWritable value = NullWritable.get();
        while (peer.readNext(key, value)) {
            DoubleVector deepCopy = key.getVector().deepCopy();
            m_cache.add(deepCopy);//from w ww .j a va  2 s  .  c o m
            // but do the assignment directly
            assignCentersInternal(newCenterArray, summationCount, deepCopy);
        }
    } else {
        // now we can iterate in memory and check against the centers
        for (DoubleVector v : m_cache) {
            assignCentersInternal(newCenterArray, summationCount, v);
        }
    }

    // now send messages about the local updates to each other peer
    for (int i = 0; i < newCenterArray.length; i++) {
        if (newCenterArray[i] != null) {
            for (String peerName : peer.getAllPeerNames()) {
                peer.send(peerName, new CenterMessage(i, summationCount[i], newCenterArray[i]));
                // Logging
                // if (m_isDebuggingEnabled) {
                // m_logger.writeChars("assignCenters,sent,peerName=" + peerName
                // + ",CenterMessage=" + i + "," + summationCount[i] + ","
                // + Arrays.toString(newCenterArray[i].toArray()) + "\n");
                // m_logger.flush();
                // }
            }
        }
    }

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »