Example usage for org.apache.hadoop.io NullWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io NullWritable get.

Prototype

public static NullWritable get()

Source Link

Document

Returns the single instance of this class.

Usage

From source file:ComRoughSetApproInputSampler.java

License:Apache License

/**
 * Write a partition file for the given job, using the Sampler provided.
 * Queries the sampler for a sample keyset, sorts by the output key
 * comparator, selects the keys for each rank, and writes to the destination
 * returned from {@link TotalOrderPartitioner#getPartitionFile}.
 *///from  w ww. j  a  va  2s  .  com
@SuppressWarnings("unchecked") // getInputFormat, getOutputKeyComparator
public static <K, V> void writePartitionFile(Job job, Sampler<K, V> sampler)
        throws IOException, ClassNotFoundException, InterruptedException {
    Configuration conf = job.getConfiguration();
    final InputFormat inf = ReflectionUtils.newInstance(job.getInputFormatClass(), conf);
    int numPartitions = job.getNumReduceTasks();
    K[] samples = (K[]) sampler.getSample(inf, job);
    LOG.info("Using " + samples.length + " samples");
    RawComparator<K> comparator = (RawComparator<K>) job.getSortComparator();
    Arrays.sort(samples, comparator);
    Path dst = new Path(TotalOrderPartitioner.getPartitionFile(conf));
    FileSystem fs = dst.getFileSystem(conf);
    if (fs.exists(dst)) {
        fs.delete(dst, false);
    }
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, dst, job.getMapOutputKeyClass(),
            NullWritable.class);
    NullWritable nullValue = NullWritable.get();
    float stepSize = samples.length / (float) numPartitions;
    int last = -1;
    for (int i = 1; i < numPartitions; ++i) {
        int k = Math.round(stepSize * i);
        while (last >= k && comparator.compare(samples[last], samples[k]) == 0) {
            ++k;
        }
        writer.append(samples[k], nullValue);
        last = k;
    }
    writer.close();
}

From source file:SleepJob.java

License:Apache License

public void map(IntWritable key, IntWritable value, OutputCollector<IntWritable, NullWritable> output,
        Reporter reporter) throws IOException {

    //it is expected that every map processes mapSleepCount number of records. 
    try {//from ww  w.j  av a2 s  .  com
        reporter.setStatus("Sleeping... (" + (mapSleepDuration * (mapSleepCount - count)) + ") ms left");
        Thread.sleep(mapSleepDuration);
    } catch (InterruptedException ex) {
        throw (IOException) new IOException("Interrupted while sleeping").initCause(ex);
    }
    ++count;
    // output reduceSleepCount * numReduce number of random values, so that
    // each reducer will get reduceSleepCount number of keys.
    int k = key.get();
    for (int i = 0; i < value.get(); ++i) {
        output.collect(new IntWritable(k + i), NullWritable.get());
    }
}

From source file:SleepJobWithArray.java

License:Apache License

public void map(IntWritable key, IntWritable value, OutputCollector<IntWritable, NullWritable> output,
        Reporter reporter) throws IOException {

    if (initBigArray) {
        // Yes, I should use log4j :-/
        System.out.println("Requesting array of " + bigArraySize);
        int[] foo = new int[bigArraySize];
    }// www.  j  a  v  a 2 s. c o  m
    //it is expected that every map processes mapSleepCount number of records. 
    try {
        reporter.setStatus("Sleeping... (" + (mapSleepDuration * (mapSleepCount - count)) + ") ms left");
        Thread.sleep(mapSleepDuration);
    } catch (InterruptedException ex) {
        throw (IOException) new IOException("Interrupted while sleeping").initCause(ex);
    }
    ++count;
    // output reduceSleepCount * numReduce number of random values, so that
    // each reducer will get reduceSleepCount number of keys.
    int k = key.get();
    for (int i = 0; i < value.get(); ++i) {
        output.collect(new IntWritable(k + i), NullWritable.get());
    }
}

From source file:$package.TextFileSetSink.java

License:Apache License

@Override
public void transform(StructuredRecord input, Emitter<KeyValue<NullWritable, Text>> emitter) throws Exception {
    StringBuilder joinedFields = new StringBuilder();
    Iterator<Schema.Field> fieldIter = input.getSchema().getFields().iterator();
    if (!fieldIter.hasNext()) {
        // shouldn't happen
        return;//from ww  w.j av a2  s . c o  m
    }

    Object val = input.get(fieldIter.next().getName());
    if (val != null) {
        joinedFields.append(val);
    }
    while (fieldIter.hasNext()) {
        String fieldName = fieldIter.next().getName();
        joinedFields.append(config.fieldSeparator);
        val = input.get(fieldName);
        if (val != null) {
            joinedFields.append(val);
        }
    }
    emitter.emit(new KeyValue<>(NullWritable.get(), new Text(joinedFields.toString())));
}

From source file:AllLab_Skeleton.Lab2.Lab2Mapper.java

@Override
public void map(Object key, Text values, Context context) {
    if (values.toString().length() > 0) {
        try {//  w  ww .ja v a2  s .c  o m
            String value[] = values.toString().split("\t");
            CompositeKeyWritable cw = new CompositeKeyWritable(value[6], value[3]);
            context.write(cw, NullWritable.get());
        } catch (IOException | InterruptedException ex) {
            Logger.getLogger(Lab2Mapper.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
}

From source file:AllLab_Skeleton.Lab2.Lab2Reducer.java

/**
 *
 * @param key/*from  www  .  ja v  a2  s  .co  m*/
 * @param values
 * @param context
 */
@Override
public void reduce(CompositeKeyWritable key, Iterable<NullWritable> values, Context context) {
    for (NullWritable val : values) {
        try {
            context.write(key, NullWritable.get());
        } catch (IOException | InterruptedException ex) {
            System.out.println("Error Message" + ex.getMessage());
        }
    }
}

From source file:Analysis.A1_Total_Unique_Artists_on_Service.Distinct_Artist_Mapper.java

public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
    // get artist info
    String[] artistInfo = value.toString().split("\t");

    // cleanup artist name by removing all whitespaces
    String aName = artistInfo[2] + "\t"; //.replaceAll("\\s+","");

    // extract artist name
    artistName = new Text(aName);

    context.write(artistName, NullWritable.get());
}

From source file:Analysis.A1_Total_Unique_Artists_on_Service.Distinct_Artist_Reducer.java

public void reduce(Text key, Iterable<IntWritable> values, Context context)
        throws IOException, InterruptedException {
    context.write(key, NullWritable.get());
}

From source file:Analysis.A2_Top_20_Most_Popular_Artists.Top_20_Most_Popular_Artist_Reducer.java

@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
    for (Map.Entry<Integer, String> entry : top20.entrySet()) {

        //Integer key = entry.getKey();
        String value = entry.getValue().substring(0, 1).toUpperCase() + entry.getValue().substring(1);

        // print atop 20 artists
        context.write(NullWritable.get(), new Text(value));
    }/*  ww w  . ja va 2s . c  om*/
}

From source file:Analysis.A6_User_Differentiation_By_Age.Partition_Users_By_Age_Reducer.java

public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
    for (Text value : values) {
        multipleOutputs.write("ageBins", value, NullWritable.get(), key.toString());
    }/*from  w w w .  j  a  va2 s  .c  o m*/
}