Example usage for org.apache.hadoop.io MapWritable size

Introduction

In this page you can find the example usage for org.apache.hadoop.io MapWritable size.

Prototype

@Override
    public int size() 

Source Link

Usage

From source file:selfy.JSONOutput.java

@Override
public RecordWriter<Text, MapWritable> getRecordWriter(final TaskAttemptContext tac)
        throws IOException, InterruptedException {
    Path p = FileOutputFormat.getOutputPath(tac);
    FileSystem fileSystem = p.getFileSystem(tac.getConfiguration());

    final int n = 1024;
    final FSDataOutputStream[] streams = new FSDataOutputStream[n];
    final boolean[] first = new boolean[n];

    for (int i = 0; i < n; i++) {
        first[i] = true;//w  w  w. ja  v a  2  s .  c  o  m
        Path output = new Path(p, i + ".json");
        streams[i] = fileSystem.create(output);
        streams[i].write("{\n".getBytes());
    }

    return new RecordWriter<Text, MapWritable>() {

        @Override
        public void write(Text k, MapWritable v) throws IOException, InterruptedException {
            long d = k.toString().hashCode();

            d = d - Integer.MIN_VALUE;
            int indexNumber = (int) (d % n);
            FSDataOutputStream stream = streams[indexNumber];
            if (!first[indexNumber])
                stream.write(",\n".getBytes());
            else
                first[indexNumber] = false;
            stream.write(("\"" + k + "\" : {\n").getBytes());
            stream.write("\t\"documentCount\" : ".getBytes());
            stream.write(("" + v.size() + ",\n").getBytes());
            stream.write("\t\"documents\": [\n".getBytes());
            int i = 0;
            for (Map.Entry<Writable, Writable> a : v.entrySet()) {
                String id = ((Text) a.getKey()).toString();
                int score = ((IntWritable) a.getValue()).get();
                stream.write("\t\t{\n".getBytes());
                stream.write(("\t\t\t\"document\" : \"" + id + "\",\n").getBytes());
                stream.write(("\t\t\t\"score\" : " + (float) score + "\n").getBytes());
                stream.write("\t\t}".getBytes());
                if (i + 1 < v.size())
                    stream.write(",\n".getBytes());
                else
                    stream.write("\n".getBytes());
                i++;
            }
            stream.write("\t]\n".getBytes());
            stream.write("}".getBytes());

        }

        @Override
        public void close(TaskAttemptContext tac) throws IOException, InterruptedException {
            for (int i = 0; i < n; i++) {
                streams[i].write("}\n".getBytes());
                streams[i].close();
            }
        }
    };
}

---

• «
• 1
• 2
• »