Example usage for org.apache.hadoop.io DoubleWritable DoubleWritable

List of usage examples for org.apache.hadoop.io DoubleWritable DoubleWritable

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. DoubleWritable
  6. DoubleWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io DoubleWritable DoubleWritable.

Prototype

public DoubleWritable(double value)

Source Link

Usage

From source file:eagle.query.aggregate.raw.RawGroupbyBucket.java

License:Apache License

/**
 * expensive operation - create objects and format the result
 * @return/*from www.j  a  v  a 2s .c  o  m*/
 */
public List<GroupbyKeyValue> groupbyKeyValues() {
    List<GroupbyKeyValue> results = new ArrayList<GroupbyKeyValue>();
    for (Map.Entry<GroupbyKey, List<Function>> entry : this.group2FunctionMap.entrySet()) {
        GroupbyValue value = new GroupbyValue();
        for (Function f : entry.getValue()) {
            value.add(new DoubleWritable(f.result()));
            value.addMeta(f.count());
        }
        results.add(new GroupbyKeyValue(entry.getKey(), value));
    }
    return results;
}

From source file:edu.brown.cs.mapreduce.generator.DataLoader.java

License:Open Source License

/**
 * @param args//from ww w . ja  v a 2s  .co  m
 */
public static void main(String[] args) {
    List<String> otherArgs = new ArrayList<String>();
    for (int i = 0; i < args.length; i++) {
        if ("-compress".equals(args[i])) {
            DataLoader.compress = true;
            DataLoader.sequence = true;
        } else if ("-sequence".equals(args[i])) {
            DataLoader.sequence = true;
        } else if ("-tuple".equals(args[i])) {
            DataLoader.tuple = true;
        } else if ("-local".equals(args[i])) {
            DataLoader.local = true;
        } else if ("-limit".equals(args[i])) {
            DataLoader.limit = Integer.parseInt(args[++i]);
        } else if ("-xargs".equals(args[i])) {
            DataLoader.xargs = true;
        } else if ("-debug".equals(args[i])) {
            DataLoader.debug = true;
        } else {
            otherArgs.add(args[i]);
        }
    } // FOR

    if (otherArgs.size() < 3 && !DataLoader.xargs) {
        System.err.println("USAGE: DataLoader <input type> <input file> <output file>");
        System.exit(1);
    }

    String input_type = otherArgs.get(0).toLowerCase();
    String input_file = otherArgs.get(1);
    String output_file = null;
    if (DataLoader.xargs) {
        output_file = input_file + ".dl";
    } else {
        output_file = otherArgs.get(2);
    }

    boolean valid = false;
    for (String type : DataLoader.VALID_TYPES) {
        if (type.equals(input_type)) {
            valid = true;
            break;
        }
    }
    if (!valid) {
        System.err.println("ERROR: Invalid input data type '" + input_type + "'");
        System.exit(1);
    }

    if (debug) {
        System.out.println("Input Type:  " + input_type);
        System.out.println("Input File:  " + input_file);
        System.out.println("Output File: " + output_file);
        System.out.println("Limit:       " + DataLoader.limit);
        System.out.println("Local:       " + DataLoader.local);
        System.out.println("XArgs:       " + DataLoader.xargs);
    }

    //
    // Get HDFS filesystem object that we can use for writing
    //
    FileSystem fs = null;
    Configuration conf = null;
    if (!DataLoader.local) {
        conf = AbstractHadoopClient.getConfiguration();
        try {
            fs = FileSystem.get(conf);
        } catch (Exception ex) {
            ex.printStackTrace();
            System.exit(-1);
        }
        if (debug)
            System.out.println("fs.default.name: " + conf.get("fs.default.name"));
    }

    //
    // Now open the file that we want to read and start writing the contents to our file system
    // For some things, like 'urls' we will want reverse the order so that the data makes sense
    // in our key->value paradigm
    //
    BufferedReader in = null;
    DataOutputStream out = null;
    SequenceFile.Writer writer = null;
    int lines = 0;
    try {
        if (input_file.equals("-")) {
            in = new BufferedReader(new InputStreamReader(System.in));
        } else {
            in = new BufferedReader(new FileReader(input_file));
        }
    } catch (FileNotFoundException ex) {
        System.err.println("ERROR: The input file '" + input_file + "' was not found : " + ex.getMessage());
        System.exit(1);
    }
    try {
        if (!DataLoader.local) {
            //
            // FileSystem Writer
            //
            if (!DataLoader.sequence) {
                out = fs.create(new Path(output_file));
                //
                // SequenceFile Writer
                //
            } else {
                if (input_type.equals("sortgrep"))
                    DataLoader.tuple = false;
                if (DataLoader.debug)
                    System.out.print("Creating " + (DataLoader.compress ? "compressed " : "")
                            + "SequenceFile.Writer for '" + output_file + "': ");
                Class<? extends Writable> key_class = Text.class;
                Class<? extends Writable> value_class = null;
                if (DataLoader.tuple) {
                    if (input_type.equals("uservisits"))
                        value_class = UserVisitsTuple.class;
                    if (input_type.equals("rankings"))
                        value_class = RankingsTuple.class;
                } else {
                    value_class = Text.class;
                }
                writer = SequenceFile.createWriter(fs, conf, new Path(output_file), key_class, value_class,
                        (DataLoader.compress ? SequenceFile.CompressionType.BLOCK
                                : SequenceFile.CompressionType.NONE));
                if (DataLoader.debug)
                    System.out.println("DONE!");
            }
            //
            // Local Filesystem
            //
        } else {
            out = new DataOutputStream(new FileOutputStream(output_file, true));
        }
    } catch (IOException ex) {
        System.err.println("ERROR: Failed to open output file '" + output_file + "' : " + ex.getMessage());
        System.exit(1);
    }
    try {
        //
        // Now read in each line of the input file and append it to our output
        //
        while (in.ready()) {
            //
            // Ignore any misformated lines
            //
            String line = null;
            String key = "";
            String value = "";
            try {
                line = in.readLine();
                String data[] = line.split("\\" + BenchmarkBase.VALUE_DELIMITER);
                //
                // Switch the two values in a rankings record
                //
                if (input_type.equals("rankings")) {
                    key = data[1];
                    value = data[0];
                    for (int i = 2; i < data.length; i++) {
                        value += BenchmarkBase.VALUE_DELIMITER + data[i];
                    } // FOR
                    //
                    // Change the comma to a tab
                    //
                } else if (input_type.equals("convert") || input_type.equals("uservisits")) {
                    key = data[0];
                    for (int i = 1; i < data.length; i++) {
                        if (i != 1)
                            value += BenchmarkBase.VALUE_DELIMITER;
                        value += data[i];
                    } // FOR
                    //
                    // Don't do anything with the SortGrep data!
                    //
                } else if (input_type.equals("sortgrep")) {
                    key = line.substring(0, 10);
                    value = line.substring(10);
                    //
                    // All others need to switch the first VALUE_DELIMITER to a KEYVALUE_DELIMITER
                    //   
                } else {
                    line = line.replaceFirst(BenchmarkBase.VALUE_DELIMITER, BenchmarkBase.KEYVALUE_DELIMITER);
                }
                if (DataLoader.local || !DataLoader.sequence) {
                    line = key + BenchmarkBase.KEYVALUE_DELIMITER + value + "\n";
                    out.write(line.getBytes());
                } else {
                    //if (DataLoader.debug) System.out.println("[" + lines + "] " + key + " => " + value);
                    if (DataLoader.tuple) {
                        try {
                            data = value.split("\\" + BenchmarkBase.VALUE_DELIMITER);
                            Writable tuple_values[] = new Writable[data.length];
                            Class<?> types[] = (input_type.equals("uservisits") ? BenchmarkBase.USERVISITS_TYPES
                                    : BenchmarkBase.RANKINGS_TYPES);
                            for (int ctr = 0; ctr < data.length; ctr++) {
                                //
                                // Important! You have to subtract one from the types list
                                // because the first one is really the key, but we're creating a tuple
                                // on just the values!!
                                //
                                if (types[ctr + 1] == Text.class) {
                                    tuple_values[ctr] = new Text(data[ctr]);
                                } else if (types[ctr + 1] == IntWritable.class) {
                                    tuple_values[ctr] = new IntWritable(Integer.valueOf(data[ctr]));
                                } else if (types[ctr + 1] == DoubleWritable.class) {
                                    tuple_values[ctr] = new DoubleWritable(Double.valueOf(data[ctr]));
                                } else if (types[ctr + 1] == LongWritable.class) {
                                    tuple_values[ctr] = new LongWritable(Long.valueOf(data[ctr]));
                                } else if (types[ctr + 1] == FloatWritable.class) {
                                    tuple_values[ctr] = new FloatWritable(Float.valueOf(data[ctr]));
                                } else {
                                    System.err.println("Unsupported Class: " + types[ctr + 1]);
                                    System.exit(1);
                                }
                                if (DataLoader.debug)
                                    System.out.println("tuple_values[" + ctr + "] = " + tuple_values[ctr]);
                            }
                            AbstractTuple tuple = (input_type.equals("uservisits")
                                    ? new UserVisitsTuple(tuple_values)
                                    : new RankingsTuple(tuple_values));
                            if (DataLoader.debug)
                                System.out.println("STORING TUPLE: " + tuple + " (DATA " + data + " | VALUE "
                                        + value + ")");
                            writer.append(new Text(key), tuple);
                        } catch (Exception ex) {
                            ex.printStackTrace();
                            System.err.println("Error[" + output_file + "]");
                            System.err.println("## Line:    " + lines);
                            System.err.println("## Content: " + line);
                        }
                    } else {
                        writer.append(new Text(key), new Text(value));
                    }
                }
                lines++;
                if (DataLoader.limit != null && lines >= DataLoader.limit)
                    break;
                if (DataLoader.debug && lines % 1000000 == 0)
                    System.out.println(
                            "\tWrote " + lines + " '" + input_type + "' records to '" + output_file + "'");
            } catch (Exception ex) {
                System.err.println("Error[" + output_file + "]");
                System.err.println("## Line:    " + lines);
                System.err.println("## Content: " + line);
                ex.printStackTrace();
                System.exit(1);
            }
        } // WHILE
    } catch (Exception ex) {
        ex.printStackTrace();
        System.exit(1);
    } finally {
        try {
            if (in != null)
                in.close();
            if (out != null)
                out.close();
            if (writer != null)
                writer.close();
        } catch (Exception ex) {
            ex.printStackTrace();
            System.exit(1);
        }
    }
    System.out.println("Wrote " + lines + " '" + input_type + "' records to '" + output_file + "'");
}

From source file:edu.ub.ahstfg.io.DocumentDistance.java

License:Open Source License

/**
 * Unparametrized constructor.// w w  w .ja  va2  s.co  m
 */
public DocumentDistance() {
    doc = new DocumentDescriptor();
    distance = new DoubleWritable(0.0);
    stub = new BooleanWritable(false);
}

From source file:edu.ub.ahstfg.io.DocumentDistance.java

License:Open Source License

/**
 * Constructor for stub documents./*from  w ww  .  ja v  a 2  s .c  o m*/
 * @param stub true to make a stub docuemnt.
 */
public DocumentDistance(boolean stub) {
    doc = new DocumentDescriptor();
    distance = new DoubleWritable(0.0);
    this.stub = new BooleanWritable(stub);
}

From source file:edu.ub.ahstfg.io.DocumentDistance.java

License:Open Source License

/**
 * Constructor specifying the document descriptor and the distance to centroid.
 * @param doc Document descriptor./*from w  ww.  j a  v a  2s  .co  m*/
 * @param distance Distance to centroid.
 */
public DocumentDistance(DocumentDescriptor doc, double distance) {
    this.doc = doc;
    this.distance = new DoubleWritable(distance);
    stub = new BooleanWritable(false);
}

From source file:edu.ub.ahstfg.kmeans.document.DocumentCentroid.java

License:Open Source License

@Override
public void write(DataOutput out) throws IOException {
    WritableConverter.shortArray2ArrayWritable(keywordVector).write(out);
    WritableConverter.shortArray2ArrayWritable(termVector).write(out);
    DoubleWritable dist = new DoubleWritable(distance);
    dist.write(out);//from  ww  w  .  j a va2s  .  c om
}

From source file:edu.uci.ics.hivesterix.serde.lazy.LazyDouble.java

License:Apache License

public LazyDouble(LazyDouble copy) {
    super(copy);
    data = new DoubleWritable(copy.data.get());
}

From source file:edu.uci.ics.pregelix.benchmark.io.TextSPInputFormat.java

License:Apache License

@Override
public TextVertexReader createVertexReader(InputSplit split, TaskAttemptContext context) throws IOException {
    return new TextVertexReaderFromEachLine() {
        StringTokenizer items;//ww  w  .  j  av  a  2s.  c om

        @Override
        protected VLongWritable getId(Text line) throws IOException {
            items = new StringTokenizer(line.toString());
            return new VLongWritable(Long.parseLong(items.nextToken()));
        }

        @Override
        protected DoubleWritable getValue(Text line) throws IOException {
            return null;
        }

        @Override
        protected Iterable<Edge<VLongWritable, DoubleWritable>> getEdges(Text line) throws IOException {
            List<Edge<VLongWritable, DoubleWritable>> edges = new ArrayList<Edge<VLongWritable, DoubleWritable>>();
            Map<VLongWritable, DoubleWritable> edgeMap = new HashMap<VLongWritable, DoubleWritable>();
            while (items.hasMoreTokens()) {
                edgeMap.put(new VLongWritable(Long.parseLong(items.nextToken())), null);
            }
            for (Entry<VLongWritable, DoubleWritable> entry : edgeMap.entrySet()) {
                MapMutableEdge<VLongWritable, DoubleWritable> edge = new MapMutableEdge<VLongWritable, DoubleWritable>();
                edge.setEntry(entry);
                edge.setValue(new DoubleWritable(1.0));
                edges.add(edge);
            }
            return edges;
        }

    };
}

From source file:edu.uci.ics.pregelix.benchmark.vertex.PageRankVertex.java

License:Apache License

@Override
public void compute(Iterable<DoubleWritable> messages) throws IOException {
    if (getSuperstep() >= 1) {
        float sum = 0;
        for (DoubleWritable message : messages) {
            sum += message.get();/*from w  w  w .  java2 s  . co m*/
        }
        getValue().set((0.15f / getTotalNumVertices()) + 0.85f * sum);
    }

    if (getSuperstep() < maxSuperStep) {
        sendMessageToAllEdges(new DoubleWritable(getValue().get() / getNumEdges()));
    } else {
        voteToHalt();
    }
}

From source file:edu.uci.ics.pregelix.benchmark.vertex.ShortestPathsVertex.java

License:Apache License

@Override
public void compute(Iterable<DoubleWritable> messages) throws IOException {
    if (getSuperstep() == 0) {
        setValue(new DoubleWritable(Double.MAX_VALUE));
    }//ww  w  . j av a 2s .c  om

    double minDist = isSource() ? 0d : Double.MAX_VALUE;
    for (DoubleWritable message : messages) {
        minDist = Math.min(minDist, message.get());
    }

    if (minDist < getValue().get()) {
        setValue(new DoubleWritable(minDist));
        for (Edge<VLongWritable, DoubleWritable> edge : getEdges()) {
            double distance = minDist + edge.getValue().get();
            sendMessage(edge.getTargetVertexId(), new DoubleWritable(distance));
        }
    }

    voteToHalt();
}