Example usage for org.apache.hadoop.io IntWritable IntWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io IntWritable IntWritable.

Prototype

public IntWritable() 

Source Link

Usage

From source file:cc.slda.DisplayTopic.java

License:Apache License

@SuppressWarnings("unchecked")
public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(Settings.HELP_OPTION, false, "print the help message");
    options.addOption(OptionBuilder.withArgName(Settings.PATH_INDICATOR).hasArg()
            .withDescription("input beta file").create(Settings.INPUT_OPTION));
    options.addOption(OptionBuilder.withArgName(Settings.PATH_INDICATOR).hasArg()
            .withDescription("term index file").create(ParseCorpus.INDEX));
    options.addOption(OptionBuilder.withArgName(Settings.INTEGER_INDICATOR).hasArg()
            .withDescription("display top terms only (default - 10)").create(TOP_DISPLAY_OPTION));

    String betaString = null;//from   w w w .j a va2  s .co m
    String indexString = null;
    int topDisplay = TOP_DISPLAY;

    CommandLineParser parser = new GnuParser();
    HelpFormatter formatter = new HelpFormatter();
    try {
        CommandLine line = parser.parse(options, args);

        if (line.hasOption(Settings.HELP_OPTION)) {
            formatter.printHelp(ParseCorpus.class.getName(), options);
            System.exit(0);
        }

        if (line.hasOption(Settings.INPUT_OPTION)) {
            betaString = line.getOptionValue(Settings.INPUT_OPTION);
        } else {
            throw new ParseException("Parsing failed due to " + Settings.INPUT_OPTION + " not initialized...");
        }

        if (line.hasOption(ParseCorpus.INDEX)) {
            indexString = line.getOptionValue(ParseCorpus.INDEX);
        } else {
            throw new ParseException("Parsing failed due to " + ParseCorpus.INDEX + " not initialized...");
        }

        if (line.hasOption(TOP_DISPLAY_OPTION)) {
            topDisplay = Integer.parseInt(line.getOptionValue(TOP_DISPLAY_OPTION));
        }
    } catch (ParseException pe) {
        System.err.println(pe.getMessage());
        formatter.printHelp(ParseCorpus.class.getName(), options);
        System.exit(0);
    } catch (NumberFormatException nfe) {
        System.err.println(nfe.getMessage());
        System.exit(0);
    }

    JobConf conf = new JobConf(DisplayTopic.class);
    FileSystem fs = FileSystem.get(conf);

    Path indexPath = new Path(indexString);
    Preconditions.checkArgument(fs.exists(indexPath) && fs.isFile(indexPath), "Invalid index path...");

    Path betaPath = new Path(betaString);
    Preconditions.checkArgument(fs.exists(betaPath) && fs.isFile(betaPath), "Invalid beta path...");

    SequenceFile.Reader sequenceFileReader = null;
    try {
        IntWritable intWritable = new IntWritable();
        Text text = new Text();
        Map<Integer, String> termIndex = new HashMap<Integer, String>();
        sequenceFileReader = new SequenceFile.Reader(fs, indexPath, conf);
        while (sequenceFileReader.next(intWritable, text)) {
            termIndex.put(intWritable.get(), text.toString());
        }

        PairOfIntFloat pairOfIntFloat = new PairOfIntFloat();
        // HMapIFW hmap = new HMapIFW();
        HMapIDW hmap = new HMapIDW();
        TreeMap<Double, Integer> treeMap = new TreeMap<Double, Integer>();
        sequenceFileReader = new SequenceFile.Reader(fs, betaPath, conf);
        while (sequenceFileReader.next(pairOfIntFloat, hmap)) {
            treeMap.clear();

            System.out.println("==============================");
            System.out.println(
                    "Top ranked " + topDisplay + " terms for Topic " + pairOfIntFloat.getLeftElement());
            System.out.println("==============================");

            Iterator<Integer> itr1 = hmap.keySet().iterator();
            int temp1 = 0;
            while (itr1.hasNext()) {
                temp1 = itr1.next();
                treeMap.put(-hmap.get(temp1), temp1);
                if (treeMap.size() > topDisplay) {
                    treeMap.remove(treeMap.lastKey());
                }
            }

            Iterator<Double> itr2 = treeMap.keySet().iterator();
            double temp2 = 0;
            while (itr2.hasNext()) {
                temp2 = itr2.next();
                if (termIndex.containsKey(treeMap.get(temp2))) {
                    System.out.println(termIndex.get(treeMap.get(temp2)) + "\t\t" + -temp2);
                } else {
                    System.out.println("How embarrassing! Term index not found...");
                }
            }
        }
    } finally {
        IOUtils.closeStream(sequenceFileReader);
    }

    return 0;
}

---

From source file:clustering.io.tuple.IntIntTupleWritable.java

License:Apache License

public IntIntTupleWritable() {
    this.left = new IntWritable();
    this.right = new IntWritable();
}

---

From source file:clustering.io.tuple.IntIntTupleWritable.java

License:Apache License

public IntIntTupleWritable(int left, int right) {
    if (this.left == null) {
        this.left = new IntWritable();
    }/* www  . j a v  a  2s. c  om*/
    if (this.right == null) {
        this.right = new IntWritable();
    }
    this.left.set(left);
    this.right.set(right);
}

---

From source file:clustering.link_back.io.Step1KeyWritable.java

License:Apache License

/**
 * Join key means the group id in mst,/*w  w  w  . j  a  va  2s. c o m*/
 * and tag is the secondary sort field,
 * 1 = group_id, 2 = content.
 */
public Step1KeyWritable() {
    this.joinKey = new IntWritable();
    this.tag = new IntWritable();
}

---

From source file:clustering.link_back.io.Step2KeyWritable.java

License:Apache License

/**
 * joinKey means entry_id@@g_no,//from  w  w  w. j  ava 2  s  .com
 * and tag is the secondary sort field,
 * 1 = cluster_id, 2 = content
 */
public Step2KeyWritable() {
    this.joinKey = new Text();
    this.tag = new IntWritable();
}

---

From source file:cn.com.warlock.SequenceFilesTest.java

License:Apache License

public static void main(String[] args) throws IOException {
    String hdfsUri = "hdfs://hlg-2p238-fandongsheng:8020";
    String pathStr = "/tmp/example/seq1";
    String compressType = "1";

    // ??windows?
    // System.setProperty("hadoop.home.dir", "E:\\tools");

    Configuration conf = new Configuration();
    conf.set("fs.defaultFS", hdfsUri);
    Path path = new Path(pathStr);

    IntWritable key = new IntWritable();
    Text value = new Text();
    SequenceFile.Writer writer = null;
    try {/*ww w .j a  v a2s .  co m*/
        SequenceFile.Writer.Option pathOpt = SequenceFile.Writer.file(path);
        SequenceFile.Writer.Option keyClassOpt = SequenceFile.Writer.keyClass(key.getClass());
        SequenceFile.Writer.Option valueClassOpt = SequenceFile.Writer.valueClass(value.getClass());
        SequenceFile.Writer.Option compressionOpt = null;

        // compress type
        if (compressType.equals("1")) {
            System.out.println("compress none");
            compressionOpt = SequenceFile.Writer.compression(CompressionType.NONE);
        } else if (compressType.equals("2")) {
            System.out.println("compress record");
            compressionOpt = SequenceFile.Writer.compression(CompressionType.RECORD);
        } else if (compressType.equals("3")) {
            System.out.println("compress block");
            compressionOpt = SequenceFile.Writer.compression(CompressionType.BLOCK);
        } else {
            System.out.println("Default : compress none");
            compressionOpt = SequenceFile.Writer.compression(CompressionType.NONE);
        }

        writer = SequenceFile.createWriter(conf, pathOpt, keyClassOpt, valueClassOpt, compressionOpt);

        for (int i = 0; i < 100; i++) {
            key.set(100 - i);
            value.set(DATA[i % DATA.length]);
            System.out.printf("[%s]\t%s\t%s\n", writer.getLength(), key, value);
            writer.append(key, value);

        }
    } finally {
        IOUtils.closeStream(writer);
    }
}

---

From source file:cn.lhfei.hadoop.ch04.MapFileWriteDemo.java

License:Apache License

public static void main(String[] args) {
    String uri = args[0];//from  w  ww .  ja  v  a2  s  .  co m
    Configuration conf = new Configuration();
    FileSystem fs = null;

    IntWritable key = new IntWritable();
    Text value = new Text();
    MapFile.Writer writer = null;
    try {
        fs = FileSystem.get(URI.create(uri), conf);
        /*writer = new MapFile.Writer(conf, fs, uri, key.getClass(),
              value.getClass());*/

        writer = new MapFile.Writer(conf, new Path(uri), Writer.keyClass(key.getClass()),
                Writer.valueClass(value.getClass()));

        for (int i = 0; i < 1024; i++) {
            key.set(i + 1);
            value.set(DATA[i % DATA.length]);
            writer.append(key, value);
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        IOUtils.closeStream(writer);
    }
}

---

From source file:cn.lhfei.hadoop.ch04.SequenceFileWriteDemo.java

License:Apache License

public static void main(String[] args) {

    String uri = args[0];/*from  w w  w .  ja  va 2 s  . co m*/
    Configuration conf = new Configuration();
    FileSystem fs = null;
    SequenceFile.Writer writer = null;

    try {
        fs = FileSystem.get(URI.create(uri), conf);
        Path path = new Path(uri);

        IntWritable key = new IntWritable();
        Text value = new Text();

        //writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());

        writer = SequenceFile.createWriter(conf, Writer.keyClass(key.getClass()),
                writer.valueClass(value.getClass()));

        for (int i = 0; i < 100; i++) {
            key.set(100 - i);
            value.set(DATA[i % DATA.length]);
            System.out.printf("[%s]\t%s\t%s\n", writer.getLength(), key, value);
            writer.append(key, value);
        }

    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        IOUtils.closeStream(writer);
    }

}

---

From source file:co.nubetech.hiho.testdata.SequenceFileWriteDemo.java

License:Apache License

public static void main(String[] args) throws IOException {
    String uri = "input2.seq";
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(URI.create(uri), conf);
    Path path = new Path(uri);
    IntWritable key = new IntWritable();
    Text value = new Text();
    SequenceFile.Writer writer = null;
    try {/*  w  w  w.  j  ava 2 s  .c o m*/
        writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
        for (int i = 0; i < 2; i++) {
            key.set(2 - i);
            value.set(DATA[i % DATA.length]);
            System.out.printf("[%s]\t%s\t%s\n", writer.getLength(), key, value);
            writer.append(key, value);
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        IOUtils.closeStream(writer);
    }
}

---

From source file:com.chimpler.example.eigenface.Helper.java

License:Apache License

public static double[][] readMatrixSequenceFile(String fileName) throws Exception {
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(configuration);
    Reader matrixReader = new SequenceFile.Reader(fs, new Path(fileName), configuration);

    List<double[]> rows = new ArrayList<double[]>();
    IntWritable key = new IntWritable();
    VectorWritable value = new VectorWritable();
    while (matrixReader.next(key, value)) {
        Vector vector = value.get();
        double[] row = new double[vector.size()];
        for (int i = 0; i < vector.getNumNondefaultElements(); i++) {
            Element element = vector.getElement(i);
            row[element.index()] = element.get();
        }//from   ww w .j a  va  2 s  .  c  o m
        rows.add(row);
    }
    return rows.toArray(new double[rows.size()][]);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• »