Example usage for org.apache.hadoop.io LongWritable set

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable set.

Prototype

public void set(long value) 

Source Link

Document

Set the value of this LongWritable.

Usage

From source file:com.yolodata.tbana.hadoop.mapred.csv.CSVLineRecordReader.java

License:Open Source License

@Override
public boolean next(LongWritable key, List<Text> value) throws IOException {
    if (key == null) {
        key = new LongWritable();
    }/*from   w  w  w.j  av a2  s.c o m*/
    key.set(pos);

    if (value == null) {
        value = new ArrayListTextWritable();
    }
    while (true) {
        if (pos >= end)
            return false;
        int newSize = 0;
        newSize = reader.readLine(value);
        pos += newSize;
        if (newSize == 0) {
            if (isZipFile) {
                ZipInputStream zis = (ZipInputStream) is;
                if (zis.getNextEntry() != null) {
                    is = zis;
                    reader = new CSVReader(new BufferedReader(new InputStreamReader(is)));
                    continue;
                }
            }
            key = null;
            value = null;
            return false;
        } else {
            removeNewLineOnLastColumn(value);
            return true;
        }
    }
}

---

From source file:com.yolodata.tbana.hadoop.mapred.shuttl.ShuttlCSVRecordReader.java

License:Open Source License

@Override
public boolean next(LongWritable key, ArrayListTextWritable value) throws IOException {
    if (pos == end) {
        return false;
    }//  www.ja  va 2 s.  co m
    if (key == null) {
        key = new LongWritable();
    }
    key.set(startKey + pos);

    if (value == null) {
        value = new ArrayListTextWritable();
    }

    int bytesRead = reader.readLine(value);

    pos += bytesRead;
    if (bytesRead == 0) {
        key = null;
        value = null;
        return false;
    }

    removeNewLineOnLastColumn(value);
    return true;

}

---

From source file:com.yolodata.tbana.hadoop.mapred.splunk.recordreader.SplunkRecordReader.java

License:Open Source License

@Override
public boolean next(LongWritable key, ArrayListTextWritable value) throws IOException {

    reader = new CSVReader(in);
    if (key == null)
        key = createKey();//from w w w.  j  a v a2 s .  c  o  m
    if (value == null)
        value = createValue();

    int bytesRead = reader.readLine(value);

    if (bytesRead == 0) {
        key = null;
        value = null;
        return false;
    }

    key.set(currentPosition++);
    return true;
}

---

From source file:edu.cuhk.hccl.SequenceFileWriter.java

License:Apache License

private static void createSeqFile(File[] files, String seqName) {
    Configuration conf = new Configuration();
    LongWritable key = new LongWritable();
    Text value = new Text();

    SequenceFile.Writer writer = null;

    try {/*from w w  w.j a v a  2  s. c o m*/
        FileSystem fs = FileSystem.get(URI.create(seqName), conf);
        writer = SequenceFile.createWriter(fs, conf, new Path(seqName), key.getClass(), value.getClass());

        for (File file : files) {
            //System.out.printf("Processing file: %s \n", file.getPath());
            key.set(Integer.parseInt(file.getName().split("_")[1]));
            value.set(FileUtils.readFileToString(file));
            writer.append(key, value);
        }
        System.out.printf("[INFO] The sequence file %s has been created for %d files! \n", seqName,
                files.length);

    } catch (IOException e) {
        System.out.println(e.getMessage());
    } finally {
        IOUtils.closeStream(writer);
    }
}

---

From source file:edu.umd.cloud9.demo.DemoPackJSON.java

License:Apache License

/**
 * Runs the demo./* w w  w . j av a2 s.c  o m*/
 */
public static void main(String[] args) throws IOException, JSONException {
    if (args.length != 2) {
        System.out.println("usage: [input] [output]");
        System.exit(-1);
    }

    String infile = args[0];
    String outfile = args[1];

    sLogger.info("input: " + infile);
    sLogger.info("output: " + outfile);

    Configuration conf = new JobConf();
    FileSystem fs = FileSystem.get(conf);
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, new Path(outfile), LongWritable.class,
            JSONObjectWritable.class);

    // read in raw text records, line separated
    BufferedReader data = new BufferedReader(new InputStreamReader(new FileInputStream(infile)));

    // the key
    LongWritable l = new LongWritable();
    JSONObjectWritable json = new JSONObjectWritable();
    long cnt = 0;

    String line;
    while ((line = data.readLine()) != null) {
        // write the record
        json.clear();
        json.put("text", line);
        l.set(cnt);
        writer.append(l, json);

        cnt++;
    }

    data.close();
    writer.close();

    sLogger.info("Wrote " + cnt + " records.");
}

---

From source file:edu.umd.cloud9.demo.DemoPackTuples1.java

License:Apache License

/**
 * Runs the demo./*from  ww w  . j  a  v  a 2  s  .  c  o  m*/
 */
public static void main(String[] args) throws IOException {
    if (args.length != 2) {
        System.out.println("usage: [input] [output]");
        System.exit(-1);
    }

    String infile = args[0];
    String outfile = args[1];

    sLogger.info("input: " + infile);
    sLogger.info("output: " + outfile);

    Configuration conf = new JobConf();
    FileSystem fs = FileSystem.get(conf);
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, new Path(outfile), LongWritable.class,
            Tuple.class);

    // read in raw text records, line separated
    BufferedReader data = new BufferedReader(new InputStreamReader(new FileInputStream(infile)));

    // the key
    LongWritable l = new LongWritable();
    long cnt = 0;

    String line;
    while ((line = data.readLine()) != null) {
        // write the record
        tuple.set(0, line);
        l.set(cnt);
        writer.append(l, tuple);

        cnt++;
    }

    data.close();
    writer.close();

    sLogger.info("Wrote " + cnt + " records.");
}

---

From source file:edu.umd.cloud9.demo.DemoPackTuples2.java

License:Apache License

/**
 * Runs the demo.//  www.j a  va 2s. c o m
 */
public static void main(String[] args) throws IOException {
    if (args.length != 2) {
        System.out.println("usage: [input] [output]");
        System.exit(-1);
    }

    String infile = args[0];
    String outfile = args[1];

    sLogger.info("input: " + infile);
    sLogger.info("output: " + outfile);

    Configuration conf = new JobConf();
    FileSystem fs = FileSystem.get(conf);
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, new Path(outfile), LongWritable.class,
            Tuple.class);

    // read in raw text records, line separated
    BufferedReader data = new BufferedReader(new InputStreamReader(new FileInputStream(infile)));

    LongWritable l = new LongWritable();
    long cnt = 0;

    String line;
    while ((line = data.readLine()) != null) {
        ArrayListWritable<Text> tokens = new ArrayListWritable<Text>();
        StringTokenizer itr = new StringTokenizer(line);
        while (itr.hasMoreTokens()) {
            tokens.add(new Text(itr.nextToken()));
        }

        // write the record
        tuple.set("length", line.length());
        tuple.set("tokens", tokens);
        l.set(cnt);
        writer.append(l, tuple);

        cnt++;
    }

    data.close();
    writer.close();

    sLogger.info("Wrote " + cnt + " records.");
}

---

From source file:edu.umd.cloud9.example.simple.DemoPackJson.java

License:Apache License

/**
 * Runs the demo./*  ww w. ja v a 2  s  .  com*/
 */
public static void main(String[] args) throws IOException {
    if (args.length != 2) {
        System.out.println("usage: [input] [output]");
        System.exit(-1);
    }

    String infile = args[0];
    String outfile = args[1];

    LOG.info("input: " + infile);
    LOG.info("output: " + outfile);

    Configuration conf = new Configuration();
    SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(new Path(outfile)),
            SequenceFile.Writer.keyClass(LongWritable.class),
            SequenceFile.Writer.valueClass(JsonWritable.class));

    // Read in raw text records, line separated.
    BufferedReader data = new BufferedReader(new InputStreamReader(new FileInputStream(infile)));

    LongWritable key = new LongWritable();
    JsonWritable json = new JsonWritable();
    long cnt = 0;

    String line;
    while ((line = data.readLine()) != null) {
        json.getJsonObject().addProperty("text", line);
        key.set(cnt);
        writer.append(key, json);

        cnt++;
    }

    data.close();
    writer.close();

    LOG.info("Wrote " + cnt + " records.");
}

---

From source file:edu.umd.cloud9.example.simple.DemoPackTuples2.java

License:Apache License

/**
 * Runs the demo./* w  w  w.j  a va 2  s  .c  om*/
 */
public static void main(String[] args) throws IOException {
    if (args.length != 2) {
        System.out.println("usage: [input] [output]");
        System.exit(-1);
    }

    String infile = args[0];
    String outfile = args[1];

    LOG.info("input: " + infile);
    LOG.info("output: " + outfile);

    Configuration conf = new Configuration();
    SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(new Path(outfile)),
            SequenceFile.Writer.keyClass(LongWritable.class),
            SequenceFile.Writer.valueClass(BinSedesTuple.class));

    // Read in raw text records, line separated.
    BufferedReader data = new BufferedReader(new InputStreamReader(new FileInputStream(infile)));

    LongWritable l = new LongWritable();
    long cnt = 0;

    String line;
    while ((line = data.readLine()) != null) {
        Tuple tuple = TUPLE_FACTORY.newTuple();
        tuple.append(new Integer(line.length()));

        StringTokenizer itr = new StringTokenizer(line);
        while (itr.hasMoreTokens()) {
            tuple.append(itr.nextToken());
        }

        l.set(cnt);
        writer.append(l, tuple);

        cnt++;
    }

    data.close();
    writer.close();

    LOG.info("Wrote " + cnt + " records.");
}

---

From source file:edu.umd.cloud9.io.PackTextFile.java

License:Apache License

public static void main(String[] args) throws IOException {

    if (args.length < 2) {
        System.err.println("args: [input-file] [output-file]");
        System.exit(-1);/* w ww .j  a va2 s  . co m*/
    }

    String inFile = args[0];
    String outFile = args[1];

    Text text = new Text();
    LongWritable lw = new LongWritable();
    long l = 0;

    JobConf config = new JobConf();

    SequenceFile.Writer writer = SequenceFile.createWriter(FileSystem.get(config), config, new Path(outFile),
            LongWritable.class, Text.class);

    BufferedReader reader = new BufferedReader(new FileReader(new File(inFile)));

    String line = "";
    while ((line = reader.readLine()) != null) {
        lw.set(l);
        text.set(line);
        writer.append(lw, text);
        l++;
    }

    reader.close();
    writer.close();

    System.out.println("Wrote a total of " + l + " records");
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• »