Example usage for org.apache.hadoop.io LongWritable set

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable set.

Prototype

public void set(long value) 

Source Link

Document

Set the value of this LongWritable.

Usage

From source file:com.aliyun.fs.oss.common.OssRecordReader.java

License:Apache License

public boolean next(LongWritable key, Text value) throws IOException {
    // We always read one extra line, which lies outside the upper
    // split limit i.e. (end - 1)
    while (getFilePosition() <= end) {
        key.set(pos);

        int newSize = in.readLine(value, maxLineLength, Math.max(maxBytesToConsume(pos), maxLineLength));
        if (newSize == 0) {
            return false;
        }// w  w  w  .  jav a  2s .  c om
        pos += newSize;
        if (newSize < maxLineLength) {
            return true;
        }

        // line too long. try again
        LOG.info("Skipped line of size " + newSize + " at pos " + (pos - newSize));
    }

    return false;
}

---

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test for input./*ww  w.  jav  a  2  s.c om*/
 * @throws Exception if failed
 */
@Test
public void input() throws Exception {
    final int count = 10000;
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class,
            Text.class)) {
        LongWritable k = new LongWritable();
        Text v = new Text();
        for (int i = 0; i < count; i++) {
            k.set(i);
            v.set("Hello, world at " + i);
            writer.append(k, v);
        }
    }

    try (ModelInput<StringOption> in = format.createInput(StringOption.class, fs, path, 0,
            fs.getFileStatus(path).getLen(), new Counter())) {
        StringOption value = new StringOption();
        for (int i = 0; i < count; i++) {
            String answer = "Hello, world at " + i;
            assertThat(answer, in.readTo(value), is(true));
            assertThat(value.getAsString(), is(answer));
        }
        assertThat("eof", in.readTo(value), is(false));
    }
}

---

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test for input./*  w w w  .j  ava 2 s  .  c  o m*/
 * @throws Exception if failed
 */
@Test
public void input_fragment() throws Exception {
    final int count = 30000;
    Random rand = new Random();
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class,
            Text.class)) {
        LongWritable k = new LongWritable();
        Text v = new Text();
        for (int i = 0; i < count; i++) {
            k.set(i);
            v.set("Hello, world at " + i);
            writer.append(k, v);
        }
    }

    long fileLen = fs.getFileStatus(path).getLen();
    StringOption value = new StringOption();
    for (int attempt = 0; attempt < 5; attempt++) {
        int index = 0;
        long offset = 0;
        while (offset < fileLen) {
            long length = SequenceFile.SYNC_INTERVAL * (rand.nextInt(10) + 2);
            length = Math.min(length, fileLen - offset);
            try (ModelInput<StringOption> in = format.createInput(StringOption.class, fs, path, offset, length,
                    new Counter())) {
                while (in.readTo(value)) {
                    String answer = "Hello, world at " + index;
                    assertThat(value.getAsString(), is(answer));
                    index++;
                }
                assertThat("eof", in.readTo(value), is(false));
            }
            offset += length;
        }
        assertThat(index, is(count));
    }
}

---

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test for input.//w  w w  .  j  av  a  2s . c  o  m
 * @throws Exception if failed
 */
@Test
public void input_largerecord() throws Exception {
    StringBuilder buf = new StringBuilder();
    for (int i = 0; i < 1000000; i++) {
        buf.append("Hello, world!");
    }
    Text record = new Text(buf.toString());

    final int count = 5;
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class,
            Text.class)) {
        LongWritable k = new LongWritable();
        Text v = new Text();
        for (int i = 0; i < count; i++) {
            k.set(i);
            v.set(record);
            writer.append(k, v);
        }
    }

    long fileLen = fs.getFileStatus(path).getLen();
    StringOption value = new StringOption();
    int index = 0;
    long offset = 0;
    while (offset < fileLen) {
        long length = SequenceFile.SYNC_INTERVAL * 2;
        length = Math.min(length, fileLen - offset);
        try (ModelInput<StringOption> in = format.createInput(StringOption.class, fs, path, offset, length,
                new Counter())) {
            while (in.readTo(value)) {
                assertThat(value.get(), is(record));
                index++;
            }
            assertThat("eof", in.readTo(value), is(false));
        }
        offset += length;
    }
    assertThat(index, is(count));
}

---

From source file:com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest.java

License:Apache License

/**
 * Uses a large sequence file with original API.
 * @throws Exception if failed/*from   w w w . j  av  a2  s.c  om*/
 */
@Test
public void original_large() throws Exception {
    Path path = new Path("large");

    LongWritable key = new LongWritable();
    LongWritable value = new LongWritable();

    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, key.getClass(),
            value.getClass())) {
        for (long i = 0; i < 300000; i++) {
            key.set(i);
            value.set(i + 1);
            writer.append(key, value);
        }
    }

    try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf)) {
        for (long i = 0; i < 300000; i++) {
            assertThat(reader.next(key, value), is(true));
            assertThat(key.get(), is(i));
            assertThat(value.get(), is(i + 1));
        }
        assertThat(reader.next(key, value), is(false));
    }
}

---

From source file:com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest.java

License:Apache License

/**
 * Reads a large sequence file./* ww  w  . j  a  v  a2  s .  c  om*/
 * @throws Exception if failed
 */
@Test
public void read_large() throws Exception {
    Path path = new Path("large");

    LongWritable key = new LongWritable();
    LongWritable value = new LongWritable();

    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, key.getClass(),
            value.getClass())) {
        for (long i = 0; i < 300000; i++) {
            key.set(i);
            value.set(i + 1);
            writer.append(key, value);
        }
    }

    FileStatus status = fs.getFileStatus(path);
    try (InputStream in = new FileInputStream(fs.pathToFile(path));
            SequenceFile.Reader reader = SequenceFileUtil.openReader(in, status, conf)) {
        for (long i = 0; i < 300000; i++) {
            assertThat(reader.next(key, value), is(true));
            assertThat(key.get(), is(i));
            assertThat(value.get(), is(i + 1));
        }
        assertThat(reader.next(key, value), is(false));
    }
}

---

From source file:com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest.java

License:Apache License

/**
 * Creates a large sequence file./* w  w  w. j  a  va  2 s .  com*/
 * @throws Exception if failed
 */
@Test
public void write_large() throws Exception {
    Path path = new Path("testing");

    LongWritable key = new LongWritable();
    LongWritable value = new LongWritable();
    try (OutputStream out = new FileOutputStream(fs.pathToFile(path));
            SequenceFile.Writer writer = SequenceFileUtil.openWriter(new BufferedOutputStream(out), conf,
                    key.getClass(), value.getClass(), null)) {
        for (long i = 0; i < 300000; i++) {
            key.set(i);
            value.set(i + 1);
            writer.append(key, value);
        }
    }

    try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf)) {
        for (long i = 0; i < 300000; i++) {
            assertThat(reader.next(key, value), is(true));
            assertThat(key.get(), is(i));
            assertThat(value.get(), is(i + 1));
        }
        assertThat(reader.next(key, value), is(false));
    }
}

---

From source file:com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest.java

License:Apache License

/**
 * Creates a compressed sequence file./*from w  w  w  .  j  av  a2 s.c o m*/
 * @throws Exception if failed
 */
@Test
public void write_compressed() throws Exception {
    DefaultCodec codec = new DefaultCodec();
    codec.setConf(conf);

    Path path = new Path("testing");

    LongWritable key = new LongWritable();
    LongWritable value = new LongWritable();
    try (OutputStream out = new FileOutputStream(fs.pathToFile(path));
            SequenceFile.Writer writer = SequenceFileUtil.openWriter(new BufferedOutputStream(out), conf,
                    key.getClass(), value.getClass(), codec);) {
        for (long i = 0; i < 300000; i++) {
            key.set(i);
            value.set(i + 1);
            writer.append(key, value);
        }
    }

    try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf)) {
        for (long i = 0; i < 300000; i++) {
            assertThat(reader.next(key, value), is(true));
            assertThat(key.get(), is(i));
            assertThat(value.get(), is(i + 1));
        }
        assertThat(reader.next(key, value), is(false));
    }
}

---

From source file:com.caseystella.analytics.distribution.RotationTest.java

License:Apache License

public static DataPoint nextDataPoint(Random r, LongWritable ts, long delta, List<DataPoint> points) {
    double val = r.nextDouble() * 1000;
    DataPoint dp = (new DataPoint(ts.get(), val, null, "foo"));
    if (points != null) {
        points.add(dp);/*from   w  w w .  java 2s  .  c  o  m*/
    }
    ts.set(ts.get() + delta);
    return dp;
}

---

From source file:com.caseystella.analytics.outlier.streaming.mad.SketchyMovingMADIntegrationTest.java

License:Apache License

@Test
public void runAccuracyBenchmark() throws IOException {
    Map<String, List<String>> benchmarks = JSONUtil.INSTANCE.load(
            new FileInputStream(new File(new File(benchmarkRoot), "combined_labels.json")),
            new TypeReference<Map<String, List<String>>>() {
            });//  ww w.ja  v a  2  s  . co m
    Assert.assertTrue(benchmarks.size() > 0);
    Map<ConfusionMatrix.ConfusionEntry, Long> overallConfusionMatrix = new HashMap<>();
    DescriptiveStatistics globalExpectedScores = new DescriptiveStatistics();
    long total = 0;
    for (Map.Entry<String, List<String>> kv : benchmarks.entrySet()) {
        File dataFile = new File(new File(benchmarkRoot), kv.getKey());
        File plotFile = new File(new File(benchmarkRoot), kv.getKey() + ".dat");
        Assert.assertTrue(dataFile.exists());
        Set<Long> expectedOutliers = Sets.newHashSet(Iterables.transform(kv.getValue(), STR_TO_TS));
        OutlierRunner runner = new OutlierRunner(outlierConfig, extractorConfigStr);
        final LongWritable numObservations = new LongWritable(0);
        final LongWritable lastTimestamp = new LongWritable(Long.MIN_VALUE);
        final DescriptiveStatistics timeDiffStats = new DescriptiveStatistics();
        final Map<Long, Outlier> outlierMap = new HashMap<>();
        final PrintWriter pw = new PrintWriter(plotFile);
        List<Outlier> outliers = runner.run(dataFile, 1, EnumSet.of(Severity.SEVERE_OUTLIER),
                new Function<Map.Entry<DataPoint, Outlier>, Void>() {
                    @Nullable
                    @Override
                    public Void apply(@Nullable Map.Entry<DataPoint, Outlier> kv) {
                        DataPoint dataPoint = kv.getKey();
                        Outlier outlier = kv.getValue();
                        pw.println(dataPoint.getTimestamp() + " " + outlier.getDataPoint().getValue() + " "
                                + ((outlier.getSeverity() == Severity.SEVERE_OUTLIER) ? "outlier" : "normal"));
                        outlierMap.put(dataPoint.getTimestamp(), outlier);
                        numObservations.set(numObservations.get() + 1);
                        if (lastTimestamp.get() != Long.MIN_VALUE) {
                            timeDiffStats.addValue(dataPoint.getTimestamp() - lastTimestamp.get());
                        }
                        lastTimestamp.set(dataPoint.getTimestamp());
                        return null;
                    }
                });
        pw.close();
        total += numObservations.get();
        Set<Long> calculatedOutliers = Sets
                .newHashSet(Iterables.transform(outliers, OutlierRunner.OUTLIER_TO_TS));
        double stdDevDiff = Math.sqrt(timeDiffStats.getVariance());
        System.out.println("Running data from " + kv.getKey() + " - E[time delta]: "
                + ConfusionMatrix.timeConversion((long) timeDiffStats.getMean()) + ", StdDev[time delta]: "
                + ConfusionMatrix.timeConversion((long) stdDevDiff) + " mean: " + runner.getMean());
        Map<ConfusionMatrix.ConfusionEntry, Long> confusionMatrix = ConfusionMatrix.getConfusionMatrix(
                expectedOutliers, calculatedOutliers, numObservations, (long) timeDiffStats.getMean(), 3 //stdDevDiff > 30000?0:3
                , outlierMap, globalExpectedScores);

        ConfusionMatrix.printConfusionMatrix(confusionMatrix);
        overallConfusionMatrix = ConfusionMatrix.merge(overallConfusionMatrix, confusionMatrix);
    }
    System.out.println("Really ran " + total);
    ConfusionMatrix.printConfusionMatrix(overallConfusionMatrix);
    ConfusionMatrix.printStats("Global Expected Outlier Scores", globalExpectedScores);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• »