Example usage for org.apache.hadoop.io LongWritable LongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable LongWritable.

Prototype

public LongWritable(long value) 

Source Link

Usage

From source file:com.justgiving.raven.kissmetrics.jsonenricher.KissmetricsJsonToEnrichedJsonMapperTest.java

License:Open Source License

@Test
public void mapper_JsonEscapedRowsURL_parsedrow() throws IOException, JSONException {
    jsonRowbuilder.setValue("url",
            "https://www.justgiving.com/BelCenMission/donate/?utm_source=website_cid247466&utm_medium=buttons&utm_content=BelCenMission&utm_campaign=donate_whiteC:\\\\Users\\\\gordonj\\\\Documents\\\\2011-2012");
    mapDriver.withInput(new LongWritable(1), new Text(jsonRowbuilder.toString()));

    jsonRowbuilder.setValue("event_timedate", "2014-04-15 16:57:33").setValue("filename", "somefile")
            .setValue("event", jsonRowbuilder.getValue("_n")).setValue("user_email", "justgiving@gmail.com")
            .setValue("user_email_back", "justgiving@gmail.com")
            .setValue("url",
                    "https://www.justgiving.com/BelCenMission/donate/?utm_source=website_cid247466&utm_medium=buttons&utm_content=BelCenMission&utm_campaign=donate_whiteC:\\\\Users\\\\gordonj\\\\Documents\\\\2011-2012")
            .setValue("user_km_id", "3lwlxqlulqe24q/jl4aqlibrtte=").setValue("km_timestamp", "1397577453")
            .setValue("bucket", "somefile");
    String expectedJSON = jsonRowbuilder.toString();

    List<Pair<Text, Text>> output = mapDriver.run();
    String actualJSON = output.get(0).getSecond().toString();

    JSONAssert.assertEquals(expectedJSON, actualJSON, true);
}

---

From source file:com.justgiving.raven.kissmetrics.jsonenricher.KissmetricsJsonToEnrichedJsonMapperTest.java

License:Open Source License

@Test
public void mapper_JsonEscapedRowsInUserAgent_parsedrow() throws IOException, JSONException {
    jsonRowbuilder.setValue("user_agent",
            "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; Tablet PC 2.0; .NET CLR 1.1.4322; BRI/2; Burton Primary School\\\\; rv:11.0) like Gecko");
    mapDriver.withInput(new LongWritable(1), new Text(jsonRowbuilder.toString()));

    jsonRowbuilder.setValue("event_timedate", "2014-04-15 16:57:33").setValue("filename", "somefile")
            .setValue("event", jsonRowbuilder.getValue("_n")).setValue("user_email", "justgiving@gmail.com")
            .setValue("user_email_back", "justgiving@gmail.com")
            .setValue("user_agent",
                    "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; Tablet PC 2.0; .NET CLR 1.1.4322; BRI/2; Burton Primary School\\\\; rv:11.0) like Gecko")
            .setValue("user_km_id", "3lwlxqlulqe24q/jl4aqlibrtte=").setValue("km_timestamp", "1397577453")
            .setValue("bucket", "somefile");
    String expectedJSON = jsonRowbuilder.toString();

    List<Pair<Text, Text>> output = mapDriver.run();
    String actualJSON = output.get(0).getSecond().toString();

    JSONAssert.assertEquals(expectedJSON, actualJSON, true);
}

---

From source file:com.justgiving.raven.kissmetrics.jsonenricher.KissmetricsJsonToEnrichedJsonMapReducerTest.java

License:Open Source License

@Test
public void mapReducer_keyAndjson_enrichedJson() throws IOException, JSONException {
    List<Text> values = new ArrayList<Text>();
    values.add(new Text(rowbuilder.toString()));

    //values.add(new IntWritable(1));
    mapReduceDriver.withInput(new LongWritable(1), new Text(rowbuilder.toString()));

    rowbuilder.setValue("event_timedate", "2014-04-15 16:57:33").setValue("filename", "somefile")
            .setValue("event", rowbuilder.getValue("_n")).setValue("user_email", "justgiving@gmail.com")
            .setValue("user_email_back", "justgiving@gmail.com")
            .setValue("user_km_id", "3lwlxqlulqe24q/jl4aqlibrtte=").setValue("km_timestamp", "1397577453")
            .setValue("bucket", "somefile");
    String expectedJSON = rowbuilder.toString();

    List<Pair<Text, NullWritable>> output = mapReduceDriver.run();
    String actualJSON = output.get(0).getFirst().toString();

    JSONAssert.assertEquals(expectedJSON, actualJSON, true);

}

---

From source file:com.justgiving.raven.kissmetrics.schema.KissmetricsJsonToSchemaMapperTest.java

License:Open Source License

@Test
public void testMapper_WithOneValidjsonRow__OneCountOfProperties() throws IOException {
    mapDriver.withInput(new LongWritable(1), new Text(
            "{\"_n\":\"viewed signup\",\"_p\":\"bob@bob.com\",\"useragent\":\"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0\",\"_t\":1397577453}"));
    mapDriver.withOutput(new Text("_n"), new Text("1\t13"));
    mapDriver.withOutput(new Text("_p"), new Text("1\t11"));
    mapDriver.withOutput(new Text("useragent"), new Text("1\t72"));
    mapDriver.withOutput(new Text("_t"), new Text("1\t10"));
    mapDriver.runTest(false);//  w w  w  . j  a v  a 2  s  . c  o m
}

---

From source file:com.justgiving.raven.kissmetrics.schema.KissmetricsJsonToSchemaMapperTest.java

License:Open Source License

@Test
public void testMapper_WithOneValidjsonRowUrl__OneCountOfProperties() throws IOException {
    mapDriver.withInput(new LongWritable(1), new Text(
            "{\"_n\":\"viewed signup\",\"_p\":\"bob@bob.com\",\"url\":\"www.bob.com\",\"useragent\":\"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0\",\"_t\":1397577453}"));
    mapDriver.withOutput(new Text("_n"), new Text("1\t13"));
    mapDriver.withOutput(new Text("_p"), new Text("1\t11"));
    mapDriver.withOutput(new Text("useragent"), new Text("1\t72"));
    mapDriver.withOutput(new Text("_t"), new Text("1\t10"));
    mapDriver.withOutput(new Text("url"), new Text("1\t11"));
    mapDriver.runTest(false);//ww w.j  ava 2  s .  co m

}

---

From source file:com.justgiving.raven.kissmetrics.schema.KissmetricsJsonToSchematMapReduceTest.java

License:Open Source License

@Test
public void testMapReduce_OneValidRow_OneCountEachProperty() throws IOException {
    mapReduceDriver.withInput(new LongWritable(1), new Text(
            "{\"_n\":\"viewed signup\",\"_p\":\"bob@bob.com\",\"useragent\":\"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0\",\"_t\":1397577453}"));
    mapReduceDriver.withOutput(new Text("_n"), new Text("1\t13"));
    mapReduceDriver.withOutput(new Text("_p"), new Text("1\t11"));
    mapReduceDriver.withOutput(new Text("useragent"), new Text("1\t72"));
    mapReduceDriver.withOutput(new Text("_t"), new Text("1\t10"));
    mapReduceDriver.runTest(false);/*from  ww w .  ja  va  2s  .  c om*/
}

---

From source file:com.justgiving.raven.kissmetrics.schema.KissmetricsJsonToSchematMapReduceTest.java

License:Open Source License

@Test
public void testMapReduce_TwoValidjsonRowOneLongerEventName_CountEachProperty() throws IOException {
    mapReduceDriver.withInput(new LongWritable(1), new Text(
            "{\"_n\":\"viewed signup\",\"_p\":\"bob@bob.com\",\"useragent\":\"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0\",\"_t\":1397577453}"));
    mapReduceDriver.withInput(new LongWritable(1),
            new Text("{\"_n\":\"viewed signup to it\",\"_p\":\"bob@bob.com\",\"_t\":1397577453}"));
    mapReduceDriver.withOutput(new Text("_n"), new Text("2\t19"));
    mapReduceDriver.withOutput(new Text("_p"), new Text("2\t11"));
    mapReduceDriver.withOutput(new Text("useragent"), new Text("1\t72"));
    mapReduceDriver.withOutput(new Text("_t"), new Text("2\t10"));
    mapReduceDriver.runTest(false);// w  ww.  ja  v a  2 s . c  om
}

---

From source file:com.kadwa.hadoop.DistExec.java

License:Open Source License

/**
 * Initialize ExecFilesMapper specific job-configuration.
 *
 * @param conf    : The dfs/mapred configuration.
 * @param jobConf : The handle to the jobConf object to be initialized.
 * @param args    Arguments/*from  www . j  a  va2  s . c o m*/
 * @return true if it is necessary to launch a job.
 */
private static boolean setup(Configuration conf, JobConf jobConf, final Arguments args) throws IOException {
    jobConf.set(DST_DIR_LABEL, args.dst.toUri().toString());
    jobConf.set(EXEC_CMD_LABEL, args.execCmd);

    //set boolean values
    jobConf.setBoolean(Options.REDIRECT_ERROR_TO_OUT.propertyname,
            args.flags.contains(Options.REDIRECT_ERROR_TO_OUT));

    final String randomId = getRandomId();
    JobClient jClient = new JobClient(jobConf);
    Path stagingArea;
    try {
        stagingArea = JobSubmissionFiles.getStagingDir(jClient, conf);
    } catch (InterruptedException e) {
        throw new IOException(e);
    }

    Path jobDirectory = new Path(stagingArea + NAME + "_" + randomId);
    FsPermission mapredSysPerms = new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION);
    FileSystem.mkdirs(FileSystem.get(jobDirectory.toUri(), conf), jobDirectory, mapredSysPerms);
    jobConf.set(JOB_DIR_LABEL, jobDirectory.toString());

    FileSystem dstfs = args.dst.getFileSystem(conf);

    // get tokens for all the required FileSystems..
    TokenCache.obtainTokensForNamenodes(jobConf.getCredentials(), new Path[] { args.dst }, conf);

    boolean dstExists = dstfs.exists(args.dst);
    boolean dstIsDir = false;
    if (dstExists) {
        dstIsDir = dstfs.getFileStatus(args.dst).isDir();
    }

    // default logPath
    Path logPath = args.log;
    if (logPath == null) {
        String filename = "_" + NAME + "_logs_" + randomId;
        if (!dstExists || !dstIsDir) {
            Path parent = args.dst.getParent();
            if (!dstfs.exists(parent)) {
                dstfs.mkdirs(parent);
            }
            logPath = new Path(parent, filename);
        } else {
            logPath = new Path(args.dst, filename);
        }
    }
    FileOutputFormat.setOutputPath(jobConf, logPath);

    // create src list, dst list
    FileSystem jobfs = jobDirectory.getFileSystem(jobConf);

    Path srcfilelist = new Path(jobDirectory, "_" + NAME + "_src_files");
    jobConf.set(SRC_LIST_LABEL, srcfilelist.toString());
    SequenceFile.Writer src_writer = SequenceFile.createWriter(jobfs, jobConf, srcfilelist, LongWritable.class,
            FilePair.class, SequenceFile.CompressionType.NONE);

    Path dstfilelist = new Path(jobDirectory, "_" + NAME + "_dst_files");
    SequenceFile.Writer dst_writer = SequenceFile.createWriter(jobfs, jobConf, dstfilelist, Text.class,
            Text.class, SequenceFile.CompressionType.NONE);

    Path dstdirlist = new Path(jobDirectory, "_" + NAME + "_dst_dirs");
    jobConf.set(DST_DIR_LIST_LABEL, dstdirlist.toString());
    SequenceFile.Writer dir_writer = SequenceFile.createWriter(jobfs, jobConf, dstdirlist, Text.class,
            FilePair.class, SequenceFile.CompressionType.NONE);

    // handle the case where the destination directory doesn't exist
    // and we've only a single src directory.
    final boolean special = (args.srcs.size() == 1 && !dstExists);
    int srcCount = 0, cnsyncf = 0, dirsyn = 0;
    long fileCount = 0L, byteCount = 0L, cbsyncs = 0L;
    try {
        for (Iterator<Path> srcItr = args.srcs.iterator(); srcItr.hasNext();) {
            final Path src = srcItr.next();
            FileSystem srcfs = src.getFileSystem(conf);
            FileStatus srcfilestat = srcfs.getFileStatus(src);
            Path root = special && srcfilestat.isDir() ? src : src.getParent();
            if (srcfilestat.isDir()) {
                ++srcCount;
            }

            Stack<FileStatus> pathstack = new Stack<FileStatus>();
            for (pathstack.push(srcfilestat); !pathstack.empty();) {
                FileStatus cur = pathstack.pop();
                FileStatus[] children = srcfs.listStatus(cur.getPath());
                for (int i = 0; i < children.length; i++) {
                    boolean skipfile = false;
                    final FileStatus child = children[i];
                    final String dst = makeRelative(root, child.getPath());
                    ++srcCount;

                    if (child.isDir()) {
                        pathstack.push(child);
                    } else {

                        if (!skipfile) {
                            ++fileCount;
                            byteCount += child.getLen();

                            if (LOG.isTraceEnabled()) {
                                LOG.trace("adding file " + child.getPath());
                            }

                            ++cnsyncf;
                            cbsyncs += child.getLen();
                            if (cnsyncf > SYNC_FILE_MAX || cbsyncs > BYTES_PER_MAP) {
                                src_writer.sync();
                                dst_writer.sync();
                                cnsyncf = 0;
                                cbsyncs = 0L;
                            }
                        }
                    }

                    if (!skipfile) {
                        src_writer.append(new LongWritable(child.isDir() ? 0 : child.getLen()),
                                new FilePair(child, dst));
                    }

                    dst_writer.append(new Text(dst), new Text(child.getPath().toString()));
                }

                if (cur.isDir()) {
                    String dst = makeRelative(root, cur.getPath());
                    dir_writer.append(new Text(dst), new FilePair(cur, dst));
                    if (++dirsyn > SYNC_FILE_MAX) {
                        dirsyn = 0;
                        dir_writer.sync();
                    }
                }
            }
        }
    } finally {
        checkAndClose(src_writer);
        checkAndClose(dst_writer);
        checkAndClose(dir_writer);
    }

    FileStatus dststatus = null;
    try {
        dststatus = dstfs.getFileStatus(args.dst);
    } catch (FileNotFoundException fnfe) {
        LOG.info(args.dst + " does not exist.");
    }

    // create dest path dir if copying > 1 file
    if (dststatus == null) {
        if (srcCount > 1 && !dstfs.mkdirs(args.dst)) {
            throw new IOException("Failed to create" + args.dst);
        }
    }

    final Path sorted = new Path(jobDirectory, "_" + NAME + "_sorted");
    checkDuplication(jobfs, dstfilelist, sorted, conf);

    Path tmpDir = new Path(
            (dstExists && !dstIsDir) || (!dstExists && srcCount == 1) ? args.dst.getParent() : args.dst,
            "_" + NAME + "_tmp_" + randomId);
    jobConf.set(TMP_DIR_LABEL, tmpDir.toUri().toString());
    LOG.info("sourcePathsCount=" + srcCount);
    LOG.info("filesToExecCount=" + fileCount);
    LOG.info("bytesToExecCount=" + StringUtils.humanReadableInt(byteCount));
    jobConf.setInt(SRC_COUNT_LABEL, srcCount);
    jobConf.setLong(TOTAL_SIZE_LABEL, byteCount);
    setMapCount(fileCount, jobConf);
    return fileCount > 0;
}

---

From source file:com.knewton.mapreduce.example.StudentEventMapper.java

License:Apache License

/**
 * {@inheritDoc}/*from w  w  w .jav a2s.c  o  m*/
 */
@Override
public void performMapTask(Long key, StudentEvent value, Context context)
        throws IOException, InterruptedException {
    StudentEventWritable seWritable = new StudentEventWritable(value, getValue().timestamp());
    context.write(new LongWritable(key), seWritable);
}

---

From source file:com.kylinolap.cube.kv.RowValueDecoderTest.java

License:Apache License

@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(this.getTestConfig()).getCube("test_kylin_cube_with_slr_ready")
            .getDescriptor();/*from   w  w w  .j  av a2 s .co  m*/
    HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0];

    MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
    codec.encode(new Object[] { sum, min, max, count }, buf);

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasureIndex(aggrFunc);
        rowValueDecoder.setIndex(index);
    }

    rowValueDecoder.decode(valueBytes);
    List<String> measureNames = rowValueDecoder.getNames();
    Object[] measureValues = rowValueDecoder.getValues();

    assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString());
    assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »