Example usage for org.apache.hadoop.io LongWritable LongWritable

List of usage examples for org.apache.hadoop.io LongWritable LongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable LongWritable.

Prototype

public LongWritable(long value) 

Source Link

Usage

From source file:com.splunk.shuttl.integration.hadoop.hbase.CSVMapperTest.java

License:Apache License

public void map_inputStringIsHeaderLineFromFile_DoesNotWriteHeaderDataToContext()
        throws IOException, InterruptedException {
    Text headerStringText = new Text("header1,header2,header3");
    LongWritable anyKey = new LongWritable(17);

    when(configuration.get(JobConfigurationConstants.HEADER_STRING)).thenReturn(headerStringText.toString());

    mapper.map(anyKey, headerStringText, context);

    verify(context, never()).write(any(), any());
}

From source file:com.splunk.shuttl.integration.hadoop.hbase.CSVMapperTest.java

License:Apache License

@Test(expectedExceptions = { HeaderNotFoundException.class })
public void map_headerContainsEmptyName_castsException() throws IOException, InterruptedException {
    Text inputValue = new Text("value1,value2,value3,value4");

    when(configuration.get(JobConfigurationConstants.HEADER_STRING)).thenReturn("h1,,h3,h4");

    LongWritable anyKey = new LongWritable(17);

    mapper.map(anyKey, inputValue, context);
}

From source file:com.stratio.deep.es.utils.UtilES.java

License:Apache License

/**
 * Returns the object inside Writable//  w w  w .  jav a 2s.  c om
 *
 * @param object
 * @return
 * @throws IllegalAccessException
 * @throws InstantiationException
 * @throws InvocationTargetException
 * @throws NoSuchMethodException
 */
private static Writable getWritableFromObject(Object object) {

    Writable writable = null;

    if (object instanceof String) {

        writable = new Text(object.toString());

    } else if (object instanceof Long) {

        writable = new LongWritable((Long) object);

    } else {

        writable = new IntWritable((Integer) object);

    }
    // writable = writable!=null?writable:new Text("");
    return writable;
}

From source file:com.streamsets.pipeline.stage.origin.hdfs.cluster.ClusterHDFSSourceIT.java

License:Apache License

@Test(timeout = 30000)
public void testProduce() throws Exception {
    ClusterHdfsConfigBean conf = new ClusterHdfsConfigBean();
    conf.hdfsUri = miniDFS.getURI().toString();
    conf.hdfsDirLocations = Arrays.asList(dir.toUri().getPath());
    conf.hdfsConfigs = new HashMap<>();
    conf.hdfsKerberos = false;/*from  w  w  w .  j  av  a 2  s  .co  m*/
    conf.hdfsConfDir = hadoopConfDir;
    conf.recursive = false;
    conf.produceSingleRecordPerMessage = false;
    conf.dataFormat = DataFormat.TEXT;
    conf.dataFormatConfig.textMaxLineLen = 1024;

    SourceRunner sourceRunner = new SourceRunner.Builder(ClusterHdfsDSource.class, createSource(conf))
            .addOutputLane("lane").setExecutionMode(ExecutionMode.CLUSTER_BATCH).setResourcesDir(resourcesDir)
            .build();

    sourceRunner.runInit();

    List<Map.Entry> list = new ArrayList<>();
    list.add(new Pair(new LongWritable(1), new Text("aaa")));
    list.add(new Pair(new LongWritable(2), new Text("bbb")));
    list.add(new Pair(new LongWritable(3), new Text("ccc")));

    Thread th = createThreadForAddingBatch(sourceRunner, list);
    try {
        StageRunner.Output output = sourceRunner.runProduce(null, 5);

        String newOffset = output.getNewOffset();
        Assert.assertEquals("3", newOffset);
        List<Record> records = output.getRecords().get("lane");
        Assert.assertEquals(3, records.size());

        for (int i = 0; i < records.size(); i++) {
            Assert.assertNotNull(records.get(i).get("/text"));
            LOG.info("Header " + records.get(i).getHeader().getSourceId());
            Assert.assertTrue(!records.get(i).get("/text").getValueAsString().isEmpty());
            Assert.assertEquals(list.get(i).getValue().toString(),
                    records.get(i).get("/text").getValueAsString());
        }

        if (sourceRunner != null) {
            sourceRunner.runDestroy();
        }
    } finally {
        th.interrupt();
    }
}

From source file:com.streamsets.pipeline.stage.origin.hdfs.cluster.TestClusterHDFSSource.java

License:Apache License

@Test(timeout = 30000)
public void testProduce() throws Exception {
    SourceRunner sourceRunner = new SourceRunner.Builder(ClusterHdfsDSource.class).addOutputLane("lane")
            .setClusterMode(true).addConfiguration("hdfsUri", miniDFS.getURI().toString())
            .addConfiguration("hdfsDirLocations", Arrays.asList(dir.toUri().getPath()))
            .addConfiguration("recursive", false).addConfiguration("hdfsConfigs", new HashMap<String, String>())
            .addConfiguration("dataFormat", DataFormat.TEXT).addConfiguration("textMaxLineLen", 1024)
            .addConfiguration("produceSingleRecordPerMessage", false).addConfiguration("regex", null)
            .addConfiguration("grokPatternDefinition", null)
            .addConfiguration("enableLog4jCustomLogFormat", false).addConfiguration("customLogFormat", null)
            .addConfiguration("fieldPathsToGroupName", null).addConfiguration("log4jCustomLogFormat", null)
            .addConfiguration("grokPattern", null).addConfiguration("hdfsKerberos", false)
            .addConfiguration("hdfsConfDir", hadoopConfDir).setResourcesDir(resourcesDir).build();
    sourceRunner.runInit();// ww  w  .  j  a  va 2  s  . com

    List<Map.Entry> list = new ArrayList<>();
    list.add(new Pair(new LongWritable(1), new Text("aaa")));
    list.add(new Pair(new LongWritable(2), new Text("bbb")));
    list.add(new Pair(new LongWritable(3), new Text("ccc")));

    Thread th = createThreadForAddingBatch(sourceRunner, list);
    try {
        StageRunner.Output output = sourceRunner.runProduce(null, 5);

        String newOffset = output.getNewOffset();
        Assert.assertEquals("3", newOffset);
        List<Record> records = output.getRecords().get("lane");
        Assert.assertEquals(3, records.size());

        for (int i = 0; i < records.size(); i++) {
            Assert.assertNotNull(records.get(i).get("/text"));
            LOG.info("Header " + records.get(i).getHeader().getSourceId());
            Assert.assertTrue(!records.get(i).get("/text").getValueAsString().isEmpty());
            Assert.assertEquals(list.get(i).getValue().toString(),
                    records.get(i).get("/text").getValueAsString());
        }

        if (sourceRunner != null) {
            sourceRunner.runDestroy();
        }
    } finally {
        th.interrupt();
    }
}

From source file:com.talis.labs.pagerank.mapreduce.CountPagesReducer.java

License:Apache License

@Override
public void reduce(Text key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
    long sum = 0;
    for (LongWritable value : values) {
        sum += value.get();/*from w  w  w. j a v a  2 s.com*/
    }
    context.write(key, new LongWritable(sum));
}

From source file:com.telefonica.iot.tidoop.apiext.hadoop.ckan.CKANRecordReader.java

License:Open Source License

@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
    if (current < length) {
        key = new LongWritable(current);
        value = new Text(records.get(current).toString());
        current++;/*from w  w  w . j  av a  2 s  .co  m*/
        return true;
    } else {
        return false;
    } // if else
}

From source file:com.telefonica.iot.tidoop.apiext.hadoop.ckan.CKANRecordReaderTest.java

License:Open Source License

/**
 * Test of getCurrentKey method, of class CKANRecordReader.
 *//*  w w w .  ja  va  2s .  c om*/
@Test
public void testGetCurrentKey() {
    System.out.println("Testing CKANRecordReader.getCurrentKey");

    try {
        recordReader.initialize(inputSplit, taskAttemptContext);
        recordReader.nextKeyValue();
        assertEquals(new LongWritable(0), recordReader.getCurrentKey());
    } catch (IOException e) {
        fail(e.getMessage());
    } catch (InterruptedException e) {
        fail(e.getMessage());
    } finally {
        assertTrue(true);
    } // try catch finally
}

From source file:com.teradata.benchto.generator.HiveTypesGeneratorTest.java

License:Apache License

@Test
public void testMapper() throws Exception {
    Configuration serializationConfiguration = new Configuration();
    MapDriver mapDriver = MapDriver.newMapDriver(new HiveTypesGenerator.HiveTypesMapper())
            .withInput(new LongWritable(0L), NullWritable.get())
            .withInput(new LongWritable(1L), NullWritable.get())
            .withInput(new LongWritable(2L), NullWritable.get())
            .withInput(new LongWritable(3L), NullWritable.get())
            .withInput(new LongWritable(4L), NullWritable.get())
            .withOutputFormat(HiveTypesGenerator.getOutputFormatClass(format),
                    HiveTypesGenerator.getInputFormatClass(format))
            .withOutputSerializationConfiguration(serializationConfiguration);

    mapDriver.getConfiguration().set(HiveTypesGenerator.FORMAT_PROPERTY_NAME, format);
    mapDriver.getConfiguration().set(HiveTypesGenerator.HIVE_TYPE_PROPERTY_NAME, type);
    mapDriver.getConfiguration().setLong(HiveTypesGenerator.NUM_ROWS_PROPERTY_NAME, 5L);
    mapDriver.getConfiguration().setInt(NUM_MAPS, 1);

    List<Pair<NullWritable, Writable>> output = mapDriver.run();

    extractMapperProperties(mapDriver);//from   w  ww  . ja v  a2s . c om

    assertEquals(expectedSerializedRow(0), output.get(0).getSecond());
    assertEquals(expectedSerializedRow(1), output.get(1).getSecond());
    assertEquals(expectedSerializedRow(2), output.get(2).getSecond());
    assertEquals(expectedSerializedRow(3), output.get(3).getSecond());
    assertEquals(expectedSerializedRow(4), output.get(4).getSecond());
}

From source file:com.test.PiEstimatorKrb.java

License:Apache License

/**
 * Run a map/reduce job for estimating Pi.
 *
 * @return the estimated value of Pi//from  w  w w .  j  a v a 2 s  .co  m
 */
public static BigDecimal estimate(int numMaps, long numPoints, JobConf jobConf) throws IOException {
    //setup job conf
    jobConf.setJobName(PiEstimatorKrb.class.getSimpleName());

    jobConf.setInputFormat(SequenceFileInputFormat.class);

    jobConf.setOutputKeyClass(BooleanWritable.class);
    jobConf.setOutputValueClass(LongWritable.class);
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    jobConf.setMapperClass(PiMapper.class);
    jobConf.setNumMapTasks(numMaps);

    jobConf.setReducerClass(PiReducer.class);
    jobConf.setNumReduceTasks(1);

    // turn off speculative execution, because DFS doesn't handle
    // multiple writers to the same file.
    jobConf.setSpeculativeExecution(false);

    //setup input/output directories
    final Path inDir = new Path(TMP_DIR, "in");
    final Path outDir = new Path(TMP_DIR, "out");
    FileInputFormat.setInputPaths(jobConf, inDir);
    FileOutputFormat.setOutputPath(jobConf, outDir);

    final FileSystem fs = FileSystem.get(jobConf);
    if (fs.exists(TMP_DIR)) {
        throw new IOException(
                "Tmp directory " + fs.makeQualified(TMP_DIR) + " already exists.  Please remove it first.");
    }
    if (!fs.mkdirs(inDir)) {
        throw new IOException("Cannot create input directory " + inDir);
    }

    try {
        //generate an input file for each map task
        for (int i = 0; i < numMaps; ++i) {
            final Path file = new Path(inDir, "part" + i);
            final LongWritable offset = new LongWritable(i * numPoints);
            final LongWritable size = new LongWritable(numPoints);
            final SequenceFile.Writer writer = SequenceFile.createWriter(fs, jobConf, file, LongWritable.class,
                    LongWritable.class, CompressionType.NONE);
            try {
                writer.append(offset, size);
            } finally {
                writer.close();
            }
            sLogger.info("Wrote input for Map #" + i);
        }

        //start a map/reduce job
        sLogger.info("Starting Job");
        final long startTime = System.currentTimeMillis();

        if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
            jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
        }

        JobClient.runJob(jobConf);
        final double duration = (System.currentTimeMillis() - startTime) / 1000.0;
        sLogger.info("Job Finished in " + duration + " seconds");

        //read outputs
        Path inFile = new Path(outDir, "reduce-out");
        LongWritable numInside = new LongWritable();
        LongWritable numOutside = new LongWritable();
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, inFile, jobConf);
        try {
            reader.next(numInside, numOutside);
        } finally {
            reader.close();
        }

        //compute estimated value
        return BigDecimal.valueOf(4).setScale(20).multiply(BigDecimal.valueOf(numInside.get()))
                .divide(BigDecimal.valueOf(numMaps)).divide(BigDecimal.valueOf(numPoints));
    } finally {
        fs.delete(TMP_DIR, true);
    }
}