Example usage for org.apache.hadoop.io LongWritable LongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable LongWritable.

Prototype

public LongWritable(long value) 

Source Link

Usage

From source file:com.splunk.shuttl.integration.hadoop.hbase.CSVMapperTest.java

License:Apache License

public void map_inputStringIsHeaderLineFromFile_DoesNotWriteHeaderDataToContext()
        throws IOException, InterruptedException {
    Text headerStringText = new Text("header1,header2,header3");
    LongWritable anyKey = new LongWritable(17);

    when(configuration.get(JobConfigurationConstants.HEADER_STRING)).thenReturn(headerStringText.toString());

    mapper.map(anyKey, headerStringText, context);

    verify(context, never()).write(any(), any());
}

---

From source file:com.splunk.shuttl.integration.hadoop.hbase.CSVMapperTest.java

License:Apache License

@Test(expectedExceptions = { HeaderNotFoundException.class })
public void map_headerContainsEmptyName_castsException() throws IOException, InterruptedException {
    Text inputValue = new Text("value1,value2,value3,value4");

    when(configuration.get(JobConfigurationConstants.HEADER_STRING)).thenReturn("h1,,h3,h4");

    LongWritable anyKey = new LongWritable(17);

    mapper.map(anyKey, inputValue, context);
}

---

From source file:com.stratio.deep.es.utils.UtilES.java

License:Apache License

/**
 * Returns the object inside Writable//  w w  w .  jav a 2s.  c om
 *
 * @param object
 * @return
 * @throws IllegalAccessException
 * @throws InstantiationException
 * @throws InvocationTargetException
 * @throws NoSuchMethodException
 */
private static Writable getWritableFromObject(Object object) {

    Writable writable = null;

    if (object instanceof String) {

        writable = new Text(object.toString());

    } else if (object instanceof Long) {

        writable = new LongWritable((Long) object);

    } else {

        writable = new IntWritable((Integer) object);

    }
    // writable = writable!=null?writable:new Text("");
    return writable;
}

---

From source file:com.streamsets.pipeline.stage.origin.hdfs.cluster.ClusterHDFSSourceIT.java

License:Apache License

@Test(timeout = 30000)
public void testProduce() throws Exception {
    ClusterHdfsConfigBean conf = new ClusterHdfsConfigBean();
    conf.hdfsUri = miniDFS.getURI().toString();
    conf.hdfsDirLocations = Arrays.asList(dir.toUri().getPath());
    conf.hdfsConfigs = new HashMap<>();
    conf.hdfsKerberos = false;/*from  w  w  w .  j  av  a 2  s  .co  m*/
    conf.hdfsConfDir = hadoopConfDir;
    conf.recursive = false;
    conf.produceSingleRecordPerMessage = false;
    conf.dataFormat = DataFormat.TEXT;
    conf.dataFormatConfig.textMaxLineLen = 1024;

    SourceRunner sourceRunner = new SourceRunner.Builder(ClusterHdfsDSource.class, createSource(conf))
            .addOutputLane("lane").setExecutionMode(ExecutionMode.CLUSTER_BATCH).setResourcesDir(resourcesDir)
            .build();

    sourceRunner.runInit();

    List<Map.Entry> list = new ArrayList<>();
    list.add(new Pair(new LongWritable(1), new Text("aaa")));
    list.add(new Pair(new LongWritable(2), new Text("bbb")));
    list.add(new Pair(new LongWritable(3), new Text("ccc")));

    Thread th = createThreadForAddingBatch(sourceRunner, list);
    try {
        StageRunner.Output output = sourceRunner.runProduce(null, 5);

        String newOffset = output.getNewOffset();
        Assert.assertEquals("3", newOffset);
        List<Record> records = output.getRecords().get("lane");
        Assert.assertEquals(3, records.size());

        for (int i = 0; i < records.size(); i++) {
            Assert.assertNotNull(records.get(i).get("/text"));
            LOG.info("Header " + records.get(i).getHeader().getSourceId());
            Assert.assertTrue(!records.get(i).get("/text").getValueAsString().isEmpty());
            Assert.assertEquals(list.get(i).getValue().toString(),
                    records.get(i).get("/text").getValueAsString());
        }

        if (sourceRunner != null) {
            sourceRunner.runDestroy();
        }
    } finally {
        th.interrupt();
    }
}

---

From source file:com.streamsets.pipeline.stage.origin.hdfs.cluster.TestClusterHDFSSource.java

License:Apache License

@Test(timeout = 30000)
public void testProduce() throws Exception {
    SourceRunner sourceRunner = new SourceRunner.Builder(ClusterHdfsDSource.class).addOutputLane("lane")
            .setClusterMode(true).addConfiguration("hdfsUri", miniDFS.getURI().toString())
            .addConfiguration("hdfsDirLocations", Arrays.asList(dir.toUri().getPath()))
            .addConfiguration("recursive", false).addConfiguration("hdfsConfigs", new HashMap<String, String>())
            .addConfiguration("dataFormat", DataFormat.TEXT).addConfiguration("textMaxLineLen", 1024)
            .addConfiguration("produceSingleRecordPerMessage", false).addConfiguration("regex", null)
            .addConfiguration("grokPatternDefinition", null)
            .addConfiguration("enableLog4jCustomLogFormat", false).addConfiguration("customLogFormat", null)
            .addConfiguration("fieldPathsToGroupName", null).addConfiguration("log4jCustomLogFormat", null)
            .addConfiguration("grokPattern", null).addConfiguration("hdfsKerberos", false)
            .addConfiguration("hdfsConfDir", hadoopConfDir).setResourcesDir(resourcesDir).build();
    sourceRunner.runInit();// ww  w  .  j  a  va 2  s  . com

    List<Map.Entry> list = new ArrayList<>();
    list.add(new Pair(new LongWritable(1), new Text("aaa")));
    list.add(new Pair(new LongWritable(2), new Text("bbb")));
    list.add(new Pair(new LongWritable(3), new Text("ccc")));

    Thread th = createThreadForAddingBatch(sourceRunner, list);
    try {
        StageRunner.Output output = sourceRunner.runProduce(null, 5);

        String newOffset = output.getNewOffset();
        Assert.assertEquals("3", newOffset);
        List<Record> records = output.getRecords().get("lane");
        Assert.assertEquals(3, records.size());

        for (int i = 0; i < records.size(); i++) {
            Assert.assertNotNull(records.get(i).get("/text"));
            LOG.info("Header " + records.get(i).getHeader().getSourceId());
            Assert.assertTrue(!records.get(i).get("/text").getValueAsString().isEmpty());
            Assert.assertEquals(list.get(i).getValue().toString(),
                    records.get(i).get("/text").getValueAsString());
        }

        if (sourceRunner != null) {
            sourceRunner.runDestroy();
        }
    } finally {
        th.interrupt();
    }
}

---

From source file:com.talis.labs.pagerank.mapreduce.CountPagesReducer.java

License:Apache License

@Override
public void reduce(Text key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
    long sum = 0;
    for (LongWritable value : values) {
        sum += value.get();/*from w  w  w. j a v a  2 s.com*/
    }
    context.write(key, new LongWritable(sum));
}

---

From source file:com.telefonica.iot.tidoop.apiext.hadoop.ckan.CKANRecordReader.java

License:Open Source License

@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
    if (current < length) {
        key = new LongWritable(current);
        value = new Text(records.get(current).toString());
        current++;/*from w  w  w . j  av a  2 s  .co  m*/
        return true;
    } else {
        return false;
    } // if else
}

---

From source file:com.telefonica.iot.tidoop.apiext.hadoop.ckan.CKANRecordReaderTest.java

License:Open Source License

/**
 * Test of getCurrentKey method, of class CKANRecordReader.
 *//*  w w w .  ja  va  2s .  c om*/
@Test
public void testGetCurrentKey() {
    System.out.println("Testing CKANRecordReader.getCurrentKey");

    try {
        recordReader.initialize(inputSplit, taskAttemptContext);
        recordReader.nextKeyValue();
        assertEquals(new LongWritable(0), recordReader.getCurrentKey());
    } catch (IOException e) {
        fail(e.getMessage());
    } catch (InterruptedException e) {
        fail(e.getMessage());
    } finally {
        assertTrue(true);
    } // try catch finally
}

---

From source file:com.teradata.benchto.generator.HiveTypesGeneratorTest.java

License:Apache License

@Test
public void testMapper() throws Exception {
    Configuration serializationConfiguration = new Configuration();
    MapDriver mapDriver = MapDriver.newMapDriver(new HiveTypesGenerator.HiveTypesMapper())
            .withInput(new LongWritable(0L), NullWritable.get())
            .withInput(new LongWritable(1L), NullWritable.get())
            .withInput(new LongWritable(2L), NullWritable.get())
            .withInput(new LongWritable(3L), NullWritable.get())
            .withInput(new LongWritable(4L), NullWritable.get())
            .withOutputFormat(HiveTypesGenerator.getOutputFormatClass(format),
                    HiveTypesGenerator.getInputFormatClass(format))
            .withOutputSerializationConfiguration(serializationConfiguration);

    mapDriver.getConfiguration().set(HiveTypesGenerator.FORMAT_PROPERTY_NAME, format);
    mapDriver.getConfiguration().set(HiveTypesGenerator.HIVE_TYPE_PROPERTY_NAME, type);
    mapDriver.getConfiguration().setLong(HiveTypesGenerator.NUM_ROWS_PROPERTY_NAME, 5L);
    mapDriver.getConfiguration().setInt(NUM_MAPS, 1);

    List<Pair<NullWritable, Writable>> output = mapDriver.run();

    extractMapperProperties(mapDriver);//from   w  ww  . ja v  a2s . c om

    assertEquals(expectedSerializedRow(0), output.get(0).getSecond());
    assertEquals(expectedSerializedRow(1), output.get(1).getSecond());
    assertEquals(expectedSerializedRow(2), output.get(2).getSecond());
    assertEquals(expectedSerializedRow(3), output.get(3).getSecond());
    assertEquals(expectedSerializedRow(4), output.get(4).getSecond());
}

---

From source file:com.test.PiEstimatorKrb.java

License:Apache License

/**
 * Run a map/reduce job for estimating Pi.
 *
 * @return the estimated value of Pi//from  w  w w .  j  a v a 2 s  .co  m
 */
public static BigDecimal estimate(int numMaps, long numPoints, JobConf jobConf) throws IOException {
    //setup job conf
    jobConf.setJobName(PiEstimatorKrb.class.getSimpleName());

    jobConf.setInputFormat(SequenceFileInputFormat.class);

    jobConf.setOutputKeyClass(BooleanWritable.class);
    jobConf.setOutputValueClass(LongWritable.class);
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    jobConf.setMapperClass(PiMapper.class);
    jobConf.setNumMapTasks(numMaps);

    jobConf.setReducerClass(PiReducer.class);
    jobConf.setNumReduceTasks(1);

    // turn off speculative execution, because DFS doesn't handle
    // multiple writers to the same file.
    jobConf.setSpeculativeExecution(false);

    //setup input/output directories
    final Path inDir = new Path(TMP_DIR, "in");
    final Path outDir = new Path(TMP_DIR, "out");
    FileInputFormat.setInputPaths(jobConf, inDir);
    FileOutputFormat.setOutputPath(jobConf, outDir);

    final FileSystem fs = FileSystem.get(jobConf);
    if (fs.exists(TMP_DIR)) {
        throw new IOException(
                "Tmp directory " + fs.makeQualified(TMP_DIR) + " already exists.  Please remove it first.");
    }
    if (!fs.mkdirs(inDir)) {
        throw new IOException("Cannot create input directory " + inDir);
    }

    try {
        //generate an input file for each map task
        for (int i = 0; i < numMaps; ++i) {
            final Path file = new Path(inDir, "part" + i);
            final LongWritable offset = new LongWritable(i * numPoints);
            final LongWritable size = new LongWritable(numPoints);
            final SequenceFile.Writer writer = SequenceFile.createWriter(fs, jobConf, file, LongWritable.class,
                    LongWritable.class, CompressionType.NONE);
            try {
                writer.append(offset, size);
            } finally {
                writer.close();
            }
            sLogger.info("Wrote input for Map #" + i);
        }

        //start a map/reduce job
        sLogger.info("Starting Job");
        final long startTime = System.currentTimeMillis();

        if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
            jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
        }

        JobClient.runJob(jobConf);
        final double duration = (System.currentTimeMillis() - startTime) / 1000.0;
        sLogger.info("Job Finished in " + duration + " seconds");

        //read outputs
        Path inFile = new Path(outDir, "reduce-out");
        LongWritable numInside = new LongWritable();
        LongWritable numOutside = new LongWritable();
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, inFile, jobConf);
        try {
            reader.next(numInside, numOutside);
        } finally {
            reader.close();
        }

        //compute estimated value
        return BigDecimal.valueOf(4).setScale(20).multiply(BigDecimal.valueOf(numInside.get()))
                .divide(BigDecimal.valueOf(numMaps)).divide(BigDecimal.valueOf(numPoints));
    } finally {
        fs.delete(TMP_DIR, true);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »