List of usage examples for org.apache.hadoop.io LongWritable LongWritable
public LongWritable(long value)
From source file:com.splunk.shuttl.integration.hadoop.hbase.CSVMapperTest.java
License:Apache License
public void map_inputStringIsHeaderLineFromFile_DoesNotWriteHeaderDataToContext() throws IOException, InterruptedException { Text headerStringText = new Text("header1,header2,header3"); LongWritable anyKey = new LongWritable(17); when(configuration.get(JobConfigurationConstants.HEADER_STRING)).thenReturn(headerStringText.toString()); mapper.map(anyKey, headerStringText, context); verify(context, never()).write(any(), any()); }
From source file:com.splunk.shuttl.integration.hadoop.hbase.CSVMapperTest.java
License:Apache License
@Test(expectedExceptions = { HeaderNotFoundException.class }) public void map_headerContainsEmptyName_castsException() throws IOException, InterruptedException { Text inputValue = new Text("value1,value2,value3,value4"); when(configuration.get(JobConfigurationConstants.HEADER_STRING)).thenReturn("h1,,h3,h4"); LongWritable anyKey = new LongWritable(17); mapper.map(anyKey, inputValue, context); }
From source file:com.stratio.deep.es.utils.UtilES.java
License:Apache License
/** * Returns the object inside Writable// w w w . jav a 2s. c om * * @param object * @return * @throws IllegalAccessException * @throws InstantiationException * @throws InvocationTargetException * @throws NoSuchMethodException */ private static Writable getWritableFromObject(Object object) { Writable writable = null; if (object instanceof String) { writable = new Text(object.toString()); } else if (object instanceof Long) { writable = new LongWritable((Long) object); } else { writable = new IntWritable((Integer) object); } // writable = writable!=null?writable:new Text(""); return writable; }
From source file:com.streamsets.pipeline.stage.origin.hdfs.cluster.ClusterHDFSSourceIT.java
License:Apache License
@Test(timeout = 30000) public void testProduce() throws Exception { ClusterHdfsConfigBean conf = new ClusterHdfsConfigBean(); conf.hdfsUri = miniDFS.getURI().toString(); conf.hdfsDirLocations = Arrays.asList(dir.toUri().getPath()); conf.hdfsConfigs = new HashMap<>(); conf.hdfsKerberos = false;/*from w w w . j av a 2 s .co m*/ conf.hdfsConfDir = hadoopConfDir; conf.recursive = false; conf.produceSingleRecordPerMessage = false; conf.dataFormat = DataFormat.TEXT; conf.dataFormatConfig.textMaxLineLen = 1024; SourceRunner sourceRunner = new SourceRunner.Builder(ClusterHdfsDSource.class, createSource(conf)) .addOutputLane("lane").setExecutionMode(ExecutionMode.CLUSTER_BATCH).setResourcesDir(resourcesDir) .build(); sourceRunner.runInit(); List<Map.Entry> list = new ArrayList<>(); list.add(new Pair(new LongWritable(1), new Text("aaa"))); list.add(new Pair(new LongWritable(2), new Text("bbb"))); list.add(new Pair(new LongWritable(3), new Text("ccc"))); Thread th = createThreadForAddingBatch(sourceRunner, list); try { StageRunner.Output output = sourceRunner.runProduce(null, 5); String newOffset = output.getNewOffset(); Assert.assertEquals("3", newOffset); List<Record> records = output.getRecords().get("lane"); Assert.assertEquals(3, records.size()); for (int i = 0; i < records.size(); i++) { Assert.assertNotNull(records.get(i).get("/text")); LOG.info("Header " + records.get(i).getHeader().getSourceId()); Assert.assertTrue(!records.get(i).get("/text").getValueAsString().isEmpty()); Assert.assertEquals(list.get(i).getValue().toString(), records.get(i).get("/text").getValueAsString()); } if (sourceRunner != null) { sourceRunner.runDestroy(); } } finally { th.interrupt(); } }
From source file:com.streamsets.pipeline.stage.origin.hdfs.cluster.TestClusterHDFSSource.java
License:Apache License
@Test(timeout = 30000) public void testProduce() throws Exception { SourceRunner sourceRunner = new SourceRunner.Builder(ClusterHdfsDSource.class).addOutputLane("lane") .setClusterMode(true).addConfiguration("hdfsUri", miniDFS.getURI().toString()) .addConfiguration("hdfsDirLocations", Arrays.asList(dir.toUri().getPath())) .addConfiguration("recursive", false).addConfiguration("hdfsConfigs", new HashMap<String, String>()) .addConfiguration("dataFormat", DataFormat.TEXT).addConfiguration("textMaxLineLen", 1024) .addConfiguration("produceSingleRecordPerMessage", false).addConfiguration("regex", null) .addConfiguration("grokPatternDefinition", null) .addConfiguration("enableLog4jCustomLogFormat", false).addConfiguration("customLogFormat", null) .addConfiguration("fieldPathsToGroupName", null).addConfiguration("log4jCustomLogFormat", null) .addConfiguration("grokPattern", null).addConfiguration("hdfsKerberos", false) .addConfiguration("hdfsConfDir", hadoopConfDir).setResourcesDir(resourcesDir).build(); sourceRunner.runInit();// ww w . j a va 2 s . com List<Map.Entry> list = new ArrayList<>(); list.add(new Pair(new LongWritable(1), new Text("aaa"))); list.add(new Pair(new LongWritable(2), new Text("bbb"))); list.add(new Pair(new LongWritable(3), new Text("ccc"))); Thread th = createThreadForAddingBatch(sourceRunner, list); try { StageRunner.Output output = sourceRunner.runProduce(null, 5); String newOffset = output.getNewOffset(); Assert.assertEquals("3", newOffset); List<Record> records = output.getRecords().get("lane"); Assert.assertEquals(3, records.size()); for (int i = 0; i < records.size(); i++) { Assert.assertNotNull(records.get(i).get("/text")); LOG.info("Header " + records.get(i).getHeader().getSourceId()); Assert.assertTrue(!records.get(i).get("/text").getValueAsString().isEmpty()); Assert.assertEquals(list.get(i).getValue().toString(), records.get(i).get("/text").getValueAsString()); } if (sourceRunner != null) { sourceRunner.runDestroy(); } } finally { th.interrupt(); } }
From source file:com.talis.labs.pagerank.mapreduce.CountPagesReducer.java
License:Apache License
@Override public void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { long sum = 0; for (LongWritable value : values) { sum += value.get();/*from w w w. j a v a 2 s.com*/ } context.write(key, new LongWritable(sum)); }
From source file:com.telefonica.iot.tidoop.apiext.hadoop.ckan.CKANRecordReader.java
License:Open Source License
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (current < length) { key = new LongWritable(current); value = new Text(records.get(current).toString()); current++;/*from w w w . j av a 2 s .co m*/ return true; } else { return false; } // if else }
From source file:com.telefonica.iot.tidoop.apiext.hadoop.ckan.CKANRecordReaderTest.java
License:Open Source License
/** * Test of getCurrentKey method, of class CKANRecordReader. *//* w w w . ja va 2s . c om*/ @Test public void testGetCurrentKey() { System.out.println("Testing CKANRecordReader.getCurrentKey"); try { recordReader.initialize(inputSplit, taskAttemptContext); recordReader.nextKeyValue(); assertEquals(new LongWritable(0), recordReader.getCurrentKey()); } catch (IOException e) { fail(e.getMessage()); } catch (InterruptedException e) { fail(e.getMessage()); } finally { assertTrue(true); } // try catch finally }
From source file:com.teradata.benchto.generator.HiveTypesGeneratorTest.java
License:Apache License
@Test public void testMapper() throws Exception { Configuration serializationConfiguration = new Configuration(); MapDriver mapDriver = MapDriver.newMapDriver(new HiveTypesGenerator.HiveTypesMapper()) .withInput(new LongWritable(0L), NullWritable.get()) .withInput(new LongWritable(1L), NullWritable.get()) .withInput(new LongWritable(2L), NullWritable.get()) .withInput(new LongWritable(3L), NullWritable.get()) .withInput(new LongWritable(4L), NullWritable.get()) .withOutputFormat(HiveTypesGenerator.getOutputFormatClass(format), HiveTypesGenerator.getInputFormatClass(format)) .withOutputSerializationConfiguration(serializationConfiguration); mapDriver.getConfiguration().set(HiveTypesGenerator.FORMAT_PROPERTY_NAME, format); mapDriver.getConfiguration().set(HiveTypesGenerator.HIVE_TYPE_PROPERTY_NAME, type); mapDriver.getConfiguration().setLong(HiveTypesGenerator.NUM_ROWS_PROPERTY_NAME, 5L); mapDriver.getConfiguration().setInt(NUM_MAPS, 1); List<Pair<NullWritable, Writable>> output = mapDriver.run(); extractMapperProperties(mapDriver);//from w ww . ja v a2s . c om assertEquals(expectedSerializedRow(0), output.get(0).getSecond()); assertEquals(expectedSerializedRow(1), output.get(1).getSecond()); assertEquals(expectedSerializedRow(2), output.get(2).getSecond()); assertEquals(expectedSerializedRow(3), output.get(3).getSecond()); assertEquals(expectedSerializedRow(4), output.get(4).getSecond()); }
From source file:com.test.PiEstimatorKrb.java
License:Apache License
/** * Run a map/reduce job for estimating Pi. * * @return the estimated value of Pi//from w w w . j a v a 2 s .co m */ public static BigDecimal estimate(int numMaps, long numPoints, JobConf jobConf) throws IOException { //setup job conf jobConf.setJobName(PiEstimatorKrb.class.getSimpleName()); jobConf.setInputFormat(SequenceFileInputFormat.class); jobConf.setOutputKeyClass(BooleanWritable.class); jobConf.setOutputValueClass(LongWritable.class); jobConf.setOutputFormat(SequenceFileOutputFormat.class); jobConf.setMapperClass(PiMapper.class); jobConf.setNumMapTasks(numMaps); jobConf.setReducerClass(PiReducer.class); jobConf.setNumReduceTasks(1); // turn off speculative execution, because DFS doesn't handle // multiple writers to the same file. jobConf.setSpeculativeExecution(false); //setup input/output directories final Path inDir = new Path(TMP_DIR, "in"); final Path outDir = new Path(TMP_DIR, "out"); FileInputFormat.setInputPaths(jobConf, inDir); FileOutputFormat.setOutputPath(jobConf, outDir); final FileSystem fs = FileSystem.get(jobConf); if (fs.exists(TMP_DIR)) { throw new IOException( "Tmp directory " + fs.makeQualified(TMP_DIR) + " already exists. Please remove it first."); } if (!fs.mkdirs(inDir)) { throw new IOException("Cannot create input directory " + inDir); } try { //generate an input file for each map task for (int i = 0; i < numMaps; ++i) { final Path file = new Path(inDir, "part" + i); final LongWritable offset = new LongWritable(i * numPoints); final LongWritable size = new LongWritable(numPoints); final SequenceFile.Writer writer = SequenceFile.createWriter(fs, jobConf, file, LongWritable.class, LongWritable.class, CompressionType.NONE); try { writer.append(offset, size); } finally { writer.close(); } sLogger.info("Wrote input for Map #" + i); } //start a map/reduce job sLogger.info("Starting Job"); final long startTime = System.currentTimeMillis(); if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) { jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION")); } JobClient.runJob(jobConf); final double duration = (System.currentTimeMillis() - startTime) / 1000.0; sLogger.info("Job Finished in " + duration + " seconds"); //read outputs Path inFile = new Path(outDir, "reduce-out"); LongWritable numInside = new LongWritable(); LongWritable numOutside = new LongWritable(); SequenceFile.Reader reader = new SequenceFile.Reader(fs, inFile, jobConf); try { reader.next(numInside, numOutside); } finally { reader.close(); } //compute estimated value return BigDecimal.valueOf(4).setScale(20).multiply(BigDecimal.valueOf(numInside.get())) .divide(BigDecimal.valueOf(numMaps)).divide(BigDecimal.valueOf(numPoints)); } finally { fs.delete(TMP_DIR, true); } }