List of usage examples for org.apache.hadoop.io LongWritable LongWritable
public LongWritable(long value)
From source file:com.bigdata.diane.MiniTestDFSIO.java
License:Apache License
private static void createControlFile(FileSystem fs, int fileSize, // in MB int nrFiles, Configuration fsConfig) throws InterruptedException, IOException { LOG.info("creating control file: " + fileSize + " mega bytes, " + nrFiles + " files"); for (int i = 0; i < nrFiles; i++) { String name = getFileName(i); Path controlFile = new Path(CONTROL_DIR, "in_file_" + name); SequenceFile.Writer writer = null; try {//from ww w .j av a2s . co m writer = SequenceFile.createWriter(fs, fsConfig, controlFile, Text.class, LongWritable.class, CompressionType.NONE); writer.append(new Text(name), new LongWritable(fileSize)); } catch (Exception e) { throw new IOException(e.getLocalizedMessage()); } finally { if (writer != null) writer.close(); writer = null; } } LOG.info("created control files for: " + nrFiles + " files now sleep 20 seconds"); Thread.sleep(20000); }
From source file:com.bizosys.hsearch.kv.indexing.KVIndexerLocal.java
License:Apache License
public void index(String data, String schemaPath, boolean skipHeader, String pluginClass) throws IOException, InterruptedException, ParseException { FieldMapping fm = FieldMapping.getInstance(schemaPath); //create table in hbase System.out.println("Accessing HBase"); HBaseAdmin admin = HBaseFacade.getInstance().getAdmin(); System.out.println(" Creating Table " + fm.tableName); if (!admin.tableExists(fm.tableName)) createTable(fm.tableName, fm.familyName); System.out.println(fm.tableName + " Table is created"); KVMapperLocal mapper = new KVMapperLocal(); LocalMapContext ctxM = mapper.getContext(); ctxM.getConfiguration().set(KVIndexer.XML_FILE_PATH, schemaPath); mapper.setupRouter(ctxM);//from www. ja v a2s. c o m AbstractList records = new AbstractList() { @Override public boolean add(String record) { if (skipHeader) { skipHeader = false; return false; } try { mapper.mapRouter(new LongWritable(i++), new Text(record), ctxM); } catch (Exception ex) { ex.printStackTrace(System.err); } return true; } }; records.ctxM = ctxM; records.mapper = mapper; records.skipHeader = skipHeader; LineReaderUtil.fastSplit(records, data, '\n'); /** * Reducer Starts here */ KVReducerLocal reducer = new KVReducerLocal(); LocalReduceContext ctxR = reducer.getContext(); ctxR.getConfiguration().set(KVIndexer.XML_FILE_PATH, schemaPath); reducer.setupRouter(ctxR); for (TextPair keyPair : ctxM.values.keySet()) { reducer.reduceRouter(keyPair, ctxM.values.get(keyPair), ctxR); } System.out.println("Data is uploaded sucessfully"); }
From source file:com.caseystella.analytics.distribution.RotationTest.java
License:Apache License
@Test public void rotationTest() throws Exception { OutlierConfig config = JSONUtil.INSTANCE.load(amountConfig, OutlierConfig.class); final IntWritable numChunksAdded = new IntWritable(0); final IntWritable numRotations = new IntWritable(0); Distribution.Context context = new Distribution.Context(0, 0) { @Override//w w w .ja v a 2 s . com protected void addChunk(Distribution d) { super.addChunk(d); numChunksAdded.set(numChunksAdded.get() + 1); } @Override protected void rotate() { super.rotate(); numRotations.set(numRotations.get() + 1); } }; GlobalStatistics globalStats = new GlobalStatistics(); Random r = new Random(0); List<DataPoint> points = new ArrayList<>(); DescriptiveStatistics stats = new DescriptiveStatistics(); LongWritable ts = new LongWritable(0L); Assert.assertEquals(context.getAmount(), 0); context.addDataPoint(nextDataPoint(r, ts, 1, points), config.getRotationPolicy(), config.getChunkingPolicy(), config.getScalingFunction(), globalStats); Assert.assertEquals(context.getAmount(), 1); Assert.assertEquals(context.getChunks().size(), 1); Assert.assertEquals(numChunksAdded.get(), 1); Assert.assertEquals(numRotations.get(), 0); for (int i = 1; i < 10; ++i) { context.addDataPoint(nextDataPoint(r, ts, 1, points), config.getRotationPolicy(), config.getChunkingPolicy(), config.getScalingFunction(), globalStats); Assert.assertEquals(context.getChunks().size(), 1); } //at the 11th point, we should create a new chunk context.addDataPoint(nextDataPoint(r, ts, 1, points), config.getRotationPolicy(), config.getChunkingPolicy(), config.getScalingFunction(), globalStats); Assert.assertEquals(context.getChunks().size(), 2); Assert.assertEquals(numChunksAdded.get(), 2); Assert.assertEquals(context.getAmount(), 11); Assert.assertEquals(numRotations.get(), 0); for (int i = 12; i <= 110; ++i) { context.addDataPoint(nextDataPoint(r, ts, 1, points), config.getRotationPolicy(), config.getChunkingPolicy(), config.getScalingFunction(), globalStats); } Assert.assertEquals(11, numChunksAdded.get()); Assert.assertEquals(0, numRotations.get()); //at the 111th point, we should create a rotation context.addDataPoint(nextDataPoint(r, ts, 1, points), config.getRotationPolicy(), config.getChunkingPolicy(), config.getScalingFunction(), globalStats); Assert.assertEquals(12, numChunksAdded.get()); Assert.assertEquals(11, context.getChunks().size()); Assert.assertEquals(1, numRotations.get()); //rotates just past the rotation cutoff (ensuring that we keep at least the last 100 entries in there) Assert.assertEquals(context.getAmount(), 101); for (int i = 111; i <= 150; ++i) { context.addDataPoint(nextDataPoint(r, ts, 1, points), config.getRotationPolicy(), config.getChunkingPolicy(), config.getScalingFunction(), globalStats); } //no matter how far we go in the stream, we always stay at 11 chunks and a total number of values in the distribution of <= 110 (i.e. between the cutoff and cutoff + a chunk) Assert.assertEquals(11, context.getChunks().size()); Assert.assertTrue(context.getAmount() <= 110); }
From source file:com.caseystella.analytics.outlier.streaming.mad.SketchyMovingMADIntegrationTest.java
License:Apache License
@Test public void runAccuracyBenchmark() throws IOException { Map<String, List<String>> benchmarks = JSONUtil.INSTANCE.load( new FileInputStream(new File(new File(benchmarkRoot), "combined_labels.json")), new TypeReference<Map<String, List<String>>>() { });// w w w . ja v a2s.com Assert.assertTrue(benchmarks.size() > 0); Map<ConfusionMatrix.ConfusionEntry, Long> overallConfusionMatrix = new HashMap<>(); DescriptiveStatistics globalExpectedScores = new DescriptiveStatistics(); long total = 0; for (Map.Entry<String, List<String>> kv : benchmarks.entrySet()) { File dataFile = new File(new File(benchmarkRoot), kv.getKey()); File plotFile = new File(new File(benchmarkRoot), kv.getKey() + ".dat"); Assert.assertTrue(dataFile.exists()); Set<Long> expectedOutliers = Sets.newHashSet(Iterables.transform(kv.getValue(), STR_TO_TS)); OutlierRunner runner = new OutlierRunner(outlierConfig, extractorConfigStr); final LongWritable numObservations = new LongWritable(0); final LongWritable lastTimestamp = new LongWritable(Long.MIN_VALUE); final DescriptiveStatistics timeDiffStats = new DescriptiveStatistics(); final Map<Long, Outlier> outlierMap = new HashMap<>(); final PrintWriter pw = new PrintWriter(plotFile); List<Outlier> outliers = runner.run(dataFile, 1, EnumSet.of(Severity.SEVERE_OUTLIER), new Function<Map.Entry<DataPoint, Outlier>, Void>() { @Nullable @Override public Void apply(@Nullable Map.Entry<DataPoint, Outlier> kv) { DataPoint dataPoint = kv.getKey(); Outlier outlier = kv.getValue(); pw.println(dataPoint.getTimestamp() + " " + outlier.getDataPoint().getValue() + " " + ((outlier.getSeverity() == Severity.SEVERE_OUTLIER) ? "outlier" : "normal")); outlierMap.put(dataPoint.getTimestamp(), outlier); numObservations.set(numObservations.get() + 1); if (lastTimestamp.get() != Long.MIN_VALUE) { timeDiffStats.addValue(dataPoint.getTimestamp() - lastTimestamp.get()); } lastTimestamp.set(dataPoint.getTimestamp()); return null; } }); pw.close(); total += numObservations.get(); Set<Long> calculatedOutliers = Sets .newHashSet(Iterables.transform(outliers, OutlierRunner.OUTLIER_TO_TS)); double stdDevDiff = Math.sqrt(timeDiffStats.getVariance()); System.out.println("Running data from " + kv.getKey() + " - E[time delta]: " + ConfusionMatrix.timeConversion((long) timeDiffStats.getMean()) + ", StdDev[time delta]: " + ConfusionMatrix.timeConversion((long) stdDevDiff) + " mean: " + runner.getMean()); Map<ConfusionMatrix.ConfusionEntry, Long> confusionMatrix = ConfusionMatrix.getConfusionMatrix( expectedOutliers, calculatedOutliers, numObservations, (long) timeDiffStats.getMean(), 3 //stdDevDiff > 30000?0:3 , outlierMap, globalExpectedScores); ConfusionMatrix.printConfusionMatrix(confusionMatrix); overallConfusionMatrix = ConfusionMatrix.merge(overallConfusionMatrix, confusionMatrix); } System.out.println("Really ran " + total); ConfusionMatrix.printConfusionMatrix(overallConfusionMatrix); ConfusionMatrix.printStats("Global Expected Outlier Scores", globalExpectedScores); }
From source file:com.cg.mapreduce.fpgrowth.mahout.fpm.ParallelCountingReducer.java
License:Apache License
@Override protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { long sum = 0; for (LongWritable value : values) { context.setStatus("Parallel Counting Reducer :" + key); sum += value.get();/*w w w . ja va2 s. c o m*/ } context.setStatus("Parallel Counting Reducer: " + key + " => " + sum); context.write(key, new LongWritable(sum)); }
From source file:com.cg.mapreduce.fpgrowth.mahout.fpm.PFPGrowth.java
License:Apache License
/** * Serializes the fList and returns the string representation of the List *//* w w w . j a va 2 s.co m*/ public static void saveFList(Iterable<Pair<String, Long>> flist, Parameters params, Configuration conf) throws IOException { Path flistPath = new Path(params.get(OUTPUT), F_LIST); FileSystem fs = FileSystem.get(flistPath.toUri(), conf); flistPath = fs.makeQualified(flistPath); HadoopUtil.delete(conf, flistPath); SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, flistPath, Text.class, LongWritable.class); try { for (Pair<String, Long> pair : flist) { writer.append(new Text(pair.getFirst()), new LongWritable(pair.getSecond())); } } finally { writer.close(); } DistributedCache.addCacheFile(flistPath.toUri(), conf); }
From source file:com.chinamobile.bcbsp.bspcontroller.Counters.java
License:Apache License
/** * Logs the current counter values./* ww w. j av a 2s.c o m*/ * @param log * The log to use. */ public void log(Log log) { log.info("Counters: " + size()); for (Group group : this) { log.info(" " + group.getDisplayName()); for (Counter counter : group) { log.info(counter.getDisplayName() + "=" + counter.getCounter()); countersmap.put(new Text(counter.getDisplayName()), new LongWritable(counter.getCounter())); } } }
From source file:com.chinamobile.bcbsp.bspcontroller.JobInProgress.java
License:Apache License
/** * update the job counters status/*from w ww .j a va2 s.c o m*/ * @author cui */ public void updCountersTJobs() { for (Group group : this.counters) { for (Counter counter : group) { LOG.info(counter.getDisplayName() + "=" + counter.getCounter()); this.mapcounterTemp.put(new Text(counter.getDisplayName()), new LongWritable(counter.getCounter())); } } this.status.setMapcounter(mapcounterTemp); }
From source file:com.cloudera.crunch.type.writable.WritablesTest.java
License:Open Source License
@Test public void testLongs() throws Exception { long j = 55;/*from w w w . ja v a 2 s .c o m*/ LongWritable w = new LongWritable(j); testInputOutputFn(Writables.longs(), j, w); }
From source file:com.cloudera.impala.hive.executor.TestUdf.java
License:Apache License
public LongWritable evaluate(LongWritable a) { if (a == null) return null; return new LongWritable(a.get()); }