List of usage examples for org.apache.commons.math3.stat.descriptive DescriptiveStatistics addValue
public void addValue(double v)
From source file:com.linkedin.pinot.perf.ForwardIndexReaderBenchmark.java
public static void singleValuedReadBenchMarkV1(File file, int numDocs, int columnSizeInBits) throws Exception { boolean signed = false; boolean isMmap = false; PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "benchmark"); BaseSingleColumnSingleValueReader reader = new com.linkedin.pinot.core.io.reader.impl.v1.FixedBitSingleValueReader( heapBuffer, numDocs, columnSizeInBits, signed); // sequential read long start, end; DescriptiveStatistics stats = new DescriptiveStatistics(); for (int run = 0; run < MAX_RUNS; run++) { start = System.currentTimeMillis(); for (int i = 0; i < numDocs; i++) { int value = reader.getInt(i); }/*from w w w. j av a 2 s .c o m*/ end = System.currentTimeMillis(); stats.addValue(end - start); } System.out.println(" v1 sequential read stats for " + file.getName()); System.out.println(stats.toString().replaceAll("\n", ", ") + " raw:" + Arrays.toString(stats.getValues())); reader.close(); heapBuffer.close(); }
From source file:main.java.repartition.SimpleTr.java
static double getDeltaLb(Cluster cluster, SimpleTr t, MigrationPlan m) { // Before migration DescriptiveStatistics current_server_data = new DescriptiveStatistics(); for (Server s : cluster.getServers()) if (t.serverDataSet.containsKey(s.getServer_id())) current_server_data.addValue(s.getServer_total_data()); // After migration DescriptiveStatistics expected_server_data = new DescriptiveStatistics(); for (Server s : cluster.getServers()) { if (t.serverDataSet.containsKey(s.getServer_id())) { if (m.fromSet.contains(s.getServer_id())) { int data_count = s.getServer_total_data() - t.serverDataSet.get(s.getServer_id()).size(); expected_server_data.addValue(data_count); } else if (m.to == s.getServer_id()) { int data_count = s.getServer_total_data() + t.serverDataSet.get(s.getServer_id()).size(); expected_server_data.addValue(data_count); }// w ww.j ava 2 s . c om } } // Calculate total lb double variance = current_server_data.getVariance() - expected_server_data.getVariance(); // Calculate delta lb double mu = current_server_data.getMean(); double delta_lb = (double) (variance / (mu * Math.sqrt(Global.servers - 1))); return delta_lb; //return ((double) delta_lb/m.req_data_mgr); }
From source file:com.linkedin.pinot.perf.ForwardIndexReaderBenchmark.java
public static void multiValuedReadBenchMarkV2(File file, int numDocs, int totalNumValues, int maxEntriesPerDoc, int columnSizeInBits) throws Exception { boolean signed = false; boolean isMmap = false; boolean readOneEachTime = true; PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "benchmarking"); com.linkedin.pinot.core.io.reader.impl.v2.FixedBitMultiValueReader reader = new com.linkedin.pinot.core.io.reader.impl.v2.FixedBitMultiValueReader( heapBuffer, numDocs, totalNumValues, columnSizeInBits, signed); int[] intArray = new int[maxEntriesPerDoc]; long start, end; // read one entry at a time if (readOneEachTime) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (int run = 0; run < MAX_RUNS; run++) { start = System.currentTimeMillis(); for (int i = 0; i < numDocs; i++) { int length = reader.getIntArray(i, intArray); }// w w w. ja va 2 s . c om end = System.currentTimeMillis(); stats.addValue((end - start)); } System.out.println("v2 multi value sequential read one stats for " + file.getName()); System.out.println( stats.toString().replaceAll("\n", ", ") + " raw:" + Arrays.toString(stats.getValues())); } reader.close(); heapBuffer.close(); }
From source file:main.java.metric.Metric.java
public static void getPartitionStatistic(Cluster cluster) { DescriptiveStatistics _data = new DescriptiveStatistics(); DescriptiveStatistics _inflow = new DescriptiveStatistics(); DescriptiveStatistics _outflow = new DescriptiveStatistics(); for (Partition partition : cluster.getPartitions()) { _data.addValue(partition.getPartition_dataSet().size()); _inflow.addValue(partition.getPartition_inflow()); _outflow.addValue(partition.getPartition_outflow()); }/* w w w . j a v a 2s . c o m*/ mean_partition_inflow.add(_inflow.getMean()); mean_partition_outflow.add(_outflow.getMean()); mean_partition_data.add(_data.getMean()); sd_partition_data.add(_data.getStandardDeviation()); }
From source file:de.tudarmstadt.ukp.experiments.argumentation.convincingness.sampling.Step1DebateFilter.java
/** * Computes statistics over full debate dataset * * @param inputDir input dir//from w w w. ja v a 2 s . com * @throws IOException IO exception */ public static void computeRawLengthStatistics(String inputDir) throws IOException { DescriptiveStatistics fullWordCountStatistics = new DescriptiveStatistics(); // read all debates and filter them for (File file : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { Debate debate = DebateSerializer.deserializeFromXML(FileUtils.readFileToString(file, "utf-8")); for (Argument argument : debate.getArgumentList()) { // we have a first-level argument if (argument.getParentId() == null) { // now check the length int wordCount = argument.getText().split("\\s+").length; fullWordCountStatistics.addValue(wordCount); } } } System.out.println("Full word count statistics"); System.out.println(fullWordCountStatistics); }
From source file:main.java.metric.Metric.java
public static void getServerStatistic(Cluster cluster) { DescriptiveStatistics _data = new DescriptiveStatistics(); DescriptiveStatistics _inflow = new DescriptiveStatistics(); DescriptiveStatistics _outflow = new DescriptiveStatistics(); DescriptiveStatistics _meDMV = new DescriptiveStatistics(); for (Server server : cluster.getServers()) { _data.addValue(server.getServer_total_data()); _inflow.addValue(server.getServer_inflow()); _outflow.addValue(server.getServer_outflow()); }//from ww w . j a v a2 s . co m // New - mutually exclusive server sets ArrayList<Integer> estimated_dmgr = new ArrayList<Integer>(); for (Entry<Pair<Integer, Integer>, Integer> entry : mutually_exclusive_serverSets.entrySet()) { _meDMV.addValue(entry.getValue()); estimated_dmgr.add(entry.getValue()); } // Sort the list containing data migration counts within individual server pairs Collections.sort(estimated_dmgr); // Sum up every floor(S/2) index values to get the estimated data migration time int estimated_mgr = 0; int j = 0; for (int i = 0; i < estimated_dmgr.size(); ++i) { j = (int) (i + Math.floor(Global.servers / 2)); estimated_mgr += estimated_dmgr.get(i); i = j; } mean_server_inflow.add(_inflow.getMean()); mean_server_outflow.add(_outflow.getMean()); mean_server_data.add(_data.getMean()); sd_server_data.add(_data.getStandardDeviation()); total_data.add((long) Global.global_dataCount); mean_pairwise_inter_server_data_mgr.add(_meDMV.getMean()); sd_pairwise_inter_server_data_mgr.add(_meDMV.getStandardDeviation()); estimated_data_mgr.add(estimated_mgr); }
From source file:edu.ucsc.barrel.cdf_gen.CDF_Gen.java
public static void fill511Gaps() { int size = data.getSize("mod32"), step_size = 1, start = 0; double delta, std_dev, med; float m, b, //values used for interpolating data fill = (Float) CDFVar.getIstpVal("FLOAT_FILL"), new_value = fill, last_value = fill; DescriptiveStatistics stats = new DescriptiveStatistics(); //generate statistics on the 511 peak jump sizes for (int peak_i = 0; peak_i < (size - 1); peak_i++) { if (data.peak511_bin[peak_i] == fill) { continue; }// w w w . ja v a 2 s.c om if (data.peak511_bin[peak_i + 1] == fill) { continue; } delta = data.peak511_bin[peak_i + 1] - data.peak511_bin[peak_i]; if (delta != 0) { stats.addValue(delta); } } std_dev = stats.getStandardDeviation(); med = stats.getPercentile(50); //find first good value for (start = 0; start < size; start++) { if (data.peak511_bin[start] != fill) { new_value = data.peak511_bin[start]; last_value = data.peak511_bin[start]; break; } } //fill any missing data before the first point Arrays.fill(data.peak511_bin, 0, start, new_value); for (int filler_i = start + 1; filler_i < size; filler_i++) { if (data.peak511_bin[filler_i] == fill) { //temporarily fill the gap with the last good value //this is done in case there is not another good value //to use for interpolation data.peak511_bin[filler_i] = last_value; step_size++; } else { //make sure jump size wasn't too big delta = data.peak511_bin[filler_i] - data.peak511_bin[filler_i - 1]; // if(Math.abs(delta - med) > (std_dev * 3)){ // data.peak511_bin[filler_i] = last_value; // step_size++; // } last_value = new_value; new_value = data.peak511_bin[filler_i]; //fill any gaps if (step_size > 1) { m = (last_value - new_value) / step_size; b = new_value - (m * filler_i); for (int fill_i = filler_i - step_size; fill_i < filler_i; fill_i++) { data.peak511_bin[fill_i] = m * fill_i + b; } step_size = 1; } } } }
From source file:com.fpuna.preproceso.PreprocesoTS.java
private static TrainingSetFeature calculoFeaturesMagnitud(List<Registro> muestras, String activity) { TrainingSetFeature Feature = new TrainingSetFeature(); DescriptiveStatistics stats_m = new DescriptiveStatistics(); double[] fft_m; double[] AR_4; muestras = Util.calcMagnitud(muestras); for (int i = 0; i < muestras.size(); i++) { stats_m.addValue(muestras.get(i).getM_1()); }//from w w w . jav a 2 s . c om //********* FFT ********* //fft_m = Util.transform(stats_m.getValues()); fft_m = FFTMixedRadix.fftPowerSpectrum(stats_m.getValues()); //******************* Calculos Magnitud *******************// //mean(s) - Arithmetic mean System.out.print(stats_m.getMean() + ","); Feature.setMeanX((float) stats_m.getMean()); //std(s) - Standard deviation System.out.print(stats_m.getStandardDeviation() + ","); Feature.setStdX((float) stats_m.getStandardDeviation()); //mad(s) - Median absolute deviation // //max(s) - Largest values in array System.out.print(stats_m.getMax() + ","); Feature.setMaxX((float) stats_m.getMax()); //min(s) - Smallest value in array System.out.print(stats_m.getMin() + ","); Feature.setMinX((float) stats_m.getMin()); //skewness(s) - Frequency signal Skewness System.out.print(stats_m.getSkewness() + ","); Feature.setSkewnessX((float) stats_m.getSkewness()); //kurtosis(s) - Frequency signal Kurtosis System.out.print(stats_m.getKurtosis() + ","); Feature.setKurtosisX((float) stats_m.getKurtosis()); //energy(s) - Average sum of the squares System.out.print(stats_m.getSumsq() / stats_m.getN() + ","); Feature.setEnergyX((float) (stats_m.getSumsq() / stats_m.getN())); //entropy(s) - Signal Entropy System.out.print(Util.calculateShannonEntropy(fft_m) + ","); Feature.setEntropyX(Util.calculateShannonEntropy(fft_m).floatValue()); //iqr (s) Interquartile range System.out.print(stats_m.getPercentile(75) - stats_m.getPercentile(25) + ","); Feature.setIqrX((float) (stats_m.getPercentile(75) - stats_m.getPercentile(25))); try { //autoregression (s) -4th order Burg Autoregression coefficients AR_4 = AutoRegression.calculateARCoefficients(stats_m.getValues(), 4, true); System.out.print(AR_4[0] + ","); System.out.print(AR_4[1] + ","); System.out.print(AR_4[2] + ","); System.out.print(AR_4[3] + ","); Feature.setArX1((float) AR_4[0]); Feature.setArX2((float) AR_4[1]); Feature.setArX3((float) AR_4[2]); Feature.setArX4((float) AR_4[3]); } catch (Exception ex) { Logger.getLogger(PreprocesoTS.class.getName()).log(Level.SEVERE, null, ex); } //meanFreq(s) - Frequency signal weighted average System.out.print(Util.meanFreq(fft_m, stats_m.getValues()) + ","); Feature.setMeanFreqx((float) Util.meanFreq(fft_m, stats_m.getValues())); //******************* Actividad *******************/ System.out.print(activity); System.out.print("\n"); Feature.setEtiqueta(activity); return Feature; }
From source file:com.linkedin.pinot.perf.ForwardIndexReaderBenchmark.java
public static void multiValuedReadBenchMarkV1(File file, int numDocs, int totalNumValues, int maxEntriesPerDoc, int columnSizeInBits) throws Exception { System.out.println("******************************************************************"); System.out.println("Analyzing " + file.getName() + " numDocs:" + numDocs + ", totalNumValues:" + totalNumValues + ", maxEntriesPerDoc:" + maxEntriesPerDoc + ", numBits:" + columnSizeInBits); long start, end; boolean readFile = true; boolean randomRead = true; boolean contextualRead = true; boolean signed = false; boolean isMmap = false; PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "benchmarking"); BaseSingleColumnMultiValueReader reader = new com.linkedin.pinot.core.io.reader.impl.v1.FixedBitMultiValueReader( heapBuffer, numDocs, totalNumValues, columnSizeInBits, signed); int[] intArray = new int[maxEntriesPerDoc]; File outfile = new File("/tmp/" + file.getName() + ".raw"); FileWriter fw = new FileWriter(outfile); for (int i = 0; i < numDocs; i++) { int length = reader.getIntArray(i, intArray); StringBuilder sb = new StringBuilder(); String delim = ""; for (int j = 0; j < length; j++) { sb.append(delim);//from w ww. j a v a 2s . com sb.append(intArray[j]); delim = ","; } fw.write(sb.toString()); fw.write("\n"); } fw.close(); // sequential read if (readFile) { DescriptiveStatistics stats = new DescriptiveStatistics(); RandomAccessFile raf = new RandomAccessFile(file, "rw"); ByteBuffer buffer = ByteBuffer.allocateDirect((int) file.length()); raf.getChannel().read(buffer); for (int run = 0; run < MAX_RUNS; run++) { long length = file.length(); start = System.currentTimeMillis(); for (int i = 0; i < length; i++) { byte b = buffer.get(i); } end = System.currentTimeMillis(); stats.addValue((end - start)); } System.out.println("v1 multi value read bytes stats for " + file.getName()); System.out.println( stats.toString().replaceAll("\n", ", ") + " raw:" + Arrays.toString(stats.getValues())); raf.close(); } if (randomRead) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (int run = 0; run < MAX_RUNS; run++) { start = System.currentTimeMillis(); for (int i = 0; i < numDocs; i++) { int length = reader.getIntArray(i, intArray); } end = System.currentTimeMillis(); stats.addValue((end - start)); } System.out.println("v1 multi value sequential read one stats for " + file.getName()); System.out.println( stats.toString().replaceAll("\n", ", ") + " raw:" + Arrays.toString(stats.getValues())); } if (contextualRead) { DescriptiveStatistics stats = new DescriptiveStatistics(); for (int run = 0; run < MAX_RUNS; run++) { MultiValueReaderContext context = (MultiValueReaderContext) reader.createContext(); start = System.currentTimeMillis(); for (int i = 0; i < numDocs; i++) { int length = reader.getIntArray(i, intArray, context); } end = System.currentTimeMillis(); // System.out.println("RUN:" + run + "Time:" + (end-start)); stats.addValue((end - start)); } System.out.println("v1 multi value sequential read one with context stats for " + file.getName()); System.out.println( stats.toString().replaceAll("\n", ", ") + " raw:" + Arrays.toString(stats.getValues())); } reader.close(); heapBuffer.close(); System.out.println("******************************************************************"); }
From source file:io.hops.experiments.results.compiler.InterleavedBMResultsAggregator.java
public static InterleavedBMResults processInterleavedResults(Collection<Object> responses, Configuration args) throws FileNotFoundException, IOException, InterruptedException { Map<BenchmarkOperations, double[][]> allOpsPercentiles = new HashMap<BenchmarkOperations, double[][]>(); System.out.println("Processing the results "); DescriptiveStatistics successfulOps = new DescriptiveStatistics(); DescriptiveStatistics failedOps = new DescriptiveStatistics(); DescriptiveStatistics speed = new DescriptiveStatistics(); DescriptiveStatistics duration = new DescriptiveStatistics(); DescriptiveStatistics opsLatency = new DescriptiveStatistics(); DescriptiveStatistics noOfNNs = new DescriptiveStatistics(); for (Object obj : responses) { if (!(obj instanceof InterleavedBenchmarkCommand.Response)) { throw new IllegalStateException("Wrong response received from the client"); } else {/*from w ww . ja v a 2 s .c om*/ InterleavedBenchmarkCommand.Response response = (InterleavedBenchmarkCommand.Response) obj; successfulOps.addValue(response.getTotalSuccessfulOps()); failedOps.addValue(response.getTotalFailedOps()); speed.addValue(response.getOpsPerSec()); duration.addValue(response.getRunTime()); opsLatency.addValue(response.getAvgOpLatency()); noOfNNs.addValue(response.getNnCount()); } } //write the response objects to files. //these files are processed by CalculatePercentiles.java int responseCount = 0; for (Object obj : responses) { if (!(obj instanceof InterleavedBenchmarkCommand.Response)) { throw new IllegalStateException("Wrong response received from the client"); } else { String filePath = args.getResultsDir(); if (!filePath.endsWith("/")) { filePath += "/"; } InterleavedBenchmarkCommand.Response response = (InterleavedBenchmarkCommand.Response) obj; filePath += "ResponseRawData" + responseCount++ + ConfigKeys.RAW_RESPONSE_FILE_EXT; System.out.println("Writing Rwaw results to " + filePath); FileOutputStream fout = new FileOutputStream(filePath); ObjectOutputStream oos = new ObjectOutputStream(fout); oos.writeObject(response); oos.close(); } } InterleavedBMResults result = new InterleavedBMResults(args.getNamenodeCount(), (int) Math.floor(noOfNNs.getMean()), args.getNdbNodesCount(), args.getInterleavedBmWorkloadName(), (successfulOps.getSum() / ((duration.getMean() / 1000))), (duration.getMean() / 1000), (successfulOps.getSum()), (failedOps.getSum()), allOpsPercentiles, opsLatency.getMean()); // // failover testing // if(args.testFailover()){ // if(responses.size() != 1){ // throw new UnsupportedOperationException("Currently we only support failover testing for one slave machine"); // } // // String prefix = args.getBenchMarkFileSystemName().toString(); // if(args.getBenchMarkFileSystemName() == BenchMarkFileSystemName.HopsFS){ // prefix+="-"+args.getNameNodeSelectorPolicy(); // } // // final String outputFolder = args.getResultsDir(); // InterleavedBenchmarkCommand.Response response = (InterleavedBenchmarkCommand.Response)responses.iterator().next(); // // // StringBuilder sb = new StringBuilder(); // for(String data : response.getFailOverLog()){ // sb.append(data).append("\n"); // } // // String datFile = prefix+"-failover.dat"; // CompileResults.writeToFile(outputFolder+"/"+datFile, sb.toString(), false); // // // StringBuilder plot = new StringBuilder("set terminal postscript eps enhanced color font \"Helvetica,18\" #monochrome\n"); // plot.append( "set output '| ps2pdf - failover.pdf'\n"); // plot.append( "#set size 1,0.75 \n "); // plot.append( "set ylabel \"ops/sec\" \n"); // plot.append( "set xlabel \"Time (sec)\" \n"); // plot.append( "set format y \"%.0s%c\"\n"); // // // StringBuilder sbx = new StringBuilder(); // String oldPt = ""; // for(String data : response.getFailOverLog()){ // // if(data.startsWith("#")) { // StringTokenizer st = new StringTokenizer(oldPt); // long time = Long.parseLong(st.nextToken()); // long spd = Long.parseLong(st.nextToken()); // sbx.append("set label 'NN-Restart' at "+time+","+spd+" rotate by 270").append("\n"); // } // oldPt = data; // } // plot.append(sbx.toString()); // // // plot.append( "plot '"+datFile+"' with linespoints ls 1"); // CompileResults.writeToFile(outputFolder+"/"+prefix+"-failover.gnu", plot.toString(), false); // // } return result; }