List of usage examples for org.apache.commons.lang.time StopWatch getTime
public long getTime()
Get the time on the stopwatch.
This is either the time between the start and the moment this method is called, or the amount of time between start and stop.
From source file:ubic.gemma.analysis.expression.coexpression.links.LinkAnalysisServiceImpl.java
/** * Persist the links to the database. This takes care of saving a 'flipped' version of the links. * <p>/*from www . j a v a 2 s .com*/ * Note that if "known genes only" is set, then links between probes that _only_ target other types of genes will * not be stored. However, because the links are stored at the probe level, the links saved will potentially * includes links between any type of gene (but always at least between two known genes). * * @param p2v * @param la */ private void saveLinks(Map<CompositeSequence, ProcessedExpressionDataVector> p2v, LinkAnalysis la) { if (useDB) { deleteOldLinks(la); // Complete and persist the new analysis object. fillInNodeDegree(la); ProbeCoexpressionAnalysis analysisObj = la.getAnalysisObj(); analysisObj = (ProbeCoexpressionAnalysis) persisterHelper.persist(analysisObj); la.setAnalysisObj(analysisObj); } log.info("Start submitting data to database."); StopWatch watch = new StopWatch(); watch.start(); ObjectArrayList links = la.getKeep(); /* * Important implementation note: For efficiency reason, it is important that they be stored in order of "x" * (where x is the index of the first DEDV in the correlation matrix, not the ID of the DEDV.) in the database: * so all all links with probe=x are clustered. (the actual order of x1 vs x2 doesn't matter, just so long they * are clustered). This makes retrievals much faster for the most common types of queries. Note that this relies * on expected behavior that links.sort() will sort by the x coordinate. It also requires that when we persist * the elements, we retain the order. */ links.sort(); if (log.isDebugEnabled()) { for (Object link : links.elements()) { if (link == null) continue; log.debug(((Link) link).getx() + " " + ((Link) link).gety()); } } if (useDB) { saveLinks(p2v, la, links, false); } /* * now create 'reversed' links, first by sorting by the 'y' coordinate. Again, this sort is critical to keep the * links in an ordering that the RDBMS can use efficiently. */ links.mergeSortFromTo(0, links.size() - 1, new Comparator<Link>() { @Override public int compare(Link a, Link b) { if (a == null || b == null) return 1; if (a.gety() < b.gety()) { return -1; } else if (a.gety() > b.gety()) { return 1; } else { if (a.getx() < b.getx()) { return -1; } else if (a.getx() > b.getx()) { return 1; } return 0; } } }); log.info("Saving flipped links"); if (log.isDebugEnabled()) { for (Object link : links.elements()) { if (link == null) continue; log.debug(((Link) link).getx() + " " + ((Link) link).gety()); } } if (useDB) { saveLinks(p2v, la, links, true); } watch.stop(); log.info("Seconds to process " + links.size() + " links plus flipped versions:" + watch.getTime() / 1000.0); }
From source file:ubic.gemma.analysis.expression.coexpression.links.MatrixRowPairPearsonAnalysis.java
/** * Calculate the linear correlation matrix of a matrix, allowing missing values. If there are no missing values, * this calls PearsonFast.// ww w .ja v a2 s. com */ @Override public void calculateMetrics() { if (this.numMissing == 0) { calculateMetricsFast(); return; } int numused; int numrows = this.dataMatrix.rows(); int numcols = this.dataMatrix.columns(); if (numcols < this.minNumUsed) { throw new IllegalArgumentException("Sorry, correlations will not be computed unless there are at least " + this.minNumUsed + " mutually present data points per vector pair, current data has only " + numcols + " columns."); } boolean docalcs = this.needToCalculateMetrics(); boolean[][] usedB = new boolean[][] {}; double[][] data = new double[][] {}; if (docalcs) { // Temporarily copy the data in this matrix, for performance. usedB = new boolean[numrows][numcols]; data = new double[numrows][numcols]; for (int i = 0; i < numrows; i++) { // first vector for (int j = 0; j < numcols; j++) { // second vector usedB[i][j] = used.get(i, j); // this is only needed if we use it below, speeds things up // slightly. data[i][j] = this.dataMatrix.get(i, j); } } rowStatistics(); } /* for each vector, compare it to all other vectors */ ExpressionDataMatrixRowElement itemA = null; StopWatch timer = new StopWatch(); timer.start(); double[] vectorA = new double[] {}; double syy, sxy, sxx, sx, sy, xj, yj; int skipped = 0; int numComputed = 0; for (int i = 0; i < numrows; i++) { // first vector itemA = this.dataMatrix.getRowElement(i); if (!this.hasGene(itemA)) { skipped++; continue; } if (docalcs) { vectorA = data[i]; } boolean thisRowHasMissing = hasMissing[i]; for (int j = i + 1; j < numrows; j++) { // second vector ExpressionDataMatrixRowElement itemB = this.dataMatrix.getRowElement(j); if (!this.hasGene(itemB)) continue; // second pass over matrix? Don't calculate it if we already have it. Just do the requisite checks. if (!docalcs || results.get(i, j) != 0.0) { keepCorrel(i, j, results.get(i, j), numcols); continue; } double[] vectorB = data[j]; /* if there are no missing values, use the faster method of calculation */ if (!thisRowHasMissing && !hasMissing[j]) { setCorrel(i, j, correlFast(vectorA, vectorB, i, j), numcols); continue; } /* do it the old fashioned way */ numused = 0; sxy = 0.0; sxx = 0.0; syy = 0.0; sx = 0.0; sy = 0.0; for (int k = 0; k < numcols; k++) { xj = vectorA[k]; yj = vectorB[k]; if (usedB[i][k] && usedB[j][k]) { /* this is a bit faster than calling Double.isNan */ sx += xj; sy += yj; sxy += xj * yj; sxx += xj * xj; syy += yj * yj; numused++; } } // avoid -1 correlations or extremely noisy values (minNumUsed should be set high enough so that degrees // of freedom isn't too low. if (numused < this.minNumUsed) setCorrel(i, j, Double.NaN, 0); else { double denom = correlationNorm(numused, sxx, sx, syy, sy); if (denom <= 0.0) { // means variance is zero for one of the vectors. setCorrel(i, j, 0.0, numused); } else { double correl = (sxy - sx * sy / numused) / Math.sqrt(denom); setCorrel(i, j, correl, numused); } } ++numComputed; } if ((i + 1) % 2000 == 0) { double t = timer.getTime() / 1000.0; log.info((i + 1) + " rows done, " + numComputed + " correlations computed, last row was " + itemA + " " + (keepers.size() > 0 ? keepers.size() + " scores retained" : "") + String.format(", time elapsed since last check: %.2f", t) + "s"); timer.reset(); timer.start(); } } log.info(skipped + " rows skipped, where probe lacks a gene annotation"); finishMetrics(); }
From source file:ubic.gemma.analysis.expression.coexpression.links.MatrixRowPairPearsonAnalysis.java
/** * Calculate a linear correlation matrix for a matrix. Use this if you know there are no missing values, or don't * care about NaNs. Rows that are not mapped to genes are skipped. * //from ww w . j a va 2 s .c om * @param duplicates The map containing information about what items are the 'same' as other items; such are * skipped. */ private void calculateMetricsFast() { int numrows = this.dataMatrix.rows(); int numcols = this.dataMatrix.columns(); boolean docalcs = this.needToCalculateMetrics(); double[][] data = new double[][] {}; if (docalcs) { rowStatistics(); // Temporarily put the data in this matrix (performance) data = new double[numrows][numcols]; for (int i = 0; i < numrows; i++) { // first vector for (int j = 0; j < numcols; j++) { // second vector data[i][j] = this.dataMatrix.get(i, j); } } } /* * For each vector, compare it to all other vectors, avoid repeating things; skip items that don't have genes * mapped to them. */ StopWatch timer = new StopWatch(); timer.start(); ExpressionDataMatrixRowElement itemA = null; ExpressionDataMatrixRowElement itemB = null; double[] vectorA = null; int skipped = 0; int numComputed = 0; for (int i = 0; i < numrows; i++) { itemA = this.dataMatrix.getRowElement(i); if (!this.hasGene(itemA)) { skipped++; continue; } if (docalcs) { vectorA = data[i]; } for (int j = i + 1; j < numrows; j++) { itemB = this.dataMatrix.getRowElement(j); if (!this.hasGene(itemB)) continue; if (!docalcs || results.get(i, j) != 0.0) { // second pass over matrix. Don't calculate it // if we // already have it. Just do the requisite checks. keepCorrel(i, j, results.get(i, j), numcols); continue; } double[] vectorB = data[j]; setCorrel(i, j, correlFast(vectorA, vectorB, i, j), numcols); ++numComputed; } if ((i + 1) % 2000 == 0) { double t = timer.getTime() / 1000.0; log.info((i + 1) + " rows done, " + numComputed + " correlations computed, last row was " + itemA + " " + (keepers.size() > 0 ? keepers.size() + " scores retained" : "") + String.format(", time elapsed since last check: %.2f", t) + "s"); timer.reset(); timer.start(); } } log.info(skipped + " rows skipped, due to no gene association"); finishMetrics(); }
From source file:ubic.gemma.analysis.expression.coexpression.ProbeLinkCoexpressionAnalyzerImpl.java
@Override public QueryGeneCoexpression linkAnalysis(Gene gene, Collection<? extends BioAssaySet> ees, int inputStringency, int limit) { int stringency = inputStringency <= 0 ? 1 : inputStringency; if (log.isDebugEnabled()) log.debug("Link query for " + gene.getName() + " stringency=" + stringency); /*/*from w w w . j a v a2 s . c o m*/ * Identify data sets the query gene is expressed in - this is fast (?) and provides an upper bound for EEs we * need to search in the first place. */ Collection<BioAssaySet> eesQueryTestedIn = probe2ProbeCoexpressionService .getExpressionExperimentsLinkTestedIn(gene, ees, false); if (eesQueryTestedIn.size() == 0) { QueryGeneCoexpression r = new QueryGeneCoexpression(gene.getId(), stringency); r.setErrorState("No experiments have coexpression data for " + gene.getOfficialSymbol()); return r; } /* * Perform the coexpression search, some postprocessing done. If eesQueryTestedIn is empty, this returns real * quick. */ QueryGeneCoexpression coexpressions = geneService.getCoexpressedGenes(gene, eesQueryTestedIn, stringency); /* * Finish the postprocessing. */ StopWatch timer = new StopWatch(); timer.start(); if (coexpressions.getAllGeneCoexpressionData(stringency).size() == 0) { return coexpressions; } // don't fill in the gene info etc if we're in batch mode. if (limit > 0) { filter(coexpressions, limit, stringency); // remove excess fillInEEInfo(coexpressions); // do first... fillInGeneInfo(stringency, coexpressions); } computeEesTestedIn(ees, coexpressions, eesQueryTestedIn, stringency, limit); timer.stop(); if (timer.getTime() > 1000) { log.info("All Post-Postprocessing: " + timer.getTime() + "ms"); } log.debug("Analysis completed"); return coexpressions; }
From source file:ubic.gemma.analysis.expression.coexpression.ProbeLinkCoexpressionAnalyzerImpl.java
/** * For each experiment, get the genes it tested and populate a map. This is slow; but once we've seen a gene, we * don't have to repeat it. We used to cache the ee->gene relationship, but doing it the other way around yields * must faster code./* ww w. ja v a 2 s. c o m*/ * * @param ees * @param eeIndexMap */ private void cacheEesGeneTestedIn(Collection<? extends BioAssaySet> ees, Map<Long, Integer> eeIndexMap) { assert ees != null; assert eeIndexMap != null; assert !ees.isEmpty(); assert !eeIndexMap.isEmpty(); StopWatch timer = new StopWatch(); timer.start(); int count = 0; for (BioAssaySet ee : ees) { Collection<Long> genes = probe2ProbeCoexpressionService.getGenesTestedBy(ee, false); // inverted map of gene -> ees tested in. Integer indexOfEEInAr = eeIndexMap.get(ee.getId()); assert indexOfEEInAr != null : "No index for EEID=" + ee.getId() + " in the eeIndexMap"; for (Long geneId : genes) { if (!genesTestedIn.containsKey(geneId)) { // initialize the boolean array for this gene. genesTestedIn.put(geneId, new ArrayList<Boolean>()); for (int i = 0; i < ees.size(); i++) { genesTestedIn.get(geneId).add(Boolean.FALSE); } } // flip to true since the gene was tested in the ee genesTestedIn.get(geneId).set(indexOfEEInAr, Boolean.TRUE); } if (++count % 100 == 0) { log.info("Got EEs gene tested in map for " + count + " experiments ... " + timer.getTime() + "ms elapsed"); } } }
From source file:ubic.gemma.analysis.expression.coexpression.ProbeLinkCoexpressionAnalyzerImpl.java
/** * For the genes that the query is coexpressed with. This is limited to the top MAX_GENES_TO_COMPUTE_EESTESTEDIN. * This is not very fast if MAX_GENES_TO_COMPUTE_EESTESTEDIN is large. We use this version for on-line requests. * //from w w w. j av a2s. c o m * @param eesQueryTestedIn, limited to the ees that the query gene is tested in. * @param coexpressionData * @see ProbeLinkCoexpressionAnalyzerImpl.computeEesTestedInBatch for the version used when requests are going to be * done for many genes, so cache is built first time. */ private void computeEesTestedIn(Collection<? extends BioAssaySet> eesQueryTestedIn, List<CoexpressedGenePairValueObject> coexpressionData) { Collection<Long> coexGeneIds = new HashSet<Long>(); StopWatch timer = new StopWatch(); timer.start(); int i = 0; Map<Long, CoexpressedGenePairValueObject> gmap = new HashMap<Long, CoexpressedGenePairValueObject>(); for (CoexpressedGenePairValueObject o : coexpressionData) { coexGeneIds.add(o.getCoexpressedGeneId()); gmap.put(o.getCoexpressedGeneId(), o); i++; if (i >= MAX_GENES_TO_COMPUTE_EESTESTEDIN) break; } log.debug("Computing EEs tested in for " + coexGeneIds.size() + " genes."); Map<Long, Collection<BioAssaySet>> eesTestedIn = probe2ProbeCoexpressionService .getExpressionExperimentsTestedIn(coexGeneIds, eesQueryTestedIn, false); for (Long g : eesTestedIn.keySet()) { CoexpressedGenePairValueObject cvo = gmap.get(g); assert cvo != null; assert eesTestedIn.get(g).size() <= eesQueryTestedIn.size(); Collection<Long> ids = EntityUtils.getIds(eesTestedIn.get(g)); cvo.setDatasetsTestedIn(ids); } timer.stop(); if (timer.getTime() > 1000) { log.info("computeEesTestedIn: " + timer.getTime() + "ms"); } }
From source file:ubic.gemma.analysis.expression.coexpression.ProbeLinkCoexpressionAnalyzerImpl.java
/** * For the genes that the query is coexpressed with; this retrieves the information for all the coexpressionData * passed in (no limit) - all of them have to be for the same query gene! * //from www. ja v a 2s . c o m * @param ees, including ALL experiments that were initially started with, NOT just the ones that the query gene was * tested in. * @param coexpressionData to check, the query gene must be the same for each of them. */ private void computeEesTestedInBatch(Collection<? extends BioAssaySet> ees, List<CoexpressedGenePairValueObject> coexpressionData) { if (coexpressionData.isEmpty()) return; StopWatch timer = new StopWatch(); timer.start(); if (log.isDebugEnabled()) { log.debug("Computing EEs tested in for " + coexpressionData.size() + " genes coexpressed with query."); } /* * Note: we assume this is actually constant as we build a cache based on it. Small risk. */ Map<Long, Integer> eeIndexMap = getOrderingMap(ees); assert eeIndexMap.size() == ees.size(); /* * Save some computation in the inner loop by assuming the query gene is constant. */ CoexpressedGenePairValueObject initializer = coexpressionData.iterator().next(); Long queryGeneId = initializer.getQueryGene(); byte[] queryGeneEETestStatusBytes; if (geneTestStatusByteCache.containsKey(queryGeneId)) { queryGeneEETestStatusBytes = geneTestStatusByteCache.get(queryGeneId); } else { queryGeneEETestStatusBytes = computeTestedDatasetVector(queryGeneId, ees, eeIndexMap); geneTestStatusByteCache.put(queryGeneId, queryGeneEETestStatusBytes); } /* * This is a potential bottleneck, because there are often >500 ees and >1000 cvos, and each EE tests >20000 * genes. Therefore we try hard not to iterate over all the CVOEEs more than we need to. So this is coded very * carefully. Once the cache is warmed up, this still can take over 500ms to run (Feb 2010). The total number of * loops _easily_ exceeds a million. (June 2012: this is rarely taking long enough to trigger the logging, after * it gets warmed up) */ int loopcount = 0; // for performance statistics for (CoexpressedGenePairValueObject cvo : coexpressionData) { Long coexGeneId = cvo.getCoexpressedGeneId(); if (!queryGeneId.equals(cvo.getQueryGene())) { throw new IllegalArgumentException( "All coexpression value objects must have the same query gene here"); } /* * Get the target gene info, from the cache if possible. */ byte[] targetGeneEETestStatus; if (geneTestStatusByteCache.containsKey(coexGeneId)) { targetGeneEETestStatus = geneTestStatusByteCache.get(coexGeneId); } else { targetGeneEETestStatus = computeTestedDatasetVector(coexGeneId, ees, eeIndexMap); geneTestStatusByteCache.put(coexGeneId, targetGeneEETestStatus); } assert targetGeneEETestStatus.length == queryGeneEETestStatusBytes.length; byte[] answer = new byte[targetGeneEETestStatus.length]; for (int i = 0; i < answer.length; i++) { answer[i] = (byte) (targetGeneEETestStatus[i] & queryGeneEETestStatusBytes[i]); loopcount++; } cvo.setDatasetsTestedInBytes(answer); } if (timer.getTime() > 100) { // It usually doesn't take this long. log.info("Compute EEs tested in (batch ): " + timer.getTime() + "ms; " + loopcount + " loops "); // provide a cache status update at the same time. Worried about memory. 1500 data sets * 50000 genes = // 75megabytes, so this // shouldn't be a big deal. log.info(geneTestStatusByteCache.size() + " gene test status stored in cache, approx " + (geneTestStatusByteCache.get(queryGeneId).length * geneTestStatusByteCache.size()) + " bytes total"); } }
From source file:ubic.gemma.analysis.expression.coexpression.ProbeLinkCoexpressionAnalyzerImpl.java
/** * @param stringency//from w w w.jav a2 s. c o m * @param coexpressions * @param coexp */ private void fillInGeneInfo(int stringency, QueryGeneCoexpression coexpressions) { StopWatch timer = new StopWatch(); timer.start(); List<CoexpressedGenePairValueObject> coexpressionData = coexpressions.getCoexpressionData(stringency); Collection<Long> geneIds = new HashSet<Long>(); for (CoexpressedGenePairValueObject cod : coexpressionData) { geneIds.add(cod.getCoexpressedGeneId()); } Collection<Gene> genes = geneService.loadMultiple(geneIds); // this can be slow if there are a lot. Map<Long, Gene> gm = new HashMap<Long, Gene>(); for (Gene g : genes) { gm.put(g.getId(), g); } for (CoexpressedGenePairValueObject cod : coexpressionData) { coexpressions.add(cod); } timer.stop(); if (timer.getTime() > 1000) { log.info("Filled in gene info: " + timer.getTime() + "ms"); } }
From source file:ubic.gemma.analysis.expression.diff.DifferentialExpressionAnalyzerServiceImpl.java
/** * Made public for testing purposes only. * //from w w w.j a va2s. c o m * @param expressionExperiment * @param analysis * @param config * @return */ @Override public DifferentialExpressionAnalysis persistAnalysis(ExpressionExperiment expressionExperiment, DifferentialExpressionAnalysis analysis, DifferentialExpressionAnalysisConfig config) { deleteOldAnalyses(expressionExperiment, analysis, config.getFactorsToInclude()); StopWatch timer = new StopWatch(); timer.start(); Collection<ExpressionAnalysisResultSet> resultSets = analysis.getResultSets(); analysis.setResultSets(new HashSet<ExpressionAnalysisResultSet>()); // first transaction, gets us an ID DifferentialExpressionAnalysis persistentAnalysis = helperService.persistStub(analysis); // second set of transactions creates the empty resultSets. for (ExpressionAnalysisResultSet rs : resultSets) { Collection<DifferentialExpressionAnalysisResult> results = rs.getResults(); rs.setResults(new HashSet<DifferentialExpressionAnalysisResult>()); ExpressionAnalysisResultSet prs = helperService.create(rs); assert prs != null; for (DifferentialExpressionAnalysisResult r : results) { r.setResultSet(prs); } analysis.getResultSets().add(prs); rs.getResults().addAll(results); prs.setQvalueThresholdForStorage(config.getQvalueThreshold()); addPvalueDistribution(prs); } // we do this here because now we have IDs for everything. expressionDataFileService.getDiffExpressionAnalysisArchiveFile(expressionExperiment, analysis, resultSets); for (ExpressionAnalysisResultSet rs : resultSets) { removeUnwantedResults(config.getQvalueThreshold(), rs.getResults()); } // third transaction - add results. log.info("Saving results"); helperService.addResults(persistentAnalysis, resultSets); // final transaction: audit. auditTrailService.addUpdateEvent(expressionExperiment, DifferentialExpressionAnalysisEvent.Factory.newInstance(), persistentAnalysis.getDescription() + "; analysis id=" + persistentAnalysis.getId()); if (timer.getTime() > 5000) { log.info("Save results: " + timer.getTime() + "ms"); } return persistentAnalysis; }
From source file:ubic.gemma.analysis.expression.diff.DifferentialExpressionAnalyzerServiceImpl.java
/** * Print the p-value and score distributions. Note that if there are multiple analyses for the experiment, each one * will get a set of files. The analysis must be 'thawed' already. * /* w w w. j av a 2 s .c o m*/ * @param expressionExperiment * @param diffExpressionAnalysis - could be on a subset of the experiment. */ private void writeDistributions(BioAssaySet expressionExperiment, DifferentialExpressionAnalysis diffExpressionAnalysis) { assert diffExpressionAnalysis.getId() != null; /* * write histograms * * Of 1) pvalues, // (following not used now) 2) scores, 3) qvalues * * Put all pvalues in one file etc so we don't get 9 files for a 2x anova with interactions. */ StopWatch timer = new StopWatch(); timer.start(); List<Histogram> pvalueHistograms = new ArrayList<Histogram>(); List<ExpressionAnalysisResultSet> resultSetList = new ArrayList<ExpressionAnalysisResultSet>(); resultSetList.addAll(diffExpressionAnalysis.getResultSets()); List<String> factorNames = new ArrayList<String>(); for (ExpressionAnalysisResultSet resultSet : resultSetList) { String factorName = ""; // these will be headings on the for (ExperimentalFactor factor : resultSet.getExperimentalFactors()) { // Make a unique column heading. factorName = factorName + (factorName.equals("") ? "" : ":") + factor.getName() + FACTOR_NAME_MANGLING_DELIMITER + factor.getId(); } factorNames.add(factorName); Histogram pvalHist = addPvalueDistribution(resultSet); this.differentialExpressionResultService.update(resultSet); pvalueHistograms.add(pvalHist); } DoubleMatrix<String, String> pvalueDists = new DenseDoubleMatrix<String, String>(100, resultSetList.size()); fillDists(factorNames, pvalueHistograms, pvalueDists); saveDistributionMatrixToFile(pvalueDists, expressionExperiment, resultSetList); if (timer.getTime() > 5000) { log.info("Done writing distributions: " + timer.getTime() + "ms"); } }