Java tutorial
/* * The Gemma project * * Copyright (c) 2007 University of British Columbia * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package ubic.gemma.web.controller.expression.experiment; import cern.colt.list.DoubleArrayList; import cern.colt.matrix.DoubleMatrix2D; import cern.colt.matrix.doublealgo.Formatter; import cern.colt.matrix.impl.DenseDoubleMatrix2D; import cern.jet.stat.Descriptive; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.jfree.chart.ChartFactory; import org.jfree.chart.ChartUtilities; import org.jfree.chart.JFreeChart; import org.jfree.chart.StandardChartTheme; import org.jfree.chart.axis.CategoryAxis; import org.jfree.chart.axis.CategoryLabelPositions; import org.jfree.chart.axis.NumberAxis; import org.jfree.chart.axis.ValueAxis; import org.jfree.chart.plot.CategoryPlot; import org.jfree.chart.plot.PlotOrientation; import org.jfree.chart.plot.XYPlot; import org.jfree.chart.renderer.category.BarRenderer; import org.jfree.chart.renderer.category.ScatterRenderer; import org.jfree.chart.renderer.xy.XYDotRenderer; import org.jfree.chart.renderer.xy.XYItemRenderer; import org.jfree.chart.renderer.xy.XYLineAndShapeRenderer; import org.jfree.data.category.CategoryDataset; import org.jfree.data.category.DefaultCategoryDataset; import org.jfree.data.statistics.DefaultMultiValueCategoryDataset; import org.jfree.data.time.Hour; import org.jfree.data.time.Minute; import org.jfree.data.time.TimeSeries; import org.jfree.data.time.TimeSeriesCollection; import org.jfree.data.xy.DefaultXYDataset; import org.jfree.data.xy.XYSeries; import org.jfree.data.xy.XYSeriesCollection; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.servlet.ModelAndView; import ubic.basecode.dataStructure.matrix.DenseDoubleMatrix; import ubic.basecode.dataStructure.matrix.DoubleMatrix; import ubic.basecode.graphics.ColorMatrix; import ubic.basecode.graphics.MatrixDisplay; import ubic.basecode.io.ByteArrayConverter; import ubic.basecode.io.writer.MatrixWriter; import ubic.basecode.math.DescriptiveWithMissing; import ubic.basecode.math.distribution.Histogram; import ubic.gemma.core.analysis.preprocess.MeanVarianceService; import ubic.gemma.core.analysis.preprocess.OutlierDetails; import ubic.gemma.core.analysis.preprocess.OutlierDetectionService; import ubic.gemma.core.analysis.preprocess.svd.SVDService; import ubic.gemma.core.analysis.preprocess.svd.SVDValueObject; import ubic.gemma.core.analysis.util.ExperimentalDesignUtils; import ubic.gemma.core.datastructure.matrix.ExperimentalDesignWriter; import ubic.gemma.core.datastructure.matrix.ExpressionDataWriterUtils; import ubic.gemma.model.analysis.expression.coexpression.CoexpCorrelationDistribution; import ubic.gemma.model.common.description.Characteristic; import ubic.gemma.model.expression.bioAssay.BioAssay; import ubic.gemma.model.expression.bioAssayData.MeanVarianceRelation; import ubic.gemma.model.expression.experiment.ExperimentalFactor; import ubic.gemma.model.expression.experiment.ExpressionExperiment; import ubic.gemma.model.expression.experiment.FactorValue; import ubic.gemma.persistence.service.analysis.expression.coexpression.CoexpressionAnalysisService; import ubic.gemma.persistence.service.analysis.expression.diff.DifferentialExpressionResultService; import ubic.gemma.persistence.service.analysis.expression.sampleCoexpression.SampleCoexpressionAnalysisService; import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService; import ubic.gemma.persistence.util.EntityUtils; import ubic.gemma.persistence.util.Settings; import ubic.gemma.web.controller.BaseController; import ubic.gemma.web.view.TextView; import javax.imageio.ImageIO; import java.awt.*; import java.awt.geom.Ellipse2D; import java.awt.geom.Rectangle2D; import java.awt.image.BufferedImage; import java.io.*; import java.text.DecimalFormat; import java.util.*; import java.util.List; // /** * @author paul */ @Controller public class ExpressionExperimentQCController extends BaseController { public static final int DEFAULT_QC_IMAGE_SIZE_PX = 200; private static final int MAX_HEATMAP_CELLSIZE = 12; @Autowired private ExpressionExperimentService expressionExperimentService; @Autowired private SVDService svdService; @Autowired private MeanVarianceService meanVarianceService; @Autowired private SampleCoexpressionAnalysisService sampleCoexpressionAnalysisService; @Autowired private OutlierDetectionService outlierDetectionService; @Autowired private DifferentialExpressionResultService differentialExpressionResultService; @Autowired private CoexpressionAnalysisService coexpressionAnalysisService; @RequestMapping("/expressionExperiment/detailedFactorAnalysis.html") public void detailedFactorAnalysis(Long id, OutputStream os) throws Exception { ExpressionExperiment ee = expressionExperimentService.load(id); if (ee == null) { log.warn("Could not load experiment with id " + id); return; } boolean ok = this.writeDetailedFactorAnalysis(ee, os); if (!ok) { this.writePlaceholderImage(os); } } @RequestMapping("/expressionExperiment/outliersRemoved.html") public ModelAndView identifyOutliersRemoved(Long id) throws IOException { if (id == null) { log.warn("No id!"); return null; } ExpressionExperiment ee = expressionExperimentService.load(id); if (ee == null) { log.warn("Could not load experiment with id " + id); return null; } ee = expressionExperimentService.thawLite(ee); Collection<BioAssay> bioAssays = new HashSet<>(); for (BioAssay assay : ee.getBioAssays()) { if (assay.getIsOutlier()) { bioAssays.add(assay); } } // and write it out StringWriter writer = new StringWriter(); StringBuffer buf = writer.getBuffer(); ExpressionDataWriterUtils.appendBaseHeader(ee, "Outliers removed", buf); ExperimentalDesignWriter edWriter = new ExperimentalDesignWriter(); ee = expressionExperimentService.thawLiter(ee); edWriter.write(writer, ee, bioAssays, false, true); ModelAndView mav = new ModelAndView(new TextView()); mav.addObject(TextView.TEXT_PARAM, buf.toString()); return mav; } @RequestMapping("/expressionExperiment/possibleOutliers.html") public ModelAndView identifyPossibleOutliers(Long id) throws IOException { if (id == null) { log.warn("No id!"); return null; } ExpressionExperiment ee = expressionExperimentService.load(id); if (ee == null) { log.warn("Could not load experiment with id " + id); return null; } // identify outliers if (!sampleCoexpressionAnalysisService.hasAnalysis(ee)) { log.warn("Experiment doesn't have correlation matrix computed (will not create right now)"); return null; } DoubleMatrix<BioAssay, BioAssay> sampleCorrelationMatrix = sampleCoexpressionAnalysisService .loadFullMatrix(ee); if (sampleCorrelationMatrix == null || sampleCorrelationMatrix.rows() < 3) { return null; } Collection<OutlierDetails> outliers = outlierDetectionService .identifyOutliersByMedianCorrelation(sampleCorrelationMatrix); Collection<BioAssay> bioAssays = new HashSet<>(); if (!outliers.isEmpty()) { for (OutlierDetails details : outliers) { bioAssays.add(details.getBioAssay()); } } // and write it out StringWriter writer = new StringWriter(); StringBuffer buf = writer.getBuffer(); ExpressionDataWriterUtils.appendBaseHeader(ee, "Sample outlier", buf); ExperimentalDesignWriter edWriter = new ExperimentalDesignWriter(); ee = expressionExperimentService.thawLiter(ee); edWriter.write(writer, ee, bioAssays, false, true); ModelAndView mav = new ModelAndView(new TextView()); mav.addObject(TextView.TEXT_PARAM, buf.toString()); return mav; } @SuppressWarnings("SameReturnValue") @RequestMapping("/expressionExperiment/pcaFactors.html") public ModelAndView pcaFactors(Long id, OutputStream os) throws Exception { if (id == null) return null; ExpressionExperiment ee = expressionExperimentService.load(id); if (ee == null) { log.warn("Could not load experiment with id " + id); // or access denied. this.writePlaceholderImage(os); return null; } SVDValueObject svdo = null; try { svdo = svdService.getSvdFactorAnalysis(ee.getId()); } catch (Exception e) { // if there is no pca // log.error( e, e ); } if (svdo != null) { this.writePCAFactors(os, ee, svdo); } else this.writePlaceholderImage(os); return null; } @SuppressWarnings("SameReturnValue") @RequestMapping("/expressionExperiment/pcaScree.html") public ModelAndView pcaScree(Long id, OutputStream os) throws Exception { ExpressionExperiment ee = expressionExperimentService.load(id); if (ee == null) { log.warn("Could not load experiment with id " + id); // or access deined. this.writePlaceholderImage(os); return null; } SVDValueObject svdo = svdService.getSvd(ee.getId()); if (svdo != null) { this.writePCAScree(os, svdo); } else { this.writePlaceholderImage(os); } return null; } /** * @param id of experiment * @param size Multiplier on the cell size. 1 or null for standard small size. * @param text if true, output a tabbed file instead of a png * @param showLabels if the row and column labels of the matrix should be shown. * @param contrVal * @param forceShowLabels forces the display of labels in the picture * @param reg uses the regressed matrix (if available). * @param os response output stream */ @RequestMapping("/expressionExperiment/visualizeCorrMat.html") public void visualizeCorrMat(Long id, Double size, String contrVal, Boolean text, Boolean showLabels, Boolean forceShowLabels, Boolean reg, OutputStream os) throws Exception { if (id == null) { log.warn("No id!"); return; } ExpressionExperiment ee = expressionExperimentService.load(id); if (ee == null) { log.warn("Could not load experiment with id " + id); return; } ee = expressionExperimentService.thawLiter(ee); DoubleMatrix<BioAssay, BioAssay> omatrix = (reg != null && reg) ? sampleCoexpressionAnalysisService.loadTryRegressedThenFull(ee) : sampleCoexpressionAnalysisService.loadFullMatrix(ee); if (omatrix == null) { log.warn("No correlation matrix for ee " + id); return; } List<String> stringNames = new ArrayList<>(); for (BioAssay ba : omatrix.getRowNames()) { stringNames.add(ba.getName() + " ID=" + ba.getId()); } DoubleMatrix<String, String> matrix = new DenseDoubleMatrix<>(omatrix.getRawMatrix()); matrix.setRowNames(stringNames); matrix.setColumnNames(stringNames); if (text != null && text) { StringWriter s = new StringWriter(); MatrixWriter<String, String> mw = new MatrixWriter<>(s, new DecimalFormat("#.##")); mw.writeMatrix(matrix, true); os.write(s.toString().replace("\uFFFD", "\t").getBytes()); // This does not solve the root issue, but I wasted too much time on it return; } /* * Blank out the diagonal so it doesn't affect the colour scale. */ for (int i = 0; i < matrix.rows(); i++) { matrix.set(i, i, Double.NaN); } ColorMatrix<String, String> cm = new ColorMatrix<>(matrix); int row = matrix.rows(); int cellsize = (int) Math.min(ExpressionExperimentQCController.MAX_HEATMAP_CELLSIZE, Math.max(1, size * ExpressionExperimentQCController.DEFAULT_QC_IMAGE_SIZE_PX / row)); MatrixDisplay<String, String> writer = new MatrixDisplay<>(cm); boolean reallyShowLabels; int minimumCellSizeForText = 9; if (forceShowLabels != null && forceShowLabels) { cellsize = Math.min(ExpressionExperimentQCController.MAX_HEATMAP_CELLSIZE, minimumCellSizeForText); reallyShowLabels = true; } else { reallyShowLabels = showLabels != null && (showLabels && cellsize >= minimumCellSizeForText); } writer.setCellSize(new Dimension(cellsize, cellsize)); boolean showScalebar = size > 2; writer.writeToPng(cm, os, reallyShowLabels, showScalebar); } /** * @param id of experiment * @param size Multiplier on the cell size. 1 or null for standard small size. * @param text if true, output a tabbed file instead of a png * @param os response output stream * @return ModelAndView object if text is true, otherwise null */ @RequestMapping("/expressionExperiment/visualizeMeanVariance.html") public ModelAndView visualizeMeanVariance(Long id, Double size, Boolean text, OutputStream os) throws Exception { if (id == null) { log.warn("No id!"); return null; } ExpressionExperiment ee = expressionExperimentService.load(id); if (ee == null) { log.warn("Could not load experiment with id " + id); return null; } MeanVarianceRelation mvr = meanVarianceService.find(ee); if (mvr == null) { return null; } if (text != null && text) { final ByteArrayConverter bac = new ByteArrayConverter(); double[] means = bac.byteArrayToDoubles(mvr.getMeans()); double[] variances = bac.byteArrayToDoubles(mvr.getVariances()); DoubleMatrix2D matrix = new DenseDoubleMatrix2D(means.length, 2); matrix.viewColumn(0).assign(means); matrix.viewColumn(1).assign(variances); String matrixString = new Formatter("%1.2G").toTitleString(matrix, null, new String[] { "mean", "variance" }, null, null, null, null); ModelAndView mav = new ModelAndView(new TextView()); mav.addObject(TextView.TEXT_PARAM, matrixString); return mav; } if (!this.writeMeanVariance(os, mvr, size)) { // FIXME might be something better to do return null; } return null; } @SuppressWarnings("SameReturnValue") @RequestMapping("/expressionExperiment/visualizeProbeCorrDist.html") public ModelAndView visualizeProbeCorrDist(Long id, OutputStream os) throws Exception { ExpressionExperiment ee = expressionExperimentService.load(id); if (ee == null) { log.warn("Could not load experiment with id " + id); return null; } this.writeProbeCorrHistImage(os, ee); return null; // nothing to return; } /** * @param id of the experiment * @param analysisId of the analysis * @param rsid resultSet Id * @param factorName deprecated, we will use rsId instead. Maintained for backwards compatibility. * @param size of the image. * @param os stream to write the image to. * @return null */ @SuppressWarnings("SameReturnValue") @RequestMapping("/expressionExperiment/visualizePvalueDist.html") public ModelAndView visualizePvalueDist(Long id, Long analysisId, Long rsid, String factorName, Integer size, OutputStream os) throws Exception { ExpressionExperiment ee = this.expressionExperimentService.load(id); if (ee == null) { this.log.warn("Could not load experiment with id " + id); return null; } if (size == null) { if (!this.writePValueHistImage(os, ee, analysisId, rsid, factorName)) { this.writePlaceholderImage(os); } } else { if (!this.writePValueHistThumbnailImage(os, ee, analysisId, rsid, factorName, size)) { this.writePlaceholderThumbnailImage(os, size); } } return null; // nothing to return; } @RequestMapping("/expressionExperiment/eigenGenes.html") public ModelAndView writeEigenGenes(Long eeid) throws IOException { ExpressionExperiment ee = expressionExperimentService.load(eeid); if (ee == null) { throw new IllegalArgumentException("Could not load experiment with id " + eeid); // or access deined. } SVDValueObject svdo = svdService.getSvd(ee.getId()); DoubleMatrix<Long, Integer> vMatrix = svdo.getvMatrix(); /* * FIXME put the biomaterial names in there instead of the IDs. */ /* * new DenseDoubleMatrix<String, String>() DoubleMatrix<String, String> matrix = new DenseDoubleMatrix<String, * String>( omatrix.getRawMatrix() ); matrix.setRowNames( stringNames ); matrix.setColumnNames( stringNames ); */ StringWriter s = new StringWriter(); MatrixWriter<Long, Integer> mw = new MatrixWriter<>(s, new DecimalFormat("#.######")); mw.writeMatrix(vMatrix, true); ModelAndView mav = new ModelAndView(new TextView()); mav.addObject(TextView.TEXT_PARAM, s.toString()); return mav; } private void addChartToGraphics(JFreeChart chart, Graphics2D g2, double x, double y, double width, double height) { chart.draw(g2, new Rectangle2D.Double(x, y, width, height), null, null); } /** * Support method for writeDetailedFactorAnalysis * * @param categories map of factor ID to text value. Strings will be unique, but possibly abbreviated and/or munged. */ private void getCategories(Map<Long, ExperimentalFactor> efIdMap, Long efId, Map<Long, String> categories) { ExperimentalFactor ef = efIdMap.get(efId); if (ef == null) return; int maxCategoryLabelLength = 10; for (FactorValue fv : ef.getFactorValues()) { String value = fv.getValue(); if (StringUtils.isBlank(value) || value.equals("null")) { for (Characteristic c : fv.getCharacteristics()) { if (StringUtils.isNotBlank(c.getValue())) { if (StringUtils.isNotBlank(value)) { value = value + "; " + c.getValue(); } else { value = c.getValue(); } } } } if (StringUtils.isBlank(value)) { value = fv.toString() + "--??"; } if (value.startsWith(ExperimentalDesignUtils.BATCH_FACTOR_NAME_PREFIX)) { value = value.replaceFirst(ExperimentalDesignUtils.BATCH_FACTOR_NAME_PREFIX, ""); } else { value = StringUtils.abbreviate(value, maxCategoryLabelLength); } while (categories.values().contains(value)) { value = value + "+";// make unique, kludge, will end up with string of ++++ } categories.put(fv.getId(), value); } } /** * @return JFreeChart XYSeries representing the histogram. * @throws FileNotFoundException - only if the coexp dist is being read from a file; when migration to db storage is * complete this can be removed * @throws IOException - only if the coexp dist is being read from a file; when migration to db storage is complete * this can be removed */ private XYSeries getCorrelHist(ExpressionExperiment ee) throws IOException { CoexpCorrelationDistribution coexpCorrelationDistribution = coexpressionAnalysisService .getCoexpCorrelationDistribution(ee); if (coexpCorrelationDistribution == null) { // try to get it from the file. return this.getCorrelHistFromFile(ee); } XYSeries series = new XYSeries(ee.getId(), true, true); byte[] binCountsBytes = coexpCorrelationDistribution.getBinCounts(); ByteArrayConverter bac = new ByteArrayConverter(); double[] binCounts = bac.byteArrayToDoubles(binCountsBytes); Integer numBins = coexpCorrelationDistribution.getNumBins(); double step = 2.0 / numBins; double lim = -1.0; for (double d : binCounts) { series.add(lim, d); lim += step; } return series; } /** * For backwards compatibility - read from the file. Remove this method when no longer needed. */ private XYSeries getCorrelHistFromFile(ExpressionExperiment ee) throws IOException { File file = this.locateProbeCorrFile(ee); // Current format is to have just one file for each analysis. if (file == null) { return null; } else if (!file.canRead()) { return null; } try (BufferedReader in = new BufferedReader(new FileReader(file))) { XYSeries series = new XYSeries(ee.getId(), true, true); DoubleArrayList counts = new DoubleArrayList(); while (in.ready()) { String line = in.readLine().trim(); if (line.startsWith("#")) continue; String[] split = StringUtils.split(line); if (split.length < 2) continue; try { double x = Double.parseDouble(split[0]); double y = Double.parseDouble(split[1]); series.add(x, y); counts.add(y); } catch (NumberFormatException e) { // line wasn't useable.. no big deal. Heading is included. } } if (!counts.isEmpty()) { // Backfill. this.corrDistFileToPersistent(file, ee, counts); } return series; } } /** * @return JFreeChart XYSeries representing the histogram for the requested result set */ private XYSeries getDiffExPvalueHistXYSeries(ExpressionExperiment ee, Long analysisId, Long rsId, String factorName) { if (ee == null || analysisId == null || rsId == null) { log.warn("Got invalid values: " + ee + " " + analysisId + " " + rsId + " " + factorName); return null; } Histogram hist = differentialExpressionResultService.loadPvalueDistribution(rsId); XYSeries xySeries; if (hist != null) { xySeries = new XYSeries(rsId, true, true); Double[] binEdges = hist.getBinEdges(); double[] counts = hist.getArray(); assert binEdges.length == counts.length; for (int i = 0; i < binEdges.length; i++) { xySeries.add(binEdges[i].doubleValue(), counts[i]); } return xySeries; } return null; } /** * Get the eigengene for the given component. * The values are rescaled so that jfreechart can cope. Small numbers give it fits. */ private Double[] getEigenGene(SVDValueObject svdo, Integer component) { DoubleArrayList eigenGeneL = new DoubleArrayList( ArrayUtils.toPrimitive(svdo.getvMatrix().getColObj(component))); DescriptiveWithMissing.standardize(eigenGeneL); return ArrayUtils.toObject(eigenGeneL.elements()); } private Map<Long, String> getFactorNames(ExpressionExperiment ee, int maxWidth) { Collection<ExperimentalFactor> factors = ee.getExperimentalDesign().getExperimentalFactors(); Map<Long, String> efs = new HashMap<>(); for (ExperimentalFactor ef : factors) { efs.put(ef.getId(), StringUtils.abbreviate(StringUtils.capitalize(ef.getName()), maxWidth)); } return efs; } /** * @param mvr MeanVarianceRelation object that contains the datapoints to plot * @return XYSeriesCollection which contains the Mean-variance and Loess series */ private XYSeriesCollection getMeanVariance(MeanVarianceRelation mvr) { final ByteArrayConverter bac = new ByteArrayConverter(); XYSeriesCollection dataset = new XYSeriesCollection(); if (mvr == null) { return dataset; } double[] means = bac.byteArrayToDoubles(mvr.getMeans()); double[] variances = bac.byteArrayToDoubles(mvr.getVariances()); if (means == null || variances == null) { return dataset; } XYSeries series = new XYSeries("Mean-variance"); for (int i = 0; i < means.length; i++) { series.add(means[i], variances[i]); } dataset.addSeries(series); return dataset; } private CategoryDataset getPCAScree(SVDValueObject svdo) { DefaultCategoryDataset series = new DefaultCategoryDataset(); Double[] variances = svdo.getVariances(); if (variances == null || variances.length == 0) { return series; } int MAX_COMPONENTS_FOR_SCREE = 10; // make constant for (int i = 0; i < Math.min(MAX_COMPONENTS_FOR_SCREE, variances.length); i++) { series.addValue(variances[i], new Integer(1), new Integer(i + 1)); } return series; } /** * For backwards compatibility only; remove when no longer needed. */ private File locateProbeCorrFile(ExpressionExperiment ee) { String shortName = ee.getShortName(); String analysisStoragePath = Settings.getAnalysisStoragePath(); String suffix = ".correlDist.txt"; return new File(analysisStoragePath + File.separatorChar + shortName + suffix); } /** * For conversion from legacy system. */ private void corrDistFileToPersistent(File file, ExpressionExperiment ee, DoubleArrayList counts) { log.info("Converting from pvalue distribution file to persistent stored version"); ByteArrayConverter bac = new ByteArrayConverter(); Double[] countArray = (Double[]) counts.toList().toArray(new Double[] {}); byte[] bytes = bac.doubleArrayToBytes(countArray); CoexpCorrelationDistribution coexpd = CoexpCorrelationDistribution.Factory.newInstance(); coexpd.setNumBins(counts.size()); coexpd.setBinCounts(bytes); try { coexpressionAnalysisService.addCoexpCorrelationDistribution(ee, coexpd); if (file.delete()) { log.info("Old file deleted"); } else { log.info("Old file could not be deleted"); } } catch (Exception e) { log.info("Could not save the corr dist: " + e.getMessage()); } } private boolean writeDetailedFactorAnalysis(ExpressionExperiment ee, OutputStream os) throws Exception { SVDValueObject svdo = svdService.getSvdFactorAnalysis(ee.getId()); if (svdo == null) return false; if (svdo.getFactors().isEmpty() && svdo.getDates().isEmpty()) { return false; } Map<Integer, Map<Long, Double>> factorCorrelations = svdo.getFactorCorrelations(); // Map<Integer, Map<Long, Double>> factorPvalues = svdo.getFactorPvalues(); Map<Integer, Double> dateCorrelations = svdo.getDateCorrelations(); assert ee.getId().equals(svdo.getId()); ee = expressionExperimentService.thawLite(ee); // need the experimental design int maxWidth = 30; Map<Long, String> efs = this.getFactorNames(ee, maxWidth); Map<Long, ExperimentalFactor> efIdMap = EntityUtils .getIdMap(ee.getExperimentalDesign().getExperimentalFactors()); Collection<Long> continuousFactors = new HashSet<>(); for (ExperimentalFactor ef : ee.getExperimentalDesign().getExperimentalFactors()) { boolean isContinous = ExperimentalDesignUtils.isContinuous(ef); if (isContinous) { continuousFactors.add(ef.getId()); } } /* * Make plots of the dates vs. PCs, factors vs. PCs. */ int MAX_COMP = 3; Map<Long, List<JFreeChart>> charts = new LinkedHashMap<>(); ChartFactory.setChartTheme(StandardChartTheme.createLegacyTheme()); /* * FACTORS */ String componentShorthand = "PC"; for (Integer component : factorCorrelations.keySet()) { if (component >= MAX_COMP) break; String xaxisLabel = componentShorthand + (component + 1); for (Long efId : factorCorrelations.get(component).keySet()) { /* * Should not happen. */ if (!efs.containsKey(efId)) { log.warn("No experimental factor with id " + efId); continue; } if (!svdo.getFactors().containsKey(efId)) { // this should not happen. continue; } boolean isCategorical = !continuousFactors.contains(efId); Map<Long, String> categories = new HashMap<>(); if (isCategorical) { this.getCategories(efIdMap, efId, categories); } if (!charts.containsKey(efId)) { charts.put(efId, new ArrayList<JFreeChart>()); } Double a = factorCorrelations.get(component).get(efId); String plotname = (efs.get(efId) == null ? "?" : efs.get(efId)) + " " + xaxisLabel; // unique? if (a != null && !Double.isNaN(a)) { String title = plotname + " " + String.format("%.2f", a); List<Double> values = svdo.getFactors().get(efId); Double[] eigenGene = this.getEigenGene(svdo, component); assert values.size() == eigenGene.length; /* * Plot eigengene vs values, add correlation to the plot */ JFreeChart chart; if (isCategorical) { /* * Categorical factor */ // use the absolute value of the correlation, since direction is arbitrary. title = plotname + " " + String.format("r=%.2f", Math.abs(a)); DefaultMultiValueCategoryDataset dataset = new DefaultMultiValueCategoryDataset(); /* * What this code does is organize the factor values by the groups. */ Map<String, List<Double>> groupedValues = new TreeMap<>(); for (int i = 0; i < values.size(); i++) { Long fvId = values.get(i).longValue(); String fvValue = categories.get(fvId); if (fvValue == null) { /* * Problem ...eg gill2006fateinocean id=1748 -- missing values. We just don't plot * anything for this sample. */ continue; // is this all we need to do? } if (!groupedValues.containsKey(fvValue)) { groupedValues.put(fvValue, new ArrayList<Double>()); } groupedValues.get(fvValue).add(eigenGene[i]); if (log.isDebugEnabled()) log.debug(fvValue + " " + values.get(i)); } for (String key : groupedValues.keySet()) { dataset.add(groupedValues.get(key), plotname, key); } // don't show the name of the X axis: it's redundant with the title. NumberAxis rangeAxis = new NumberAxis(xaxisLabel); rangeAxis.setAutoRangeIncludesZero(false); // rangeAxis.setAutoRange( false ); rangeAxis.setAutoRangeMinimumSize(4.0); // rangeAxis.setRange( new Range( -2, 2 ) ); CategoryPlot plot = new CategoryPlot(dataset, new CategoryAxis(null), rangeAxis, new ScatterRenderer()); plot.setRangeGridlinesVisible(false); plot.setDomainGridlinesVisible(false); chart = new JFreeChart(title, new Font("SansSerif", Font.BOLD, 12), plot, false); ScatterRenderer renderer = (ScatterRenderer) plot.getRenderer(); float saturationDrop = (float) Math.min(1.0, component * 0.8f / MAX_COMP); renderer.setSeriesFillPaint(0, Color.getHSBColor(0.0f, 1.0f - saturationDrop, 0.7f)); renderer.setSeriesShape(0, new Ellipse2D.Double(0, 0, 3, 3)); renderer.setUseOutlinePaint(false); renderer.setUseFillPaint(true); renderer.setBaseFillPaint(Color.white); CategoryAxis domainAxis = plot.getDomainAxis(); domainAxis.setCategoryLabelPositions(CategoryLabelPositions.UP_45); } else { /* * Continuous value factor */ DefaultXYDataset series = new DefaultXYDataset(); series.addSeries(plotname, new double[][] { ArrayUtils.toPrimitive(values.toArray(new Double[] {})), ArrayUtils.toPrimitive(eigenGene) }); // don't show x-axis label, which would otherwise be efs.get( efId ) chart = ChartFactory.createScatterPlot(title, null, xaxisLabel, series, PlotOrientation.VERTICAL, false, false, false); XYPlot plot = chart.getXYPlot(); plot.setRangeGridlinesVisible(false); plot.setDomainGridlinesVisible(false); XYItemRenderer renderer = plot.getRenderer(); renderer.setBasePaint(Color.white); renderer.setSeriesShape(0, new Ellipse2D.Double(0, 0, 3, 3)); float saturationDrop = (float) Math.min(1.0, component * 0.8f / MAX_COMP); renderer.setSeriesPaint(0, Color.getHSBColor(0.0f, 1.0f - saturationDrop, 0.7f)); plot.setRenderer(renderer); } chart.getTitle().setFont(new Font("SansSerif", Font.BOLD, 12)); charts.get(efId).add(chart); } } } /* * DATES */ charts.put(-1L, new ArrayList<JFreeChart>()); for (Integer component : dateCorrelations.keySet()) { String xaxisLabel = componentShorthand + (component + 1); List<Date> dates = svdo.getDates(); if (dates.isEmpty()) break; long secspan = ubic.basecode.util.DateUtil.numberOfSecondsBetweenDates(dates); if (component >= MAX_COMP) break; Double a = dateCorrelations.get(component); if (a != null && !Double.isNaN(a)) { Double[] eigenGene = svdo.getvMatrix().getColObj(component); /* * Plot eigengene vs values, add correlation to the plot */ TimeSeries series = new TimeSeries("Dates vs. eigen" + (component + 1)); int i = 0; for (Date d : dates) { // if span is less than an hour, retain the minute. if (secspan < 60 * 60) { series.addOrUpdate(new Minute(d), eigenGene[i++]); } else { series.addOrUpdate(new Hour(d), eigenGene[i++]); } } TimeSeriesCollection dataset = new TimeSeriesCollection(); dataset.addSeries(series); JFreeChart chart = ChartFactory.createTimeSeriesChart( "Dates: " + xaxisLabel + " " + String.format("r=%.2f", a), null, xaxisLabel, dataset, false, false, false); XYPlot xyPlot = chart.getXYPlot(); chart.getTitle().setFont(new Font("SansSerif", Font.BOLD, 12)); // standard renderer makes lines. XYDotRenderer renderer = new XYDotRenderer(); renderer.setBaseFillPaint(Color.white); renderer.setDotHeight(3); renderer.setDotWidth(3); renderer.setSeriesShape(0, new Ellipse2D.Double(0, 0, 3, 3)); // has no effect, need dotheight. float saturationDrop = (float) Math.min(1.0, component * 0.8f / MAX_COMP); renderer.setSeriesPaint(0, Color.getHSBColor(0.0f, 1.0f - saturationDrop, 0.7f)); ValueAxis domainAxis = xyPlot.getDomainAxis(); domainAxis.setVerticalTickLabels(true); xyPlot.setRenderer(renderer); xyPlot.setRangeGridlinesVisible(false); xyPlot.setDomainGridlinesVisible(false); charts.get(-1L).add(chart); } } /* * Plot in a grid, with each factor as a column. FIXME What if we have too many factors to fit on the screen? */ int columns = (int) Math.ceil(charts.size()); int perChartSize = ExpressionExperimentQCController.DEFAULT_QC_IMAGE_SIZE_PX; BufferedImage image = new BufferedImage(columns * perChartSize, MAX_COMP * perChartSize, BufferedImage.TYPE_INT_ARGB); Graphics2D g2 = image.createGraphics(); int currentX = 0; int currentY = 0; for (Long id : charts.keySet()) { for (JFreeChart chart : charts.get(id)) { this.addChartToGraphics(chart, g2, currentX, currentY, perChartSize, perChartSize); if (currentY + perChartSize < MAX_COMP * perChartSize) { currentY += perChartSize; } else { currentY = 0; currentX += perChartSize; } } } os.write(ChartUtilities.encodeAsPNG(image)); return true; } /** * @param os response output stream * @param mvr MeanVarianceRelation object to plot * @return true if mvr data points were plotted */ private boolean writeMeanVariance(OutputStream os, MeanVarianceRelation mvr, Double size) throws Exception { // if number of datapoints > THRESHOLD then alpha = TRANSLUCENT, else alpha = OPAQUE final int THRESHOLD = 1000; final int TRANSLUCENT = 50; final int OPAQUE = 255; // Set maximum plot range to Y_MAX + YRANGE * OFFSET to leave some extra white space final double OFFSET_FACTOR = 0.05f; // set the final image size to be the minimum of MAX_IMAGE_SIZE_PX or size final int MAX_IMAGE_SIZE_PX = 5; if (mvr == null) { return false; } // get data points XYSeriesCollection collection = this.getMeanVariance(mvr); if (collection.getSeries().size() == 0) { return false; } ChartFactory.setChartTheme(StandardChartTheme.createLegacyTheme()); JFreeChart chart = ChartFactory.createScatterPlot("", "mean (log2)", "variance (log2)", collection, PlotOrientation.VERTICAL, false, false, false); // adjust colors and shapes XYRegressionRenderer renderer = new XYRegressionRenderer(); renderer.setBasePaint(Color.white); XYSeries series = collection.getSeries(0); int alpha = series.getItemCount() > THRESHOLD ? TRANSLUCENT : OPAQUE; renderer.setSeriesPaint(0, new Color(0, 0, 0, alpha)); renderer.setSeriesPaint(1, Color.red); renderer.setSeriesStroke(1, new BasicStroke(1)); renderer.setSeriesShape(0, new Ellipse2D.Double(4, 4, 4, 4)); renderer.setSeriesShapesFilled(0, false); renderer.setSeriesLinesVisible(0, false); renderer.setSeriesLinesVisible(1, true); renderer.setSeriesShapesVisible(1, false); XYPlot plot = chart.getXYPlot(); plot.setRenderer(renderer); plot.setRangeGridlinesVisible(false); plot.setDomainGridlinesVisible(false); // adjust the chart domain and ranges double yRange = series.getMaxY() - series.getMinY(); double xRange = series.getMaxX() - series.getMinX(); if (xRange < 0) { log.warn("Min X was greater than Max X: Max=" + series.getMaxY() + " Min= " + series.getMinY()); return false; } double ybuffer = (yRange) * OFFSET_FACTOR; double xbuffer = (xRange) * OFFSET_FACTOR; double newYMin = series.getMinY() - ybuffer; double newYMax = series.getMaxY() + ybuffer; double newXMin = series.getMinX() - xbuffer; double newXMax = series.getMaxX() + xbuffer; ValueAxis yAxis = new NumberAxis("Variance"); yAxis.setRange(newYMin, newYMax); ValueAxis xAxis = new NumberAxis("Mean"); xAxis.setRange(newXMin, newXMax); chart.getXYPlot().setRangeAxis(yAxis); chart.getXYPlot().setDomainAxis(xAxis); int finalSize = (int) Math.min( MAX_IMAGE_SIZE_PX * ExpressionExperimentQCController.DEFAULT_QC_IMAGE_SIZE_PX, size * ExpressionExperimentQCController.DEFAULT_QC_IMAGE_SIZE_PX); ChartUtilities.writeChartAsPNG(os, chart, finalSize, finalSize); return true; } /** * Remove outliers from the MeanVarianceRelation by removing those points which have: (zscore(mean) > zscoreMax || * zscore(variance) > zscoreMax) */ @SuppressWarnings("unused") private MeanVarianceRelation removeMVOutliers(MeanVarianceRelation mvr, double zscoreMax) { MeanVarianceRelation ret = MeanVarianceRelation.Factory.newInstance(); ByteArrayConverter bac = new ByteArrayConverter(); DoubleArrayList vars = new DoubleArrayList(bac.byteArrayToDoubles(mvr.getVariances())); DoubleArrayList means = new DoubleArrayList(bac.byteArrayToDoubles(mvr.getMeans())); DoubleArrayList filteredMeans = new DoubleArrayList(); DoubleArrayList filteredVars = new DoubleArrayList(); DoubleArrayList zVars = this.zscore(vars); DoubleArrayList zMeans = this.zscore(means); // clip outliers for (int i = 0; i < zMeans.size(); i++) { if (Math.abs(zMeans.getQuick(i)) > zscoreMax || Math.abs(zVars.getQuick(i)) > zscoreMax) { continue; } filteredMeans.add(means.getQuick(i)); filteredVars.add(vars.getQuick(i)); } log.debug(filteredMeans.size() + " (out of " + means.size() + ") MV points had mean or variance zscore < " + zscoreMax + ". Max mean,variance is ( " + Descriptive.max(filteredMeans) + "," + Descriptive.max(filteredVars) + ")."); ret.setVariances(bac.doubleArrayToBytes(filteredVars)); ret.setMeans(bac.doubleArrayToBytes(filteredMeans)); return ret; } /** * @return zscores */ private DoubleArrayList zscore(DoubleArrayList d) { DoubleArrayList z = new DoubleArrayList(); double mean = Descriptive.mean(d); double sd = Descriptive .standardDeviation(Descriptive.variance(d.size(), Descriptive.sum(d), Descriptive.sumOfSquares(d))); for (int i = 0; i < d.size(); i++) { z.add(Math.abs(d.getQuick(i) - mean) / sd); } assert z.size() == d.size(); return z; } /** * Visualization of the correlation of principal components with factors or the date samples were run. * * @param svdo SVD value object */ private void writePCAFactors(OutputStream os, ExpressionExperiment ee, SVDValueObject svdo) throws Exception { Map<Integer, Map<Long, Double>> factorCorrelations = svdo.getFactorCorrelations(); // Map<Integer, Map<Long, Double>> factorPvalues = svdo.getFactorPvalues(); Map<Integer, Double> dateCorrelations = svdo.getDateCorrelations(); assert ee.getId().equals(svdo.getId()); if (factorCorrelations.isEmpty() && dateCorrelations.isEmpty()) { this.writePlaceholderImage(os); return; } ee = expressionExperimentService.thawLite(ee); // need the experimental design int maxWidth = 10; Map<Long, String> efs = this.getFactorNames(ee, maxWidth); DefaultCategoryDataset series = new DefaultCategoryDataset(); /* * With two groups, or a continuous factor, we get rank correlations */ int MAX_COMP = 3; double STUB = 0.05; // always plot a little thing so we know its there. for (Integer component : factorCorrelations.keySet()) { if (component >= MAX_COMP) break; for (Long efId : factorCorrelations.get(component).keySet()) { Double a = factorCorrelations.get(component).get(efId); String facname = efs.get(efId) == null ? "?" : efs.get(efId); if (a != null && !Double.isNaN(a)) { Double corr = Math.max(STUB, Math.abs(a)); series.addValue(corr, "PC" + (component + 1), facname); } } } for (Integer component : dateCorrelations.keySet()) { if (component >= MAX_COMP) break; Double a = dateCorrelations.get(component); if (a != null && !Double.isNaN(a)) { Double corr = Math.max(STUB, Math.abs(a)); series.addValue(corr, "PC" + (component + 1), "Date run"); } } ChartFactory.setChartTheme(StandardChartTheme.createLegacyTheme()); JFreeChart chart = ChartFactory.createBarChart("", "Factors", "Component assoc.", series, PlotOrientation.VERTICAL, true, false, false); chart.getCategoryPlot().getRangeAxis().setRange(0, 1); BarRenderer renderer = (BarRenderer) chart.getCategoryPlot().getRenderer(); renderer.setBasePaint(Color.white); renderer.setShadowVisible(false); chart.getCategoryPlot().setRangeGridlinesVisible(false); chart.getCategoryPlot().setDomainGridlinesVisible(false); ChartUtilities.applyCurrentTheme(chart); CategoryAxis domainAxis = chart.getCategoryPlot().getDomainAxis(); domainAxis.setCategoryLabelPositions(CategoryLabelPositions.UP_45); for (int i = 0; i < MAX_COMP; i++) { /* * Hue is straightforward; brightness is set medium to make it muted; saturation we vary from high to low. */ float saturationDrop = (float) Math.min(1.0, i * 1.3f / MAX_COMP); renderer.setSeriesPaint(i, Color.getHSBColor(0.0f, 1.0f - saturationDrop, 0.7f)); } /* * Give figure more room .. up to a limit */ int width = ExpressionExperimentQCController.DEFAULT_QC_IMAGE_SIZE_PX; if (chart.getCategoryPlot().getCategories().size() > 3) { width = width + 40 * (chart.getCategoryPlot().getCategories().size() - 2); } int MAX_QC_IMAGE_SIZE_PX = 500; width = Math.min(width, MAX_QC_IMAGE_SIZE_PX); ChartUtilities.writeChartAsPNG(os, chart, width, ExpressionExperimentQCController.DEFAULT_QC_IMAGE_SIZE_PX); } private boolean writePCAScree(OutputStream os, SVDValueObject svdo) throws Exception { /* * Make a scree plot. */ CategoryDataset series = this.getPCAScree(svdo); if (series.getColumnCount() == 0) { return false; } int MAX_COMPONENTS_FOR_SCREE = 10; ChartFactory.setChartTheme(StandardChartTheme.createLegacyTheme()); JFreeChart chart = ChartFactory.createBarChart("", "Component (up to" + MAX_COMPONENTS_FOR_SCREE + ")", "Fraction of var.", series, PlotOrientation.VERTICAL, false, false, false); BarRenderer renderer = (BarRenderer) chart.getCategoryPlot().getRenderer(); renderer.setBasePaint(Color.white); renderer.setShadowVisible(false); chart.getCategoryPlot().setRangeGridlinesVisible(false); chart.getCategoryPlot().setDomainGridlinesVisible(false); ChartUtilities.writeChartAsPNG(os, chart, ExpressionExperimentQCController.DEFAULT_QC_IMAGE_SIZE_PX, ExpressionExperimentQCController.DEFAULT_QC_IMAGE_SIZE_PX); return true; } /** * Write a blank image so user doesn't see the broken icon. */ private void writePlaceholderImage(OutputStream os) throws IOException { int placeholderSize = (int) (ExpressionExperimentQCController.DEFAULT_QC_IMAGE_SIZE_PX * 0.75); BufferedImage buffer = new BufferedImage(placeholderSize, placeholderSize, BufferedImage.TYPE_INT_RGB); Graphics g = buffer.createGraphics(); g.setColor(Color.lightGray); g.fillRect(0, 0, placeholderSize, placeholderSize); g.setColor(Color.black); g.drawString("Not available", placeholderSize / 4, placeholderSize / 4); ImageIO.write(buffer, "png", os); } /** * Write a blank thumbnail image so user doesn't see the broken icon. */ private void writePlaceholderThumbnailImage(OutputStream os, int placeholderSize) throws IOException { // Make the image a bit bigger to account for the empty space around the generated image. // If we can find a way to remove this empty space, we don't need to make the chart bigger. BufferedImage buffer = new BufferedImage(placeholderSize + 16, placeholderSize + 9, BufferedImage.TYPE_INT_RGB); Graphics g = buffer.createGraphics(); g.setColor(Color.white); g.fillRect(0, 0, placeholderSize + 16, placeholderSize + 9); g.setColor(Color.gray); g.drawLine(8, placeholderSize + 5, placeholderSize + 8, placeholderSize + 5); // x-axis g.drawLine(8, 5, 8, placeholderSize + 5); // y-axis g.setColor(Color.black); Font font = g.getFont(); g.setFont(new Font(font.getName(), font.getStyle(), 8)); g.drawString("N/A", 9, placeholderSize); ImageIO.write(buffer, "png", os); } private boolean writeProbeCorrHistImage(OutputStream os, ExpressionExperiment ee) throws IOException { XYSeries series = this.getCorrelHist(ee); if (series == null || series.getItemCount() == 0) { return false; } ChartFactory.setChartTheme(StandardChartTheme.createLegacyTheme()); XYSeriesCollection xySeriesCollection = new XYSeriesCollection(); xySeriesCollection.addSeries(series); JFreeChart chart = ChartFactory.createXYLineChart("", "Correlation", "Frequency", xySeriesCollection, PlotOrientation.VERTICAL, false, false, false); chart.getXYPlot().setRangeGridlinesVisible(false); chart.getXYPlot().setDomainGridlinesVisible(false); XYItemRenderer renderer = chart.getXYPlot().getRenderer(); renderer.setBasePaint(Color.white); int size = (int) (ExpressionExperimentQCController.DEFAULT_QC_IMAGE_SIZE_PX * 0.8); ChartUtilities.writeChartAsPNG(os, chart, size, size); return true; } /** * Has to handle the situation where there might be more than one ResultSet. */ private boolean writePValueHistImage(OutputStream os, ExpressionExperiment ee, Long analysisId, Long rsId, String factorName) throws IOException { XYSeries series = this.getDiffExPvalueHistXYSeries(ee, analysisId, rsId, factorName); if (series == null) { return false; } XYSeriesCollection xySeriesCollection = new XYSeriesCollection(series); ChartFactory.setChartTheme(StandardChartTheme.createLegacyTheme()); JFreeChart chart = ChartFactory.createXYLineChart("", "P-value", "Frequency", xySeriesCollection, PlotOrientation.VERTICAL, false, false, false); chart.getXYPlot().setRangeGridlinesVisible(false); chart.getXYPlot().setDomainGridlinesVisible(false); XYItemRenderer renderer = chart.getXYPlot().getRenderer(); renderer.setBasePaint(Color.white); ChartUtilities.writeChartAsPNG(os, chart, (int) (ExpressionExperimentQCController.DEFAULT_QC_IMAGE_SIZE_PX * 1.4), ExpressionExperimentQCController.DEFAULT_QC_IMAGE_SIZE_PX); return true; } /** * Write p-value histogram thumbnail image. */ private boolean writePValueHistThumbnailImage(OutputStream os, ExpressionExperiment ee, Long analysisId, Long rsId, String factorName, int size) throws IOException { XYSeries series = this.getDiffExPvalueHistXYSeries(ee, analysisId, rsId, factorName); if (series == null) { return false; } series.add(-0.01, 0.0); XYSeriesCollection xySeriesCollection = new XYSeriesCollection(series); ChartFactory.setChartTheme(StandardChartTheme.createLegacyTheme()); JFreeChart chart = ChartFactory.createXYLineChart("", "", "", xySeriesCollection, PlotOrientation.VERTICAL, false, false, false); chart.getXYPlot().setBackgroundPaint(new Color(230, 230, 230)); chart.getXYPlot().setRangeGridlinesVisible(false); chart.getXYPlot().setDomainGridlinesVisible(false); chart.getXYPlot().setOutlineVisible(false); // around the plot chart.getXYPlot().getRangeAxis().setTickMarksVisible(false); chart.getXYPlot().getRangeAxis().setTickLabelsVisible(false); chart.getXYPlot().getRangeAxis().setAxisLineVisible(false); chart.getXYPlot().getDomainAxis().setTickMarksVisible(false); chart.getXYPlot().getDomainAxis().setTickLabelsVisible(false); chart.getXYPlot().getDomainAxis().setAxisLineVisible(false); chart.getXYPlot().getRenderer().setSeriesPaint(0, Color.RED); // chart.getXYPlot().getRenderer().setSeriesStroke( 0, new BasicStroke( 1 ) ); // Make the chart a bit bigger to account for the empty space around the generated image. // If we can find a way to remove this empty space, we don't need to make the chart bigger. ChartUtilities.writeChartAsPNG(os, chart, size + 16, size + 9); return true; } /** * Overrides XYLineAndShapeRenderer such that lines are drawn on top of points. */ private class XYRegressionRenderer extends XYLineAndShapeRenderer { private static final long serialVersionUID = 1L; @Override protected boolean isLinePass(int pass) { return pass == 1; } @Override protected boolean isItemPass(int pass) { return pass == 0; } } }