List of usage examples for java.lang Float isNaN
public static boolean isNaN(float v)
From source file:org.chromium.chrome.browser.tab.Tab.java
/** * @param manager The fullscreen manager that should be notified of changes to this tab (if * set to null, no more updates will come from this tab). *///w w w . ja v a 2s. co m public void setFullscreenManager(FullscreenManager manager) { mFullscreenManager = manager; if (mFullscreenManager != null) { boolean topOffsetsInitialized = !Float.isNaN(mPreviousTopControlsOffsetY) && !Float.isNaN(mPreviousContentOffsetY); boolean bottomOffsetsInitialized = !Float.isNaN(mPreviousBottomControlsOffsetY); boolean isChromeHomeEnabled = FeatureUtilities.isChromeHomeEnabled(); // Make sure the dominant control offsets have been set. if ((!topOffsetsInitialized && !isChromeHomeEnabled) || (!bottomOffsetsInitialized && isChromeHomeEnabled)) { mFullscreenManager.setPositionsForTabToNonFullscreen(); } else { mFullscreenManager.setPositionsForTab(mPreviousTopControlsOffsetY, mPreviousBottomControlsOffsetY, mPreviousContentOffsetY); } updateFullscreenEnabledState(); } }
From source file:com.actelion.research.table.view.JVisualization.java
public String getStatisticalValues() { if (mChartType == cChartTypeScatterPlot) return "Incompatible chart type."; String[][] categoryList = new String[6][]; int[] categoryColumn = new int[6]; int categoryColumnCount = 0; if (isSplitView()) { for (int i = 0; i < 2; i++) { if (mSplittingColumn[i] != cColumnUnassigned) { categoryColumn[categoryColumnCount] = mSplittingColumn[i]; categoryList[categoryColumnCount] = mTableModel.getCategoryList(mSplittingColumn[i]); categoryColumnCount++;/*from w w w. j av a 2s . c o m*/ } } } for (int axis = 0; axis < mDimensions; axis++) { if (mIsCategoryAxis[axis]) { categoryColumn[categoryColumnCount] = mAxisIndex[axis]; categoryList[categoryColumnCount] = new String[mCategoryVisibleCount[axis]]; if (mAxisIndex[axis] == cColumnUnassigned) { // box/whisker plot with unassigned category axis categoryList[categoryColumnCount][0] = "<All Rows>"; } else { String[] list = mTableModel.getCategoryList(categoryColumn[categoryColumnCount]); for (int j = 0; j < mCategoryVisibleCount[axis]; j++) categoryList[categoryColumnCount][j] = list[mCategoryMin[axis] + j]; } categoryColumnCount++; } } if (isCaseSeparationDone()) { categoryColumn[categoryColumnCount] = mCaseSeparationColumn; categoryList[categoryColumnCount] = mTableModel.getCategoryList(mCaseSeparationColumn); categoryColumnCount++; } boolean includePValue = false; boolean includeFoldChange = false; int pValueColumn = getPValueColumn(); int referenceCategoryIndex = -1; if (pValueColumn != cColumnUnassigned) { referenceCategoryIndex = getCategoryIndex(pValueColumn, mPValueRefCategory); if (referenceCategoryIndex != -1) { includePValue = mBoxplotShowPValue; includeFoldChange = mBoxplotShowFoldChange; } } StringWriter stringWriter = new StringWriter(1024); BufferedWriter writer = new BufferedWriter(stringWriter); try { // construct the title line for (int i = 0; i < categoryColumnCount; i++) { String columnTitle = (categoryColumn[i] == -1) ? "Category" : mTableModel.getColumnTitleWithSpecialType(categoryColumn[i]); writer.append(columnTitle + "\t"); } if (mChartType != cChartTypeBoxPlot) writer.append("Rows in Category"); if ((mChartType == cChartTypeBars || mChartType == cChartTypePies) && mChartMode != cChartModeCount) { String name = mTableModel.getColumnTitleWithSpecialType(mChartColumn); writer.append((mChartMode == cChartModePercent) ? "\tPercent of Rows" : (mChartMode == cChartModeMean) ? "\tMean of " + name : (mChartMode == cChartModeMean) ? "\tSum of " + name : (mChartMode == cChartModeMin) ? "\tMinimum of " + name : (mChartMode == cChartModeMax) ? "\tMaximum of " + name : ""); } if (mChartType == cChartTypeBoxPlot || mChartType == cChartTypeWhiskerPlot) { if (mChartType == cChartTypeBoxPlot) { writer.append("Total Count"); writer.append("\tOutlier Count"); } writer.append("\tMean Value"); writer.append("\t1st Quartile"); writer.append("\tMedian"); writer.append("\t3rd Quartile"); writer.append("\tLower Adjacent Limit"); writer.append("\tUpper Adjacent Limit"); /* if (includeFoldChange || includePValue) don't use additional column writer.append("\tIs Reference Group"); */ if (includeFoldChange) writer.append( mTableModel.isLogarithmicViewMode(mAxisIndex[((BoxPlotViewInfo) mChartInfo).barAxis]) ? "\tlog2(Fold Change)" : "\tFold Change"); if (includePValue) writer.append("\tp-Value"); } writer.newLine(); int[] categoryIndex = new int[6]; while (categoryIndex[0] < categoryList[0].length) { int columnIndex = 0; int hv = 0; if (isSplitView()) { if (mSplittingColumn[0] != cColumnUnassigned) hv += categoryIndex[columnIndex++]; if (mSplittingColumn[1] != cColumnUnassigned) hv += categoryIndex[columnIndex++] * categoryList[0].length; } int cat = 0; for (int axis = 0; axis < mDimensions; axis++) if (mIsCategoryAxis[axis]) cat += categoryIndex[columnIndex++] * mCombinedCategoryCount[axis]; if (isCaseSeparationDone()) cat += categoryIndex[columnIndex++]; if (mChartInfo.pointsInCategory[hv][cat] != 0) { for (int i = 0; i < categoryColumnCount; i++) { writer.append(categoryList[i][categoryIndex[i]]); if ((includeFoldChange || includePValue) && pValueColumn == categoryColumn[i] && mPValueRefCategory.equals(categoryList[i][categoryIndex[i]])) writer.append(" (ref)"); writer.append("\t"); } if (mChartType != cChartTypeBoxPlot) writer.append("" + mChartInfo.pointsInCategory[hv][cat]); if ((mChartType == cChartTypeBars || mChartType == cChartTypePies) && mChartMode != cChartModeCount) writer.append("\t" + formatValue(mChartInfo.barValue[hv][cat], mChartColumn)); if (mChartType == cChartTypeBoxPlot || mChartType == cChartTypeWhiskerPlot) { BoxPlotViewInfo vi = (BoxPlotViewInfo) mChartInfo; if (mChartType == cChartTypeBoxPlot) { writer.append("" + (mChartInfo.pointsInCategory[hv][cat] + vi.outlierCount[hv][cat])); writer.append("\t" + vi.outlierCount[hv][cat]); } int column = mAxisIndex[((BoxPlotViewInfo) mChartInfo).barAxis]; writer.append("\t" + formatValue(vi.barValue[hv][cat], column)); writer.append("\t" + formatValue(vi.boxQ1[hv][cat], column)); writer.append("\t" + formatValue(vi.median[hv][cat], column)); writer.append("\t" + formatValue(vi.boxQ3[hv][cat], column)); writer.append("\t" + formatValue(vi.boxLAV[hv][cat], column)); writer.append("\t" + formatValue(vi.boxUAV[hv][cat], column)); /* if (includeFoldChange || includePValue) { don't use additional column int refHV = getReferenceHV(hv, pValueColumn, referenceCategoryIndex); int refCat = getReferenceCat(cat, pValueColumn, referenceCategoryIndex, new int[1+mDimensions]); writer.append("\t"+((hv==refHV && cat==refCat) ? "yes" : "no")); } */ if (includeFoldChange) { writer.append("\t"); if (!Float.isNaN(vi.foldChange[hv][cat])) writer.append(new DecimalFormat("#.#####").format(vi.foldChange[hv][cat])); } if (includePValue) { writer.append("\t"); if (!Float.isNaN(vi.pValue[hv][cat])) writer.append(new DecimalFormat("#.#####").format(vi.pValue[hv][cat])); } } writer.newLine(); } // update category indices for next row for (int i = categoryColumnCount - 1; i >= 0; i--) { if (++categoryIndex[i] < categoryList[i].length || i == 0) break; categoryIndex[i] = 0; } } writer.close(); } catch (IOException ioe) { } return stringWriter.toString(); }
From source file:com.actelion.research.table.view.JVisualization.java
/** * Formats a numerical value for displaying it. * This includes proper rounding and potential de-logarithmization of the original value. * @param value is the logarithm, if the column is in logarithmic view mode * @param column the column this value refers to * @return//from w ww.j av a 2s .c o m */ protected String formatValue(float value, int column) { if (mTableModel.isLogarithmicViewMode(column) && !Float.isNaN(value) && !Float.isInfinite(value)) value = (float) Math.pow(10, value); return DoubleFormat.toString(value); }
From source file:com.actelion.research.table.view.JVisualization.java
public void setMarkerSizeZoomAdaption(boolean adapt) { if (Float.isNaN(mMarkerSizeZoomAdaption) == adapt) { if (adapt) mMarkerSizeZoomAdaption = calculateZoomState(); else/*from w ww . jav a 2s . c o m*/ mMarkerSizeZoomAdaption = Float.NaN; invalidateOffImage(false); } }
From source file:tufts.vue.LWComponent.java
protected boolean validateInitialValues() { boolean bad = false; // Note that if ANY component in the map has a NaN coordinate or dimension, // it can put the AWT graphics routines into an unrecoverable state that // may prevent the entire map from drawing sanely or at all. if (Float.isNaN(x)) { Log.warn("bad x " + this); x = 0;// w w w . j av a2 s . c om bad = true; } if (Float.isNaN(y)) { Log.warn("bad y " + this); y = 0; bad = true; } if (Float.isNaN(width)) { Log.warn("bad width " + this); width = 0; bad = true; } if (Float.isNaN(height)) { Log.warn("bad height " + this); height = 0; bad = true; } if (supportsProperty(KEY_FontSize) && mFontSize.get() < 1) { Log.warn("bad font size " + mFontSize.get() + " " + this); mFontSize.take(1); // don't risk triggering an event at a bad time bad = true; } return bad; }
From source file:com.actelion.research.table.view.JVisualization.java
public void updateVisibleRange(int axis, float low, float high, boolean isAdjusting) { if (axis < mDimensions && mAxisIndex[axis] != cColumnUnassigned) { mPruningBarLow[axis] = low;// ww w. j a v a2 s. c om mPruningBarHigh[axis] = high; if (calculateVisibleRange(axis)) { updateLocalZoomExclusion(axis); applyLocalExclusion(isAdjusting); if (!Float.isNaN(mMarkerSizeZoomAdaption)) mMarkerSizeZoomAdaption = calculateZoomState(); invalidateOffImage(true); } } }
From source file:com.actelion.research.table.view.JVisualization.java
/** * Sets axis related zoom flags to false and sets NaN flags depending on * whether the value is NaN regardless whether the axis is showing floats or categories. * @param axis/*from w w w . j av a 2 s . c o m*/ */ private void initializeLocalExclusion(int axis) { // flags 0-2: set if invisible due to view zooming // flags 3-5: set if invisible because of empty data (applies only for non-category ) // flags 6 : set if invisible because point is not part of currently shown detail graph int column = mAxisIndex[axis]; byte nanFlag = (byte) (EXCLUSION_FLAG_NAN_0 << axis); byte bothFlags = (byte) ((EXCLUSION_FLAG_ZOOM_0 | EXCLUSION_FLAG_NAN_0) << axis); // reset all flags and then // flag all records with empty data for (int i = 0; i < mDataPoints; i++) { mPoint[i].exclusionFlags &= ~bothFlags; if (column != -1 && !mTableModel.isDescriptorColumn(column) && Float.isNaN(mPoint[i].record.getDouble(column))) mPoint[i].exclusionFlags |= nanFlag; } }
From source file:gov.noaa.pfel.erddap.dataset.EDDTableFromNcFiles.java
/** NOT FOR GENERAL USE. Bob uses this to consolidate the individual GTSPP * data files into 30 x 30 x 1 month files (tiles). * 30 x 30 leads to 12x6=72 files for a given time point, so a request * for a short time but entire world opens ~72 files. * There are ~240 months worth of data, so a request for a small lon lat * range for all time opens ~240 files.// w w w . ja v a2 s .c o m * * <p>Why tile? Because there are ~10^6 profiles/year now, so ~10^7 total. * And if 100 bytes of info per file for EDDTableFromFiles fileTable, that's 1 GB!. * So there needs to be fewer files. * We want to balance number of files for 1 time point (all region tiles), * and number of time point files (I'll stick with their use of 1 month). * The tiling size selected is ok, but searches for single profile (by name) * are slow since a given file may have a wide range of station_ids. * * <p>Quality flags * <br>https://www.nodc.noaa.gov/GTSPP/document/qcmans/GTSPP_RT_QC_Manual_20090916.pdf * <br>http://www.ifremer.fr/gosud/formats/gtspp_qcflags.htm * <br>CODE SIGNIFICATION * <br>0 NOT CONTROLLED VALUE * <br>1 CORRECT VALUE * <br>2 VALUE INCONSISTENT WITH STATISTICS * <br>3 DOUBTFUL VALUE (spike, ...) * <br>4 FALSE VALUE (out of scale, constant profile, vertical instability, ...) * <br>5 VALUE MODIFIED DURING QC (only for interpolate location or date) * <br>6-8 Not USED * <br>9 NO VALUE * <br> * <br>I interpret as: okay values are 1, 2, 5 * * @param firstYear e.g., 1990 * @param firstMonth e.g., 1 (1..) * @param lastYear e.g., 2010 * @param lastMonth e.g., 12 (1..) * @param testMode if true, this just processes .nc files * already in testTempDir f:/data/gtspp/testTemp/ * and puts results in testDestDir f:/data/gtspp/testDest/. * So the first/last/Year/Month params are ignored. */ public static void bobConsolidateGtsppTgz(int firstYear, int firstMonth, int lastYear, int lastMonth, boolean testMode) throws Throwable { int chunkSize = 45; //lon width, lat height of a tile, in degrees int minLat = -90; int maxLat = 90; int minLon = -180; int maxLon = 180; String today = Calendar2.getCurrentISODateTimeStringZulu().substring(0, 10); //to nearest day String sevenZip = "c:\\progra~1\\7-Zip\\7z"; String zipDir = "c:\\data\\gtspp\\bestNcZip\\"; //gtspp_at199001.tgz String destDir = "c:\\data\\gtspp\\bestNcConsolidated\\"; String tempDir = "c:\\data\\gtspp\\temp\\"; String testTempDir = "c:\\data\\gtspp\\testTemp\\"; //tempDir if testMode=true String testDestDir = "c:\\data\\gtspp\\testDest\\"; //destDir if testMode=true String logFile = "c:\\data\\gtspp\\log" + String2.replaceAll(today, "-", "") + ".txt"; File2.makeDirectory(tempDir); //https://www.nodc.noaa.gov/GTSPP/document/qcmans/qcflags.htm //1=correct, 2=probably correct, 5=modified (so now correct) //pre 2012-04-15 was {1,2,5} //pre 2012-05-25 was {1,2} int okQF[] = { 1, 2, 5 }; String okQFCsv = String2.toCSSVString(okQF); float depthMV = 99999; //was -99; float temperatureMV = 99999; //was -99; float salinityMV = 99999; //was -99; int qMV = 9; String timeUnits = "days since 1900-01-01 00:00:00"; //causes roundoff error(!) double timeBaseAndFactor[] = Calendar2.getTimeBaseAndFactor(timeUnits); //impossible values: float minDepth = -0.4f, maxDepth = 10000; //-0.4 allows for imprecise values float minTemperature = -4, maxTemperature = 40; float minSalinity = 0, maxSalinity = 41; if (testMode) { firstYear = 1990; firstMonth = 1; lastYear = 1990; lastMonth = 1; } SSR.verbose = false; String2.setupLog(true, false, logFile, false, 1000000000); String2.log("*** starting bobConsolidateGtsppTgz " + Calendar2.getCurrentISODateTimeStringLocalTZ() + "\n" + "logFile=" + String2.logFileName() + "\n" + String2.standardHelpAboutMessage()); long elapsedTime = System.currentTimeMillis(); //q_pos (position quality flag), q_date_time (time quality flag) int stationCol = -1, organizationCol = -1, dataTypeCol = -1, platformCol = -1, cruiseCol = -1, longitudeCol = -1, latitudeCol = -1, timeCol = -1, depthCol = -1, temperatureCol = -1, salinityCol = -1; int totalNGoodStation = 0, totalNGoodPos = 0, totalNGoodTime = 0, totalNGoodDepth = 0, totalNGoodTemperature = 0, totalNGoodSalinity = 0; int totalNBadStation = 0, totalNBadPos = 0, totalNBadTime = 0, totalNBadDepth = 0, totalNBadTemperature = 0, totalNBadSalinity = 0, totalNWarnings = 0, totalNExceptions = 0; long totalNGoodRows = 0, totalNBadRows = 0; StringArray impossibleNanLat = new StringArray(); StringArray impossibleMinLat = new StringArray(); StringArray impossibleMaxLat = new StringArray(); StringArray impossibleNanLon = new StringArray(); StringArray impossibleMinLon = new StringArray(); StringArray impossibleMaxLon = new StringArray(); //StringArray impossibleNaNDepth = new StringArray(); StringArray impossibleMinDepth = new StringArray(); StringArray impossibleMaxDepth = new StringArray(); //StringArray impossibleNanTemperature = new StringArray(); StringArray impossibleMinTemperature = new StringArray(); StringArray impossibleMaxTemperature = new StringArray(); //StringArray impossibleNanSalinity = new StringArray(); StringArray impossibleMinSalinity = new StringArray(); StringArray impossibleMaxSalinity = new StringArray(); int nLons = 0, nLats = 0, nFiles = 0; int lonSum = 0, latSum = 0; long profilesSum = 0; long rowsSum = 0; //*** process a month's data int year = firstYear; int month = firstMonth; long chunkTime = System.currentTimeMillis(); while (year <= lastYear) { String2.log("\n*** " + Calendar2.getCurrentISODateTimeStringLocalTZ() + " start processing year=" + year + " month=" + month); String zMonth = String2.zeroPad("" + month, 2); String zMonth1 = String2.zeroPad("" + (month + 1), 2); double minEpochSeconds = Calendar2.isoStringToEpochSeconds(year + "-" + zMonth + "-01"); double maxEpochSeconds = Calendar2.isoStringToEpochSeconds(year + "-" + zMonth1 + "-01"); //destination directory String tDestDir = testMode ? testDestDir : destDir + year + "\\" + zMonth + "\\"; File2.makeDirectory(tDestDir); HashMap tableHashMap = new HashMap(); //make sure all files are deleted int waitSeconds = 2; int nAttempts = 10; long cmdTime = System.currentTimeMillis(); String cmd = "del/q " + tDestDir + "*.*"; for (int attempt = 0; attempt < nAttempts; attempt++) { if (attempt % 8 == 0) { String2.log(cmd); SSR.dosShell(cmd, 30 * 60); //10 minutes*60 seconds //File2.deleteAllFiles(tempDir); //previous method } Math2.gc(waitSeconds * 1000); //gtspp: give OS time to settle File destDirFile = new File(tDestDir); File files[] = destDirFile.listFiles(); String2.log(" nRemainingFiles=" + files.length); if (files.length == 0) break; waitSeconds = 2 * nAttempts; } String2.log(" cmd total time=" + Calendar2.elapsedTimeString(System.currentTimeMillis() - cmdTime)); //unzip all atlantic, indian, and pacific .zip files for that month String region2[] = { "at", "in", "pa" }; int nRegions = testMode ? 1 : 3; for (int region = 0; region < nRegions; region++) { String sourceBaseName = "gtspp4_" + region2[region] + year + zMonth; String sourceZipJustFileName = sourceBaseName + ".tgz"; String sourceZipName = zipDir + sourceZipJustFileName; if (!testMode) { //delete all files in tempDir waitSeconds = 2; nAttempts = 10; cmdTime = System.currentTimeMillis(); cmd = "del/q " + tempDir + "*.*"; String2.log(""); //blank line for (int attempt = 0; attempt < nAttempts; attempt++) { String2.log(cmd); SSR.dosShell(cmd, 30 * 60); //30 minutes*60 seconds //File2.deleteAllFiles(tempDir); //previous method //delete dirs too File2.deleteAllFiles(tempDir, true, true); Math2.gc(waitSeconds * 1000); //gtspp: give OS time to settle String2.log(" " + Math2.memoryString()); File tempDirFile = new File(tempDir); File files[] = tempDirFile.listFiles(); String2.log(" nRemainingFiles=" + files.length); if (files.length == 0) break; waitSeconds = 2 * nAttempts; } String2.log(" cmd total time=" + Calendar2.elapsedTimeString(System.currentTimeMillis() - cmdTime)); //unzip file into tempDir //gtspp_at199001.zip cmd = sevenZip + " -y e " + sourceZipName + " -o" + tempDir + " -r"; cmdTime = System.currentTimeMillis(); String2.log("\n*** " + cmd); if (File2.isFile(sourceZipName)) { try { SSR.dosShell(cmd, 30 * 60); //10 minutes*60 seconds String2.log(" cmd time=" + Calendar2.elapsedTimeString(System.currentTimeMillis() - cmdTime)); //extract from the .tar file //gtspp4_at199001.tar cmd = sevenZip + " -y e " + tempDir + sourceBaseName + ".tar -o" + tempDir + " -r"; cmdTime = System.currentTimeMillis(); String2.log("\n*** " + cmd); SSR.dosShell(cmd, 120 * 60); //120 minutes*60 seconds String2.log(" cmd time=" + Calendar2.elapsedTimeString(System.currentTimeMillis() - cmdTime)); } catch (Exception e) { String2.log("Caught exception: " + MustBe.throwableToString(e)); } } //previous method //SSR.unzip(sourceZipName, // tempDir, true, 100 * 60, null); //ignoreZipDirectories, timeOutSeconds 100 minutes } //read each file and put data in proper table String tTempDir = testMode ? testTempDir : tempDir; File tTempDirAsFile = new File(tTempDir); String sourceFileNames[] = tTempDirAsFile.list(); //just the file names String2.log("\nunzipped " + sourceFileNames.length + " files"); int nSourceFileNames = //testMode? 100 : sourceFileNames.length; int nGoodStation = 0, nGoodPos = 0, nGoodTime = 0, nGoodDepth = 0, nGoodTemperature = 0, nGoodSalinity = 0, nGoodRows = 0; int nBadStation = 0, nBadPos = 0, nBadTime = 0, nBadDepth = 0, nBadTemperature = 0, nBadSalinity = 0, nBadRows = 0, nWarnings = 0, nExceptions = 0; long fileReadTime = System.currentTimeMillis(); profilesSum += nSourceFileNames; for (int sfi = 0; sfi < nSourceFileNames; sfi++) { String sourceFileName = sourceFileNames[sfi]; if (sfi % 10000 == 0) { //if (sfi > 0) //2012-12-13 commented out. Let Java handle it. // Math2.gc(3 * 1000); //gtspp: give OS time to settle //high water mark is ~160 MB, so memory not a problem String2.log("file #" + sfi + " " + Math2.memoryString()); } if (!sourceFileName.endsWith(".nc")) { //String2.log("ERROR: not a .nc file: " + sourceFileName); continue; } NetcdfFile ncFile = null; try { //get the station name //gtspp_13635162_te_111.nc gtspp_10313692_cu_111.nc if (!sourceFileName.matches("gtspp_[0-9]+_.*\\.nc")) { //was "\\d+")) {//all digits nBadStation++; throw new SimpleException("Invalid sourceFileName=" + sourceFileName); } int po = sourceFileName.indexOf('_', 6); if (po < 0) { nBadStation++; throw new SimpleException("Invalid sourceFileName=" + sourceFileName); } int station = String2.parseInt(sourceFileName.substring(6, po)); nGoodStation++; String key = sourceZipJustFileName + " " + sourceFileName; //open the file ncFile = NcHelper.openFile(tTempDir + sourceFileName); Variable var; Attributes tVarAtts = new Attributes(); String tUnits; //get all of the data //stream_ident var = ncFile.findVariable("stream_ident"); String organization = ""; String dataType = ""; if (var == null) { nWarnings++; String2.log("WARNING: No stream_ident in " + sourceFileName); } else { PrimitiveArray streamPA = NcHelper.getPrimitiveArray(var); if (streamPA instanceof StringArray && streamPA.size() > 0) { String stream = streamPA.getString(0); if (stream.length() >= 4) { organization = stream.substring(0, 2).trim(); dataType = stream.substring(2, 4).trim(); } else { String2.log("WARNING: stream_ident isn't a 4 char string: " + stream); } } else { String2.log("WARNING: stream_ident isn't a StringArray: " + streamPA.toString()); } } //platform_code var = ncFile.findVariable("gtspp_platform_code"); String platform = ""; if (var == null) { //a small percentage have this problem //nWarnings++; //String2.log("WARNING: No gtspp_platform_code in " + sourceFileName); } else { PrimitiveArray pa = NcHelper.getPrimitiveArray(var); if (pa instanceof StringArray && pa.size() > 0) { platform = pa.getString(0).trim(); //String2.log("platform_code=" + platform_code); } else { String2.log("WARNING: gtspp_platform_code isn't a StringArray: " + pa.toString()); } } //cruise var = ncFile.findVariable("cruise_id"); String cruise = ""; if (var == null) { nWarnings++; String2.log("WARNING: No cruise_id in " + sourceFileName); } else { PrimitiveArray cruisePA = NcHelper.getPrimitiveArray(var); if (cruisePA instanceof StringArray && cruisePA.size() > 0) { cruise = cruisePA.getString(0).trim(); } else { String2.log("WARNING: cruise_id isn't a StringArray: " + cruisePA.toString()); } } //prof_type is TEMP or PSAL so don't save it. /*var = ncFile.findVariable("prof_type"); String prof_type = ""; if (var == null) { nWarnings++; String2.log("WARNING: No prof_type in " + sourceFileName); } else { PrimitiveArray pa = NcHelper.getPrimitiveArray(var); if (pa instanceof StringArray && pa.size() > 0) { prof_type = pa.getString(0).trim(); String2.log("prof_type=" + prof_type); } else { String2.log("WARNING: prof_type isn't a StringArray: " + pa.toString()); } }*/ //position quality flag var = ncFile.findVariable("position_quality_flag"); //was "q_pos"); if (var == null) { nWarnings++; String2.log("WARNING: No position_quality_flag in " + sourceFileName); } else { PrimitiveArray q_pos = NcHelper.getPrimitiveArray(var); if (!(q_pos instanceof IntArray) || q_pos.size() != 1) throw new SimpleException("Invalid position_quality_flag=" + q_pos); int ti = q_pos.getInt(0); if (String2.indexOf(okQF, ti) < 0) { nBadPos++; continue; } //nGoodPos++; is below } //time quality flag var = ncFile.findVariable("time_quality_flag"); //q_date_time"); if (var == null) { nWarnings++; String2.log("WARNING: No time_quality_flag in " + sourceFileName); } else { PrimitiveArray q_date_time = NcHelper.getPrimitiveArray(var); if (!(q_date_time instanceof IntArray) || q_date_time.size() != 1) throw new SimpleException("Invalid time_quality_flag=" + q_date_time); int ti = q_date_time.getInt(0); if (String2.indexOf(okQF, ti) < 0) { nBadTime++; continue; } //nGoodTime is below } //time var = ncFile.findVariable("time"); if (var == null) throw new SimpleException("No time!"); tVarAtts.clear(); NcHelper.getVariableAttributes(var, tVarAtts); tUnits = tVarAtts.getString("units"); if (!timeUnits.equals(tUnits)) throw new SimpleException("Invalid time units=" + tUnits); PrimitiveArray time = NcHelper.getPrimitiveArray(var); if (!(time instanceof DoubleArray) || time.size() != 1) throw new SimpleException("Invalid time=" + time); double tTime = Calendar2.unitsSinceToEpochSeconds(timeBaseAndFactor[0], timeBaseAndFactor[1], time.getDouble(0)); if (tTime < minEpochSeconds || tTime > maxEpochSeconds) throw new SimpleException( "Invalid tTime=" + Calendar2.safeEpochSecondsToIsoStringTZ(tTime, "")); //original times (that I looked at) are to nearest second //so round to nearest second (fix .99999 problems) tTime = Math.rint(tTime); nGoodTime++; //longitude (position qFlag is good) var = ncFile.findVariable("longitude"); if (var == null) { impossibleNanLon.add(key + " lon=null"); continue; } PrimitiveArray longitude = NcHelper.getPrimitiveArray(var); if (!(longitude instanceof FloatArray) || longitude.size() != 1) { impossibleNanLon.add(key + " lon=wrongTypeOrSize"); continue; } float lon = longitude.getFloat(0); if (Float.isNaN(lon)) { impossibleNanLon.add(key + " lon=NaN"); continue; } else if (lon < minLon) { impossibleMinLon.add(key + " lon=" + lon); //fall through } else if (lon > maxLon) { impossibleMaxLon.add(key + " lon=" + lon); //fall through } lon = (float) Math2.anglePM180(lon); //latitude (position qFlag is good) var = ncFile.findVariable("latitude"); if (var == null) { impossibleNanLat.add(key + " lat=null"); continue; } PrimitiveArray latitude = NcHelper.getPrimitiveArray(var); if (!(latitude instanceof FloatArray) || latitude.size() != 1) { impossibleNanLat.add(key + " lat=wrongTypeOrSize"); continue; } float lat = latitude.getFloat(0); if (Float.isNaN(lat)) { impossibleNanLat.add(key + " lat=NaN"); continue; } else if (lat < minLat) { impossibleMinLat.add(key + " lat=" + lat); continue; } else if (lat > maxLat) { impossibleMaxLat.add(key + " lat=" + lat); continue; } nGoodPos++; //depth var = ncFile.findVariable("z"); if (var == null) throw new SimpleException("No z!"); PrimitiveArray depth = NcHelper.getPrimitiveArray(var); if (!(depth instanceof FloatArray) || depth.size() == 0) throw new SimpleException("Invalid z=" + depth); int nDepth = depth.size(); //DEPH_qparm var = ncFile.findVariable("z_variable_quality_flag"); //DEPH_qparm"); if (var == null) throw new SimpleException("No z_variable_quality_flag!"); PrimitiveArray DEPH_qparm = NcHelper.getPrimitiveArray(var); if (!(DEPH_qparm instanceof IntArray) || DEPH_qparm.size() != nDepth) throw new SimpleException("Invalid z_variable_quality_flag=" + DEPH_qparm); //nGoodDepth is below //temperature var = ncFile.findVariable("temperature"); PrimitiveArray temperature; PrimitiveArray TEMP_qparm; float temperatureFV = temperatureMV; if (var == null) { //nWarnings++; //String2.log("WARNING: No temperature in " + sourceFileName); reasonably common temperature = PrimitiveArray.factory(float.class, nDepth, "" + temperatureMV); TEMP_qparm = PrimitiveArray.factory(int.class, nDepth, "" + qMV); } else { temperature = NcHelper.getPrimitiveArray(var); if (!(temperature instanceof FloatArray) || temperature.size() != nDepth) throw new SimpleException("Invalid temperature=" + temperature); tVarAtts.clear(); NcHelper.getVariableAttributes(var, tVarAtts); temperatureFV = tVarAtts.getFloat("_FillValue"); if (!Float.isNaN(temperatureFV) && temperatureFV != temperatureMV) throw new SimpleException("Invalid temperature _FillValue=" + temperatureFV); //TEMP_qparm var = ncFile.findVariable("temperature_quality_flag"); //TEMP_qparm"); if (var == null) { nWarnings++; String2.log("WARNING: No temperature_quality_flag in " + sourceFileName); TEMP_qparm = PrimitiveArray.factory(int.class, nDepth, "" + qMV); } else { TEMP_qparm = NcHelper.getPrimitiveArray(var); if (!(TEMP_qparm instanceof IntArray) || TEMP_qparm.size() != nDepth) throw new SimpleException("Invalid temperature_quality_flag=" + TEMP_qparm); } } //salinity var = ncFile.findVariable("salinity"); PrimitiveArray salinity; PrimitiveArray PSAL_qparm; float salinityFV = salinityMV; if (var == null) { //String2.log("WARNING: No salinity in " + sourceFileName); //very common salinity = PrimitiveArray.factory(float.class, nDepth, "" + salinityMV); PSAL_qparm = PrimitiveArray.factory(int.class, nDepth, "" + qMV); } else { salinity = NcHelper.getPrimitiveArray(var); if (!(salinity instanceof FloatArray) || salinity.size() != nDepth) throw new SimpleException("Invalid salinity=" + salinity); tVarAtts.clear(); NcHelper.getVariableAttributes(var, tVarAtts); salinityFV = tVarAtts.getFloat("_FillValue"); if (!Float.isNaN(salinityFV) && salinityFV != salinityMV) throw new SimpleException("Invalid salinity _FillValue=" + salinityFV); //PSAL_qparm var = ncFile.findVariable("salinity_quality_flag"); //PSAL_qparm"); if (var == null) { nWarnings++; String2.log("WARNING: No salinity_quality_flag in " + sourceFileName); PSAL_qparm = PrimitiveArray.factory(int.class, nDepth, "" + qMV); } else { PSAL_qparm = NcHelper.getPrimitiveArray(var); if (!(PSAL_qparm instanceof IntArray) || PSAL_qparm.size() != nDepth) throw new SimpleException("Invalid salinity_quality_flag=" + PSAL_qparm); } } //clean the data //(good to do it here so memory usage is low -- table remains as small as possible) //Change "impossible" data to NaN //(from https://www.nodc.noaa.gov/GTSPP/document/qcmans/GTSPP_RT_QC_Manual_20090916.pdf //pg 61 has Table 2.1: Global Impossible Parameter Values). BitSet keep = new BitSet(); keep.set(0, nDepth); //all true //find worst impossible depth/temperature/salinity for this station //boolean tImpossibleNanDepth = false; //boolean tImpossibleNanTemperature = false; //boolean tImpossibleNanSalinity = false; float tImpossibleMinDepth = minDepth; float tImpossibleMaxDepth = maxDepth; float tImpossibleMinTemperature = minTemperature; float tImpossibleMaxTemperature = maxTemperature; float tImpossibleMinSalinity = minSalinity; float tImpossibleMaxSalinity = maxSalinity; for (int row = 0; row < nDepth; row++) { //DEPH_qparm int qs = DEPH_qparm.getInt(row); float f = depth.getFloat(row); if (String2.indexOf(okQF, qs) < 0) { nBadDepth++; keep.clear(row); continue; } else if (Float.isNaN(f) || f == depthMV) { //"impossible" depth //tImpossibleNanDepth = true; nBadDepth++; keep.clear(row); continue; } else if (f < minDepth) { tImpossibleMinDepth = Math.min(tImpossibleMinDepth, f); nBadDepth++; keep.clear(row); continue; } else if (f > maxDepth) { tImpossibleMaxDepth = Math.max(tImpossibleMaxDepth, f); nBadDepth++; keep.clear(row); continue; } nGoodDepth++; boolean hasData = false; //temperature qs = TEMP_qparm.getInt(row); f = temperature.getFloat(row); if (String2.indexOf(okQF, qs) < 0) { temperature.setString(row, ""); //so bad value is now NaN nBadTemperature++; } else if (Float.isNaN(f) || f == temperatureMV) { temperature.setString(row, ""); //so missing value is now NaN nBadTemperature++; } else if (f < minTemperature) { //"impossible" water temperature tImpossibleMinTemperature = Math.min(tImpossibleMinTemperature, f); temperature.setString(row, ""); //so impossible value is now NaN nBadTemperature++; } else if (f > maxTemperature) { //"impossible" water temperature tImpossibleMaxTemperature = Math.max(tImpossibleMaxTemperature, f); temperature.setString(row, ""); //so impossible value is now NaN nBadTemperature++; } else { nGoodTemperature++; hasData = true; } //salinity qs = PSAL_qparm.getInt(row); f = salinity.getFloat(row); if (String2.indexOf(okQF, qs) < 0) { salinity.setString(row, ""); //so bad value is now NaN nBadSalinity++; } else if (Float.isNaN(f) || f == salinityMV) { salinity.setString(row, ""); //so missing value is now NaN nBadSalinity++; } else if (f < minSalinity) { //"impossible" salinity tImpossibleMinSalinity = Math.min(tImpossibleMinSalinity, f); salinity.setString(row, ""); //so impossible value is now NaN nBadSalinity++; } else if (f > maxSalinity) { //"impossible" salinity tImpossibleMaxSalinity = Math.max(tImpossibleMaxSalinity, f); salinity.setString(row, ""); //so impossible value is now NaN nBadSalinity++; } else { nGoodSalinity++; hasData = true; } //no valid temperature or salinity data? if (!hasData) { keep.clear(row); } } //ensure sizes still correct Test.ensureEqual(depth.size(), nDepth, "depth.size changed!"); Test.ensureEqual(temperature.size(), nDepth, "temperature.size changed!"); Test.ensureEqual(salinity.size(), nDepth, "salinity.size changed!"); //actually remove the bad rows int tnGood = keep.cardinality(); if (testMode && verbose) String2.log( sourceFileName + ": nGoodRows=" + tnGood + " nBadRows=" + (nDepth - tnGood)); nGoodRows += tnGood; nBadRows += nDepth - tnGood; depth.justKeep(keep); temperature.justKeep(keep); salinity.justKeep(keep); nDepth = depth.size(); //impossible //if (tImpossibleNanDepth) // impossibleNanDepth.add(key + " hasNaN=true"); //if (tImpossibleNanTemperature) // impossibleNanTemperature.add(key + " hasNaN=true"); //if (tImpossibleNanSalinity) // impossibleNanSalinity.add(key + " hasNaN=true"); if (tImpossibleMinDepth < minDepth) impossibleMinDepth.add(key + " worst = " + tImpossibleMinDepth); if (tImpossibleMaxDepth > maxDepth) impossibleMaxDepth.add(key + " worst = " + tImpossibleMaxDepth); if (tImpossibleMinTemperature < minTemperature) impossibleMinTemperature.add(key + " worst = " + tImpossibleMinTemperature); if (tImpossibleMaxTemperature > maxTemperature) impossibleMaxTemperature.add(key + " worst = " + tImpossibleMaxTemperature); if (tImpossibleMinSalinity < minSalinity) impossibleMinSalinity.add(key + " worst = " + tImpossibleMinSalinity); if (tImpossibleMaxSalinity > maxSalinity) impossibleMaxSalinity.add(key + " worst = " + tImpossibleMaxSalinity); //which table if (tnGood == 0) continue; int loni = Math2 .roundToInt(Math.floor((Math.min(lon, maxLon - 0.1f) - minLon) / chunkSize)); int lati = Math2 .roundToInt(Math.floor((Math.min(lat, maxLat - 0.1f) - minLat) / chunkSize)); String outTableName = (minLon + loni * chunkSize) + "E_" + (minLat + lati * chunkSize) + "N"; //String2.replaceAll(cruise + "_" + organization + dataType, ' ', '_'); //too many: 3000+/month in 2011 Table tTable = (Table) tableHashMap.get(outTableName); if (tTable == null) { Attributes ncGlobalAtts = new Attributes(); NcHelper.getGlobalAttributes(ncFile, ncGlobalAtts); String tHistory = ncGlobalAtts.getString("history"); tHistory = tHistory != null && tHistory.length() > 0 ? tHistory + "\n" : ""; //make a table for this platform tTable = new Table(); Attributes ga = tTable.globalAttributes(); String ack = "These data were acquired from the US NOAA National Oceanographic Data Center (NODC) on " + today + " from https://www.nodc.noaa.gov/GTSPP/."; ga.add("acknowledgment", ack); ga.add("license", "These data are openly available to the public. " + "Please acknowledge the use of these data with:\n" + ack + "\n\n" + "[standard]"); ga.add("history", tHistory + ".tgz files from ftp.nodc.noaa.gov /pub/gtspp/best_nc/ (https://www.nodc.noaa.gov/GTSPP/)\n" + today + " Most recent ingest, clean, and reformat at ERD (bob.simons at noaa.gov)."); ga.add("infoUrl", "https://www.nodc.noaa.gov/GTSPP/"); ga.add("institution", "NOAA NODC"); ga.add("title", "Global Temperature and Salinity Profile Programme (GTSPP) Data"); String attName = "gtspp_ConventionVersion"; String attValue = ncGlobalAtts.getString(attName); if (attValue != null && attValue.length() > 0) ga.add(attName, attValue); attName = "gtspp_program"; attValue = ncGlobalAtts.getString(attName); if (attValue != null && attValue.length() > 0) ga.add(attName, attValue); attName = "gtspp_programVersion"; attValue = ncGlobalAtts.getString(attName); if (attValue != null && attValue.length() > 0) ga.add(attName, attValue); attName = "gtspp_handbook_version"; attValue = ncGlobalAtts.getString(attName); if (attValue != null && attValue.length() > 0) ga.add(attName, attValue); organizationCol = tTable.addColumn(tTable.nColumns(), "org", new StringArray(), new Attributes()); platformCol = tTable.addColumn(tTable.nColumns(), "platform", new StringArray(), new Attributes()); dataTypeCol = tTable.addColumn(tTable.nColumns(), "type", new StringArray(), new Attributes()); cruiseCol = tTable.addColumn(tTable.nColumns(), "cruise", new StringArray(), new Attributes()); stationCol = tTable.addColumn(tTable.nColumns(), "station_id", new IntArray(), new Attributes()); longitudeCol = tTable.addColumn(tTable.nColumns(), "longitude", new FloatArray(), (new Attributes()).add("units", EDV.LON_UNITS)); latitudeCol = tTable.addColumn(tTable.nColumns(), "latitude", new FloatArray(), (new Attributes()).add("units", EDV.LAT_UNITS)); timeCol = tTable.addColumn(tTable.nColumns(), "time", new DoubleArray(), (new Attributes()).add("units", EDV.TIME_UNITS)); depthCol = tTable.addColumn(tTable.nColumns(), "depth", new FloatArray(), (new Attributes()).add("units", "m")); temperatureCol = tTable.addColumn(tTable.nColumns(), "temperature", new FloatArray(), (new Attributes()).add("units", "degree_C")); salinityCol = tTable.addColumn(tTable.nColumns(), "salinity", new FloatArray(), (new Attributes()).add("units", "1e-3")); //PSU changed to 1e-3 with CF std names 25 tableHashMap.put(outTableName, tTable); } //put data in tTable int oNRows = tTable.nRows(); ((StringArray) tTable.getColumn(organizationCol)).addN(nDepth, organization); ((StringArray) tTable.getColumn(platformCol)).addN(nDepth, platform); ((StringArray) tTable.getColumn(dataTypeCol)).addN(nDepth, dataType); ((StringArray) tTable.getColumn(cruiseCol)).addN(nDepth, cruise); ((IntArray) tTable.getColumn(stationCol)).addN(nDepth, station); ((FloatArray) tTable.getColumn(longitudeCol)).addN(nDepth, lon); ((FloatArray) tTable.getColumn(latitudeCol)).addN(nDepth, lat); ((DoubleArray) tTable.getColumn(timeCol)).addN(nDepth, tTime); ((FloatArray) tTable.getColumn(depthCol)).append(depth); ((FloatArray) tTable.getColumn(temperatureCol)).append(temperature); ((FloatArray) tTable.getColumn(salinityCol)).append(salinity); //ensure the table is valid (same size for each column) tTable.ensureValid(); } catch (Throwable t) { nExceptions++; String2.log( "ERROR while processing " + sourceFileName + "\n " + MustBe.throwableToString(t)); } finally { //always close the ncFile if (ncFile != null) { try { ncFile.close(); } catch (Throwable t) { String2.log("ERROR: unable to close " + sourceFileName + "\n" + MustBe.getShortErrorMessage(t)); } } } } String2.log("\n time to read all those files = " + Calendar2.elapsedTimeString(System.currentTimeMillis() - fileReadTime)); //end of region loop String2.log("\nIn zip=" + sourceZipName + "\n nExceptions= " + nExceptions + " nWarnings=" + nWarnings + "\n nBadStation= " + nBadStation + " nGoodStation=" + nGoodStation + "\n nBadPos= " + nBadPos + " nGoodPos=" + nGoodPos + "\n nBadTime= " + nBadTime + " nGoodTime=" + nGoodTime + "\n nBadDepth= " + nBadDepth + " nGoodDepth=" + nGoodDepth + "\n nBadTemperature=" + nBadTemperature + " nGoodTemperature=" + nGoodTemperature + "\n nBadSalinity= " + nBadSalinity + " nGoodSalinity=" + nGoodSalinity); totalNGoodStation += nGoodStation; totalNGoodPos += nGoodPos; totalNGoodTime += nGoodTime; totalNGoodDepth += nGoodDepth; totalNGoodTemperature += nGoodTemperature; totalNGoodSalinity += nGoodSalinity; totalNGoodRows += nGoodRows; totalNBadPos += nBadPos; totalNBadTime += nBadTime; totalNBadDepth += nBadDepth; totalNBadTemperature += nBadTemperature; totalNBadSalinity += nBadSalinity; totalNBadRows += nBadRows; totalNWarnings += nWarnings; totalNExceptions += nExceptions; } //end of region loop //save by outTableName boolean filePrinted = false; Object keys[] = tableHashMap.keySet().toArray(); int nKeys = keys.length; String2.log("\n*** saving nFiles=" + nKeys); for (int keyi = 0; keyi < nKeys; keyi++) { String key = keys[keyi].toString(); Table tTable = (Table) tableHashMap.remove(key); if (tTable == null || tTable.nRows() == 0) { String2.log("Unexpected: no table for key=" + key); continue; } //sort by time, station, depth //depth matches the source files: from surface to deepest tTable.sort(new int[] { timeCol, stationCol, depthCol }, new boolean[] { true, true, true }); //is this saving a small lat lon range? double stationStats[] = tTable.getColumn(stationCol).calculateStats(); //double lonStats[] = tTable.getColumn(longitudeCol).calculateStats(); //double latStats[] = tTable.getColumn(latitudeCol).calculateStats(); //nLats++; //double latRange = latStats[PrimitiveArray.STATS_MAX] - latStats[PrimitiveArray.STATS_MIN]; //latSum += latRange; rowsSum += tTable.nRows(); String2.log(" stationRange=" + Math2.roundToInt( stationStats[PrimitiveArray.STATS_MAX] - stationStats[PrimitiveArray.STATS_MIN]) + //" lonRange=" + Math2.roundToInt(lonStats[ PrimitiveArray.STATS_MAX] - lonStats[ PrimitiveArray.STATS_MIN]) + //" latRange=" + Math2.roundToInt(latRange) + " nRows=" + tTable.nRows()); //save it String tName = tDestDir + String2.encodeFileNameSafe(key); /*if (lonStats[PrimitiveArray.STATS_MAX] > 45 && lonStats[PrimitiveArray.STATS_MIN] < -45) { //NO MORE: This happened with 1 file/cruise, // but won't happen now with lon/lat tiles. //crosses dateline (or widely across lon=0)? split into 2 files Table ttTable = (Table)tTable.clone(); ttTable.oneStepApplyConstraint(0, "longitude", "<", "0"); ttTable.saveAsFlatNc(tName + "_W.nc", "row", false); double lonStatsW[] = ttTable.getColumn(longitudeCol).calculateStats(); nLons++; double lonRangeW = lonStatsW[PrimitiveArray.STATS_MAX] - lonStatsW[PrimitiveArray.STATS_MIN]; lonSum += lonRangeW; ttTable = (Table)tTable.clone(); ttTable.oneStepApplyConstraint(0, "longitude", ">=", "0"); ttTable.saveAsFlatNc(tName + "_E.nc", "row", false); double lonStatsE[] = ttTable.getColumn(longitudeCol).calculateStats(); nLons++; double lonRangeE = lonStatsE[PrimitiveArray.STATS_MAX] - lonStatsE[PrimitiveArray.STATS_MIN]; lonSum += lonRangeE; String2.log(" westLonRange=" + Math2.roundToInt(lonRangeW) + " eastLonRange=" + Math2.roundToInt(lonRangeE)); } else */ { //nLons++; nFiles++; //create trajectory variable: platform + cruise StringArray pl = (StringArray) tTable.getColumn("platform"); StringArray cr = (StringArray) tTable.getColumn("cruise"); StringArray or = (StringArray) tTable.getColumn("org"); StringArray ty = (StringArray) tTable.getColumn("type"); StringArray tr = new StringArray(); int n = pl.size(); for (int i = 0; i < n; i++) { pl.set(i, String2.whitespacesToSpace(pl.get(i))); cr.set(i, String2.whitespacesToSpace(cr.get(i))); or.set(i, String2.whitespacesToSpace(or.get(i))); ty.set(i, String2.whitespacesToSpace(ty.get(i))); tr.add(or.getString(i) + "_" + ty.getString(i) + "_" + pl.getString(i) + "_" + cr.getString(i)); } tTable.addColumn(0, "trajectory", tr, new Attributes()); tTable.saveAsFlatNc(tName + ".nc", "row", false); //convertToFakeMissingValues (keep mv's as NaNs) } //print a file if (testMode && !filePrinted) { filePrinted = true; String2.log(NcHelper.dumpString(tName, true)); } } String2.log("\ncumulative nProfiles=" + profilesSum + " nRows=" + rowsSum + " mean nRows/file=" + (rowsSum / Math.max(1, nFiles))); //if (nLats > 0) // String2.log( "cumulative nLats=" + nLats + " meanLatRange=" + (float)(latSum / nLats)); //if (nLons > 0) { // String2.log( "cumulative nLons=" + nLons + " meanLonRange=" + (float)(lonSum / nLons)); // String2.log("mean nRows per saved file = " + (rowsSum / nLons)); //} //print list of impossible at end of year or end of run if (month == 12 || (year == lastYear && month == lastMonth)) { String2.log("\n*** " + Calendar2.getCurrentISODateTimeStringLocalTZ() + " bobConsolidateGtsppTgz finished the chunk ending " + year + "-" + month + "\n" + "chunkTime=" + Calendar2.elapsedTimeString(System.currentTimeMillis() - chunkTime)); chunkTime = System.currentTimeMillis(); //print impossible statistics String2.log("\nCumulative number of stations with:\n" + "impossibleNanLon = " + impossibleNanLon.size() + "\n" + "impossibleMinLon = " + impossibleMinLon.size() + "\n" + "impossibleMaxLon = " + impossibleMaxLon.size() + "\n" + "impossibleNanLat = " + impossibleNanLat.size() + "\n" + "impossibleMinLat = " + impossibleMinLat.size() + "\n" + "impossibleMaxLat = " + impossibleMaxLat.size() + "\n" + "impossibleMinDepth = " + impossibleMinDepth.size() + "\n" + "impossibleMaxDepth = " + impossibleMaxDepth.size() + "\n" + //"impossibleLatLon = " + impossibleLatLon.size() + "\n" + "impossibleMinTemperature = " + impossibleMinTemperature.size() + "\n" + "impossibleMaxTemperature = " + impossibleMaxTemperature.size() + "\n" + "impossibleMinSalinity = " + impossibleMinSalinity.size() + "\n" + "impossibleMaxSalinity = " + impossibleMaxSalinity.size() + "\n"); //lon String2.log("\n*** " + impossibleNanLon.size() + " stations had invalid lon" + " and good pos quality flags (" + okQFCsv + ")."); impossibleNanLon.sortIgnoreCase(); String2.log(impossibleNanLon.toNewlineString()); String2.log("\n*** " + impossibleMinLon.size() + " stations had lon<" + minLon + " and good pos quality flags (" + okQFCsv + ")."); impossibleMinLon.sortIgnoreCase(); String2.log(impossibleMinLon.toNewlineString()); String2.log("\n*** " + impossibleMaxLon.size() + " stations had lon>" + maxLon + " and good pos quality flags (" + okQFCsv + ")."); impossibleMaxLon.sortIgnoreCase(); String2.log(impossibleMaxLon.toNewlineString()); //lat String2.log("\n*** " + impossibleNanLat.size() + " stations had invalid lat" + " and good pos quality flags (" + okQFCsv + ")."); impossibleNanLat.sortIgnoreCase(); String2.log(impossibleNanLat.toNewlineString()); String2.log("\n*** " + impossibleMinLat.size() + " stations had lat<" + minLat + " and good pos quality flags (" + okQFCsv + ")."); impossibleMinLat.sortIgnoreCase(); String2.log(impossibleMinLat.toNewlineString()); String2.log("\n*** " + impossibleMaxLat.size() + " stations had lat>" + maxLat + " and good pos quality flags (" + okQFCsv + ")."); impossibleMaxLat.sortIgnoreCase(); String2.log(impossibleMaxLat.toNewlineString()); //depth String2.log("\n*** " + impossibleMinDepth.size() + " stations had depth<" + minDepth + " and good depth quality flags (" + okQFCsv + ")."); impossibleMinDepth.sortIgnoreCase(); String2.log(impossibleMinDepth.toNewlineString()); String2.log("\n*** " + impossibleMaxDepth.size() + " stations had depth>" + maxDepth + " and good depth quality flags (" + okQFCsv + ")."); impossibleMaxDepth.sortIgnoreCase(); String2.log(impossibleMaxDepth.toNewlineString()); //sa = impossibleLatLon.toArray(); //Arrays.sort(sa); //String2.log("\n*** " + sa.length + " stations had impossible latitude or longitude values" + // " and good q_pos quality flags."); //String2.log(String2.toNewlineString(sa)); String2.log("\n*** " + impossibleMinTemperature.size() + " stations had temperature<" + minTemperature + " and good temperature quality flags (" + okQFCsv + ")."); impossibleMinTemperature.sortIgnoreCase(); String2.log(impossibleMinTemperature.toNewlineString()); String2.log("\n*** " + impossibleMaxTemperature.size() + " stations had temperature>" + maxTemperature + " and good temperature quality flags (" + okQFCsv + ")."); impossibleMaxTemperature.sortIgnoreCase(); String2.log(impossibleMaxTemperature.toNewlineString()); String2.log("\n*** " + impossibleMinSalinity.size() + " stations had salinity<" + minSalinity + " and good salinity quality flags (" + okQFCsv + ")."); impossibleMinSalinity.sortIgnoreCase(); String2.log(impossibleMinSalinity.toNewlineString()); String2.log("\n*** " + impossibleMaxSalinity.size() + " stations had salinity>" + maxSalinity + " and good salinity quality flags (" + okQFCsv + ")."); impossibleMaxSalinity.sortIgnoreCase(); String2.log(impossibleMaxSalinity.toNewlineString()); } //are we done? if (year == lastYear && month == lastMonth) break; //increment the month month++; if (month == 13) { year++; month = 1; } } //end of month/year loop String2.log("\n*** bobConsolidateGtspp completely finished " + firstYear + "-" + firstMonth + " through " + lastYear + "-" + lastMonth); String2.log("\n***" + "\ntotalNExceptions= " + totalNExceptions + " totalNWarnings= " + totalNWarnings + "\ntotalNBadStation= " + totalNBadStation + " totalNGoodStation= " + totalNGoodStation + "\ntotalNBadPos= " + totalNBadPos + " totalNGoodPos= " + totalNGoodPos + "\ntotalNBadTime= " + totalNBadTime + " totalNGoodTime= " + totalNGoodTime + "\ntotalNBadDepth= " + totalNBadDepth + " totalNGoodDepth= " + totalNGoodDepth + "\ntotalNBadTemperature=" + totalNBadTemperature + " totalNGoodTemperature=" + totalNGoodTemperature + "\ntotalNBadSalinity= " + totalNBadSalinity + " totalNGoodSalinity= " + totalNGoodSalinity + "\ntotalNBadRows= " + totalNBadRows + " totalNGoodRows= " + totalNGoodRows + "\nlogFile=F:/data/gtspp/log.txt" + "\n\n*** all finished time=" + Calendar2.elapsedTimeString(System.currentTimeMillis() - elapsedTime)); String2.returnLoggingToSystemOut(); }
From source file:fr.amap.lidar.amapvox.gui.MainFrameController.java
private void editVoxelSpace(final File voxelFile) { if (editingFrameOpened) { return;/*ww w . j a v a 2 s . c o m*/ } editingFrameOpened = true; voxelsToRemove.clear(); final String attributeToView = "PadBVTotal"; //window size ObservableList<Screen> screens = Screen.getScreens(); if (screens != null && screens.size() > 0) { SCREEN_WIDTH = screens.get(0).getBounds().getWidth(); SCREEN_HEIGHT = screens.get(0).getBounds().getHeight(); } try { Service s = new Service() { @Override protected Task createTask() { return new Task() { @Override protected Object call() throws Exception { SimpleViewer viewer3D = new SimpleViewer((int) (SCREEN_WIDTH / 4.0d), (int) (SCREEN_HEIGHT / 4.0d), (int) (SCREEN_WIDTH / 1.5d), (int) (SCREEN_HEIGHT / 2.0d), voxelFile.toString()); fr.amap.viewer3d.object.scene.Scene scene = viewer3D.getScene(); /** * *VOXEL SPACE** */ updateMessage("Loading voxel space: " + voxelFile.getAbsolutePath()); final VoxelSpaceSceneObject voxelSpace = new VoxelSpaceSceneObject(voxelFile); voxelSpace.setMousePickable(true); voxelSpace.addVoxelSpaceListener(new VoxelSpaceAdapter() { @Override public void voxelSpaceCreationProgress(int progress) { updateProgress(progress, 100); } }); voxelSpace.loadVoxels(); float voxelResolution = voxelSpace.data.getVoxelSpaceInfos().getResolution(); VoxelFileReader reader = new VoxelFileReader(voxelFile); VoxelSpaceInfos infos = reader.getVoxelSpaceInfos(); final SceneObject boundingBox = new SimpleSceneObject(); boundingBox.setMesh(GLMeshFactory.createBoundingBox((float) infos.getMinCorner().x, (float) infos.getMinCorner().y, (float) infos.getMinCorner().z, (float) infos.getMaxCorner().x, (float) infos.getMaxCorner().y, (float) infos.getMaxCorner().z)); SimpleShader s = new SimpleShader(); s.setColor(new Vec3F(1, 0, 0)); boundingBox.setShader(s); boundingBox.setDrawType(GLMesh.DrawType.LINES); scene.addSceneObject(boundingBox); /* * Voxel information */ StringToImage stringToImage = new StringToImage(1024, 1024); stringToImage.setAdaptableFontSize(true); stringToImage.setBackgroundColor(new Color(255, 255, 255, 127)); stringToImage.setTextColor(new Color(0, 0, 0, 255)); BufferedImage image = new BufferedImage(1024, 1024, BufferedImage.TYPE_INT_ARGB); Texture texture = new Texture(image); int pickingInfoObjectWidth = viewer3D.getWidth() / 5; int pickingInfoObjectHeight = viewer3D.getHeight() / 5; SceneObject pickingInfoObject = SceneObjectFactory.createTexturedPlane( new Vec3F(viewer3D.getWidth() - pickingInfoObjectWidth, viewer3D.getHeight() - pickingInfoObjectHeight, 0), pickingInfoObjectWidth, pickingInfoObjectHeight, texture); pickingInfoObject.setShader(new TextureShader()); pickingInfoObject.setDrawType(GLMesh.DrawType.TRIANGLES); SceneObject sceneObjectSelectedVox = new SimpleSceneObject( GLMeshFactory.createBoundingBox(-voxelResolution / 2.0f, -voxelResolution / 2.0f, -voxelResolution / 2.0f, voxelResolution / 2.0f, voxelResolution / 2.0f, voxelResolution / 2.0f), false); SimpleShader simpleShader = new SimpleShader(); simpleShader.setColor(new Vec3F(1, 0, 0)); sceneObjectSelectedVox.setVisible(false); sceneObjectSelectedVox.setShader(simpleShader); sceneObjectSelectedVox.setDrawType(GLMesh.DrawType.LINES); viewer3D.getScene().addSceneObject(sceneObjectSelectedVox); final SimpleObjectProperty<VoxelObject> selectedVoxel = new SimpleObjectProperty<>(); SceneObjectListener listener = new SceneObjectListener() { @Override public void clicked(SceneObject sceneObject, MousePicker mousePicker, Point3D intersection) { Vec3F camLocation = viewer3D.getScene().getCamera().getLocation(); selectedVoxel.set(voxelSpace.doPicking(mousePicker)); if (selectedVoxel.get() != null) { String[][] lines = new String[voxelSpace.getColumnsNames().length][2]; for (int i = 0; i < voxelSpace.getColumnsNames().length; i++) { lines[i][0] = voxelSpace.getColumnsNames()[i]; float attribut = selectedVoxel.get().getAttributs()[i]; if (Float.isNaN(attribut)) { lines[i][1] = "NaN"; } else { lines[i][1] = String .valueOf(Math.round(attribut * 1000.0f) / 1000.0f); } } arrangeText(lines); String text = ""; for (int i = 0; i < voxelSpace.getColumnsNames().length; i++) { String attribut = lines[i][0] + " " + lines[i][1]; text += attribut + "\n"; } stringToImage.setText(text, 0, 0); texture.setBufferedImage(stringToImage.buildImage()); Point3f voxelPosition = voxelSpace.getVoxelPosition(selectedVoxel.get().$i, selectedVoxel.get().$j, selectedVoxel.get().$k); sceneObjectSelectedVox.setPosition( new Point3F(voxelPosition.x, voxelPosition.y, voxelPosition.z)); sceneObjectSelectedVox.setVisible(true); pickingInfoObject.setVisible(true); } else { sceneObjectSelectedVox.setVisible(false); pickingInfoObject.setVisible(false); } } }; final SimpleIntegerProperty currentZCropIndex = new SimpleIntegerProperty(0); viewer3D.addEventListener(new EventManager(null, new InputKeyListener()) { @Override public void updateEvents() { if (this.keyboard.isKeyClicked(KeyEvent.VK_DELETE)) { if (selectedVoxel.get() != null) { selectedVoxel.get().setAlpha(0); selectedVoxel.get().isHidden = true; voxelSpace.updateVao(); sceneObjectSelectedVox.setVisible(false); pickingInfoObject.setVisible(false); voxelsToRemove.add(new Point3i(selectedVoxel.get().$i, selectedVoxel.get().$j, selectedVoxel.get().$k)); } } } }); voxelSpace.addSceneObjectListener(listener); voxelSpace.changeCurrentAttribut(attributeToView); voxelSpace.setShader(new InstanceLightedShader()); voxelSpace.setDrawType(GLMesh.DrawType.TRIANGLES); scene.addSceneObject(voxelSpace); scene.addSceneObjectAsHud(pickingInfoObject); /** * *scale** */ updateMessage("Generating scale"); final Texture scaleTexture = new Texture(ScaleGradient.createColorScaleBufferedImage( voxelSpace.getGradient(), voxelSpace.getAttributValueMin(), voxelSpace.getAttributValueMax(), viewer3D.getWidth() - 80, (int) (viewer3D.getHeight() / 20), ScaleGradient.Orientation.HORIZONTAL, 5, 8)); SceneObject scalePlane = SceneObjectFactory.createTexturedPlane(new Vec3F(40, 20, 0), (int) (viewer3D.getWidth() - 80), (int) (viewer3D.getHeight() / 20), scaleTexture); scalePlane.setShader(new TextureShader()); scalePlane.setDrawType(GLMesh.DrawType.TRIANGLES); scene.addSceneObjectAsHud(scalePlane); voxelSpace.addPropertyChangeListener("gradientUpdated", new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { BufferedImage image = ScaleGradient.createColorScaleBufferedImage( voxelSpace.getGradient(), voxelSpace.getAttributValueMin(), voxelSpace.getAttributValueMax(), viewer3D.getWidth() - 80, (int) (viewer3D.getHeight() / 20), ScaleGradient.Orientation.HORIZONTAL, 5, 8); scaleTexture.setBufferedImage(image); } }); /** * *light** */ scene.setLightPosition( new Point3F(voxelSpace.getGravityCenter().x, voxelSpace.getGravityCenter().y, voxelSpace.getGravityCenter().z + voxelSpace.widthZ + 100)); /** * *camera** */ TrackballCamera trackballCamera = new TrackballCamera(); trackballCamera.setPivot(voxelSpace); trackballCamera .setLocation(new Vec3F(voxelSpace.getGravityCenter().x + voxelSpace.widthX, voxelSpace.getGravityCenter().y + voxelSpace.widthY, voxelSpace.getGravityCenter().z + voxelSpace.widthZ)); viewer3D.getScene().setCamera(trackballCamera); Platform.runLater(new Runnable() { @Override public void run() { final Stage toolBarFrameStage = new Stage(); final FXMLLoader loader = new FXMLLoader( getClass().getResource("/fxml/ToolBoxFrame.fxml")); try { stage.setAlwaysOnTop(false); Parent root = loader.load(); toolBarFrameStage.setScene(new Scene(root)); toolBarFrameStage.initStyle(StageStyle.UNDECORATED); toolBarFrameStage.setAlwaysOnTop(true); ToolBoxFrameController toolBarFrameController = loader.getController(); toolBarFrameController.setStage(toolBarFrameStage); toolBarFrameStage.setX(viewer3D.getPosition().getX()); toolBarFrameStage.setY(viewer3D.getPosition().getY()); toolBarFrameController.setJoglListener(viewer3D.getJoglContext()); toolBarFrameController.setAttributes(attributeToView, voxelSpace.data.getVoxelSpaceInfos().getColumnNames()); toolBarFrameStage.focusedProperty() .addListener(new ChangeListener<Boolean>() { @Override public void changed( ObservableValue<? extends Boolean> observable, Boolean oldValue, Boolean newValue) { if (newValue) { toolBarFrameStage.setAlwaysOnTop(true); toolBarFrameStage.setX(viewer3D.getPosition().getX()); toolBarFrameStage.setY(viewer3D.getPosition().getY()); } else if (!viewer3D.isFocused()) { toolBarFrameStage.setAlwaysOnTop(false); } } }); toolBarFrameController.initContent(voxelSpace); toolBarFrameStage.setAlwaysOnTop(true); toolBarFrameStage.show(); double maxToolBoxHeight = toolBarFrameStage.getHeight(); viewer3D.getJoglContext().setStartX((int) toolBarFrameStage.getWidth()); viewer3D.getRenderFrame().addWindowListener(new WindowAdapter() { @Override public void windowResized(com.jogamp.newt.event.WindowEvent we) { Window window = (Window) we.getSource(); final double height = window.getHeight(); Platform.runLater(new Runnable() { @Override public void run() { if (height < maxToolBoxHeight) { toolBarFrameStage.setHeight(height); } else { toolBarFrameStage.setHeight(maxToolBoxHeight); } toolBarFrameStage.setX(viewer3D.getPosition().getX()); toolBarFrameStage.setY(viewer3D.getPosition().getY()); } }); } @Override public void windowMoved(com.jogamp.newt.event.WindowEvent we) { Platform.runLater(new Runnable() { @Override public void run() { toolBarFrameStage.setX(viewer3D.getPosition().getX()); toolBarFrameStage.setY(viewer3D.getPosition().getY()); } }); } @Override public void windowDestroyed(com.jogamp.newt.event.WindowEvent we) { Platform.runLater(new Runnable() { @Override public void run() { toolBarFrameStage.close(); if (fcSaveVoxelFileForAreaExtracting == null) { fcSaveVoxelFileForAreaExtracting = new FileChooserContext(); if (fcOpenVoxelFileForAreaExtracting != null) { fcSaveVoxelFileForAreaExtracting.fc .setInitialDirectory( fcOpenVoxelFileForAreaExtracting.lastSelectedFile .getParentFile()); } } File selectedFile = fcSaveVoxelFileForAreaExtracting .showSaveDialog(stage); if (selectedFile != null) { VoxelFileReader reader; BufferedWriter writer = null; try { reader = new VoxelFileReader(voxelFile, true); VoxelSpaceInfos infos = reader.getVoxelSpaceInfos(); writer = new BufferedWriter( new FileWriter(selectedFile)); writer.write(infos.toString() + "\n"); Iterator<Voxel> iterator = reader.iterator(); while (iterator.hasNext()) { Voxel voxel = iterator.next(); if (voxelsToRemove.contains(new Point3i( voxel.$i, voxel.$j, voxel.$k))) { voxel.PadBVTotal = 0; voxel.nbEchos = 0; voxel.transmittance = 1; voxel.bvIntercepted = 0; } writer.write(voxel + "\n"); } writer.close(); addFileToProductsList(selectedFile); } catch (Exception ex) { showErrorDialog(ex); } finally { try { if (writer != null) { writer.close(); } } catch (IOException ex) { showErrorDialog(ex); } } } editingFrameOpened = false; } }); } @Override public void windowGainedFocus(com.jogamp.newt.event.WindowEvent we) { viewer3D.setIsFocused(true); Platform.runLater(new Runnable() { @Override public void run() { if (!toolBarFrameStage.isShowing()) { toolBarFrameStage.toFront(); } toolBarFrameStage.setIconified(false); toolBarFrameStage.setAlwaysOnTop(true); toolBarFrameStage.setX(viewer3D.getPosition().getX()); toolBarFrameStage.setY(viewer3D.getPosition().getY()); } }); } @Override public void windowLostFocus(com.jogamp.newt.event.WindowEvent e) { viewer3D.setIsFocused(false); Platform.runLater(new Runnable() { @Override public void run() { if (!toolBarFrameStage.focusedProperty().get()) { toolBarFrameStage.setIconified(true); toolBarFrameStage.setAlwaysOnTop(false); } } }); } }); viewer3D.show(); toolBarFrameStage.setAlwaysOnTop(true); } catch (IOException e) { showErrorDialog(new Exception("Loading ToolBarFrame.fxml failed", e)); } catch (Exception e) { showErrorDialog(new Exception("Error occured during toolbar init", e)); } } }); return null; } }; } }; ProgressDialog d = new ProgressDialog(s); d.show(); s.start(); } catch (Exception ex) { showErrorDialog(new Exception("Cannot launch 3d view", ex)); } }
From source file:fr.amap.lidar.amapvox.gui.MainFrameController.java
private void fillWeightingData(float[][] weightingData) { String text = ""; for (int i = 0; i < weightingData.length; i++) { for (int j = 0; j < weightingData.length; j++) { if (j != 0) { text += " "; }//ww w .j a v a 2 s .co m if (!Float.isNaN(weightingData[i][j])) { text += weightingData[i][j]; } if (j == weightingData.length - 1 && i != weightingData.length - 1) { text += "\n"; } } } textAreaWeighting.setText(text); }