List of usage examples for org.apache.commons.math3.util MultidimensionalCounter getCounts
public int[] getCounts(int index) throws OutOfRangeException
From source file:uk.ac.diamond.scisoft.ncd.reduction.LazyNcdProcessing.java
/** * //from w ww .jav a 2s . com * @param monitor * @throws HDF5Exception */ public void execute(final IProgressMonitor monitor) throws HDF5Exception { int sliceDim = 0; int sliceSize = (int) frames[0]; // We will slice only 2D data. 1D data is loaded into memory completely if (dim == 2) { estimateFrameBatchSize(dim, frames); // Find dimension that needs to be sliced MultidimensionalCounter dimCounter = new MultidimensionalCounter( Arrays.copyOfRange(frames_int, 0, rank - dim)); if (dimCounter.getSize() > frameBatch) { int[] sliceIdx = dimCounter.getCounts(frameBatch); for (int i = 0; i < sliceIdx.length; i++) { if (sliceIdx[i] != 0) { sliceDim = i; break; } } sliceSize = sliceIdx[sliceDim]; } } SliceSettings sliceParams = new SliceSettings(frames, sliceDim, sliceSize); int[] iter_array = Arrays.copyOfRange(frames_int, 0, sliceDim + 1); int[] start = new int[iter_array.length]; int[] step = new int[iter_array.length]; Arrays.fill(start, 0); Arrays.fill(step, 1); step[sliceDim] = sliceSize; SliceND slice = new SliceND(iter_array, null, iter_array, step); IndexIterator iter = new SliceIterator(iter_array, AbstractDataset.calcSize(iter_array), slice); if (flags.isEnableSector() && dim == 2) { ArrayList<Job> sectorJobList = new ArrayList<Job>(); ArrayList<Job> runningJobList = new ArrayList<Job>(); while (iter.hasNext()) { DataReductionJob sectorJob = new DataReductionJob("Sector Integration") { @Override protected IStatus run(IProgressMonitor jobmonitor) { try { Dataset data; try { lock.acquire(); data = NcdNexusUtils.sliceInputData(currentSliceParams, tmp_ids); if (tmp_errors_ids != null) { if (tmp_errors_ids.dataset_id >= 0) { Dataset errors = NcdNexusUtils.sliceInputData(currentSliceParams, tmp_errors_ids); data.setError(errors); } else { tmp_errors_ids.setSlice(currentSliceParams); } } } catch (Exception e) { throw e; } finally { lock.release(); } if (flags.isEnableDetectorResponse()) { jobmonitor.setTaskName(monitorFile + " : Correct for detector response"); data = lazyDetectorResponse.execute(dim, data, currentSliceParams, lock); } jobmonitor.setTaskName(monitorFile + " : Performing sector integration"); data = lazySectorIntegration.execute(dim, data, currentSliceParams, lock)[1]; } catch (Exception e) { e.printStackTrace(); return Status.CANCEL_STATUS; } return Status.OK_STATUS; } }; sectorJob.tmp_ids = new DataSliceIdentifiers(input_ids); sectorJob.tmp_errors_ids = new DataSliceIdentifiers(input_errors_ids); sliceParams.setStart(iter.getPos()); sectorJob.currentSliceParams = new SliceSettings(sliceParams); sectorJobList.add(sectorJob); } monitor.beginTask(monitorFile + " : Running Sector Integration Stage", sectorJobList.size()); for (Job job : sectorJobList) { if (monitor.isCanceled()) { sectorJobList.clear(); for (Job runningJob : runningJobList) { runningJob.cancel(); } break; } while (runningJobList.size() >= cores) { try { runningJobList.get(0).join(); runningJobList.remove(0); monitor.worked(1); } catch (InterruptedException e) { e.printStackTrace(); } } job.schedule(); runningJobList.add(job); } for (Job job : sectorJobList) { try { job.join(); } catch (InterruptedException e) { e.printStackTrace(); } } if (monitor.isCanceled()) { return; } monitor.done(); dim = 1; rank = secRank; sliceDim = 0; sliceSize = (int) secFrames[0]; frames = secFrames; frames_int = (int[]) ConvertUtils.convert(secFrames, int[].class); sliceParams = new SliceSettings(frames, sliceDim, sliceSize); IntegerDataset idx_dataset = new IntegerDataset(new int[] { sliceSize }, new int[] { 1 }); iter = idx_dataset.getSliceIterator(new int[] { 0 }, new int[] { 1 }, new int[] { 1 }); input_ids.setIDs(lazySectorIntegration.sec_group_id, lazySectorIntegration.sec_data_id); input_errors_ids.setIDs(lazySectorIntegration.sec_group_id, lazySectorIntegration.sec_errors_id); } ArrayList<DataReductionJob> processingJobList = new ArrayList<DataReductionJob>(); ArrayList<DataReductionJob> runningJobList = new ArrayList<DataReductionJob>(); while (iter.hasNext()) { DataReductionJob processingJob = new DataReductionJob("Data Reduction") { @Override protected IStatus run(IProgressMonitor jobmonitor) { try { Dataset data; int finalSliceDim = currentSliceParams.getSliceDim(); int finalSliceSize = currentSliceParams.getSliceSize(); try { lock.acquire(); data = NcdNexusUtils.sliceInputData(currentSliceParams, tmp_ids); if (tmp_errors_ids != null) { if (tmp_errors_ids.dataset_id >= 0) { Dataset errors = NcdNexusUtils.sliceInputData(currentSliceParams, tmp_errors_ids); data.setError(errors); } else { tmp_errors_ids.setSlice(currentSliceParams); } } } catch (Exception e) { throw e; } finally { lock.release(); } if (flags.isEnableDetectorResponse() && !flags.isEnableSector()) { jobmonitor.setTaskName(monitorFile + " : Correct for detector response"); data = lazyDetectorResponse.execute(dim, data, currentSliceParams, lock); } if (flags.isEnableNormalisation()) { jobmonitor.setTaskName(monitorFile + " : Normalising data"); data = lazyNormalisation.execute(dim, data, currentSliceParams, lock); } if (flags.isEnableBackground()) { jobmonitor.setTaskName(monitorFile + " : Subtracting background"); long[] bgFrames = lazyBackgroundSubtraction.bgFrames; int[] bgFrames_int = lazyBackgroundSubtraction.bgFrames_int; int bgSliceSize = Math.min(finalSliceSize, bgFrames_int[finalSliceDim]); int[] bgStart = new int[finalSliceDim + 1]; for (int i = 0; i <= finalSliceDim; i++) { bgStart[i] = Math.min(currentSliceParams.getStart()[i], bgFrames_int[i] - 1); } SliceSettings bgSliceParams = new SliceSettings(bgFrames, finalSliceDim, bgSliceSize); bgSliceParams.setStart(bgStart); Dataset bgData = NcdNexusUtils.sliceInputData(bgSliceParams, tmp_bgIds); if (tmp_errors_bgIds != null) { if (tmp_errors_bgIds.dataset_id >= 0) { Dataset bgErrors = NcdNexusUtils.sliceInputData(bgSliceParams, tmp_errors_bgIds); bgData.setError(bgErrors); } else { tmp_errors_bgIds.setSlice(bgSliceParams); } } Dataset[] remapData = NcdDataUtils.matchDataDimensions(data, bgData); Dataset[] remapErrors = NcdDataUtils.matchDataDimensions(data.getError(), bgData.getError()); remapData[0].setError(remapErrors[0]); remapData[1].setError(remapErrors[1]); Dataset res = lazyBackgroundSubtraction.execute(dim, remapData[0], remapData[1], currentSliceParams, lock); remapData[0] = res; remapErrors[0] = res.getError(); // restore original axis order in output dataset data = DatasetUtils.transpose(remapData[0], (int[]) remapData[2].getBuffer()); data.setError( DatasetUtils.transpose(remapErrors[0], (int[]) remapErrors[2].getBuffer())); } if (flags.isEnableInvariant()) { jobmonitor.setTaskName(monitorFile + " : Calculating invariant"); SliceSettings invSliceParam = new SliceSettings(lazyInvariant.invFrames, finalSliceDim, finalSliceSize); invSliceParam.setStart(Arrays.copyOf(currentSliceParams.getStart(), finalSliceDim + 1)); lazyInvariant.execute(dim, data, invSliceParam, lock); } } catch (Exception e) { e.printStackTrace(); return Status.CANCEL_STATUS; } return Status.OK_STATUS; } }; processingJob.tmp_ids = new DataSliceIdentifiers(input_ids); processingJob.tmp_errors_ids = new DataSliceIdentifiers(input_errors_ids); sliceParams.setStart(iter.getPos()); processingJob.currentSliceParams = new SliceSettings(sliceParams); if (flags.isEnableBackground()) { processingJob.tmp_bgIds = new DataSliceIdentifiers(lazyBackgroundSubtraction.bgIds); processingJob.tmp_errors_bgIds = new DataSliceIdentifiers(lazyBackgroundSubtraction.bgErrorsIds); } processingJobList.add(processingJob); } monitor.beginTask(monitorFile + " : Running NCD Data Reduction stages", processingJobList.size()); for (DataReductionJob job : processingJobList) { if (monitor.isCanceled()) { processingJobList.clear(); for (Job runningJob : runningJobList) { runningJob.cancel(); } break; } while (runningJobList.size() >= cores) { try { runningJobList.get(0).join(); runningJobList.remove(0); monitor.worked(1); } catch (InterruptedException e) { e.printStackTrace(); } } job.schedule(); runningJobList.add(job); } for (DataReductionJob job : processingJobList) { try { job.join(); } catch (InterruptedException e) { e.printStackTrace(); } } if (monitor.isCanceled()) { return; } monitor.done(); if (flags.isEnableBackground()) { input_ids.setIDs(lazyBackgroundSubtraction.bg_group_id, lazyBackgroundSubtraction.bg_data_id); input_errors_ids.setIDs(lazyBackgroundSubtraction.bg_group_id, lazyBackgroundSubtraction.bg_errors_id); } else if (flags.isEnableNormalisation()) { input_ids.setIDs(lazyNormalisation.norm_group_id, lazyNormalisation.norm_data_id); input_errors_ids.setIDs(lazyNormalisation.norm_group_id, lazyNormalisation.norm_errors_id); } else if (flags.isEnableDetectorResponse() && !flags.isEnableSector()) { input_ids.setIDs(lazyDetectorResponse.dr_group_id, lazyDetectorResponse.dr_data_id); input_errors_ids.setIDs(lazyDetectorResponse.dr_group_id, lazyDetectorResponse.dr_errors_id); } if (flags.isEnableAverage()) { monitor.beginTask(monitorFile + " : Averaging datasets", IProgressMonitor.UNKNOWN); int[] averageIndices = new int[] { frames.length - dim }; if (gridAverage != null) { averageIndices = NcdDataUtils.createGridAxesList(gridAverage, frames.length - dim + 1); } lazyAverage = new LazyAverage(); lazyAverage.setAverageIndices(averageIndices); lazyAverage.setMonitor(monitor); lazyAverage.configure(dim, frames_int, processing_group_id, frameBatch); lazyAverage.execute(input_ids, input_errors_ids); if (monitor.isCanceled()) { return; } if (qaxis != null) { lazyAverage.setQaxis(qaxis, qaxisUnit); lazyAverage.writeQaxisData(frames_int.length, input_ids.datagroup_id); } lazyAverage.writeNcdMetadata(input_ids.datagroup_id); monitor.done(); } H5.H5Lcopy(input_ids.datagroup_id, "./data", result_group_id, "./data", HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); if (input_errors_ids.dataset_id != -1) { H5.H5Lcopy(input_errors_ids.datagroup_id, "./errors", result_group_id, "./errors", HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); } if (qaxis != null) { H5.H5Lcopy(input_ids.datagroup_id, "./q", result_group_id, "./q", HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); if (input_errors_ids.dataset_id != -1 && qaxis.hasErrors()) { H5.H5Lcopy(input_ids.datagroup_id, "./q_errors", result_group_id, "./q_errors", HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); } } if (flags.isEnableLogLogPlot()) { SaxsPlotData plotData = new LogLogPlotData(); addPlotData(plotData, detector, qaxis); } if (flags.isEnableGuinierPlot()) { SaxsPlotData plotData = new GuinierPlotData(); addPlotData(plotData, detector, qaxis); } if (flags.isEnablePorodPlot()) { SaxsPlotData plotData = new PorodPlotData(); addPlotData(plotData, detector, qaxis); } if (flags.isEnableKratkyPlot()) { SaxsPlotData plotData = new KratkyPlotData(); addPlotData(plotData, detector, qaxis); } if (flags.isEnableZimmPlot()) { SaxsPlotData plotData = new ZimmPlotData(); addPlotData(plotData, detector, qaxis); } if (flags.isEnableDebyeBuechePlot()) { SaxsPlotData plotData = new DebyeBuechePlotData(); addPlotData(plotData, detector, qaxis); } }