List of usage examples for org.apache.commons.math3.util MultidimensionalCounter MultidimensionalCounter
public MultidimensionalCounter(int... size) throws NotStrictlyPositiveException
From source file:uk.ac.diamond.scisoft.ncd.passerelle.actors.core.NcdProcessingObjectTransformer.java
@Override protected void process(ActorContext ctxt, ProcessRequest request, ProcessResponse response) throws ProcessingException { ManagedMessage receivedMsg = request.getMessage(input); try {//from www. j a va 2 s . c om NcdProcessingObject receivedObject = (NcdProcessingObject) receivedMsg.getBodyContent(); inputGroupID = receivedObject.getInputGroupID(); inputDataID = receivedObject.getInputDataID(); inputErrorsID = receivedObject.getInputErrorsID(); lock = receivedObject.getLock(); lock.lock(); configureActorParameters(); int[] resultShape = getResultDataShape(); MultidimensionalCounter frameCounter = new MultidimensionalCounter(selectedShape); Iterator iter = frameCounter.iterator(); Dataset data = DatasetFactory.zeros(resultShape, Dataset.FLOAT32); data.setName(datasetName); Dataset errors = null; if (hasErrors) { errors = DatasetFactory.zeros(resultShape, Dataset.FLOAT64); } while (iter.hasNext()) { iter.next(); int[] frame = iter.getCounts(); int[] gridFrame = new int[resultShape.length]; Arrays.fill(gridFrame, 0); for (int i = 0; i < selectedShape.length; i++) { gridFrame[i] = indexList.get(i)[frame[i]]; } SliceSettings sliceData = new SliceSettings(frames, frames.length - dimension - 1, 1); sliceData.setStart(gridFrame); DataSliceIdentifiers ids = new DataSliceIdentifiers(); ids.setIDs(inputGroupID, inputDataID); Dataset value = NcdNexusUtils.sliceInputData(sliceData, ids); int[] start = Arrays.copyOf(frame, resultShape.length); int[] stop = Arrays.copyOf(resultShape, resultShape.length); for (int i = 0; i < selectedShape.length; i++) { stop[i] = start[i] + 1; } data.setSlice(value, start, stop, null); if (hasErrors && errors != null) { DataSliceIdentifiers errors_ids = new DataSliceIdentifiers(); errors_ids.setIDs(inputGroupID, inputErrorsID); Dataset error = NcdNexusUtils.sliceInputData(sliceData, errors_ids); errors.setSlice(error, start, stop, null); } } if (hasErrors && errors != null) { data.setErrorBuffer(errors); } ManagedMessage outputMsg = createMessageFromCauses(receivedMsg); DataMessageComponent obj = new DataMessageComponent(); obj.addList(datasetName, data); outputMsg.setBodyContent(obj, "application/octet-stream"); response.addOutputMessage(output, outputMsg); } catch (MessageException e) { throw new ProcessingException(ErrorCode.ACTOR_EXECUTION_ERROR, e.getMessage(), this, e); } catch (HDF5Exception e) { throw new ProcessingException(ErrorCode.ACTOR_EXECUTION_ERROR, e.getMessage(), this, e); } finally { if (lock != null && lock.isHeldByCurrentThread()) { lock.unlock(); } } }
From source file:uk.ac.diamond.scisoft.ncd.rcp.utils.NcdNexusUtilsTest.java
@Test public void testSliceString() throws HDF5Exception { int[] datDimMake = Arrays.copyOfRange(frames, 0, frames.length - dim); ArrayList<int[]> list = NcdDataUtils.createSliceList(format, datDimMake); for (int i = 0; i < datDimMake.length; i++) datDimMake[i] = list.get(i).length; DataSliceIdentifiers dr_id = readDataId(inputPath, detector, "data", null)[0]; long[] frames_long = (long[]) ConvertUtils.convert(frames, long[].class); SliceSettings drSlice = new SliceSettings(frames_long, 0, 1); int[] start = new int[] { 0, 0, 0, 0 }; drSlice.setStart(start);/*from w w w. j av a 2s . c o m*/ Dataset data = NcdNexusUtils.sliceInputData(drSlice, dr_id); Dataset result = sliceInputData(dim, frames, format, dr_id); MultidimensionalCounter resultCounter = new MultidimensionalCounter(result.getShape()); Iterator resIter = resultCounter.iterator(); while (resIter.hasNext()) { resIter.next(); int[] gridFrame = Arrays.copyOf(resIter.getCounts(), frames.length); for (int i = 0; i < datDimMake.length; i++) gridFrame[i] = list.get(i)[gridFrame[i]]; float valResult = result.getFloat(resIter.getCounts()); float valData = data.getFloat(gridFrame); double acc = Math.max(1e-6 * Math.abs(Math.sqrt(valResult * valResult + valData * valData)), 1e-10); assertEquals(String.format("Data slicing test for pixel %s has failed.", Arrays.toString(resIter.getCounts())), valData, valResult, acc); } }
From source file:uk.ac.diamond.scisoft.ncd.rcp.utils.NcdNexusUtilsTest.java
public static Dataset sliceInputData(int dim, int[] frames, String format, DataSliceIdentifiers ids) throws HDF5Exception { int[] datDimMake = Arrays.copyOfRange(frames, 0, frames.length - dim); int[] imageSize = Arrays.copyOfRange(frames, frames.length - dim, frames.length); ArrayList<int[]> list = NcdDataUtils.createSliceList(format, datDimMake); for (int i = 0; i < datDimMake.length; i++) datDimMake[i] = list.get(i).length; int[] framesTotal = ArrayUtils.addAll(datDimMake, imageSize); long[] block = new long[frames.length]; block = Arrays.copyOf((long[]) ConvertUtils.convert(frames, long[].class), block.length); Arrays.fill(block, 0, block.length - dim, 1); int[] block_int = (int[]) ConvertUtils.convert(block, int[].class); long[] count = new long[frames.length]; Arrays.fill(count, 1);//from w ww . j a va2 s . c o m int dtype = HDF5Loader.getDtype(ids.dataclass_id, ids.datasize_id); Dataset data = DatasetFactory.zeros(block_int, dtype); Dataset result = null; MultidimensionalCounter bgFrameCounter = new MultidimensionalCounter(datDimMake); Iterator iter = bgFrameCounter.iterator(); while (iter.hasNext()) { iter.next(); long[] bgFrame = (long[]) ConvertUtils.convert(iter.getCounts(), long[].class); long[] gridFrame = new long[datDimMake.length]; for (int i = 0; i < datDimMake.length; i++) gridFrame[i] = list.get(i)[(int) bgFrame[i]]; long[] start = new long[frames.length]; start = Arrays.copyOf(gridFrame, frames.length); int memspace_id = H5.H5Screate_simple(block.length, block, null); H5.H5Sselect_hyperslab(ids.dataspace_id, HDF5Constants.H5S_SELECT_SET, start, block, count, block); H5.H5Dread(ids.dataset_id, ids.datatype_id, memspace_id, ids.dataspace_id, HDF5Constants.H5P_DEFAULT, data.getBuffer()); if (result == null) { result = data.clone(); } else { result = DatasetUtils.append(result, data, block.length - dim - 1); } } if (result != null) { result.setShape(framesTotal); } return result; }
From source file:uk.ac.diamond.scisoft.ncd.reduction.LazyNcdProcessing.java
/** * //www . j a v a 2 s . c om * @param monitor * @throws HDF5Exception */ public void execute(final IProgressMonitor monitor) throws HDF5Exception { int sliceDim = 0; int sliceSize = (int) frames[0]; // We will slice only 2D data. 1D data is loaded into memory completely if (dim == 2) { estimateFrameBatchSize(dim, frames); // Find dimension that needs to be sliced MultidimensionalCounter dimCounter = new MultidimensionalCounter( Arrays.copyOfRange(frames_int, 0, rank - dim)); if (dimCounter.getSize() > frameBatch) { int[] sliceIdx = dimCounter.getCounts(frameBatch); for (int i = 0; i < sliceIdx.length; i++) { if (sliceIdx[i] != 0) { sliceDim = i; break; } } sliceSize = sliceIdx[sliceDim]; } } SliceSettings sliceParams = new SliceSettings(frames, sliceDim, sliceSize); int[] iter_array = Arrays.copyOfRange(frames_int, 0, sliceDim + 1); int[] start = new int[iter_array.length]; int[] step = new int[iter_array.length]; Arrays.fill(start, 0); Arrays.fill(step, 1); step[sliceDim] = sliceSize; SliceND slice = new SliceND(iter_array, null, iter_array, step); IndexIterator iter = new SliceIterator(iter_array, AbstractDataset.calcSize(iter_array), slice); if (flags.isEnableSector() && dim == 2) { ArrayList<Job> sectorJobList = new ArrayList<Job>(); ArrayList<Job> runningJobList = new ArrayList<Job>(); while (iter.hasNext()) { DataReductionJob sectorJob = new DataReductionJob("Sector Integration") { @Override protected IStatus run(IProgressMonitor jobmonitor) { try { Dataset data; try { lock.acquire(); data = NcdNexusUtils.sliceInputData(currentSliceParams, tmp_ids); if (tmp_errors_ids != null) { if (tmp_errors_ids.dataset_id >= 0) { Dataset errors = NcdNexusUtils.sliceInputData(currentSliceParams, tmp_errors_ids); data.setError(errors); } else { tmp_errors_ids.setSlice(currentSliceParams); } } } catch (Exception e) { throw e; } finally { lock.release(); } if (flags.isEnableDetectorResponse()) { jobmonitor.setTaskName(monitorFile + " : Correct for detector response"); data = lazyDetectorResponse.execute(dim, data, currentSliceParams, lock); } jobmonitor.setTaskName(monitorFile + " : Performing sector integration"); data = lazySectorIntegration.execute(dim, data, currentSliceParams, lock)[1]; } catch (Exception e) { e.printStackTrace(); return Status.CANCEL_STATUS; } return Status.OK_STATUS; } }; sectorJob.tmp_ids = new DataSliceIdentifiers(input_ids); sectorJob.tmp_errors_ids = new DataSliceIdentifiers(input_errors_ids); sliceParams.setStart(iter.getPos()); sectorJob.currentSliceParams = new SliceSettings(sliceParams); sectorJobList.add(sectorJob); } monitor.beginTask(monitorFile + " : Running Sector Integration Stage", sectorJobList.size()); for (Job job : sectorJobList) { if (monitor.isCanceled()) { sectorJobList.clear(); for (Job runningJob : runningJobList) { runningJob.cancel(); } break; } while (runningJobList.size() >= cores) { try { runningJobList.get(0).join(); runningJobList.remove(0); monitor.worked(1); } catch (InterruptedException e) { e.printStackTrace(); } } job.schedule(); runningJobList.add(job); } for (Job job : sectorJobList) { try { job.join(); } catch (InterruptedException e) { e.printStackTrace(); } } if (monitor.isCanceled()) { return; } monitor.done(); dim = 1; rank = secRank; sliceDim = 0; sliceSize = (int) secFrames[0]; frames = secFrames; frames_int = (int[]) ConvertUtils.convert(secFrames, int[].class); sliceParams = new SliceSettings(frames, sliceDim, sliceSize); IntegerDataset idx_dataset = new IntegerDataset(new int[] { sliceSize }, new int[] { 1 }); iter = idx_dataset.getSliceIterator(new int[] { 0 }, new int[] { 1 }, new int[] { 1 }); input_ids.setIDs(lazySectorIntegration.sec_group_id, lazySectorIntegration.sec_data_id); input_errors_ids.setIDs(lazySectorIntegration.sec_group_id, lazySectorIntegration.sec_errors_id); } ArrayList<DataReductionJob> processingJobList = new ArrayList<DataReductionJob>(); ArrayList<DataReductionJob> runningJobList = new ArrayList<DataReductionJob>(); while (iter.hasNext()) { DataReductionJob processingJob = new DataReductionJob("Data Reduction") { @Override protected IStatus run(IProgressMonitor jobmonitor) { try { Dataset data; int finalSliceDim = currentSliceParams.getSliceDim(); int finalSliceSize = currentSliceParams.getSliceSize(); try { lock.acquire(); data = NcdNexusUtils.sliceInputData(currentSliceParams, tmp_ids); if (tmp_errors_ids != null) { if (tmp_errors_ids.dataset_id >= 0) { Dataset errors = NcdNexusUtils.sliceInputData(currentSliceParams, tmp_errors_ids); data.setError(errors); } else { tmp_errors_ids.setSlice(currentSliceParams); } } } catch (Exception e) { throw e; } finally { lock.release(); } if (flags.isEnableDetectorResponse() && !flags.isEnableSector()) { jobmonitor.setTaskName(monitorFile + " : Correct for detector response"); data = lazyDetectorResponse.execute(dim, data, currentSliceParams, lock); } if (flags.isEnableNormalisation()) { jobmonitor.setTaskName(monitorFile + " : Normalising data"); data = lazyNormalisation.execute(dim, data, currentSliceParams, lock); } if (flags.isEnableBackground()) { jobmonitor.setTaskName(monitorFile + " : Subtracting background"); long[] bgFrames = lazyBackgroundSubtraction.bgFrames; int[] bgFrames_int = lazyBackgroundSubtraction.bgFrames_int; int bgSliceSize = Math.min(finalSliceSize, bgFrames_int[finalSliceDim]); int[] bgStart = new int[finalSliceDim + 1]; for (int i = 0; i <= finalSliceDim; i++) { bgStart[i] = Math.min(currentSliceParams.getStart()[i], bgFrames_int[i] - 1); } SliceSettings bgSliceParams = new SliceSettings(bgFrames, finalSliceDim, bgSliceSize); bgSliceParams.setStart(bgStart); Dataset bgData = NcdNexusUtils.sliceInputData(bgSliceParams, tmp_bgIds); if (tmp_errors_bgIds != null) { if (tmp_errors_bgIds.dataset_id >= 0) { Dataset bgErrors = NcdNexusUtils.sliceInputData(bgSliceParams, tmp_errors_bgIds); bgData.setError(bgErrors); } else { tmp_errors_bgIds.setSlice(bgSliceParams); } } Dataset[] remapData = NcdDataUtils.matchDataDimensions(data, bgData); Dataset[] remapErrors = NcdDataUtils.matchDataDimensions(data.getError(), bgData.getError()); remapData[0].setError(remapErrors[0]); remapData[1].setError(remapErrors[1]); Dataset res = lazyBackgroundSubtraction.execute(dim, remapData[0], remapData[1], currentSliceParams, lock); remapData[0] = res; remapErrors[0] = res.getError(); // restore original axis order in output dataset data = DatasetUtils.transpose(remapData[0], (int[]) remapData[2].getBuffer()); data.setError( DatasetUtils.transpose(remapErrors[0], (int[]) remapErrors[2].getBuffer())); } if (flags.isEnableInvariant()) { jobmonitor.setTaskName(monitorFile + " : Calculating invariant"); SliceSettings invSliceParam = new SliceSettings(lazyInvariant.invFrames, finalSliceDim, finalSliceSize); invSliceParam.setStart(Arrays.copyOf(currentSliceParams.getStart(), finalSliceDim + 1)); lazyInvariant.execute(dim, data, invSliceParam, lock); } } catch (Exception e) { e.printStackTrace(); return Status.CANCEL_STATUS; } return Status.OK_STATUS; } }; processingJob.tmp_ids = new DataSliceIdentifiers(input_ids); processingJob.tmp_errors_ids = new DataSliceIdentifiers(input_errors_ids); sliceParams.setStart(iter.getPos()); processingJob.currentSliceParams = new SliceSettings(sliceParams); if (flags.isEnableBackground()) { processingJob.tmp_bgIds = new DataSliceIdentifiers(lazyBackgroundSubtraction.bgIds); processingJob.tmp_errors_bgIds = new DataSliceIdentifiers(lazyBackgroundSubtraction.bgErrorsIds); } processingJobList.add(processingJob); } monitor.beginTask(monitorFile + " : Running NCD Data Reduction stages", processingJobList.size()); for (DataReductionJob job : processingJobList) { if (monitor.isCanceled()) { processingJobList.clear(); for (Job runningJob : runningJobList) { runningJob.cancel(); } break; } while (runningJobList.size() >= cores) { try { runningJobList.get(0).join(); runningJobList.remove(0); monitor.worked(1); } catch (InterruptedException e) { e.printStackTrace(); } } job.schedule(); runningJobList.add(job); } for (DataReductionJob job : processingJobList) { try { job.join(); } catch (InterruptedException e) { e.printStackTrace(); } } if (monitor.isCanceled()) { return; } monitor.done(); if (flags.isEnableBackground()) { input_ids.setIDs(lazyBackgroundSubtraction.bg_group_id, lazyBackgroundSubtraction.bg_data_id); input_errors_ids.setIDs(lazyBackgroundSubtraction.bg_group_id, lazyBackgroundSubtraction.bg_errors_id); } else if (flags.isEnableNormalisation()) { input_ids.setIDs(lazyNormalisation.norm_group_id, lazyNormalisation.norm_data_id); input_errors_ids.setIDs(lazyNormalisation.norm_group_id, lazyNormalisation.norm_errors_id); } else if (flags.isEnableDetectorResponse() && !flags.isEnableSector()) { input_ids.setIDs(lazyDetectorResponse.dr_group_id, lazyDetectorResponse.dr_data_id); input_errors_ids.setIDs(lazyDetectorResponse.dr_group_id, lazyDetectorResponse.dr_errors_id); } if (flags.isEnableAverage()) { monitor.beginTask(monitorFile + " : Averaging datasets", IProgressMonitor.UNKNOWN); int[] averageIndices = new int[] { frames.length - dim }; if (gridAverage != null) { averageIndices = NcdDataUtils.createGridAxesList(gridAverage, frames.length - dim + 1); } lazyAverage = new LazyAverage(); lazyAverage.setAverageIndices(averageIndices); lazyAverage.setMonitor(monitor); lazyAverage.configure(dim, frames_int, processing_group_id, frameBatch); lazyAverage.execute(input_ids, input_errors_ids); if (monitor.isCanceled()) { return; } if (qaxis != null) { lazyAverage.setQaxis(qaxis, qaxisUnit); lazyAverage.writeQaxisData(frames_int.length, input_ids.datagroup_id); } lazyAverage.writeNcdMetadata(input_ids.datagroup_id); monitor.done(); } H5.H5Lcopy(input_ids.datagroup_id, "./data", result_group_id, "./data", HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); if (input_errors_ids.dataset_id != -1) { H5.H5Lcopy(input_errors_ids.datagroup_id, "./errors", result_group_id, "./errors", HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); } if (qaxis != null) { H5.H5Lcopy(input_ids.datagroup_id, "./q", result_group_id, "./q", HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); if (input_errors_ids.dataset_id != -1 && qaxis.hasErrors()) { H5.H5Lcopy(input_ids.datagroup_id, "./q_errors", result_group_id, "./q_errors", HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); } } if (flags.isEnableLogLogPlot()) { SaxsPlotData plotData = new LogLogPlotData(); addPlotData(plotData, detector, qaxis); } if (flags.isEnableGuinierPlot()) { SaxsPlotData plotData = new GuinierPlotData(); addPlotData(plotData, detector, qaxis); } if (flags.isEnablePorodPlot()) { SaxsPlotData plotData = new PorodPlotData(); addPlotData(plotData, detector, qaxis); } if (flags.isEnableKratkyPlot()) { SaxsPlotData plotData = new KratkyPlotData(); addPlotData(plotData, detector, qaxis); } if (flags.isEnableZimmPlot()) { SaxsPlotData plotData = new ZimmPlotData(); addPlotData(plotData, detector, qaxis); } if (flags.isEnableDebyeBuechePlot()) { SaxsPlotData plotData = new DebyeBuechePlotData(); addPlotData(plotData, detector, qaxis); } }
From source file:uk.ac.diamond.scisoft.ncd.reduction.LazySelection.java
public DataSliceIdentifiers[] execute(int dim, DataSliceIdentifiers ids, DataSliceIdentifiers error_ids, int output_group_id) throws HDF5Exception { int[] datDimMake = Arrays.copyOfRange(frames, 0, frames.length - dim); int[] imageSize = Arrays.copyOfRange(frames, frames.length - dim, frames.length); ArrayList<int[]> list = NcdDataUtils.createSliceList(format, datDimMake); for (int i = 0; i < datDimMake.length; i++) { datDimMake[i] = list.get(i).length; }/*from w ww . j av a2 s . co m*/ long[] framesTotal = (long[]) ConvertUtils.convert(ArrayUtils.addAll(datDimMake, imageSize), long[].class); long[] block = new long[frames.length]; block = Arrays.copyOf((long[]) ConvertUtils.convert(frames, long[].class), block.length); Arrays.fill(block, 0, block.length - dim, 1); int[] block_int = (int[]) ConvertUtils.convert(block, int[].class); long[] count = new long[frames.length]; Arrays.fill(count, 1); int dtype = HDF5Utils.getDtype(ids.dataclass_id, ids.datasize_id); Dataset data = DatasetFactory.zeros(block_int, dtype); int output_data_id = NcdNexusUtils.makedata(output_group_id, "data", ids.datatype_id, framesTotal, true, "counts"); int output_dataspace_id = H5.H5Dget_space(output_data_id); Dataset errors = DatasetFactory.zeros(block_int, dtype); int errors_datatype_id = H5.H5Tcopy(HDF5Constants.H5T_NATIVE_DOUBLE); int errors_data_id = NcdNexusUtils.makedata(output_group_id, "errors", errors_datatype_id, framesTotal, true, "counts"); int errors_dataspace_id = H5.H5Dget_space(errors_data_id); MultidimensionalCounter frameCounter = new MultidimensionalCounter(datDimMake); Iterator iter = frameCounter.iterator(); while (iter.hasNext()) { iter.next(); long[] frame = (long[]) ConvertUtils.convert(iter.getCounts(), long[].class); long[] gridFrame = new long[datDimMake.length]; for (int i = 0; i < datDimMake.length; i++) { gridFrame[i] = list.get(i)[(int) frame[i]]; } long[] start = new long[frames.length]; start = Arrays.copyOf(gridFrame, frames.length); long[] writePosition = new long[frames.length]; writePosition = Arrays.copyOf(frame, frames.length); int memspace_id = H5.H5Screate_simple(block.length, block, null); H5.H5Sselect_hyperslab(ids.dataspace_id, HDF5Constants.H5S_SELECT_SET, start, block, count, block); H5.H5Dread(ids.dataset_id, ids.datatype_id, memspace_id, ids.dataspace_id, HDF5Constants.H5P_DEFAULT, data.getBuffer()); int errors_memspace_id = H5.H5Screate_simple(block.length, block, null); if (error_ids.dataset_id >= 0) { H5.H5Sselect_hyperslab(error_ids.dataspace_id, HDF5Constants.H5S_SELECT_SET, start, block, count, block); H5.H5Dread(error_ids.dataset_id, error_ids.datatype_id, errors_memspace_id, error_ids.dataspace_id, HDF5Constants.H5P_DEFAULT, errors.getBuffer()); data.setError(errors); } else { Object obj = DatasetUtils.createJavaArray(data); Dataset error = DatasetFactory.createFromObject(obj); error.ipower(0.5); data.setError(error); } H5.H5Sselect_hyperslab(output_dataspace_id, HDF5Constants.H5S_SELECT_SET, writePosition, block, count, block); H5.H5Dwrite(output_data_id, ids.datatype_id, memspace_id, output_dataspace_id, HDF5Constants.H5P_DEFAULT, data.getBuffer()); H5.H5Sselect_hyperslab(errors_dataspace_id, HDF5Constants.H5S_SELECT_SET, writePosition, block, count, block); H5.H5Dwrite(errors_data_id, errors_datatype_id, errors_memspace_id, errors_dataspace_id, HDF5Constants.H5P_DEFAULT, data.getError().getBuffer()); if (monitor.isCanceled()) { return null; } monitor.worked(1); } DataSliceIdentifiers outputDataIds = new DataSliceIdentifiers(); outputDataIds.setIDs(output_group_id, output_data_id); DataSliceIdentifiers outputErrorsIds = new DataSliceIdentifiers(); outputErrorsIds.setIDs(output_group_id, errors_data_id); return new DataSliceIdentifiers[] { outputDataIds, outputErrorsIds }; }