Java tutorial
/** * The contents of this file are subject to the license and copyright * detailed in the LICENSE file at the root of the source * tree and available online at * * https://github.com/keeps/roda */ package org.roda.core.plugins.plugins.base; import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.roda.core.RodaCoreFactory; import org.roda.core.data.common.RodaConstants; import org.roda.core.data.common.RodaConstants.PreservationEventType; import org.roda.core.data.exceptions.InvalidParameterException; import org.roda.core.data.v2.LiteOptionalWithCause; import org.roda.core.data.v2.ip.AIP; import org.roda.core.data.v2.jobs.Job; import org.roda.core.data.v2.jobs.PluginParameter; import org.roda.core.data.v2.jobs.PluginParameter.PluginParameterType; import org.roda.core.data.v2.jobs.PluginType; import org.roda.core.data.v2.jobs.Report; import org.roda.core.index.IndexService; import org.roda.core.model.ModelService; import org.roda.core.plugins.AbstractPlugin; import org.roda.core.plugins.Plugin; import org.roda.core.plugins.PluginException; import org.roda.core.plugins.RODAObjectProcessingLogic; import org.roda.core.plugins.RODAProcessingLogic; import org.roda.core.plugins.orchestrate.SimpleJobPluginInfo; import org.roda.core.plugins.plugins.PluginHelper; import org.roda.core.storage.StorageService; import org.roda.core.storage.fs.FSUtils; import org.roda.core.util.IdUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class InventoryReportPlugin extends AbstractPlugin<AIP> { private static final Logger LOGGER = LoggerFactory.getLogger(InventoryReportPlugin.class); public static final String EXPORT_CSV_TEMP_FOLDER = "CSV"; public static final String CSV_FILE_FIELDS = "parameter.csv.file.fields"; public static final String CSV_FILE_OUTPUT = "parameter.csv.file.output"; public static final String CSV_FILE_HEADERS = "parameter.csv.file.headers"; public static final String CSV_FILE_OUTPUT_DATA = "parameter.csv.file.output.data"; public static final String CSV_FILE_OUTPUT_DESCRIPTIVE = "parameter.csv.file.output.descriptive"; public static final String CSV_FILE_OTHER_METADATA_TYPES = "parameter.csv.file.output.other"; public static final String CSV_FIELD_SIP_ID = "sipId"; public static final String CSV_FIELD_AIP_ID = "aipId"; public static final String CSV_FIELD_REPRESENTATION_ID = "representationId"; public static final String CSV_FIELD_FILE_PATH = "filePath"; public static final String CSV_FIELD_FILE_ID = "fileId"; public static final String CSV_FIELD_ISDIRECTORY = "isDirectory"; public static final String CSV_FIELD_CHECKSUM_SHA1 = "SHA-1"; public static final String CSV_FIELD_CHECKSUM_SHA256 = "SHA-256"; public static final String CSV_FIELD_CHECKSUM_MD5 = "MD5"; public static final String CSV_FILE_TYPE = "type"; public enum CSV_LINE_TYPE { DATA, METADATA_DESCRIPTIVE, METADATA_OTHER } protected static final List<String> CHECKSUM_ALGORITHMS = Arrays.asList(CSV_FIELD_CHECKSUM_MD5, CSV_FIELD_CHECKSUM_SHA1, CSV_FIELD_CHECKSUM_SHA256); public static final String CSV_DEFAULT_FIELDS = StringUtils .join(Arrays.asList(CSV_FIELD_SIP_ID, CSV_FIELD_AIP_ID, CSV_FIELD_REPRESENTATION_ID, CSV_FIELD_FILE_PATH, CSV_FIELD_FILE_ID, CSV_FIELD_ISDIRECTORY, CSV_FILE_TYPE, CSV_FIELD_CHECKSUM_SHA256, CSV_FIELD_CHECKSUM_MD5, CSV_FIELD_CHECKSUM_SHA1), ","); public static final String CSV_DEFAULT_OUTPUT = "/tmp/output.csv"; public static final String CSV_DEFAULT_HEADERS = "true"; public static final String CSV_DEFAULT_OTHER_METADATA = "tika,siegfried"; private List<String> fields = null; private Path output; private boolean enableHeaders; private boolean outputDataInformation; private boolean outputDescriptiveMetadataInformation; private List<String> otherMetadataTypes; private static Map<String, PluginParameter> pluginParameters = new HashMap<>(); // TODO -> add plugin parameter type "LIST"... static { pluginParameters.put(CSV_FILE_FIELDS, new PluginParameter(CSV_FILE_FIELDS, "Attributes to include in the report", PluginParameterType.STRING, CSV_DEFAULT_FIELDS, true, false, "List of file attributes to include in the inventory export. The example includes all the possible options. Remove attributes as necessary.")); pluginParameters.put(CSV_FILE_OUTPUT, new PluginParameter(CSV_FILE_OUTPUT, "Report file path", PluginParameterType.STRING, CSV_DEFAULT_OUTPUT, true, false, "The full path and file name on the server where the inventory report file should be created.")); pluginParameters.put(CSV_FILE_HEADERS, new PluginParameter(CSV_FILE_HEADERS, "Include header line", PluginParameterType.BOOLEAN, CSV_DEFAULT_HEADERS, true, false, "Include a header line in the CSV inventory report.")); pluginParameters.put(CSV_FILE_OUTPUT_DATA, new PluginParameter(CSV_FILE_OUTPUT_DATA, "Include data files", PluginParameterType.BOOLEAN, CSV_DEFAULT_HEADERS, true, false, "Include in the inventory report information about data files that exist inside AIPs.")); pluginParameters.put(CSV_FILE_OUTPUT_DESCRIPTIVE, new PluginParameter(CSV_FILE_OUTPUT_DESCRIPTIVE, "Include descriptive metadata files", PluginParameterType.BOOLEAN, CSV_DEFAULT_HEADERS, true, false, "Include in the inventory report information about descriptive metadata files that exist inside AIPs.")); pluginParameters.put(CSV_FILE_OTHER_METADATA_TYPES, new PluginParameter(CSV_FILE_OTHER_METADATA_TYPES, "Include other metadata files", PluginParameterType.STRING, CSV_DEFAULT_OTHER_METADATA, true, false, "Include in the inventory report information about other metadata files that exist inside AIPs.")); } @Override public void init() throws PluginException { // do nothing } @Override public void shutdown() { // do nothing } @Override public String getName() { return "Inventory report"; } @Override public String getDescription() { return "Creates a report in CSV format that includes a listing of all AIP and its inner files (data and metadata) which also includes some of " + "their technical properties (e.g. sipId, aipId, representationId, filePath, SHA-256, MD5, SHA-1). The report will be stored in a folder on " + "the server side as defined by the user. To obtain the report, one needs access to the storage layer of the repository server.\nThis report" + " may be used to validate the completeness and correctness of an ingest process."; } @Override public String getVersionImpl() { return "1.0"; } @Override public List<PluginParameter> getParameters() { ArrayList<PluginParameter> parameters = new ArrayList<>(); parameters.add(pluginParameters.get(CSV_FILE_FIELDS)); PluginParameter outputPluginParameter = pluginParameters.get(CSV_FILE_OUTPUT); SimpleDateFormat df = new SimpleDateFormat(RodaConstants.DEFAULT_DATETIME_FORMAT); String reportName = "inventory_report_" + df.format(new Date()) + ".csv"; outputPluginParameter.setDefaultValue(RodaCoreFactory.getReportsDirectory().resolve(reportName).toString()); parameters.add(outputPluginParameter); parameters.add(pluginParameters.get(CSV_FILE_HEADERS)); parameters.add(pluginParameters.get(CSV_FILE_OUTPUT_DATA)); parameters.add(pluginParameters.get(CSV_FILE_OUTPUT_DESCRIPTIVE)); parameters.add(pluginParameters.get(CSV_FILE_OTHER_METADATA_TYPES)); return parameters; } @Override public void setParameterValues(Map<String, String> parameters) throws InvalidParameterException { super.setParameterValues(parameters); if (parameters.containsKey(CSV_FILE_FIELDS)) { String fieldsSTR = parameters.get(CSV_FILE_FIELDS); if (fieldsSTR != null && !"".equals(fieldsSTR.trim())) { fields = new ArrayList<>(); fields.addAll(Arrays.asList(fieldsSTR.split(","))); } } if (parameters.containsKey(CSV_FILE_OUTPUT)) { try { output = Paths.get(parameters.get(CSV_FILE_OUTPUT)); Path parent = output.getParent(); Files.createDirectories(parent); } catch (IOException e) { LOGGER.error("Error creating output parent folder.", e); } } if (parameters.containsKey(CSV_FILE_HEADERS)) { enableHeaders = Boolean.parseBoolean(parameters.get(CSV_FILE_HEADERS)); } if (parameters.containsKey(CSV_FILE_OUTPUT_DATA)) { outputDataInformation = Boolean.parseBoolean(parameters.get(CSV_FILE_OUTPUT_DATA)); } if (parameters.containsKey(CSV_FILE_OUTPUT_DESCRIPTIVE)) { outputDescriptiveMetadataInformation = Boolean .parseBoolean(parameters.get(CSV_FILE_OUTPUT_DESCRIPTIVE)); } if (parameters.containsKey(CSV_FILE_OTHER_METADATA_TYPES)) { String otherMetadataSTR = parameters.get(CSV_FILE_OTHER_METADATA_TYPES); if (otherMetadataSTR != null && !"".trim().equalsIgnoreCase(otherMetadataSTR)) { otherMetadataTypes = new ArrayList<>(); otherMetadataTypes.addAll(Arrays.asList(otherMetadataSTR.split(","))); } } } @Override public Report execute(IndexService index, ModelService model, StorageService storage, List<LiteOptionalWithCause> liteList) throws PluginException { final CSVPrinter csvFilePrinter = createCSVPrinter(); return PluginHelper.processObjects(this, new RODAObjectProcessingLogic<AIP>() { @Override public void process(IndexService index, ModelService model, StorageService storage, Report report, Job cachedJob, SimpleJobPluginInfo jobPluginInfo, Plugin<AIP> plugin, AIP object) { processAIP(model, storage, jobPluginInfo, csvFilePrinter, object); } }, new RODAProcessingLogic<AIP>() { @Override public void process(IndexService index, ModelService model, StorageService storage, Report report, Job cachedJob, SimpleJobPluginInfo jobPluginInfo, Plugin<AIP> plugin) { IOUtils.closeQuietly(csvFilePrinter); } }, index, model, storage, liteList); } private CSVPrinter createCSVPrinter() { Path jobCSVTempFolder = getJobCSVTempFolder(); Path csvTempFile = jobCSVTempFolder.resolve(IdUtils.createUUID() + ".csv"); CSVFormat csvFileFormat = CSVFormat.DEFAULT.withRecordSeparator("\n"); try (BufferedWriter fileWriter = Files.newBufferedWriter(csvTempFile)) { return new CSVPrinter(fileWriter, csvFileFormat); } catch (IOException e) { LOGGER.error("Unable to instantiate CSVPrinter", e); return null; } } private void processAIP(ModelService model, StorageService storage, SimpleJobPluginInfo jobPluginInfo, CSVPrinter csvFilePrinter, AIP aip) { if (csvFilePrinter == null) { LOGGER.warn("CSVPrinter is NULL! Skipping..."); return; } try { if (outputDataInformation && aip.getRepresentations() != null) { List<List<String>> dataInformation = InventoryReportPluginUtils.getDataInformation(fields, aip, model, storage); csvFilePrinter.printRecords(dataInformation); } if (outputDescriptiveMetadataInformation && aip.getDescriptiveMetadata() != null) { List<List<String>> dataInformation = InventoryReportPluginUtils .getDescriptiveMetadataInformation(fields, aip, model, storage); csvFilePrinter.printRecords(dataInformation); } if (otherMetadataTypes != null && !otherMetadataTypes.isEmpty()) { for (String otherMetadataType : otherMetadataTypes) { List<List<String>> otherMetadataInformation = InventoryReportPluginUtils .getOtherMetadataInformation(fields, otherMetadataType, aip, model, storage); csvFilePrinter.printRecords(otherMetadataInformation); } } jobPluginInfo.incrementObjectsProcessedWithSuccess(); } catch (IOException e) { jobPluginInfo.incrementObjectsProcessedWithFailure(); } } @Override public Report beforeAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { try { Path jobCSVTempFolder = getJobCSVTempFolder(); Files.createDirectories(jobCSVTempFolder); } catch (IOException e) { LOGGER.error("Error while creating plugin working dir", e); } try { Path reportsFolder = RodaCoreFactory.getRodaHomePath().resolve(RodaConstants.CORE_REPORT_FOLDER); if (FSUtils.exists(reportsFolder)) { Files.createDirectories(reportsFolder); } } catch (IOException e) { LOGGER.error("Error while creating report dir", e); } return new Report(); } private Path getJobCSVTempFolder() { Path wd = RodaCoreFactory.getWorkingDirectory(); Path csvExportTempFolder = wd.resolve(InventoryReportPlugin.EXPORT_CSV_TEMP_FOLDER); return csvExportTempFolder.resolve(PluginHelper.getJobId(this)); } @Override public Report afterAllExecute(IndexService index, ModelService model, StorageService storage) throws PluginException { CSVFormat csvFileFormat = CSVFormat.DEFAULT.withRecordSeparator("\n"); Path csvTempFolder = getJobCSVTempFolder(); if (csvTempFolder != null) { List<Path> partials = new ArrayList<>(); try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(csvTempFolder); FileWriter fileWriter = new FileWriter(output.toFile()); CSVPrinter csvFilePrinter = new CSVPrinter(fileWriter, csvFileFormat);) { if (enableHeaders) { csvFilePrinter.printRecord(fields); } for (Path path : directoryStream) { partials.add(path); } } catch (IOException e) { LOGGER.error("Error while merging partial CSVs", e); } try { InventoryReportPluginUtils.mergeFiles(partials, output); FSUtils.deletePathQuietly(csvTempFolder); } catch (IOException e) { LOGGER.error("Error while merging partial CSVs", e); } } return new Report(); } @Override public Plugin<AIP> cloneMe() { return new InventoryReportPlugin(); } @Override public PluginType getType() { return PluginType.MISC; } @Override public boolean areParameterValuesValid() { return true; } @Override public PreservationEventType getPreservationEventType() { return PreservationEventType.VALIDATION; } @Override public String getPreservationEventDescription() { return "Created a report in CSV format"; } @Override public String getPreservationEventSuccessMessage() { return "Created a report in CSV format successfully"; } @Override public String getPreservationEventFailureMessage() { return "Create of a report in CSV format failed"; } @Override public List<String> getCategories() { return Arrays.asList(RodaConstants.PLUGIN_CATEGORY_MANAGEMENT); } @Override public List<Class<AIP>> getObjectClasses() { return Arrays.asList(AIP.class); } }