Java tutorial
/* * Copyright (C) 2013 Vladimir Lahoda, Robert Simonovsky * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ package cz.cas.lib.proarc.desa; import java.io.File; import java.io.FileFilter; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.security.MessageDigest; import java.util.GregorianCalendar; import java.util.List; import java.util.logging.FileHandler; import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.Logger; import java.util.logging.SimpleFormatter; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.ws.Holder; import cz.cas.lib.proarc.desa.pspsip.ObjectFactory; import cz.cas.lib.proarc.desa.pspsip.PSPSIP; import cz.cas.lib.proarc.desa.pspsip.ResultType; import cz.cas.lib.proarc.desa.pspsip.SipType; import cz.cas.lib.proarc.desa.soap.FileHashAlg; import cz.cas.lib.proarc.desa.soap.SIPSubmission; import cz.cas.lib.proarc.desa.soap.SIPSubmissionFault; import org.apache.commons.io.FileUtils; /** * Main class for import of the SIP packages to DESA repository and for checking * the progress of the import. */ public final class SIP2DESATransporter { private static final Logger log = Logger.getLogger(SIP2DESATransporter.class.toString()); private static JAXBContext PSPSIP_JAXB; private final SIPSubmission desaPort; private final DesaClient desaClient; private final String desaFolderPath; private final String operatorName; private final String producerCode; private boolean checkMTDPSP = false; private final boolean useRest; private Marshaller marshaller = null; private Unmarshaller unmarshaller = null; private int filesIndex = -1; private int mtdpspIndex = -1; private String packageid = null; private final ObjectFactory resultsFactory = new ObjectFactory(); private PSPSIP results = resultsFactory.createPSPSIP(); /** * Returns the parsed result from the result file * * @return */ public PSPSIP getResults() { return results; } /** used just in case non REST usage! */ private File desaFolder; private String logRoot = ""; // /** // * The transporter entry point // * // * @param args // * command line arguments for transport mode : transport // * <input-folder> <results-folder> <log-folder> command line // * arguments for checkStatus mode : checkStatus <results-file> // * <log-folder> // */ // public static void main(String[] args) { // try { // if (args.length == 4 && "transport".equalsIgnoreCase(args[0])) { // String importRoot = args[1]; // String resultsRoot = args[2]; // String logRoot = args[3]; // new SIP2DESATransporter().transport(importRoot, resultsRoot, logRoot); // } else if (args.length == 3 && "checkStatus".equalsIgnoreCase(args[0])) { // String resultsRoot = args[1]; // String logRoot = args[2]; // new SIP2DESATransporter().checkStatus(resultsRoot, logRoot); // } else if (args.length == 2 && "adminUpload".equalsIgnoreCase(args[0])) { // String importRoot = args[1]; // new SIP2DESATransporter().adminUpload(importRoot); // } else { // System.out.println("SIP to DESA transporter.\n"); // System.out.println("Usage for transport: sip2desa.SIP2DESATransporter transport <input-folder> <results-folder> <log-folder>"); // System.out.println("Usage for checkStatus: sip2desa.SIP2DESATransporter checkStatus <results-file> <log-folder>"); // System.out.println("Usage for adminUpload: sip2desa.SIP2DESATransporter adminUpload <input-folder>"); // // System.exit(1); // } // } catch (Exception e) { // log.log(Level.SEVERE, e.getMessage(), e); // } // } /** * Submits SIPs through REST interface. */ SIP2DESATransporter(DesaClient desaClient, String operator, String producerCode) { this(desaClient, true, null, operator, producerCode); } /** * Ignores REST interface. */ SIP2DESATransporter(DesaClient desaClient, String desaFolderPath, String operator, String producerCode) { this(desaClient, false, desaFolderPath, operator, producerCode); } private SIP2DESATransporter(DesaClient desaClient, boolean useRest, String desaFolderPath, String operator, String producerCode) { this.desaPort = desaClient.getSoapClient(); this.desaClient = desaClient; this.useRest = useRest; this.desaFolderPath = desaFolderPath; this.operatorName = operator; this.producerCode = producerCode; } /** * Main execution method for the transport mode. Imports the SIP files in * the sourceRoot to DESA repository (configured in config property file). * The status of the import of each SIP is written in the results XML file * named TRANSF_packageid.xml in the resultsRoot folder. The JDK logging * output is mirrored in the packageid.log file in the logRoot folder. * * @param sourceRoot * @param resultsRoot * @param logRootIn */ public int[] transport(String sourceRoot, String resultsRoot, String logRootIn) { long timeStart = System.currentTimeMillis(); logRoot = logRootIn; Handler handler = null; File[] sourceFiles; File resultsFolder; try { try { initJAXB(); File sourceFolder = new File(sourceRoot); if (!sourceFolder.exists()) { handler = setupLogHandler(sourceFolder.getName()); log.log(Level.SEVERE, "Source folder doesn't exist: " + sourceFolder.getAbsolutePath()); throw new IllegalStateException( "Source folder doesn't exist: " + sourceFolder.getAbsolutePath()); } sourceFiles = sourceFolder.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return (pathname.getName().endsWith(".zip")); } }); if (sourceFiles == null || sourceFiles.length == 0) { handler = setupLogHandler(sourceFolder.getName()); log.log(Level.SEVERE, "Empty source folder: " + sourceFolder.getAbsolutePath()); throw new IllegalStateException("Empty source folder: " + sourceFolder.getAbsolutePath()); } preprocessFiles(sourceFolder, sourceFiles); handler = setupLogHandler(packageid); resultsFolder = checkDirectory(resultsRoot); log.info("Loaded source folder: " + sourceFolder); log.info("Results directory: " + resultsFolder); } catch (Exception e) { log.log(Level.SEVERE, "Error in transporter initialization.", e); throw new IllegalStateException("Error in transporter initialization.", e); } try { for (int i = 0; i < sourceFiles.length; i++) { if (i != filesIndex && i != mtdpspIndex) { uploadFile(sourceFiles[i], SipType.RECORD, true); } } if (mtdpspIndex > -1) { uploadFile(sourceFiles[mtdpspIndex], SipType.RECORD, true); } if (filesIndex >= 0) { uploadFile(sourceFiles[filesIndex], SipType.FILE, true); } } catch (Throwable th) { log.log(Level.SEVERE, "Error in file upload: ", th); throw new IllegalStateException("Error in file upload: ", th); } finally { results.setPspResultCode("progress"); try { marshaller.marshal(results, new File(resultsFolder, "TRANSF_" + packageid + ".xml")); } catch (JAXBException e) { log.log(Level.SEVERE, "Error writing results file: ", e); throw new IllegalStateException(e); } } long timeFinish = System.currentTimeMillis(); log.info("Elapsed time: " + ((timeFinish - timeStart) / 1000.0) + " seconds. " + sourceFiles.length + " SIP packages transported."); log.info("RESULT: OK"); } finally { if (handler != null) { removeLogHandler(handler); } } return countSip(); } private Handler setupLogHandler(String fileName) { Handler handler; try { checkDirectory(logRoot); handler = new FileHandler(logRoot + System.getProperty("file.separator") + fileName + ".txt", 0, 1, true); handler.setFormatter(new SimpleFormatter()); } catch (IOException e) { throw new RuntimeException(e); } Logger.getLogger("").addHandler(handler); return handler; } private static void removeLogHandler(Handler handler) { Logger.getLogger("").removeHandler(handler); handler.close(); } public void adminUpload(String sourceRoot) { long timeStart = System.currentTimeMillis(); File[] sourceFiles; try { initJAXB(); File sourceFolder = new File(sourceRoot); if (!sourceFolder.exists()) { log.log(Level.SEVERE, "Source folder doesn't exist: " + sourceFolder.getAbsolutePath()); throw new IllegalStateException("Source folder doesn't exist: " + sourceFolder.getAbsolutePath()); } sourceFiles = sourceFolder.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return (pathname.getName().endsWith(".zip")); } }); if (sourceFiles == null || sourceFiles.length == 0) { log.log(Level.SEVERE, "Empty source folder: " + sourceFolder.getAbsolutePath()); throw new IllegalStateException("Empty source folder: " + sourceFolder.getAbsolutePath()); } log.info("Loaded source folder: " + sourceFolder); } catch (Exception e) { log.log(Level.SEVERE, "Error in transporter initialization.", e); throw new IllegalStateException("Error in transporter initialization.", e); } try { for (int i = 0; i < sourceFiles.length; i++) { uploadFile(sourceFiles[i], null, false); } } catch (Throwable th) { log.log(Level.SEVERE, "Error in file upload: ", th); throw new IllegalStateException("Error in file upload: ", th); } long timeFinish = System.currentTimeMillis(); log.info("Elapsed time: " + ((timeFinish - timeStart) / 1000.0) + " seconds. " + sourceFiles.length + " SIP packages transported."); log.info("RESULT: OK"); } /** * Gets the cached thread safe JAXB context. */ private static JAXBContext getPspipJaxb() throws JAXBException { if (PSPSIP_JAXB == null) { PSPSIP_JAXB = JAXBContext.newInstance(PSPSIP.class); } return PSPSIP_JAXB; } /** * Initialize JAXB transformers * * @throws JAXBException */ private void initJAXB() throws JAXBException { if (marshaller != null) { return; } JAXBContext jaxbContext = getPspipJaxb(); marshaller = jaxbContext.createMarshaller(); marshaller.setProperty(Marshaller.JAXB_ENCODING, "utf-8"); marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); unmarshaller = jaxbContext.createUnmarshaller(); } /** * Upload one SIP file from the input folder. First call the DESA API method * asyncSubmitPackageStart, then copy the zip file to the mapped DESA target * folder and finally call the DESA API method asyncSubmitPackageEnd. Add * the corresponding entry into the results file JAXB representation. * * @param file * @param sipType * @param writeResults */ private void uploadFile(File file, SipType sipType, boolean writeResults) { Holder<String> sipId = new Holder<String>(getSipId(file)); Holder<String> idSipVersion = new Holder<String>(); String checksum = getMD5Checksum(file); log.info("Transporting file: " + file.getName()); if (useRest) { idSipVersion.value = desaClient.submitPackage(file, operatorName, producerCode, sipId.value, FileHashAlg.MD_5, checksum, "cs"); log.info("Received idSipVersion:" + idSipVersion.value); if (idSipVersion.value == null || "".equals(idSipVersion.value)) { throw new RuntimeException("DESA REST call did not return idSipVersion for file " + file.getName()); } } else { try { desaPort.asyncSubmitPackageStart(0, producerCode, operatorName, sipId, (int) file.length(), checksum, FileHashAlg.MD_5, idSipVersion); } catch (SIPSubmissionFault sipSubmissionFault) { throw new RuntimeException(sipSubmissionFault); } if (idSipVersion.value == null || "".equals(idSipVersion.value)) { throw new RuntimeException("DESA SOAP call did not return idSipVersion for file " + file.getName()); } File target = new File(getDesaFolder(), idSipVersion.value + ".sip"); try { FileUtils.copyFile(file, target); } catch (IOException e) { throw new RuntimeException(e); } log.info("Received idSipVersion:" + idSipVersion.value); try { desaPort.asyncSubmitPackageEnd(0, producerCode, operatorName, idSipVersion.value); } catch (SIPSubmissionFault sipSubmissionFault) { throw new RuntimeException(sipSubmissionFault); } } if (writeResults) { XMLGregorianCalendar currentDate = desaClient.getXmlTypes() .newXMLGregorianCalendar(new GregorianCalendar()); PSPSIP.SIP entry = resultsFactory.createPSPSIPSIP(); entry.setIdentifier(sipId.value); entry.setIdSIPVersion(idSipVersion.value); entry.setResultTime(currentDate); entry.setResultCode(ResultType.PROGRESS); entry.setType(sipType); results.getSIP().add(entry); } } private String getSipId(File sipFile) { return sipFile.getName().replace(".zip", ""); } /** * Check the contents of the input directory and find packageid and FILES * and MTDPSP SIP packages (which will be imported last) */ private void preprocessFiles(File sourceFolder, File[] sourceFiles) { for (int i = 0; i < sourceFiles.length; i++) { if (sourceFiles[i].getName().endsWith("_FILE.zip")) { packageid = sourceFiles[i].getName().replace("_FILE.zip", ""); filesIndex = i; } else if (sourceFiles[i].getName().endsWith("_MTDPSP.zip")) { mtdpspIndex = i; } } if (packageid == null) { for (int i = 0; i < sourceFiles.length; i++) { if (sourceFiles[i].getName().endsWith(".zip")) { if (sourceFiles[i].getName().indexOf("_") > 0) { packageid = sourceFiles[i].getName().substring(0, sourceFiles[i].getName().indexOf("_")); } else { packageid = sourceFiles[i].getName().replace(".zip", ""); } break; } } } if (packageid == null) { Handler handler = setupLogHandler(sourceFolder.getName()); log.log(Level.SEVERE, "Invalid source folder contents. Missing FILE.zip."); removeLogHandler(handler); throw new IllegalStateException("Invalid source folder contents. Missing FILE.zip."); } if (checkMTDPSP && mtdpspIndex == -1) { Handler handler = setupLogHandler(sourceFolder.getName()); log.log(Level.SEVERE, "Invalid source folder contents. Missing MTDPSP.zip."); removeLogHandler(handler); throw new IllegalStateException("Invalid source folder contents. Missing MTDPSP.zip."); } results.setPackageID(packageid); } /** * Main execution method for the checkStatus mode. The status of the import * of each SIP in the results XML file named TRANSF_packageid.xml in the * resultsRoot folder is checked by call to the DESA API getPackageStatus * method. The results file is then updated accordingly. The JDK logging * output is mirrored in the packageid.log file in the logRoot folder. * * @param resultsFileStr * @param logRootIn */ public int[] checkStatus(String resultsFileStr, String logRootIn) { long timeStart = System.currentTimeMillis(); logRoot = logRootIn; Handler handler = null; try { File resultsFile = new File(resultsFileStr); if (!resultsFile.exists()) { handler = setupLogHandler(resultsFile.getName()); log.log(Level.SEVERE, "Results file does not exist."); throw new IllegalStateException("Results file does not exist."); } try { initJAXB(); log.info("Loaded results file: " + resultsFile); parseResultsFile(resultsFile); } catch (Exception e) { handler = setupLogHandler(resultsFile.getName()); log.log(Level.SEVERE, "Error in transporter initialization.", e); throw new IllegalStateException("Error in transporter initialization.", e); } handler = setupLogHandler(packageid); int objectCounter = 0; try { boolean finishedAll = true; for (PSPSIP.SIP sip : results.getSIP()) { if (!checkSIP(sip)) { finishedAll = false; } ; objectCounter++; } if (finishedAll) { results.setPspResultCode("finished"); } else { results.setPspResultCode("progress"); } marshaller.marshal(results, resultsFile); } catch (Exception e) { log.log(Level.SEVERE, "Error checking results: ", e); throw new IllegalStateException("Error checking results: ", e); } long timeFinish = System.currentTimeMillis(); log.info("Elapsed time: " + ((timeFinish - timeStart) / 1000.0) + " seconds. " + objectCounter + " SIP packages checked."); log.info("RESULT: OK"); } finally { if (handler != null) { removeLogHandler(handler); } } return countSip(); } private int[] countSip() { List<PSPSIP.SIP> sipList = results.getSIP(); int sipCount = sipList.size(); int sipFinishedCount = 0; for (PSPSIP.SIP sip : sipList) { if (ResultType.FINISHED.equals(sip.getResultCode())) { sipFinishedCount++; } } return new int[] { sipCount, sipFinishedCount }; } /** * Convert the existing results file to JAXB representation * * @param resultsFile */ private void parseResultsFile(File resultsFile) { try { results = (PSPSIP) unmarshaller.unmarshal(resultsFile); packageid = results.getPackageID(); } catch (Exception e) { Handler handler = setupLogHandler(resultsFile.getName()); log.log(Level.SEVERE, "Error in parsing results file.", e); removeLogHandler(handler); throw new IllegalStateException("Error in parsing results file.", e); } } /** * Check the status of the SIP of one entry in the results file and update * the entry in the JAXB object. * * @param sip * @return */ private boolean checkSIP(PSPSIP.SIP sip) { Holder<String> sipId = new Holder<String>(sip.getIdentifier()); Holder<String> idSipVersion = new Holder<String>(sip.getIdSIPVersion()); Holder<String> packageStateCode = new Holder<String>(); Holder<String> packageStateText = new Holder<String>(); Holder<String> errorCode = new Holder<String>(); log.info("Checking file: " + sipId.value); /* * if(config.getBoolean("desa.rest")){ try { URLConnection connection = * new * URL(config.getString("desa.restapi")+"/packagestatus"+"?userName=" * +config.getString("desa.user") * +"&producerCode="+config.getString("desa.producer" * )+"&producerSipId="+ * sipId.value+"&idSIPVersion="+idSipVersion.value).openConnection(); * packageStateCode * .value=connection.getHeaderField("X-DEA-packageStateCode"); * packageStateText * .value=connection.getHeaderField("X-DEA-packageStateText"); * errorCode.value=connection.getHeaderField("X-DEA-errorCode"); } catch * (Exception e) { throw new RuntimeException(e); } }else{ */ try { desaPort.getPackageStatus(0, producerCode, operatorName, idSipVersion, sipId, packageStateCode, packageStateText, errorCode); } catch (SIPSubmissionFault sipSubmissionFault) { throw new RuntimeException(sipSubmissionFault); } catch (Exception e1) { log.log(Level.SEVERE, "DESA exception", e1); throw new IllegalStateException("DESA exception", e1); } /* } */ log.info("Status: " + packageStateCode.value + " (" + packageStateText.value + ")" + (errorCode.value != null ? (", errorCode:" + errorCode.value) : "")); XMLGregorianCalendar currentDate = desaClient.getXmlTypes() .newXMLGregorianCalendar(new GregorianCalendar()); ; boolean retval = false; sip.setResultTime(currentDate); if ("AI_ACC_OK".equalsIgnoreCase(packageStateCode.value)) { sip.setResultCode(ResultType.FINISHED); retval = true; } else if ("AI_ERROR".equalsIgnoreCase(packageStateCode.value) || "AI_INVALID".equalsIgnoreCase(packageStateCode.value) || "AI_REJECT".equalsIgnoreCase(packageStateCode.value) || "AI_INFECTED".equalsIgnoreCase(packageStateCode.value) || "AI_QA_ERR".equalsIgnoreCase(packageStateCode.value)) { sip.setResultCode(ResultType.ERROR); } else { sip.setResultCode(ResultType.PROGRESS); } return retval; } /** Gets target folder for SOAP submit package. */ private File getDesaFolder() { if (!useRest) { if (desaFolder == null) { desaFolder = checkDirectory(desaFolderPath); } } return desaFolder; } /** * Check if the directory with the given path exists and create it if * necessary * * @param name * The path of the requested directory * @return The File representation of the requested directory */ private static File checkDirectory(String name) { File directory = new File(name); try { FileUtils.forceMkdir(directory); return directory; } catch (IOException e) { throw new RuntimeException(e); } } /** * Calculate the MD5 checksum of the given file * * @param file * @return */ private static String getMD5Checksum(File file) { try { InputStream fis = new FileInputStream(file); byte[] buffer = new byte[1024]; MessageDigest complete = MessageDigest.getInstance("MD5"); int numRead; do { numRead = fis.read(buffer); if (numRead > 0) { complete.update(buffer, 0, numRead); } } while (numRead != -1); fis.close(); byte[] bytes = complete.digest(); StringBuilder sb = new StringBuilder(2 * bytes.length); for (byte b : bytes) { sb.append(hexDigits[(b >> 4) & 0xf]).append(hexDigits[b & 0xf]); } return sb.toString(); } catch (Exception e) { throw new RuntimeException(e); } } private static final char[] hexDigits = "0123456789abcdef".toCharArray(); }