Java tutorial
/* * Copyright (c) 2012. The Genome Analysis Centre, Norwich, UK * MISO project contacts: Robert Davey, Mario Caccamo @ TGAC * ********************************************************************* * * This file is part of MISO. * * MISO is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * MISO is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with MISO. If not, see <http://www.gnu.org/licenses/>. * * ********************************************************************* */ package uk.ac.bbsrc.tgac.miso.core.manager; import net.sf.json.JSONArray; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpPost; import org.apache.http.conn.ClientConnectionManager; import org.apache.http.conn.scheme.Scheme; import org.apache.http.conn.scheme.SchemeRegistry; import org.apache.http.conn.ssl.SSLSocketFactory; import org.apache.http.entity.mime.MultipartEntity; import org.apache.http.entity.mime.content.FileBody; import org.apache.http.impl.client.DefaultHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.w3c.dom.Document; import org.w3c.dom.NodeList; import uk.ac.bbsrc.tgac.miso.core.data.*; import uk.ac.bbsrc.tgac.miso.core.service.submission.*; import uk.ac.bbsrc.tgac.miso.core.util.LimsUtils; import uk.ac.bbsrc.tgac.miso.core.util.SubmissionUtils; import uk.ac.bbsrc.tgac.miso.core.exception.SubmissionException; import uk.ac.bbsrc.tgac.miso.core.factory.submission.ERASubmissionFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import java.io.*; import java.net.URI; import java.net.URL; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; /** * Manager class that holds state for a submission connection to the EBI SRA submission service, and facilitates the submission process * * @author Rob Davey * @since 0.0.2 */ public class ERASubmissionManager implements SubmissionManager<Set<Submittable<Document>>, URL, Document> { @Autowired private Properties submissionProperties; public void setSubmissionProperties(Properties submissionProperties) { this.submissionProperties = submissionProperties; } @Autowired private RequestManager requestManager; public void setRequestManager(RequestManager requestManager) { this.requestManager = requestManager; } @Autowired private FilesManager misoFileManager; public void setMisoFileManager(FilesManager misoFileManager) { this.misoFileManager = misoFileManager; } /** Field log */ protected static final Logger log = LoggerFactory.getLogger(ERASubmissionManager.class); private String centreName; private String accountName; private String dropBox; private String authKey; //private String proxyHost; //private String proxyUser; //private String proxyPass; private URL submissionEndPoint; private String submissionStoragePath; private Map<Long, UploadReport> uploadReports = new HashMap<Long, UploadReport>(); private DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); /** * Sets the centreName of this ERASubmissionManager object. * * @param centreName centreName. */ public void setCentreName(String centreName) { this.centreName = centreName; } /** * Sets the accountName of this ERASubmissionManager object. * * @param accountName accountName. */ public void setAccountName(String accountName) { this.accountName = accountName; } /** * Sets the dropBox of this ERASubmissionManager object. * * @param dropBox dropBox. */ public void setDropBox(String dropBox) { this.dropBox = dropBox; } /** * Sets the authKey of this ERASubmissionManager object. * * @param authKey authKey. */ public void setAuthKey(String authKey) { this.authKey = authKey; } /** * Sets the proxyHost of this ERASubmissionManager object if a proxy needs to be traversed * * @param proxyHost proxyHost. */ // public void setProxyHost(String proxyHost) { // this.proxyHost = proxyHost; // } /** * Sets the proxyUser of this ERASubmissionManager object if a proxy needs to be traversed * * @param proxyUser proxyUser. */ // public void setProxyUser(String proxyUser) { // this.proxyUser = proxyUser; // } /** * Sets the proxyPass of this ERASubmissionManager object if a proxy needs to be traversed * * @param proxyPass proxyPass. */ // public void setProxyPass(String proxyPass) { // this.proxyPass = proxyPass; // } /** * Sets the submissionEndPoint of this ERASubmissionManager object. * * @param submissionEndPoint submissionEndPoint. */ public void setSubmissionEndPoint(URL submissionEndPoint) { this.submissionEndPoint = submissionEndPoint; } /** * Returns the submissionEndPoint of this ERASubmissionManager object. * * @return URL submissionEndPoint. */ public URL getSubmissionEndPoint() { return this.submissionEndPoint; } /** * Sets the submissionStoragePath of this ERASubmissionManager object. * * @param submissionStoragePath submissionStoragePath. */ public void setSubmissionStoragePath(String submissionStoragePath) { this.submissionStoragePath = submissionStoragePath; } /** * Returns the submissionStoragePath of this ERASubmissionManager object. * * @return String submissionStoragePath. */ public String getSubmissionStoragePath() { return submissionStoragePath; } @Override public String generateSubmissionMetadata(Submission submission) throws SubmissionException { File subPath = new File(misoFileManager.getFileStorageDirectory() + "/submission/" + submission.getName()); //File subPath = null; StringBuilder sb = new StringBuilder(); try { //subPath = misoFileManager.storeFile(submission.getClass(),submission.getName()); DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); String d = df.format(new Date()); submissionProperties.put("submissionDate", d); if (LimsUtils.checkDirectory(subPath, true)) { Map<String, List<Submittable<Document>>> map = new HashMap<String, List<Submittable<Document>>>(); map.put("study", new ArrayList<Submittable<Document>>()); map.put("sample", new ArrayList<Submittable<Document>>()); map.put("experiment", new ArrayList<Submittable<Document>>()); map.put("run", new ArrayList<Submittable<Document>>()); Set<Submittable<Document>> subs = submission.getSubmissionElements(); for (Submittable<Document> sub : subs) { if (sub instanceof Study) { map.get("study").add(sub); } else if (sub instanceof Sample) { map.get("sample").add(sub); } else if (sub instanceof Experiment) { map.get("experiment").add(sub); } else if (sub instanceof SequencerPoolPartition) { map.get("run").add(sub); } } for (String key : map.keySet()) { List<Submittable<Document>> submittables = map.get(key); Document submissionDocument = docBuilder.newDocument(); ERASubmissionFactory.generateSubmissionXML(submissionDocument, submittables, key, submissionProperties); //generate xml files on disk File f = new File(subPath, File.separator + submission.getName() + "_" + key + "_" + d + ".xml"); if (f.exists()) { f.delete(); } SubmissionUtils.transform(submissionDocument, f); sb.append(SubmissionUtils.transform(submissionDocument, true)); } Document submissionDocument = docBuilder.newDocument(); ERASubmissionFactory.generateParentSubmissionXML(submissionDocument, submission, submissionProperties); File f = new File(subPath, File.separator + submission.getName() + "_submission_" + d + ".xml"); if (f.exists()) { f.delete(); } SubmissionUtils.transform(submissionDocument, f, true); sb.append(SubmissionUtils.transform(submissionDocument, true)); } } catch (ParserConfigurationException e) { throw new SubmissionException(e.getMessage()); } catch (TransformerException e) { throw new SubmissionException(e.getMessage()); } catch (IOException e) { throw new SubmissionException("Cannot write to submission storage directory: " + subPath + ". Please check this directory exists and is writable."); } finally { submissionProperties.remove("submissionDate"); } return sb.toString(); } /** * Submits the given set of Submittables to the ERA submission service endpoint * * @param submissionData of type Set<Submittable<Document>> * @return Document * @throws SubmissionException when an error occurred with the submission process */ public Document submit(Set<Submittable<Document>> submissionData) throws SubmissionException { try { DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); if (submissionEndPoint == null) { throw new SubmissionException( "No submission endpoint configured. Please check your submission.properties file."); } if (submissionData == null || submissionData.size() == 0) { throw new SubmissionException("No submission data set."); } if (accountName == null || dropBox == null || authKey == null) { throw new SubmissionException("An accountName, dropBox and authKey must be supplied!"); } if (centreName == null) { throw new SubmissionException( "No centreName configured. Please check your submission.properties file and specify your Center Name as given by the SRA."); } String curl = "curl -k "; StringBuilder sb = new StringBuilder(); sb.append(curl); String proxyHost = submissionProperties.getProperty("submission.proxyHost"); String proxyUser = submissionProperties.getProperty("submission.proxyUser"); String proxyPass = submissionProperties.getProperty("submission.proxyPass"); if (proxyHost != null && !proxyHost.equals("")) { sb.append("-x ").append(proxyHost); if (proxyUser != null && !proxyUser.equals("")) { sb.append("-U ").append(proxyUser); if (proxyPass != null && !proxyPass.equals("")) { sb.append(":").append(proxyPass); } } } //submit via REST to endpoint try { Map<String, List<Submittable<Document>>> map = new HashMap<String, List<Submittable<Document>>>(); map.put("study", new ArrayList<Submittable<Document>>()); map.put("sample", new ArrayList<Submittable<Document>>()); map.put("experiment", new ArrayList<Submittable<Document>>()); map.put("run", new ArrayList<Submittable<Document>>()); Document submissionXml = docBuilder.newDocument(); String subName = null; String d = df.format(new Date()); submissionProperties.put("submissionDate", d); for (Submittable<Document> s : submissionData) { if (s instanceof Submission) { //s.buildSubmission(); ERASubmissionFactory.generateParentSubmissionXML(submissionXml, (Submission) s, submissionProperties); subName = ((Submission) s).getName(); } else if (s instanceof Study) { map.get("study").add(s); } else if (s instanceof Sample) { map.get("sample").add(s); } else if (s instanceof Experiment) { map.get("experiment").add(s); } else if (s instanceof SequencerPoolPartition) { map.get("run").add(s); } } if (submissionXml != null && subName != null) { String url = getSubmissionEndPoint() + "?auth=ERA%20" + dropBox + "%20" + authKey; HttpClient httpclient = getEvilTrustingTrustManager(new DefaultHttpClient()); HttpPost httppost = new HttpPost(url); MultipartEntity reqEntity = new MultipartEntity(); String submissionXmlFileName = subName + File.separator + subName + "_submission_" + d + ".xml"; File subtmp = new File(submissionStoragePath + submissionXmlFileName); SubmissionUtils.transform(submissionXml, subtmp, true); reqEntity.addPart("SUBMISSION", new FileBody(subtmp)); for (String key : map.keySet()) { List<Submittable<Document>> submittables = map.get(key); String submittableXmlFileName = subName + File.separator + subName + "_" + key.toLowerCase() + "_" + d + ".xml"; File elementTmp = new File(submissionStoragePath + submittableXmlFileName); Document submissionDocument = docBuilder.newDocument(); ERASubmissionFactory.generateSubmissionXML(submissionDocument, submittables, key, submissionProperties); SubmissionUtils.transform(submissionDocument, elementTmp, true); reqEntity.addPart(key.toUpperCase(), new FileBody(elementTmp)); } httppost.setEntity(reqEntity); HttpResponse response = httpclient.execute(httppost); if (response.getStatusLine().getStatusCode() == 200) { HttpEntity resEntity = response.getEntity(); try { Document submissionReport = docBuilder.newDocument(); SubmissionUtils.transform(resEntity.getContent(), submissionReport); File savedReport = new File( submissionStoragePath + subName + File.separator + "report_" + d + ".xml"); SubmissionUtils.transform(submissionReport, savedReport); return submissionReport; } catch (IOException e) { e.printStackTrace(); } catch (TransformerException e) { e.printStackTrace(); } finally { submissionProperties.remove("submissionDate"); } } else { throw new SubmissionException("Response from submission endpoint (" + url + ") was not OK (200). Was: " + response.getStatusLine().getStatusCode()); } } else { throw new SubmissionException( "Could not find a Submission in the supplied set of Submittables"); } } catch (IOException e) { e.printStackTrace(); } catch (TransformerException e) { e.printStackTrace(); } finally { submissionProperties.remove("submissionDate"); } } catch (ParserConfigurationException e) { e.printStackTrace(); } return null; } public Map<String, Object> parseResponse(Document report) { Map<String, Object> responseMap = new HashMap<String, Object>(); NodeList errors = report.getElementsByTagName("ERROR"); NodeList infos = report.getElementsByTagName("INFO"); ArrayList<String> errorList = new ArrayList<String>(); if (errors.getLength() > 0) { for (int i = 0; i < errors.getLength(); i++) { errorList.add(errors.item(i).getTextContent()); } responseMap.put("errors", JSONArray.fromObject(errorList)); } ArrayList<String> infoList = new ArrayList<String>(); if (infos.getLength() > 0) { for (int i = 0; i < infos.getLength(); i++) { infoList.add(infos.item(i).getTextContent()); } responseMap.put("infos", JSONArray.fromObject(infoList)); } return responseMap; } public String submitSequenceData(Submission s) { Set<File> dataFiles = new HashSet<File>(); FilePathGenerator FPG = new TGACIlluminaFilepathGenerator(); //FilePathGenerator FPG = new FakeFilepathGenerator(); for (Object o : s.getSubmissionElements()) { if (o instanceof SequencerPoolPartition) { SequencerPoolPartition l = (SequencerPoolPartition) o; // if( l.getPool()!=null){ // Collection<LibraryDilution> ld=l.getPool().getDilutions(); // LibraryDilution libd=ld.iterator().next(); try { dataFiles = FPG.generateFilePaths(l); } catch (SubmissionException submissionException) { submissionException.printStackTrace(); } } } if (dataFiles.size() > 0) { TransferMethod t = new FTPTransferMethod(); EndPoint end = new ERAEndpoint(); end.setDestination(URI.create("ftp://localhost")); try { UploadReport report = t.uploadSequenceData(dataFiles, end); uploadReports.put(s.getId(), report); return ("Attempting to upload files..."); } catch (Exception e) { e.printStackTrace(); return ("There was an error: " + e.getMessage()); } } else return ("No datafiles were found to upload"); } @Override public UploadReport getUploadProgress(Long submissionId) { try { return uploadReports.get(submissionId); } catch (Exception e) { e.printStackTrace(); } return null; } public void setTransferMethod(TransferMethod transferMethod) { } /* public FTPUploadReport getUploadReport(Submission submission){ return uploadReports.get(submission); } */ public String prettifySubmissionMetadata(Submission submission) throws SubmissionException { StringBuilder sb = new StringBuilder(); try { Collection<File> files = misoFileManager.getFiles(Submission.class, submission.getName()); Date latestDate = null; //get latest submitted xmls try { for (File f : files) { if (f.getName().contains("submission_")) { String d = f.getName().substring(f.getName().lastIndexOf("_") + 1, f.getName().lastIndexOf(".")); Date test = df.parse(d); if (latestDate == null || test.after(latestDate)) { latestDate = test; } } } } catch (ParseException e) { log.error("No timestamped submission metadata documents. Falling back to simple names: " + e.getMessage()); } String dateStr = ""; if (latestDate != null) { dateStr = "_" + df.format(latestDate); } InputStream in = null; for (File f : files) { if (f.getName().contains("submission" + dateStr)) { in = ERASubmissionManager.class.getResourceAsStream("/submission/xsl/eraSubmission.xsl"); if (in != null) { String xsl = LimsUtils.inputStreamToString(in); sb.append(SubmissionUtils.xslTransform(SubmissionUtils.transform(f, true), xsl)); } } } for (File f : files) { if (f.getName().contains("study" + dateStr)) { in = ERASubmissionManager.class.getResourceAsStream("/submission/xsl/eraStudy.xsl"); if (in != null) { String xsl = LimsUtils.inputStreamToString(in); sb.append(SubmissionUtils.xslTransform(SubmissionUtils.transform(f, true), xsl)); } } } for (File f : files) { if (f.getName().contains("sample" + dateStr)) { in = ERASubmissionManager.class.getResourceAsStream("/submission/xsl/eraSample.xsl"); if (in != null) { String xsl = LimsUtils.inputStreamToString(in); sb.append(SubmissionUtils.xslTransform(SubmissionUtils.transform(f, true), xsl)); } } } for (File f : files) { if (f.getName().contains("experiment" + dateStr)) { in = ERASubmissionManager.class.getResourceAsStream("/submission/xsl/eraExperiment.xsl"); if (in != null) { String xsl = LimsUtils.inputStreamToString(in); sb.append(SubmissionUtils.xslTransform(SubmissionUtils.transform(f, true), xsl)); } } } for (File f : files) { if (f.getName().contains("run" + dateStr)) { in = ERASubmissionManager.class.getResourceAsStream("/submission/xsl/eraRun.xsl"); if (in != null) { String xsl = LimsUtils.inputStreamToString(in); sb.append(SubmissionUtils.xslTransform(SubmissionUtils.transform(f, true), xsl)); } } } } catch (IOException e) { e.printStackTrace(); } catch (TransformerException e) { e.printStackTrace(); } return sb.toString(); } /** * Builds a "trusting" trust manager. This is totally horrible and basically ignores everything that SSL stands for. * This allows connection to self-signed certificate hosts, bypassing the normal validation exceptions that occur. * <p/> * Use at your own risk - again, this is horrible! */ public DefaultHttpClient getEvilTrustingTrustManager(DefaultHttpClient httpClient) { try { // First create a trust manager that won't care about any SSL self-cert problems - eurgh! X509TrustManager trustManager = new X509TrustManager() { public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { log.warn("BYPASSING CLIENT TRUSTED CHECK!"); } public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { log.warn("BYPASSING SERVER TRUSTED CHECK!"); } public X509Certificate[] getAcceptedIssuers() { log.warn("BYPASSING CERTIFICATE ISSUER CHECKS!"); return null; } }; // Now put the trust manager into an SSLContext SSLContext sslcontext = SSLContext.getInstance("TLS"); sslcontext.init(null, new TrustManager[] { trustManager }, null); SSLSocketFactory sf = new SSLSocketFactory(sslcontext); sf.setHostnameVerifier(SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); // If you want a thread safe client, use the ThreadSafeConManager, but // otherwise just grab the one from the current client, and get hold of its // schema registry. THIS IS THE KEY THING. ClientConnectionManager ccm = httpClient.getConnectionManager(); SchemeRegistry schemeRegistry = ccm.getSchemeRegistry(); // Register our new socket factory with the typical SSL port and the // correct protocol name. schemeRegistry.register(new Scheme("https", sf, 443)); // Finally, apply the ClientConnectionManager to the Http Client // or, as in this example, create a new one. return new DefaultHttpClient(ccm, httpClient.getParams()); } catch (Throwable t) { log.warn("Something nasty happened with the EvilTrustingTrustManager. Warranty is null and void!"); t.printStackTrace(); return null; } } }