List of usage examples for java.io InputStreamReader close
public void close() throws IOException
From source file:carnero.cgeo.cgBase.java
public cgResponse request(boolean secure, String host, String path, String method, String params, int requestId, Boolean xContentType) {//from w w w . ja va 2s . com URL u = null; int httpCode = -1; String httpMessage = null; String httpLocation = null; if (requestId == 0) { requestId = (int) (Math.random() * 1000); } if (method == null || (method.equalsIgnoreCase("GET") == false && method.equalsIgnoreCase("POST") == false)) { method = "POST"; } else { method = method.toUpperCase(); } // https String scheme = "http://"; if (secure) { scheme = "https://"; } // prepare cookies String cookiesDone = null; if (cookies == null || cookies.isEmpty() == true) { if (cookies == null) { cookies = new HashMap<String, String>(); } final Map<String, ?> prefsAll = prefs.getAll(); final Set<String> prefsKeys = prefsAll.keySet(); for (String key : prefsKeys) { if (key.matches("cookie_.+") == true) { final String cookieKey = key.substring(7); final String cookieValue = (String) prefsAll.get(key); cookies.put(cookieKey, cookieValue); } } } if (cookies != null && !cookies.isEmpty() && cookies.keySet().size() > 0) { final Object[] keys = cookies.keySet().toArray(); final ArrayList<String> cookiesEncoded = new ArrayList<String>(); for (int i = 0; i < keys.length; i++) { String value = cookies.get(keys[i].toString()); cookiesEncoded.add(keys[i] + "=" + value); } if (cookiesEncoded.size() > 0) { cookiesDone = implode("; ", cookiesEncoded.toArray()); } } if (cookiesDone == null) { Map<String, ?> prefsValues = prefs.getAll(); if (prefsValues != null && prefsValues.size() > 0 && prefsValues.keySet().size() > 0) { final Object[] keys = prefsValues.keySet().toArray(); final ArrayList<String> cookiesEncoded = new ArrayList<String>(); final int length = keys.length; for (int i = 0; i < length; i++) { if (keys[i].toString().length() > 7 && keys[i].toString().substring(0, 7).equals("cookie_") == true) { cookiesEncoded .add(keys[i].toString().substring(7) + "=" + prefsValues.get(keys[i].toString())); } } if (cookiesEncoded.size() > 0) { cookiesDone = implode("; ", cookiesEncoded.toArray()); } } } if (cookiesDone == null) { cookiesDone = ""; } URLConnection uc = null; HttpURLConnection connection = null; Integer timeout = 30000; StringBuffer buffer = null; for (int i = 0; i < 5; i++) { if (i > 0) { Log.w(cgSettings.tag, "Failed to download data, retrying. Attempt #" + (i + 1)); } buffer = new StringBuffer(); timeout = 30000 + (i * 10000); try { if (method.equals("GET")) { // GET u = new URL(scheme + host + path + "?" + params); uc = u.openConnection(); uc.setRequestProperty("Host", host); uc.setRequestProperty("Cookie", cookiesDone); if (xContentType == true) { uc.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); } if (settings.asBrowser == 1) { uc.setRequestProperty("Accept", "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"); // uc.setRequestProperty("Accept-Encoding", "gzip"); // not supported via cellular network uc.setRequestProperty("Accept-Charset", "utf-8, iso-8859-1, utf-16, *;q=0.7"); uc.setRequestProperty("Accept-Language", "en-US"); uc.setRequestProperty("User-Agent", idBrowser); uc.setRequestProperty("Connection", "keep-alive"); uc.setRequestProperty("Keep-Alive", "300"); } connection = (HttpURLConnection) uc; connection.setReadTimeout(timeout); connection.setRequestMethod(method); HttpURLConnection.setFollowRedirects(false); connection.setDoInput(true); connection.setDoOutput(false); } else { // POST u = new URL(scheme + host + path); uc = u.openConnection(); uc.setRequestProperty("Host", host); uc.setRequestProperty("Cookie", cookiesDone); if (xContentType == true) { uc.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); } if (settings.asBrowser == 1) { uc.setRequestProperty("Accept", "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"); // uc.setRequestProperty("Accept-Encoding", "gzip"); // not supported via cellular network uc.setRequestProperty("Accept-Charset", "utf-8, iso-8859-1, utf-16, *;q=0.7"); uc.setRequestProperty("Accept-Language", "en-US"); uc.setRequestProperty("User-Agent", idBrowser); uc.setRequestProperty("Connection", "keep-alive"); uc.setRequestProperty("Keep-Alive", "300"); } connection = (HttpURLConnection) uc; connection.setReadTimeout(timeout); connection.setRequestMethod(method); HttpURLConnection.setFollowRedirects(false); connection.setDoInput(true); connection.setDoOutput(true); final OutputStream out = connection.getOutputStream(); final OutputStreamWriter wr = new OutputStreamWriter(out); wr.write(params); wr.flush(); wr.close(); } String headerName = null; final SharedPreferences.Editor prefsEditor = prefs.edit(); for (int j = 1; (headerName = uc.getHeaderFieldKey(j)) != null; j++) { if (headerName != null && headerName.equalsIgnoreCase("Set-Cookie")) { int index; String cookie = uc.getHeaderField(j); index = cookie.indexOf(";"); if (index > -1) { cookie = cookie.substring(0, cookie.indexOf(";")); } index = cookie.indexOf("="); if (index > -1 && cookie.length() > (index + 1)) { String name = cookie.substring(0, cookie.indexOf("=")); String value = cookie.substring(cookie.indexOf("=") + 1, cookie.length()); cookies.put(name, value); prefsEditor.putString("cookie_" + name, value); } } } prefsEditor.commit(); final String encoding = connection.getContentEncoding(); InputStream ins; if (encoding != null && encoding.equalsIgnoreCase("gzip")) { ins = new GZIPInputStream(connection.getInputStream()); } else if (encoding != null && encoding.equalsIgnoreCase("deflate")) { ins = new InflaterInputStream(connection.getInputStream(), new Inflater(true)); } else { ins = connection.getInputStream(); } final InputStreamReader inr = new InputStreamReader(ins); final BufferedReader br = new BufferedReader(inr); readIntoBuffer(br, buffer); httpCode = connection.getResponseCode(); httpMessage = connection.getResponseMessage(); httpLocation = uc.getHeaderField("Location"); final String paramsLog = params.replaceAll(passMatch, "password=***"); if (buffer != null && connection != null) { Log.i(cgSettings.tag + "|" + requestId, "[" + method + " " + (int) (params.length() / 1024) + "k | " + httpCode + " | " + (int) (buffer.length() / 1024) + "k] Downloaded " + scheme + host + path + "?" + paramsLog); } else { Log.i(cgSettings.tag + "|" + requestId, "[" + method + " | " + httpCode + "] Failed to download " + scheme + host + path + "?" + paramsLog); } connection.disconnect(); br.close(); ins.close(); inr.close(); } catch (IOException e) { Log.e(cgSettings.tag, "cgeoBase.request.IOException: " + e.toString()); } catch (Exception e) { Log.e(cgSettings.tag, "cgeoBase.request: " + e.toString()); } if (buffer != null && buffer.length() > 0) { break; } } cgResponse response = new cgResponse(); String data = null; try { if (httpCode == 302 && httpLocation != null) { final Uri newLocation = Uri.parse(httpLocation); if (newLocation.isRelative() == true) { response = request(secure, host, path, "GET", new HashMap<String, String>(), requestId, false, false, false); } else { boolean secureRedir = false; if (newLocation.getScheme().equals("https")) { secureRedir = true; } response = request(secureRedir, newLocation.getHost(), newLocation.getPath(), "GET", new HashMap<String, String>(), requestId, false, false, false); } } else { if (buffer != null && buffer.length() > 0) { data = replaceWhitespace(buffer); buffer = null; if (data != null) { response.setData(data); } else { response.setData(""); } response.setStatusCode(httpCode); response.setStatusMessage(httpMessage); response.setUrl(u.toString()); } } } catch (Exception e) { Log.e(cgSettings.tag, "cgeoBase.page: " + e.toString()); } return response; }
From source file:au.org.theark.lims.util.BioCollectionSpecimenUploader.java
/** * //from w w w. j a va 2 s .co m * Upload the biocollection file data. * * Where N is any number of columns * * @param fileInputStream * is the input stream of a file * @param inLength * is the length of a file * @throws FileFormatException * file format Exception * @throws ArkBaseException * general ARK Exception * @return the upload report detailing the upload process */ public StringBuffer uploadAndReportMatrixBiocollectionFile(InputStream fileInputStream, long inLength, String inFileFormat, char inDelimChr) throws FileFormatException, ArkSystemException { delimiterCharacter = inDelimChr; uploadReport = new StringBuffer(); curPos = 0; InputStreamReader inputStreamReader = null; CsvReader csvReader = null; DecimalFormat decimalFormat = new DecimalFormat("0.00"); // If Excel, convert to CSV for validation if (inFileFormat.equalsIgnoreCase("XLS")) { Workbook w; try { w = Workbook.getWorkbook(fileInputStream); delimiterCharacter = ','; XLStoCSV xlsToCsv = new XLStoCSV(delimiterCharacter); fileInputStream = xlsToCsv.convertXlsToCsv(w); fileInputStream.reset(); } catch (BiffException e) { log.error(e.getMessage()); } catch (IOException e) { log.error(e.getMessage()); } } try { inputStreamReader = new InputStreamReader(fileInputStream); csvReader = new CsvReader(inputStreamReader, delimiterCharacter); srcLength = inLength; if (srcLength <= 0) { uploadReport.append("The input size was not greater than 0. Actual length reported: "); uploadReport.append(srcLength); uploadReport.append("\n"); throw new FileFormatException( "The input size was not greater than 0. Actual length reported: " + srcLength); } timer = new StopWatch(); timer.start(); csvReader.readHeaders(); srcLength = inLength - csvReader.getHeaders().toString().length(); log.debug("Header length: " + csvReader.getHeaders().toString().length()); // Loop through all rows in file while (csvReader.readRecord()) { log.info("At record: " + recordCount); String subjectUID = csvReader.get("SUBJECTUID"); String biocollectionUID = csvReader.get("BIOCOLLECTIONUID"); LinkSubjectStudy linkSubjectStudy = iArkCommonService.getSubjectByUIDAndStudy(subjectUID, study); //this is validated in prior step and should never happen if (linkSubjectStudy == null) { log.error( "\n\n\n\n\n\n\n\n\n\n\n\nUnexpected subject? a shouldnt happen...we should have errored this in validation"); break;//TODO : log appropriately or do some handling } BioCollection bioCollection = iLimsService.getBioCollectionForStudySubjectByUID(biocollectionUID, study, linkSubjectStudy); if (bioCollection == null) { bioCollection = new BioCollection(); if (study.getAutoGenerateBiocollectionUid()) { // if biocollection not in the system we have to create a new biocollection uid. bioCollection.setBiocollectionUid(iLimsService.getNextGeneratedBiospecimenUID(study)); } else { bioCollection.setBiocollectionUid(biocollectionUID); } } else {// if exsists we do not want to auto genetared the uid. bioCollection.setBiocollectionUid(biocollectionUID); } bioCollection.setStudy(study); bioCollection.setLinkSubjectStudy(linkSubjectStudy); if (csvReader.getIndex("NAME") > 0) { String name = csvReader.get("NAME"); bioCollection.setName(name); } if (csvReader.getIndex("COLLECTIONDATE") > 0) { String collectionDate = csvReader.get("COLLECTIONDATE"); bioCollection.setCollectionDate(simpleDateFormat.parse(collectionDate)); } if (csvReader.getIndex("COMMENTS") > 0) { String comments = csvReader.get("COMMENTS"); bioCollection.setComments(comments); } //validation SHOULD make sure these cases will work. TODO: test scripts if (bioCollection.getId() == null) { insertBiocollections.add(bioCollection); StringBuffer sb = new StringBuffer(); sb.append("BioCollectionUID: "); sb.append(bioCollection.getBiocollectionUid()); sb.append(" has been created successfully."); sb.append("\n"); uploadReport.append(sb); insertCount++; } else { updateBiocollections.add(bioCollection); StringBuffer sb = new StringBuffer(); sb.append("BioCollectionUID: "); sb.append(bioCollection.getBiocollectionUid()); sb.append(" has been updated successfully."); sb.append("\n"); uploadReport.append(sb); updateCount++; } recordCount++; } } catch (IOException ioe) { uploadReport.append("Unexpected I/O exception whilst reading the biospecimen data file\n"); log.error("processMatrixBiospecimenFile IOException stacktrace:", ioe); throw new ArkSystemException("Unexpected I/O exception whilst reading the biospecimen data file"); } catch (Exception ex) { uploadReport.append("Unexpected exception whilst reading the biospecimen data file\n"); log.error("processMatrixBiospecimenFile Exception stacktrace:", ex); throw new ArkSystemException( "Unexpected exception occurred when trying to process biospecimen data file"); } finally { // Clean up the IO objects timer.stop(); uploadReport.append("\n"); uploadReport.append("Total elapsed time: "); uploadReport.append(timer.getTime()); uploadReport.append(" ms or "); uploadReport.append(decimalFormat.format(timer.getTime() / 1000.0)); uploadReport.append(" s"); uploadReport.append("\n"); uploadReport.append("Total file size: "); uploadReport.append(inLength); uploadReport.append(" B or "); uploadReport.append(decimalFormat.format(inLength / 1024.0 / 1024.0)); uploadReport.append(" MB"); uploadReport.append("\n"); if (timer != null) timer = null; if (csvReader != null) { try { csvReader.close(); } catch (Exception ex) { log.error("Cleanup operation failed: csvRdr.close()", ex); } } if (inputStreamReader != null) { try { inputStreamReader.close(); } catch (Exception ex) { log.error("Cleanup operation failed: isr.close()", ex); } } // Restore the state of variables srcLength = -1; } uploadReport.append("Processed "); uploadReport.append(recordCount); uploadReport.append(" records."); uploadReport.append("\n"); uploadReport.append("Inserted "); uploadReport.append(insertCount); uploadReport.append(" records."); uploadReport.append("\n"); uploadReport.append("Updated "); uploadReport.append(updateCount); uploadReport.append(" records."); uploadReport.append("\n"); // Batch insert/update iLimsService.batchInsertBiocollections(insertBiocollections); iLimsService.batchUpdateBiocollections(updateBiocollections); return uploadReport; }
From source file:com.cohort.util.String2.java
/** * This reads the text contents of the specified text file. * //from w w w . j a v a2 s . c o m * <P>This method uses try/catch to ensure that all possible * exceptions are caught and returned as the error String * (throwable.toString()). * * <P>This method is generally appropriate for small and medium-sized * files. For very large files or files that need additional processing, * it may be better to write a custom method to * read the file line-by-line, processing as it goes. * * @param fileName is the (usually canonical) path (dir+name) for the file * @param charset e.g., ISO-8859-1, UTF-8, or "" or null for the default (ISO-8859-1) * @param maxAttempt e.g. 3 (the tries are 1 second apart) * @return a String array with two strings. * Using a String array gets around Java's limitation of * only returning one value from a method. * String #0 is an error String (or "" if no error). * String #1 has the contents of the file as one big string * (with any end-of-line characters converted to \n). * If the error String is not "", String #1 * may not have all the contents of the file. * ***This ensures that the last character in the file (if any) is \n. * This behavior varies from other implementations of readFromFile. */ public static String[] readFromFile(String fileName, String charset, int maxAttempt) { //declare the BufferedReader variable //declare the results variable: String results[] = {"", ""}; //BufferedReader and results are declared outside try/catch so //that they can be accessed from within either try/catch block. long time = System.currentTimeMillis(); FileInputStream fis = null; InputStreamReader isr = null; BufferedReader bufferedReader = null; String results[] = { "", "" }; int errorIndex = 0; int contentsIndex = 1; try { //open the file //To deal with problems in multithreaded apps //(when deleting and renaming files, for an instant no file with that name exists), maxAttempt = Math.max(1, maxAttempt); for (int attempt = 1; attempt <= maxAttempt; attempt++) { try { fis = new FileInputStream(fileName); isr = new InputStreamReader(fis, charset == null || charset.length() == 0 ? ISO_8859_1 : charset); } catch (Exception e) { if (attempt == maxAttempt) { log(ERROR + ": String2.readFromFile was unable to read " + fileName); throw e; } else { log("WARNING #" + attempt + ": String2.readFromFile is having trouble. It will try again to read " + fileName); if (attempt == 1) Math2.gcAndWait(); //trouble! Give OS/Java a time and gc to deal with trouble else Math2.sleep(1000); } } } bufferedReader = new BufferedReader(isr); //get the text from the file //This uses bufferedReader.readLine() to repeatedly //read lines from the file and thus can handle various //end-of-line characters. //The lines (with \n added at the end) are added to a //StringBuilder. StringBuilder sb = new StringBuilder(8192); String s = bufferedReader.readLine(); while (s != null) { //null = end-of-file sb.append(s); sb.append('\n'); s = bufferedReader.readLine(); } //save the contents as results[1] results[contentsIndex] = sb.toString(); } catch (Exception e) { results[errorIndex] = MustBe.throwable("fileName=" + fileName, e); } //close the bufferedReader try { //close the highest level file object available if (bufferedReader != null) bufferedReader.close(); else if (isr != null) isr.close(); else if (fis != null) fis.close(); } catch (Exception e) { if (results[errorIndex].length() == 0) results[errorIndex] = e.toString(); //else ignore the error (the first one is more important) } //return results //log(" String2.readFromFile " + fileName + " time=" + // (System.currentTimeMillis() - time) + "ms"); return results; }
From source file:edu.slu.action.ObjectAction.java
/** * All actions come here to process the request body. We check if it is JSON. * DELETE is a special case because the content could be JSON or just the @id string and it only has to specify a content type if passing a JSONy object. * and pretty format it. Returns pretty stringified JSON or fail to null. * Methods that call this should handle requestBody==null as unexpected. * @param http_request Incoming request to check. * @param supportStringID The request may be allowed to pass the @id as the body. * @return String of anticipated JSON format. * @throws java.io.IOException//from ww w . j a v a2 s . c om * @throws javax.servlet.ServletException * @throws java.lang.Exception */ public String processRequestBody(HttpServletRequest http_request, boolean supportStringID) throws IOException, ServletException, Exception { String cType = http_request.getContentType(); http_request.setCharacterEncoding("UTF-8"); String requestBody; JSONObject complianceInfo = new JSONObject(); /* UTF-8 special character support for requests */ //http://biercoff.com/malformedinputexception-input-length-1-exception-solution-for-scala-and-java/ ServletInputStream input = http_request.getInputStream(); //CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder(); //decoder.onMalformedInput(CodingErrorAction.REPLACE); InputStreamReader reader = new InputStreamReader(input, "utf-8"); //bodyReader = new BufferedReader( reader ); //System.out.println("Process req body..."); bodyReader = new BufferedReader(reader); //System.out.println("...got reader"); bodyString = new StringBuilder(); String line; JSONObject test; JSONArray test2; if (null != cType && (cType.contains("application/json") || cType.contains("application/ld+json"))) { //System.out.println("Processing because it was jsony"); //Note special characters cause this to break right here. try { while ((line = bodyReader.readLine()) != null) { bodyString.append(line); } } catch (Exception e) { System.out.println("Couldn't access body to read"); System.out.println(e); } //System.out.println("built body string"); requestBody = bodyString.toString(); //System.out.println("here is the bodyString"); //System.out.println(requestBody); try { //JSONObject test test = JSONObject.fromObject(requestBody); } catch (Exception ex) { System.out.println("not a json object for processing"); if (supportStringID) { //We do not allow arrays of ID's for DELETE, so if it failed JSONObject parsing then this is a hard fail for DELETE. //They attempted to provide a JSON object for DELETE but it was not valid JSON writeErrorResponse("The data passed was not valid JSON. Could not get @id: " + requestBody, HttpServletResponse.SC_BAD_REQUEST); requestBody = null; } else { //Maybe it was an action on a JSONArray, check that before failing JSON parse test. try { //JSONArray test test2 = JSONArray.fromObject(requestBody); } catch (Exception ex2) { // not a JSONObject or a JSONArray. writeErrorResponse("The data passed was not valid JSON:\n" + requestBody, HttpServletResponse.SC_BAD_REQUEST); requestBody = null; } } } // no-catch: Is either JSONObject or JSON Array } else { if (supportStringID) { //Content type is not JSONy, looking for @id string as body while ((line = bodyReader.readLine()) != null) { bodyString.append(line); } requestBody = bodyString.toString(); try { test = JSONObject.fromObject(requestBody); if (test.containsKey("@id")) { requestBody = test.getString("@id"); if ("".equals(requestBody)) { //No ID provided writeErrorResponse( "Must provide an id or a JSON object containing @id of object to perform this action.", HttpServletResponse.SC_BAD_REQUEST); requestBody = null; } else { // This string could be ANYTHING. ANY string is valid at this point. Create a wrapper JSONObject for elegant handling in deleteObject(). // We will check against the string for existing objects in deleteObject(), processing the body is completed as far as this method is concerned. JSONObject modifiedDeleteRequest = new JSONObject(); modifiedDeleteRequest.element("@id", requestBody); requestBody = modifiedDeleteRequest.toString(); } } } catch (Exception e) { //This is good, they should not be using a JSONObject if ("".equals(requestBody)) { //No ID provided writeErrorResponse( "Must provide an id or a JSON object containing @id of object to delete.", HttpServletResponse.SC_BAD_REQUEST); requestBody = null; } else { // This string could be ANYTHING. ANY string is valid at this point. Create a wrapper JSONObject for elegant handling in deleteObject(). // We will check against the string for existing objects in deleteObject(), processing the body is completed as far as this method is concerned. JSONObject modifiedDeleteRequest = new JSONObject(); modifiedDeleteRequest.element("@id", requestBody); requestBody = modifiedDeleteRequest.toString(); } } } else { //This is an error, actions must use the correct content type writeErrorResponse("Invalid Content-Type. Please use 'application/json' or 'application/ld+json'", HttpServletResponse.SC_BAD_REQUEST); requestBody = null; } } //@cubap @theHabes TODO IIIF compliance handling on action objects /* if(null != requestBody){ complianceInfo = checkIIIFCompliance(requestBody, "2.1"); if(complianceInfo.getInt("okay") < 1){ writeErrorResponse(complianceInfo.toString(), HttpServletResponse.SC_CONFLICT); requestBody = null; } } */ reader.close(); input.close(); content = requestBody; response.setContentType("application/json; charset=utf-8"); // We create JSON objects for the return body in most cases. response.addHeader("Access-Control-Allow-Headers", "Content-Type"); response.addHeader("Access-Control-Allow-Methods", "GET,OPTIONS,HEAD,PUT,PATCH,DELETE,POST"); // Must have OPTIONS for @webanno return requestBody; }
From source file:com.amalto.workbench.export.ImportItemsWizard.java
protected void parses(boolean importFromArchieve, String zipFilePath, IProgressMonitor monitor) { // init var for progressMonitor int total = 500, zipCount = 200, readCount = 100; int step = 1, interval = 1; ///*from ww w . j a va2s . c om*/ monitor.beginTask(Messages.ImportItemsWizard_7, total); if (importFromArchieve) { checkUpExchangeImport(true); try { Util.unZipFile(zipFilePath, importFolder, zipCount, monitor); } catch (Exception e) { log.error(e.getMessage(), e); } } monitor.worked(zipCount); monitor.setTaskName(Messages.ImportItemsWizard_8); InputStreamReader reader = null; try { reader = new InputStreamReader(new FileInputStream(importFolder + "/exportitems.xml"), "UTF-8");//$NON-NLS-1$//$NON-NLS-2$ final Exports exports = (Exports) Unmarshaller.unmarshal(Exports.class, reader); String[] orgSchemas = exports.getSchemas(); int idx = 0; if (orgSchemas != null) { for (String orgSchema : orgSchemas) { String orgSchemaCpy = new String(orgSchema); for (Map.Entry<String, String> pair : ICoreConstants.rolesConvert.oldRoleToNewRoleMap .entrySet()) { orgSchemaCpy = orgSchemaCpy.replaceAll(pair.getKey().toString(), pair.getValue().toString()); } if (!orgSchemaCpy.equals(orgSchema)) { orgSchemas[idx] = orgSchemaCpy; } idx++; } } try { LocalTreeObjectRepository.getInstance().makeUpDocWithImportCategory(orgSchemas, serverRoot); } catch (Exception e) { } // import autoincrement if (exports.getAutoIncrement() != null) { try { service.getAutoIncrement(new WSAutoIncrement(exports.getAutoIncrement())); } catch (Exception e) { } } // new server root final TreeParent reserverRoot = new TreeParent(serverRoot.getDisplayName(), null, TreeObject._SERVER_, serverRoot.getWsKey(), serverRoot.getWsObject()); reserverRoot.setUser(serverRoot.getUser()); // serverRoot=reserverRoot; TreeParent clusters = new TreeParent(EXtentisObjects.DataCluster.getDisplayName(), reserverRoot, TreeObject.DATA_CLUSTER, null, null); TreeParent models = new TreeParent(EXtentisObjects.DataMODEL.getDisplayName(), reserverRoot, TreeObject.DATA_MODEL, null, null); TreeParent menus = new TreeParent(EXtentisObjects.Menu.getDisplayName(), reserverRoot, TreeObject.MENU, null, null); TreeParent roles = new TreeParent(EXtentisObjects.Role.getDisplayName(), reserverRoot, TreeObject.ROLE, null, null); TreeParent eventManager = new TreeParent(EXtentisObjects.EventManagement.getDisplayName(), reserverRoot, TreeObject.EVENT_MANAGEMENT, null, null); TreeParent routingrules = new TreeParent(EXtentisObjects.RoutingRule.getDisplayName(), reserverRoot, TreeObject.ROUTING_RULE, null, null); TreeParent storeprocedures = new TreeParent(EXtentisObjects.StoredProcedure.getDisplayName(), reserverRoot, TreeObject.STORED_PROCEDURE, null, null); TreeParent transformers = new TreeParent(EXtentisObjects.Transformer.getDisplayName(), reserverRoot, TreeObject.TRANSFORMER, null, null); TreeParent pictures = new TreeParent(EXtentisObjects.PICTURESRESOURCE.getDisplayName(), reserverRoot, TreeObject.PICTURES_RESOURCE, null, null); // add by ymli TreeParent workflow = new TreeParent(EXtentisObjects.Workflow.getDisplayName(), reserverRoot, TreeObject.WORKFLOW, null, null); TreeParent views = new TreeParent(EXtentisObjects.View.getDisplayName(), reserverRoot, TreeObject.VIEW, null, null); reserverRoot.addChild(clusters); reserverRoot.addChild(models); reserverRoot.addChild(menus); reserverRoot.addChild(roles); reserverRoot.addChild(storeprocedures); eventManager.addChild(transformers); eventManager.addChild(routingrules); reserverRoot.addChild(eventManager); reserverRoot.addChild(pictures); reserverRoot.addChild(workflow); reserverRoot.addChild(views); monitor.worked(readCount); // caculate step and interval float val = (total - zipCount - readCount) / exports.getItems().length; if (val > 0) { interval = 1; step = Math.round(val); } else { step = 1; interval = Math.round(exports.getItems().length / (total - zipCount - readCount) + 0.5f); } // System.out.println("count:" + exports.getItems().length + "\tinterval:" + interval + "\tstep:" + step); monitor.setTaskName(Messages.ImportItemsWizard_9); // int tmp = 1; for (TreeObject obj : exports.getItems()) { obj.setServerRoot(reserverRoot); switch (obj.getType()) { case TreeObject.DATA_CLUSTER: clusters.addChild(obj); break; case TreeObject.DATA_MODEL: models.addChild(obj); break; case TreeObject.MENU: menus.addChild(obj); break; case TreeObject.ROLE: roles.addChild(obj); break; case TreeObject.ROUTING_RULE: routingrules.addChild(obj); break; case TreeObject.STORED_PROCEDURE: storeprocedures.addChild(obj); break; case TreeObject.TRANSFORMER: transformers.addChild(obj); break; case TreeObject.PICTURES_RESOURCE: pictures.addChild(obj); break; case TreeObject.WORKFLOW_PROCESS: workflow.addChild(obj); break; case TreeObject.VIEW: views.addChild(obj); break; default: if (obj.getItems() != null && obj.getItems().length > 0) { for (int i = 0; i < obj.getItems().length; i++) { if (obj.getItems()[i].split(Messages.ImportItemsWizard_10)[1] != null) { dataClusterContent.put(obj.getItems()[i].split(Messages.ImportItemsWizard_11)[1], obj.getItems()); } } } } // update monitor if (interval == 1) { monitor.worked(step); } else { if (tmp >= interval) { monitor.worked(step); tmp = 1; } else { tmp++; } } } Display.getDefault().syncExec(new Runnable() { public void run() { treeViewer.setRoot(reserverRoot); treeViewer.getViewer().setInput(null); treeViewer.setCheckItems(Arrays.asList(exports.getItems())); GridData gd = (GridData) treeViewer.getViewer().getControl().getLayoutData(); gd.heightHint = 300; treeViewer.getViewer().getControl().getParent().layout(true); treeViewer.getViewer().getControl().getShell().layout(true); try { Object[] checkNodes = treeViewer.getCheckNodes(); if (checkNodes != null) { LocalTreeObjectRepository.getInstance() .setOriginalXobjectsToImport((TreeObject[]) checkNodes); } } catch (Exception e) { } } }); } catch (Exception e) { log.error(e.getMessage(), e); } finally { try { if (reader != null) { reader.close(); } } catch (Exception e) { } monitor.done(); } }
From source file:be.fedict.eidviewer.lib.file.imports.Version35CSVFile.java
public void load(File file) throws CertificateException, FileNotFoundException, IOException { logger.fine("Loading Version 35X CSV File"); String line = null;/* ww w . ja v a2 s.c o m*/ certificateFactory = CertificateFactory.getInstance("X.509"); InputStreamReader inputStreamReader = new InputStreamReader(new FileInputStream(file), "utf-8"); BufferedReader bufferedReader = new BufferedReader(inputStreamReader); line = bufferedReader.readLine(); if (line != null) { String[] tokens = line.split(";"); for (int tokenNumber = 0; (variableCertCount < 0) && (tokenNumber < tokens.length); tokenNumber++) { String token = tokens[tokenNumber]; logger.log(Level.FINEST, "token #{0} : [{1}]", new Object[] { tokenNumber, token }); try { switch (tokenNumber) { case DOCUMENTTYPE: identity = new Identity(); identity.documentType = DocumentType.toDocumentType(token.getBytes("utf-8")); if (identity.documentType == null) logger.log(Level.SEVERE, "Unknown Document Type \"{0}\"", token); break; case FIRSTNAMES: TextFormatHelper.setFirstNamesFromString(identity, token); break; case LASTNAME: identity.name = token; break; case GENDER: identity.gender = token.equals("M") ? Gender.MALE : Gender.FEMALE; break; case BIRTHDATE: { logger.fine("field BIRTHDATE"); DateOfBirthDataConvertor dateOfBirthConvertor = new DateOfBirthDataConvertor(); identity.dateOfBirth = dateOfBirthConvertor.convert(token.getBytes("utf-8")); } break; case PLACEOFBIRTH: identity.placeOfBirth = token; break; case NATIONALITY: identity.nationality = token; break; case NATIONALNUMBER: identity.nationalNumber = token; break; case CARDNUMBER: identity.cardNumber = token; break; case CARDCHIPNUMBER: identity.chipNumber = token; break; case CARDVALIDFROM: { GregorianCalendar validityBeginCalendar = new GregorianCalendar(); try { validityBeginCalendar.setTime(dateFormat.parse(token)); identity.cardValidityDateBegin = validityBeginCalendar; } catch (ParseException ex) { logger.log(Level.SEVERE, "Failed to parse Card Validity Start Date \"" + token + "\"", ex); } } break; case CARDVALIDUNTIL: GregorianCalendar validityEndCalendar = new GregorianCalendar(); try { validityEndCalendar.setTime(dateFormat.parse(token)); identity.cardValidityDateEnd = validityEndCalendar; } catch (ParseException ex) { logger.log(Level.SEVERE, "Failed to parse Card Validity End Date \"" + token + "\"", ex); } break; case CARDISSUINGMUNICIPALITY: identity.cardDeliveryMunicipality = token; break; case STREETANDNUMBER: address = new Address(); address.streetAndNumber = token; break; case ZIP: address.zip = token; break; case MUNICIPALITY: address.municipality = token; break; case PHOTO: byte[] tokenBytes = token.getBytes(); eidData.setPhoto(Base64.decodeBase64(tokenBytes)); break; case RRNCERTIFICATE: logger.finer("Gathering RRN Certificate"); try { rrnCert = (X509Certificate) certificateFactory.generateCertificate( new ByteArrayInputStream(Base64.decodeBase64(token.getBytes()))); } catch (CertificateException ex) { logger.log(Level.SEVERE, "Failed to RRN Certificate", ex); } break; case CERTCOUNT: logger.finer("Certificate Count: [" + token + "]"); variableCertCount = Integer.parseInt(token); break; } } catch (UnsupportedEncodingException ex) { logger.log(Level.SEVERE, "Unsupported Encoding for Token " + tokenNumber, ex); } catch (DataConvertorException ex) { logger.log(Level.SEVERE, "Couldn't Convert Date \"" + tokens[tokenNumber] + "\" in Token " + tokenNumber, ex); } } // done looping over fixed parts.. if (identity != null) { TextFormatHelper.setFirstNamesFromStrings(identity, identity.getFirstName(), identity.getMiddleName()); eidData.setIdentity(identity); } if (address != null) eidData.setAddress(address); // get variableCertCount variable certs for (int i = 0; i < variableCertCount; i++) { X509Certificate thisCertificate = null; int certOffset = CERTBASE + (CERTFIELDS * i); String certType = tokens[certOffset + CERT_NAME_OFFSET]; String certData = tokens[certOffset + CERT_DATA_OFFSET]; logger.finer("Gathering " + certType + " Certificate"); try { thisCertificate = (X509Certificate) certificateFactory.generateCertificate( new ByteArrayInputStream(Base64.decodeBase64(certData.getBytes()))); } catch (CertificateException ex) { logger.log(Level.SEVERE, "Failed to Convert Signing Certificate", ex); thisCertificate = null; } if (thisCertificate != null) { if (certType.equalsIgnoreCase("Authentication")) authenticationCert = thisCertificate; else if (certType.equalsIgnoreCase("Signature")) signingCert = thisCertificate; else if (certType.equalsIgnoreCase("CA")) citizenCert = thisCertificate; else if (certType.equalsIgnoreCase("Root")) rootCert = thisCertificate; } } } if (rootCert != null && citizenCert != null) { logger.fine("Certificates were gathered"); if (rrnCert != null) { logger.fine("Setting RRN Certificate Chain"); List<X509Certificate> rrnChain = new LinkedList<X509Certificate>(); rrnChain.add(rrnCert); rrnChain.add(rootCert); eidData.setRRNCertChain(new X509CertificateChainAndTrust( TrustServiceDomains.BELGIAN_EID_NATIONAL_REGISTRY_TRUST_DOMAIN, rrnChain)); } if (authenticationCert != null) { logger.fine("Setting Authentication Certificate Chain"); List<X509Certificate> authChain = new LinkedList<X509Certificate>(); authChain.add(authenticationCert); authChain.add(citizenCert); authChain.add(rootCert); eidData.setAuthCertChain(new X509CertificateChainAndTrust( TrustServiceDomains.BELGIAN_EID_AUTH_TRUST_DOMAIN, authChain)); } if (signingCert != null) { logger.fine("Setting Signing Certificate Chain"); List<X509Certificate> signChain = new LinkedList<X509Certificate>(); signChain.add(signingCert); signChain.add(citizenCert); signChain.add(rootCert); eidData.setSignCertChain(new X509CertificateChainAndTrust( TrustServiceDomains.BELGIAN_EID_NON_REPUDIATION_TRUST_DOMAIN, signChain)); } } inputStreamReader.close(); }
From source file:au.org.theark.lims.util.BioCollectionSpecimenUploader.java
/** * // w ww . ja v a2 s . com * Upload the biospecimen file data. * * Where N is any number of columns * * @param fileInputStream * is the input stream of a file * @param inLength * is the length of a file * @throws FileFormatException * file format Exception * @throws ArkBaseException * general ARK Exception * @return the upload report detailing the upload process */ public StringBuffer uploadAndReportMatrixBiospecimenFile(InputStream fileInputStream, long inLength, String inFileFormat, char inDelimChr) throws FileFormatException, ArkSystemException { delimiterCharacter = inDelimChr; uploadReport = new StringBuffer(); curPos = 0; InputStreamReader inputStreamReader = null; CsvReader csvReader = null; DecimalFormat decimalFormat = new DecimalFormat("0.00"); // If Excel, convert to CSV for validation if (inFileFormat.equalsIgnoreCase("XLS")) { Workbook w; try { w = Workbook.getWorkbook(fileInputStream); delimiterCharacter = ','; XLStoCSV xlsToCsv = new XLStoCSV(delimiterCharacter); fileInputStream = xlsToCsv.convertXlsToCsv(w); fileInputStream.reset(); } catch (BiffException e) { log.error(e.getMessage()); } catch (IOException e) { log.error(e.getMessage()); } } try { inputStreamReader = new InputStreamReader(fileInputStream); csvReader = new CsvReader(inputStreamReader, delimiterCharacter); srcLength = inLength; if (srcLength <= 0) { uploadReport.append("The input size was not greater than 0. Actual length reported: "); uploadReport.append(srcLength); uploadReport.append("\n"); throw new FileFormatException( "The input size was not greater than 0. Actual length reported: " + srcLength); } timer = new StopWatch(); timer.start(); csvReader.readHeaders(); srcLength = inLength - csvReader.getHeaders().toString().length(); log.debug("Header length: " + csvReader.getHeaders().toString().length()); // Loop through all rows in file while (csvReader.readRecord()) { log.info("At record: " + recordCount); String subjectUID = csvReader.get("SUBJECTUID"); String biospecimenUID = csvReader.get("BIOSPECIMENUID"); LinkSubjectStudy linkSubjectStudy = iArkCommonService.getSubjectByUIDAndStudy(subjectUID, study); //this is validated in prior step and should never happen if (linkSubjectStudy == null) { log.error( "\n\n\n\n\n\n\n\n\n\n\n\n Unexpected subject? a shouldnt happen...we should have errored this in validation"); break;//TODO : log appropriately or do some handling } //Always create a new biospecimen in this time //exsisting biospecimen are not allow to update in here. Biospecimen biospecimen = iLimsService.getBiospecimenByUid(biospecimenUID, study); if (biospecimen == null) { biospecimen = new Biospecimen(); } else { log.error( "\n\n\n\n\n\n\n\n\n....We should NEVER have existing biospecimens this should be validated in prior step"); break; } biospecimen.setStudy(study); biospecimen.setLinkSubjectStudy(linkSubjectStudy); if (csvReader.getIndex("BIOCOLLECTIONUID") > 0) { String biocollectionUid = csvReader.get("BIOCOLLECTIONUID"); BioCollection bioCollection = iLimsService.getBioCollectionByUID(biocollectionUid, this.study.getId(), subjectUID); if (bioCollection == null) { log.error( "\n\n\n\n\n\n\n\n\n....We already validated for the exsisting biocollection and we never created " + "new one if it does not exsists."); break; } else { biospecimen.setBioCollection(bioCollection); } } if (csvReader.getIndex("SAMPLETYPE") > 0) { String name = csvReader.get("SAMPLETYPE"); BioSampletype sampleType = new BioSampletype(); sampleType = iLimsService.getBioSampleTypeByName(name); biospecimen.setSampleType(sampleType); } if (csvReader.getIndex("QUANTITY") > 0) { String quantity = csvReader.get("QUANTITY"); biospecimen.setQuantity(new Double(quantity)); } if (csvReader.getIndex("CONCENTRATION") > 0) { String concentration = csvReader.get("CONCENTRATION"); if (concentration != null && !concentration.isEmpty()) { try { biospecimen.setConcentration(Double.parseDouble(concentration)); } catch (NumberFormatException ne) { log.error("Already validated in the previous step and never happen the for error"); } } } if (csvReader.getIndex("UNITS") > 0) { String name = csvReader.get("UNITS"); Unit unit = iLimsService.getUnitByName(name); biospecimen.setUnit(unit); } if (csvReader.getIndex("TREATMENT") > 0) { String name = csvReader.get("TREATMENT"); TreatmentType treatmentType = iLimsService.getTreatmentTypeByName(name); biospecimen.setTreatmentType(treatmentType); } Set<BioTransaction> bioTransactions = new HashSet<BioTransaction>(0); // Inheriently create a transaction for the initial quantity BioTransaction bioTransaction = new BioTransaction(); bioTransaction.setBiospecimen(biospecimen); bioTransaction.setTransactionDate(Calendar.getInstance().getTime()); bioTransaction.setQuantity(biospecimen.getQuantity()); bioTransaction.setReason(au.org.theark.lims.web.Constants.BIOTRANSACTION_STATUS_INITIAL_QUANTITY); BioTransactionStatus initialStatus = iLimsService.getBioTransactionStatusByName( au.org.theark.lims.web.Constants.BIOTRANSACTION_STATUS_INITIAL_QUANTITY); bioTransaction.setStatus(initialStatus); //ensure that the initial transaction can be identified bioTransactions.add(bioTransaction); biospecimen.setBioTransactions(bioTransactions); //validation SHOULD make sure these cases will work. TODO: test scripts if (study.getAutoGenerateBiospecimenUid()) { biospecimen.setBiospecimenUid(iLimsService.getNextGeneratedBiospecimenUID(study)); } else { biospecimen.setBiospecimenUid(biospecimenUID); } insertBiospecimens.add(biospecimen); StringBuffer sb = new StringBuffer(); sb.append("Biospecimen UID: "); sb.append(biospecimen.getBiospecimenUid()); sb.append(" has been created successfully."); sb.append("\n"); uploadReport.append(sb); insertCount++; // Allocation details String siteName = csvReader.get("SITE"); String freezerName = csvReader.get("FREEZER"); String rackName = csvReader.get("RACK"); String boxName = csvReader.get("BOX"); String row = csvReader.get("ROW"); String column = csvReader.get("COLUMN"); InvCell invCell = iInventoryService.getInvCellByLocationNames(siteName, freezerName, rackName, boxName, row, column); //Biospecimen was supposed to locate in the following valid, empty inventory cell // inventory cell is not persist with biospeciman. So we have to update the valid inventory cell location with the //biospecimen uid which we will do it while bispecimen creates. biospecimen.setInvCell(invCell); recordCount++; } } catch (IOException ioe) { uploadReport.append("Unexpected I/O exception whilst reading the biospecimen data file\n"); log.error("processMatrixBiospecimenFile IOException stacktrace:", ioe); throw new ArkSystemException("Unexpected I/O exception whilst reading the biospecimen data file"); } catch (Exception ex) { uploadReport.append("Unexpected exception whilst reading the biospecimen data file\n"); log.error("processMatrixBiospecimenFile Exception stacktrace:", ex); throw new ArkSystemException( "Unexpected exception occurred when trying to process biospecimen data file"); } finally { // Clean up the IO objects timer.stop(); uploadReport.append("\n"); uploadReport.append("Total elapsed time: "); uploadReport.append(timer.getTime()); uploadReport.append(" ms or "); uploadReport.append(decimalFormat.format(timer.getTime() / 1000.0)); uploadReport.append(" s"); uploadReport.append("\n"); uploadReport.append("Total file size: "); uploadReport.append(inLength); uploadReport.append(" B or "); uploadReport.append(decimalFormat.format(inLength / 1024.0 / 1024.0)); uploadReport.append(" MB"); uploadReport.append("\n"); if (timer != null) timer = null; if (csvReader != null) { try { csvReader.close(); } catch (Exception ex) { log.error("Cleanup operation failed: csvRdr.close()", ex); } } if (inputStreamReader != null) { try { inputStreamReader.close(); } catch (Exception ex) { log.error("Cleanup operation failed: isr.close()", ex); } } // Restore the state of variables srcLength = -1; } uploadReport.append("Processed "); uploadReport.append(recordCount); uploadReport.append(" records."); uploadReport.append("\n"); uploadReport.append("Inserted "); uploadReport.append(insertCount); uploadReport.append(" records."); uploadReport.append("\n"); uploadReport.append("Updated "); uploadReport.append(updateCount); uploadReport.append(" records."); uploadReport.append("\n"); // Batch insert/update iLimsService.batchInsertBiospecimensAndUpdateInventoryCell(insertBiospecimens); return uploadReport; }
From source file:au.org.theark.study.util.DataUploader.java
public StringBuffer uploadAndReportSubjectConsentDataFile(InputStream inputStream, long size, String fileFormat, char delimChar) throws FileFormatException, ArkSystemException { uploadReport = new StringBuffer(); long rowCount = 0; long insertFieldsCount = 0; long updateFieldsCount = 0; List<Consent> consentFieldsToUpdate = new ArrayList<Consent>(); List<Consent> consentFieldsToInsert = new ArrayList<Consent>(); delimiterCharacter = delimChar;/*from w ww. j a v a 2 s. com*/ InputStreamReader inputStreamReader = null; CsvReader csvReader = null; DecimalFormat decimalFormat = new DecimalFormat("0.00"); if (fileFormat.equalsIgnoreCase("XLS")) { Workbook w; try { w = Workbook.getWorkbook(inputStream); delimiterCharacter = ','; XLStoCSV xlsToCsv = new XLStoCSV(delimiterCharacter); inputStream = xlsToCsv.convertXlsToCsv(w); inputStream.reset(); } catch (BiffException e) { log.error(e.getMessage()); } catch (IOException e) { log.error(e.getMessage()); } } try { inputStreamReader = new InputStreamReader(inputStream); csvReader = new CsvReader(inputStreamReader, delimiterCharacter); csvReader.readHeaders(); String[] stringLineArray; List<StudyComp> studyComList = iArkCommonService.getStudyComponentByStudy(study); Map<String, StudyComp> studyCompMap = new HashMap<String, StudyComp>(); for (StudyComp studuComp : studyComList) { studyCompMap.put(studuComp.getName().toUpperCase(), studuComp); } List<StudyCompStatus> studyCompStatusList = iArkCommonService.getStudyComponentStatus(); Map<String, StudyCompStatus> studyCompStatusMap = new HashMap<String, StudyCompStatus>(); for (StudyCompStatus studyCompStatus : studyCompStatusList) { studyCompStatusMap.put(studyCompStatus.getName().toUpperCase(), studyCompStatus); } List<ConsentType> consentTypeList = iArkCommonService.getConsentType(); Map<String, ConsentType> consentTypeMap = new HashMap<String, ConsentType>(); for (ConsentType consentType : consentTypeList) { consentTypeMap.put(consentType.getName().toUpperCase(), consentType); } List<ConsentStatus> consentStatusList = iArkCommonService.getConsentStatus(); Map<String, ConsentStatus> consentStatusMap = new HashMap<String, ConsentStatus>(); for (ConsentStatus consentStatus : consentStatusList) { consentStatusMap.put(consentStatus.getName().toUpperCase(), consentStatus); } List<YesNo> consentDownloadedList = iArkCommonService.getYesNoList(); Map<String, YesNo> consentDownloadedMap = new HashMap<String, YesNo>(); for (YesNo consentDownloaded : consentDownloadedList) { consentDownloadedMap.put(consentDownloaded.getName().toUpperCase(), consentDownloaded); } ConsentVO consentVO = new ConsentVO(); consentVO.getConsent().setStudy(study); int subjectUidIndex = csvReader.getIndex("SUBJECTUID"); int studyComponentIndex = csvReader.getIndex("STUDY_COMPONENT"); int studyComponentStatusIndex = csvReader.getIndex("STUDY_COMPONENT_STATUS"); int consentTypeIndex = csvReader.getIndex("CONSENT_TYPE"); int consentStatusIndex = csvReader.getIndex("CONSENT_STATUS"); int consentDownloadedIndex = csvReader.getIndex("CONSENT_DOWNLOADED"); int consentedByIndex = csvReader.getIndex("CONSENTED_BY"); int consentDateIndex = csvReader.getIndex("CONSENT_DATE"); int commentIndex = csvReader.getIndex("COMMENT"); int completedDateIndex = csvReader.getIndex("COMPLETED_DATE"); while (csvReader.readRecord()) { ++rowCount; stringLineArray = csvReader.getValues(); String subjectUID = stringLineArray[subjectUidIndex]; LinkSubjectStudy subject = iArkCommonService.getSubjectByUID(subjectUID, study); consentVO.getConsent().setLinkSubjectStudy(subject); consentVO.getConsent() .setStudyComp(studyCompMap.get(stringLineArray[studyComponentIndex].toUpperCase())); List<Consent> existingConcentList = iStudyService.searchConsent(consentVO); if (existingConcentList.size() > 0) { ++updateFieldsCount; Consent existingConsent = existingConcentList.get(0); existingConsent.setStudyComponentStatus( studyCompStatusMap.get(stringLineArray[studyComponentStatusIndex].toUpperCase())); existingConsent .setConsentType(consentTypeMap.get(stringLineArray[consentTypeIndex].toUpperCase())); existingConsent.setConsentStatus( consentStatusMap.get(stringLineArray[consentStatusIndex].toUpperCase())); existingConsent.setConsentDownloaded( consentDownloadedMap.get(stringLineArray[consentDownloadedIndex].toUpperCase())); if (stringLineArray.length > consentedByIndex) { existingConsent.setConsentedBy(stringLineArray[consentedByIndex]); } if (stringLineArray.length > consentDateIndex) { String consentDate = stringLineArray[consentDateIndex]; if (consentDate != null && consentDate.trim().length() > 0) { existingConsent.setConsentDate(simpleDateFormat.parse(consentDate)); } } if (stringLineArray.length > commentIndex) { existingConsent.setComments(stringLineArray[commentIndex]); } if ("Completed".equalsIgnoreCase(existingConsent.getStudyComponentStatus().getName())) { try { existingConsent .setCompletedDate(simpleDateFormat.parse(stringLineArray[completedDateIndex])); } catch (Exception e) { existingConsent.setCompletedDate(null); } } else { existingConsent.setCompletedDate(null); } consentFieldsToUpdate.add(existingConsent); } else { ++insertFieldsCount; Consent consent = new Consent(); consent.setStudy(study); consent.setLinkSubjectStudy(subject); consent.setStudyComp( studyCompMap.get(stringLineArray[studyComponentIndex].toUpperCase().trim())); consent.setStudyComponentStatus(studyCompStatusMap .get(stringLineArray[studyComponentStatusIndex].toUpperCase().trim())); consent.setConsentType( consentTypeMap.get(stringLineArray[consentTypeIndex].toUpperCase().trim())); consent.setConsentStatus( consentStatusMap.get(stringLineArray[consentStatusIndex].toUpperCase().trim())); consent.setConsentDownloaded( consentDownloadedMap.get(stringLineArray[consentDownloadedIndex].toUpperCase().trim())); if (stringLineArray.length > consentedByIndex) { consent.setConsentedBy(stringLineArray[consentedByIndex]); } if (stringLineArray.length > consentDateIndex) { String consentDate = stringLineArray[consentDateIndex].trim(); if (consentDate != null && consentDate.trim().length() > 0) { try { consent.setConsentDate(simpleDateFormat.parse(consentDate)); } catch (Exception e) { consent.setConsentDate(simpleDateFormat.parse(null)); } } } if (stringLineArray.length > commentIndex) { consent.setComments(stringLineArray[commentIndex].trim()); } if ("Completed".equalsIgnoreCase(consent.getStudyComponentStatus().getName())) { try { consent.setCompletedDate( simpleDateFormat.parse(stringLineArray[completedDateIndex].trim())); } catch (Exception e) { consent.setCompletedDate(null); } } consentFieldsToInsert.add(consent); } } } catch (Exception e) { e.printStackTrace(); throw new ArkSystemException(e.getMessage()); } finally { uploadReport.append("Total file size: "); uploadReport.append(decimalFormat.format(size / 1024.0 / 1024.0)); uploadReport.append(" MB"); uploadReport.append("\n"); if (csvReader != null) { try { csvReader.close(); } catch (Exception ex) { log.error("Cleanup operation failed: csvRdr.close()", ex); } } if (inputStreamReader != null) { try { inputStreamReader.close(); } catch (Exception ex) { log.error("Cleanup operation failed: isr.close()", ex); } } } uploadReport.append("Process "); uploadReport.append(rowCount); uploadReport.append(" rows of data"); uploadReport.append("\n"); uploadReport.append(insertFieldsCount); uploadReport.append(" fields were inserted."); uploadReport.append("\n"); uploadReport.append(updateFieldsCount); uploadReport.append(" fields were updated."); uploadReport.append("\n"); try { iStudyService.processSubjectConsentBatch(consentFieldsToUpdate, consentFieldsToInsert); } catch (Exception e) { e.printStackTrace(); throw new ArkSystemException(e.getMessage()); } return uploadReport; }
From source file:au.org.theark.study.util.DataUploader.java
/** * Imports the subject data file to the database tables, and creates report on the process Assumes the file is in the default "matrix" file format: * SUBJECTUID,FIELD1,FIELD2,FIELDN... 1,01/01/1900,99.99,99.99,, ... * // w w w . j a va2 s.c o m * Where N is any number of columns * * @param fileInputStream * is the input stream of a file * @param inLength * is the length of a file * @throws FileFormatException * file format Exception * @throws ArkBaseException * general ARK Exception * @return the upload report detailing the upload process **/ @SuppressWarnings("unchecked") public StringBuffer uploadAndReportMatrixSubjectFile(InputStream fileInputStream, long inLength, String inFileFormat, char inDelimChr, List<String> uidsWhichNeedUpdating) throws FileFormatException, ArkSystemException { List<LinkSubjectStudy> insertSubjects = new ArrayList<LinkSubjectStudy>(); List<LinkSubjectStudy> updateSubjects = new ArrayList<LinkSubjectStudy>(); long rowCount = 0; long subjectCount = 0; long insertCount = 0; long updateCount = 0; long srcLength = -1; // -1 means nothing being processed delimiterCharacter = inDelimChr; uploadReport = new StringBuffer(); InputStreamReader inputStreamReader = null; CsvReader csvReader = null; DecimalFormat decimalFormat = new DecimalFormat("0.00"); // If Excel, convert to CSV for validation if (inFileFormat.equalsIgnoreCase("XLS")) { Workbook w; try { w = Workbook.getWorkbook(fileInputStream); delimiterCharacter = ','; XLStoCSV xlsToCsv = new XLStoCSV(delimiterCharacter); fileInputStream = xlsToCsv.convertXlsToCsv(w); fileInputStream.reset(); } catch (BiffException e) { log.error(e.getMessage()); } catch (IOException e) { log.error(e.getMessage()); } } try { inputStreamReader = new InputStreamReader(fileInputStream); csvReader = new CsvReader(inputStreamReader, delimiterCharacter); String[] stringLineArray; // this is a list of all our somewhat enum-like reference tables... // much better to call this once than each one n times in the for loop...plus each ones default is n times // should save 200,000-250,000 selects for a 17K insert. may still wish to evaluate whats best here Collection<MaritalStatus> maritalStatiiPossible = iArkCommonService.getMaritalStatus(); Collection<EmailStatus> emailStatiiPossible = iArkCommonService.getAllEmailStatuses(); Collection<SubjectStatus> subjectStatiiPossible = iArkCommonService.getSubjectStatus(); Collection<GenderType> genderTypesPossible = iArkCommonService.getGenderTypes(); Collection<TitleType> titleTypesPossible = iArkCommonService.getTitleType(); List<AddressType> addressTypesPossible = iArkCommonService.getAddressTypes(); List<AddressStatus> addressStatiiPossible = iArkCommonService.getAddressStatuses(); List<PhoneType> phoneTypesPossible = iArkCommonService.getPhoneTypes(); List<PhoneStatus> phoneStatiiPossible = iArkCommonService.getPhoneStatuses(); List<Country> countriesPossible = iArkCommonService.getCountries(); // List<State> statesPossible = iArkCommonService.getStates(country); Collection<VitalStatus> vitalStatiiPossible = iArkCommonService.getVitalStatus(); Collection<PersonContactMethod> personContactMethodPossible = iArkCommonService .getPersonContactMethodList(); // Collection<MaritalStatus> yesNoList = iArkCommonService.getYesNoList(); //TODO would boolean not be better? YesNo yes = iArkCommonService.getYes();// TODO: boolean YesNo no = iArkCommonService.getNo(); // things inherant... "CONSENT_ID", "CONSENT_STUDY_ID", "CONSENT_LINK_SUBJECT_STUDY_ID", // things needed from file... "CONSENT_STUDY_COMP_ID", "CONSENT_STUDY_COMP_STATUS_ID", "CONSENT_CONSENT_STATUS_ID", // "CONSENT_CONSENT_TYPE_ID", "CONSENT_CONSENT_DATE", // "CONSENT_CONSENTED_BY", "CONSENT_COMMENTS", "CONSENT_REQUESTED_DATE", "CONSENT_RECEIVED_DATE", "CONSENT_COMPLETED_DATE", // "CONSENT_CONSENT_DOWNLOADED_ID"? /* * int consentComponentIndex = csvReader.getIndex("CONSENT_STUDY_COMP_ID"); int consentComponentStatusIndex = * csvReader.getIndex("CONSENT_STUDY_COMP_STATUS_ID"); int consentStatusIndex = csvReader.getIndex("CONSENT_CONSENT_STATUS_ID"); int * consentTypeIndex = csvReader.getIndex("CONSENT_CONSENT_TYPE_ID"); int consentDateIndex = csvReader.getIndex("CONSENT_CONSENT_DATE"); int * consentByIndex = csvReader.getIndex("CONSENT_CONSENTED_BY"); int consentCommentsIndex = csvReader.getIndex("CONSENT_COMMENTS"); int * consentRequestedDateIndex = csvReader.getIndex("CONSENT_REQUESTED_DATE"); int consentReceivedDateIndex = * csvReader.getIndex("CONSENT_RECEIVED_DATE"); int consentCompletedDateIndex = csvReader.getIndex("CONSENT_COMPLETED_DATE"); //??? * "CONSENT_CONSENT_DOWNLOADED_ID"; * * * * * "CONSENT_TO_ACTIVE_CONTACT_ID", "CONSENT_TO_PASSIVE_DATA_GATHERING_ID", "CONSENT_TO_USE_DATA_ID", "CONSENT_STATUS_ID", "CONSENT_TYPE_ID", * "CONSENT_DATE", "CONSENT_DOWNLOADED" consent_option c_o c_o c_status c_type date study.yes_no each of these appears to have a link to * consent_option table which has a number 1-6 for id each representing a status...there appears to be no default...therefore may need * validation * * the diffrence being consent statuys and consent type link to consent_status and consent_type tables...so look ups and validation needed */ boolean autoConsent = study.getAutoConsent(); SubjectStatus defaultSubjectStatus = iStudyService.getDefaultSubjectStatus(); TitleType defaultTitleType = iStudyService.getDefaultTitleType(); AddressType defaultAddressType = iStudyService.getDefaultAddressType(); AddressStatus defaultAddressStatus = iStudyService.getDefaultAddressStatus(); PhoneType defaultPhoneType = iStudyService.getDefaultPhoneType(); PhoneStatus defaultPhoneStatus = iStudyService.getDefaultPhoneStatus(); GenderType defaultGenderType = iStudyService.getDefaultGenderType(); VitalStatus defaultVitalStatus = iStudyService.getDefaultVitalStatus(); MaritalStatus defaultMaritalStatus = iStudyService.getDefaultMaritalStatus(); EmailStatus defaultEmailStatus = iStudyService.getDefaultEmailStatus(); ConsentOption concentOptionOfYes = iStudyService.getConsentOptionForBoolean(true);// sounds a lot like boolean blah = true???? ConsentStatus consentStatusOfConsented = iStudyService.getConsentStatusByName("Consented"); ConsentType consentTypeOfElectronic = iStudyService.getConsentTypeByName("Electronic"); List<ConsentOption> consentOptionsPossible = iStudyService.getConsentOptions(); List<ConsentStatus> consentStatusPossible = iStudyService.getConsentStatus(); List<ConsentType> consentTypePossible = iStudyService.getConsentType(); List<LinkSubjectStudy> allSubjectWhichWillBeUpdated = null; if (uidsWhichNeedUpdating.size() > 0) { // TODO analyse performance of bringing all back and having to iterate everytime, vs conditional query + looping through less // TODO analyze performance of getting that big list of UIDs and doing a .contains(subjectuid) VS getting all the entities and doing a // .getSubjectUID.equals(subjectUID) allSubjectWhichWillBeUpdated = iArkCommonService.getUniqueSubjectsWithTheseUIDs(study, uidsWhichNeedUpdating); } else { allSubjectWhichWillBeUpdated = new ArrayList(); } srcLength = inLength; if (srcLength <= 0) { uploadReport.append("ERROR: The input size was not greater than 0. Actual length reported: "); uploadReport.append(srcLength); uploadReport.append("\n"); throw new FileFormatException( "The input size was not greater than 0. Actual length reported: " + srcLength); } csvReader.readHeaders(); srcLength = inLength - csvReader.getHeaders().toString().length(); int firstNameIndex = csvReader.getIndex("FIRST_NAME"); int middleNameIndex = csvReader.getIndex("MIDDLE_NAME"); int lastNameIndex = csvReader.getIndex("LAST_NAME"); int previousLastNameIndex = csvReader.getIndex("PREVIOUS_LAST_NAME"); int preferredNameIndex = csvReader.getIndex("PREFERRED_NAME"); int preferredEmailIndex = csvReader.getIndex("EMAIL"); int preferredEmailStatusIndex = csvReader.getIndex("EMAIL_STATUS"); int otherEmailIndex = csvReader.getIndex("OTHER_EMAIL"); int otherEmailStatusIndex = csvReader.getIndex("OTHER_EMAIL_STATUS"); int heardAboutStudyIndex = csvReader.getIndex("HEARD_ABOUT_STUDY"); int commentsIndex = csvReader.getIndex("COMMENTS"); int titleIndex = csvReader.getIndex("TITLE"); int vitalStatusIndex = csvReader.getIndex("VITAL_STATUS"); int maritalStatusIndex = csvReader.getIndex("MARITAL_STATUS"); int statusIndex = csvReader.getIndex("STATUS"); int addressLine1Index = csvReader.getIndex("BUILDING_NAME"); int addressLine2Index = csvReader.getIndex("STREET_ADDRESS"); int suburbIndex = csvReader.getIndex("SUBURB"); int stateIndex = csvReader.getIndex("STATE"); int countryIndex = csvReader.getIndex("COUNTRY"); int postCodeIndex = csvReader.getIndex("POST_CODE"); int addressSourceIndex = csvReader.getIndex("ADDRESS_SOURCE"); int addressStatusIndex = csvReader.getIndex("ADDRESS_STATUS"); int addressTypeIndex = csvReader.getIndex("ADDRESS_TYPE"); int addressReceivedIndex = csvReader.getIndex("ADDRESS_DATE_RECEIVED"); int addressCommentsIndex = csvReader.getIndex("ADDRESS_COMMENTS"); int isPreferredMailingIndex = csvReader.getIndex("IS_PREFERRED_MAILING_ADDRESS"); int phoneNumberIndex = csvReader.getIndex("PHONE_NUMBER"); int areaCodeIndex = csvReader.getIndex("PHONE_AREA_CODE"); int phoneStatusIndex = csvReader.getIndex("PHONE_STATUS"); int phoneTypeIndex = csvReader.getIndex("PHONE_TYPE"); int phoneSourceIndex = csvReader.getIndex("PHONE_SOURCE"); int phoneCommentsIndex = csvReader.getIndex("PHONE_COMMENTS"); int phoneDateReceivedIndex = csvReader.getIndex("PHONE_DATE_RECEIVED"); int phoneSilentIndex = csvReader.getIndex("SILENT"); // if(PERSON_CONTACT_METHOD is in headers, use it, // else, if CONTACT_METHOD, us IT, else, just set to -1 int personContactIndex = ((csvReader.getIndex("PERSON_CONTACT_METHOD") > 0) ? csvReader.getIndex("PERSON_CONTACT_METHOD") : ((csvReader.getIndex("CONTACT_METHOD") > 0) ? csvReader.getIndex("CONTACT_METHOD") : -1)); int dateOfBirthIndex = ((csvReader.getIndex("DATE_OF_BIRTH") > 0) ? csvReader.getIndex("DATE_OF_BIRTH") : ((csvReader.getIndex("DOB") > 0) ? csvReader.getIndex("DOB") : -1)); int dateOfDeathIndex = ((csvReader.getIndex("DATE_OF_DEATH") > 0) ? csvReader.getIndex("DATE_OF_DEATH") : ((csvReader.getIndex("DODEATH") > 0) ? csvReader.getIndex("DODEATH") : -1)); int dateLastKnownAliveIndex = ((csvReader.getIndex("DATE_LAST_KNOWN_ALIVE") > 0) ? csvReader.getIndex("DATE_LAST_KNOWN_ALIVE") : ((csvReader.getIndex("LAST_KNOWN_ALIVE") > 0) ? csvReader.getIndex("LAST_KNOWN_ALIVE") : -1)); int causeOfDeathIndex = ((csvReader.getIndex("CAUSE_OF_DEATH") > 0) ? csvReader.getIndex("CAUSE_OF_DEATH") : ((csvReader.getIndex("CODEATH") > 0) ? csvReader.getIndex("CODEATH") : -1)); // in reality, validation doesnt permit this yet anyway...but probably not bad to align it over in validation int genderIndex = ((csvReader.getIndex("GENDER_TYPE") > 0) ? csvReader.getIndex("GENDER_TYPE") : ((csvReader.getIndex("GENDER") > 0) ? csvReader.getIndex("GENDER") : ((csvReader.getIndex("SEX") > 0) ? csvReader.getIndex("SEX") : -1))); boolean isAutoGen = study.getAutoGenerateSubjectUid(); while (csvReader.readRecord()) { rowCount++; LinkSubjectStudy subject = null; // Hack to ensure XLS rows contain all cells (ie empty strings for cells after the right-hand non-null value List<String> stringList = new ArrayList<String>(csvReader.getHeaders().length); for (int i = 0; i < csvReader.getHeaders().length; i++) { stringList.add(csvReader.get(i)); } stringLineArray = stringList.toArray(new String[csvReader.getHeaders().length]); String subjectUID = stringLineArray[0]; boolean hasSomeData = false; for (String next : stringLineArray) { if (next != null && !next.isEmpty()) { hasSomeData = true; } } if (!isAutoGen && (subjectUID == null || subjectUID.isEmpty())) { if (!hasSomeData) { uploadReport.append("Warning/Info: Row " + rowCount + ": There appeared to be no data on this row, so we ignored this line"); } else { // THIS SHOULD NEVER EVER HAPPEN IF VALIDATION IS CORRECT uploadReport.append("Error: Row " + rowCount + ": There is no subject UID on this row, " + "yet the study is not set up to auto generate subject UIDs. This line was ignored. Please remove this line or provide an ID"); } } else if (isAutoGen && (subjectUID == null || subjectUID.isEmpty()) && !hasSomeData) { uploadReport.append("Warning/Info: Row " + rowCount + ": There appeared to be no data on this row, so we ignored this line"); } else { subject = getSubjectByUIDFromExistList(allSubjectWhichWillBeUpdated, subjectUID); boolean thisSubjectAlreadyExists = (subject != null); Person person = null; if (thisSubjectAlreadyExists) { person = subject.getPerson(); } else { subject = new LinkSubjectStudy(); subject.setSubjectUID(subjectUID);// note: this will be overwritten IF study.isautogenerate subject.setStudy(study); person = new Person(); } if (firstNameIndex > 0) person.setFirstName(stringLineArray[firstNameIndex]); if (heardAboutStudyIndex > 0) subject.setHeardAboutStudy(stringLineArray[heardAboutStudyIndex]); if (commentsIndex > 0) subject.setComment(stringLineArray[commentsIndex]); if (middleNameIndex > 0) person.setMiddleName(stringLineArray[middleNameIndex]); if (lastNameIndex > 0) { String lastnameFromFile = stringLineArray[lastNameIndex]; if (thisSubjectAlreadyExists && lastnameFromFile != null && !lastnameFromFile.isEmpty() && !lastnameFromFile.equalsIgnoreCase(person.getLastName()) && person.getLastName() != null) { PersonLastnameHistory personLastNameHistory = new PersonLastnameHistory(); personLastNameHistory.setPerson(person); personLastNameHistory.setLastName(person.getLastName()); person.getPersonLastnameHistory().add(personLastNameHistory);// TODO analyze this } else if (!thisSubjectAlreadyExists) { if (previousLastNameIndex > 0) { String previousLastName = (stringLineArray[previousLastNameIndex]); if (previousLastName != null && !previousLastName.isEmpty()) { PersonLastnameHistory personLastNameHistory = new PersonLastnameHistory(); personLastNameHistory.setPerson(person); personLastNameHistory.setLastName(previousLastName); person.getPersonLastnameHistory().add(personLastNameHistory); } } } person.setLastName(lastnameFromFile); } else { // just in the odd instance of no last name but has previous lastname known if (!thisSubjectAlreadyExists) { if (previousLastNameIndex > 0) { String previousLastName = (stringLineArray[previousLastNameIndex]); if (previousLastName != null && !previousLastName.isEmpty()) { PersonLastnameHistory personLastNameHistory = new PersonLastnameHistory(); personLastNameHistory.setPerson(person); personLastNameHistory.setLastName(previousLastName); person.getPersonLastnameHistory().add(personLastNameHistory); } } } } if (preferredNameIndex > 0) { person.setPreferredName(stringLineArray[preferredNameIndex]); } if (genderIndex > 0) { if (stringLineArray[genderIndex] != null && stringLineArray[genderIndex].length() > 0) { for (GenderType boygirl : genderTypesPossible) { if (boygirl.getName().equalsIgnoreCase(stringLineArray[genderIndex])) { person.setGenderType(boygirl); } } } if (person.getGenderType() == null || StringUtils.isBlank(person.getGenderType().getName())) { person.setGenderType(defaultGenderType); } } if (person.getGenderType() == null) { person.setGenderType(defaultGenderType); } if (dateOfBirthIndex > 0) { Date dateOfBirth = new Date(); if (stringLineArray[dateOfBirthIndex] != null && stringLineArray[dateOfBirthIndex].length() > 0) { dateOfBirth = simpleDateFormat.parse(stringLineArray[dateOfBirthIndex]); person.setDateOfBirth(dateOfBirth); } } if (dateOfDeathIndex > 0) { Date dateOfDeath = new Date(); if (stringLineArray[dateOfDeathIndex] != null && stringLineArray[dateOfDeathIndex].length() > 0) { dateOfDeath = simpleDateFormat.parse(stringLineArray[dateOfDeathIndex]); person.setDateOfDeath(dateOfDeath); } } if (dateLastKnownAliveIndex > 0) { Date dateLastKnownAlive = new Date(); if (stringLineArray[dateLastKnownAliveIndex] != null && stringLineArray[dateLastKnownAliveIndex].length() > 0) { dateLastKnownAlive = simpleDateFormat.parse(stringLineArray[dateLastKnownAliveIndex]); person.setDateLastKnownAlive(dateLastKnownAlive); } } if (causeOfDeathIndex > 0) { if (stringLineArray[causeOfDeathIndex] != null && stringLineArray[causeOfDeathIndex].length() > 0) { person.setCauseOfDeath(stringLineArray[causeOfDeathIndex]); } } if (vitalStatusIndex > 0) { String vitalStatusStr = (stringLineArray[vitalStatusIndex]); for (VitalStatus vitalStat : vitalStatiiPossible) { if (vitalStat.getName().equalsIgnoreCase(vitalStatusStr)) { person.setVitalStatus(vitalStat); } } if (person.getVitalStatus() == null || StringUtils.isBlank(person.getVitalStatus().getName())) { person.setVitalStatus(defaultVitalStatus); } } if (person.getVitalStatus() == null) { person.setVitalStatus(defaultVitalStatus); } if (preferredEmailIndex > 0) { person.setPreferredEmail(stringLineArray[preferredEmailIndex]); } if (otherEmailIndex > 0) { person.setOtherEmail(stringLineArray[otherEmailIndex]); } if (preferredEmailStatusIndex > 0) { String preferredEmailStatusStr = (stringLineArray[preferredEmailStatusIndex]); for (EmailStatus possibleEmailStat : emailStatiiPossible) { if (possibleEmailStat.getName().equalsIgnoreCase(preferredEmailStatusStr)) { person.setPreferredEmailStatus(possibleEmailStat); } } if (person.getPreferredEmailStatus() == null || StringUtils.isBlank(person.getPreferredEmailStatus().getName())) { person.setPreferredEmailStatus(defaultEmailStatus); } } if (person.getPreferredEmailStatus() == null) { person.setPreferredEmailStatus(defaultEmailStatus); } if (otherEmailStatusIndex > 0) { String OtherEmailStatusStr = (stringLineArray[otherEmailStatusIndex]); for (EmailStatus possibleEmailStat : emailStatiiPossible) { if (possibleEmailStat.getName().equalsIgnoreCase(OtherEmailStatusStr)) { person.setOtherEmailStatus(possibleEmailStat); } } if (person.getOtherEmailStatus() == null || StringUtils.isBlank(person.getOtherEmailStatus().getName())) { person.setOtherEmailStatus(defaultEmailStatus); } } if (person.getOtherEmailStatus() == null) { person.setOtherEmailStatus(defaultEmailStatus); } if (titleIndex > 0) { String titleStr = (stringLineArray[titleIndex]); for (TitleType titleType : titleTypesPossible) { if (titleType.getName().equalsIgnoreCase(titleStr)) { person.setTitleType(titleType); } } if (person.getTitleType() == null || StringUtils.isBlank(person.getTitleType().getName())) { person.setTitleType(defaultTitleType); } } if (person.getTitleType() == null) { person.setTitleType(defaultTitleType); } if (maritalStatusIndex > 0) { String maritalStatusStr = (stringLineArray[maritalStatusIndex]); for (MaritalStatus maritalStat : maritalStatiiPossible) { if (maritalStat.getName().equalsIgnoreCase(maritalStatusStr)) { person.setMaritalStatus(maritalStat); } } if (person.getMaritalStatus() == null || StringUtils.isBlank(person.getMaritalStatus().getName())) { person.setMaritalStatus(defaultMaritalStatus); } } if (person.getMaritalStatus() == null) { person.setMaritalStatus(defaultMaritalStatus); } if (personContactIndex > 0) { String personContactMethodStr = null; personContactMethodStr = (stringLineArray[personContactIndex]); for (PersonContactMethod possibleMethod : personContactMethodPossible) { if (possibleMethod.getName().equalsIgnoreCase(personContactMethodStr)) { person.setPersonContactMethod(possibleMethod); } } // TODO if we get to the end and personcontactmethod doesnt exist...what do we do? do we want a default or does it get ignored } if (statusIndex > 0) { String statusStr = (stringLineArray[statusIndex]); for (SubjectStatus subjectStat : subjectStatiiPossible) { if (subjectStat.getName().equalsIgnoreCase(statusStr)) { subject.setSubjectStatus(subjectStat); } } if (subject.getSubjectStatus() == null || StringUtils.isBlank(subject.getSubjectStatus().getName())) { subject.setSubjectStatus(defaultSubjectStatus); } } else { subject.setSubjectStatus(defaultSubjectStatus); } // if the study is autoconsent...then there are some defaults we have to set TODO get rid of hardcoding subject.setUpdateConsent(false); if (autoConsent && subject.getSubjectStatus().getName().equalsIgnoreCase("Subject")) { subject.setConsentDate(new Date()); subject.setConsentStatus(consentStatusOfConsented); subject.setConsentType(consentTypeOfElectronic); ConsentOption defaultConsentOption = concentOptionOfYes; subject.setConsentToActiveContact(defaultConsentOption); subject.setConsentToPassiveDataGathering(defaultConsentOption); subject.setConsentToUseData(defaultConsentOption); } else { // Manual Consent details String consentDate = csvReader.get("CONSENT_DATE"); String consentStatusStr = csvReader.get("CONSENT_STATUS"); String consentTypeStr = csvReader.get("CONSENT_TYPE"); String passiveDataStr = csvReader.get("CONSENT_TO_PASSIVE_DATA_GATHERING"); String activeContactStr = csvReader.get("CONSENT_TO_ACTIVE_CONTACT"); String useDataStr = csvReader.get("CONSENT_TO_USE_DATA"); if (!consentDate.isEmpty() || !consentStatusStr.isEmpty() || !consentTypeStr.isEmpty() || !passiveDataStr.isEmpty() || !activeContactStr.isEmpty() || !useDataStr.isEmpty()) { LinkSubjectStudy newSubject = new LinkSubjectStudy(); if (!consentDate.isEmpty()) { newSubject.setConsentDate(simpleDateFormat.parse(consentDate)); } if (!consentStatusStr.isEmpty()) { for (ConsentStatus consentStatus : consentStatusPossible) { if (consentStatus.getName().equalsIgnoreCase(consentStatusStr)) { newSubject.setConsentStatus(consentStatus); } } } if (!consentTypeStr.isEmpty()) { for (ConsentType consentType : consentTypePossible) { if (consentType.getName().equalsIgnoreCase(consentTypeStr)) { newSubject.setConsentType(consentType); } } } if (!passiveDataStr.isEmpty() || !activeContactStr.isEmpty() || !useDataStr.isEmpty()) { for (ConsentOption consentOption : consentOptionsPossible) { if (consentOption.getName().equalsIgnoreCase(passiveDataStr)) { newSubject.setConsentToPassiveDataGathering(consentOption); } if (consentOption.getName().equalsIgnoreCase(activeContactStr)) { newSubject.setConsentToActiveContact(consentOption); } if (consentOption.getName().equalsIgnoreCase(useDataStr)) { newSubject.setConsentToUseData(consentOption); } } } if (thisSubjectAlreadyExists) { // Existing Subject to compare if consent actually changed (inherently handles when no consent previously) LinkSubjectStudyConsentHistoryComparator comparator = new LinkSubjectStudyConsentHistoryComparator(); if (comparator.compare(subject, newSubject) != 0) { subject.setUpdateConsent(true); subject.setConsentDate(newSubject.getConsentDate()); subject.setConsentStatus(newSubject.getConsentStatus()); subject.setConsentType(newSubject.getConsentType()); subject.setConsentToPassiveDataGathering( newSubject.getConsentToPassiveDataGathering()); subject.setConsentToActiveContact(newSubject.getConsentToActiveContact()); subject.setConsentToUseData(newSubject.getConsentToUseData()); } else { subject.setUpdateConsent(false); } } else { // New Subject with consent details subject.setConsentDate(newSubject.getConsentDate()); subject.setConsentStatus(newSubject.getConsentStatus()); subject.setConsentType(newSubject.getConsentType()); subject.setConsentToPassiveDataGathering( newSubject.getConsentToPassiveDataGathering()); subject.setConsentToActiveContact(newSubject.getConsentToActiveContact()); subject.setConsentToUseData(newSubject.getConsentToUseData()); } } } // if no address info - ignore if (addressLine1Index > 0 || addressLine1Index > 0) { boolean updateAddress = false; String address1String = stringLineArray[addressLine1Index]; String address2String = stringLineArray[addressLine2Index]; String suburb = stringLineArray[suburbIndex]; String countryString = stringLineArray[countryIndex]; String stateString = stringLineArray[stateIndex]; String postCode = stringLineArray[postCodeIndex]; if ((address1String == null || StringUtils.isBlank(address1String)) && (address2String == null || StringUtils.isBlank(address2String)) && (suburb == null || StringUtils.isBlank(suburb)) && (postCode == null || StringUtils.isBlank(postCode)) && (countryString == null || StringUtils.isBlank(countryString)) && (stateString == null || StringUtils.isBlank(stateString))) { // then lets just jump out as there is no address to validate. lay down to user that they must have data if they want an update } else { boolean usingDefaultType = false; boolean usingDefaultStatus = false; Address addressToAttachToPerson = null; if (thisSubjectAlreadyExists) { String typeString = null; String statusString = null; if (addressTypeIndex > 0) { typeString = stringLineArray[addressTypeIndex]; if (typeString == null || typeString.isEmpty()) { typeString = defaultAddressType.getName(); usingDefaultType = true; } } if (addressStatusIndex > 0) { statusString = stringLineArray[addressStatusIndex]; if (statusString == null || statusString.isEmpty()) { statusString = defaultPhoneStatus.getName(); usingDefaultStatus = true; } } for (Address a : person.getAddresses()) { if (a.getAddressStatus().getName().equalsIgnoreCase(statusString) && a.getAddressType().getName().equalsIgnoreCase(typeString)) { addressToAttachToPerson = a; updateAddress = true; } } } if (addressToAttachToPerson == null) { log.info("address was null"); addressToAttachToPerson = new Address(); } else { log.info("address was not null"); } AddressType type = findAddressType(addressTypesPossible, stringLineArray[addressTypeIndex]); if (type == null) { type = defaultAddressType; usingDefaultType = true; } AddressStatus status = findAddressStatus(addressStatiiPossible, stringLineArray[addressTypeIndex]); if (status == null) { status = defaultAddressStatus; usingDefaultStatus = true; } String addressComments = stringLineArray[addressCommentsIndex]; Country country = findCountry(countriesPossible, countryString); if (country != null) { addressToAttachToPerson.setCountry(country); // TODO one option: all possible states locally and test where it matches might work...or lets see how the entity goes first, // and if it hits db again! // State state = findState(statesPossible, stateString, country); State state = findStateWithinThisCountry(stateString, country); if (state == null) { uploadReport.append("Warning/Info: could not find a state named '" + stateString + "' in " + country.getName() + " for row " + rowCount + ", but will proceed.\n"); addressToAttachToPerson.setOtherState(stateString); } else { addressToAttachToPerson.setState(state); } } else { uploadReport.append("Warning/Info: Could not find country '" + countryString + " for row " + rowCount + ", but will proceed.\n"); } String addressSource = stringLineArray[addressSourceIndex]; String dateReceivedString = stringLineArray[addressReceivedIndex]; String isPreferredMailingString = stringLineArray[isPreferredMailingIndex]; addressToAttachToPerson.setAddressType(type); addressToAttachToPerson.setAddressStatus(status); if (postCode != null && !postCode.isEmpty()) addressToAttachToPerson.setPostCode(postCode); if (address1String != null && !address1String.isEmpty()) addressToAttachToPerson.setAddressLineOne(address1String); if (address2String != null && !address2String.isEmpty()) addressToAttachToPerson.setStreetAddress(address2String);// yeah.. if (dateReceivedString != null && !dateReceivedString.isEmpty()) { // TODO dateconvert and set Date dateReceived = new Date(); dateReceived = simpleDateFormat.parse(dateReceivedString); addressToAttachToPerson.setDateReceived(dateReceived); } if (suburb != null && !suburb.isEmpty()) addressToAttachToPerson.setCity(suburb); if (addressComments != null && !addressComments.isEmpty()) addressToAttachToPerson.setComments(addressComments); if (DataConversionAndManipulationHelper .isSomethingLikeABoolean(isPreferredMailingString)) { if (DataConversionAndManipulationHelper .isSomethingLikeTrue(isPreferredMailingString)) { // isPreferredMailingString!=null && // !isPreferredMailingString.isEmpty()){ addressToAttachToPerson.setPreferredMailingAddress(true); } else { addressToAttachToPerson.setPreferredMailingAddress(false); } } else { addressToAttachToPerson.setPreferredMailingAddress(false); } if (usingDefaultStatus && usingDefaultType) { uploadReport .append("Info: Using the default status '" + defaultAddressStatus.getName() + "' and the default type '" + defaultAddressType.getName() + " for row " + rowCount + ", but will proceed.\n"); } else if (usingDefaultType) { uploadReport.append("Info: Using the default type '" + defaultAddressType.getName() + "' for row " + rowCount + ", but will proceed.\n"); } else if (usingDefaultStatus) { uploadReport .append("Info: Using the default status '" + defaultAddressStatus.getName() + " for row " + rowCount + ", but will proceed.\n"); } if (addressSource != null && !addressSource.isEmpty()) addressToAttachToPerson.setSource(addressSource); if (!updateAddress) { // TODO check this works in all cases addressToAttachToPerson.setPerson(person); person.getAddresses().add(addressToAttachToPerson); } } } // if no address info - ignore if (phoneNumberIndex > 0) { boolean updatePhones = false; boolean usingDefaultType = false; boolean usingDefaultStatus = false; String phoneNumberString = stringLineArray[phoneNumberIndex]; if (phoneNumberString == null || StringUtils.isBlank(phoneNumberString)) { // then lets just jump out as there is no phone to validate. lay down to user that they must have data if they want an update } else { Phone phoneToAttachToPerson = null; if (thisSubjectAlreadyExists) { String typeString = null; String statusString = null; if (phoneTypeIndex > 0) { typeString = stringLineArray[phoneTypeIndex]; if (typeString == null || typeString.isEmpty()) { typeString = defaultPhoneType.getName(); usingDefaultType = true; } } if (phoneStatusIndex > 0) { statusString = stringLineArray[phoneStatusIndex]; if (statusString == null || statusString.isEmpty()) { statusString = defaultPhoneStatus.getName(); usingDefaultStatus = true; } } for (Phone phone : person.getPhones()) { if (phone.getPhoneStatus().getName().equalsIgnoreCase(statusString) && phone.getPhoneType().getName().equalsIgnoreCase(typeString)) { phoneToAttachToPerson = phone; updatePhones = true; } } } if (phoneToAttachToPerson == null) { phoneToAttachToPerson = new Phone(); } PhoneType type = findPhoneTypeOrSetDefault(phoneTypesPossible, defaultPhoneType, stringLineArray[phoneTypeIndex]); PhoneStatus status = findPhoneStatusOrSetDefault(phoneStatiiPossible, defaultPhoneStatus, stringLineArray[phoneTypeIndex]); String phoneComments = stringLineArray[phoneCommentsIndex]; String areaCode = stringLineArray[areaCodeIndex]; String silentString = stringLineArray[phoneSilentIndex]; String phoneSource = stringLineArray[phoneSourceIndex]; String phoneDateReceivedString = stringLineArray[phoneDateReceivedIndex]; // log.warn("phone Date Reveived = " + phoneDateReceivedString + " for index = " + phoneDateReceivedIndex); phoneToAttachToPerson.setPhoneType(type); phoneToAttachToPerson.setPhoneStatus(status); if (areaCode != null && !areaCode.isEmpty()) phoneToAttachToPerson.setAreaCode(areaCode); if (phoneNumberString != null && !phoneNumberString.isEmpty()) phoneToAttachToPerson.setPhoneNumber(phoneNumberString); if (phoneDateReceivedString != null && !phoneDateReceivedString.isEmpty()) { // TODO dateconvert and set Date dateReceived = new Date(); dateReceived = simpleDateFormat.parse(phoneDateReceivedString); phoneToAttachToPerson.setDateReceived(dateReceived); } if (DataConversionAndManipulationHelper.isSomethingLikeABoolean(silentString)) { if (DataConversionAndManipulationHelper.isSomethingLikeTrue(silentString)) { phoneToAttachToPerson.setSilentMode(yes); } else { phoneToAttachToPerson.setSilentMode(no); } } if (phoneComments != null && !phoneComments.isEmpty()) phoneToAttachToPerson.setComment(phoneComments); if (phoneSource != null && !phoneSource.isEmpty()) phoneToAttachToPerson.setSource(phoneSource); if (usingDefaultStatus && usingDefaultType) { uploadReport .append("Info: Using the default status '" + defaultAddressStatus.getName() + "' and the default type '" + defaultAddressType.getName() + " for row " + rowCount + ", but will proceed.\n"); } else if (usingDefaultType) { uploadReport.append("Info: Using the default type '" + defaultAddressType.getName() + "' for row " + rowCount + ", but will proceed.\n"); } else if (usingDefaultStatus) { uploadReport .append("Info: Using the default status '" + defaultAddressStatus.getName() + " for row " + rowCount + ", but will proceed.\n"); } if (!updatePhones) { // TODO check this works in all cases phoneToAttachToPerson.setPerson(person); person.getPhones().add(phoneToAttachToPerson); } } } /* * * * //if no address info - ignore if(consentByIndex >0 && consentCompletedDateIndex >0 && consentDateIndex >0 && * consentCompletedDateIndex >0 && consentCompletedDateIndex >0 && consentCompletedDateIndex >0 && consentCompletedDateIndex >0 && * consentCompletedDateIndex >0 ){ * * * * * boolean updatePhones= false; boolean usingDefaultType = false; boolean usingDefaultStatus = false; String phoneNumberString = * stringLineArray[phoneNumberIndex]; * * if(phoneNumberString == null || StringUtils.isBlank(phoneNumberString)){ //then lets just jump out as there is no phone to validate. * lay down to user that they must have data if they want an update } else{ Phone phoneToAttachToPerson = null; * if(thisSubjectAlreadyExists){ String typeString = null; String statusString = null; * * if (phoneTypeIndex > 0){ typeString = stringLineArray[phoneTypeIndex]; if(typeString==null || typeString.isEmpty()){ typeString = * defaultPhoneType.getName(); usingDefaultType = true; } } if (phoneStatusIndex > 0){ statusString = * stringLineArray[phoneStatusIndex]; if(statusString==null || statusString.isEmpty()){ statusString = defaultPhoneStatus.getName(); * usingDefaultStatus = true; } } for(Phone phone : person.getPhones()){ * if(phone.getPhoneStatus().getName().equalsIgnoreCase(statusString) && phone.getPhoneType().getName().equalsIgnoreCase(typeString)){ * phoneToAttachToPerson = phone; updatePhones = true; } } } if(phoneToAttachToPerson==null){ phoneToAttachToPerson = new Phone(); } * * PhoneType type = findPhoneTypeOrSetDefault(phoneTypesPossible, defaultPhoneType, stringLineArray[phoneTypeIndex]); PhoneStatus * status = findPhoneStatusOrSetDefault(phoneStatiiPossible, defaultPhoneStatus, stringLineArray[phoneTypeIndex]); String phoneComments * = stringLineArray[phoneCommentsIndex]; * * String areaCode = stringLineArray[areaCodeIndex]; String silentString = stringLineArray[phoneSilentIndex]; String phoneSource = * stringLineArray[phoneSourceIndex]; String phoneDateReceivedString = stringLineArray[phoneDateReceivedIndex]; * //log.warn("phone Date Reveived = " + phoneDateReceivedString + " for index = " + phoneDateReceivedIndex); * * phoneToAttachToPerson.setPhoneType(type); phoneToAttachToPerson.setPhoneStatus(status); if(areaCode!=null && !areaCode.isEmpty()) * phoneToAttachToPerson.setAreaCode(areaCode); if(phoneNumberString !=null && !phoneNumberString.isEmpty()) * phoneToAttachToPerson.setPhoneNumber(phoneNumberString); if(phoneDateReceivedString!=null && !phoneDateReceivedString.isEmpty()){ // * TODO dateconvert and set Date dateReceived = new Date(); dateReceived = simpleDateFormat.parse(phoneDateReceivedString); * phoneToAttachToPerson.setDateReceived(dateReceived); } * * if(DataConversionAndManipulationHelper.isSomethingLikeABoolean(silentString)){ * if(DataConversionAndManipulationHelper.isSomethingLikeTrue(silentString)){ phoneToAttachToPerson.setSilentMode(yes); } else{ * phoneToAttachToPerson.setSilentMode(no); } } if(phoneComments!=null && !phoneComments.isEmpty()) * phoneToAttachToPerson.setComment(phoneComments); if(phoneSource!=null && !phoneSource.isEmpty()) * phoneToAttachToPerson.setSource(phoneSource); * * * if(usingDefaultStatus && usingDefaultType){ uploadReport.append("Info: Using the default status '" + defaultAddressStatus.getName() * + "' and the default type '" + defaultAddressType.getName() + " for row " + rowCount + ", but will proceed.\n"); } else * if(usingDefaultType){ uploadReport.append("Info: Using the default type '" + defaultAddressType.getName() + "' for row " + rowCount * + ", but will proceed.\n"); } else if(usingDefaultStatus){ uploadReport.append("Info: Using the default status '" + * defaultAddressStatus.getName() + " for row " + rowCount + ", but will proceed.\n"); } * * if(!updatePhones){ //TODO check this works in all cases phoneToAttachToPerson.setPerson(person); * person.getPhones().add(phoneToAttachToPerson); } } } */ subject.setPerson(person); if (subject.getId() == null || subject.getPerson().getId() == 0) { insertSubjects.add(subject); /* * StringBuffer sb = new StringBuffer(); //does this report have to happen? ... and in reality it hasnt had success yet * sb.append("\nCreated subject from original Subject UID: "); sb.append(subject.getSubjectUID()); * //sb.append(" has been created successfully and linked to the study: "); //sb.append(study.getName()); //sb.append("\n"); * uploadReport.append(sb); */ insertCount++; } else { // iStudyService.updateSubject(subjectVo); updateSubjects.add(subject); /* * StringBuffer sb = new StringBuffer(); sb.append("\nUpdate subject with Subject UID: "); sb.append(subject.getSubjectUID()); * //sb.append(" has been updated successfully and linked to the study: "); //sb.append(study.getName()); //sb.append("\n"); * uploadReport.append(sb); */ updateCount++; } subjectCount++; } } } catch (IOException ioe) { uploadReport.append("System Error: Unexpected I/O exception whilst reading the subject data file\n"); log.error("processMatrixSubjectFile IOException stacktrace:", ioe); throw new ArkSystemException("Unexpected I/O exception whilst reading the subject data file"); } catch (Exception ex) { uploadReport.append("System Error: Unexpected exception whilst reading the subject data file\n"); log.error("processMatrixSubjectFile Exception stacktrace:", ex); throw new ArkSystemException("Unexpected exception occurred when trying to process subject data file"); } finally { uploadReport.append("Total file size: "); uploadReport.append(decimalFormat.format(inLength / 1024.0 / 1024.0)); uploadReport.append(" MB"); uploadReport.append("\n"); if (csvReader != null) { try { csvReader.close(); } catch (Exception ex) { log.error("Cleanup operation failed: csvRdr.close()", ex); } } if (inputStreamReader != null) { try { inputStreamReader.close(); } catch (Exception ex) { log.error("Cleanup operation failed: isr.close()", ex); } } // Restore the state of variables srcLength = -1; } /* * uploadReport.append("Processed "); uploadReport.append(subjectCount); uploadReport.append(" rows for "); uploadReport.append(subjectCount); * uploadReport.append(" subjects."); uploadReport.append("\n"); uploadReport.append(insertCount); * uploadReport.append(" fields were inserted."); uploadReport.append("\n"); uploadReport.append(updateCount); * uploadReport.append(" fields were updated."); uploadReport.append("\n"); */ uploadReport.append("Processed "); uploadReport.append(subjectCount); uploadReport.append(" rows."); uploadReport.append("\n"); uploadReport.append("Inserted "); uploadReport.append(insertCount); uploadReport.append(" subjects."); uploadReport.append("\n"); uploadReport.append("Updated "); uploadReport.append(updateCount); uploadReport.append(" subjects."); uploadReport.append("\n"); // TODO better exceptionhandling iStudyService.processBatch(insertSubjects, study, updateSubjects); return uploadReport; }