List of usage examples for java.io IOException toString
public String toString()
From source file:gov.nih.nci.ncicb.cadsr.common.security.jboss.DBLoginModule.java
protected String[] getUsernameAndPassword() throws LoginException { String[] info = { null, null }; if (callbackHandler == null) { throw new LoginException("Error: no CallbackHandler available to collect authentication information"); }// w w w . j av a2s. c o m NameCallback nc = new NameCallback("User name: ", "guest"); PasswordCallback pc = new PasswordCallback("Password: ", false); Callback[] callbacks = { nc, pc }; String username = null; String password = null; try { callbackHandler.handle(callbacks); username = nc.getName(); char[] tmpPassword = pc.getPassword(); if (tmpPassword != null) { credential = new char[tmpPassword.length]; System.arraycopy(tmpPassword, 0, credential, 0, tmpPassword.length); pc.clearPassword(); password = new String(credential); } } catch (IOException ioe) { throw new LoginException(ioe.toString()); } catch (UnsupportedCallbackException uce) { throw new LoginException("CallbackHandler does not support: " + uce.getCallback()); } info[0] = username; info[1] = password; logger.debug("Username=" + username); return info; }
From source file:com.streamsets.pipeline.stage.destination.recordstolocalfilesystem.RecordsToLocalFileSystemTarget.java
@Override public void write(Batch batch) throws StageException { Iterator<Record> it = batch.getRecords(); try {//from w ww. j a v a2s. c o m while (it.hasNext()) { if (generator == null || hasToRotate()) { //rotating file because of rotation interval or size limit. creates new file as we need to write records //or we don't have a writer and need to create one rotate(true); } generator.write(it.next()); } if (generator != null) { generator.flush(); } if (hasToRotate()) { // rotating file because of rotation interval in case of empty batches. new file is not created. rotate(false); } } catch (IOException ex) { throw new StageException(Errors.RECORDFS_05, activeFile, ex.toString(), ex); } }
From source file:de.escidoc.core.common.business.TripleStoreConnector.java
/** * @param spoQuery/* w w w .j a v a2 s . c o m*/ * @param outputFormat * @throws TripleStoreSystemException TODO move to TriplestoreUtility implementation * @throws de.escidoc.core.common.exceptions.application.invalid.InvalidTripleStoreOutputFormatException * @throws de.escidoc.core.common.exceptions.application.invalid.InvalidTripleStoreQueryException * @return */ public String requestMPT(final String spoQuery, final String outputFormat) throws TripleStoreSystemException, InvalidTripleStoreOutputFormatException, InvalidTripleStoreQueryException { try { final List<NameValuePair> params = new ArrayList<NameValuePair>(); params.add(new BasicNameValuePair("format", outputFormat)); params.add(new BasicNameValuePair("query", spoQuery)); params.add(new BasicNameValuePair("type", TYPE_MPT)); params.add(new BasicNameValuePair("lang", LANG_MPT)); // The flush parameter tells the resource index to ensure // that any recently-added/modified/deleted triples are // flushed to the triplestore before executing the query. params.add(new BasicNameValuePair("flush", FLUSH)); final String url = this.fedoraUrl + "/risearch"; final HttpPost httpPost = new HttpPost(url); final HttpEntity entity = new UrlEncodedFormEntity(params, XmlUtility.CHARACTER_ENCODING); httpPost.setEntity(entity); connectionUtility.setAuthentication(new URL(url), this.fedoraUser, this.fedoraPassword); final HttpResponse httpResponse = connectionUtility.getHttpClient(url).execute(httpPost); String responseContent = connectionUtility.readResponse(httpResponse).trim(); if (responseContent == null || responseContent.length() == 0) { return null; } if (responseContent.startsWith("<html")) { final Pattern p = Pattern.compile(QUERY_ERROR); final Matcher m = p.matcher(responseContent); final Pattern p1 = Pattern.compile(PARSE_ERROR); final Matcher m1 = p1.matcher(responseContent); final Pattern p2 = Pattern.compile(FORMAT_ERROR); final Matcher m2 = p2.matcher(responseContent); if (m.find()) { LOGGER.error(responseContent); responseContent = XmlUtility.CDATA_START + responseContent + XmlUtility.CDATA_END; if (m1.find()) { throw new InvalidTripleStoreQueryException(responseContent); } else if (m2.find()) { throw new InvalidTripleStoreOutputFormatException(responseContent); } } else { responseContent = XmlUtility.CDATA_START + responseContent + XmlUtility.CDATA_END; throw new TripleStoreSystemException("Request to MPT failed." + responseContent); } } return responseContent; } catch (final IOException e) { throw new TripleStoreSystemException(e.toString(), e); } catch (final WebserverSystemException e) { throw new TripleStoreSystemException(e.toString(), e); } }
From source file:com.photon.phresco.nativeapp.unit.test.testcases.A_MainActivityTest.java
/** * get the application configuration from web server * *///from w w w . ja v a 2 s . c om public void testSplash() { try { PhrescoLogger.info(TAG + " testSplash -------------- START "); JSONObject appConfigJSONObj = AppConfig.getAppConfigJSONObject( Constants.getWebContextURL() + Constants.getRestAPI() + Constants.CONFIG_URL); assertNotNull(appConfigJSONObj); JSONArray appConfigArray; appConfigArray = appConfigJSONObj.getJSONArray("featureLayout"); assertTrue(appConfigArray.length() > 0); PhrescoLogger.info(TAG + " testSplash -------------- END "); } catch (IOException ex) { PhrescoLogger.info(TAG + "testSplash - IOException: " + ex.toString()); PhrescoLogger.warning(ex); } catch (JSONException ex) { PhrescoLogger.info(TAG + "testSplash - JSONException: " + ex.toString()); PhrescoLogger.warning(ex); } }
From source file:de.uzk.hki.da.service.CSVQueryHandler.java
@SuppressWarnings("serial") public void generateReportBasedOnFile(File csvFile, File outCsvFile) { logger.info("generating Report file on " + csvFile + " to " + outCsvFile); try {/*from w ww. j a v a 2 s . c o m*/ csvFileHandler.parseFile(csvFile); evalStates(); csvFileHandler.persistStates(csvFile); FolderUtils.deleteQuietlySafe(outCsvFile); FileUtils.moveFile(csvFile, outCsvFile); } catch (IOException e) { logger.error("catched " + e.toString() + " while working with " + csvFile.getAbsolutePath()); throw new RuntimeException("CSV File operations not possible " + csvFile.getAbsolutePath(), e) { }; } }
From source file:it.anyplace.sync.discovery.utils.AddressRanker.java
private int doTestTcpConnection(SocketAddress socketAddress) { logger.debug("test tcp connection to address = {}", socketAddress); Stopwatch stopwatch = Stopwatch.createStarted(); try (Socket socket = new Socket()) { socket.setSoTimeout(TCP_CONNECTION_TIMEOUT); socket.connect(socketAddress, TCP_CONNECTION_TIMEOUT); } catch (IOException ex) { logger.debug("address unreacheable = {} ({})", socketAddress, ex.toString()); logger.trace("address unreacheable", ex); return -1; }/* ww w . j a va 2s . c o m*/ int time = (int) stopwatch.elapsed(TimeUnit.MILLISECONDS); logger.debug("tcp connection to address = {} is ok, time = {} ms", socketAddress, time); return time; }
From source file:com.streamsets.pipeline.kafka.impl.KafkaValidationUtil08.java
@Override public int getPartitionCount(String metadataBrokerList, String topic, Map<String, Object> kafkaClientConfigs, int messageSendMaxRetries, long retryBackoffMs) throws StageException { List<HostAndPort> kafkaBrokers = getKafkaBrokers(metadataBrokerList); TopicMetadata topicMetadata;/* ww w . j av a 2s. c om*/ try { topicMetadata = KafkaValidationUtil08.getTopicMetadata(kafkaBrokers, topic, messageSendMaxRetries, retryBackoffMs); if (topicMetadata == null) { // Could not get topic metadata from any of the supplied brokers throw new StageException(KafkaErrors.KAFKA_03, topic, metadataBrokerList); } if (topicMetadata.errorCode() == ErrorMapping.UnknownTopicOrPartitionCode()) { // Topic does not exist throw new StageException(KafkaErrors.KAFKA_04, topic); } if (topicMetadata.errorCode() != 0) { // Topic metadata returned error code other than ErrorMapping.UnknownTopicOrPartitionCode() throw new StageException(KafkaErrors.KAFKA_03, topic, metadataBrokerList); } } catch (IOException e) { LOG.error(KafkaErrors.KAFKA_11.getMessage(), topic, kafkaBrokers, e.toString(), e); throw new StageException(KafkaErrors.KAFKA_11, topic, kafkaBrokers, e.toString()); } return topicMetadata.partitionsMetadata().size(); }
From source file:com.grendelscan.commons.http.apache_overrides.client.CustomHttpClient.java
public HttpResponse customExecute(final HttpHost target, final HttpRequest request, HttpContext context) throws HttpException { if (context == null) { context = createHttpContext();//from ww w . ja v a 2 s . co m } context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, target); prepareRequest(request); int tries = 0; HttpResponse response = null; String message = ""; while (response == null && tries++ < Scan.getScanSettings().getMaxRequestRetries()) { try { response = super.execute(target, request, context); } catch (IOException e) { message += e.toString() + " "; LOGGER.error("ERROR: Connection failure to server: " + e.toString(), e); } synchronized (failedRequestsLock) { if (response == null) { if (++failedRequests > Scan.getScanSettings().getMaxConsecutiveFailedRequests()) { Scan.getInstance().shutdown("Maximum consecutive failed requests exceeded"); } } else { failedRequests = 0; } } } if (response == null) { throw new HttpException( "Request failed." + (message.equals("") ? "" : " This might be the cause: " + message)); } return response; }
From source file:de.uzk.hki.da.service.CSVQueryHandler.java
@SuppressWarnings({ "serial", "unchecked" })
public void generateRetrievalRequests(File csvFile, File outCsvFile) {
try {// w w w .j a v a 2 s. c o m
csvFileHandler.parseFile(csvFile);
Object o = null;
for (Map<String, java.lang.Object> csvEntry : csvFileHandler.getCsvEntries()) {
logger.debug("Evaluating " + csvEntry.get("origName"));
String origName = String.valueOf(csvEntry.get("origName"));
o = fetchObject(origName);
if (o != null) {
createRetrievalJob(o);
}
}
FolderUtils.deleteQuietlySafe(outCsvFile);
FileUtils.moveFile(csvFile, outCsvFile);
} catch (IOException e) {
logger.error("catched " + e.toString() + " while working with " + csvFile.getAbsolutePath());
throw new RuntimeException("CSV File operations not possible " + csvFile.getAbsolutePath(), e) {
};
}
}
From source file:eu.sisob.uma.extractors.adhoc.email.EmailExtractor.java
/** * * @param input_file/*from w ww. j a v a2s. com*/ * @param data_dir * @param output_file * @param norepeat_output_file * @param notfound_output_file * @param notfound_norepeat_output_file * @param filters * @param error_sw */ public static void extract_emails(File input_file, File data_dir, File output_file, File norepeat_output_file, File notfound_output_file, File notfound_norepeat_output_file, List<String> filters, StringWriter error_sw) { CSVReader reader = null; try { reader = new CSVReader(new FileReader(input_file), CSV_SEPARATOR); } catch (FileNotFoundException ex) { Logger.getRootLogger().error("Error reading " + input_file.getName() + " - " + ex.toString()); } int idStaffIdentifier = -1; int idName = -1; int idFirstName = -1; int idLastName = -1; int idInitials = -1; int idUnitOfAssessment_Description = -1; int idInstitutionName = -1; int idWebAddress = -1; int idResearchGroupDescription = -1; int idResearcherWebAddress = -1; int idResearcherWebAddressType = -1; int idResearcherWebAddressExt = -1; int idScoreUrl = -1; String filter_literal = "("; for (String filter : filters) { filter_literal += filter + ","; } filter_literal += ")"; String[] nextLine; try { if ((nextLine = reader.readNext()) != null) { //Locate indexes //Locate indexes for (int i = 0; i < nextLine.length; i++) { String column_name = nextLine[i]; if (column_name.equals(FileFormatConversor.CSV_COL_ID)) idStaffIdentifier = i; else if (column_name.equals(FileFormatConversor.CSV_COL_NAME)) idName = i; else if (column_name.equals(FileFormatConversor.CSV_COL_FIRSTNAME)) idFirstName = i; else if (column_name.equals(FileFormatConversor.CSV_COL_LASTNAME)) idLastName = i; else if (column_name.equals(FileFormatConversor.CSV_COL_INITIALS)) idInitials = i; else if (column_name.equals(FileFormatConversor.CSV_COL_SUBJECT)) idUnitOfAssessment_Description = i; else if (column_name.equals(FileFormatConversor.CSV_COL_INSTITUTION_NAME)) idInstitutionName = i; else if (column_name.equals(FileFormatConversor.CSV_COL_INSTITUTION_URL)) idWebAddress = i; else if (column_name.equals(FileFormatConversor.CSV_COL_RESEARCHER_PAGE_URL)) idResearcherWebAddress = i; else if (column_name.equals(FileFormatConversor.CSV_COL_RESEARCHER_PAGE_TYPE)) idResearcherWebAddressType = i; else if (column_name.equals(FileFormatConversor.CSV_COL_RESEARCHER_PAGE_EXT)) idResearcherWebAddressExt = i; else if (column_name.equals(FileFormatConversor.CSV_COL_SCORE_URL)) idScoreUrl = i; } } } catch (Exception ex) { String error_msg = "Error reading headers of " + input_file.getName(); Logger.getRootLogger().error(error_msg + " - " + ex.toString()); if (error_sw != null) error_sw.append(error_msg + "\r\n"); return; } if (idResearcherWebAddress != -1 && idStaffIdentifier != -1 && idLastName != -1 && idInitials != -1) { //if(!test_only_output) { try { String header = ""; header += "\"" + FileFormatConversor.CSV_COL_ID + "\"" + CSV_SEPARATOR; header += "\"" + FileFormatConversor.CSV_COL_LASTNAME + "\"" + CSV_SEPARATOR; header += "\"" + FileFormatConversor.CSV_COL_INITIALS + "\"" + CSV_SEPARATOR; if (idFirstName != -1) header += "\"" + FileFormatConversor.CSV_COL_INITIALS + "\"" + CSV_SEPARATOR; if (idName != -1) header += "\"" + FileFormatConversor.CSV_COL_NAME + "\"" + CSV_SEPARATOR; header += "\"" + FileFormatConversor.CSV_COL_EMAIL + "\"" + CSV_SEPARATOR; if (idInstitutionName != -1) header += "\"" + FileFormatConversor.CSV_COL_INSTITUTION_NAME + "\"" + CSV_SEPARATOR; if (idWebAddress != -1) header += "\"" + FileFormatConversor.CSV_COL_INSTITUTION_URL + "\"" + CSV_SEPARATOR; header += "\"" + FileFormatConversor.CSV_COL_RESEARCHER_PAGE_URL + "\"" + CSV_SEPARATOR; if (idResearcherWebAddressExt != -1) header += "\"" + FileFormatConversor.CSV_COL_RESEARCHER_PAGE_EXT + "\"" + CSV_SEPARATOR; if (idResearcherWebAddressType != -1) header += "\"" + FileFormatConversor.CSV_COL_RESEARCHER_PAGE_TYPE + "\"" + CSV_SEPARATOR; if (idScoreUrl != -1) header += "\"" + FileFormatConversor.CSV_COL_SCORE_URL + "\"" + CSV_SEPARATOR; header += "\"" + FileFormatConversor.CSV_COL_SCORE_EMAIL + "\""; header += "\r\n"; FileUtils.write(output_file, header, "UTF-8", false); header = ""; header += "\"" + FileFormatConversor.CSV_COL_ID + "\"" + CSV_SEPARATOR; header += "\"" + FileFormatConversor.CSV_COL_LASTNAME + "\"" + CSV_SEPARATOR; header += "\"" + FileFormatConversor.CSV_COL_INITIALS + "\"" + CSV_SEPARATOR; if (idFirstName != -1) header += "\"" + FileFormatConversor.CSV_COL_INITIALS + "\"" + CSV_SEPARATOR; if (idName != -1) header += "\"" + FileFormatConversor.CSV_COL_NAME + "\"" + CSV_SEPARATOR; if (idInstitutionName != -1) header += "\"" + FileFormatConversor.CSV_COL_INSTITUTION_NAME + "\"" + CSV_SEPARATOR; if (idWebAddress != -1) header += "\"" + FileFormatConversor.CSV_COL_INSTITUTION_URL + "\"" + CSV_SEPARATOR; header += "\"" + FileFormatConversor.CSV_COL_RESEARCHER_PAGE_URL + "\"" + CSV_SEPARATOR; if (idResearcherWebAddressExt != -1) header += "\"" + FileFormatConversor.CSV_COL_RESEARCHER_PAGE_EXT + "\"" + CSV_SEPARATOR; if (idResearcherWebAddressType != -1) header += "\"" + FileFormatConversor.CSV_COL_RESEARCHER_PAGE_TYPE + "\"" + CSV_SEPARATOR; if (idScoreUrl != -1) header += "\"" + FileFormatConversor.CSV_COL_SCORE_URL + "\""; header += "\r\n"; FileUtils.write(notfound_output_file, header, "UTF-8", false); } catch (IOException ex) { Logger.getLogger("root").error(ex.toString()); error_sw.append("Error creating output files\r\n"); } } try { //if(!test_only_output) { Pattern p1 = Pattern.compile("([a-zA-Z0-9#._-]+)+"); while ((nextLine = reader.readNext()) != null) { nextLine[idLastName] = nextLine[idLastName].replaceAll("[^a-zA-Z]", " ").toLowerCase(); nextLine[idInitials] = nextLine[idInitials].replaceAll("[^a-zA-Z]", " ").toLowerCase(); if (idFirstName != -1) nextLine[idFirstName] = nextLine[idFirstName].replaceAll("[^a-zA-Z]", " ") .toLowerCase(); if (idName != -1) nextLine[idName] = nextLine[idName].replaceAll("[^a-zA-Z]", " ").toLowerCase(); String content = ""; String researcher_page_url = nextLine[idResearcherWebAddress]; Logger.getLogger("root").info("Go with " + researcher_page_url); if (p1.matcher(researcher_page_url).matches()) { File f = new File(data_dir, researcher_page_url); if (researcher_page_url.endsWith(".doc") || researcher_page_url.endsWith(".docx")) { Logger.getLogger("root") .error("The document " + researcher_page_url + " could not loaded"); error_sw.append("The document " + researcher_page_url + " could not loaded"); } else if (researcher_page_url.endsWith(".pdf")) { PDFParser parser = null; PDFTextStripper pdfStripper = null; PDDocument pdDoc = null; COSDocument cosDoc = null; try { parser = new PDFParser(new FileInputStream(f)); } catch (IOException e) { Logger.getLogger("root").error(e.toString()); error_sw.append("Unable to open PDF called " + researcher_page_url); } if (parser != null) { try { parser.parse(); cosDoc = parser.getDocument(); pdfStripper = new PDFTextStripper(); pdDoc = new PDDocument(cosDoc); pdfStripper.setStartPage(1); pdfStripper.setEndPage(2); content = pdfStripper.getText(pdDoc); } catch (Exception e) { Logger.getLogger("root").error(e.toString()); error_sw.append("An exception occured in parsing the PDF Document."); } finally { try { if (cosDoc != null) cosDoc.close(); if (pdDoc != null) pdDoc.close(); } catch (Exception e) { Logger.getLogger("root").error(e.toString()); } } } } } else { try { Logger.getRootLogger().info("Reading " + researcher_page_url); File temp; temp = File.createTempFile("temp-file-name", ".tmp"); URL fetched_url = Downloader.fetchURL(researcher_page_url); FileUtils.copyURLToFile(fetched_url, temp); long sizeInBytes = temp.length(); long sizeInMb = sizeInBytes / (1024 * 1024); if (sizeInMb > 100) { content = ""; } else { content = FileUtils.readFileToString(temp); temp.delete(); } } catch (Exception ex) { Logger.getLogger("root").error("" + researcher_page_url + " could not loaded", ex); error_sw.append("" + researcher_page_url + " could not loaded"); content = ""; } catch (java.lang.OutOfMemoryError ex2) { Logger.getLogger("root").error( researcher_page_url + " could not loaded (Jsoup OutOfMemoryError)", ex2); error_sw.append("" + researcher_page_url + " could not loaded"); content = ""; } } if (!content.equals("")) { //final String RE_MAIL = "([\\w\\-]([\\.\\w])+[\\w]+@([\\w\\-]+\\.)+[A-Za-z]{2,4})"; final String RE_MAIL = "([\\w\\-]([\\.\\w]){1,16}[\\w]{1,16}@([\\w\\-]{1,16}\\.){1,16}[A-Za-z]{2,4})"; Pattern p = Pattern.compile(RE_MAIL); Matcher m = p.matcher(content); List<String> emails = new ArrayList<String>(); while (m.find()) { String email = m.group(1); if (!emails.contains(email)) { // Apply filter boolean pass = true; if (filters.size() > 0) { pass = false; for (String filter : filters) { String filter2 = filter.replace("*", ".*?"); Pattern pattern = Pattern.compile(filter2); if (pattern.matcher(email).matches()) { pass = true; break; } else { } } } if (pass) { Logger.getRootLogger().info(researcher_page_url + " => " + email + " PASS FILTER! " + filter_literal); emails.add(email); } else { Logger.getRootLogger().info(researcher_page_url + " => " + email + " REFUSE BY FILTER! " + filter_literal); } } } if (emails.size() < MAX_MAIL_PER_PAGE) { for (String email : emails) { String score_email = ""; String lastname = nextLine[idLastName]; if (lastname.length() > 5) lastname = lastname.substring(0, 6); if (email.toLowerCase().contains(lastname)) { score_email = "A"; } else { int temp_id = idFirstName; if (temp_id == -1) temp_id = idInitials; if (!nextLine[idInitials].trim().equals("")) { String firstname = nextLine[temp_id].split(" ")[0]; if (firstname.length() > 5) firstname = firstname.substring(0, 5); if (firstname.length() > 1) { if (email.toLowerCase().contains(firstname)) { score_email = "A"; } } } if (score_email.equals("")) { String initials = ""; String[] arr = nextLine[temp_id].split(" "); for (int i = 0; i < arr.length; i++) { if (arr[i].length() > 0) initials += arr[i].charAt(0); } initials += nextLine[idLastName].charAt(0); if (email.toLowerCase().contains(initials)) { score_email = "B"; } else { score_email = "Z"; } } } String result = ""; result += "\"" + nextLine[idStaffIdentifier] + "\"" + CSV_SEPARATOR; result += "\"" + nextLine[idLastName] + "\"" + CSV_SEPARATOR; result += "\"" + nextLine[idInitials] + "\"" + CSV_SEPARATOR; if (idFirstName != -1) result += "\"" + nextLine[idFirstName] + "\"" + CSV_SEPARATOR; if (idName != -1) result += "\"" + nextLine[idName] + "\"" + CSV_SEPARATOR; result += "\"" + email + "\"" + CSV_SEPARATOR; if (idInstitutionName != -1) result += "\"" + nextLine[idInstitutionName] + "\"" + CSV_SEPARATOR; if (idWebAddress != -1) result += "\"" + nextLine[idWebAddress] + "\"" + CSV_SEPARATOR; result += "\"" + nextLine[idResearcherWebAddress] + "\"" + CSV_SEPARATOR; if (idResearcherWebAddressExt != -1) result += "\"" + nextLine[idResearcherWebAddressExt] + "\"" + CSV_SEPARATOR; if (idResearcherWebAddressType != -1) result += "\"" + nextLine[idResearcherWebAddressType] + "\"" + CSV_SEPARATOR; if (idScoreUrl != -1) result += "\"" + nextLine[idScoreUrl] + "\"" + CSV_SEPARATOR; result += "\"" + score_email + "\""; result += "\r\n"; try { FileUtils.write(output_file, result, "UTF-8", true); } catch (IOException ex) { Logger.getLogger("root").error(ex.toString()); } } } else { content = ""; } if (emails.size() == 0) content = ""; } if (content == "") { String result = ""; result += "\"" + nextLine[idStaffIdentifier] + "\"" + CSV_SEPARATOR; result += "\"" + nextLine[idLastName] + "\"" + CSV_SEPARATOR; result += "\"" + nextLine[idInitials] + "\"" + CSV_SEPARATOR; if (idFirstName != -1) result += "\"" + nextLine[idFirstName] + "\"" + CSV_SEPARATOR; if (idName != -1) result += "\"" + nextLine[idName] + "\"" + CSV_SEPARATOR; if (idInstitutionName != -1) result += "\"" + nextLine[idInstitutionName] + "\"" + CSV_SEPARATOR; if (idWebAddress != -1) result += "\"" + nextLine[idWebAddress] + "\"" + CSV_SEPARATOR; result += "\"" + nextLine[idResearcherWebAddress] + "\"" + CSV_SEPARATOR; if (idResearcherWebAddressExt != -1) result += "\"" + nextLine[idResearcherWebAddressExt] + "\"" + CSV_SEPARATOR; if (idResearcherWebAddressType != -1) result += "\"" + nextLine[idResearcherWebAddressType] + "\"" + CSV_SEPARATOR; if (idScoreUrl != -1) result += "\"" + nextLine[idScoreUrl] + "\""; result += "\r\n"; try { FileUtils.write(notfound_output_file, result, "UTF-8", true); } catch (IOException ex) { Logger.getLogger("root").error(ex.toString()); } } } reader.close(); } Logger.getLogger("root").info("Applying deduplication algoritm - Counting duplications"); boolean finish = false; String alternate_filename_1 = "file1"; String alternate_filename_2 = "file2"; File alternate_file_s = new File(output_file.getParentFile(), alternate_filename_1); File alternate_file_d = new File(output_file.getParentFile(), alternate_filename_2); FileUtils.copyFile(output_file, alternate_file_s); //FileUtils.write(output_file_wor_notfound, "", "UTF-8", false); FileUtils.write(norepeat_output_file, "", "UTF-8", false); while (!finish) { reader = null; try { reader = new CSVReader(new FileReader(alternate_file_s), CSV_SEPARATOR); } catch (FileNotFoundException ex) { Logger.getRootLogger() .error("Error reading " + input_file.getName() + " - " + ex.toString()); } HashMap<String, Integer> count_dictionary = new HashMap<String, Integer>(); int idEmail = 3; if (idFirstName != -1) idEmail++; if (idName != -1) idEmail++; try { FileUtils.write(alternate_file_d, "", "UTF-8", false); } catch (IOException ex) { Logger.getLogger("root").error(ex.toString()); } finish = true; while ((nextLine = reader.readNext()) != null) { Integer count = 1; if (count_dictionary.containsKey(nextLine[idEmail].toString())) count = count_dictionary.get(nextLine[idEmail].toString()); else { if (count_dictionary.size() < max_in_mem) { count_dictionary.put(nextLine[idEmail].toString(), count + 1); } else { try { for (int i = 0; i < nextLine.length; i++) nextLine[i] = "\"" + nextLine[i] + "\""; FileUtils.write(alternate_file_d, StringUtil.join(Arrays.asList(nextLine), String.valueOf(CSV_SEPARATOR)) + "\r\n", "UTF-8", true); finish = false; } catch (IOException ex) { Logger.getLogger("root").error(ex.toString()); } } } } reader.close(); Logger.getLogger("root").info("Applying deduplication algoritm - Removing duplications"); reader = null; try { reader = new CSVReader(new FileReader(alternate_file_s), CSV_SEPARATOR); } catch (FileNotFoundException ex) { Logger.getRootLogger() .error("Error reading " + input_file.getName() + " - " + ex.toString()); } String previous_id = "%previous%"; String previous_email = "%previous_email%"; List<String[]> cache = new ArrayList<String[]>(); while ((nextLine = reader.readNext()) != null) { String id = nextLine[idStaffIdentifier].toString(); if (previous_id.equals(id)) { cache.add(nextLine); previous_id = id; } else { //Process String[] winner_line = null; String max_score = "Z"; for (String[] act_line : cache) { String act_score = "Z"; try { act_score = act_line[act_line.length - 1]; } catch (Exception ex) { } String email = act_line[idEmail].toString(); if (count_dictionary.containsKey(email) && count_dictionary.get(email) > 0) { if (max_score.compareTo(act_score) > 0 && !act_score.equals("")) { winner_line = act_line; max_score = act_score; } count_dictionary.put(email, 0); } } if (winner_line != null) { try { for (int i = 0; i < winner_line.length; i++) winner_line[i] = "\"" + winner_line[i] + "\""; FileUtils.write(norepeat_output_file, StringUtil.join(Arrays.asList(winner_line), String.valueOf(CSV_SEPARATOR)) + "\r\n", "UTF-8", true); } catch (IOException ex) { Logger.getLogger("root").error(ex.toString()); } } else { // try { // FileUtils.write(output_file_wor_notfound, StringUtil.join(Arrays.asList(winner_line), String.valueOf(CSV_SEPARATOR)) + "\r\n", "UTF-8", true); // } catch (IOException ex) { // Logger.getLogger("root").error(ex.toString()); // } } cache.clear(); cache.add(nextLine); previous_id = id; } } //Process if (cache.size() > 0) { String[] winner_line = null; String max_score = "Z"; for (String[] act_line : cache) { String act_score = "Z"; try { act_score = (act_line[act_line.length - 1]); } catch (Exception ex) { } String email = act_line[idEmail]; if (count_dictionary.containsKey(email) && count_dictionary.get(email) > 0) { if (max_score.compareTo(act_score) > 0 && !act_score.equals("")) { winner_line = act_line; max_score = act_score; } count_dictionary.put(email, 0); } } if (winner_line != null) { try { for (int i = 0; i < winner_line.length; i++) winner_line[i] = "\"" + winner_line[i] + "\""; FileUtils.write(norepeat_output_file, StringUtil.join(Arrays.asList(winner_line), String.valueOf(CSV_SEPARATOR)) + "\r\n", "UTF-8", true); } catch (IOException ex) { Logger.getLogger("root").error(ex.toString()); } } else { // try { // FileUtils.write(output_file_wor_notfound, StringUtil.join(Arrays.asList(winner_line), String.valueOf(CSV_SEPARATOR)) + "\r\n", "UTF-8", true); // } catch (IOException ex) { // Logger.getLogger("root").error(ex.toString()); // } } } reader.close(); // if (!finish) { FileUtils.copyFile(alternate_file_d, alternate_file_s); alternate_file_s = new File(output_file.getParentFile(), alternate_filename_1); alternate_file_d = new File(output_file.getParentFile(), alternate_filename_2); } } FileUtils.forceDelete(alternate_file_s); FileUtils.forceDelete(alternate_file_d); Logger.getLogger("root").info("Applying deduplication algoritm - Finish"); } catch (Exception ex) { String error_msg = "Error extracting emails from extractor " + input_file.getName(); Logger.getRootLogger().error(error_msg + " - " + ex.toString()); if (error_sw != null) error_sw.append(error_msg + "\r\n"); return; } } }