List of usage examples for org.apache.commons.csv CSVFormat DEFAULT
CSVFormat DEFAULT
To view the source code for org.apache.commons.csv CSVFormat DEFAULT.
Click Source Link
From source file:com.x460dot10.b.registrar.StartupManager.java
/** * Imports data/passwords.txt into <code>University.passwords</code> * * @return Indicates import of passwords was successful * @throws IOException /*from ww w .jav a 2s .c o m*/ */ public boolean importPasswords() throws IOException { Boolean importPasswordsSuccessful = true; File file = new File("data/mockpasswords.dat"); FileReader reader = null; Object nextPassword; ArrayList<Password> filePasswords = new ArrayList<Password>(); try { reader = new FileReader(file); CSVFormat format = CSVFormat.DEFAULT; List<CSVRecord> records = new CSVParser(reader, format).getRecords(); for (CSVRecord record : records) { String idAsString = record.values[0]; Integer id = Integer.parseInt(idAsString); String userName = record.values[1]; String password = record.values[2]; nextPassword = Password.getStaticInstance(id, userName, password).clone(); filePasswords.add((Password) nextPassword); } uni.passwordManager.importPasswords(filePasswords); } catch (Exception ex) { // TODO send error message to a log file System.err.println("Error: " + ex.getMessage()); importPasswordsSuccessful = false; } finally { if (reader != null) reader.close(); } return importPasswordsSuccessful; }
From source file:eu.fthevenet.binjr.data.codec.CsvDecoder.java
@Override public Map<TimeSeriesInfo<T>, TimeSeriesProcessor<T>> decode(InputStream in, List<TimeSeriesInfo<T>> seriesInfo) throws IOException, DecodingDataFromAdapterException { try (Profiler ignored = Profiler.start("Building time series from csv data", logger::trace)) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, encoding))) { CSVFormat csvFormat = CSVFormat.DEFAULT.withAllowMissingColumnNames(false).withFirstRecordAsHeader() .withSkipHeaderRecord().withDelimiter(delimiter); Iterable<CSVRecord> records = csvFormat.parse(reader); Map<TimeSeriesInfo<T>, TimeSeriesProcessor<T>> series = new HashMap<>(); final AtomicLong nbpoints = new AtomicLong(0); for (CSVRecord csvRecord : records) { nbpoints.incrementAndGet(); ZonedDateTime timeStamp = dateParser.apply(csvRecord.get(0)); for (TimeSeriesInfo<T> info : seriesInfo) { T val = numberParser.apply(csvRecord.get(info.getBinding().getLabel())); XYChart.Data<ZonedDateTime, T> point = new XYChart.Data<>(timeStamp, val); TimeSeriesProcessor<T> l = series.computeIfAbsent(info, k -> timeSeriesFactory.create()); l.addSample(point);/*from w w w. ja va 2 s. com*/ } } logger.trace(() -> String.format("Built %d series with %d samples each (%d total samples)", seriesInfo.size(), nbpoints.get(), seriesInfo.size() * nbpoints.get())); return series; } } }
From source file:canreg.client.dataentry.Import.java
/** * * @param task//from w w w . j a v a2 s. c o m * @param doc * @param map * @param file * @param server * @param io * @return * @throws java.sql.SQLException * @throws java.rmi.RemoteException * @throws canreg.server.database.RecordLockedException */ public static boolean importFile(Task<Object, String> task, Document doc, List<canreg.client.dataentry.Relation> map, File file, CanRegServerInterface server, ImportOptions io) throws SQLException, RemoteException, SecurityException, RecordLockedException { //public static boolean importFile(canreg.client.gui.management.CanReg4MigrationInternalFrame.MigrationTask task, Document doc, List<canreg.client.dataentry.Relation> map, File file, CanRegServerInterface server, ImportOptions io) throws SQLException, RemoteException, SecurityException, RecordLockedException { boolean success = false; Set<String> noNeedToLookAtPatientVariables = new TreeSet<String>(); noNeedToLookAtPatientVariables.add(io.getPatientIDVariableName()); noNeedToLookAtPatientVariables.add(io.getPatientRecordIDVariableName()); String firstNameVariableName = io.getFirstNameVariableName(); String sexVariableName = io.getSexVariableName(); CSVParser parser = null; CSVFormat format = CSVFormat.DEFAULT.withFirstRecordAsHeader().withDelimiter(io.getSeparator()); int linesToRead = io.getMaxLines(); HashMap mpCodes = new HashMap(); int numberOfLinesRead = 0; Map<String, Integer> nameSexTable = server.getNameSexTables(); try { // FileInputStream fis = new FileInputStream(file); // BufferedReader bsr = new BufferedReader(new InputStreamReader(fis, io.getFileCharset())); // Logger.getLogger(Import.class.getName()).log(Level.CONFIG, "Name of the character encoding {0}"); int numberOfRecordsInFile = canreg.common.Tools.numberOfLinesInFile(file.getAbsolutePath()); if (linesToRead > 0) { linesToRead = Math.min(numberOfRecordsInFile, linesToRead); } else { linesToRead = numberOfRecordsInFile; } parser = CSVParser.parse(file, io.getFileCharset(), format); for (CSVRecord csvRecord : parser) { numberOfLinesRead++; // We allow for null tasks... boolean needToSavePatientAgain = true; int patientDatabaseRecordID = -1; if (task != null) { task.firePropertyChange("progress", (numberOfLinesRead - 1) * 100 / linesToRead, (numberOfLinesRead) * 100 / linesToRead); } // Build patient part Patient patient = new Patient(); for (int i = 0; i < map.size(); i++) { Relation rel = map.get(i); if (rel.getDatabaseTableVariableID() >= 0 && rel.getDatabaseTableName().equalsIgnoreCase("patient")) { if (rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getVariableType().equalsIgnoreCase("Number")) { if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) { try { patient.setVariable(rel.getDatabaseVariableName(), Integer.parseInt(csvRecord.get(rel.getFileColumnNumber()))); } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Number format error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } } } else { patient.setVariable(rel.getDatabaseVariableName(), StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber()))); } } else { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Something wrong with patient part of line " + numberOfLinesRead + ".", new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: " + rel.getDatabaseVariableName())); } } } // debugOut(patient.toString()); // Build tumour part Tumour tumour = new Tumour(); for (canreg.client.dataentry.Relation rel : map) { if (rel.getDatabaseTableVariableID() >= 0 && rel.getDatabaseTableName().equalsIgnoreCase("tumour")) { if (rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getVariableType().equalsIgnoreCase("Number")) { if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) { try { tumour.setVariable(rel.getDatabaseVariableName(), Integer.parseInt(csvRecord.get(rel.getFileColumnNumber()))); } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Number format error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } } } else { tumour.setVariable(rel.getDatabaseVariableName(), StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber()))); } } else { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Something wrong with tumour part of line " + numberOfLinesRead + ".", new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: " + rel.getDatabaseVariableName())); } } } // Build source part Set<Source> sources = Collections.synchronizedSet(new LinkedHashSet<Source>()); Source source = new Source(); for (canreg.client.dataentry.Relation rel : map) { if (rel.getDatabaseTableVariableID() >= 0 && rel.getDatabaseTableName().equalsIgnoreCase(Globals.SOURCE_TABLE_NAME)) { if (rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getVariableType().equalsIgnoreCase("Number")) { if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) { try { source.setVariable(rel.getDatabaseVariableName(), Integer.parseInt(csvRecord.get(rel.getFileColumnNumber()))); } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Number format error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } } } else { source.setVariable(rel.getDatabaseVariableName(), StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber()))); } } else { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Something wrong with source part of line " + numberOfLinesRead + ".", new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: " + rel.getDatabaseVariableName())); } } } sources.add(source); tumour.setSources(sources); // debugOut(tumour.toString()); // add patient to the database Object patientID = patient.getVariable(io.getPatientIDVariableName()); Object patientRecordID = patient.getVariable(io.getPatientRecordIDVariableName()); if (patientID == null) { // save the record to get the new patientID; patientDatabaseRecordID = server.savePatient(patient); patient = (Patient) server.getRecord(patientDatabaseRecordID, Globals.PATIENT_TABLE_NAME, false); patientID = patient.getVariable(io.getPatientIDVariableName()); patientRecordID = patient.getVariable(io.getPatientRecordIDVariableName()); } if (io.isDataFromPreviousCanReg()) { // set update date for the patient the same as for the tumour Object updateDate = tumour.getVariable(io.getTumourUpdateDateVariableName()); patient.setVariable(io.getPatientUpdateDateVariableName(), updateDate); // Set the patientID the same as the tumourID initially // Object tumourSequence = tumour.getVariable(io.getTumourSequenceVariableName()); Object tumourSequence = "1"; String tumourSequenceString = tumourSequence + ""; while (tumourSequenceString.length() < Globals.ADDITIONAL_DIGITS_FOR_PATIENT_RECORD) { tumourSequenceString = "0" + tumourSequenceString; } patientRecordID = patientID + "" + tumourSequenceString; // If this is a multiple primary tumour... String mpCodeString = (String) tumour.getVariable(io.getMultiplePrimaryVariableName()); if (mpCodeString != null && mpCodeString.length() > 0) { patientID = lookUpPatientID(mpCodeString, patientID, mpCodes); // rebuild sequenceNumber Tumour[] tumours = new Tumour[0]; try { tumours = CanRegClientApp.getApplication() .getTumourRecordsBasedOnPatientID(patientID + "", false); } catch (DistributedTableDescriptionException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (UnknownTableException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } tumourSequenceString = (tumours.length + 1) + ""; while (tumourSequenceString.length() < Globals.ADDITIONAL_DIGITS_FOR_PATIENT_RECORD) { tumourSequenceString = "0" + tumourSequenceString; } patientRecordID = patientID + "" + tumourSequenceString; Patient[] oldPatients = null; try { oldPatients = CanRegClientApp.getApplication().getPatientRecordsByID((String) patientID, false); } catch (RemoteException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (SecurityException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (DistributedTableDescriptionException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (RecordLockedException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (SQLException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (UnknownTableException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } for (Patient oldPatient : oldPatients) { if (!Tools.newRecordContainsNewInfo(patient, oldPatient, noNeedToLookAtPatientVariables)) { needToSavePatientAgain = false; patient = oldPatient; patientRecordID = oldPatient.getVariable(io.getPatientRecordIDVariableName()); } } } Object tumourID = patientRecordID + "" + tumourSequenceString; // patient.setVariable(io.getPatientIDVariableName(), patientID); tumour.setVariable(io.getTumourIDVariablename(), tumourID); // And store the record ID patient.setVariable(io.getPatientRecordIDVariableName(), patientRecordID); // Set the patient ID number on the tumour tumour.setVariable(io.getPatientIDTumourTableVariableName(), patientID); tumour.setVariable(io.getPatientRecordIDTumourTableVariableName(), patientRecordID); // Set the deprecated flag to 0 - no obsolete records from CR4 tumour.setVariable(io.getObsoleteTumourFlagVariableName(), "0"); patient.setVariable(io.getObsoletePatientFlagVariableName(), "0"); } // Set the name in the firstName database String sex = (String) patient.getVariable(sexVariableName); if (sex != null && sex.length() > 0) { Integer sexCode = Integer.parseInt(sex); String firstNames = (String) patient.getVariable(firstNameVariableName); if (firstNames != null) { String[] firstNamesArray = firstNames.split(" "); for (String firstName : firstNamesArray) { if (firstName != null && firstName.trim().length() > 0) { // here we use the locale specific toUpperCase Integer registeredSexCode = nameSexTable.get(firstName); if (registeredSexCode == null) { NameSexRecord nsr = new NameSexRecord(); nsr.setName(firstName); nsr.setSex(sexCode); server.saveNameSexRecord(nsr, false); nameSexTable.put(firstName, sexCode); } else if (registeredSexCode != sexCode) { if (registeredSexCode != 9) { sexCode = 9; NameSexRecord nsr = new NameSexRecord(); nsr.setName(firstName); nsr.setSex(sexCode); server.saveNameSexRecord(nsr, true); nameSexTable.remove(firstName); nameSexTable.put(firstName, sexCode); } } } } } } if (needToSavePatientAgain) { if (patientDatabaseRecordID > 0) { server.editPatient(patient); } else { patientDatabaseRecordID = server.savePatient(patient); } } if (patient != null && tumour != null) { String icd10 = (String) tumour.getVariable(io.getICD10VariableName()); if (icd10 == null || icd10.trim().length() == 0) { ConversionResult[] conversionResult = canreg.client.CanRegClientApp.getApplication() .performConversions(Converter.ConversionName.ICDO3toICD10, patient, tumour); tumour.setVariable(io.getICD10VariableName(), conversionResult[0].getValue()); } String iccc = (String) tumour.getVariable(io.getICCCVariableName()); if (iccc == null || iccc.trim().length() == 0) { ConversionResult[] conversionResult = canreg.client.CanRegClientApp.getApplication() .performConversions(Converter.ConversionName.ICDO3toICCC3, patient, tumour); tumour.setVariable(io.getICCCVariableName(), conversionResult[0].getValue()); } } if (tumour.getVariable(io.getPatientIDTumourTableVariableName()) == null) { tumour.setVariable(io.getPatientIDTumourTableVariableName(), patientID); } if (tumour.getVariable(io.getPatientRecordIDTumourTableVariableName()) == null) { tumour.setVariable(io.getPatientRecordIDTumourTableVariableName(), patientRecordID); } int tumourDatabaseIDNumber = server.saveTumour(tumour); if (Thread.interrupted()) { //We've been interrupted: no more importing. throw new InterruptedException(); } } task.firePropertyChange("finished", null, null); success = true; } catch (IOException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } catch (InterruptedException ex) { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Interupted on line: " + (numberOfLinesRead + 1) + ". ", ex); success = true; } catch (IndexOutOfBoundsException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } catch (SQLException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } finally { if (parser != null) { try { parser.close(); } catch (IOException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } } } return success; }
From source file:GUI.ReadFile.java
public List<Phase> readPhase(String fileName) { FileReader fileReader;/*from ww w . j a v a 2 s .c om*/ CSVParser csvFileParser; CSVFormat csvFileFormat = CSVFormat.DEFAULT.withHeader(PHASE_HEADER_MAPPING); List<Phase> phase_list = new ArrayList<>(); try { fileReader = new FileReader(fileName); //initialize CSVParser object csvFileParser = new CSVParser(fileReader, csvFileFormat); //Get a list of CSV file records List<CSVRecord> csvRecords = csvFileParser.getRecords(); //Read the CSV file records starting from the second record to skip the header for (int i = 1; i < csvRecords.size(); i++) { CSVRecord record = csvRecords.get(i); Phase p = new Phase(record.get(st_time), record.get(end_time), record.get(category)); phase_list.add(p); } fileReader.close(); csvFileParser.close(); System.out.println(fileName + " Phase file read!"); } catch (FileNotFoundException e) { System.out.println(fileName + " Phase file missing ..."); return null; } catch (IOException ex) { System.out.println(fileName + " csv file error !!!"); return null; } catch (ParseException ex) { System.out.println(fileName + " phase parsing error !!!"); return null; } return phase_list; }
From source file:io.ecarf.core.utils.UsageParser.java
public void parse() throws FileNotFoundException, IOException { for (String file : files) { try (BufferedReader reader = new BufferedReader(new FileReader(file), Constants.GZIP_BUF_SIZE);) { Iterable<CSVRecord> records = CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord().parse(reader); for (CSVRecord record : records) { String[] values = record.values(); String measurement = StringUtils.remove(values[1], MEASURE_PREFIX); this.measurementIds.add(measurement); if (!measurement.contains(CONTAINER_ENGINE_VM)) { if (measurement.contains(VM)) { this.numberOfVms++; this.vms.add(values[4]); }//from w ww . ja v a2 s.com Usage usage = this.usages.get(measurement); if (usage == null) { usage = new Usage(); this.usages.put(measurement, usage); } long value = Long.parseLong(values[2]); usage.raw += value; if (measurement.contains(VM)) { long adjusted = value; // minimum billable is 10 minutes for VMs if (adjusted < MIN_BILL) { adjusted = MIN_BILL; } // round up value to the nearest minute adjusted = (long) (MINUTE * Math.ceil(adjusted / 60.0)); usage.value += adjusted; // hourly based billing adjusted = value; if (adjusted < HOUR) { adjusted = HOUR; } else { adjusted = (long) (HOUR * Math.ceil(adjusted / 3600.0)); } usage.adjusted += adjusted; } } else { // container engine vms System.out.println(StringUtils.join(values, ',')); } } } } for (String measureId : this.measurementIds) { System.out.println(measureId); } System.out.println("Total number of VMs: " + this.numberOfVms); System.out.println(this.vms); System.out.println(this.vms.size()); for (Entry<String, Usage> entry : this.usages.entrySet()) { Usage usage = entry.getValue(); System.out.println(entry.getKey() + ',' + usage.raw + ',' + usage.value + ',' + usage.adjusted); } }
From source file:com.rodaxsoft.mailgun.CampaignManager.java
/** * Saves campaign events to a CSV file with the following format: * <code><campaign name>_(<campaign id>)_<timestamp>.csv</code> * @param campaignId The campaign ID//from www .ja va 2 s . c om * @throws ContextedException if a processing error occurs * @throws IOException if an I/O error occurs */ void saveCampaignEventsToCSV(String campaignId) throws ContextedException, IOException { Campaign campaign = getCampaign(campaignId); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); String dateTime = format.format(new Date()); String fileName; if (campaign != null) { String name = StringUtils.replace(campaign.getName(), " ", "_"); fileName = new StringBuilder(name).append("_(").append(campaignId).append(")_").append(dateTime) .append(".csv").toString(); } else { fileName = campaignId + "_" + dateTime + ".csv"; } CSVPrinter csvPrinter = null; PrintWriter pw = null; CSVFormat csvFormat = null; try { pw = new PrintWriter(fileName); final List<Map<String, Object>> events = getEvents(campaignId); for (Map<String, Object> map : events) { if (null == csvPrinter) { final Set<String> keySet = map.keySet(); int size = keySet.size(); String[] keys = keySet.toArray(new String[size]); csvFormat = CSVFormat.DEFAULT.withHeader(keys); csvPrinter = new CSVPrinter(pw, csvFormat); } // city domain tags timestamp region ip country recipient event user_vars String[] headers = csvFormat.getHeader(); for (String key : headers) { csvPrinter.print(map.get(key)); } csvPrinter.println(); } } finally { if (csvPrinter != null) { csvPrinter.flush(); } IOUtils.closeQuietly(csvPrinter); } }
From source file:com.linkedin.pinot.core.data.readers.CSVRecordReader.java
private CSVFormat getFormatFromConfig() { String format = (_config != null) ? _config.getCsvFileFormat() : null; if (format == null) { return CSVFormat.DEFAULT; }/*from ww w .j av a2 s. c om*/ format = format.toUpperCase(); if ((format.equals("DEFAULT"))) { return CSVFormat.DEFAULT; } else if (format.equals("EXCEL")) { return CSVFormat.EXCEL; } else if (format.equals("MYSQL")) { return CSVFormat.MYSQL; } else if (format.equals("RFC4180")) { return CSVFormat.RFC4180; } else if (format.equals("TDF")) { return CSVFormat.TDF; } else { return CSVFormat.DEFAULT; } }
From source file:javalibs.CSVExtractor.java
private void readCSV() { try {//from www. j av a 2 s. co m CSVParser parser = new CSVParser(Files.newBufferedReader(Paths.get(this.inCSV)), CSVFormat.DEFAULT.withHeader().withIgnoreHeaderCase().withTrim()); // Get all headers Map<String, Integer> rawHeaders = parser.getHeaderMap(); // Store the inRecords this.inRecords = parser.getRecords(); parser.close(); orderHeaders(rawHeaders); } catch (IOException e) { log_.die(e); } }
From source file:com.streamsets.pipeline.lib.csv.TestCsvParser.java
@Test public void testMaxObjectLen() throws Exception { CsvParser parser = new CsvParser(new StringReader("a,b,c\naa,bb,cc\ne,f,g\n"), CSVFormat.DEFAULT.withHeader((String[]) null).withSkipHeaderRecord(false), 6); try {/*from ww w . ja v a 2s . c o m*/ Assert.assertEquals(0, parser.getReaderPosition()); String[] record = parser.read(); Assert.assertEquals(6, parser.getReaderPosition()); Assert.assertNotNull(record); Assert.assertArrayEquals(new String[] { "a", "b", "c" }, record); try { parser.read(); Assert.fail(); } catch (ObjectLengthException ex) { } Assert.assertEquals(15, parser.getReaderPosition()); record = parser.read(); Assert.assertNotNull(record); Assert.assertArrayEquals(new String[] { "e", "f", "g" }, record); Assert.assertNull(parser.read()); } finally { parser.close(); } }
From source file:it.sayservice.platform.smartplanner.utils.LegGenerator.java
private static List<String[]> readFileGetLines(String fileName) throws IOException { FileInputStream fis = new FileInputStream(new File(fileName)); UnicodeReader ur = new UnicodeReader(fis, "UTF-8"); List<String[]> lines = new ArrayList<String[]>(); for (CSVRecord record : CSVFormat.DEFAULT.parse(ur)) { String[] line = Iterables.toArray(record, String.class); lines.add(line);/* w ww. j a v a2 s . c om*/ } lines.get(0)[0] = lines.get(0)[0].replaceAll(UTF8_BOM, ""); return lines; }