Example usage for org.apache.commons.csv CSVFormat MYSQL

Introduction

In this page you can find the example usage for org.apache.commons.csv CSVFormat MYSQL.

Prototype

CSVFormat MYSQL

To view the source code for org.apache.commons.csv CSVFormat MYSQL.

Click Source Link

Document

Default MySQL format used by the SELECT INTO OUTFILE and LOAD DATA INFILE operations.

Usage

From source file:com.lithium.flow.util.CsvFormats.java

@Nonnull
public static CSVFormat fromConfig(@Nonnull Config config) {
    checkNotNull(config);/*  w  w  w  . jav  a 2s .co  m*/
    switch (config.getString("csv.format", "default")) {
    case "default":
        return CSVFormat.DEFAULT;
    case "excel":
        return CSVFormat.EXCEL;
    case "mysql":
        return CSVFormat.MYSQL;
    case "rfc4180":
        return CSVFormat.RFC4180;
    case "tdf":
        return CSVFormat.TDF;
    case "custom":
        return CSVFormat.newFormat(getChar(config, "csv.delimiter", ','))
                .withAllowMissingColumnNames(getBoolean(config, "csv.allowMissingColumnNames"))
                .withCommentMarker(getChar(config, "csv.commentMarker"))
                .withEscape(getChar(config, "csv.escape")).withHeader(getHeader(config, "csv.header"))
                .withIgnoreEmptyLines(getBoolean(config, "csv.ignoreEmptyLines"))
                .withIgnoreSurroundingSpaces(getBoolean(config, "csv.ignoreSurroundingSpaces"))
                .withNullString(getString(config, "csv.nullString")).withQuote(getChar(config, "csv.quote"))
                .withQuoteMode(getQuoteMode(config, "csv.quoteMode"))
                .withRecordSeparator(getString(config, "csv.recordSeparator"))
                .withSkipHeaderRecord(getBoolean(config, "csv.skipHeaderRecord"));
    default:
        return CSVFormat.DEFAULT;
    }
}

---

From source file:co.cask.hydrator.transforms.ParseCSV.java

@Override
public void initialize(TransformContext context) throws Exception {
    super.initialize(context);

    String csvFormatString = config.format.toLowerCase();
    switch (csvFormatString) {
    case "default":
        csvFormat = CSVFormat.DEFAULT;// ww w  .java2 s  .c  om
        break;

    case "excel":
        csvFormat = CSVFormat.EXCEL;
        break;

    case "mysql":
        csvFormat = CSVFormat.MYSQL;
        break;

    case "rfc4180":
        csvFormat = CSVFormat.RFC4180;
        break;

    case "tdf":
        csvFormat = CSVFormat.TDF;
        break;

    default:
        throw new IllegalArgumentException(
                "Format {} specified is not one of the allowed format. Allowed formats are"
                        + "DEFAULT, EXCEL, MYSQL, RFC4180 and TDF");
    }

    try {
        outSchema = Schema.parseJson(config.schema);
        fields = outSchema.getFields();
    } catch (IOException e) {
        throw new IllegalArgumentException("Format of schema specified is invalid. Please check the format.");
    }
}

---

From source file:co.cask.hydrator.transforms.CSVParser2.java

@Override
public void initialize(TransformContext context) throws Exception {
    super.initialize(context);

    String csvFormatString = config.format.toLowerCase();
    switch (csvFormatString) {
    case "default":
        csvFormat = CSVFormat.DEFAULT;// ww w. jav a  2  s.  c  o m
        break;

    case "excel":
        csvFormat = CSVFormat.EXCEL;
        break;

    case "mysql":
        csvFormat = CSVFormat.MYSQL;
        break;

    case "rfc4180":
        csvFormat = CSVFormat.RFC4180;
        break;

    case "tdf":
        csvFormat = CSVFormat.TDF;
        break;

    default:
        throw new IllegalArgumentException(
                "Format {} specified is not one of the allowed format. Allowed formats are"
                        + "DEFAULT, EXCEL, MYSQL, RFC4180 and TDF");
    }

    if (config.field == null || config.field.isEmpty()) {
        throw new IllegalArgumentException("Field for applying transformation is not specified.");
    }

    try {
        outSchema = Schema.parseJson(config.schema);
        fields = outSchema.getFields();
    } catch (IOException e) {
        throw new IllegalArgumentException("Format of schema specified is invalid. Please check the format.");
    }
}

---

From source file:com.linkedin.pinot.core.data.readers.CSVRecordReader.java

private CSVFormat getFormatFromConfig() {
    String format = (_config != null) ? _config.getCsvFileFormat() : null;

    if (format == null) {
        return CSVFormat.DEFAULT;
    }//from   w w w .  j a v  a 2  s  .com

    format = format.toUpperCase();
    if ((format.equals("DEFAULT"))) {
        return CSVFormat.DEFAULT;

    } else if (format.equals("EXCEL")) {
        return CSVFormat.EXCEL;

    } else if (format.equals("MYSQL")) {
        return CSVFormat.MYSQL;

    } else if (format.equals("RFC4180")) {
        return CSVFormat.RFC4180;

    } else if (format.equals("TDF")) {
        return CSVFormat.TDF;
    } else {
        return CSVFormat.DEFAULT;
    }
}

---

From source file:com.streamsets.pipeline.lib.jdbc.JdbcLoadRecordWriter.java

@Override
public List<OnRecordErrorException> writeBatch(Iterator<Record> recordIterator) throws StageException {
    final List<OnRecordErrorException> errorRecords = new LinkedList<>();
    if (!recordIterator.hasNext()) {
        return errorRecords;
    }//from  w w  w.j a v a2  s  . c  o  m

    // Assume all records have the same columns.
    final Record first = recordIterator.next();
    SortedMap<String, String> columnsToParameters = recordReader.getColumnsToParameters(first,
            OperationType.LOAD_CODE, getColumnsToParameters(), getColumnsToFields());
    if (columnsToParameters.isEmpty()) {
        throw new StageException(JdbcErrors.JDBC_22);
    }

    final Set<String> columnNames = columnsToParameters.keySet();
    final String loadSql = "LOAD DATA LOCAL INFILE '' " + duplicateKeyAction.getKeyword() + " INTO TABLE "
            + getTableName() + " (" + Joiner.on(", ").join(columnNames) + ")";
    try (Connection connection = getDataSource().getConnection()) {
        Connection conn = connection.unwrap(Connection.class);
        try (PreparedStatement statement = conn.prepareStatement(loadSql)) {
            PipedInputStream is = new PipedInputStream();
            PipedOutputStream os = new PipedOutputStream(is);
            statement.getClass().getMethod("setLocalInfileInputStream", InputStream.class).invoke(statement,
                    is);

            Future<?> future = loadOutputExecutor.submit(() -> {
                try (OutputStreamWriter writer = new OutputStreamWriter(os)) {
                    CSVPrinter printer = new CSVPrinter(writer, CSVFormat.MYSQL);
                    Record record = first;
                    while (record != null) {
                        int opCode = getOperationCode(record, errorRecords);
                        if (opCode == OperationType.LOAD_CODE) {
                            for (String column : columnNames) {
                                Field field = record.get(getColumnsToFields().get(column));
                                printer.print(field.getValue());
                            }
                            printer.println();
                        } else if (opCode > 0) {
                            LOG.debug("Sending record to error due to unsupported operation {}", opCode);
                            errorRecords.add(new OnRecordErrorException(record, JdbcErrors.JDBC_70, opCode));
                        } else {
                            // It should be added to the error records.
                        }
                        record = recordIterator.hasNext() ? recordIterator.next() : null;
                    }
                    ;
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            });

            if (LOG.isDebugEnabled()) {
                LOG.debug("Executing query: {}", statement.toString());
            }
            statement.execute();
            future.get();
        }
        connection.commit();
    } catch (SQLException e) {
        handleSqlException(e);
    } catch (Exception e) {
        throw new StageException(JdbcErrors.JDBC_14, e.getMessage(), e);
    }
    return errorRecords;
}

---

From source file:co.cask.hydrator.plugin.CSVParser.java

@Override
public void initialize(TransformContext context) throws Exception {
    super.initialize(context);

    String csvFormatString = config.format.toLowerCase();
    switch (csvFormatString) {
    case "default":
        csvFormat = CSVFormat.DEFAULT;//from   w ww.  j av  a 2s. c om
        break;

    case "excel":
        csvFormat = CSVFormat.EXCEL;
        break;

    case "mysql":
        csvFormat = CSVFormat.MYSQL;
        break;

    case "rfc4180":
        csvFormat = CSVFormat.RFC4180;
        break;

    case "tdf":
        csvFormat = CSVFormat.TDF;
        break;

    case "pdl":
        csvFormat = PDL;
        break;

    default:
        throw new IllegalArgumentException(
                "Format {} specified is not one of the allowed format. Allowed formats are"
                        + "DEFAULT, EXCEL, MYSQL, RFC4180, PDL and TDF");
    }

    try {
        outSchema = Schema.parseJson(config.schema);
        fields = outSchema.getFields();
    } catch (IOException e) {
        throw new IllegalArgumentException("Format of schema specified is invalid. Please check the format.");
    }
}

---

From source file:com.ggvaidya.scinames.ui.DatasetImporterController.java

private Dataset loadDataset() throws IOException {
    String format = fileFormatComboBox.getSelectionModel().getSelectedItem();
    CSVFormat csvFormat = null;/* ww w  .  j  a  v  a  2s  .  co  m*/
    if (format == null) {
        csvFormat = CSVFormat.DEFAULT;
    } else {
        switch (format) {
        case "List of names":
            return Checklist.fromListInFile(currentFile);
        case "Default CSV":
            csvFormat = CSVFormat.DEFAULT;
            break;
        case "Microsoft Excel CSV":
            csvFormat = CSVFormat.EXCEL;
            break;
        case "RFC 4180 CSV":
            csvFormat = CSVFormat.RFC4180;
            break;
        case "Oracle MySQL CSV":
            csvFormat = CSVFormat.MYSQL;
            break;
        case "Tab-delimited file":
            csvFormat = CSVFormat.TDF;
            break;
        case "TaxDiff file":
            return ChecklistDiff.fromTaxDiffFile(currentFile);
        case "Excel file":
            return new ExcelImporter(currentFile).asDataset(0);
        }
    }

    if (csvFormat == null) {
        LOGGER.info("Could not determine CSV format from format '" + format + "', using CSV default.");
        csvFormat = CSVFormat.DEFAULT;
    }

    return Dataset.fromCSV(csvFormat, currentFile);
}

---

From source file:com.hurence.logisland.service.cache.CSVKeyValueCacheService.java

@Override
// @OnEnabled//from  w w  w  . j a va  2 s.  c  om
public void init(ControllerServiceInitializationContext context) throws InitializationException {
    super.init(context);
    try {

        if (context.getPropertyValue(DATABASE_FILE_URI).isSet()) {
            dbUri = context.getPropertyValue(DATABASE_FILE_URI).asString();
        }

        if (context.getPropertyValue(DATABASE_FILE_PATH).isSet()) {
            dbPath = context.getPropertyValue(DATABASE_FILE_PATH).asString();
        }

        if ((dbUri == null) && (dbPath == null)) {
            throw new Exception(
                    "You must declare " + DATABASE_FILE_URI.getName() + " or " + DATABASE_FILE_PATH.getName());
        }

        InputStream is = null;
        if (dbUri != null) {
            logger.info("opening csv database from hdfs : " + dbUri);
            is = initFromUri(dbUri);
        }

        if (dbPath != null) {
            logger.info("opening csv database from local fs : " + dbPath);
            is = initFromPath(context, dbPath);
        }

        if (is == null) {
            throw new InitializationException("Something went wrong while initializing csv db from "
                    + DATABASE_FILE_URI.getName() + " or " + DATABASE_FILE_PATH.getName());
        }

        // final Reader reader = new InputStreamReader(is);
        CSVFormat format = CSVFormat.DEFAULT;
        if (context.getPropertyValue(CSV_FORMAT).asString().equals(CSV_EXCEL.getValue())) {
            format = CSVFormat.EXCEL;
        } else if (context.getPropertyValue(CSV_FORMAT).asString().equals(CSV_EXCEL_FR.getValue())) {
            format = CSVFormat.EXCEL.withDelimiter(';');
        } else if (context.getPropertyValue(CSV_FORMAT).asString().equals(CSV_MYSQL.getValue())) {
            format = CSVFormat.MYSQL;
        } else if (context.getPropertyValue(CSV_FORMAT).asString().equals(CSV_RFC4180.getValue())) {
            format = CSVFormat.RFC4180;
        } else if (context.getPropertyValue(CSV_FORMAT).asString().equals(CSV_TDF.getValue())) {
            format = CSVFormat.TDF;
        }

        if (context.getPropertyValue(CSV_HEADER).isSet()) {
            String[] columnNames = context.getPropertyValue(CSV_HEADER).asString().split(",");
            for (String name : columnNames) {
                headers.get().put(name, "string");
            }
            format = format.withHeader(columnNames);
        } else if (context.getPropertyValue(FIRST_LINE_HEADER).isSet()) {
            format = format.withFirstRecordAsHeader();
        } else {
            throw new InitializationException("unable to get headers from somewhere");
        }

        Charset charset = Charset.forName("UTF-8");
        if (context.getPropertyValue(ENCODING_CHARSET).isSet()) {
            String encoding = context.getPropertyValue(ENCODING_CHARSET).asString();
            charset = Charset.forName(encoding);
        }

        rowKey = context.getPropertyValue(ROW_KEY).asString();
        CSVParser parser = CSVParser.parse(is, charset, format); //new CSVParser(reader, format);

        /*
        *    CSVParser parser = null;
                
        if (context.getPropertyValue(ENCODING_CHARSET).isSet()) {
        String encoding = context.getPropertyValue(ENCODING_CHARSET).asString();
        parser = CSVParser.parse(reader, Charset.forName(encoding), format);
        } else {
        parser = CSVParser.parse(reader, format);
        }
        */
        long count = 0;
        try {
            final Set<String> columnNames = parser.getHeaderMap().keySet();
            for (final CSVRecord record : parser) {

                Record logislandRecord = new StandardRecord();
                for (final String column : columnNames) {
                    logislandRecord.setStringField(column, record.get(column));
                }

                set(logislandRecord.getField(rowKey).asString(), logislandRecord);
                count++;
            }
        } finally {
            logger.info("successfully loaded " + count + " records from CSV file");

            parser.close();
            is.close();
        }

    } catch (Exception e) {
        getLogger().error("Could not load database file: {}", new Object[] { e.getMessage() });
        throw new InitializationException(e);
    }
}

---

From source file:norbert.mynemo.dataimport.fileformat.input.MovieLensIdConverter.java

/**
 * Loads the mapping file.//from   w  ww.  ja va  2s.  c  o  m
 *
 * <p>
 * The columns of the mapping file are:
 * <ol>
 * <li>MovieLens id of the movie</li>
 * <li>rating</li>
 * <li>average</li>
 * <li>IMDb id of the movie</li>
 * <li>title and year</li>
 * </ol>
 *
 * @param mappingFilepath the file that contains the mapping
 */
public MovieLensIdConverter(String mappingFilepath) throws IOException {
    checkArgument(new File(mappingFilepath).exists(), "The mapping file must exist.");

    CSVParser parser = new CSVParser(
            new CleanRatingsReader(new BufferedReader(new FileReader(mappingFilepath))), CSVFormat.MYSQL);

    mappings = new HashMap<>();
    for (CSVRecord record : parser) {
        if (record.size() != RECORD_SIZE) {
            parser.close();
            throw new IllegalStateException("Error: unable to parse the movie file \"" + mappingFilepath
                    + "\". A list of five tab separated values is expected. Approximate" + " line number: "
                    + record.getRecordNumber());
        }
        mappings.put(record.get(MOVIELENS_MOVIE_ID_INDEX), record.get(IMDB_MOVIE_ID_INDEX));
    }

    parser.close();
}

---

From source file:norbert.mynemo.dataimport.fileformat.input.MovieLensRatingImporter.java

private static CSVParser createParser(String filepath) throws IOException {
    return new CSVParser(new CleanRatingsReader(new BufferedReader(new FileReader(filepath))), CSVFormat.MYSQL);
}

---

• «
• 1
• 2
• »