Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.internal.csv; import java.io.IOException; import java.io.Reader; import java.util.ArrayList; /** * Parses CSV files according to the specified configuration. * * Because CSV appears in many different dialects, the parser supports many * configuration settings by allowing the specification of a {@link CSVStrategy}. * * <p>Parsing of a csv-string having tabs as separators, * '"' as an optional value encapsulator, and comments starting with '#':</p> * <pre> * String[][] data = * (new CSVParser(new StringReader("a\tb\nc\td"), new CSVStrategy('\t','"','#'))).getAllValues(); * </pre> * * <p>Parsing of a csv-string in Excel CSV format</p> * <pre> * String[][] data = * (new CSVParser(new StringReader("a;b\nc;d"), CSVStrategy.EXCEL_STRATEGY)).getAllValues(); * </pre> * * <p> * Internal parser state is completely covered by the strategy * and the reader-state.</p> * * <p>see <a href="package-summary.html">package documentation</a> * for more details</p> */ public class CSVParser { /** length of the initial token (content-)buffer */ private static final int INITIAL_TOKEN_LENGTH = 50; // the token types /** Token has no valid content, i.e. is in its initialized state. */ protected static final int TT_INVALID = -1; /** Token with content, at beginning or in the middle of a line. */ protected static final int TT_TOKEN = 0; /** Token (which can have content) when end of file is reached. */ protected static final int TT_EOF = 1; /** Token with content when end of a line is reached. */ protected static final int TT_EORECORD = 2; /** Immutable empty String array. */ private static final String[] EMPTY_STRING_ARRAY = new String[0]; // the input stream private final ExtendedBufferedReader in; private final CSVStrategy strategy; // the following objects are shared to reduce garbage /** A record buffer for getLine(). Grows as necessary and is reused. */ private final ArrayList record = new ArrayList(); private final Token reusableToken = new Token(); private final CharBuffer wsBuf = new CharBuffer(); private final CharBuffer code = new CharBuffer(4); /** * Token is an internal token representation. * * It is used as contract between the lexer and the parser. */ static class Token { /** Token type, see TT_xxx constants. */ int type = TT_INVALID; /** The content buffer. */ CharBuffer content = new CharBuffer(INITIAL_TOKEN_LENGTH); /** Token ready flag: indicates a valid token with content (ready for the parser). */ boolean isReady; Token reset() { content.clear(); type = TT_INVALID; isReady = false; return this; } } // ====================================================== // the constructor // ====================================================== /** * CSV parser using the default {@link CSVStrategy}. * * @param input a Reader containing "csv-formatted" input */ public CSVParser(Reader input) { this(input, CSVStrategy.DEFAULT_STRATEGY); } /** * Customized CSV parser using the given {@link CSVStrategy} * * @param input a Reader containing "csv-formatted" input * @param strategy the CSVStrategy used for CSV parsing */ public CSVParser(Reader input, CSVStrategy strategy) { this.in = new ExtendedBufferedReader(input); this.strategy = strategy; } // ====================================================== // the parser // ====================================================== /** * Parses the CSV according to the given strategy * and returns the content as an array of records * (whereas records are arrays of single values). * <p> * The returned content starts at the current parse-position in * the stream. * * @return matrix of records x values ('null' when end of file) * @throws IOException on parse error or input read-failure */ public String[][] getAllValues() throws IOException { ArrayList records = new ArrayList(); String[] values; String[][] ret = null; while ((values = getLine()) != null) { records.add(values); } if (records.size() > 0) { ret = new String[records.size()][]; records.toArray(ret); } return ret; } /** * Parses the CSV according to the given strategy * and returns the next csv-value as string. * * @return next value in the input stream ('null' when end of file) * @throws IOException on parse error or input read-failure */ public String nextValue() throws IOException { Token tkn = nextToken(); String ret = null; switch (tkn.type) { case TT_TOKEN: case TT_EORECORD: ret = tkn.content.toString(); break; case TT_EOF: ret = null; break; case TT_INVALID: default: // error no token available (or error) throw new IOException("(line " + getLineNumber() + ") invalid parse sequence"); // unreachable: break; } return ret; } /** * Parses from the current point in the stream til * the end of the current line. * * @return array of values til end of line * ('null' when end of file has been reached) * @throws IOException on parse error or input read-failure */ public String[] getLine() throws IOException { String[] ret = EMPTY_STRING_ARRAY; record.clear(); while (true) { reusableToken.reset(); nextToken(reusableToken); switch (reusableToken.type) { case TT_TOKEN: record.add(reusableToken.content.toString()); break; case TT_EORECORD: record.add(reusableToken.content.toString()); break; case TT_EOF: if (reusableToken.isReady) { record.add(reusableToken.content.toString()); } else { ret = null; } break; case TT_INVALID: default: // error: throw IOException throw new IOException("(line " + getLineNumber() + ") invalid parse sequence"); // unreachable: break; } if (reusableToken.type != TT_TOKEN) { break; } } if (!record.isEmpty()) { ret = (String[]) record.toArray(new String[record.size()]); } return ret; } /** * Returns the current line number in the input stream. * * ATTENTION: in case your csv has multiline-values the returned * number does not correspond to the record-number * * @return current line number */ public int getLineNumber() { return in.getLineNumber(); } // ====================================================== // the lexer(s) // ====================================================== /** * Convenience method for <code>nextToken(null)</code>. */ protected Token nextToken() throws IOException { return nextToken(new Token()); } /** * Returns the next token. * * A token corresponds to a term, a record change or an * end-of-file indicator. * * @param tkn an existing Token object to reuse. The caller is responsible to initialize the * Token. * @return the next token found * @throws IOException on stream access error */ protected Token nextToken(Token tkn) throws IOException { wsBuf.clear(); // resuse // get the last read char (required for empty line detection) int lastChar = in.readAgain(); // read the next char and set eol /* note: unfortunately isEndOfLine may consumes a character silently. * this has no effect outside of the method. so a simple workaround * is to call 'readAgain' on the stream... * uh: might using objects instead of base-types (jdk1.5 autoboxing!) */ int c = in.read(); boolean eol = isEndOfLine(c); c = in.readAgain(); // empty line detection: eol AND (last char was EOL or beginning) while (strategy.getIgnoreEmptyLines() && eol && (lastChar == '\n' || lastChar == ExtendedBufferedReader.UNDEFINED) && !isEndOfFile(lastChar)) { // go on char ahead ... lastChar = c; c = in.read(); eol = isEndOfLine(c); c = in.readAgain(); // reached end of file without any content (empty line at the end) if (isEndOfFile(c)) { tkn.type = TT_EOF; return tkn; } } // did we reached eof during the last iteration already ? TT_EOF if (isEndOfFile(lastChar) || (lastChar != strategy.getDelimiter() && isEndOfFile(c))) { tkn.type = TT_EOF; return tkn; } // important: make sure a new char gets consumed in each iteration while (!tkn.isReady && tkn.type != TT_EOF) { // ignore whitespaces at beginning of a token while (strategy.getIgnoreLeadingWhitespaces() && isWhitespace(c) && !eol) { wsBuf.append((char) c); c = in.read(); eol = isEndOfLine(c); } // ok, start of token reached: comment, encapsulated, or token if (c == strategy.getCommentStart()) { // ignore everything till end of line and continue (incr linecount) in.readLine(); tkn = nextToken(tkn.reset()); } else if (c == strategy.getDelimiter()) { // empty token return TT_TOKEN("") tkn.type = TT_TOKEN; tkn.isReady = true; } else if (eol) { // empty token return TT_EORECORD("") //noop: tkn.content.append(""); tkn.type = TT_EORECORD; tkn.isReady = true; } else if (c == strategy.getEncapsulator()) { // consume encapsulated token encapsulatedTokenLexer(tkn, c); } else if (isEndOfFile(c)) { // end of file return TT_EOF() //noop: tkn.content.append(""); tkn.type = TT_EOF; tkn.isReady = true; } else { // next token must be a simple token // add removed blanks when not ignoring whitespace chars... if (!strategy.getIgnoreLeadingWhitespaces()) { tkn.content.append(wsBuf); } simpleTokenLexer(tkn, c); } } return tkn; } /** * A simple token lexer * * Simple token are tokens which are not surrounded by encapsulators. * A simple token might contain escaped delimiters (as \, or \;). The * token is finished when one of the following conditions become true: * <ul> * <li>end of line has been reached (TT_EORECORD)</li> * <li>end of stream has been reached (TT_EOF)</li> * <li>an unescaped delimiter has been reached (TT_TOKEN)</li> * </ul> * * @param tkn the current token * @param c the current character * @return the filled token * * @throws IOException on stream access error */ private Token simpleTokenLexer(Token tkn, int c) throws IOException { for (;;) { if (isEndOfLine(c)) { // end of record tkn.type = TT_EORECORD; tkn.isReady = true; break; } else if (isEndOfFile(c)) { // end of file tkn.type = TT_EOF; tkn.isReady = true; break; } else if (c == strategy.getDelimiter()) { // end of token tkn.type = TT_TOKEN; tkn.isReady = true; break; } else if (c == '\\' && strategy.getUnicodeEscapeInterpretation() && in.lookAhead() == 'u') { // interpret unicode escaped chars (like \u0070 -> p) tkn.content.append((char) unicodeEscapeLexer(c)); } else if (c == strategy.getEscape()) { tkn.content.append((char) readEscape(c)); } else { tkn.content.append((char) c); } c = in.read(); } if (strategy.getIgnoreTrailingWhitespaces()) { tkn.content.trimTrailingWhitespace(); } return tkn; } /** * An encapsulated token lexer * * Encapsulated tokens are surrounded by the given encapsulating-string. * The encapsulator itself might be included in the token using a * doubling syntax (as "", '') or using escaping (as in \", \'). * Whitespaces before and after an encapsulated token are ignored. * * @param tkn the current token * @param c the current character * @return a valid token object * @throws IOException on invalid state */ private Token encapsulatedTokenLexer(Token tkn, int c) throws IOException { // save current line int startLineNumber = getLineNumber(); // ignore the given delimiter // assert c == delimiter; for (;;) { c = in.read(); if (c == '\\' && strategy.getUnicodeEscapeInterpretation() && in.lookAhead() == 'u') { tkn.content.append((char) unicodeEscapeLexer(c)); } else if (c == strategy.getEscape()) { tkn.content.append((char) readEscape(c)); } else if (c == strategy.getEncapsulator()) { if (in.lookAhead() == strategy.getEncapsulator()) { // double or escaped encapsulator -> add single encapsulator to token c = in.read(); tkn.content.append((char) c); } else { // token finish mark (encapsulator) reached: ignore whitespace till delimiter for (;;) { c = in.read(); if (c == strategy.getDelimiter()) { tkn.type = TT_TOKEN; tkn.isReady = true; return tkn; } else if (isEndOfFile(c)) { tkn.type = TT_EOF; tkn.isReady = true; return tkn; } else if (isEndOfLine(c)) { // ok eo token reached tkn.type = TT_EORECORD; tkn.isReady = true; return tkn; } else if (!isWhitespace(c)) { // error invalid char between token and next delimiter throw new IOException("(line " + getLineNumber() + ") invalid char between encapsulated token end delimiter"); } } } } else if (isEndOfFile(c)) { // error condition (end of file before end of token) throw new IOException( "(startline " + startLineNumber + ")" + "eof reached before encapsulated token finished"); } else { // consume character tkn.content.append((char) c); } } } /** * Decodes Unicode escapes. * * Interpretation of "\\uXXXX" escape sequences * where XXXX is a hex-number. * @param c current char which is discarded because it's the "\\" of "\\uXXXX" * @return the decoded character * @throws IOException on wrong unicode escape sequence or read error */ protected int unicodeEscapeLexer(int c) throws IOException { int ret = 0; // ignore 'u' (assume c==\ now) and read 4 hex digits c = in.read(); code.clear(); try { for (int i = 0; i < 4; i++) { c = in.read(); if (isEndOfFile(c) || isEndOfLine(c)) { throw new NumberFormatException("number too short"); } code.append((char) c); } ret = Integer.parseInt(code.toString(), 16); } catch (NumberFormatException e) { throw new IOException("(line " + getLineNumber() + ") Wrong unicode escape sequence found '" + code.toString() + "'" + e.toString()); } return ret; } private int readEscape(int c) throws IOException { // assume c is the escape char (normally a backslash) c = in.read(); int out; switch (c) { case 'r': out = '\r'; break; case 'n': out = '\n'; break; case 't': out = '\t'; break; case 'b': out = '\b'; break; case 'f': out = '\f'; break; default: out = c; } return out; } // ====================================================== // strategies // ====================================================== /** * Obtain the specified CSV Strategy. This should not be modified. * * @return strategy currently being used */ public CSVStrategy getStrategy() { return this.strategy; } // ====================================================== // Character class checker // ====================================================== /** * @return true if the given char is a whitespace character */ private boolean isWhitespace(int c) { return Character.isWhitespace((char) c) && (c != strategy.getDelimiter()); } /** * Greedy - accepts \n and \r\n * This checker consumes silently the second control-character... * * @return true if the given character is a line-terminator */ private boolean isEndOfLine(int c) throws IOException { // check if we have \r\n... if (c == '\r') { if (in.lookAhead() == '\n') { // note: does not change c outside of this method !! c = in.read(); } } return (c == '\n'); } /** * @return true if the given character indicates end of file */ private boolean isEndOfFile(int c) { return c == ExtendedBufferedReader.END_OF_STREAM; } }