List of usage examples for org.antlr.v4.runtime CommonToken getText
@Override
public String getText()
From source file:PhpBaseLexer.java
License:Open Source License
@Override public Token nextToken() { CommonToken token = (CommonToken) super.nextToken(); if (token.getType() == PhpLexer.PHPEnd || token.getType() == PhpLexer.PHPEndSingleLineComment) { if (_mode == PhpLexer.SingleLineCommentMode) { // SingleLineCommentMode for such allowed syntax: // <?php echo "Hello world"; // comment ?> popMode(); // exit from SingleLineComment mode. }/*from www . j a v a2 s .co m*/ popMode(); // exit from PHP mode. if (token.getText().equals("</script>")) { _phpScript = false; token.setType(PhpLexer.ScriptClose); } else { // Add semicolon to the end of statement if it is absente. // For example: <?php echo "Hello world" ?> if (_prevTokenType == PhpLexer.SemiColon || _prevTokenType == PhpLexer.Colon || _prevTokenType == PhpLexer.OpenCurlyBracket || _prevTokenType == PhpLexer.CloseCurlyBracket) { token.setChannel(PhpLexer.SkipChannel); } else { token = new CommonToken(PhpLexer.SemiColon); } } } else if (token.getType() == PhpLexer.HtmlName) { _htmlNameText = token.getText(); } else if (token.getType() == PhpLexer.HtmlDoubleQuoteString) { if (token.getText().equals("php") && _htmlNameText.equals("language")) { _phpScript = true; } } else if (_mode == PhpLexer.HereDoc) { // Heredoc and Nowdoc syntax support: http://php.net/manual/en/language.types.string.php#language.types.string.syntax.heredoc switch (token.getType()) { case PhpLexer.StartHereDoc: case PhpLexer.StartNowDoc: _heredocIdentifier = token.getText().substring(3).trim().replace("\'", ""); break; case PhpLexer.HereDocText: if (CheckHeredocEnd(token.getText())) { popMode(); String heredocIdentifier = GetHeredocIdentifier(token.getText()); if (token.getText().trim().endsWith(";")) { token = new CommonToken(PhpLexer.SemiColon, heredocIdentifier + ";\n"); } else { token = (CommonToken) super.nextToken(); token.setText(heredocIdentifier + "\n;"); } } break; } } else if (_mode == PhpLexer.PHP) { if (_channel != PhpLexer.HIDDEN) { _prevTokenType = token.getType(); } } return token; }
From source file:com.github.jknack.css.CSS.java
License:Apache License
public StyleSheet parse(final String content) { CssLexer lexer = new CssLexer(new ANTLRInputStream(content)); // lexer.removeErrorListeners(); if (!debug) { lexer.addErrorListener(new ErrorReporter()); }/* w w w . ja v a 2s . c o m*/ CommonTokenStream tokens = new CommonTokenStream(lexer); tokens.fill(); CssParser parser = new CssParser(tokens); // parser.setErrorHandler(new BailErrorStrategy()); // parser.removeParseListeners(); if (!debug) { parser.addErrorListener(new ErrorReporter()); } if (debug) { for (Token tok : tokens.getTokens()) { CommonToken ct = (CommonToken) tok; String[] tokenNames = lexer.getTokenNames(); int type = ct.getType(); System.out.println((type > 0 ? tokenNames[type] : "EOF") + "(" + ct.getText() + ")"); } } StyleSheetContext tree = parser.styleSheet(); if (debug) { System.out.println(tree.toStringTree(parser)); } CSSBuilder builder = new CSSBuilder(); return (StyleSheet) builder.visit(tree); }
From source file:com.github.jknack.handlebars.internal.HbsErrorReporter.java
License:Apache License
@Override public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line, final int charPositionInLine, final String msg, final RecognitionException e) { int column = Math.max(1, charPositionInLine); CommonToken offendingToken = (CommonToken) offendingSymbol; StringBuilder message = new StringBuilder(); message.append(filename).append(":").append(line).append(":").append(column).append(": "); String stacktrace = ""; int reasonStart = message.length(); if (offendingToken == null) { String[] parts = StringUtils.split(msg, "\n"); message.append(parts[0]);//w ww . ja v a 2s. c o m stacktrace = "\n" + join(parts, "\n", 1, parts.length); } else { message.append("found: '").append(offendingToken.getText()).append("', "); message.append("expected: '").append(msg).append("'"); } String reason = message.substring(reasonStart); message.append("\n"); int evidenceStat = message.length(); String[] lines = lines(recognizer); underline(message, lines, line, column); String prevLine = lineAt(lines, line > lines.length ? lines.length : line - 2); String nextLine = lineAt(lines, line); String evidence = prevLine + "\n" + message.substring(evidenceStat) + "\n" + nextLine; message.append(stacktrace); HandlebarsError error = new HandlebarsError(filename, line, column, reason.replace("<EOF>", "EOF"), evidence, message.toString()); throw new HandlebarsException(error); }
From source file:com.github.jknack.handlebars.internal.MustacheSpec.java
License:Apache License
@Override public void enterSpaces(final SpacesContext ctx) { CommonToken space = (CommonToken) ctx.SPACE().getSymbol(); line.append(space.getText()); spaces.add(space);/*ww w. j av a 2s .com*/ }
From source file:com.github.s4ke.moar.regex.parser.RegexCompiler.java
License:Open Source License
public static Regex compile(String regexStr) { StringBuilder additionalMessage = new StringBuilder(); RegexParser parser = regexParser(regexStr); parser.getErrorListeners().clear();// ww w. jav a2s . co m parser.addErrorListener(new BaseErrorListener() { @Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { additionalMessage.append("SyntaxEception in Regex: \"").append(regexStr).append("\": ").append(msg); if (offendingSymbol instanceof CommonToken) { CommonToken token = (CommonToken) offendingSymbol; if (token.getText().equals("*") || token.getText().equals("+") || token.getText().equals("?")) { additionalMessage.append(", dangling metacharacter: '") .append(((CommonToken) offendingSymbol).getText()).append("' at line ") .append(token.getLine()).append(", pos ").append(token.getCharPositionInLine()); } } } }); RegexParser.RegexContext regexTree = parser.regex(); if (parser.getNumberOfSyntaxErrors() > 0) { throw new IllegalArgumentException( "malformed regex found : " + regexStr + "\n" + additionalMessage.toString()); } ParseTreeWalker walker = new ParseTreeWalker(); RegexGroupNameListener nameListener = new RegexGroupNameListener(); walker.walk(nameListener, regexTree); RegexTreeListener listener = new RegexTreeListener(nameListener.getGroupNames()); walker.walk(listener, regexTree); return listener.finalRegex(); }
From source file:com.globalforge.infix.FixRulesParserErrorListener.java
License:Open Source License
/** * @see BaseErrorListener#reportAmbiguity *//*from w w w. j a v a2 s .c o m*/ @Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { List<String> stack = ((Parser) recognizer).getRuleInvocationStack(); Collections.reverse(stack); String logMsg = "Parser ERROR: line " + line + ":" + charPositionInLine + " at " + offendingSymbol + ": " + msg; CommonToken tok = (CommonToken) offendingSymbol; String s = tok.getText(); logMsg += ": offending token " + s; if (s.equals("<EOF>")) { logMsg += ". Look for tag=(null or empty)."; } else { try { Integer.parseInt(s); } catch (NumberFormatException ex) { logMsg += " not a number. "; } } FixRulesParserErrorListener.logger.error(logMsg + " Tree = {}", stack); throw new RuntimeException(logMsg); }
From source file:com.jmcalc.utils.DescriptiveErrorListener.java
License:Apache License
@Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { CommonToken offendingToken = ((CommonToken) offendingSymbol); String sourceName = offendingToken.getInputStream() .getText(new Interval(0, offendingToken.getInputStream().size())); if (e != null) { throw new ExpressionParseException(sourceName + " -> line " + line + ":" + charPositionInLine + " mismatched input '" + e.getOffendingToken().getText() + "'"); } else {/*w w w. j a v a2s. c o m*/ throw new ExpressionParseException(sourceName + " -> line " + line + ":" + charPositionInLine + " unexpected token: " + offendingToken.getText()); } }
From source file:com.khubla.jvmbasic.jvmbasicc.compiler.TreePrinter.java
License:Open Source License
@Override public void token(CommonToken commonToken, int ctxlevel) { if (commonToken.getType() != -1) { logger.info(indentString(ctxlevel) + "[" + commonToken.getType() + " " + jvmBasicParser.tokenNames[commonToken.getType()] + "] " + commonToken.getText()); }/*from w w w .j a v a 2s . c om*/ }
From source file:controle.analiseTexto.AnalisePeriodo.java
public void etiquetarPadrao(String sentencaAnalisada) { ArrayList<String> palavras = new ArrayList<String>(); CommonToken ob = null; SrsGrammarLexer lexer = new SrsGrammarLexer(new ANTLRInputStream(sentencaAnalisada)); tokens = new CommonTokenStream(lexer); System.out.println(tokens.getText()); Iterator it = tokens.getTokens().iterator(); while (it.hasNext()) { ob = (CommonToken) it.next();//from w ww . j av a 2 s. c o m palavras.add(ob.getText().toLowerCase()); } tagger = new Etiquetador(palavras); tagger.etiquetar(); }
From source file:controle.dicionario.Dicionario.java
private void analisaAssunto(String nomeArquivo, String texto) throws Exception { //ArrayList<String> tags = null; ArrayList<String> tagsMorf = null; CommonTokenStream tokensMorf = null; CommonTokenStream tokens = null;//from w w w . j a va 2s . c o m Referencia referenciaBanco = referenciasHash.get(nomeArquivo); referenciaBanco.setQtd(0); referenciaDAO.atualiza(referenciaBanco); this.nomeArquivo = nomeArquivo; SrsGrammarLexer lexer = new SrsGrammarLexer(new ANTLRInputStream(texto)); tokens = new CommonTokenStream(lexer); SrsGrammarParser parser = new SrsGrammarParser(tokens); System.out.println(tokens.getText()); analisePeriodo = new AnalisePeriodo(texto); SrsGrammarLexer lexerMorf = new SrsGrammarLexer(new ANTLRInputStream(texto)); tokensMorf = new CommonTokenStream(lexerMorf); System.out.println(tokensMorf.getText()); tagsMorf = etiquetar(tokensMorf); try { Sigla sigla = new Sigla(); siglas.putAll(sigla.reconheceSiglas(tokens)); } catch (Exception e) { e.printStackTrace(); } Iterator it1 = tokens.getTokens().iterator(); CommonToken ob = null; int qtdPalavrasRelevantes = 0; int qtdRadicais = 0; while (it1.hasNext()) { ob = (CommonToken) it1.next(); //Retirando o caractere '-' do inicio da palavra if (ob.getText().length() > 1 && ob.getText().charAt(0) == '-') { ob.setText(ob.getText().substring(1)); } //Retirando o caractere '-' do fim da palavra if (ob.getText().length() > 1 && (ob.getText().charAt(ob.getText().length() - 1) == '-')) { ob.setText(ob.getText().substring(0, ob.getText().length() - 2)); } String palavra = ob.getText().toLowerCase(); String palavraLemmaa = ob.getText(); //Filtrando as palavras relevantes do texto if ((ob.getType() == SrsGrammarLexer.PALAVRA) && (ob.getText().length() > 4) && stopWordsHash.get(palavra) == null && !ehTudoIgual(palavra)) { qtdPalavrasRelevantes++; String radical; //Recuperando radical da palavra //Obtem radical utilizando o PTStemmer radical = getRadicalPTStemmer(palavra); String lemma; //Obtem lemma utilizando o TreeTagger // lemma = AnalisePeriodo.tagger.getLemma(palavraLemmaa); lemma = AnalisePeriodo.recuperaLemma(palavraLemmaa); palavraLemma.put(palavraLemmaa, lemma); Lemma lemmaBanco = new Lemma(null, lemma); Radical radicalBanco = new Radical(null, radical); Palavra palavraBanco = new Palavra(null, palavra, lemmaBanco, radicalBanco); if (!palavrasHash.containsKey(palavra)) { palavrasHash.put(palavra, palavraBanco); } if (!radicaisHash.containsKey(radical)) { radicaisHash.put(radical, radicalBanco); } if (!lemmaHash.containsKey(lemma)) { lemmaHash.put(lemma, lemmaBanco); } Dados dadosBanco = new Dados(); dadosBanco.setF(1); dadosBanco.setIdPalavra(palavraBanco); dadosBanco.setIdRadical(radicalBanco); dadosBanco.setIdReferencia(referenciaBanco); dadosBanco.setIdProjeto(projetoSelecionado); String chaveDados = projetoSelecionado.getId() + referenciaBanco.getId() + radical + palavra; if (!dadosHash.containsKey(chaveDados)) { dadosHash.put(chaveDados, dadosBanco); palavraBanco = palavrasHash.get(palavra); dadosBanco.setQtdDoc(dadosBanco.getQtdDoc() + 1); } else { Integer i = dadosHash.get(chaveDados).getF() + 1; dadosBanco.setF(i); dadosBanco.setQtdDoc(1); dadosHash.replace(chaveDados, dadosBanco); } DadoRadical dadoRadicalBanco = new DadoRadical(); dadoRadicalBanco.setF(1); dadoRadicalBanco.setIdRadical(radicalBanco); dadoRadicalBanco.setIdReferencia(referenciaBanco); dadoRadicalBanco.setIdProjeto(projetoSelecionado); String chaveDadoRadical = projetoSelecionado.getId() + referenciaBanco.getId() + radical; if (!dadoRadicalHash.containsKey(chaveDadoRadical)) { dadoRadicalHash.put(chaveDadoRadical, dadoRadicalBanco); radicalBanco = radicaisHash.get(radical); dadoRadicalBanco.setQtdDoc(dadoRadicalBanco.getQtdDoc() + 1); } else { Integer i = dadoRadicalHash.get(chaveDadoRadical).getF() + 1; dadoRadicalBanco.setF(i); dadoRadicalBanco.setQtdDoc(1); dadoRadicalHash.replace(chaveDadoRadical, dadoRadicalBanco); } } } referenciaBanco.setQtdRelevante(qtdPalavrasRelevantes); referenciaBanco.setPercRelevante(((float) qtdPalavrasRelevantes / tokens.size()) * 100); referenciaDAO.atualiza(referenciaBanco); tagsMorf = etiquetar(tokensMorf); ParseTree tree; tree = parser.init(); oracoes = parser.oracoes; ArrayList<String> oracoes1 = new ArrayList<>(); for (String oracao : oracoes) { if (!oracao.equals("\n")) { oracoes1.add(oracao); } } parser.oracoes.clear(); parser.oracoes.addAll(oracoes1); addAbreviatura(); }