Example usage for org.antlr.v4.runtime CommonTokenStream size

List of usage examples for org.antlr.v4.runtime CommonTokenStream size

  1. HOME
  2. Java
  3. org.antlr
  4. org.antlr.v4.runtime.*
  5. CommonTokenStream
  6. size

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream size.

Prototype

@Override
    public int size()

Source Link

Usage

From source file:com.espertech.esper.epl.parse.ASTUtil.java

License:Open Source License

/**
 * Print the token stream to the logger.
 * @param tokens to print//from  www  .ja v a 2  s .  c o m
 */
public static void printTokens(CommonTokenStream tokens) {
    if (log.isDebugEnabled()) {
        List tokenList = tokens.getTokens();

        StringWriter writer = new StringWriter();
        PrintWriter printer = new PrintWriter(writer);
        for (int i = 0; i < tokens.size(); i++) {
            Token t = (Token) tokenList.get(i);
            String text = t.getText();
            if (text.trim().length() == 0) {
                printer.print("'" + text + "'");
            } else {
                printer.print(text);
            }
            printer.print('[');
            printer.print(t.getType());
            printer.print(']');
            printer.print(" ");
        }
        printer.println();
        log.debug("Tokens: " + writer.toString());
    }
}

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static ScriptResult rewriteTokensScript(CommonTokenStream tokens) {
    List<String> scripts = new ArrayList<String>();

    List<UniformPair<Integer>> scriptTokenIndexRanges = new ArrayList<UniformPair<Integer>>();
    for (int i = 0; i < tokens.size(); i++) {
        if (tokens.get(i).getType() == EsperEPL2GrammarParser.EXPRESSIONDECL) {
            Token tokenBefore = getTokenBefore(i, tokens);
            boolean isCreateExpressionClause = tokenBefore != null
                    && tokenBefore.getType() == EsperEPL2GrammarParser.CREATE;
            Pair<String, Integer> nameAndNameStart = findScriptName(i + 1, tokens);

            int startIndex = findStartTokenScript(nameAndNameStart.getSecond(), tokens,
                    EsperEPL2GrammarParser.LBRACK);
            if (startIndex != -1) {
                int endIndex = findEndTokenScript(startIndex + 1, tokens, EsperEPL2GrammarParser.RBRACK,
                        EsperEPL2GrammarParser.getAfterScriptTokens(), !isCreateExpressionClause);
                if (endIndex != -1) {

                    StringWriter writer = new StringWriter();
                    for (int j = startIndex + 1; j < endIndex; j++) {
                        writer.append(tokens.get(j).getText());
                    }//from w w w  . j a v a 2  s  .co m
                    scripts.add(writer.toString());
                    scriptTokenIndexRanges.add(new UniformPair<Integer>(startIndex, endIndex));
                }
            }
        }
    }

    String rewrittenEPL = rewriteScripts(scriptTokenIndexRanges, tokens);
    return new ScriptResult(rewrittenEPL, scripts);
}

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static Pair<String, Integer> findScriptName(int start, CommonTokenStream tokens) {
    String lastIdent = null;/*from  www  .  java 2  s.c om*/
    int lastIdentIndex = 0;
    for (int i = start; i < tokens.size(); i++) {
        if (tokens.get(i).getType() == EsperEPL2GrammarParser.IDENT) {
            lastIdent = tokens.get(i).getText();
            lastIdentIndex = i;
        }
        if (tokens.get(i).getType() == EsperEPL2GrammarParser.LPAREN) {
            break;
        }
        // find beginning of script, ignore brackets
        if (tokens.get(i).getType() == EsperEPL2GrammarParser.LBRACK
                && tokens.get(i + 1).getType() != EsperEPL2GrammarParser.RBRACK) {
            break;
        }
    }
    if (lastIdent == null) {
        throw new IllegalStateException("Failed to parse expression name");
    }
    return new Pair<String, Integer>(lastIdent, lastIdentIndex);
}

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static String rewriteScripts(List<UniformPair<Integer>> ranges, CommonTokenStream tokens) {
    if (ranges.isEmpty()) {
        return tokens.getText();
    }//from  w  w w . ja  va2s. co m
    StringWriter writer = new StringWriter();
    int rangeIndex = 0;
    UniformPair<Integer> current = ranges.get(rangeIndex);
    for (int i = 0; i < tokens.size(); i++) {
        Token t = tokens.get(i);
        if (t.getType() == EsperEPL2GrammarLexer.EOF) {
            break;
        }
        if (i < current.getFirst()) {
            writer.append(t.getText());
        } else if (i == current.getFirst()) {
            writer.append(t.getText());
            writer.append("'");
        } else if (i == current.getSecond()) {
            writer.append("'");
            writer.append(t.getText());
            rangeIndex++;
            if (ranges.size() > rangeIndex) {
                current = ranges.get(rangeIndex);
            } else {
                current = new UniformPair<Integer>(-1, -1);
            }
        } else {
            if (t.getType() == EsperEPL2GrammarParser.QUOTED_STRING_LITERAL && i > current.getFirst()
                    && i < current.getSecond()) {
                writer.append("\\'");
                writer.append(t.getText().substring(1, t.getText().length() - 1));
                writer.append("\\'");
            } else {
                writer.append(t.getText());
            }
        }
    }
    return writer.toString();
}

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static int findEndTokenScript(int startIndex, CommonTokenStream tokens, int tokenTypeSearch,
        Set<Integer> afterScriptTokens, boolean requireAfterScriptToken) {
    int found = -1;
    for (int i = startIndex; i < tokens.size(); i++) {
        if (tokens.get(i).getType() == tokenTypeSearch) {
            if (!requireAfterScriptToken) {
                return i;
            }/*  w  w w  . j  a  v  a  2  s.c  o m*/
            // The next non-comment token must be among the afterScriptTokens, i.e. SELECT/INSERT/ON/DELETE/UPDATE
            // Find next non-comment token.
            for (int j = i + 1; j < tokens.size(); j++) {
                Token next = tokens.get(j);
                if (next.getChannel() == 0) {
                    if (afterScriptTokens.contains(next.getType())) {
                        found = i;
                    }
                    break;
                }
            }
        }
        if (found != -1) {
            break;
        }
    }
    return found;
}

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static boolean isContainsScriptExpression(CommonTokenStream tokens) {
    for (int i = 0; i < tokens.size(); i++) {
        if (tokens.get(i).getType() == EsperEPL2GrammarParser.EXPRESSIONDECL) {
            int startIndex = findStartTokenScript(i + 1, tokens, EsperEPL2GrammarParser.LBRACK);
            if (startIndex != -1) {
                return true;
            }//ww  w .  ja v  a 2  s  .c  o  m
        }
    }
    return false;
}

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static int findStartTokenScript(int startIndex, CommonTokenStream tokens, int tokenTypeSearch) {
    int found = -1;
    for (int i = startIndex; i < tokens.size(); i++) {
        if (tokens.get(i).getType() == tokenTypeSearch) {
            return i;
        }//from  w ww .ja  v  a 2 s  .  co m
    }
    return found;
}

From source file:com.sri.ai.praise.sgsolver.demo.editor.HOGMCodeArea.java

License:Open Source License

private static StyleSpans<Collection<String>> computeHighlighting(String text) {
    StyleSpansBuilder<Collection<String>> spansBuilder = new StyleSpansBuilder<>();
    int lastTokenEnd = 0;
    ANTLRInputStream input = new ANTLRInputStream(text);
    HOGMLexer lexer = new HOGMLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();//from w w w .  java 2  s .  c o  m
    for (int i = 0; i < tokens.size(); i++) {
        Token t = tokens.get(i);
        if (t.getType() == Token.EOF) {
            break;
        }
        String styleClass;
        if (t.getType() == HOGMLexer.COMMENT || t.getType() == HOGMLexer.LINE_COMMENT) {
            styleClass = "hogmCodeComment";
        } else if (HOGMTerminalSymbols.isTerminalSymbol(t.getText())) {
            styleClass = "hogmCodeKeyword";
        } else {
            styleClass = "hogmCodeOther";
        }
        int spacing = t.getStartIndex() - lastTokenEnd;
        if (spacing > 0) {
            spansBuilder.add(Collections.emptyList(), spacing);
        }
        int stylesize = (t.getStopIndex() - t.getStartIndex()) + 1;
        spansBuilder.add(Collections.singleton(styleClass), stylesize);
        lastTokenEnd = t.getStopIndex() + 1;
    }

    return spansBuilder.create();
}

From source file:controle.dicionario.Dicionario.java

private void analisaAssunto(String nomeArquivo, String texto) throws Exception {
    //ArrayList<String> tags = null;
    ArrayList<String> tagsMorf = null;
    CommonTokenStream tokensMorf = null;
    CommonTokenStream tokens = null;
    Referencia referenciaBanco = referenciasHash.get(nomeArquivo);
    referenciaBanco.setQtd(0);/*from  w w w .jav  a2  s. c om*/
    referenciaDAO.atualiza(referenciaBanco);
    this.nomeArquivo = nomeArquivo;
    SrsGrammarLexer lexer = new SrsGrammarLexer(new ANTLRInputStream(texto));
    tokens = new CommonTokenStream(lexer);
    SrsGrammarParser parser = new SrsGrammarParser(tokens);
    System.out.println(tokens.getText());

    analisePeriodo = new AnalisePeriodo(texto);

    SrsGrammarLexer lexerMorf = new SrsGrammarLexer(new ANTLRInputStream(texto));
    tokensMorf = new CommonTokenStream(lexerMorf);
    System.out.println(tokensMorf.getText());
    tagsMorf = etiquetar(tokensMorf);

    try {
        Sigla sigla = new Sigla();
        siglas.putAll(sigla.reconheceSiglas(tokens));

    } catch (Exception e) {
        e.printStackTrace();
    }

    Iterator it1 = tokens.getTokens().iterator();
    CommonToken ob = null;
    int qtdPalavrasRelevantes = 0;
    int qtdRadicais = 0;

    while (it1.hasNext()) {
        ob = (CommonToken) it1.next();
        //Retirando o caractere '-' do inicio da palavra
        if (ob.getText().length() > 1 && ob.getText().charAt(0) == '-') {
            ob.setText(ob.getText().substring(1));
        }
        //Retirando o caractere '-' do fim da palavra
        if (ob.getText().length() > 1 && (ob.getText().charAt(ob.getText().length() - 1) == '-')) {
            ob.setText(ob.getText().substring(0, ob.getText().length() - 2));
        }

        String palavra = ob.getText().toLowerCase();
        String palavraLemmaa = ob.getText();

        //Filtrando as palavras relevantes do texto
        if ((ob.getType() == SrsGrammarLexer.PALAVRA) && (ob.getText().length() > 4)
                && stopWordsHash.get(palavra) == null && !ehTudoIgual(palavra)) {
            qtdPalavrasRelevantes++;

            String radical;
            //Recuperando radical da palavra
            //Obtem radical utilizando o PTStemmer
            radical = getRadicalPTStemmer(palavra);

            String lemma;
            //Obtem lemma utilizando o TreeTagger
            //                lemma = AnalisePeriodo.tagger.getLemma(palavraLemmaa);
            lemma = AnalisePeriodo.recuperaLemma(palavraLemmaa);
            palavraLemma.put(palavraLemmaa, lemma);

            Lemma lemmaBanco = new Lemma(null, lemma);
            Radical radicalBanco = new Radical(null, radical);
            Palavra palavraBanco = new Palavra(null, palavra, lemmaBanco, radicalBanco);

            if (!palavrasHash.containsKey(palavra)) {
                palavrasHash.put(palavra, palavraBanco);
            }

            if (!radicaisHash.containsKey(radical)) {
                radicaisHash.put(radical, radicalBanco);
            }

            if (!lemmaHash.containsKey(lemma)) {
                lemmaHash.put(lemma, lemmaBanco);
            }

            Dados dadosBanco = new Dados();
            dadosBanco.setF(1);
            dadosBanco.setIdPalavra(palavraBanco);
            dadosBanco.setIdRadical(radicalBanco);
            dadosBanco.setIdReferencia(referenciaBanco);
            dadosBanco.setIdProjeto(projetoSelecionado);
            String chaveDados = projetoSelecionado.getId() + referenciaBanco.getId() + radical + palavra;
            if (!dadosHash.containsKey(chaveDados)) {
                dadosHash.put(chaveDados, dadosBanco);
                palavraBanco = palavrasHash.get(palavra);
                dadosBanco.setQtdDoc(dadosBanco.getQtdDoc() + 1);
            } else {
                Integer i = dadosHash.get(chaveDados).getF() + 1;
                dadosBanco.setF(i);
                dadosBanco.setQtdDoc(1);
                dadosHash.replace(chaveDados, dadosBanco);
            }

            DadoRadical dadoRadicalBanco = new DadoRadical();
            dadoRadicalBanco.setF(1);
            dadoRadicalBanco.setIdRadical(radicalBanco);
            dadoRadicalBanco.setIdReferencia(referenciaBanco);
            dadoRadicalBanco.setIdProjeto(projetoSelecionado);
            String chaveDadoRadical = projetoSelecionado.getId() + referenciaBanco.getId() + radical;
            if (!dadoRadicalHash.containsKey(chaveDadoRadical)) {
                dadoRadicalHash.put(chaveDadoRadical, dadoRadicalBanco);
                radicalBanco = radicaisHash.get(radical);
                dadoRadicalBanco.setQtdDoc(dadoRadicalBanco.getQtdDoc() + 1);
            } else {
                Integer i = dadoRadicalHash.get(chaveDadoRadical).getF() + 1;
                dadoRadicalBanco.setF(i);
                dadoRadicalBanco.setQtdDoc(1);
                dadoRadicalHash.replace(chaveDadoRadical, dadoRadicalBanco);
            }

        }
    }
    referenciaBanco.setQtdRelevante(qtdPalavrasRelevantes);
    referenciaBanco.setPercRelevante(((float) qtdPalavrasRelevantes / tokens.size()) * 100);
    referenciaDAO.atualiza(referenciaBanco);

    tagsMorf = etiquetar(tokensMorf);

    ParseTree tree;
    tree = parser.init();
    oracoes = parser.oracoes;
    ArrayList<String> oracoes1 = new ArrayList<>();
    for (String oracao : oracoes) {
        if (!oracao.equals("\n")) {
            oracoes1.add(oracao);
        }
    }
    parser.oracoes.clear();
    parser.oracoes.addAll(oracoes1);

    addAbreviatura();

}

From source file:de.bioviz.parser.BioParser.java

License:Open Source License

/**
 * Parses the annotations in a file./*from  w  w  w  .  j  a va2s .c o  m*/
 * @param input an ANTLRInputStream
 * @param channel the channel to parse
 * @return A List of Strings containing the annotations.
 */
private static List<String> parseChannel(final ANTLRInputStream input, final int channel) {
    BioLexerGrammar lexer = new BioLexerGrammar(input);

    lexer.reset();
    CommonTokenStream cts = new CommonTokenStream(lexer);
    List<String> channelTokens = new ArrayList<>();

    // this one gets everything that is in the stream.
    cts.getText();
    // now we can use size() to run over the tokens
    for (int i = 0; i < cts.size(); i++) {
        Token token = cts.get(i);
        // and check here if the token is on the right channel
        if (token.getChannel() == channel) {
            logger.trace("Parsing Comment: " + token.getText());
            channelTokens.add(token.getText());
        }
    }

    return channelTokens;
}