Example usage for org.antlr.v4.runtime CommonTokenStream get

List of usage examples for org.antlr.v4.runtime CommonTokenStream get

  1. HOME
  2. Java
  3. org.antlr
  4. org.antlr.v4.runtime.*
  5. CommonTokenStream
  6. get

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream get.

Prototype

@Override
    public Token get(int i)

Source Link

Usage

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static ScriptResult rewriteTokensScript(CommonTokenStream tokens) {
    List<String> scripts = new ArrayList<String>();

    List<UniformPair<Integer>> scriptTokenIndexRanges = new ArrayList<UniformPair<Integer>>();
    for (int i = 0; i < tokens.size(); i++) {
        if (tokens.get(i).getType() == EsperEPL2GrammarParser.EXPRESSIONDECL) {
            Token tokenBefore = getTokenBefore(i, tokens);
            boolean isCreateExpressionClause = tokenBefore != null
                    && tokenBefore.getType() == EsperEPL2GrammarParser.CREATE;
            Pair<String, Integer> nameAndNameStart = findScriptName(i + 1, tokens);

            int startIndex = findStartTokenScript(nameAndNameStart.getSecond(), tokens,
                    EsperEPL2GrammarParser.LBRACK);
            if (startIndex != -1) {
                int endIndex = findEndTokenScript(startIndex + 1, tokens, EsperEPL2GrammarParser.RBRACK,
                        EsperEPL2GrammarParser.getAfterScriptTokens(), !isCreateExpressionClause);
                if (endIndex != -1) {

                    StringWriter writer = new StringWriter();
                    for (int j = startIndex + 1; j < endIndex; j++) {
                        writer.append(tokens.get(j).getText());
                    }//from  w w  w .  j  a  v a2s  .  c  om
                    scripts.add(writer.toString());
                    scriptTokenIndexRanges.add(new UniformPair<Integer>(startIndex, endIndex));
                }
            }
        }
    }

    String rewrittenEPL = rewriteScripts(scriptTokenIndexRanges, tokens);
    return new ScriptResult(rewrittenEPL, scripts);
}

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static Token getTokenBefore(int i, CommonTokenStream tokens) {
    int position = i - 1;
    while (position >= 0) {
        Token t = tokens.get(position);
        if (t.getChannel() != 99 && t.getType() != EsperEPL2GrammarLexer.WS) {
            return t;
        }// w  ww.  j  a va2s  .  c  om
        position--;
    }
    return null;
}

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static Pair<String, Integer> findScriptName(int start, CommonTokenStream tokens) {
    String lastIdent = null;/*from   w  w  w.j a va  2 s. c om*/
    int lastIdentIndex = 0;
    for (int i = start; i < tokens.size(); i++) {
        if (tokens.get(i).getType() == EsperEPL2GrammarParser.IDENT) {
            lastIdent = tokens.get(i).getText();
            lastIdentIndex = i;
        }
        if (tokens.get(i).getType() == EsperEPL2GrammarParser.LPAREN) {
            break;
        }
        // find beginning of script, ignore brackets
        if (tokens.get(i).getType() == EsperEPL2GrammarParser.LBRACK
                && tokens.get(i + 1).getType() != EsperEPL2GrammarParser.RBRACK) {
            break;
        }
    }
    if (lastIdent == null) {
        throw new IllegalStateException("Failed to parse expression name");
    }
    return new Pair<String, Integer>(lastIdent, lastIdentIndex);
}

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static String rewriteScripts(List<UniformPair<Integer>> ranges, CommonTokenStream tokens) {
    if (ranges.isEmpty()) {
        return tokens.getText();
    }/* w w  w .  ja  v  a  2 s . co  m*/
    StringWriter writer = new StringWriter();
    int rangeIndex = 0;
    UniformPair<Integer> current = ranges.get(rangeIndex);
    for (int i = 0; i < tokens.size(); i++) {
        Token t = tokens.get(i);
        if (t.getType() == EsperEPL2GrammarLexer.EOF) {
            break;
        }
        if (i < current.getFirst()) {
            writer.append(t.getText());
        } else if (i == current.getFirst()) {
            writer.append(t.getText());
            writer.append("'");
        } else if (i == current.getSecond()) {
            writer.append("'");
            writer.append(t.getText());
            rangeIndex++;
            if (ranges.size() > rangeIndex) {
                current = ranges.get(rangeIndex);
            } else {
                current = new UniformPair<Integer>(-1, -1);
            }
        } else {
            if (t.getType() == EsperEPL2GrammarParser.QUOTED_STRING_LITERAL && i > current.getFirst()
                    && i < current.getSecond()) {
                writer.append("\\'");
                writer.append(t.getText().substring(1, t.getText().length() - 1));
                writer.append("\\'");
            } else {
                writer.append(t.getText());
            }
        }
    }
    return writer.toString();
}

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static int findEndTokenScript(int startIndex, CommonTokenStream tokens, int tokenTypeSearch,
        Set<Integer> afterScriptTokens, boolean requireAfterScriptToken) {
    int found = -1;
    for (int i = startIndex; i < tokens.size(); i++) {
        if (tokens.get(i).getType() == tokenTypeSearch) {
            if (!requireAfterScriptToken) {
                return i;
            }/* ww  w .  ja va2 s .co m*/
            // The next non-comment token must be among the afterScriptTokens, i.e. SELECT/INSERT/ON/DELETE/UPDATE
            // Find next non-comment token.
            for (int j = i + 1; j < tokens.size(); j++) {
                Token next = tokens.get(j);
                if (next.getChannel() == 0) {
                    if (afterScriptTokens.contains(next.getType())) {
                        found = i;
                    }
                    break;
                }
            }
        }
        if (found != -1) {
            break;
        }
    }
    return found;
}

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static boolean isContainsScriptExpression(CommonTokenStream tokens) {
    for (int i = 0; i < tokens.size(); i++) {
        if (tokens.get(i).getType() == EsperEPL2GrammarParser.EXPRESSIONDECL) {
            int startIndex = findStartTokenScript(i + 1, tokens, EsperEPL2GrammarParser.LBRACK);
            if (startIndex != -1) {
                return true;
            }/* w  ww  . ja  va 2s  . c  om*/
        }
    }
    return false;
}

From source file:com.espertech.esper.epl.parse.ParseHelper.java

License:Open Source License

private static int findStartTokenScript(int startIndex, CommonTokenStream tokens, int tokenTypeSearch) {
    int found = -1;
    for (int i = startIndex; i < tokens.size(); i++) {
        if (tokens.get(i).getType() == tokenTypeSearch) {
            return i;
        }//from ww  w  . j a v a 2s .  co m
    }
    return found;
}

From source file:com.sri.ai.praise.sgsolver.demo.editor.HOGMCodeArea.java

License:Open Source License

private static StyleSpans<Collection<String>> computeHighlighting(String text) {
    StyleSpansBuilder<Collection<String>> spansBuilder = new StyleSpansBuilder<>();
    int lastTokenEnd = 0;
    ANTLRInputStream input = new ANTLRInputStream(text);
    HOGMLexer lexer = new HOGMLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();/*from   w  w w.j  a v a2 s  .co  m*/
    for (int i = 0; i < tokens.size(); i++) {
        Token t = tokens.get(i);
        if (t.getType() == Token.EOF) {
            break;
        }
        String styleClass;
        if (t.getType() == HOGMLexer.COMMENT || t.getType() == HOGMLexer.LINE_COMMENT) {
            styleClass = "hogmCodeComment";
        } else if (HOGMTerminalSymbols.isTerminalSymbol(t.getText())) {
            styleClass = "hogmCodeKeyword";
        } else {
            styleClass = "hogmCodeOther";
        }
        int spacing = t.getStartIndex() - lastTokenEnd;
        if (spacing > 0) {
            spansBuilder.add(Collections.emptyList(), spacing);
        }
        int stylesize = (t.getStopIndex() - t.getStartIndex()) + 1;
        spansBuilder.add(Collections.singleton(styleClass), stylesize);
        lastTokenEnd = t.getStopIndex() + 1;
    }

    return spansBuilder.create();
}

From source file:controle.analiseTexto.AnalisePeriodo.java

public static ArrayList<String> etiquetar(CommonTokenStream tokens) {

    ultimaPalavraSentenca = tokens.getTokens().get(tokens.getTokens().size() - 2).getText();

    for (int i = 0; i < tokens.getTokens().size() - 1; i++) {
        //Deteco e atribuio das etiquetas aos seus respectivos tokens
        String palavraEtiquetada = tokens.getTokens().get(i).getText().toLowerCase();
        String etiqueta = tagger.getHashEtiquetas().get(palavraEtiquetada);
        String lemma = "";
        try {//from   w  ww  .j  a v a  2  s  .com
            lemma = tagger.getLemma(palavraEtiquetada);

        } catch (Exception e) {
            e.printStackTrace();
        }
        if (Character.isDigit(palavraEtiquetada.charAt(0))) {
            ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.NUMERO);

        } else {
            switch (tagger.getHashEtiquetas().get(palavraEtiquetada).charAt(0)) {
            case 'V':
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.VERB);
                break;
            case 'N':
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.SUBS);
                break;
            case 'D':
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.ART);
                break;
            case 'P':
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.PRON);
                break;
            case 'S':
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.PREP);
                break;
            case 'A':
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.ADJ);
                break;
            case 'Z':
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.NUM);
                break;
            case 'C':
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.CONJ);
                break;

            case 'F':
                try {
                    if (tagger.getEtiquetas().get(i).charAt(1) == 'c') {
                        ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.TERMINAL);
                    } else if (tagger.getEtiquetas().get(i).charAt(1) == 'd') {
                    } else {
                        ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.SIMBOLOS);
                    }
                } catch (Exception e) {
                    ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.PALAVRAESTRANGEIRA);
                }
                break;
            case 'R':
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.ADV);
                break;
            } //Case
        } //if chaAt
    }
    return (ArrayList<String>) tagger.getEtiquetas();
}

From source file:controle.analiseTexto.AnalisePeriodo.java

public static ArrayList<String> etiquetarDescricaoFuncionalidade(CommonTokenStream tokens) {
    String verbos = "#dever#poder#";
    elementosDaFraseSRS = new ElementosFrase();
    etiquetasSentenca = "";

    ultimaPalavraSentenca = tokens.getTokens().get(tokens.getTokens().size() - 2).getText();

    for (int i = 0; i < tokens.getTokens().size() - 1; i++) {
        //Deteco e atribuio das etiquetas aos seus respectivos tokens
        String palavraEtiquetada = tokens.getTokens().get(i).getText().toLowerCase();
        String lemma = "";
        try {/*from ww  w .ja v a  2 s.c o m*/
            lemma = tagger.getLemma(palavraEtiquetada);
        } catch (Exception e) {
            //   System.out.println("erro em: " + palavraEtiquetada);
            e.printStackTrace();
        }

        switch (tagger.getHashEtiquetas().get(palavraEtiquetada).charAt(0)) {
        case 'V':
            String verbo = tokens.getTokens().get(i).getText().toLowerCase();
            verbo = Etiquetador.lemmas.get(verbo.toLowerCase());
            if (verbo.toLowerCase().equals("permitir")) {
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.PERMITIR);
                etiquetasSentenca = etiquetasSentenca + "PERMITIR";
                //System.out.print(palavraEtiquetada + ":" + "PERMITIR ");
                elementosDaFraseSRS.addElemento(palavraEtiquetada, "PERMITIR");
            } else {
                if (verbos.contains(verbo.toLowerCase())) {
                    ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.DEVE);
                    etiquetasSentenca = etiquetasSentenca + "DEVE";
                    // System.out.print(palavraEtiquetada + ":" + "DEVE ");
                    elementosDaFraseSRS.addElemento(palavraEtiquetada, "DEVE");
                } else {
                    ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.VERB);
                    etiquetasSentenca = etiquetasSentenca + "VERB";
                    //System.out.print(palavraEtiquetada + ":" + "VERB ");
                    elementosDaFraseSRS.addElemento(palavraEtiquetada, "VERB");
                }
                elementosDaFraseSRS.addVerbo(verbo);
            }
            break;
        case 'N':
            if (tokens.getTokens().get(i).getText().toLowerCase().equals("sistema") && i <= 2) {
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.SISTEMA);
                etiquetasSentenca = etiquetasSentenca + "SISTEMA";
                //System.out.print(palavraEtiquetada + ":" + "SISTEMA ");
                elementosDaFraseSRS.addElemento(palavraEtiquetada, "SISTEMA");
            } else {
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.SUBS);
                etiquetasSentenca = etiquetasSentenca + "SUBS";
                //System.out.print(palavraEtiquetada + ":" + "SUBS ");
                elementosDaFraseSRS.addElemento(palavraEtiquetada, "SUBS");
            }
            break;
        case 'D':
            ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.ART);
            //System.out.print(palavraEtiquetada + ":" + "ART ");
            elementosDaFraseSRS.addElemento(palavraEtiquetada, "ART");
            break;
        case 'P':
            ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.PRON);
            //System.out.print(palavraEtiquetada + ":" + "PRON ");
            elementosDaFraseSRS.addElemento(palavraEtiquetada, "PRON");
            break;
        case 'S':
            ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.PREP);
            //System.out.print(palavraEtiquetada + ":" + "PREP ");
            elementosDaFraseSRS.addElemento(palavraEtiquetada, "PREP");
            break;
        case 'A':
            ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.ADJ);
            //System.out.print(palavraEtiquetada + ":" + "ADJ ");
            elementosDaFraseSRS.addElemento(palavraEtiquetada, "ADJ");
            break;
        case 'Z':
            ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.NUM);
            //System.out.print(palavraEtiquetada + ":" + "NUM ");
            elementosDaFraseSRS.addElemento(palavraEtiquetada, "NUM");
            break;
        case 'C':
            ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.CONJ);
            //System.out.print(palavraEtiquetada + ":" + "CONJ ");
            elementosDaFraseSRS.addElemento(palavraEtiquetada, "CONJ");
            break;
        case 'F':
            try {
                if (tagger.getHashEtiquetas().get(palavraEtiquetada).charAt(1) == 'c') {
                    ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.SEPARADOR);
                    //System.out.print(palavraEtiquetada + ":" + "SEPARADOR ");
                    elementosDaFraseSRS.addElemento(palavraEtiquetada, "SEPARADOR");
                } else if (tagger.getHashEtiquetas().get(palavraEtiquetada).charAt(1) == 'd') {
                } else {
                    if (tokens.getTokens().get(i).getText().toLowerCase().equals(".")) {
                        ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.PONTO);
                        //System.out.print(palavraEtiquetada + ":" + "PONTO ");
                        elementosDaFraseSRS.addElemento(palavraEtiquetada, "PONTO");
                    } else {
                        ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.SIMBOLOS);
                        //System.out.print(palavraEtiquetada + ":" + "SIMBOLOS ");
                        elementosDaFraseSRS.addElemento(palavraEtiquetada, "SIMBOLOS");
                    }
                }
            } catch (Exception e) {
                ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.PALAVRAESTRANGEIRA);
                //System.out.print(palavraEtiquetada + ":" + "PALAVRAESTRANGEIRA ");
                elementosDaFraseSRS.addElemento(palavraEtiquetada, "PALAVRAESTRANGEIRA");
            }
            break;
        case 'R':
            ((CommonToken) tokens.getTokens().get(i)).setType(SrsGrammarParser.ADV);
            //System.out.print(palavraEtiquetada + ":" + "ADV ");
            elementosDaFraseSRS.addElemento(palavraEtiquetada, "ADV");
            break;
        }
    }
    return (ArrayList<String>) tagger.getEtiquetas();
}