Example usage for org.antlr.v4.runtime Token getText

Introduction

In this page you can find the example usage for org.antlr.v4.runtime Token getText.

Prototype

String getText();

Source Link

Document

Get the text of the token.

Usage

From source file:SparqlMain.java

License:Apache License

/**
 *
 * @param args/*from  w ww. j av a 2 s .  co m*/
 */
public static void main(String args[]) throws Exception {

    System.out.println("Work on file " + args[0]);

    int lineWidth = 80;
    if (args.length >= 2) {
        lineWidth = Integer.parseInt(args[1]);
    }

    SparqlLexer lex = null;
    try {
        lex = new SparqlLexer(new ANTLRFileStream(args[0]));
    } catch (IOException ex) {
        Logger.getLogger(SparqlMain.class.getName()).log(Level.SEVERE, null, ex);
    }
    CommonTokenStream tokens = new CommonTokenStream(lex);

    System.out.println("Tokens: -------------------------------");

    tokens.fill();
    System.out.println("Number of tokens " + tokens.getTokens().size());

    List tokenList = tokens.getTokens();

    System.out.println("TokenList: -------------------------------");
    Iterator it = tokenList.iterator();
    while (it.hasNext()) {
        Token t = (Token) it.next();
        System.out.println(t.toString());
    }
    System.out.flush();

    System.out.println("Input from token list: -------------------------------");

    it = tokenList.iterator();
    while (it.hasNext()) {
        Token t = (Token) it.next();
        if (t.getType() != SparqlParser.EOF) {
            if (t.getType() == SparqlParser.WS || t.getType() == SparqlParser.COMMENT) {
                String s = t.getText();
                s = s.replace("\r\n", "\n");
                if (!System.lineSeparator().equals("\n")) {
                    s = s.replace("\n", System.lineSeparator());
                }
                System.out.print(s);
            } else {
                System.out.print(t.getText());
            }
        }
    }
    System.out.flush();

    SparqlParser parser = new SparqlParser(tokens);
    parser.setBuildParseTree(true);

    System.out.println("Start parsing: -------------------------------");
    System.out.flush();

    ParserRuleContext t = parser.query();

    System.out.flush();
    System.out.println("Parse tree: -------------------------------");
    System.out.println(t.toStringTree(parser));

    // visualize parse tree in dialog box 
    t.inspect(parser);

    if (parser.getNumberOfSyntaxErrors() <= 0) {

        //ParseTreeWalker walker = new ParseTreeWalker();

        String groupFile = "ident.stg";
        if (args.length > 1) {
            groupFile = args[1];
        }
        System.out.println("Read StringTemplate Group File: " + groupFile + "-------------------------------");

        STGroup g = new STGroupFile(groupFile);
        IdentVisitor visitor = new IdentVisitor();
        visitor.setSTGroup(g);
        ST query = visitor.visit(t);

        System.out.println("Emit reformatted query: -------------------------------");

        System.out.println(query.render(lineWidth));

        System.out.println("Emit original query: -------------------------------");

        String q = query.render(lineWidth);

        /* get common token stream */
        File tmpFile = File.createTempFile("query_", ".rq");
        FileOutputStream fo = new FileOutputStream(tmpFile);
        OutputStreamWriter ow = new OutputStreamWriter(fo, "UTF8");
        ow.write(q);
        ow.close();
        /* transformation pipline
         * step 1: Unicode pre-processing
         * step 2: Lexical analysis
         */
        lex = new SparqlLexer(new ANTLRFileStream(tmpFile.getCanonicalPath(), "UTF8"));
        tokens = new CommonTokenStream(lex);

        List formattedTokenList = tokens.getTokens();

        it = tokenList.iterator();
        Iterator fit = formattedTokenList.iterator();

        boolean lineSeparatorHasToBeModified = !System.lineSeparator().equals("\n");

        while (it.hasNext()) {
            Token originalToken = (Token) it.next();
            if (originalToken.getType() != SparqlParser.EOF) {
                if (originalToken.getType() == SparqlParser.WS
                        || originalToken.getType() == SparqlParser.COMMENT) {
                    String s = originalToken.getText();
                    s = s.replace("\r\n", "\n");
                    if (lineSeparatorHasToBeModified) {
                        s = s.replace("\n", System.lineSeparator());
                    }
                    System.out.print(s);
                } else {
                    System.out.print(originalToken.getText());
                }
            }
        }
        System.out.flush();

    }
    System.out.println("-------------------------------");
    System.out.println("Number of errors encountered: " + parser.getNumberOfSyntaxErrors());
}

---

From source file:AST.java

License:Open Source License

@Override
public String toString() {

    StringBuilder builder = new StringBuilder();

    AST ast = this;
    List<AST> firstStack = new ArrayList<>();
    firstStack.add(ast);//from   w ww.ja  v a  2s  .co m

    List<List<AST>> childListStack = new ArrayList<>();
    childListStack.add(firstStack);

    while (!childListStack.isEmpty()) {

        List<AST> childStack = childListStack.get(childListStack.size() - 1);

        if (childStack.isEmpty()) {
            childListStack.remove(childListStack.size() - 1);
        } else {
            ast = childStack.remove(0);
            String caption;

            if (ast.payload instanceof Token) {
                Token token = (Token) ast.payload;
                /*
                caption = String.format("TOKEN[type: %s, text: %s]",
                    token.getType(), token.getText().replace("\n", "\\n"));
                 */
                String symbolicName = "" + token.getType();
                if (token.getType() < 0) {
                    symbolicName = "EOF";
                } else //tokenName = this.parser.getTokenNames()[token.getType()];
                {
                    symbolicName = this.context.parser.getVocabulary().getSymbolicName(token.getType());
                }
                caption = String.format("TOKEN[type: %s, text: %s]", symbolicName,
                        token.getText().replace("\n", "\\n"));
                //caption = String.format("TOKEN[type: %s, text: %s, %d-%d]", symbolicName, token.getText().replace("\n", "\\n"), token.getStartIndex(), token.getStopIndex());
                //String text = this.context.code.substring(token.getStartIndex(), token.getStopIndex() + 1);
                //caption = String.format("TOKEN[type: %s, text: %s, %d-%d=%s]", symbolicName, token.getText().replace("\n", "\\n"), token.getStartIndex(), token.getStopIndex(), text);
            } else {
                caption = String.valueOf(ast.payload);
                ////caption = String.valueOf(ast.payload) + " [" + ast.payload.getClass().getName() + "]";
                //Interval interval = ast.node.getSourceInterval();
                //caption = String.valueOf(ast.payload) + " <== " + ast.node.getText() + " " + interval.a + "->" + interval.b; //.replace("\n","\\n");
                caption = String.valueOf(ast.payload) + " <== "
                        + this.context.code.substring(AST.getTokenStart(ast), AST.getTokenStop(ast) + 1)
                                .replace("\r\n", "\n").replace("\n", "\\n");
                /*
                caption = String.valueOf(ast.payload) + " <== "
                    + my.IOUTils.newUtf8String(
                            Arrays.copyOfRange(this.context.code, AST.getTokenStart(ast), AST.getTokenStop(ast)+1))
                            .replace("\r\n", "\n").replace("\n", "\\n")
                    ;*/
            }

            String indent = "";

            for (int i = 0; i < childListStack.size() - 1; i++) {
                indent += (childListStack.get(i).size() > 0) ? "|  " : "   ";
            }

            builder.append(indent).append(childStack.isEmpty() ? "`- " : "|- ").append(caption).append("\n");

            if (ast.children.size() > 0) {
                List<AST> children = new ArrayList<>();
                for (int i = 0; i < ast.children.size(); i++) {
                    children.add(ast.children.get(i));
                }
                childListStack.add(children);
            }
        }
    }

    return builder.toString();
}

---

From source file:Alert.java

public static void error(String s, Token t) {
    if (t != null)
        System.out.println("ERROR: in line: " + t.getLine() + " : around: " + t.getText() + " :\t" + s);
    else//from w  ww .  j  a va2  s.  c  om
        System.out.println("ERROR: " + s);
}

---

From source file:Alert.java

public static void info(String s, Token t) {
    if (t != null)
        System.out.println("INFO: in line: " + t.getLine() + " : around: " + t.getText() + " :\t" + s);
    else//from  ww w .  jav a  2 s .  c o m
        System.out.println("ERROR: " + s);
}

---

From source file:ai.grakn.graql.internal.parser.QueryVisitor.java

License:Open Source License

private Var getVariable(Token variable) {
    // Remove '$' prefix
    return Graql.var(variable.getText().substring(1));
}

---

From source file:android.databinding.tool.store.Location.java

License:Apache License

public Location(Token start, Token end) {
    if (start == null) {
        startLine = startOffset = NaN;//from  w w  w.  jav a2s .  c o m
    } else {
        startLine = start.getLine() - 1; //token lines start from 1
        startOffset = start.getCharPositionInLine();
    }

    if (end == null) {
        endLine = endOffset = NaN;
    } else {
        endLine = end.getLine() - 1; // token lines start from 1
        String endText = end.getText();
        int lastLineStart = endText.lastIndexOf(System.lineSeparator());
        String lastLine = lastLineStart < 0 ? endText : endText.substring(lastLineStart + 1);
        endOffset = end.getCharPositionInLine() + lastLine.length() - 1;//end is inclusive
    }
}

---

From source file:android.databinding.tool.util.XmlEditor.java

License:Apache License

private static Position toEndPosition(Token token) {
    return new Position(token.getLine() - 1, token.getCharPositionInLine() + token.getText().length());
}

---

From source file:annis.ql.parser.AnnisParserAntlr.java

License:Apache License

public static ParsedEntityLocation getLocation(Token start, Token stop) {
    if (start == null) {
        return new ParsedEntityLocation();
    }/*from   ww w  .  j a  v  a2 s.  co m*/
    if (stop == null) {
        stop = start;
    }

    int startLine = start.getLine();
    int endLine = stop.getLine();

    int startColumn = start.getCharPositionInLine();
    // We assume a token can be only one line (newline character is whitespace and a separator).
    // Thus the end column of a token is the start position plus its actual text length;
    String stopTokenText = stop.getText();
    int endColumn = stop.getCharPositionInLine();
    if (stopTokenText != null && !stopTokenText.isEmpty()) {
        endColumn += stopTokenText.length() - 1;
    }

    return new ParsedEntityLocation(startLine, startColumn, endLine, endColumn);
}

---

From source file:annis.ql.parser.JoinListener.java

License:Apache License

private QueryNode nodeByRef(Token ref) {
    return alternativeNodes[alternativeIndex].get("" + ref.getText().substring(1));
}

---

From source file:annis.ql.parser.LogicClause.java

License:Apache License

@Override
public String toString() {
    if (op == Operator.AND) {
        return "(" + Joiner.on(" & ").join(children) + ")";
    } else if (op == Operator.OR) {
        return "(" + Joiner.on(" \n| \n").join(children) + ")";
    }/*from www  . ja  v  a 2  s  . com*/

    LinkedList<String> texts = new LinkedList<>();
    if (content != null) {
        for (Token t : content) {
            texts.add(t.getText());
        }
    }
    return Joiner.on(" ").join(texts);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• »