Example usage for org.antlr.v4.runtime Token getType

Introduction

In this page you can find the example usage for org.antlr.v4.runtime Token getType.

Prototype

int getType();

Source Link

Document

Get the token type of the token

Usage

From source file:SparqlMain.java

License:Apache License

/**
 *
 * @param args/*w  w  w  .j  a v  a  2s .c o  m*/
 */
public static void main(String args[]) throws Exception {

    System.out.println("Work on file " + args[0]);

    int lineWidth = 80;
    if (args.length >= 2) {
        lineWidth = Integer.parseInt(args[1]);
    }

    SparqlLexer lex = null;
    try {
        lex = new SparqlLexer(new ANTLRFileStream(args[0]));
    } catch (IOException ex) {
        Logger.getLogger(SparqlMain.class.getName()).log(Level.SEVERE, null, ex);
    }
    CommonTokenStream tokens = new CommonTokenStream(lex);

    System.out.println("Tokens: -------------------------------");

    tokens.fill();
    System.out.println("Number of tokens " + tokens.getTokens().size());

    List tokenList = tokens.getTokens();

    System.out.println("TokenList: -------------------------------");
    Iterator it = tokenList.iterator();
    while (it.hasNext()) {
        Token t = (Token) it.next();
        System.out.println(t.toString());
    }
    System.out.flush();

    System.out.println("Input from token list: -------------------------------");

    it = tokenList.iterator();
    while (it.hasNext()) {
        Token t = (Token) it.next();
        if (t.getType() != SparqlParser.EOF) {
            if (t.getType() == SparqlParser.WS || t.getType() == SparqlParser.COMMENT) {
                String s = t.getText();
                s = s.replace("\r\n", "\n");
                if (!System.lineSeparator().equals("\n")) {
                    s = s.replace("\n", System.lineSeparator());
                }
                System.out.print(s);
            } else {
                System.out.print(t.getText());
            }
        }
    }
    System.out.flush();

    SparqlParser parser = new SparqlParser(tokens);
    parser.setBuildParseTree(true);

    System.out.println("Start parsing: -------------------------------");
    System.out.flush();

    ParserRuleContext t = parser.query();

    System.out.flush();
    System.out.println("Parse tree: -------------------------------");
    System.out.println(t.toStringTree(parser));

    // visualize parse tree in dialog box 
    t.inspect(parser);

    if (parser.getNumberOfSyntaxErrors() <= 0) {

        //ParseTreeWalker walker = new ParseTreeWalker();

        String groupFile = "ident.stg";
        if (args.length > 1) {
            groupFile = args[1];
        }
        System.out.println("Read StringTemplate Group File: " + groupFile + "-------------------------------");

        STGroup g = new STGroupFile(groupFile);
        IdentVisitor visitor = new IdentVisitor();
        visitor.setSTGroup(g);
        ST query = visitor.visit(t);

        System.out.println("Emit reformatted query: -------------------------------");

        System.out.println(query.render(lineWidth));

        System.out.println("Emit original query: -------------------------------");

        String q = query.render(lineWidth);

        /* get common token stream */
        File tmpFile = File.createTempFile("query_", ".rq");
        FileOutputStream fo = new FileOutputStream(tmpFile);
        OutputStreamWriter ow = new OutputStreamWriter(fo, "UTF8");
        ow.write(q);
        ow.close();
        /* transformation pipline
         * step 1: Unicode pre-processing
         * step 2: Lexical analysis
         */
        lex = new SparqlLexer(new ANTLRFileStream(tmpFile.getCanonicalPath(), "UTF8"));
        tokens = new CommonTokenStream(lex);

        List formattedTokenList = tokens.getTokens();

        it = tokenList.iterator();
        Iterator fit = formattedTokenList.iterator();

        boolean lineSeparatorHasToBeModified = !System.lineSeparator().equals("\n");

        while (it.hasNext()) {
            Token originalToken = (Token) it.next();
            if (originalToken.getType() != SparqlParser.EOF) {
                if (originalToken.getType() == SparqlParser.WS
                        || originalToken.getType() == SparqlParser.COMMENT) {
                    String s = originalToken.getText();
                    s = s.replace("\r\n", "\n");
                    if (lineSeparatorHasToBeModified) {
                        s = s.replace("\n", System.lineSeparator());
                    }
                    System.out.print(s);
                } else {
                    System.out.print(originalToken.getText());
                }
            }
        }
        System.out.flush();

    }
    System.out.println("-------------------------------");
    System.out.println("Number of errors encountered: " + parser.getNumberOfSyntaxErrors());
}

---

From source file:AST.java

License:Open Source License

@Override
public String toString() {

    StringBuilder builder = new StringBuilder();

    AST ast = this;
    List<AST> firstStack = new ArrayList<>();
    firstStack.add(ast);/*from  w  w  w.  j a  v  a  2 s.  c o m*/

    List<List<AST>> childListStack = new ArrayList<>();
    childListStack.add(firstStack);

    while (!childListStack.isEmpty()) {

        List<AST> childStack = childListStack.get(childListStack.size() - 1);

        if (childStack.isEmpty()) {
            childListStack.remove(childListStack.size() - 1);
        } else {
            ast = childStack.remove(0);
            String caption;

            if (ast.payload instanceof Token) {
                Token token = (Token) ast.payload;
                /*
                caption = String.format("TOKEN[type: %s, text: %s]",
                    token.getType(), token.getText().replace("\n", "\\n"));
                 */
                String symbolicName = "" + token.getType();
                if (token.getType() < 0) {
                    symbolicName = "EOF";
                } else //tokenName = this.parser.getTokenNames()[token.getType()];
                {
                    symbolicName = this.context.parser.getVocabulary().getSymbolicName(token.getType());
                }
                caption = String.format("TOKEN[type: %s, text: %s]", symbolicName,
                        token.getText().replace("\n", "\\n"));
                //caption = String.format("TOKEN[type: %s, text: %s, %d-%d]", symbolicName, token.getText().replace("\n", "\\n"), token.getStartIndex(), token.getStopIndex());
                //String text = this.context.code.substring(token.getStartIndex(), token.getStopIndex() + 1);
                //caption = String.format("TOKEN[type: %s, text: %s, %d-%d=%s]", symbolicName, token.getText().replace("\n", "\\n"), token.getStartIndex(), token.getStopIndex(), text);
            } else {
                caption = String.valueOf(ast.payload);
                ////caption = String.valueOf(ast.payload) + " [" + ast.payload.getClass().getName() + "]";
                //Interval interval = ast.node.getSourceInterval();
                //caption = String.valueOf(ast.payload) + " <== " + ast.node.getText() + " " + interval.a + "->" + interval.b; //.replace("\n","\\n");
                caption = String.valueOf(ast.payload) + " <== "
                        + this.context.code.substring(AST.getTokenStart(ast), AST.getTokenStop(ast) + 1)
                                .replace("\r\n", "\n").replace("\n", "\\n");
                /*
                caption = String.valueOf(ast.payload) + " <== "
                    + my.IOUTils.newUtf8String(
                            Arrays.copyOfRange(this.context.code, AST.getTokenStart(ast), AST.getTokenStop(ast)+1))
                            .replace("\r\n", "\n").replace("\n", "\\n")
                    ;*/
            }

            String indent = "";

            for (int i = 0; i < childListStack.size() - 1; i++) {
                indent += (childListStack.get(i).size() > 0) ? "|  " : "   ";
            }

            builder.append(indent).append(childStack.isEmpty() ? "`- " : "|- ").append(caption).append("\n");

            if (ast.children.size() > 0) {
                List<AST> children = new ArrayList<>();
                for (int i = 0; i < ast.children.size(); i++) {
                    children.add(ast.children.get(i));
                }
                childListStack.add(children);
            }
        }
    }

    return builder.toString();
}

---

From source file:ai.grakn.graql.internal.parser.QueryParser.java

License:Open Source License

/**
 * Consume a single query from the given token stream.
 *
 * @param tokenStream the {@link TokenStream} to consume
 * @return a new {@link TokenSource} containing the tokens comprising the query
 *///from   w  w w  . j  a v  a 2 s . c om
private TokenSource consumeOneQuery(TokenStream tokenStream) {
    List<Token> tokens = new ArrayList<>();

    boolean startedQuery = false;

    while (true) {
        Token token = tokenStream.LT(1);
        boolean isNewQuery = NEW_QUERY_TOKENS.contains(token.getType());
        boolean isEndOfTokenStream = token.getType() == IntStream.EOF;
        boolean isEndOfFirstQuery = startedQuery && isNewQuery;

        // Stop parsing tokens after reaching the end of the first query
        if (isEndOfTokenStream || isEndOfFirstQuery)
            break;

        if (isNewQuery)
            startedQuery = true;

        tokens.add(token);
        tokenStream.consume();
    }

    return new ListTokenSource(tokens);
}

---

From source file:cfml.parsing.cfscript.CFAssignmentExpression.java

License:Open Source License

public CFAssignmentExpression(Token t, CFExpression _left, CFExpression _right) {
    super(t);
    left = _left;
    right = _right;
    type = t.getType();
}

---

From source file:cfml.parsing.cfscript.CFBinaryExpression.java

License:Open Source License

public CFBinaryExpression(Token t, CFExpression left, CFExpression right) {
    super(t);//from   w w  w .  jav  a 2 s  . c  o  m
    _kind = t.getType();
    operatorImage = t.getText();
    if (_kind == CFSCRIPTLexer.ANDOPERATOR) {
        _kind = CFSCRIPTLexer.AND;
    } else if (_kind == CFSCRIPTLexer.OROPERATOR) {
        _kind = CFSCRIPTLexer.OR;
    } else if (_kind == CFSCRIPTLexer.MODOPERATOR) {
        _kind = CFSCRIPTLexer.MOD;
    }
    _left = left;
    _right = right;
}

---

From source file:cfml.parsing.cfscript.CFLiteral.java

License:Open Source License

public CFLiteral(Token _t) {
    super(_t);//  w w  w . java2  s. co  m
    kind = _t.getType();
    image = _t.getText();
    switch (kind) {
    case CFSCRIPTLexer.FLOATING_POINT_LITERAL:
    case CFSCRIPTLexer.INTEGER_LITERAL:
        val = _t.getText();
        break;
    case CFSCRIPTLexer.STRING_LITERAL:
        // create a String, stripping off the surrounding quotes and
        // replacing any escaped quotes with a single quote
        String quote = _t.getText().substring(0, 1);
        String str = _t.getText().substring(1, _t.getText().length() - 1);
        str = str.replaceAll(quote + quote, quote);
        image = str;
        val = str;
        break;
    case CFSCRIPTLexer.BOOLEAN_LITERAL:
        val = _t.getText();
        break;
    // CFML doesn't do nulls, to my knowledge
    // case CFSCRIPTLexer.NULL:
    // val = "";
    // break;
    default:
        break;
    }
}

---

From source file:cfml.parsing.cfscript.CFUnaryExpression.java

License:Open Source License

public CFUnaryExpression(org.antlr.v4.runtime.Token _t, CFExpression _sub) {
    super(_t);
    kind = _t.getType();
    sub = _sub;
}

---

From source file:ch.raffael.contracts.processor.cel.ast.Nodes.java

License:Apache License

private final static <T extends Enum> T kind(Map<Integer, T> map, Token tok) {
    T kind = map.get(tok.getType());
    if (kind == null) {
        throw new IllegalArgumentException("Cannot map token " + tok + " to kind");
    }/*  w ww . j a v  a2  s  .  c om*/
    return kind;
}

---

From source file:com.blazebit.persistence.parser.expression.JPQLSelectExpressionVisitorImpl.java

License:Apache License

private PredicateQuantifier toQuantifier(Token token) {
    PredicateQuantifier quantifier;/*  w  w w .  j  a  va 2  s  . c o  m*/
    if (token == null) {
        quantifier = PredicateQuantifier.ONE;
    } else {
        switch (token.getType()) {
        case JPQLSelectExpressionLexer.ANY:
        case JPQLSelectExpressionLexer.SOME:
            quantifier = PredicateQuantifier.ANY;
            break;
        case JPQLSelectExpressionLexer.ALL:
            quantifier = PredicateQuantifier.ALL;
            break;
        default:
            quantifier = PredicateQuantifier.ONE;
        }
    }
    return quantifier;
}

---

From source file:com.cisco.yangide.core.parser.YangTokenFormatter.java

License:Open Source License

/**
 * @param token/*from   w w w.ja v  a 2s  .c  o  m*/
 * @return <code>true</code> if token processed by import case
 */
private boolean processImportCase(Token token) {
    if (!compactImport) {
        return false;
    }

    if (token.getType() == YangLexer.IMPORT_KEYWORD) {
        importScope = true;
        importStatement = new StringBuilder();
    }

    if (importScope) {
        if (token.getType() == RIGHT_BRACE) {
            importScope = false;
            printIndent();
            sb.append(importStatement.toString()).append(' ');
            wasWS = true;
            nlCount = 0;
            // add indent for right brace processing
            currIndent += indent;
            return false;
        } else if (!isNewLine(token) && !isWS(token)) {
            if (importStatement.length() > 0 && token.getType() != SEMICOLON) {
                importStatement.append(' ');
            }
            importStatement.append(token.getText());
        }
        return true;
    }

    return importScope;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• »