Example usage for org.antlr.v4.runtime Token getStopIndex

Introduction

In this page you can find the example usage for org.antlr.v4.runtime Token getStopIndex.

Prototype

int getStopIndex();

Source Link

Document

The last character index of the token.

Usage

From source file:org.kaazing.k3po.lang.internal.parser.ScriptParseStrategy.java

License:Open Source License

private static int stopIndex(Token token) {
    return (token != null) ? token.getStopIndex() : 0;
}

---

From source file:org.ledyba.sora.parser.FortranTokenStream.java

License:Open Source License

public void addTokenTo(int index, int line, int col, int tokenType, String tokenText) {
    try {/*w  w  w .j a va2 s . c o  m*/
        // for example:
        // index = 1
        // packedList == label T_CONTINUE T_EOS  (size is 3)
        // newTokenList.size() == 22
        // 22-3+1=20
        // so, inserted between the label and T_CONTINUE
        Token last = this.tokens.get(index <= 0 ? 0 : index - 1);
        Pair<TokenSource, CharStream> src = new Pair<>(last.getTokenSource(), last.getInputStream());
        FortranToken token = new FortranToken(src, tokenType, Token.DEFAULT_CHANNEL, last.getStopIndex(),
                last.getStopIndex());
        token.setText(tokenText == null ? "" : tokenText);
        token.setLine(line);
        token.setCharPositionInLine(col);
        this.packedList.add(index, token);
    } catch (Exception e) {
        e.printStackTrace();
        System.exit(1);
    }

    return;
}

---

From source file:org.ng200.tslk.ide.editors.TSLKRepairer.java

License:Open Source License

@Override
public void createPresentation(TextPresentation presentation, ITypedRegion region) {
    // Use tokens provided by the lexer to highlight keywords, etc...
    // Seems fast enough to skip Eclipse partitioning. Infact, the Eclipse
    // partitioner seems to slow everything down...
    TSLKGrammarLexer lexer = new TSLKGrammarLexer(new ANTLRInputStream(document.get()));
    Token t = null;
    while ((t = lexer.nextToken()).getType() != Token.EOF) {
        if (t.getStartIndex() > region.getOffset() + region.getLength())
            break;
        int start = t.getStartIndex();
        int end = t.getStopIndex();
        RGB foreground = null;//from   w  w  w.j  a  v  a2 s . c  o  m
        RGB background = null;
        int style = SWT.NORMAL;
        switch (t.getType()) { // TODO: Make keywords customisable
        case TSLKGrammarLexer.WHILE:
        case TSLKGrammarLexer.FOR:
        case TSLKGrammarLexer.FUNC:
        case TSLKGrammarLexer.IF:
        case TSLKGrammarLexer.THEN:
        case TSLKGrammarLexer.DO:
        case TSLKGrammarLexer.END:
            foreground = ColorManager.KEYWORD;
            style = SWT.BOLD;
            break;
        case TSLKGrammarLexer.STRING:
            foreground = ColorManager.STRING;
            break;
        case TSLKGrammarLexer.SLCOMMENT:
            foreground = ColorManager.SINGLE_LINE_COMMENT;
            break;
        case TSLKGrammarLexer.MLCOMMENT:
            foreground = ColorManager.MULTI_LINE_COMMENT;
            break;
        default:
            foreground = ColorManager.DEFAULT;
            break;
        }
        presentation.addStyleRange(new StyleRange(start, end - start + 1, colorManager.getColor(foreground),
                colorManager.getColor(background), style));
    }
}

---

From source file:org.shirolang.playground.SyntaxHighlighter.java

License:Open Source License

private void add(TerminalNode ident, String style) {
    if (ident != null) {
        Token t = ident.getSymbol();

        int spacer = t.getStartIndex() - lastEnd;
        if (spacer > 0) {
            spansBuilder.add(Collections.emptyList(), spacer);

            int gap = t.getText().length();
            spansBuilder.add(Collections.singleton(style), gap);
            lastEnd = t.getStopIndex() + 1;
        }//from  www . j  a  va 2 s  . com
    }
}

---

From source file:org.smallpearl.compiler.DescriptiveErrorListener.java

License:BSD License

protected void underlineError(Recognizer recognizer, Token offendingToken, int line, int charPositionInLine) {
    CommonTokenStream tokens = (CommonTokenStream) recognizer.getInputStream();
    String input = tokens.getTokenSource().getInputStream().toString();
    String[] lines = input.split("\n");
    String errorLine = lines[line - 1];
    System.err.println(errorLine);
    for (int i = 0; i < charPositionInLine; i++)
        System.err.print(" ");
    int start = offendingToken.getStartIndex();
    int stop = offendingToken.getStopIndex();
    if (start >= 0 && stop >= 0) {
        for (int i = start; i < stop; i++)
            System.err.print(" ");
    }/*from w  w w .j av  a2 s  .c o  m*/
    System.err.println("^");
}

---

From source file:org.sourcepit.ltk.jcomment.AbstractCommentLexer.java

License:Apache License

@Override
public Token nextToken() {
    if (cachedToken != null) {
        Token t = cachedToken;
        cachedToken = null;//from  w  w w. jav  a2 s.  c  o m
        return t;
    }
    Token t = _nextToken();
    CommonToken aggregate = null;
    while (t.getType() == CommentLexer.CommentText) {
        if (aggregate == null) {
            aggregate = (CommonToken) t;
        } else {
            aggregate.setStopIndex(t.getStopIndex());
        }
        t = _nextToken();
    }
    if (aggregate == null) {
        return t;
    }
    cachedToken = t;
    return aggregate;
}

---

From source file:org.tinygroup.template.parser.TinyTemplateErrorListener.java

License:GNU General Public License

public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine,
        String msg, RecognitionException e) {

    CommonTokenStream tokens = (CommonTokenStream) recognizer.getInputStream();
    String input = tokens.getTokenSource().getInputStream().toString();
    String[] sourceLines = input.split("\r?\n", -1);
    Token offendingToken = (Token) offendingSymbol;

    StringBuilder sb = new StringBuilder(128);
    sb.append("Template parse failed.\n");
    sb.append(recognizer.getInputStream().getSourceName());
    sb.append(':');
    sb.append(line);/*  w w w.j  a  va2 s .  c  om*/
    sb.append(':');
    sb.append(charPositionInLine);
    sb.append("\nmessage: ");
    sb.append(msg);
    sb.append('\n');
    sb.append(MemorySourceCompiler.getPrettyError(sourceLines, line, charPositionInLine + 1,
            offendingToken.getStartIndex(), offendingToken.getStopIndex(), 5));

    if (e != null) {
        throw new SyntaxErrorException(sb.toString(), line, charPositionInLine, e);
    } else {
        throw new SyntaxErrorException(sb.toString(), line, charPositionInLine);
    }
}

---

From source file:org.tvl.goworks.editor.go.completion.GoCompletionProvider.java

License:Open Source License

@Override
public int getAutoQueryTypes(JTextComponent component, String typedText) {
    if (typedText == null || typedText.length() != 1) {
        return 0;
    }//from ww w .  j a  v  a  2s.c  om

    boolean triggered = getCompletionAutoPopupTriggers().indexOf(typedText.charAt(0)) >= 0;
    if (triggered || (autoPopupOnIdentifierPart() && GoCompletionQuery.isIdentifierPart(typedText))) {
        int offset = component.getSelectionStart() - 1;
        Token contextToken = getContext(component, offset);
        if (contextToken == null) {
            return 0;
        }

        if (!triggered) {
            // the caret must be at the end of the identifier. note that the
            // offset is already 1 position before the caret, so no need to
            // add 1 to contextToken.getStopIndex().
            if (offset != contextToken.getStopIndex()) {
                return 0;
            }

            // only trigger for the first character of the identifier
            if (contextToken.getStopIndex() > contextToken.getStartIndex()) {
                return 0;
            }
        }

        boolean allowInStrings = false;
        if (isGoContext(contextToken, offset, allowInStrings)) {
            return COMPLETION_QUERY_TYPE | AUTO_QUERY_TYPE;
        }
    }

    return 0;
}

---

From source file:org.tvl.goworks.editor.go.completion.GoCompletionProvider.java

License:Open Source License

@Override
public Token getContext(Document document, int offset) {
    Parameters.notNull("document", document);

    if (document instanceof AbstractDocument) {
        ((AbstractDocument) document).readLock();
    }//  www  .java  2s.  c  om

    try {
        //            try {
        ParserTaskManager taskManager = Lookup.getDefault().lookup(ParserTaskManager.class);
        DocumentSnapshot snapshot = VersionedDocumentUtilities.getVersionedDocument(document)
                .getCurrentSnapshot();
        Future<ParserData<Tagger<TokenTag<Token>>>> futureTokensData = taskManager.getData(snapshot,
                GoParserDataDefinitions.LEXER_TOKENS, EnumSet.of(ParserDataOptions.SYNCHRONOUS));
        if (futureTokensData == null) {
            return null;
        }

        Tagger<TokenTag<Token>> tagger;
        try {
            tagger = futureTokensData.get().getData();
            if (tagger == null) {
                return null;
            }
        } catch (InterruptedException | ExecutionException ex) {
            LOGGER.log(Level.WARNING, "An exception occurred while getting tokens.", ex);
            return null;
        }

        // get the token(s) at the cursor position, with affinity both directions
        OffsetRegion region = OffsetRegion.fromBounds(Math.max(0, offset - 1),
                Math.min(snapshot.length(), offset + 1));
        Iterable<TaggedPositionRegion<TokenTag<Token>>> tags = tagger.getTags(
                new NormalizedSnapshotPositionRegionCollection(new SnapshotPositionRegion(snapshot, region)));

        // TODO: cache tokens
        //                ANTLRInputStream input = new ANTLRInputStream(document.getText(0, document.getLength()));
        //                GoLexer lexer = new GoLexer(input);
        //                CommonTokenStream tokenStream = new CommonTokenStream(lexer);
        Token token = null;
        //                for (token = tokenStream.LT(1); token != null && token.getType() != Token.EOF; token = tokenStream.LT(1)) {
        //                    tokenStream.consume();
        //                    if (token.getStartIndex() <= offset && token.getStopIndex() >= offset) {
        //                        break;
        //                    }
        //                }
        for (TaggedPositionRegion<TokenTag<Token>> taggedRegion : tags) {
            if (taggedRegion.getTag().getToken().getChannel() != Lexer.DEFAULT_TOKEN_CHANNEL) {
                continue;
            }

            token = taggedRegion.getTag().getToken();
            if (token.getStartIndex() <= offset && token.getStopIndex() >= offset) {
                break;
            }
        }

        if (token == null) {
            // try again without skipping off-channel tokens
            for (TaggedPositionRegion<TokenTag<Token>> taggedRegion : tags) {
                token = taggedRegion.getTag().getToken();
                if (token.getStartIndex() <= offset && token.getStopIndex() >= offset) {
                    break;
                }
            }
        }

        return token;
        //List<Token> tokens;
        //            } catch (BadLocationException ex) {
        //                Exceptions.printStackTrace(ex);
        //                return null;
        //            }
    } finally {
        if (document instanceof AbstractDocument) {
            ((AbstractDocument) document).readUnlock();
        }
    }
}

---

From source file:org.tvl.goworks.editor.go.highlighter.MarkOccurrencesHighlighter.java

License:Open Source License

@Override
protected void addHighlights(List<Tuple2<OffsetRegion, AttributeSet>> intermediateContainer,
        DocumentSnapshot sourceSnapshot, DocumentSnapshot currentSnapshot, Collection<Token> tokens,
        AttributeSet attributes) {
    for (Token token : tokens) {
        TrackingPositionRegion trackingRegion = sourceSnapshot.createTrackingRegion(
                OffsetRegion.fromBounds(token.getStartIndex(), token.getStopIndex() + 1),
                TrackingPositionRegion.Bias.Forward);
        SnapshotPositionRegion region = trackingRegion.getRegion(currentSnapshot);
        intermediateContainer.add(Tuple.create(region.getRegion(), attributes));
        markPositions.add(region.getStart());
    }/* www  . j  av  a 2 s  . c o m*/
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• »