List of usage examples for org.antlr.v4.runtime Token getText
String getText();
From source file:org.semanticwb.rdf.sparql.SparqlMain.java
License:Apache License
/** * * @param args//from w w w .j av a2s . c om */ public static void main(String args[]) throws Exception { // System.out.println("Work on file " + args[0]); int lineWidth = 80; if (args.length >= 2) { lineWidth = Integer.parseInt(args[1]); } SparqlLexer lex = null; try { String q = "PREFIX map: <http://datosabiertos.gob.mx/ontology/mapas.owl#>\n" + "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n" + "\n" + "SELECT \n" + " (CONCAT(?descripcionRecursoGeografico,\" (\",?tituloCapa,\")\") as ?titulo)\n" + " (CONCAT(\n" + " \"<h2>\",?titulo,\"</h2>\",\n" + " \"Estado:\",?estado,\"<br/>\",\n" + " \"Colonia:\",?colonia,\"<br/>\",\n" + " \"Calle:\",?calle,\"<br/>\",\n" + " \"CP:\",?cp,\"<br/>\",\n" + " \"(\",?latitud,\", \",?longitud,\")\"\n" + " ) as ?descripcion) \n" + " ?latitud \n" + " ?longitud \n" + "WHERE {\n" + " ?uri rdf:type map:RecursoGeografico .\n" + " ?uri map:capa ?capa .\n" + " ?capa map:titulo ?tituloCapa .\n" + " ?uri map:descripcionRecursoGeografico ?descripcionRecursoGeografico .\n" + " ?uri map:estado ?estado .\n" + " ?uri map:colonia ?colonia .\n" + " ?uri map:calle ?calle .\n" + " ?uri map:cp ?cp .\n" + " ?uri map:latitud ?latitud .\n" + " ?uri map:longitud ?longitud .\n" + " filter( (?latitud>\"19.2\"^^xsd:double) && (?latitud<\"19.3\"^^xsd:double) && (?longitud<\"-99.1\"^^xsd:double) && (?longitud>\"-99.2\"^^xsd:double) ) .\n" + "}\n" + "LIMIT 100"; //lex = new SparqlLexer(new ANTLRFileStream(args[0])); lex = new SparqlLexer(new ANTLRInputStream( "select (count(*) as ?c) ?s ?p ?o where {?s a ?o; hola:asd <http://sdf.ser:sadasd>. ?s ?p2 ?o2}")); //lex = new SparqlLexer(new ANTLRInputStream(q)); } catch (Exception ex) { Logger.getLogger(SparqlMain.class.getName()).log(Level.SEVERE, null, ex); } CommonTokenStream tokens = new CommonTokenStream(lex); System.out.println("Tokens: -------------------------------"); tokens.fill(); System.out.println("Number of tokens " + tokens.getTokens().size()); List tokenList = tokens.getTokens(); System.out.println("TokenList: -------------------------------"); Iterator it = tokenList.iterator(); while (it.hasNext()) { Token t = (Token) it.next(); System.out.println(t.toString()); } System.out.flush(); System.out.println("Input from token list: -------------------------------"); it = tokenList.iterator(); while (it.hasNext()) { Token t = (Token) it.next(); if (t.getType() != SparqlParser.EOF) { if (t.getType() == SparqlParser.WS || t.getType() == SparqlParser.COMMENT) { String s = t.getText(); s = s.replace("\r\n", "\n"); System.out.print(s); } else { System.out.print(t.getText()); } } } System.out.flush(); SparqlParser parser = new SparqlParser(tokens); parser.setBuildParseTree(true); System.out.println("Start parsing: -------------------------------"); System.out.flush(); ParserRuleContext t = parser.query(); System.out.flush(); System.out.println("Parse tree: -------------------------------"); System.out.println(t.toStringTree(parser)); int x = t.getRuleIndex(); String rnames[] = parser.getRuleNames(); getTreeText(t, rnames); //if(true)return; // visualize parse tree in dialog box t.inspect(parser); if (parser.getNumberOfSyntaxErrors() <= 0) { //ParseTreeWalker walker = new ParseTreeWalker(); String groupFile = "/programming/proys/SWB4/swb/SWBPlatform/src/org/semanticwb/rdf/sparql/ident.stg"; if (args.length > 1) { groupFile = args[1]; } System.out.println("Read StringTemplate Group File: " + groupFile + "-------------------------------"); STGroup g = new STGroupFile(groupFile); // IdentVisitor visitor = new IdentVisitor(); // visitor.setSTGroup(g); // ST query = visitor.visit(t); // // System.out.println("Emit reformatted query: -------------------------------"); // // System.out.println(query.render(lineWidth)); // // System.out.println("Emit original query: -------------------------------"); // // String q = query.render(lineWidth); // // /* get common token stream */ // File tmpFile = File.createTempFile("query_", ".rq"); // FileOutputStream fo = new FileOutputStream(tmpFile); // OutputStreamWriter ow = new OutputStreamWriter(fo, "UTF8"); // ow.write(q); // ow.close(); // /* transformation pipline // * step 1: Unicode pre-processing // * step 2: Lexical analysis // */ // lex = new SparqlLexer(new ANTLRFileStream(tmpFile.getCanonicalPath(), "UTF8")); tokens = new CommonTokenStream(lex); List formattedTokenList = tokens.getTokens(); it = tokenList.iterator(); Iterator fit = formattedTokenList.iterator(); while (it.hasNext()) { Token originalToken = (Token) it.next(); if (originalToken.getType() != SparqlParser.EOF) { if (originalToken.getType() == SparqlParser.WS || originalToken.getType() == SparqlParser.COMMENT) { String s = originalToken.getText(); s = s.replace("\r\n", "\n"); System.out.print(s); } else { System.out.print(originalToken.getText()); } } } System.out.flush(); } System.out.println("-------------------------------"); System.out.println("Number of errors encountered: " + parser.getNumberOfSyntaxErrors()); }
From source file:org.shirolang.playground.SyntaxHighlighter.java
License:Open Source License
private void add(TerminalNode ident, String style) { if (ident != null) { Token t = ident.getSymbol(); int spacer = t.getStartIndex() - lastEnd; if (spacer > 0) { spansBuilder.add(Collections.emptyList(), spacer); int gap = t.getText().length(); spansBuilder.add(Collections.singleton(style), gap); lastEnd = t.getStopIndex() + 1; }/* www .ja v a 2s . c o m*/ } }
From source file:org.smallpearl.compiler.Symbol.java
License:BSD License
public Symbol(Token t, int type) { this(t.getText(), t.getLine(), t.getCharPositionInLine(), type); }
From source file:org.sourcepit.ltk.format.CommentParserDelegate.java
License:Apache License
@Override public int getLen(RuleNode parent, Token token, List<Token> hiddenTokensToRight, TokenStream tokenStream) { int len = 0;//w w w . j a v a 2s .com for (org.antlr.v4.runtime.Token hiddenToken : hiddenTokensToRight) { final String text = hiddenToken.getText(); if (TokenUtils.isWs(text)) { len++; } else { if (hiddenToken.getCharPositionInLine() > token.getCharPositionInLine()) { len++; } else { break; } } } return len; }
From source file:org.sourcepit.ltk.parser.LittleJParserDelegate.java
License:Apache License
@Override public ParseResult parseNestedLanguage(Class<? extends Lexer> sourceType, Token token) { if (LittleJLexer.class == sourceType) { if (LittleJLexer.COMMENT == token.getType()) { final CharStream charStream = new ANTLRInputStream(token.getText()); final Lexer lexer = new CommentLexer(charStream); final BufferedTokenStream tokenStream = new CommonTokenStream(lexer); final CommentParser parser = new CommentParser(tokenStream); try { return new ParseResult(lexer, tokenStream, parser, parser.comment()); } catch (RecognitionException e) { return new ParseResult(lexer, tokenStream, parser, e); }/*w ww.j a v a 2s . c o m*/ } } return null; }
From source file:org.sourcepit.ltk.parser.ParseTreeBuilder.java
License:Apache License
private ParseNode handleTerminalNode(Terminal origin, Rule parent, TerminalNode terminalNode) { final ParseResult parseResult = parseResultStack.peek(); final Class<? extends Lexer> sourceType = parseResult.getLexer().getClass(); final org.antlr.v4.runtime.Token antlrToken = terminalNode.getSymbol(); final int tokenType = antlrToken.getType(); final int offset = offsetStack.peek().intValue() + antlrToken.getStartIndex(); final int channel = antlrToken.getChannel(); final String text = antlrToken.getText(); final TokenType type = new TokenType(sourceType, tokenType); final Token token = new Token(type, channel, offset, text); final Terminal previous = terminals.isEmpty() ? null : terminals.getLast(); final Terminal terminal = new Terminal(previous, parent, token, origin); final ParseResult nestedParseResult = parserDelegeate.parseNestedLanguage(sourceType, antlrToken); if (nestedParseResult == null) { terminals.add(terminal);/*from w w w .j a v a2s .c o m*/ return terminal; } else { offsetStack.push(Integer.valueOf(antlrToken.getStartIndex())); try { return handleParseResult(terminal, parent, nestedParseResult); } finally { offsetStack.pop(); } } }
From source file:org.structr.core.graphql.GraphQLRequest.java
License:Open Source License
public static Document parse(final Parser parser, final String query) throws FrameworkException { try {/*from www . j ava2 s .com*/ return parser.parseDocument(query); } catch (Throwable t) { String message = t.getMessage(); if (message == null) { message = t.getClass().getName(); if (t instanceof ParseCancellationException) { final Throwable cause = t.getCause(); if (cause instanceof RecognitionException) { final RecognitionException err = (RecognitionException) cause; final Token offendingToken = err.getOffendingToken(); if (offendingToken != null) { final int line = offendingToken.getLine(); final int column = offendingToken.getCharPositionInLine(); final String text = offendingToken.getText(); message = "Parse error at " + text + " in line " + line + ", column " + column; } } } } final FrameworkException fex = new FrameworkException(422, message); final Map<String, String> data = new LinkedHashMap<>(); // do not output an empty array of errors fex.setErrorBuffer(null); fex.setData(data); data.put("query", query); throw fex; } }
From source file:org.tinygroup.template.parser.TinyTemplateCodeVisitor.java
License:Open Source License
public CodeBlock visitText(@NotNull TinyTemplateParser.TextContext ctx) { Token token = ((TerminalNode) ctx.getChild(0)).getSymbol(); String text = token.getText(); switch (token.getType()) { case TinyTemplateParser.TEXT_PLAIN: if (strictFormat) text = text.trim();//from ww w . ja v a 2 s .c o m break; case TinyTemplateParser.TEXT_CDATA: text = text.substring(3, text.length() - 3); break; case TinyTemplateParser.TEXT_ESCAPED_CHAR: text = text.substring(1); break; default: break; } if (text.length() == 0) { return null; } return new CodeBlock().header(new CodeLet().code("write($writer,\"") .code(escapeJavaStyleString(text).toString()).lineCode("\");")); }
From source file:org.tvl.goworks.editor.go.parser.GoParser.java
License:Open Source License
@Override protected boolean isPackageName(Token token) { return token != null && packageNames.contains(token.getText()); }
From source file:org.tvl.goworks.editor.go.parser.GoParser.java
License:Open Source License
@Override protected boolean isBuiltInMethodName(Token token) { return token != null && SemanticHighlighter.PREDEFINED_FUNCTIONS.contains(token.getText()); }