List of usage examples for javax.xml.stream.events Attribute getLocation
javax.xml.stream.Location getLocation();
From source file:edu.jhu.hlt.concrete.ingesters.webposts.WebPostIngester.java
@Override public Communication fromCharacterBasedFile(final Path path) throws IngestException { if (!Files.exists(path)) throw new IngestException("No file at: " + path.toString()); AnalyticUUIDGeneratorFactory f = new AnalyticUUIDGeneratorFactory(); AnalyticUUIDGenerator g = f.create(); Communication c = new Communication(); c.setUuid(g.next());/*from w w w . j a v a 2 s . c o m*/ c.setType(this.getKind()); c.setMetadata(TooledMetadataConverter.convert(this)); try { ExistingNonDirectoryFile ef = new ExistingNonDirectoryFile(path); c.setId(ef.getName().split("\\.")[0]); } catch (NoSuchFileException | NotFileException e) { // might throw if path is a directory. throw new IngestException(path.toString() + " is not a file, or is a directory."); } String content; try (InputStream is = Files.newInputStream(path); BufferedInputStream bin = new BufferedInputStream(is, 1024 * 8 * 8);) { content = IOUtils.toString(bin, StandardCharsets.UTF_8); c.setText(content); } catch (IOException e) { throw new IngestException(e); } try (InputStream is = Files.newInputStream(path); BufferedInputStream bin = new BufferedInputStream(is, 1024 * 8 * 8); BufferedReader reader = new BufferedReader(new InputStreamReader(bin, StandardCharsets.UTF_8));) { XMLEventReader rdr = null; try { rdr = inF.createXMLEventReader(reader); // Below method moves the reader // to the headline end element. Section headline = this.handleBeginning(rdr, content, c); headline.setUuid(g.next()); c.addToSectionList(headline); TextSpan sts = headline.getTextSpan(); LOGGER.debug("headline text: {}", c.getText().substring(sts.getStart(), sts.getEnding())); int sectNumber = 1; int subSect = 0; int currOff = -1; // Big amounts of characters. while (rdr.hasNext()) { XMLEvent nextEvent = rdr.nextEvent(); currOff = nextEvent.getLocation().getCharacterOffset(); // First: see if document is going to end. // If yes: exit. if (nextEvent.isEndDocument()) break; // region // enables ingestion of quotes inside a usenet webpost. // by Tongfei Chen if (nextEvent.isStartElement() && nextEvent.asStartElement().getName().equals(QName.valueOf("QUOTE"))) { Attribute attrQuote = nextEvent.asStartElement() .getAttributeByName(QName.valueOf("PREVIOUSPOST")); String quote = StringEscapeUtils.escapeXml(attrQuote.getValue()); int location = attrQuote.getLocation().getCharacterOffset() + "<QUOTE PREVIOUSPOST=\"".length(); Section quoteSection = new Section(g.next(), "quote") .setTextSpan(new TextSpan(location, location + quote.length())); c.addToSectionList(quoteSection); } // endregion // Check if start element. if (nextEvent.isCharacters()) { Characters chars = nextEvent.asCharacters(); if (!chars.isWhiteSpace()) { String fpContent = chars.getData(); LOGGER.debug("Character offset: {}", currOff); LOGGER.debug("Character based data: {}", fpContent); SimpleImmutableEntry<Integer, Integer> pads = trimSpacing(fpContent); final int tsb = currOff + pads.getKey(); final int tse = currOff + fpContent.replace("\"", """).replace("<", "<") .replace(">", ">").length() - (pads.getValue()); // MAINTAIN CORRECT TEXT SPAN // CANNOT USE StringEscapeUtils.escapeXml because it will escape "'", which // is not escaped in the data // @tongfei LOGGER.debug("Section text: {}", content.substring(tsb, tse)); TextSpan ts = new TextSpan(tsb, tse); String sk; if (subSect == 0) sk = "poster"; else if (subSect == 1) sk = "postdate"; else sk = "post"; Section s = new Section(); s.setKind(sk); s.setTextSpan(ts); s.setUuid(g.next()); List<Integer> intList = new ArrayList<>(); intList.add(sectNumber); intList.add(subSect); s.setNumberList(intList); c.addToSectionList(s); subSect++; } } else if (nextEvent.isEndElement()) { EndElement ee = nextEvent.asEndElement(); currOff = ee.getLocation().getCharacterOffset(); QName name = ee.getName(); String localName = name.getLocalPart(); LOGGER.debug("Hit end element: {}", localName); if (localName.equalsIgnoreCase(POST_LOCAL_NAME)) { LOGGER.debug("Switching to new post."); sectNumber++; subSect = 0; } else if (localName.equalsIgnoreCase(TEXT_LOCAL_NAME)) { // done with document. break; } } } return c; } catch (XMLStreamException | ConcreteException | StringIndexOutOfBoundsException | ClassCastException x) { throw new IngestException(x); } finally { if (rdr != null) try { rdr.close(); } catch (XMLStreamException e) { // not likely. LOGGER.info("Error closing XMLReader.", e); } } } catch (IOException e) { throw new IngestException(e); } }
From source file:edu.jhu.hlt.concrete.ingesters.bolt.BoltForumPostIngester.java
@Override public Communication fromCharacterBasedFile(final Path path) throws IngestException { if (!Files.exists(path)) throw new IngestException("No file at: " + path.toString()); AnalyticUUIDGeneratorFactory f = new AnalyticUUIDGeneratorFactory(); AnalyticUUIDGenerator gen = f.create(); Communication c = new Communication(); c.setUuid(gen.next());/*from w w w .j ava 2s .c o m*/ c.setType(this.getKind()); c.setMetadata(TooledMetadataConverter.convert(this)); try { ExistingNonDirectoryFile ef = new ExistingNonDirectoryFile(path); c.setId(ef.getName().split("\\.")[0]); } catch (NoSuchFileException | NotFileException e) { // might throw if path is a directory. throw new IngestException(path.toString() + " is not a file, or is a directory."); } String content; try (InputStream is = Files.newInputStream(path); BufferedInputStream bin = new BufferedInputStream(is, 1024 * 8 * 8);) { content = IOUtils.toString(bin, StandardCharsets.UTF_8); c.setText(content); } catch (IOException e) { throw new IngestException(e); } try (InputStream is = Files.newInputStream(path); BufferedInputStream bin = new BufferedInputStream(is, 1024 * 8 * 8); BufferedReader reader = new BufferedReader(new InputStreamReader(bin, StandardCharsets.UTF_8));) { XMLEventReader rdr = null; try { rdr = inF.createXMLEventReader(reader); // Below method moves the reader // to the first post element. Section headline = handleHeadline(rdr, content); headline.setUuid(gen.next()); c.addToSectionList(headline); int start = headline.getTextSpan().getStart(); int ending = headline.getTextSpan().getEnding(); if (ending < start) ending = start; // @tongfei: handle empty headlines String htxt = c.getText().substring(start, ending); LOGGER.debug("headline text: {}", htxt); // Section indices. int sectNumber = 1; int subSect = 0; // Move iterator to post start element. this.iterateToPosts(rdr); // Offset pointer. int currOff = -1; SectionFactory sf = new SectionFactory(gen); // First post element. while (rdr.hasNext()) { XMLEvent nextEvent = rdr.nextEvent(); currOff = nextEvent.getLocation().getCharacterOffset(); if (currOff > 0) { int currOffPlus = currOff + 20; int currOffLess = currOff - 20; LOGGER.debug("Offset: {}", currOff); if (currOffPlus < content.length()) LOGGER.debug("Surrounding text: {}", content.substring(currOffLess, currOffPlus)); } // First: see if document is going to end. // If yes: exit. if (nextEvent.isEndDocument()) break; // XMLEvent peeker = rdr.peek(); // Check if start element. if (nextEvent.isStartElement()) { StartElement se = nextEvent.asStartElement(); QName name = se.getName(); final String localName = name.getLocalPart(); LOGGER.debug("Hit start element: {}", localName); //region // Add sections for authors and datetimes for each bolt post // by Tongfei Chen Attribute attrAuthor = se.getAttributeByName(QName.valueOf("author")); Attribute attrDateTime = se.getAttributeByName(QName.valueOf("datetime")); if (attrAuthor != null && attrDateTime != null) { int loc = attrAuthor.getLocation().getCharacterOffset(); int sectAuthorBeginningOffset = loc + "<post author=\"".length(); Section sectAuthor = sf.fromTextSpan(new TextSpan(sectAuthorBeginningOffset, sectAuthorBeginningOffset + attrAuthor.getValue().length()), "author"); c.addToSectionList(sectAuthor); int sectDateTimeBeginningOffset = sectAuthorBeginningOffset + attrAuthor.getValue().length() + " datetime=".length(); Section sectDateTime = sf.fromTextSpan( new TextSpan(sectDateTimeBeginningOffset, sectDateTimeBeginningOffset + attrDateTime.getValue().length()), "datetime"); c.addToSectionList(sectDateTime); } //endregion // Move past quotes, images, and links. if (localName.equals(QUOTE_LOCAL_NAME)) { this.handleQuote(rdr); } else if (localName.equals(IMG_LOCAL_NAME)) { this.handleImg(rdr); } else if (localName.equals(LINK_LOCAL_NAME)) { this.handleLink(rdr); } // not a start element } else if (nextEvent.isCharacters()) { Characters chars = nextEvent.asCharacters(); int coff = chars.getLocation().getCharacterOffset(); if (!chars.isWhiteSpace()) { // content to be captured String fpContent = chars.getData(); LOGGER.debug("Character offset: {}", coff); LOGGER.debug("Character based data: {}", fpContent); // LOGGER.debug("Character data via offset diff: {}", content.substring(coff - fpContent.length(), coff)); SimpleImmutableEntry<Integer, Integer> pads = trimSpacing(fpContent); final int tsb = currOff + pads.getKey(); final int tse = currOff + fpContent.length() - pads.getValue(); final String subs = content.substring(tsb, tse); if (subs.replaceAll("\\p{Zs}", "").replaceAll("\\n", "").isEmpty()) { LOGGER.info("Found empty section: skipping."); continue; } LOGGER.debug("Section text: {}", subs); TextSpan ts = new TextSpan(tsb, tse); Section s = sf.fromTextSpan(ts, "post"); List<Integer> intList = new ArrayList<>(); intList.add(sectNumber); intList.add(subSect); s.setNumberList(intList); c.addToSectionList(s); subSect++; } } else if (nextEvent.isEndElement()) { EndElement ee = nextEvent.asEndElement(); currOff = ee.getLocation().getCharacterOffset(); QName name = ee.getName(); String localName = name.getLocalPart(); LOGGER.debug("Hit end element: {}", localName); if (localName.equalsIgnoreCase(POST_LOCAL_NAME)) { sectNumber++; subSect = 0; } } } return c; } catch (XMLStreamException | ConcreteException | StringIndexOutOfBoundsException x) { throw new IngestException(x); } finally { if (rdr != null) try { rdr.close(); } catch (XMLStreamException e) { // not likely. LOGGER.info("Error closing XMLReader.", e); } } } catch (IOException e) { throw new IngestException(e); } }