List of usage examples for org.apache.commons.io IOUtils toByteArray
public static byte[] toByteArray(Reader input, String encoding) throws IOException
Reader as a byte[] using the specified character encoding. From source file:io.fluo.commoncrawl.data.util.ArchiveUtil.java
public static Page buildPage(ArchiveRecord archiveRecord) throws IOException, ParseException { if (archiveRecord.getHeader().getMimetype().equalsIgnoreCase("application/json")) { byte[] rawData = IOUtils.toByteArray(archiveRecord, archiveRecord.available()); if (rawData.length == 0) { return Page.EMPTY; }//from www. j a v a 2 s .c o m String jsonString = new String(rawData); if (jsonString.isEmpty()) { return Page.EMPTY; } JSONObject json; try { json = new JSONObject(new String(rawData)); } catch (JSONException e) { throw new ParseException(e.getMessage(), 0); } String pageUrl = archiveRecord.getHeader().getUrl(); if (!LinkUtil.isValid(pageUrl)) { return Page.EMPTY; } String pageDomain = LinkUtil.getTopPrivate(pageUrl); Page page = new Page(archiveRecord.getHeader().getUrl()); page.setCrawlDate(archiveRecord.getHeader().getDate()); try { JSONObject responseMeta = json.getJSONObject("Envelope").getJSONObject("Payload-Metadata") .getJSONObject("HTTP-Response-Metadata"); if (archiveRecord.getHeader().getMimetype().equals("application/json")) { try { JSONArray links = responseMeta.getJSONObject("HTML-Metadata").getJSONArray("Links"); for (int i = 0; i < links.length(); i++) { JSONObject link = links.getJSONObject(i); if (link.has("path") && link.get("path").equals("A@/href") && link.has("url")) { String anchorText = ""; if (link.has("text")) { anchorText = link.getString("text"); } else if (link.has("title")) { anchorText = link.getString("title"); } String linkUrl = link.getString("url"); if (LinkUtil.isValid(linkUrl)) { String linkDomain = LinkUtil.getTopPrivate(linkUrl); if (!pageDomain.equalsIgnoreCase(linkDomain)) { page.addOutboundLink(linkUrl, anchorText); } } } } } catch (JSONException e) { log.debug("Exception trying retrieve links", e); } } try { page.setTitle( responseMeta.getJSONObject("HTML-Metadata").getJSONObject("Head").getString("Title")); } catch (JSONException e) { log.debug("Failed to retrieve title", e); } try { page.setServer(responseMeta.getJSONObject("Headers").getString("Server")); } catch (JSONException e) { log.debug("Failed to retrieve server", e); } } catch (JSONException e) { log.debug("Exception trying retrieve responseMeta", e); } return page; } return Page.EMPTY; }
From source file:io.fluo.webindex.data.util.ArchiveUtil.java
public static Page buildPage(ArchiveRecord archiveRecord) throws IOException, ParseException { if (archiveRecord.getHeader().getMimetype().equalsIgnoreCase("application/json")) { byte[] rawData = IOUtils.toByteArray(archiveRecord, archiveRecord.available()); if (rawData.length == 0) { return Page.EMPTY; }/*w ww . j a va 2 s .c o m*/ String jsonString = new String(rawData); if (jsonString.isEmpty()) { return Page.EMPTY; } JSONObject json; try { json = new JSONObject(new String(rawData)); } catch (JSONException e) { throw new ParseException(e.getMessage(), 0); } String rawPageUrl = archiveRecord.getHeader().getUrl(); URL pageUrl; try { pageUrl = DataUrl.from(rawPageUrl); } catch (IllegalArgumentException e) { return Page.EMPTY; } catch (Exception e) { log.error("Unexpected exception while parsing raw page URL: " + rawPageUrl, e); return Page.EMPTY; } Page page = new Page(pageUrl.toPageID()); page.setCrawlDate(archiveRecord.getHeader().getDate()); try { JSONObject responseMeta = json.getJSONObject("Envelope").getJSONObject("Payload-Metadata") .getJSONObject("HTTP-Response-Metadata"); if (archiveRecord.getHeader().getMimetype().equals("application/json")) { try { JSONArray links = responseMeta.getJSONObject("HTML-Metadata").getJSONArray("Links"); for (int i = 0; i < links.length(); i++) { JSONObject link = links.getJSONObject(i); if (link.has("path") && link.get("path").equals("A@/href") && link.has("url")) { String anchorText = ""; if (link.has("text")) { anchorText = link.getString("text"); } else if (link.has("title")) { anchorText = link.getString("title"); } String rawLinkUrl = link.getString("url"); URL linkUrl; try { linkUrl = DataUrl.from(rawLinkUrl); if (!page.getDomain().equals(linkUrl.getDomain())) { page.addOutbound(Link.of(linkUrl, anchorText)); } } catch (IllegalArgumentException e) { log.debug("Failed to parse link: " + rawLinkUrl); } catch (Exception e) { log.error("Unexpected exception while parsing link URL: " + rawLinkUrl, e); } } } } catch (JSONException e) { log.debug("Exception trying retrieve links", e); } } try { page.setTitle( responseMeta.getJSONObject("HTML-Metadata").getJSONObject("Head").getString("Title")); } catch (JSONException e) { log.debug("Failed to retrieve title", e); } try { page.setServer(responseMeta.getJSONObject("Headers").getString("Server")); } catch (JSONException e) { log.debug("Failed to retrieve server", e); } } catch (JSONException e) { log.debug("Exception trying retrieve responseMeta", e); } return page; } return Page.EMPTY; }
From source file:com.scistor.tab.auth.controller.HomeRestController.java
@RequestMapping(value = "/license", method = RequestMethod.POST, consumes = "text/plain") public void postLicense(HttpServletRequest request) throws IOException { int size = request.getContentLength(); if (size > KeyValue.MAX_SIZE) { }/*from w w w. ja v a 2 s . c om*/ byte[] data = IOUtils.toByteArray(request.getInputStream(), size); License license = new LicenseValidator().decryptAndVerifyLicense(request.getInputStream()); keyValueRepository.save(new KeyValue("license", data)); }
From source file:com.github.zdsiyan.maven.plugin.smartconfig.internal.JSONPathConfigurator.java
@Override public ByteArrayOutputStream execute(InputStream in, Charset charset, List<PointHandle> pointhandles) throws IOException { Object json = JSON.parse(IOUtils.toByteArray(new InputStreamReader(in), charset), Feature.OrderedField); //JSONObject jsonObject = JSON.parseObject(IOUtils.toString(in, charset), Feature.OrderedField); JSONPath path;//from ww w .j a v a 2s .c om for (PointHandle point : pointhandles) { path = JSONPath.compile(point.getExpression()); switch (point.getMode()) { case insert: //jsonObject.put(path, point.getValue()); path.set(json, point.getValue()); break; case delete: //jsonObject.remove(path); path.set(json, ""); break; case replace: default: path.set(json, point.getValue()); //path.set(jsonObject, point.getValue()); break; } } String text = JSON.toJSONString(json, true); ByteArrayOutputStream out = new ByteArrayOutputStream(); //out.write(jsonObject.toString().getBytes(charset)); out.write(text.getBytes(charset)); return out; }
From source file:com.epam.wilma.test.server.PerformanceTestHandler.java
/** * Constructor reads xml from file.// w ww. j a va 2s . co m * @throws IOException when converting to byte array fails. */ public PerformanceTestHandler() throws IOException { InputStream xmlLarge = getXmlFromFile("response_large.fis"); xmlLargeAsBytes = IOUtils.toByteArray(xmlLarge, xmlLarge.available()); InputStream xmlMax = getXmlFromFile("response_max.fis"); xmlMaxAsBytes = IOUtils.toByteArray(xmlMax, xmlMax.available()); InputStream xmlMin = getXmlFromFile("response_min.fis"); xmlMinAsBytes = IOUtils.toByteArray(xmlMin, xmlMin.available()); InputStream xmlSmall = getXmlFromFile("response_small.fis"); xmlSmallAsBytes = IOUtils.toByteArray(xmlSmall, xmlSmall.available()); }
From source file:com.bikfalvi.java.web.WebState.java
/** * Sets the data.//w w w. jav a 2s . c om * @param data The data as a string. * @param charset The charset. * @throws IOException */ public void setData(String data, Charset charset) throws IOException { StringReader reader = new StringReader(data); this.data = IOUtils.toByteArray(reader, charset); }