List of usage examples for java.io ByteArrayInputStream close
public void close() throws IOException
From source file:com.digitalpebble.storm.crawler.bolt.ParserBolt.java
@Override public void execute(Tuple tuple) { eventCounter.scope("tuple_in").incrBy(1); byte[] content = tuple.getBinaryByField("content"); String url = tuple.getStringByField("url"); Metadata metadata = (Metadata) tuple.getValueByField("metadata"); long start = System.currentTimeMillis(); // rely on mime-type provided by server or guess? ByteArrayInputStream bais = new ByteArrayInputStream(content); org.apache.tika.metadata.Metadata md = new org.apache.tika.metadata.Metadata(); LinkContentHandler linkHandler = new LinkContentHandler(); ContentHandler textHandler = new BodyContentHandler(-1); TeeContentHandler teeHandler = new TeeContentHandler(linkHandler, textHandler); ParseContext parseContext = new ParseContext(); try {// w w w . j a v a 2s. c om parseContext.set(HtmlMapper.class, (HtmlMapper) HTMLMapperClass.newInstance()); } catch (Exception e) { LOG.error("Exception while specifying HTMLMapper {}", url, e); } // build a DOM if required by the parseFilters DocumentFragment root = null; if (parseFilters.needsDOM()) { HTMLDocumentImpl doc = new HTMLDocumentImpl(); doc.setErrorChecking(false); root = doc.createDocumentFragment(); DOMBuilder domhandler = new DOMBuilder(doc, root); domhandler.setUpperCaseElementNames(upperCaseElementNames); domhandler.setDefaultNamespaceURI(XHTMLContentHandler.XHTML); teeHandler = new TeeContentHandler(linkHandler, textHandler, domhandler); } // parse String text; try { tika.getParser().parse(bais, teeHandler, md, parseContext); text = textHandler.toString(); } catch (Exception e) { String errorMessage = "Exception while parsing " + url + ": " + e; LOG.error(errorMessage); // send to status stream in case another component wants to update // its status metadata.setValue(Constants.STATUS_ERROR_SOURCE, "content parsing"); metadata.setValue(Constants.STATUS_ERROR_MESSAGE, errorMessage); collector.emit(StatusStreamName, tuple, new Values(url, metadata, Status.ERROR)); collector.ack(tuple); // Increment metric that is context specific eventCounter.scope("error_content_parsing_" + e.getClass().getSimpleName()).incrBy(1); // Increment general metric eventCounter.scope("parse exception").incrBy(1); return; } finally { try { bais.close(); } catch (IOException e) { LOG.error("Exception while closing stream", e); } } // add parse md to metadata for (String k : md.names()) { String[] values = md.getValues(k); metadata.setValues("parse." + k, values); } long duration = System.currentTimeMillis() - start; LOG.info("Parsed {} in {} msec", url, duration); // filter and convert the outlinks List<Outlink> outlinks = toOutlinks(url, linkHandler.getLinks(), metadata); // apply the parse filters if any try { parseFilters.filter(url, content, root, metadata, outlinks); } catch (RuntimeException e) { String errorMessage = "Exception while running parse filters on " + url + ": " + e; LOG.error(errorMessage); metadata.setValue(Constants.STATUS_ERROR_SOURCE, "content filtering"); metadata.setValue(Constants.STATUS_ERROR_MESSAGE, errorMessage); collector.emit(StatusStreamName, tuple, new Values(url, metadata, Status.ERROR)); collector.ack(tuple); // Increment metric that is context specific eventCounter.scope("error_content_filtering_" + e.getClass().getSimpleName()).incrBy(1); // Increment general metric eventCounter.scope("parse exception").incrBy(1); return; } if (emitOutlinks) { for (Outlink outlink : outlinks) { collector.emit(StatusStreamName, tuple, new Values(outlink.getTargetURL(), outlink.getMetadata(), Status.DISCOVERED)); } } collector.emit(tuple, new Values(url, content, metadata, text.trim())); collector.ack(tuple); eventCounter.scope("tuple_success").incrBy(1); }
From source file:com.digitalpebble.storm.crawler.tika.ParserBolt.java
@Override public void execute(Tuple tuple) { eventCounter.scope("tuple_in").incrBy(1); byte[] content = tuple.getBinaryByField("content"); String url = tuple.getStringByField("url"); Metadata metadata = (Metadata) tuple.getValueByField("metadata"); long start = System.currentTimeMillis(); // rely on mime-type provided by server or guess? ByteArrayInputStream bais = new ByteArrayInputStream(content); org.apache.tika.metadata.Metadata md = new org.apache.tika.metadata.Metadata(); LinkContentHandler linkHandler = new LinkContentHandler(); ContentHandler textHandler = new BodyContentHandler(-1); TeeContentHandler teeHandler = new TeeContentHandler(linkHandler, textHandler); ParseContext parseContext = new ParseContext(); try {//from w w w.j a v a 2s .c om parseContext.set(HtmlMapper.class, (HtmlMapper) HTMLMapperClass.newInstance()); } catch (Exception e) { LOG.error("Exception while specifying HTMLMapper {}", url, e); } // build a DOM if required by the parseFilters DocumentFragment root = null; if (parseFilters.needsDOM()) { HTMLDocumentImpl doc = new HTMLDocumentImpl(); doc.setErrorChecking(false); root = doc.createDocumentFragment(); DOMBuilder domhandler = new DOMBuilder(doc, root); domhandler.setUpperCaseElementNames(upperCaseElementNames); domhandler.setDefaultNamespaceURI(XHTMLContentHandler.XHTML); teeHandler = new TeeContentHandler(linkHandler, textHandler, domhandler); } // parse String text; try { tika.getParser().parse(bais, teeHandler, md, parseContext); text = textHandler.toString(); } catch (Exception e) { String errorMessage = "Exception while parsing " + url + ": " + e; LOG.error(errorMessage); // send to status stream in case another component wants to update // its status metadata.setValue(Constants.STATUS_ERROR_SOURCE, "content parsing"); metadata.setValue(Constants.STATUS_ERROR_MESSAGE, errorMessage); collector.emit(StatusStreamName, tuple, new Values(url, metadata, Status.ERROR)); collector.ack(tuple); // Increment metric that is context specific eventCounter.scope("error_content_parsing_" + e.getClass().getSimpleName()).incrBy(1); // Increment general metric eventCounter.scope("parse exception").incrBy(1); return; } finally { try { bais.close(); } catch (IOException e) { LOG.error("Exception while closing stream", e); } } // add parse md to metadata for (String k : md.names()) { String[] values = md.getValues(k); metadata.setValues("parse." + k, values); } long duration = System.currentTimeMillis() - start; LOG.info("Parsed {} in {} msec", url, duration); // filter and convert the outlinks List<Outlink> outlinks = toOutlinks(url, linkHandler.getLinks(), metadata); ParseResult parse = new ParseResult(); parse.setOutlinks(outlinks); // parse data of the parent URL ParseData parseData = parse.get(url); parseData.setMetadata(metadata); parseData.setText(text); parseData.setContent(content); // apply the parse filters if any try { parseFilters.filter(url, content, root, parse); } catch (RuntimeException e) { String errorMessage = "Exception while running parse filters on " + url + ": " + e; LOG.error(errorMessage); metadata.setValue(Constants.STATUS_ERROR_SOURCE, "content filtering"); metadata.setValue(Constants.STATUS_ERROR_MESSAGE, errorMessage); collector.emit(StatusStreamName, tuple, new Values(url, metadata, Status.ERROR)); collector.ack(tuple); // Increment metric that is context specific eventCounter.scope("error_content_filtering_" + e.getClass().getSimpleName()).incrBy(1); // Increment general metric eventCounter.scope("parse exception").incrBy(1); return; } if (emitOutlinks) { for (Outlink outlink : outlinks) { collector.emit(StatusStreamName, tuple, new Values(outlink.getTargetURL(), outlink.getMetadata(), Status.DISCOVERED)); } } // emit each document/subdocument in the ParseResult object // there should be at least one ParseData item for the "parent" URL for (Map.Entry<String, ParseData> doc : parse) { ParseData parseDoc = doc.getValue(); collector.emit(tuple, new Values(doc.getKey(), parseDoc.getContent(), parseDoc.getMetadata(), parseDoc.getText())); } collector.ack(tuple); eventCounter.scope("tuple_success").incrBy(1); }
From source file:bftsmart.tom.core.TOMLayer.java
/** * This method is called by the MessageHandler each time it received messages related * to the leader change/* ww w.j ava 2s. c o m*/ * @param msg Message received from the other replica */ public void deliverTimeoutRequest(LCMessage msg) { ByteArrayInputStream bis = null; ObjectInputStream ois = null; switch (msg.getType()) { case TOMUtil.STOP: // message STOP { // this message is for the next leader change? if (msg.getReg() == lcManager.getLastReg() + 1) { Logger.println("(TOMLayer.deliverTimeoutRequest) received regency change request"); try { // deserialize the content of the STOP message bis = new ByteArrayInputStream(msg.getPayload()); ois = new ObjectInputStream(bis); boolean hasReqs = ois.readBoolean(); clientsManager.getClientsLock().lock(); if (hasReqs) { // Store requests that the other replica did not manage to order //TODO: The requests have to be verified! byte[] temp = (byte[]) ois.readObject(); BatchReader batchReader = new BatchReader(temp, controller.getStaticConf().getUseSignatures() == 1); TOMMessage[] requests = batchReader.deserialiseRequests(controller); } clientsManager.getClientsLock().unlock(); ois.close(); bis.close(); } catch (IOException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } catch (ClassNotFoundException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } // store information about the message STOP lcManager.addStop(msg.getReg(), msg.getSender()); evaluateStops(msg.getReg()); // evaluate STOP messages } } break; case TOMUtil.STOPDATA: { // STOPDATA messages int regency = msg.getReg(); // Am I the new leader, and am I expecting this messages? if (regency == lcManager.getLastReg() && this.controller.getStaticConf().getProcessId() == lm .getCurrentLeader()/*(regency % this.reconfManager.getCurrentViewN())*/) { Logger.println("(TOMLayer.deliverTimeoutRequest) I'm the new leader and I received a STOPDATA"); //TODO: It is necessary to verify the proof of the last decided consensus and the signature of the state of the current consensus! LastEidData lastData = null; SignedObject signedCollect = null; int last = -1; byte[] lastValue = null; Set<PaxosMessage> proof = null; try { // deserialize the content of the message bis = new ByteArrayInputStream(msg.getPayload()); ois = new ObjectInputStream(bis); if (ois.readBoolean()) { // content of the last decided eid last = ois.readInt(); lastValue = (byte[]) ois.readObject(); proof = (Set<PaxosMessage>) ois.readObject(); //TODO: Proof is missing! } lastData = new LastEidData(msg.getSender(), last, lastValue, proof); lcManager.addLastEid(regency, lastData); // conteudo do eid a executar signedCollect = (SignedObject) ois.readObject(); ois.close(); bis.close(); lcManager.addCollect(regency, signedCollect); int bizantineQuorum = (controller.getCurrentViewN() + controller.getCurrentViewF()) / 2; int cftQuorum = (controller.getCurrentViewN()) / 2; // I already got messages from a Byzantine/Crash quorum, // related to the last eid as well as for the current? boolean conditionBFT = (controller.getStaticConf().isBFT() && lcManager.getLastEidsSize(regency) > bizantineQuorum && lcManager.getCollectsSize(regency) > bizantineQuorum); boolean conditionCFT = (lcManager.getLastEidsSize(regency) > cftQuorum && lcManager.getCollectsSize(regency) > cftQuorum); if (conditionBFT || conditionCFT) catch_up(regency); } catch (IOException ex) { ex.printStackTrace(System.err); } catch (ClassNotFoundException ex) { ex.printStackTrace(System.err); } } } break; case TOMUtil.SYNC: // message SYNC { int regency = msg.getReg(); // I am waiting for this message, and I received it from the new leader? if (msg.getReg() == lcManager.getLastReg() && msg.getReg() == lcManager.getNextReg() && msg .getSender() == lm.getCurrentLeader()/*(regency % this.reconfManager.getCurrentViewN())*/) { LastEidData lastHighestEid = null; int currentEid = -1; HashSet<SignedObject> signedCollects = null; byte[] propose = null; int batchSize = -1; try { // deserialization of the message content bis = new ByteArrayInputStream(msg.getPayload()); ois = new ObjectInputStream(bis); lastHighestEid = (LastEidData) ois.readObject(); currentEid = ois.readInt(); signedCollects = (HashSet<SignedObject>) ois.readObject(); propose = (byte[]) ois.readObject(); batchSize = ois.readInt(); lcManager.setCollects(regency, signedCollects); // Is the predicate "sound" true? Is the certificate for LastEid valid? if (lcManager.sound(lcManager.selectCollects(regency, currentEid)) && (!controller.getStaticConf().isBFT() || lcManager.hasValidProof(lastHighestEid))) { finalise(regency, lastHighestEid, currentEid, signedCollects, propose, batchSize, false); } ois.close(); bis.close(); } catch (IOException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } catch (ClassNotFoundException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } } } break; } }
From source file:loci.formats.in.LIFReader.java
private Element getMetadataRoot(String xml) throws FormatException, IOException { ByteArrayInputStream s = null; try {//from www . j av a 2 s. co m DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder parser = factory.newDocumentBuilder(); s = new ByteArrayInputStream(xml.getBytes(ENCODING)); return parser.parse(s).getDocumentElement(); } catch (ParserConfigurationException e) { throw new FormatException(e); } catch (SAXException e) { throw new FormatException(e); } finally { if (s != null) s.close(); } }
From source file:org.dasein.cloud.vcloud.vCloudMethod.java
private @Nonnull Version getVersion() throws CloudException, InternalException { Cache<Version> cache = Cache.getInstance(provider, "vCloudVersions", Version.class, CacheLevel.CLOUD, new TimePeriod<Day>(1, TimePeriod.DAY)); ProviderContext ctx = provider.getContext(); if (ctx == null) { throw new CloudException("No context was defined for this request"); }//from ww w .ja v a2 s . c o m { Iterable<Version> versions = cache.get(ctx); Iterator<Version> it = (versions == null ? null : versions.iterator()); if (it != null && it.hasNext()) { return it.next(); } } // TODO: how does vCHS do version discovery? if (ctx.getCloud().getEndpoint().startsWith("https://vchs")) { // This is a complete hack that needs to be changed to reflect vCHS version discovery Version version = new Version(); version.loginUrl = ctx.getCloud().getEndpoint() + "/api/vchs/sessions"; version.version = "5.6"; cache.put(ctx, Collections.singletonList(version)); return version; } if (wire.isDebugEnabled()) { wire.debug(""); wire.debug(">>> [GET (" + (new Date()) + ")] -> " + ctx.getCloud().getEndpoint() + " >--------------------------------------------------------------------------------------"); } try { final String[] preferred = provider.getVersionPreference(); HttpClient client = getClient(false); HttpGet method = new HttpGet(ctx.getCloud().getEndpoint() + "/api/versions"); if (wire.isDebugEnabled()) { wire.debug(method.getRequestLine().toString()); for (Header header : method.getAllHeaders()) { wire.debug(header.getName() + ": " + header.getValue()); } wire.debug(""); } HttpResponse response; StatusLine status; try { APITrace.trace(provider, "GET versions"); response = client.execute(method); if (wire.isDebugEnabled()) { wire.debug(response.getStatusLine().toString()); for (Header header : response.getAllHeaders()) { wire.debug(header.getName() + ": " + header.getValue()); } wire.debug(""); } status = response.getStatusLine(); } catch (IOException e) { throw new CloudException(e); } if (status.getStatusCode() == HttpServletResponse.SC_OK) { HttpEntity entity = response.getEntity(); String body; try { body = EntityUtils.toString(entity); if (wire.isDebugEnabled()) { wire.debug(body); wire.debug(""); } } catch (IOException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } try { ByteArrayInputStream bas = new ByteArrayInputStream(body.getBytes()); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder parser = factory.newDocumentBuilder(); Document doc = parser.parse(bas); bas.close(); NodeList versions = doc.getElementsByTagName("VersionInfo"); TreeSet<Version> set = new TreeSet<Version>(new Comparator<Version>() { public int compare(Version version1, Version version2) { if (version1.equals(version2)) { return 0; } if (preferred != null) { for (String v : preferred) { if (v.equals(version1.version)) { return -1; } else if (v.equals(version2.version)) { return 1; } } } for (String v : VERSIONS) { if (v.equals(version1.version)) { return -1; } else if (v.equals(version2.version)) { return 1; } } return -version1.version.compareTo(version2.version); } }); for (int i = 0; i < versions.getLength(); i++) { Node versionInfo = versions.item(i); NodeList vattrs = versionInfo.getChildNodes(); String version = null; String url = null; for (int j = 0; j < vattrs.getLength(); j++) { Node attr = vattrs.item(j); if (attr.getNodeName().equalsIgnoreCase("Version") && attr.hasChildNodes()) { version = attr.getFirstChild().getNodeValue().trim(); } else if (attr.getNodeName().equalsIgnoreCase("LoginUrl") && attr.hasChildNodes()) { url = attr.getFirstChild().getNodeValue().trim(); } } if (version == null || url == null || !isSupported(version)) { continue; } Version v = new Version(); v.version = version; v.loginUrl = url; set.add(v); } if (set.isEmpty()) { throw new CloudException("Unable to identify a supported version"); } Version v = set.iterator().next(); cache.put(ctx, set); return v; } catch (IOException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } catch (ParserConfigurationException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } catch (SAXException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } } else { logger.error("Expected OK for GET request, got " + status.getStatusCode()); String xml = null; try { HttpEntity entity = response.getEntity(); if (entity != null) { xml = EntityUtils.toString(entity); if (wire.isDebugEnabled()) { wire.debug(xml); wire.debug(""); } } } catch (IOException e) { logger.error("Failed to read response error due to a cloud I/O error: " + e.getMessage()); throw new CloudException(e); } vCloudException.Data data = null; if (xml != null && !xml.equals("")) { Document doc = parseXML(xml); String docElementTagName = doc.getDocumentElement().getTagName(); String nsString = ""; if (docElementTagName.contains(":")) nsString = docElementTagName.substring(0, docElementTagName.indexOf(":") + 1); NodeList errors = doc.getElementsByTagName(nsString + "Error"); if (errors.getLength() > 0) { data = vCloudException.parseException(status.getStatusCode(), errors.item(0)); } } if (data == null) { throw new vCloudException(CloudErrorType.GENERAL, status.getStatusCode(), response.getStatusLine().getReasonPhrase(), "No further information"); } logger.error("[" + status.getStatusCode() + " : " + data.title + "] " + data.description); throw new vCloudException(data); } } finally { if (wire.isDebugEnabled()) { wire.debug("<<< [GET (" + (new Date()) + ")] -> " + ctx.getEndpoint() + " <--------------------------------------------------------------------------------------"); wire.debug(""); } } }
From source file:org.syncope.core.rest.controller.ReportController.java
@PreAuthorize("hasRole('REPORT_READ')") @RequestMapping(method = RequestMethod.GET, value = "/execution/export/{executionId}") @Transactional(readOnly = true)//from ww w .java2s . com public void exportExecutionResult(final HttpServletResponse response, @PathVariable("executionId") final Long executionId, @RequestParam(value = "fmt", required = false) final ReportExecExportFormat fmt) throws NotFoundException { ReportExec reportExec = reportExecDAO.find(executionId); if (reportExec == null) { throw new NotFoundException("Report execution " + executionId); } if (!ReportExecStatus.SUCCESS.name().equals(reportExec.getStatus()) || reportExec.getExecResult() == null) { SyncopeClientCompositeErrorException sccee = new SyncopeClientCompositeErrorException( HttpStatus.BAD_REQUEST); SyncopeClientException sce = new SyncopeClientException(SyncopeClientExceptionType.InvalidReportExec); sce.addElement(reportExec.getExecResult() == null ? "No report data produced" : "Report did not run successfully"); sccee.addException(sce); throw sccee; } ReportExecExportFormat format = fmt == null ? ReportExecExportFormat.XML : fmt; LOG.debug("Exporting result of {} as {}", reportExec, format); response.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE); response.addHeader("Content-Disposition", "attachment; filename=" + reportExec.getReport().getName() + "." + format.name().toLowerCase()); // streaming SAX handler from a compressed byte array stream ByteArrayInputStream bais = new ByteArrayInputStream(reportExec.getExecResult()); ZipInputStream zis = new ZipInputStream(bais); try { // a single ZipEntry in the ZipInputStream (see ReportJob) zis.getNextEntry(); Pipeline<SAXPipelineComponent> pipeline = new NonCachingPipeline<SAXPipelineComponent>(); pipeline.addComponent(new XMLGenerator(zis)); Map<String, Object> parameters = new HashMap<String, Object>(); parameters.put("status", reportExec.getStatus()); parameters.put("message", reportExec.getMessage()); parameters.put("startDate", reportExec.getStartDate()); parameters.put("endDate", reportExec.getEndDate()); switch (format) { case HTML: XSLTTransformer xsl2html = new XSLTTransformer(getClass().getResource("/report/report2html.xsl")); xsl2html.setParameters(parameters); pipeline.addComponent(xsl2html); pipeline.addComponent(XMLSerializer.createXHTMLSerializer()); break; case PDF: XSLTTransformer xsl2pdf = new XSLTTransformer(getClass().getResource("/report/report2fo.xsl")); xsl2pdf.setParameters(parameters); pipeline.addComponent(xsl2pdf); pipeline.addComponent(new FopSerializer(MimeConstants.MIME_PDF)); break; case RTF: XSLTTransformer xsl2rtf = new XSLTTransformer(getClass().getResource("/report/report2fo.xsl")); xsl2rtf.setParameters(parameters); pipeline.addComponent(xsl2rtf); pipeline.addComponent(new FopSerializer(MimeConstants.MIME_RTF)); break; case XML: default: pipeline.addComponent(XMLSerializer.createXMLSerializer()); } pipeline.setup(response.getOutputStream()); pipeline.execute(); LOG.debug("Result of {} successfully exported as {}", reportExec, format); } catch (Throwable t) { LOG.error("While exporting content", t); } finally { try { zis.close(); bais.close(); } catch (IOException e) { LOG.error("While closing stream for execution result", e); } } }
From source file:de.innovationgate.utils.WGUtils.java
/** * Unzips compressed bytes previously produced by {@link #zip(byte[])@} * @param input The compressed bytes/*w w w. j a v a2 s. c om*/ */ public static byte[] unzip(byte[] input) throws IOException { String unzipped = null; ByteArrayInputStream byteIn = new ByteArrayInputStream(input); PatchedGZIPInputStream zipIn = new PatchedGZIPInputStream(byteIn); ByteArrayOutputStream out = new ByteArrayOutputStream(); int numBytesRead = 0; byte[] tempBytes = new byte[1024]; while ((numBytesRead = zipIn.read(tempBytes, 0, tempBytes.length)) != -1) { out.write(tempBytes, 0, numBytesRead); } byte[] outBytes = out.toByteArray(); out.close(); zipIn.close(); byteIn.close(); return outBytes; }
From source file:org.dasein.cloud.vcloud.vCloudMethod.java
private void loadVDCs(@Nonnull Org org) throws CloudException, InternalException { if (wire.isDebugEnabled()) { wire.debug(""); wire.debug(">>> [GET (" + (new Date()) + ")] -> " + org.url + " >--------------------------------------------------------------------------------------"); }//www . j a v a 2s . com try { HttpClient client = getClient(false); HttpGet method = new HttpGet(org.url); method.addHeader("Accept", "application/*+xml;version=" + org.version.version + ",application/*+xml;version=" + org.version.version); addAuth(method, org.token); if (wire.isDebugEnabled()) { wire.debug(method.getRequestLine().toString()); for (Header header : method.getAllHeaders()) { wire.debug(header.getName() + ": " + header.getValue()); } wire.debug(""); } HttpResponse response; StatusLine status; try { APITrace.trace(provider, "GET org"); response = client.execute(method); if (wire.isDebugEnabled()) { wire.debug(response.getStatusLine().toString()); for (Header header : response.getAllHeaders()) { wire.debug(header.getName() + ": " + header.getValue()); } wire.debug(""); } status = response.getStatusLine(); } catch (IOException e) { throw new CloudException(e); } if (status.getStatusCode() == HttpServletResponse.SC_OK) { HttpEntity entity = response.getEntity(); String body; try { body = EntityUtils.toString(entity); if (wire.isDebugEnabled()) { wire.debug(body); wire.debug(""); } } catch (IOException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } try { ByteArrayInputStream bas = new ByteArrayInputStream(body.getBytes()); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder parser = factory.newDocumentBuilder(); ArrayList<VDC> vdcs = new ArrayList<VDC>(); Document doc = parser.parse(bas); bas.close(); NodeList links = doc.getElementsByTagName("Link"); for (int i = 0; i < links.getLength(); i++) { Node link = links.item(i); if (link.hasAttributes()) { Node type = link.getAttributes().getNamedItem("type"); if (type != null && type.getNodeValue().trim().equals("application/vnd.vmware.vcloud.vdc+xml")) { Node name = link.getAttributes().getNamedItem("name"); if (name != null) { DataCenter dc = new DataCenter(); VDC vdc = new VDC(); vdc.actions = new HashMap<String, String>(); dc.setActive(true); dc.setAvailable(true); dc.setName(name.getNodeValue().trim()); dc.setRegionId(org.region.getProviderRegionId()); Node href = link.getAttributes().getNamedItem("href"); if (href != null) { String id = provider.toID(href.getNodeValue().trim()); dc.setProviderDataCenterId(id); vdc.dataCenter = dc; loadVDC(vdc, id); vdcs.add(vdc); } } } } } org.setVdcs(vdcs); } catch (IOException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } catch (ParserConfigurationException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } catch (SAXException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } } else { logger.error("Expected OK for GET request, got " + status.getStatusCode()); String xml = null; try { HttpEntity entity = response.getEntity(); if (entity != null) { xml = EntityUtils.toString(entity); if (wire.isDebugEnabled()) { wire.debug(xml); wire.debug(""); } } } catch (IOException e) { logger.error("Failed to read response error due to a cloud I/O error: " + e.getMessage()); throw new CloudException(e); } vCloudException.Data data = null; if (xml != null && !xml.equals("")) { Document doc = parseXML(xml); String docElementTagName = doc.getDocumentElement().getTagName(); String nsString = ""; if (docElementTagName.contains(":")) nsString = docElementTagName.substring(0, docElementTagName.indexOf(":") + 1); NodeList errors = doc.getElementsByTagName(nsString + "Error"); if (errors.getLength() > 0) { data = vCloudException.parseException(status.getStatusCode(), errors.item(0)); } } if (data == null) { throw new vCloudException(CloudErrorType.GENERAL, status.getStatusCode(), response.getStatusLine().getReasonPhrase(), "No further information"); } logger.error("[" + status.getStatusCode() + " : " + data.title + "] " + data.description); throw new vCloudException(data); } } finally { if (wire.isDebugEnabled()) { wire.debug("<<< [GET (" + (new Date()) + ")] -> " + org.url + " <--------------------------------------------------------------------------------------"); wire.debug(""); } } }
From source file:com.knowgate.dfs.FileSystem.java
/** * Download an HTML page and all its referenced files into a ZIP * @param sBasePath String Base path for page and its referenced files * @param sFilePath String File path from sBasePath * @param oOutStrm OutputStream where ZIP is written * @param sDefaultEncoding Character encoding of file to be downloaded * @throws IOException// w w w .ja v a2s .c o m * @since 7.0 */ public void downloadhtmlpage(String sBasePath, String sFilePath, OutputStream oOutStrm, String sDefaultEncoding) throws IOException { if (DebugFile.trace) { DebugFile.writeln("Begin FileSystem.downloadhtmlpage(" + sBasePath + "," + sFilePath + ",[OutputStream]," + sDefaultEncoding + ")"); DebugFile.incIdent(); } String sEncoding = sDefaultEncoding; String sBaseHref = ""; boolean bAutoDetectEncoding = (sDefaultEncoding == null); TreeSet<String> oFiles = new TreeSet<String>(); TreeSet<String> oEntries = new TreeSet<String>(); Perl5Matcher oMatcher = new Perl5Matcher(); Perl5Matcher oReplacer = new Perl5Matcher(); Perl5Compiler oCompiler = new Perl5Compiler(); if (sDefaultEncoding == null) sDefaultEncoding = "ASCII"; try { String sHtml = readfilestr(sBasePath + sFilePath, sDefaultEncoding); if (null == sHtml) { if (DebugFile.trace) { DebugFile.writeln("Could not read file " + sBasePath + sFilePath); DebugFile.decIdent(); throw new IOException("Could not read file " + sBasePath + sFilePath); } } if (DebugFile.trace) { DebugFile.writeln( String.valueOf(sHtml.length()) + " characters readed from file " + sBasePath + sFilePath); } if (bAutoDetectEncoding) { if (oMatcher.contains(sHtml, oCompiler.compile( "<meta\\x20+http-equiv=(\"|')?Content-Type(\"|')?\\x20+content=(\"|')?text/html;\\x20+charset=(\\w|-){3,32}(\"|')?>", Perl5Compiler.CASE_INSENSITIVE_MASK))) { if (DebugFile.trace) DebugFile.writeln("<meta http-equiv> tag found"); String sHttpEquiv = oMatcher.getMatch().toString(); int iCharset = Gadgets.indexOfIgnoreCase(sHttpEquiv, "charset="); if (iCharset > 0) { int iQuoute = sHttpEquiv.indexOf('"', iCharset); if (iQuoute < 0) iQuoute = sHttpEquiv.indexOf((char) 39, iCharset); if (iQuoute < 0) { bAutoDetectEncoding = true; } else { sEncoding = sHttpEquiv.substring(iCharset + 8, iQuoute); if (DebugFile.trace) DebugFile.writeln("setting charset encoding to " + sEncoding); bAutoDetectEncoding = false; try { byte[] aTest = new String("Test").getBytes(sEncoding); } catch (UnsupportedEncodingException uex) { bAutoDetectEncoding = true; } } } else { bAutoDetectEncoding = true; } } else { bAutoDetectEncoding = true; } } if (bAutoDetectEncoding) { if (DebugFile.trace) DebugFile.writeln("Autodetecting encoding"); ByteArrayInputStream oHtmlStrm = new ByteArrayInputStream(sHtml.getBytes(sDefaultEncoding)); sEncoding = new CharacterSetDetector().detect(oHtmlStrm, sDefaultEncoding); oHtmlStrm.close(); if (DebugFile.trace) DebugFile.writeln("Encoding set to " + sEncoding); } Pattern oPattern = oCompiler.compile("<base(\\x20)+href=(\"|')?([^'\"\\r\\n]+)(\"|')?(\\x20)*/?>", Perl5Compiler.CASE_INSENSITIVE_MASK); if (oMatcher.contains(sHtml, oPattern)) { sBaseHref = Gadgets.chomp(oMatcher.getMatch().group(3), "/"); if (DebugFile.trace) DebugFile.writeln("<base href=" + sBaseHref + ">"); } PatternMatcherInput oMatchInput = new PatternMatcherInput(sHtml); oPattern = oCompiler.compile( "\\x20(src=|background=|background-image:url\\x28)(\"|')?([^'\"\\r\\n]+)(\"|')?(\\x20|\\x29|/|>)", Perl5Compiler.CASE_INSENSITIVE_MASK); StringSubstitution oSrcSubs = new StringSubstitution(); int nMatches = 0; while (oMatcher.contains(oMatchInput, oPattern)) { nMatches++; String sMatch = oMatcher.getMatch().toString(); String sAttr = oMatcher.getMatch().group(1); String sQuo = oMatcher.getMatch().group(2); if (sQuo == null) sQuo = ""; String sSrc = oMatcher.getMatch().group(3); if (DebugFile.trace) DebugFile.writeln("Source file found at " + sSrc); String sEnd = oMatcher.getMatch().group(5); if (!oFiles.contains(sSrc)) oFiles.add(sSrc); String sFilename = sSrc.substring(sSrc.replace('\\', '/').lastIndexOf('/') + 1); if (DebugFile.trace) DebugFile.writeln("StringSubstitution.setSubstitution(" + sMatch + " replace with " + sMatch.substring(0, sAttr.length() + 1) + sQuo + sFilename + sQuo + sEnd + ")"); oSrcSubs.setSubstitution(sMatch.substring(0, sAttr.length() + 1) + sQuo + sFilename + sQuo + sEnd); sHtml = Util.substitute(oReplacer, oCompiler.compile(sMatch), oSrcSubs, sHtml, Util.SUBSTITUTE_ALL); } //wend oMatchInput = new PatternMatcherInput(sHtml); oPattern = oCompiler.compile( "<link\\x20+(rel=(\"|')?stylesheet(\"|')?\\x20+)?(type=(\"|')?text/css(\"|')?\\x20+)?href=(\"|')?([^'\"\\r\\n]+)(\"|')?"); while (oMatcher.contains(oMatchInput, oPattern)) { nMatches++; String sMatch = oMatcher.getMatch().toString(); String sSrc = oMatcher.getMatch().group(8); String sFilename = sSrc.substring(sSrc.replace('\\', '/').lastIndexOf('/') + 1); if (!oFiles.contains(sSrc)) oFiles.add(sSrc); if (DebugFile.trace) DebugFile.writeln("StringSubstitution.setSubstitution(" + sMatch + " replace with " + Gadgets.replace(sMatch, sSrc, sFilename) + ")"); oSrcSubs.setSubstitution(Gadgets.replace(sMatch, sSrc, sFilename)); sHtml = Util.substitute(oReplacer, oCompiler.compile(sMatch), oSrcSubs, sHtml); } // wend if (DebugFile.trace) { DebugFile.writeln(String.valueOf(nMatches) + " matches found"); DebugFile.write("\n" + sHtml + "\n"); } ZipOutputStream oZOut = new ZipOutputStream(oOutStrm); String sLocalName = sFilePath.substring(sFilePath.replace('\\', '/').lastIndexOf('/') + 1); int iDot = sLocalName.lastIndexOf('.'); if (iDot > 0) sLocalName = Gadgets.ASCIIEncode(sLocalName.substring(0, iDot)).toLowerCase() + ".html"; else sLocalName = Gadgets.ASCIIEncode(sLocalName).toLowerCase(); oEntries.add(sLocalName); if (DebugFile.trace) DebugFile.writeln("Putting entry " + sLocalName + " into ZIP"); oZOut.putNextEntry(new ZipEntry(sLocalName)); StringBufferInputStream oHtml = new StringBufferInputStream(sHtml); new StreamPipe().between(oHtml, oZOut); oHtml.close(); oZOut.closeEntry(); for (String sName : oFiles) { String sZipEntryName = sName.substring(sName.replace('\\', '/').lastIndexOf('/') + 1); if (!oEntries.contains(sZipEntryName)) { oEntries.add(sZipEntryName); if (DebugFile.trace) DebugFile.writeln("Putting entry " + sZipEntryName + " into ZIP"); oZOut.putNextEntry(new ZipEntry(sZipEntryName)); if (sName.startsWith("http://") || sName.startsWith("https://") || sName.startsWith("file://") || sBaseHref.length() > 0) { try { new StreamPipe().between(new ByteArrayInputStream(readfilebin(sBaseHref + sName)), oZOut); } catch (IOException ioe) { if (DebugFile.trace) { DebugFile.decIdent(); DebugFile.writeln("Could not download file " + sName); } } } else { try { byte[] aFile = readfilebin( sBasePath + (sName.startsWith("/") ? sName.substring(1) : sName)); if (null != aFile) { if (aFile.length > 0) new StreamPipe().between(new ByteArrayInputStream(aFile), oZOut); } else { DebugFile.writeln("Could not find file " + sBasePath + (sName.startsWith("/") ? sName.substring(1) : sName)); } } catch (IOException ioe) { if (DebugFile.trace) { DebugFile.decIdent(); DebugFile.writeln("Could not download file " + sBasePath + (sName.startsWith("/") ? sName.substring(1) : sName)); } } } oZOut.closeEntry(); } // fi (sName!=sLocalName) } // next oZOut.close(); } catch (MalformedPatternException mpe) { } catch (FTPException ftpe) { } if (DebugFile.trace) { DebugFile.decIdent(); DebugFile.writeln("End FileSystem.downloadhtmlpage()"); } }
From source file:org.dasein.cloud.vcloud.vCloudMethod.java
public @Nonnull Org authenticate(boolean force) throws CloudException, InternalException { Cache<Org> cache = Cache.getInstance(provider, "vCloudOrgs", Org.class, CacheLevel.CLOUD_ACCOUNT, new TimePeriod<Minute>(25, TimePeriod.MINUTE)); ProviderContext ctx = provider.getContext(); if (ctx == null) { throw new CloudException("No context was defined for this request"); }//ww w . j av a 2 s .co m String accountNumber = ctx.getAccountNumber(); Iterable<Org> orgs = cache.get(ctx); Iterator<Org> it = ((force || orgs == null) ? null : orgs.iterator()); if (it == null || !it.hasNext()) { String endpoint = getVersion().loginUrl; if (wire.isDebugEnabled()) { wire.debug(""); wire.debug(">>> [POST (" + (new Date()) + ")] -> " + endpoint + " >--------------------------------------------------------------------------------------"); } try { HttpClient client = getClient(true); HttpPost method = new HttpPost(endpoint); Org org = new Org(); org.version = getVersion(); method.addHeader("Accept", "application/*+xml;version=" + org.version.version + ",application/*+xml;version=" + org.version.version); String accessPublic = null; String accessPrivate = null; try { List<ContextRequirements.Field> fields = provider.getContextRequirements() .getConfigurableValues(); for (ContextRequirements.Field f : fields) { if (f.type.equals(ContextRequirements.FieldType.KEYPAIR)) { byte[][] keyPair = (byte[][]) provider.getContext().getConfigurationValue(f); accessPublic = new String(keyPair[0], "utf-8"); accessPrivate = new String(keyPair[1], "utf-8"); } } } catch (UnsupportedEncodingException e) { throw new InternalException(e); } String password = accessPrivate; String userName; if (matches(getAPIVersion(), "0.8", "0.8")) { userName = accessPublic; } else if (getAPIVersion().equals("5.6")) { userName = accessPublic; } else { userName = accessPublic + "@" + ctx.getAccountNumber(); } String auth = new String(Base64.encodeBase64((userName + ":" + password).getBytes())); method.addHeader("Authorization", "Basic " + auth); if (wire.isDebugEnabled()) { wire.debug(method.getRequestLine().toString()); for (Header header : method.getAllHeaders()) { wire.debug(header.getName() + ": " + header.getValue()); } wire.debug(""); } HttpResponse response; StatusLine status; try { APITrace.trace(provider, "POST sessions"); response = client.execute(method); if (wire.isDebugEnabled()) { wire.debug(response.getStatusLine().toString()); for (Header header : response.getAllHeaders()) { wire.debug(header.getName() + ": " + header.getValue()); } wire.debug(""); } status = response.getStatusLine(); } catch (IOException e) { throw new CloudException(e); } if (status.getStatusCode() == HttpServletResponse.SC_OK) { if (matches(getAPIVersion(), "0.8", "0.8")) { for (Header h : response.getHeaders("Set-Cookie")) { String value = h.getValue(); if (value != null) { value = value.trim(); if (value.startsWith("vcloud-token")) { value = value.substring("vcloud-token=".length()); int idx = value.indexOf(";"); if (idx == -1) { org.token = value; } else { org.token = value.substring(0, idx); } } } } } else { org.token = response.getFirstHeader("x-vcloud-authorization").getValue(); } if (org.token == null) { throw new CloudException(CloudErrorType.AUTHENTICATION, 200, "Token Empty", "No token was provided"); } HttpEntity entity = response.getEntity(); String body; try { body = EntityUtils.toString(entity); if (wire.isDebugEnabled()) { wire.debug(body); wire.debug(""); } } catch (IOException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } try { ByteArrayInputStream bas = new ByteArrayInputStream(body.getBytes()); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder parser = factory.newDocumentBuilder(); Document doc = parser.parse(bas); bas.close(); if (matches(org.version.version, "1.5", null)) { NodeList orgNodes = doc.getElementsByTagName("Link"); String orgList = null; for (int i = 0; i < orgNodes.getLength(); i++) { Node orgNode = orgNodes.item(i); if (orgNode.hasAttributes()) { Node type = orgNode.getAttributes().getNamedItem("type"); if (type != null && type.getNodeValue().trim().equals(getMediaTypeForOrg())) { Node name = orgNode.getAttributes().getNamedItem("name"); if (name != null && name.getNodeValue().trim().equals(accountNumber)) { Node href = orgNode.getAttributes().getNamedItem("href"); if (href != null) { Region region = new Region(); String url = href.getNodeValue().trim(); region.setActive(true); region.setAvailable(true); if (provider.isCompat()) { region.setProviderRegionId( "/org/" + url.substring(url.lastIndexOf('/') + 1)); } else { region.setProviderRegionId( url.substring(url.lastIndexOf('/') + 1)); } region.setJurisdiction("US"); region.setName(name.getNodeValue().trim()); org.endpoint = url.substring(0, url.lastIndexOf("/api/org")); org.region = region; org.url = url; } } } if (type != null && type.getNodeValue().trim().equals(getMediaTypeForOrgList())) { Node href = orgNode.getAttributes().getNamedItem("href"); if (href != null) { orgList = href.getNodeValue().trim(); } } } } if (org.endpoint == null && orgList != null) { loadOrg(orgList, org, accountNumber); } } else { NodeList orgNodes = doc.getElementsByTagName("Org"); for (int i = 0; i < orgNodes.getLength(); i++) { Node orgNode = orgNodes.item(i); if (orgNode.hasAttributes()) { Node name = orgNode.getAttributes().getNamedItem("name"); Node href = orgNode.getAttributes().getNamedItem("href"); if (href != null) { String url = href.getNodeValue().trim(); Region region = new Region(); if (!url.endsWith("/org/" + accountNumber)) { continue; } region.setActive(true); region.setAvailable(true); if (provider.isCompat()) { region.setProviderRegionId( "/org/" + url.substring(url.lastIndexOf('/') + 1)); } else { region.setProviderRegionId(url.substring(url.lastIndexOf('/') + 1)); } region.setJurisdiction("US"); region.setName(name == null ? accountNumber : name.getNodeValue().trim()); org.endpoint = url.substring(0, url.lastIndexOf("/org/")); org.region = region; org.url = url; } } } } } catch (IOException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } catch (ParserConfigurationException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } catch (SAXException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } } else { HttpEntity entity = response.getEntity(); if (entity != null) { String body; try { body = EntityUtils.toString(entity); if (wire.isDebugEnabled()) { wire.debug(body); wire.debug(""); } } catch (IOException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } vCloudException.Data data = null; if (body != null && !body.equals("")) { Document doc = parseXML(body); String docElementTagName = doc.getDocumentElement().getTagName(); String nsString = ""; if (docElementTagName.contains(":")) nsString = docElementTagName.substring(0, docElementTagName.indexOf(":") + 1); NodeList errors = doc.getElementsByTagName(nsString + "Error"); if (errors.getLength() > 0) { data = vCloudException.parseException(status.getStatusCode(), errors.item(0)); } } if (data == null) { throw new vCloudException(CloudErrorType.GENERAL, status.getStatusCode(), response.getStatusLine().getReasonPhrase(), "No further information"); } logger.error("[" + status.getStatusCode() + " : " + data.title + "] " + data.description); throw new vCloudException(data); } throw new CloudException(CloudErrorType.AUTHENTICATION, status.getStatusCode(), status.getReasonPhrase(), "Authentication failed"); } if (org.endpoint == null) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), "No Org", "No org was identified for " + ctx.getAccountNumber()); } cache.put(ctx, Collections.singletonList(org)); loadVDCs(org); return org; } finally { if (wire.isDebugEnabled()) { wire.debug("<<< [POST (" + (new Date()) + ")] -> " + endpoint + " <--------------------------------------------------------------------------------------"); wire.debug(""); } } } else { return it.next(); } }