List of usage examples for java.net URI toASCIIString
public String toASCIIString()
From source file:org.paxle.filter.robots.impl.RobotsTxtManager.java
/** * Downloads a <i>robots.txt</i> file from the given url and parses it * @param robotsUrlStr the URL to the robots.txt. This must be a http(s) resource * @return the parsed robots.txt file as a {@link RobotsTxt}-object * @throws IOException//from w w w.ja v a2 s . com * @throws URISyntaxException */ RobotsTxt getFromWeb(URI robotsURL) throws IOException, URISyntaxException { String hostPort = this.getHostPort(robotsURL); String statusLine = null; if (!robotsURL.getScheme().startsWith("http")) { throw new IOException(String.format("Unsupported protocol: %s", robotsURL.getScheme())); } InputStream inputStream = null; HttpMethod getMethod = null; try { getMethod = new GetMethod(robotsURL.toASCIIString()); int code = this.httpClient.executeMethod(getMethod); statusLine = getMethod.getStatusLine().toString(); if (code == HttpStatus.SC_UNAUTHORIZED || code == HttpStatus.SC_FORBIDDEN) { // access to the whole website is restricted return new RobotsTxt(hostPort, RobotsTxt.RELOAD_INTERVAL_DEFAULT, statusLine, true); } else if (code == HttpStatus.SC_NOT_FOUND) { // no robots.txt provided return new RobotsTxt(hostPort, RobotsTxt.RELOAD_INTERVAL_DEFAULT, statusLine); } else if (code != HttpStatus.SC_OK) { // the robots.txt seems not to be deliverable return new RobotsTxt(hostPort, RobotsTxt.RELOAD_INTERVAL_DEFAULT, statusLine); } Header contentTypeHeader = getMethod.getResponseHeader("Content-Type"); if (contentTypeHeader != null && !contentTypeHeader.getValue().startsWith("text/plain")) { // the robots.txt seems not to be available return new RobotsTxt(hostPort, RobotsTxt.RELOAD_INTERVAL_ERROR, "Wrong mimeType " + contentTypeHeader.getValue()); } inputStream = getMethod.getResponseBodyAsStream(); RobotsTxt robotsTxt = new RobotsTxt(hostPort, RobotsTxt.RELOAD_INTERVAL_DEFAULT, statusLine); return this.parseRobotsTxt(robotsTxt, inputStream); } catch (IOException e) { long reloadInterval = RobotsTxt.RELOAD_INTERVAL_TEMP_ERROR; String status = e.getMessage(); if (e instanceof UnknownHostException) { reloadInterval = RobotsTxt.RELOAD_INTERVAL_ERROR; status = "Unknown host"; logger.info(String.format("Unknown host '%s'.", robotsURL.getHost())); } else if (e instanceof CircularRedirectException || e instanceof RedirectException || e instanceof InvalidRedirectLocationException) { reloadInterval = RobotsTxt.RELOAD_INTERVAL_ERROR; logger.info(String.format("Invalid redirection on host '%s'.", hostPort)); } else if (e instanceof SocketTimeoutException || e instanceof ConnectTimeoutException || e instanceof NoHttpResponseException) { logger.debug(String.format("TimeOut while loading robots.txt from host '%s'.", hostPort)); } else if (!(e instanceof ConnectException || e instanceof SocketException)) { logger.error("Exception while loading robots.txt from " + hostPort, e); } return new RobotsTxt(hostPort, reloadInterval, status); } catch (IllegalArgumentException e) { // occurs if redirected to an invalid URI, see https://bugs.pxl.li/view.php?id=172 // we treat it like a 404, see above logger.info(String.format("Invalid redirection URI on host '%s'.", hostPort)); return new RobotsTxt(hostPort, RobotsTxt.RELOAD_INTERVAL_DEFAULT, "Redirected to illegal URI"); } catch (IllegalStateException e) { // occurs if redirected to an URI with an invalid protocol, see https://bugs.pxl.li/view.php?id=169 // we treat it like a 404, see above logger.info(String.format("Invalid redirection URI on host '%s'.", hostPort)); return new RobotsTxt(hostPort, RobotsTxt.RELOAD_INTERVAL_DEFAULT, "Redirected to illegal URI"); } finally { if (inputStream != null) try { inputStream.close(); } catch (Exception e) { this.logger.error(e); } if (getMethod != null) getMethod.releaseConnection(); } }
From source file:io.fabric8.core.jmx.FabricManager.java
@Override public String mavenProxyDownloadUrl() { URI uri = getFabricService().getMavenRepoURI(); if (uri != null) { return uri.toASCIIString(); } else {// w ww . ja v a 2 s. c o m return null; } }
From source file:io.fabric8.core.jmx.FabricManager.java
@Override public String mavenProxyUploadUrl() { URI uri = getFabricService().getMavenRepoUploadURI(); if (uri != null) { return uri.toASCIIString(); } else {//from w w w . ja v a2s . c o m return null; } }
From source file:com.emc.esu.api.rest.AbstractEsuRestApi.java
/** * Builds a new URL to the given resource * @throws URISyntaxException /*from ww w .jav a 2s. c o m*/ * @throws MalformedURLException */ protected URL buildUrl(String resource, String query) throws URISyntaxException, MalformedURLException { int uriport = 0; if ("http".equals(proto) && port == 80) { // Default port uriport = -1; } else if ("https".equals(proto) && port == 443) { uriport = -1; } else { uriport = port; } URI uri = new URI(proto, null, host, uriport, resource, query, null); l4j.debug("URI: " + uri); URL u = new URL(uri.toASCIIString()); l4j.debug("URL: " + u); return u; }
From source file:net.es.sense.rm.api.SenseRmController.java
/** * Returns a list of available SENSE service API resource URLs. * * Operation: GET /api/sense/v1/*from w ww . j ava 2 s . com*/ * * @return A RESTful response. * @throws java.net.MalformedURLException */ @ApiOperation(value = "Get a list of supported SENSE resources.", notes = "Returns a list of available SENSE resource URL.", response = DeltaResource.class, responseContainer = "List") @ApiResponses(value = { @ApiResponse(code = HttpConstants.OK_CODE, message = HttpConstants.OK_MSG, response = Resource.class), @ApiResponse(code = HttpConstants.UNAUTHORIZED_CODE, message = HttpConstants.UNAUTHORIZED_MSG, response = Error.class), @ApiResponse(code = HttpConstants.FORBIDDEN_CODE, message = HttpConstants.FORBIDDEN_MSG, response = Error.class), @ApiResponse(code = HttpConstants.INTERNAL_ERROR_CODE, message = HttpConstants.INTERNAL_ERROR_MSG, response = Error.class), }) @RequestMapping(method = RequestMethod.GET, produces = { MediaType.APPLICATION_JSON_VALUE }) @ResponseBody public ResponseEntity<?> getResources() throws MalformedURLException { try { // We need the request URL to build fully qualified resource URLs. final URI location = ServletUriComponentsBuilder.fromCurrentRequestUri().build().toUri(); log.info("[SenseRmController] GET operation = {}", location); // We will populate some HTTP response headers. final HttpHeaders headers = new HttpHeaders(); headers.add("Content-Location", location.toASCIIString()); List<Resource> resources = new ArrayList<>(); Method[] methods = SenseRmController.class.getMethods(); for (Method m : methods) { if (m.isAnnotationPresent(ResourceAnnotation.class)) { ResourceAnnotation ra = m.getAnnotation(ResourceAnnotation.class); RequestMapping rm = m.getAnnotation(RequestMapping.class); if (ra == null || rm == null) { continue; } Resource resource = new Resource(); resource.setId(ra.name()); resource.setVersion(ra.version()); UriComponentsBuilder path = utilities.getPath(location.toASCIIString()); path.path(rm.value()[0]); resource.setHref(path.build().toUriString()); resources.add(resource); } } return new ResponseEntity<>(resources, headers, HttpStatus.OK); } catch (SecurityException | MalformedURLException ex) { log.error("[SenseRmController] Exception caught", ex); Error error = Error.builder().error(HttpStatus.INTERNAL_SERVER_ERROR.getReasonPhrase()) .error_description(ex.getMessage()).build(); return new ResponseEntity<>(error, HttpStatus.INTERNAL_SERVER_ERROR); } }
From source file:org.paxle.crawler.http.impl.HttpCrawler.java
public ICrawlerDocument request(URI requestUri) { if (requestUri == null) throw new NullPointerException("URL was null"); this.logger.debug(String.format("Crawling URL '%s' ...", requestUri)); ICrawlerDocument doc = null;/*from w w w . j av a 2 s . c om*/ HttpMethod method = null; try { final ICrawlerContext ctx = this.contextLocal.getCurrentContext(); // creating an empty crawler-document doc = ctx.createDocument(); doc.setLocation(requestUri); final String uriAsciiString = requestUri.toASCIIString(); /* ============================================================================== * HTTP HEAD request * * first use the HEAD method to determine whether the MIME-type is supported * and to compare the content-length with the maximum allowed download size * (both only if the server provides this information, if not, the file is * fetched) * ============================================================================== */ method = new HeadMethod(uriAsciiString); // automatically follows redirects this.initRequestMethod(method); int statusCode = this.getHttpClient().executeMethod(method); final boolean headUnsupported = (statusCode == HttpStatus.SC_METHOD_FAILURE || statusCode == HttpStatus.SC_METHOD_NOT_ALLOWED); if (!headUnsupported) { if (statusCode != HttpStatus.SC_OK) { // RFC 2616 states that the GET and HEAD methods _must_ be supported by any // general purpose servers (which are in fact the ones we are connecting to here) if (statusCode == HttpStatus.SC_NOT_FOUND) { doc.setStatus(ICrawlerDocument.Status.NOT_FOUND); } else { doc.setStatus(ICrawlerDocument.Status.UNKNOWN_FAILURE, String.format("Server returned: %s", method.getStatusLine())); } this.logger.warn(String.format("Crawling of URL '%s' failed. Server returned: %s", requestUri, method.getStatusLine())); return doc; } // getting the mimetype and charset Header contentTypeHeader = method.getResponseHeader(HTTPHEADER_CONTENT_TYPE); if (!handleContentTypeHeader(contentTypeHeader, doc)) return doc; // reject the document if content-length is above our limit Header contentLengthHeader = method.getResponseHeader(HTTPHEADER_CONTENT_LENGTH); if (!handleContentLengthHeader(contentLengthHeader, doc)) return doc; // FIXME: we've been redirected, re-enqueue the new URL and abort processing //if (!requestUri.equals(method.getURI())) ; } /* ============================================================================== * HTTP GET request * * secondly - if everything is alright up to now - proceed with getting the * actual document * ============================================================================== */ HttpMethod getMethod = new GetMethod(uriAsciiString); // automatically follows redirects method.releaseConnection(); method = getMethod; this.initRequestMethod(method); // send the request to the server statusCode = this.getHttpClient().executeMethod(method); // check the response status code if (statusCode != HttpStatus.SC_OK) { if (statusCode == HttpStatus.SC_NOT_FOUND) { doc.setStatus(ICrawlerDocument.Status.NOT_FOUND); } else { doc.setStatus(ICrawlerDocument.Status.UNKNOWN_FAILURE, String.format("Server returned: %s", method.getStatusLine())); } this.logger.warn(String.format("Crawling of URL '%s' failed. Server returned: %s", requestUri, method.getStatusLine())); return doc; } // FIXME: we've been redirected, re-enqueue the new URL and abort processing // if (!requestUri.equals(method.getURI())) ; /* * HTTP Content-Type * - getting the mimetype and charset */ Header contentTypeHeader = method.getResponseHeader(HTTPHEADER_CONTENT_TYPE); if (!handleContentTypeHeader(contentTypeHeader, doc)) return doc; /* * HTTP Content-Length * - Reject the document if content-length is above our limit * * We do this a second time here because some servers may have set the content-length * of the head response to <code>0</code> */ Header contentLengthHeader = method.getResponseHeader(HTTPHEADER_CONTENT_LENGTH); if (!handleContentLengthHeader(contentLengthHeader, doc)) return doc; extractHttpHeaders(method, doc); // externalised into this method to cleanup here a bit // getting the response body InputStream respBody = method.getResponseBodyAsStream(); // handle the content-encoding, i.e. decompress the server's response Header contentEncodingHeader = method.getResponseHeader(HTTPHEADER_CONTENT_ENCODING); try { respBody = handleContentEncoding(contentEncodingHeader, respBody); /* Limit the max allowed length of the content to copy. -1 is used for no limit. * * We need to set a limit if: * a) the user has configured a max-download-size AND * b) the server returned no content-length header */ int copyLimit = (this.maxDownloadSize <= 0 || contentLengthHeader != null) ? -1 : this.maxDownloadSize; // copy the content to file final ICrawlerTools crawlerTools = ctx.getCrawlerTools(); crawlerTools.saveInto(doc, respBody, lrc, copyLimit); doc.setStatus(ICrawlerDocument.Status.OK); this.logger.debug(String.format("Crawling of URL '%s' finished.", requestUri)); } catch (IOException e) { String msg = e.getMessage(); if (msg == null || !msg.equals("Corrupt GZIP trailer")) throw e; setHostSetting(method.getURI().getHost(), PREF_NO_ENCODING); msg = String.format("server sent a corrupt gzip trailer at URL '%s'", requestUri); logger.warn(msg); // FIXME re-enqueue command doc.setStatus(ICrawlerDocument.Status.UNKNOWN_FAILURE, msg); } finally { respBody.close(); } } catch (NoRouteToHostException e) { this.logger.warn(String.format("Error crawling %s: %s", requestUri, e.getMessage())); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (UnknownHostException e) { this.logger.warn(String.format("Error crawling %s: Unknown host.", requestUri)); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (ConnectException e) { this.logger.warn(String.format("Error crawling %s: Unable to connect to host.", requestUri)); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (ConnectTimeoutException e) { this.logger.warn(String.format("Error crawling %s: %s.", requestUri, e.getMessage())); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (SocketTimeoutException e) { this.logger.warn(String.format("Error crawling %s: Connection timeout.", requestUri)); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (CircularRedirectException e) { this.logger.warn(String.format("Error crawling %s: %s", requestUri, e.getMessage())); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (NoHttpResponseException e) { this.logger.warn(String.format("Error crawling %s: %s", requestUri, e.getMessage())); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (ContentLengthLimitExceededException e) { this.logger.warn(String.format("Error crawling %s: %s", requestUri, e.getMessage())); doc.setStatus(ICrawlerDocument.Status.UNKNOWN_FAILURE, e.getMessage()); } catch (Throwable e) { String errorMsg; if (e instanceof HttpException) { errorMsg = "Unrecovered protocol exception: [%s] %s"; } else if (e instanceof IOException) { errorMsg = "Transport exceptions: [%s] %s"; } else { errorMsg = "Unexpected exception: [%s] %s"; } errorMsg = String.format(errorMsg, e.getClass().getName(), e.getMessage()); this.logger.error(String.format("Error crawling %s: %s", requestUri, errorMsg)); doc.setStatus(ICrawlerDocument.Status.UNKNOWN_FAILURE, errorMsg); e.printStackTrace(); } finally { if (method != null) method.releaseConnection(); } return doc; }
From source file:com.almende.eve.agent.Agent.java
@Override public void receive(final Object msg, final URI senderUrl, final String tag) { JsonNode id = null;/* w ww. j ava 2 s . c om*/ try { final JSONMessage jsonMsg = jsonConvert(msg); if (jsonMsg != null) { id = jsonMsg.getId(); if (jsonMsg instanceof JSONRequest) { final RequestParams params = new RequestParams(); params.put(Sender.class, senderUrl.toASCIIString()); final JSONRequest request = (JSONRequest) jsonMsg; final AgentInterface me = this; host.getPool().execute(new Runnable() { @Override public void run() { final Object[] signalData = new Object[2]; signalData[0] = request; signalData[1] = params; signalAgent(new AgentSignal<Object[]>(AgentSignal.INVOKE, signalData)); final JSONResponse response = JSONRPC.invoke(me, request, params, me); signalAgent(new AgentSignal<JSONResponse>(AgentSignal.RESPOND, response)); try { send(response, senderUrl, null, tag); } catch (final IOException e) { LOG.log(Level.WARNING, getId() + ": Failed to send response.", e); } } }); } else if (jsonMsg instanceof JSONResponse && callbacks != null && id != null && !id.isNull()) { final JSONResponse response = (JSONResponse) jsonMsg; final AsyncCallback<JSONResponse> callback = callbacks.pull(id); if (callback != null) { host.getPool().execute(new Runnable() { @Override public void run() { signalAgent(new AgentSignal<JSONResponse>(AgentSignal.RESPONSE, response)); if (response.getError() != null) { callback.onFailure(response.getError()); } else { callback.onSuccess(response); } } }); } } } else { LOG.log(Level.WARNING, getId() + ": Received non-JSON message:'" + msg + "'"); } } catch (final Exception e) { // generate JSON error response, skipped if it was an incoming // notification i.s.o. request. final JSONRPCException jsonError = new JSONRPCException(JSONRPCException.CODE.INTERNAL_ERROR, e.getMessage(), e); LOG.log(Level.WARNING, "Exception in receiving message", jsonError); final JSONResponse response = new JSONResponse(jsonError); response.setId(id); signalAgent(new AgentSignal<JSONResponse>(AgentSignal.EXCEPTION, response)); try { send(response, senderUrl, null, tag); } catch (final Exception e1) { LOG.log(Level.WARNING, getId() + ": failed to send '" + e.getLocalizedMessage() + "' error to remote agent.", e1); } } }
From source file:org.nuxeo.ecm.platform.annotations.service.AnnotationsOnVersionTest.java
@Test public void testDeleteAnnotationsOnVersions() throws Exception { DocumentModel docModel = createDocument(session.getRootDocument().getPathAsString(), "fileDeleteAnnotationsOnVersions"); String url = viewCodecManager.getUrlFromDocumentView(new DocumentViewImpl(docModel), true, SERVER); assertNotNull(url);//from ww w .j a v a2 s. c o m addAnnotationsOn(url, 3); URI uri = translator.getNuxeoUrn(docModel.getRepositoryName(), docModel.getId()); List<Annotation> annotations = service.queryAnnotations(uri, user); assertEquals(3, annotations.size()); DocumentRef version1ref = checkIn(docModel.getRef()); List<DocumentModel> versions = session.getVersions(docModel.getRef()); assertEquals(1, versions.size()); DocumentModel versionModel1 = session.getDocument(version1ref); URI uriVersion1 = translator.getNuxeoUrn(versionModel1.getRepositoryName(), versionModel1.getId()); annotations = service.queryAnnotations(uriVersion1, user); assertEquals(3, annotations.size()); addAnnotationsOn(url, 3); annotations = service.queryAnnotations(uri, user); assertEquals(3, annotations.size()); DocumentRef version2ref = checkIn(docModel.getRef()); versions = session.getVersions(docModel.getRef()); assertEquals(2, versions.size()); DocumentModel versionModel2 = session.getDocument(version2ref); URI uriVersion2 = translator.getNuxeoUrn(versionModel2.getRepositoryName(), versionModel2.getId()); annotations = service.queryAnnotations(uriVersion2, user); assertEquals(3, annotations.size()); // the text 'zombie' is not found in the document DocumentModelList results = session.query("SELECT * FROM Document WHERE ecm:fulltext = 'zombie'", 10); assertEquals(0, results.size()); // put an text annotation a checked in version to check the fulltext // indexing Annotation annotationVersion1 = getAnnotation(uriVersion1.toASCIIString(), 1); annotationVersion1.setBodyText("zombie annotation"); service.addAnnotation(annotationVersion1, user, uriVersion1.toASCIIString()); session.save(); nextTransaction(); sleepForFulltext(); results = session.query("SELECT * FROM Document WHERE ecm:fulltext = 'zombie'", 10); assertEquals(1, results.size()); assertEquals(version1ref, results.get(0).getRef()); // Delete annotations for version 1 annotations = service.queryAnnotations(uriVersion1, user); for (Annotation annotation : annotations) { service.deleteAnnotationFor(uriVersion1, annotation, user); } annotations = service.queryAnnotations(uriVersion1, user); assertEquals(0, annotations.size()); // Delete annotations for version 2 annotations = service.queryAnnotations(uriVersion2, user); for (Annotation annotation : annotations) { service.deleteAnnotationFor(uriVersion2, annotation, user); } annotations = service.queryAnnotations(uriVersion2, user); assertEquals(0, annotations.size()); // restore version 1 docModel = restoreToVersion(docModel.getRef(), version1ref); annotations = service.queryAnnotations(uri, user); // still no annotation on the current document assertEquals(0, annotations.size()); }
From source file:org.teiid.olingo.TeiidODataJsonSerializer.java
public SerializerResult complexCollection(final ServiceMetadata metadata, final List<List<ComplexReturnType>> result, final ContextURL contextURL, final URI nextLink) throws SerializerException { CircleStreamBuffer buffer = new CircleStreamBuffer(); try {/* www .ja v a 2 s.c o m*/ JsonGenerator json = new JsonFactory().createGenerator(buffer.getOutputStream()); json.writeStartObject(); if (contextURL != null && (isODataMetadataFull || !isODataMetadataNone)) { json.writeStringField(Constants.JSON_CONTEXT, ContextURLBuilder.create(contextURL).toASCIIString()); } json.writeFieldName(Constants.VALUE); json.writeStartArray(); for (List<ComplexReturnType> ct : result) { json.writeStartObject(); for (final ComplexReturnType type : ct) { if (!type.isExpand()) { json.writeStringField(type.getName() + Constants.JSON_NAVIGATION_LINK, type.getEntity().getId().toASCIIString()); } else { json.writeFieldName(type.getName()); writeEntity(metadata, type.getEdmEntityType(), type.getEntity(), null, null, null, null, false, null, type.getName(), json); } } json.writeEndObject(); } json.writeEndArray(); if (nextLink != null) { json.writeStringField(Constants.JSON_NEXT_LINK, nextLink.toASCIIString()); } json.close(); } catch (final IOException | DecoderException e) { throw new SerializerException("An I/O exception occurred.", e, SerializerException.MessageKeys.IO_EXCEPTION); } return SerializerResultImpl.with().content(buffer.getInputStream()).build(); }