List of usage examples for java.net HttpURLConnection setFollowRedirects
public static void setFollowRedirects(boolean set)
From source file:org.kaazing.maven.plugins.TrustStoreMojo.java
Map<String, String> getCertificates(String location) throws Exception { Map<String, String> certs = new HashMap<String, String>(); Pattern labelPattern = Pattern.compile("^CKA_LABEL\\s+[A-Z0-9]+\\s+\\\"(.*)\\\""); Pattern beginContentPattern = Pattern.compile("^CKA_VALUE MULTILINE_OCTAL"); Pattern endContentPattern = Pattern.compile("^END"); Pattern untrustedPattern = Pattern.compile( "^CKA_TRUST_SERVER_AUTH\\s+CK_TRUST\\s+CKT_NSS_NOT_TRUSTED$|^CKA_TRUST_SERVER_AUTH\\s+CK_TRUST\\s+CKT_NSS_TRUST_UNKNOWN$"); URI certsURI = new URI(location); // This should be the default, but make sure it's set anyway HttpURLConnection.setFollowRedirects(true); HttpURLConnection conn = (HttpURLConnection) certsURI.toURL().openConnection(); if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) { throw new Exception(String.format("Error connecting to %s: %d %s", location, conn.getResponseCode(), conn.getResponseMessage())); }/*from w ww . j a v a 2s. c o m*/ InputStream is = conn.getInputStream(); InputStreamReader isr = new InputStreamReader(is); BufferedReader br = new BufferedReader(isr); String alias = null; byte[] certData = new byte[MAX_CERT_SIZE]; int certDataLen = 0; String line = br.readLine(); while (line != null) { // Skip comments and empty lines if (line.startsWith("#")) { line = br.readLine(); continue; } if (line.trim().length() == 0) { line = br.readLine(); continue; } Matcher m = labelPattern.matcher(line); if (m.find()) { alias = m.group(1).toLowerCase().replaceAll("/", "-").replaceAll("\\s+", ""); line = br.readLine(); continue; } m = beginContentPattern.matcher(line); if (m.find()) { line = br.readLine(); while (true) { m = endContentPattern.matcher(line); if (m.find()) { StringBuilder pem = new StringBuilder(); pem.append("-----BEGIN CERTIFICATE-----\n"); String base64Data = Base64.encodeBase64String(Arrays.copyOf(certData, certDataLen)); pem.append(base64Data); pem.append("\n-----END CERTIFICATE-----\n"); certs.put(alias, pem.toString()); // Prepare for another certificate/trust section alias = null; certData = new byte[MAX_CERT_SIZE]; certDataLen = 0; line = br.readLine(); break; } String[] octets = line.split("\\\\"); // We start at index 1, not zero, because the first element // in the array will always be an empty string. The first // character in the string is a backslash; String.split() // thus populates the element before that backslash as an // empty string. for (int i = 1; i < octets.length; i++) { int octet = Integer.parseInt(octets[i], 8); certData[certDataLen++] = (byte) octet; } line = br.readLine(); } continue; } m = untrustedPattern.matcher(line); if (m.find()) { // Remove untrusted certs from our map certs.remove(alias); line = br.readLine(); continue; } line = br.readLine(); } return certs; }
From source file:horriblev3.Cloudflare.java
public List<HttpCookie> scrape() { if (strUrl == null) { System.out.println("URL == NULL"); return null; }// w ww . j a va 2s . c o m try { CookieManager manager = new CookieManager(); manager.setCookiePolicy(CookiePolicy.ACCEPT_ALL); CookieHandler.setDefault(manager); URL url = new URL(strUrl); HttpURLConnection con; con = (HttpURLConnection) url.openConnection(); con.setRequestProperty("User-Agent", USERAGENT); InputStream _is; if (con.getResponseCode() == 200) { return retrieveCookies(manager); } else { _is = con.getErrorStream(); StringBuilder result = new StringBuilder(); try (BufferedReader rd = new BufferedReader(new InputStreamReader(_is))) { String line; while ((line = rd.readLine()) != null) { result.append(line); } } String source = result.toString(); //extract challenge String challenge = regex(source, "name=\"jschl_vc\" value=\"(\\w+)\""); String challenge_pass = regex(source, "name=\"pass\" value=\"(.+?)\""); //prepare String builder = regex(source, "setTimeout\\(function\\(\\)\\{\\s+(var s,t,o,p,b,r,e,a,k,i,n,g,f.+?\\r?[\\s\\S]+?a\\.value =.+?)\\r?\\s+';"); builder = builder.replaceFirst("\\s{3,}[a-z](?: = ).+form'\\);\\s+;", "").replaceFirst( "a\\.value = parseInt\\(.+?\\).+", regex(builder, "a\\.value = (parseInt\\(.+?\\)).+")); //Execute&solve System.out.println(builder); long solved = Long.parseLong(solveJS2(builder)); solved += url.getHost().length(); System.out.println("SOLVED: " + solved); try { Thread.sleep(3000); } catch (InterruptedException e) { System.out.println(e.getMessage()); } URI tmp = UrlToUri(url); List<NameValuePair> qparams = new ArrayList<>(); qparams.add(new BasicNameValuePair("jschl_vc", challenge)); qparams.add(new BasicNameValuePair("jschl_answer", String.valueOf(solved))); qparams.add(new BasicNameValuePair("pass", challenge_pass)); URIBuilder uriBuilder = new URIBuilder().setScheme(tmp.getScheme()).setPath("/cdn-cgi/l/chk_jschl") .setHost(tmp.getHost()).setParameters(qparams); HttpURLConnection cookie_req = (HttpURLConnection) new URL(uriBuilder.toString()).openConnection(); cookie_req.setRequestProperty("Referer", url.toString()); cookie_req.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:48.0) Gecko/20100101 Firefox/48.0"); HttpURLConnection.setFollowRedirects(false); cookie_req.connect(); System.out.println("ResponseCode: " + cookie_req.getResponseCode()); if (cookie_req.getResponseCode() == 200) { return retrieveCookies(manager); } else { System.out.println("Something went wrong!"); return null; } } } catch (IOException e1) { System.out.println(e1.getMessage()); return null; } }
From source file:com.centurylink.mdw.ant.taskdef.HttpTransfer.java
protected void download(URL srcURL, File file) { try {//from w ww. ja va 2 s . c om log("source url... " + srcURL.toString()); HttpURLConnection conn = (HttpURLConnection) srcURL.openConnection(); HttpURLConnection.setFollowRedirects(true); FileOutputStream fos = new FileOutputStream(file); log("Downloading ... " + file); byte[] buffer = new byte[2048]; InputStream is = conn.getInputStream(); while (true) { int bytesRead = is.read(buffer); if (bytesRead == -1) break; fos.write(buffer, 0, bytesRead); } fos.close(); is.close(); conn.disconnect(); log("Downloaded ... " + file); } catch (IOException e) { String msg = "Unable to connect to URL: " + srcURL + " - Unable to download file. "; if (isFailOnError()) throw new BuildException(msg + e.getMessage()); else log(msg); } }
From source file:org.agmip.ui.utils.WeatherFileSystem.java
public static void WriteToFile() { // prepare the file name from the organization name and the site index number and then write to that file String orgName = DssatUtil.getInfo(AppKeys.FARM); String siteIndex = DssatUtil.getInfo("SiteIndex"); File file = new File(PathGenerator.getWeatherFilePath()); String writeBuffer = null;/*from w w w . j a va 2s . com*/ PrintWriter pr = null; try { pr = new PrintWriter(file); writeBuffer = new String("*WEATHER : \n"); pr.println(writeBuffer); writeBuffer = new String("@ INSI LAT LONG ELEV TAV AMP REFHT WNDHT"); pr.println(writeBuffer); double latitude = -34.23; double longitude = -34.23; double elevation = -99; double tavg = -99; double tamp = -99; double tmht = -99; double wmht = -99; writeBuffer = new String(); //Empty 2 Char, Org 2 Char, Site 2Char, Space 1 Char, latitude 8Chars with 3 dec writeBuffer = String.format( "%.2s%.2s%.2s%.1s%8.3f%.1s%8.3f%.1s%5.0f%.1s%5.1f%.1s%5.1f%.1s%5.1f%.1s%5.1f", " ", orgName, siteIndex, " ", latitude, " ", longitude, " ", elevation, " ", tavg, " ", tamp, " ", tmht, " ", wmht); pr.println(writeBuffer); int julianday = 56001; double solarrad = -99; double tmax = -99; double tmin = -99; double rain = -99; double dewp = -99; double wind = -99; double par = -99; writeBuffer = new String("@DATE SRAD TMAX TMIN RAIN RHUM WIND TDEW PAR"); pr.println(writeBuffer); writeBuffer = new String(); URL url = null; InputStream is = null; Integer plantingyear; if (DssatUtil.getInfo("PlantingYear") != null) { plantingyear = Integer.parseInt(DssatUtil.getInfo("PlantingYear")); } else { Date d = new Date(); plantingyear = d.getYear() + 1900; } /*Bring the Weather details for last 10 Years and write the details to the WTH File. */ try { String query = new String("SELECT%20*%20FROM%20FAWN_historic_daily_20140212%20" + "WHERE%20(%20yyyy%20BETWEEN%20" + (plantingyear - 10) + "%20AND%20" + plantingyear.toString() + ")" + "%20AND%20(%20LocId%20=%20" + DssatUtil.getInfo(AppKeys.STATION_LOCATION_ID) + ")" + "%20ORDER%20BY%20yyyy%20DESC%20"); String urlStr = "http://abe.ufl.edu/bmpmodel/xmlread/rohit.php?sql=" + query; System.out.println("********************************"); System.out.println(urlStr); //System.out.println(urlStr); URL uflconnection = new URL(urlStr); HttpURLConnection huc = (HttpURLConnection) uflconnection.openConnection(); HttpURLConnection.setFollowRedirects(false); huc.setConnectTimeout(15 * 1000); huc.setRequestMethod("GET"); huc.connect(); InputStream input = huc.getInputStream(); BufferedReader in = new BufferedReader(new InputStreamReader(input)); String inputLine; while ((inputLine = in.readLine()) != null) { if (!(inputLine.startsWith("S") || inputLine.startsWith("s"))) { String[] fields = inputLine.split(","); int M = Integer.parseInt(fields[3]); int D = Integer.parseInt(fields[4]); int a = (14 - M) / 12; int y = Integer.parseInt(fields[2]) + 4800 - a; int m = M + 12 * a - 3; long JD = D + (153 * m + 2) / 5 + 365 * y + y / 4 - y / 100 + y / 400 - 32045; String tempDate = String.format("%05d", Integer.parseInt(fields[2].substring(2)) * 1000 + Integer.parseInt(fields[5])); String SRad; if (fields[13].length() > 1) { SRad = String.format("%.1f", Double.parseDouble(fields[13])); } else { SRad = ""; } String TMax = String.format("%.1f", Double.parseDouble(fields[9])); String TMin = String.format("%.1f", Double.parseDouble(fields[8])); String RAIN = String.format("%.1f", Double.parseDouble(fields[12])); String RHUM = String.format("%.1f", Double.parseDouble(fields[11])); String WIND = String.format("%.1f", Double.parseDouble(fields[14])); String TDEW = String.format("%.1f", Double.parseDouble(fields[10])); String PAR = ""; String line = String.format("%5s %5s %5s %5s %5s %5s %5s %5s", tempDate, SRad, TMax, TMin, RAIN, RHUM, WIND, TDEW); pr.println(line); } } in.close(); } catch (MalformedURLException me) { me.printStackTrace(); //return 21.4; } catch (Exception e) { e.printStackTrace(); } } catch (Exception e) { e.printStackTrace(); } finally { if (pr != null) { pr.close(); } } }
From source file:org.pentaho.pat.server.servlet.PentahoServlet.java
private final boolean verifyXmlaUrl(final String xmlaUrl) { if (xmlaUrl == null) { return false; }//from ww w . ja va2 s . c om try { final URL url = new URL(xmlaUrl); // TODO support connection timeout. final URLConnection connection = url.openConnection(); if (connection instanceof HttpURLConnection) { HttpURLConnection.setFollowRedirects(true); final HttpURLConnection httpConnection = (HttpURLConnection) connection; httpConnection.connect(); return true; } else { return false; } } catch (Throwable t) { // TODO log this return false; } }
From source file:io.github.bonigarcia.wdm.Downloader.java
private static HttpURLConnection getConnection(URL url) throws IOException { Proxy proxy = createProxy();//from w w w. j a v a2 s. co m URLConnection conn1 = proxy != null ? url.openConnection(proxy) : url.openConnection(); HttpURLConnection conn = (HttpURLConnection) conn1; conn.setRequestProperty("User-Agent", "Mozilla/5.0"); conn.addRequestProperty("Connection", "keep-alive"); conn.setInstanceFollowRedirects(true); HttpURLConnection.setFollowRedirects(true); conn.connect(); return conn; }
From source file:com.epam.catgenome.manager.externaldb.HttpDataManager.java
private String getResultFromURL(final String location) throws ExternalDbUnavailableException { HttpURLConnection conn = null; try {// w w w. j av a2 s. com conn = createConnection(location); HttpURLConnection.setFollowRedirects(true); conn.setDoInput(true); conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON); conn.connect(); int status = conn.getResponseCode(); while (true) { long wait = 0; String header = conn.getHeaderField(HTTP_HEADER_RETRY_AFTER); if (header != null) { wait = Integer.valueOf(header); } if (wait == 0) { break; } LOGGER.info(String.format(WAITING_FORMAT, wait)); conn.disconnect(); Thread.sleep(wait * MILLIS_IN_SECOND); conn = (HttpURLConnection) new URL(location).openConnection(); conn.setDoInput(true); conn.connect(); status = conn.getResponseCode(); } return getHttpResult(status, location, conn); } catch (InterruptedException | IOException e) { throw new ExternalDbUnavailableException(String.format(EXCEPTION_MESSAGE, location), e); } finally { if (conn != null) { conn.disconnect(); } resetProxy(); } }
From source file:net.sbbi.upnp.messages.StateVariableMessage.java
/** * Executes the state variable query and retuns the UPNP device response, according to the UPNP specs, * this method could take up to 30 secs to process ( time allowed for a device to respond to a request ) * @return a state variable response object containing the variable value * @throws IOException if some IOException occurs during message send and reception process * @throws UPNPResponseException if an UPNP error message is returned from the server * or if some parsing exception occurs ( detailErrorCode = 899, detailErrorDescription = SAXException message ) *//* w w w .ja v a 2 s. c om*/ public StateVariableResponse service() throws IOException, UPNPResponseException { StateVariableResponse rtrVal = null; UPNPResponseException upnpEx = null; IOException ioEx = null; StringBuffer body = new StringBuffer(256); body.append("<?xml version=\"1.0\"?>\r\n"); body.append("<s:Envelope xmlns:s=\"http://schemas.xmlsoap.org/soap/envelope/\""); body.append(" s:encodingStyle=\"http://schemas.xmlsoap.org/soap/encoding/\">"); body.append("<s:Body>"); body.append("<u:QueryStateVariable xmlns:u=\"urn:schemas-upnp-org:control-1-0\">"); body.append("<u:varName>").append(serviceStateVar.getName()).append("</u:varName>"); body.append("</u:QueryStateVariable>"); body.append("</s:Body>"); body.append("</s:Envelope>"); if (log.isDebugEnabled()) log.debug("POST prepared for URL " + service.getControlURL()); URL url = new URL(service.getControlURL().toString()); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setDoInput(true); conn.setDoOutput(true); conn.setUseCaches(false); conn.setRequestMethod("POST"); HttpURLConnection.setFollowRedirects(false); //conn.setConnectTimeout( 30000 ); conn.setRequestProperty("HOST", url.getHost() + ":" + url.getPort()); conn.setRequestProperty("SOAPACTION", "\"urn:schemas-upnp-org:control-1-0#QueryStateVariable\""); conn.setRequestProperty("CONTENT-TYPE", "text/xml; charset=\"utf-8\""); conn.setRequestProperty("CONTENT-LENGTH", Integer.toString(body.length())); OutputStream out = conn.getOutputStream(); out.write(body.toString().getBytes()); out.flush(); conn.connect(); InputStream input = null; if (log.isDebugEnabled()) log.debug("executing query :\n" + body); try { input = conn.getInputStream(); } catch (IOException ex) { // java can throw an exception if he error code is 500 or 404 or something else than 200 // but the device sends 500 error message with content that is required // this content is accessible with the getErrorStream input = conn.getErrorStream(); } if (input != null) { int response = conn.getResponseCode(); String responseBody = getResponseBody(input); if (log.isDebugEnabled()) log.debug("received response :\n" + responseBody); SAXParserFactory saxParFact = SAXParserFactory.newInstance(); saxParFact.setValidating(false); saxParFact.setNamespaceAware(true); StateVariableResponseParser msgParser = new StateVariableResponseParser(serviceStateVar); StringReader stringReader = new StringReader(responseBody); InputSource src = new InputSource(stringReader); try { SAXParser parser = saxParFact.newSAXParser(); parser.parse(src, msgParser); } catch (ParserConfigurationException confEx) { // should never happen // we throw a runtimeException to notify the env problem throw new RuntimeException( "ParserConfigurationException during SAX parser creation, please check your env settings:" + confEx.getMessage()); } catch (SAXException saxEx) { // kind of tricky but better than nothing.. upnpEx = new UPNPResponseException(899, saxEx.getMessage()); } finally { try { input.close(); } catch (IOException ex) { // ignoring } } if (upnpEx == null) { if (response == HttpURLConnection.HTTP_OK) { rtrVal = msgParser.getStateVariableResponse(); } else if (response == HttpURLConnection.HTTP_INTERNAL_ERROR) { upnpEx = msgParser.getUPNPResponseException(); } else { ioEx = new IOException("Unexpected server HTTP response:" + response); } } } try { out.close(); } catch (IOException ex) { // ignore } conn.disconnect(); if (upnpEx != null) { throw upnpEx; } if (rtrVal == null && ioEx == null) { ioEx = new IOException("Unable to receive a response from the UPNP device"); } if (ioEx != null) { throw ioEx; } return rtrVal; }
From source file:com.anritsu.mcreleaseportal.tcintegration.ProcessTCRequest.java
public boolean URLExists(String url) { try {//ww w . j a v a2 s .co m HttpURLConnection.setFollowRedirects(true); HttpURLConnection con = (HttpURLConnection) new URL(url).openConnection(); con.setRequestMethod("HEAD"); return (con.getResponseCode() == HttpURLConnection.HTTP_OK); } catch (Exception ex) { Logger.getLogger(ProcessTCRequest.class.getName()).log(Level.SEVERE, null, url + "\n" + ex); return false; } }
From source file:org.m2x.rssreader.service.FetcherService.java
public FetcherService() { super(SERVICENAME); HttpURLConnection.setFollowRedirects(true); }