List of usage examples for org.apache.http.util EntityUtils consume
public static void consume(HttpEntity httpEntity) throws IOException
From source file:net.cloudkit.relaxation.HttpClientTest.java
public static void main(String[] args) throws Exception { InetAddress[] addresss = InetAddress.getAllByName("google.com"); for (InetAddress address : addresss) { System.out.println(address); }//from w w w .j a va 2 s . co m CloseableHttpClient httpclient = HttpClients.createDefault(); String __VIEWSTATE = ""; String __EVENTVALIDATION = ""; HttpGet httpGet = new HttpGet("http://query.customs.gov.cn/MNFTQ/MRoadQuery.aspx?" + Math.random() * 1000); httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); httpGet.setHeader("Accept-Encoding", "gzip, deflate, sdch"); httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6"); httpGet.setHeader("Cache-Control", "no-cache"); // httpGet.setHeader("Connection", "keep-alive"); httpGet.setHeader("Host", "query.customs.gov.cn"); httpGet.setHeader("Pragma", "no-cache"); httpGet.setHeader("Upgrade-Insecure-Requests", "1"); httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36"); HttpClientContext context = HttpClientContext.create(); // CloseableHttpResponse response1 = httpclient.execute(httpGet, context); CloseableHttpResponse response1 = httpclient.execute(httpGet); // Header[] headers = response1.getHeaders(HttpHeaders.CONTENT_TYPE); // System.out.println("context cookies:" + context.getCookieStore().getCookies()); // String setCookie = response1.getFirstHeader("Set-Cookie").getValue(); // System.out.println("context cookies:" + setCookie); try { System.out.println(response1.getStatusLine()); HttpEntity entity1 = response1.getEntity(); // do something useful with the response body and ensure it is fully consumed String result = IOUtils.toString(entity1.getContent(), "GBK"); // System.out.println(result); Matcher m1 = Pattern.compile( "<input type=\\\"hidden\\\" name=\\\"__VIEWSTATE\\\" id=\\\"__VIEWSTATE\\\" value=\\\"(.*)\\\" />") .matcher(result); __VIEWSTATE = m1.find() ? m1.group(1) : ""; Matcher m2 = Pattern.compile( "<input type=\\\"hidden\\\" name=\\\"__EVENTVALIDATION\\\" id=\\\"__EVENTVALIDATION\\\" value=\\\"(.*)\\\" />") .matcher(result); __EVENTVALIDATION = m2.find() ? m2.group(1) : ""; System.out.println(__VIEWSTATE); System.out.println(__EVENTVALIDATION); /* File storeFile = new File("D:\\customs\\customs"+ i +".jpg"); FileOutputStream output = new FileOutputStream(storeFile); IOUtils.copy(input, output); output.close(); */ EntityUtils.consume(entity1); } finally { response1.close(); } HttpPost httpPost = new HttpPost( "http://query.customs.gov.cn/MNFTQ/MRoadQuery.aspx?" + Math.random() * 1000); httpPost.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); httpPost.setHeader("Accept-Encoding", "gzip, deflate"); httpPost.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6"); httpPost.setHeader("Cache-Control", "no-cache"); // httpPost.setHeader("Connection", "keep-alive"); httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded"); httpPost.setHeader("Cookie", "ASP.NET_SessionId=t1td453hcuy4oqiplekkqe55"); httpPost.setHeader("Host", "query.customs.gov.cn"); httpPost.setHeader("Origin", "http://query.customs.gov.cn"); httpPost.setHeader("Pragma", "no-cache"); httpPost.setHeader("Referer", "http://query.customs.gov.cn/MNFTQ/MRoadQuery.aspx"); httpPost.setHeader("Upgrade-Insecure-Requests", "1"); httpPost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36"); List<NameValuePair> nvps = new ArrayList<NameValuePair>(); nvps.add(new BasicNameValuePair("__VIEWSTATE", __VIEWSTATE)); nvps.add(new BasicNameValuePair("__EVENTVALIDATION", __EVENTVALIDATION)); nvps.add(new BasicNameValuePair("ScrollTop", "")); nvps.add(new BasicNameValuePair("__essVariable", "")); nvps.add(new BasicNameValuePair("MRoadQueryCtrl1$txtManifestID", "5100312462240")); nvps.add(new BasicNameValuePair("MRoadQueryCtrl1$txtBillNo", "7PH650021105")); nvps.add(new BasicNameValuePair("MRoadQueryCtrl1$txtCode", "a778")); nvps.add(new BasicNameValuePair("MRoadQueryCtrl1$btQuery", " ")); nvps.add(new BasicNameValuePair("select", "")); nvps.add(new BasicNameValuePair("select1", "")); nvps.add(new BasicNameValuePair("select2", "")); nvps.add(new BasicNameValuePair("select3", "")); nvps.add(new BasicNameValuePair("select4", "")); nvps.add(new BasicNameValuePair("select5", "??")); nvps.add(new BasicNameValuePair("select6", "")); nvps.add(new BasicNameValuePair("select7", "")); nvps.add(new BasicNameValuePair("select8", "")); httpPost.setEntity(new UrlEncodedFormEntity(nvps, "GBK")); CloseableHttpResponse response2 = httpclient.execute(httpPost); try { System.out.println(response2.getStatusLine()); HttpEntity entity2 = response2.getEntity(); // do something useful with the response body // and ensure it is fully consumed // System.out.println(entity2.getContent()); System.out.println(IOUtils.toString(response2.getEntity().getContent(), "GBK")); EntityUtils.consume(entity2); } finally { response2.close(); } }
From source file:edu.harvard.hul.ois.drs.pdfaconvert.clients.FormFileUploaderClientApplication.java
/** * Run the program.//from ww w . j ava2s . co m * * @param args First argument is path to the file to analyze; second (optional) is path to server for overriding default value. */ public static void main(String[] args) { // takes file path from first program's argument if (args.length < 1) { logger.error("****** Path to input file must be first argument to program! *******"); logger.error("===== Exiting Program ====="); System.exit(1); } String filePath = args[0]; File uploadFile = new File(filePath); if (!uploadFile.exists()) { logger.error("****** File does not exist at expected locations! *******"); logger.error("===== Exiting Program ====="); System.exit(1); } if (args.length > 1) { serverUrl = args[1]; } logger.info("File to upload: " + filePath); CloseableHttpClient httpclient = HttpClients.createDefault(); try { HttpPost httppost = new HttpPost(serverUrl); FileBody bin = new FileBody(uploadFile); HttpEntity reqEntity = MultipartEntityBuilder.create().addPart(FORM_FIELD_DATAFILE, bin).build(); httppost.setEntity(reqEntity); logger.info("executing request " + httppost.getRequestLine()); CloseableHttpResponse response = httpclient.execute(httppost); try { logger.info("HTTP Response Status Line: " + response.getStatusLine()); // Expecting a 200 Status Code if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { String reason = response.getStatusLine().getReasonPhrase(); logger.warn("Unexpected HTTP response status code:[" + response.getStatusLine().getStatusCode() + "] -- Reason (if available): " + reason); } else { HttpEntity resEntity = response.getEntity(); InputStream is = resEntity.getContent(); BufferedReader in = new BufferedReader(new InputStreamReader(is)); String output; StringBuilder sb = new StringBuilder("Response data received:"); while ((output = in.readLine()) != null) { sb.append(System.getProperty("line.separator")); sb.append(output); } logger.info(sb.toString()); in.close(); EntityUtils.consume(resEntity); } } finally { response.close(); } } catch (Exception e) { logger.error("Caught exception: " + e.getMessage(), e); } finally { try { httpclient.close(); } catch (IOException e) { logger.warn("Exception closing HTTP client: " + e.getMessage(), e); } logger.info("DONE"); } }
From source file:com.atlbike.etl.service.ClientFormLogin.java
/** * @param args//from w ww .j a va 2 s .co m * @throws Exception */ public static void main(String[] args) throws Exception { String authenticityToken = ""; BasicCookieStore cookieStore = new BasicCookieStore(); CloseableHttpClient httpclient = HttpClients.custom().setDefaultCookieStore(cookieStore) .setRedirectStrategy(new LaxRedirectStrategy()).build(); try { HttpGet httpget = new HttpGet("https://atlbike.nationbuilder.com/login"); CloseableHttpResponse response1 = httpclient.execute(httpget); try { HttpEntity entity = response1.getEntity(); System.out.println("Content Length: " + entity.getContentLength()); System.out.println("Login form get: " + response1.getStatusLine()); // EntityUtils.consume(entity); String content = EntityUtils.toString(entity); Document doc = Jsoup.parse(content); Elements metaElements = doc.select("META"); for (Element elem : metaElements) { System.out.println(elem); if (elem.hasAttr("name") && "csrf-token".equals(elem.attr("name"))) { System.out.println("Value: " + elem.attr("content")); authenticityToken = elem.attr("content"); } } System.out.println("Initial set of cookies:"); List<Cookie> cookies = cookieStore.getCookies(); if (cookies.isEmpty()) { System.out.println("None"); } else { for (int i = 0; i < cookies.size(); i++) { System.out.println("- " + cookies.get(i).toString()); } } } finally { response1.close(); } HttpUriRequest login = RequestBuilder.post() .setUri(new URI("https://atlbike.nationbuilder.com/forms/user_sessions")) .addParameter("email_address", "").addParameter("user_session[email]", "email@domain") .addParameter("user_session[password]", "magicCookie") .addParameter("user_session[remember_me]", "1").addParameter("commit", "Sign in with email") .addParameter("authenticity_token", authenticityToken).build(); CloseableHttpResponse response2 = httpclient.execute(login); try { HttpEntity entity = response2.getEntity(); // for (Header h : response2.getAllHeaders()) { // System.out.println(h); // } System.out.println("Content Length: " + entity.getContentLength()); System.out.println("Login form get: " + response2.getStatusLine()); EntityUtils.consume(entity); System.out.println("Post logon cookies:"); List<Cookie> cookies = cookieStore.getCookies(); if (cookies.isEmpty()) { System.out.println("None"); } else { for (int i = 0; i < cookies.size(); i++) { System.out.println("- " + cookies.get(i).toString()); } } } finally { response2.close(); } httpget = new HttpGet( // HttpUriRequest file = RequestBuilder // .post() // .setUri(new URI( "https://atlbike.nationbuilder.com/admin/membership_types/14/download"); // .build(); // CloseableHttpResponse response3 = httpclient.execute(file); CloseableHttpResponse response3 = httpclient.execute(httpget); try { HttpEntity entity = response3.getEntity(); System.out.println("Content Length: " + entity.getContentLength()); System.out.println("File Get: " + response3.getStatusLine()); saveEntity(entity); // EntityUtils.consume(entity); System.out.println("Post file get cookies:"); List<Cookie> cookies = cookieStore.getCookies(); if (cookies.isEmpty()) { System.out.println("None"); } else { for (int i = 0; i < cookies.size(); i++) { System.out.println("- " + cookies.get(i).toString()); } } } finally { response3.close(); } } finally { httpclient.close(); } }
From source file:edu.harvard.hul.ois.fits.clients.FormFileUploaderClientApplication.java
/** * Run the program.//from w ww . j a va 2s . c om * * @param args First argument is path to the file to analyze; second (optional) is path to server for overriding default value. */ public static void main(String[] args) { // takes file path from first program's argument if (args.length < 1) { logger.error("****** Path to input file must be first argument to program! *******"); logger.error("===== Exiting Program ====="); System.exit(1); } String filePath = args[0]; File uploadFile = new File(filePath); if (!uploadFile.exists()) { logger.error("****** File does not exist at expected locations! *******"); logger.error("===== Exiting Program ====="); System.exit(1); } if (args.length > 1) { serverUrl = args[1]; } logger.info("File to upload: " + filePath); CloseableHttpClient httpclient = HttpClients.createDefault(); try { HttpPost httppost = new HttpPost(serverUrl + "false"); FileBody bin = new FileBody(uploadFile); MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.addPart(FITS_FORM_FIELD_DATAFILE, bin); HttpEntity reqEntity = builder.build(); httppost.setEntity(reqEntity); logger.info("executing request " + httppost.getRequestLine()); CloseableHttpResponse response = httpclient.execute(httppost); try { logger.info("HTTP Response Status Line: " + response.getStatusLine()); // Expecting a 200 Status Code if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { String reason = response.getStatusLine().getReasonPhrase(); logger.warn("Unexpected HTTP response status code:[" + response.getStatusLine().getStatusCode() + "] -- Reason (if available): " + reason); } else { HttpEntity resEntity = response.getEntity(); InputStream is = resEntity.getContent(); BufferedReader in = new BufferedReader(new InputStreamReader(is)); String output; StringBuilder sb = new StringBuilder(); while ((output = in.readLine()) != null) { sb.append(output); sb.append(System.getProperty("line.separator")); } logger.info(sb.toString()); in.close(); EntityUtils.consume(resEntity); } } finally { response.close(); } } catch (Exception e) { logger.error("Caught exception: " + e.getMessage(), e); } finally { try { httpclient.close(); } catch (IOException e) { logger.warn("Exception closing HTTP client: " + e.getMessage(), e); } logger.info("DONE"); } }
From source file:CourserankConnector.java
public static void main(String[] args) throws Exception { /////////////////////////////////////// //Tagger init //MaxentTagger tagger = new MaxentTagger("models/english-left3words-distsim.tagger"); ////* w w w . j a v a 2s.c om*/ //CLIENT INITIALIZATION ImportData importCourse = new ImportData(); HttpClient httpclient = new DefaultHttpClient(); httpclient = WebClientDevWrapper.wrapClient(httpclient); try { /* httpclient.getCredentialsProvider().setCredentials( new AuthScope(null, -1), new UsernamePasswordCredentials("eadrian", "eactresp1")); */ ////////////////////////////////////////////////// //Get Course Bulletin Departments page List<Course> courses = new ArrayList<Course>(); HttpGet httpget = new HttpGet("http://explorecourses.stanford.edu"); System.out.println("executing request" + httpget.getRequestLine()); HttpResponse response = httpclient.execute(httpget); HttpEntity entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); String bulletinpage = ""; //STORE RETURNED HTML TO BULLETINPAGE if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { bulletinpage += line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); /////////////////////////////////////////////////////////////////////////////// //Login to Courserank httpget = new HttpGet("https://courserank.com/stanford/main"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); String page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page += line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); //////////////////////////////////////////////////// //POST REQUEST LOGIN HttpPost post = new HttpPost("https://www.courserank.com/stanford/main"); List<NameValuePair> pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("RT", "")); pairs.add(new BasicNameValuePair("action", "login")); pairs.add(new BasicNameValuePair("password", "trespass")); pairs.add(new BasicNameValuePair("username", "eaconte@stanford.edu")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); HttpResponse resp = httpclient.execute(post); HttpEntity ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /////////////////////////////////////////////////// //THIS STEP MAY NOT BE NEEDED BUT GETS MAIN PROFILE PAGE HttpGet gethome = new HttpGet("https://www.courserank.com/stanford/home"); System.out.println("executing request" + gethome.getRequestLine()); HttpResponse gresp = httpclient.execute(gethome); HttpEntity gent = gresp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + gent.getContentLength()); InputStream i = gent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } ///////////////////////////////////////////////////////////////////////////////// //Parse Bulletin String results = getToken(bulletinpage, "RESULTS HEADER", "Additional Searches"); String[] depts = results.split("href"); //SPLIT FOR EACH DEPARTMENT LINK, ITERATE boolean ready = false; for (int i = 1; i < depts.length; i++) { //EXTRACT LINK, DEPARTMENT NAME AND ABBREVIATION String dept = new String(depts[i]); String abbr = getToken(dept, "(", ")"); String name = getToken(dept, ">", "("); name.trim(); //System.out.println(tagger.tagString(name)); String link = getToken(dept, "=\"", "\">"); System.out.println(name + " : " + abbr + " : " + link); System.out.println("======================================================================"); if (i <= 10 || i >= 127) //values to keep it to undergraduate courses. Excludes law, med, business, overseas continue; /*if (i<=46) continue; */ //Start at BIOHOP /*if (abbr.equals("INTNLREL")) ready = true; if (!ready) continue;*/ //Construct department course search URL //Then request page String URL = "http://explorecourses.stanford.edu/" + link + "&filter-term-Autumn=on&filter-term-Winter=on&filter-term-Spring=on"; httpget = new HttpGet(URL); //System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); //ystem.out.println("----------------------------------------"); //System.out.println(response.getStatusLine()); String rpage = ""; if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream in = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String line; while ((line = br.readLine()) != null) { rpage += line; //System.out.println(line); } br.close(); in.close(); } EntityUtils.consume(entity); //Process results page List<Course> deptCourses = new ArrayList<Course>(); List<Course> result = processResultPage(rpage); deptCourses.addAll(result); //While there are more result pages, keep going boolean more = (!(result.size() == 0) && (result.get((result.size() - 1)).courseNumber < 299)); boolean morepages = anotherPage(rpage); while (morepages && more) { URL = nextURL(URL); httpget = new HttpGet(URL); //System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); //System.out.println("----------------------------------------"); //System.out.println(response.getStatusLine()); rpage = ""; if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream in = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String line; while ((line = br.readLine()) != null) { rpage += line; //System.out.println(line); } br.close(); in.close(); } EntityUtils.consume(entity); morepages = anotherPage(rpage); result = processResultPage(rpage); deptCourses.addAll(result); more = (!(result.size() == 0) && (result.get((result.size() - 1)).courseNumber < 299)); /*String mores = more? "yes": "no"; String pagess = morepages?"yes":"no"; System.out.println("more: "+mores+" morepages: "+pagess); System.out.println("more");*/ } //Get course ratings for all department courses via courserank deptCourses = getRatings(httpclient, abbr, deptCourses); for (int j = 0; j < deptCourses.size(); j++) { Course c = deptCourses.get(j); System.out.println("" + c.title + " : " + c.rating); c.tags = name; c.code = c.code.trim(); c.department = name; c.deptAB = abbr; c.writeToDatabase(); //System.out.println(tagger.tagString(c.title)); } } if (!page.equals("")) return; /////////////////////////////////////////////////// //Get Course Bulletin Department courses /* httpget = new HttpGet("https://courserank.com/stanford/main"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page +=line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); //////////////////////////////////////////////////// //POST REQUEST LOGIN HttpPost post = new HttpPost("https://www.courserank.com/stanford/main"); List<NameValuePair> pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("RT", "")); pairs.add(new BasicNameValuePair("action", "login")); pairs.add(new BasicNameValuePair("password", "trespass")); pairs.add(new BasicNameValuePair("username", "eaconte@stanford.edu")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); HttpResponse resp = httpclient.execute(post); HttpEntity ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /////////////////////////////////////////////////// //THIS STEP MAY NOT BE NEEDED BUT GETS MAIN PROFILE PAGE HttpGet gethome = new HttpGet("https://www.courserank.com/stanford/home"); System.out.println("executing request" + gethome.getRequestLine()); HttpResponse gresp = httpclient.execute(gethome); HttpEntity gent = gresp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + gent.getContentLength()); InputStream i = gent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } //////////////////////////////////////// //GETS FIRST PAGE OF RESULTS EntityUtils.consume(gent); post = new HttpPost("https://www.courserank.com/stanford/search_results"); pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("filter_term_currentYear", "on")); pairs.add(new BasicNameValuePair("query", "")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); resp = httpclient.execute(post); ent = resp.getEntity(); System.out.println("----------------------------------------"); String rpage = ""; if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { rpage += line; System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); //////////////////////////////////////////////////// //PARSE FIRST PAGE OF RESULTS //int index = rpage.indexOf("div class=\"searchItem"); String []classSplit = rpage.split("div class=\"searchItem"); for (int i=1; i<classSplit.length; i++) { String str = classSplit[i]; //ID String CID = getToken(str, "course?id=","\">"); // CODE String CODE = getToken(str,"class=\"code\">" ,":</"); //TITLE String NAME = getToken(str, "class=\"title\">","</"); //DESCRIP String DES = getToken(str, "class=\"description\">","</"); //TERM String TERM = getToken(str, "Terms:", "|"); //UNITS String UNITS = getToken(str, "Units:", "<br/>"); //WORKLOAD String WLOAD = getToken(str, "Workload:", "|"); //GER String GER = getToken(str, "GERs:", "</d"); //RATING int searchIndex = 0; float rating = 0; while (true) { int ratingIndex = str.indexOf("large_Full", searchIndex); if (ratingIndex ==-1) { int halfratingIndex = str.indexOf("large_Half", searchIndex); if (halfratingIndex == -1) break; else rating += .5; break; } searchIndex = ratingIndex+1; rating++; } String RATING = ""+rating; //GRADE String GRADE = getToken(str, "div class=\"unofficialGrade\">", "</"); if (GRADE.equals("NOT FOUND")) { GRADE = getToken(str, "div class=\"officialGrade\">", "</"); } //REVIEWS String REVIEWS = getToken(str, "class=\"ratings\">", " ratings"); System.out.println(""+CODE+" : "+NAME + " : "+CID); System.out.println("----------------------------------------"); System.out.println("Term: "+TERM+" Units: "+UNITS+ " Workload: "+WLOAD + " Grade: "+ GRADE); System.out.println("Rating: "+RATING+ " Reviews: "+REVIEWS); System.out.println("=========================================="); System.out.println(DES); System.out.println("=========================================="); } /////////////////////////////////////////////////// //GETS SECOND PAGE OF RESULTS post = new HttpPost("https://www.courserank.com/stanford/search_results"); pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("filter_term_currentYear", "on")); pairs.add(new BasicNameValuePair("page", "2")); pairs.add(new BasicNameValuePair("query", "")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); resp = httpclient.execute(post); ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /* httpget = new HttpGet("https://github.com/"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page +=line; System.out.println(line); } br.close(); i.close(); }*/ EntityUtils.consume(entity); } finally { // When HttpClient instance is no longer needed, // shut down the connection manager to ensure // immediate deallocation of all system resources httpclient.getConnectionManager().shutdown(); } }
From source file:org.nmdp.b12s.mac.client.http.X509Config.java
public static void main(String[] args) throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException, UnrecoverableKeyException { URL trustKeyStoreUrl = X509Config.class.getResource("/trusted.jks"); URL clientKeyStoreUri = X509Config.class.getResource("/test-client.jks"); SSLContext sslContext = SSLContexts.custom() // Configure trusted certs .loadTrustMaterial(trustKeyStoreUrl, "changeit".toCharArray()) // Configure client certificate .loadKeyMaterial(clientKeyStoreUri, "changeit".toCharArray(), "changeit".toCharArray()).build(); try (TextHttpClient httpClient = new TextHttpClient("https://macbeta.b12x.org/mac/api", sslContext)) { }/* w ww . j a v a 2 s . c o m*/ // Allow TLSv1 protocol only SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, new String[] { "TLSv1" }, null, SSLConnectionSocketFactory.getDefaultHostnameVerifier()); try (CloseableHttpClient httpclient = HttpClients.custom().setSSLSocketFactory(sslsf).build()) { HttpGet httpget = new HttpGet("https://macbeta.b12x.org/mac/api/codes/AA"); System.out.println("executing request " + httpget.getRequestLine()); try (CloseableHttpResponse response = httpclient.execute(httpget)) { HttpEntity entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); if (entity != null) { Charset charset = StandardCharsets.UTF_8; for (Header contentType : response.getHeaders("Content-Type")) { System.out.println("Content-Type: " + contentType); for (String part : contentType.getValue().split(";")) { if (part.startsWith("charset=")) { String charsetName = part.split("=")[1]; charset = Charset.forName(charsetName); } } } System.out.println("Response content length: " + entity.getContentLength()); String content = EntityUtils.toString(entity, charset); System.out.println(content); } EntityUtils.consume(entity); } } }
From source file:com.dlmu.heipacker.crawler.client.ClientKerberosAuthentication.java
public static void main(String[] args) throws Exception { System.setProperty("java.security.auth.login.config", "login.conf"); System.setProperty("java.security.krb5.conf", "krb5.conf"); System.setProperty("sun.security.krb5.debug", "true"); System.setProperty("javax.security.auth.useSubjectCredsOnly", "false"); DefaultHttpClient httpclient = new DefaultHttpClient(); try {//from w ww . j av a2s .co m httpclient.getAuthSchemes().register(AuthPolicy.SPNEGO, new SPNegoSchemeFactory()); Credentials use_jaas_creds = new Credentials() { public String getPassword() { return null; } public Principal getUserPrincipal() { return null; } }; httpclient.getCredentialsProvider().setCredentials(new AuthScope(null, -1, null), use_jaas_creds); HttpUriRequest request = new HttpGet("http://kerberoshost/"); HttpResponse response = httpclient.execute(request); HttpEntity entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); System.out.println("----------------------------------------"); if (entity != null) { System.out.println(EntityUtils.toString(entity)); } System.out.println("----------------------------------------"); // This ensures the connection gets released back to the manager EntityUtils.consume(entity); } finally { // When HttpClient instance is no longer needed, // shut down the connection manager to ensure // immediate deallocation of all system resources httpclient.getConnectionManager().shutdown(); } }
From source file:com.renren.ntc.sg.util.wxpay.https.ClientCustomSSL.java
public final static void main(String[] args) throws Exception { KeyStore keyStore = KeyStore.getInstance("PKCS12"); FileInputStream instream = new FileInputStream( new File("/Users/allenz/Downloads/wx_cert/apiclient_cert.p12")); try {//from www . j a va 2 s. c o m keyStore.load(instream, Constants.mch_id.toCharArray()); } finally { instream.close(); } // Trust own CA and all self-signed certs SSLContext sslcontext = SSLContexts.custom().loadKeyMaterial(keyStore, Constants.mch_id.toCharArray()) .build(); // Allow TLSv1 protocol only SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext, new String[] { "TLSv1" }, null, SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER); CloseableHttpClient httpclient = HttpClients.custom().setSSLSocketFactory(sslsf).build(); try { HttpPost post = new HttpPost("https://api.mch.weixin.qq.com/mmpaymkttransfers/promotion/transfers"); System.out.println("executing request" + post.getRequestLine()); String openid = "oQfDLjmZD7Lgynv6vuoBlWXUY_ic"; String nonce_str = Sha1Util.getNonceStr(); String orderId = SUtils.getOrderId(); String re_user_name = "?"; String amount = "1"; String desc = ""; String spbill_create_ip = "123.56.102.224"; String txt = TXT.replace("{mch_appid}", Constants.mch_appid); txt = txt.replace("{mchid}", Constants.mch_id); txt = txt.replace("{openid}", openid); txt = txt.replace("{nonce_str}", nonce_str); txt = txt.replace("{partner_trade_no}", orderId); txt = txt.replace("{check_name}", "FORCE_CHECK"); txt = txt.replace("{re_user_name}", re_user_name); txt = txt.replace("{amount}", amount); txt = txt.replace("{desc}", desc); txt = txt.replace("{spbill_create_ip}", spbill_create_ip); SortedMap<String, String> map = new TreeMap<String, String>(); map.put("mch_appid", Constants.mch_appid); map.put("mchid", Constants.mch_id); map.put("openid", openid); map.put("nonce_str", nonce_str); map.put("partner_trade_no", orderId); //FORCE_CHECK| OPTION_CHECK | NO_CHECK map.put("check_name", "OPTION_CHECK"); map.put("re_user_name", re_user_name); map.put("amount", amount); map.put("desc", desc); map.put("spbill_create_ip", spbill_create_ip); String sign = SUtils.createSign(map).toUpperCase(); txt = txt.replace("{sign}", sign); post.setEntity(new StringEntity(txt, "utf-8")); CloseableHttpResponse response = httpclient.execute(post); try { HttpEntity entity = response.getEntity(); System.out.println(response.getStatusLine()); if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(entity.getContent())); String text; StringBuffer sb = new StringBuffer(); while ((text = bufferedReader.readLine()) != null) { sb.append(text); } String resp = sb.toString(); LoggerUtils.getInstance().log(String.format("req %s rec %s", txt, resp)); if (isOk(resp)) { String payment_no = getValue(resp, "payment_no"); LoggerUtils.getInstance() .log(String.format("order %s pay OK payment_no %s", orderId, payment_no)); } } EntityUtils.consume(entity); } finally { response.close(); } } finally { httpclient.close(); } }
From source file:simauthenticator.SimAuthenticator.java
/** * @param args the command line arguments *//* w ww .j av a 2 s . c om*/ public static void main(String[] args) throws Exception { cliOpts = new Options(); cliOpts.addOption("U", "url", true, "Connection URL"); cliOpts.addOption("u", "user", true, "User name"); cliOpts.addOption("p", "password", true, "User password"); cliOpts.addOption("d", "domain", true, "Domain name"); cliOpts.addOption("v", "verbose", false, "Verbose output"); cliOpts.addOption("k", "keystore", true, "KeyStore path"); cliOpts.addOption("K", "keystorepass", true, "KeyStore password"); cliOpts.addOption("h", "help", false, "Print help info"); CommandLineParser clip = new GnuParser(); cmd = clip.parse(cliOpts, args); if (cmd.hasOption("help")) { help(); return; } else { boolean valid = init(args); if (!valid) { return; } } HttpClientContext clientContext = HttpClientContext.create(); KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); char[] keystorePassword = passwk.toCharArray(); FileInputStream kfis = null; try { kfis = new FileInputStream(keyStorePath); ks.load(kfis, keystorePassword); } finally { if (kfis != null) { kfis.close(); } } SSLContext sslContext = SSLContexts.custom().useSSL().loadTrustMaterial(ks).build(); SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext); HttpClientBuilder httpClientBuilder = HttpClientBuilder.create().setSslcontext(sslContext) .setSSLSocketFactory(sslsf).setUserAgent(userAgent); ; cookieStore = new BasicCookieStore(); /* BasicClientCookie cookie = new BasicClientCookie("SIM authenticator", "Utility for getting event details"); cookie.setVersion(0); cookie.setDomain(".astelit.ukr"); cookie.setPath("/"); cookieStore.addCookie(cookie);*/ CloseableHttpClient client = httpClientBuilder.build(); try { NTCredentials creds = new NTCredentials(usern, passwu, InetAddress.getLocalHost().getHostName(), domain); CredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(AuthScope.ANY, creds); HttpClientContext context = HttpClientContext.create(); context.setCredentialsProvider(credsProvider); context.setCookieStore(cookieStore); HttpGet httpget = new HttpGet(eventUrl); if (verbose) { System.out.println("executing request " + httpget.getRequestLine()); } HttpResponse response = client.execute(httpget, context); HttpEntity entity = response.getEntity(); HttpPost httppost = new HttpPost(eventUrl); List<Cookie> cookies = cookieStore.getCookies(); if (verbose) { System.out.println("----------------------------------------------"); System.out.println(response.getStatusLine()); System.out.print("Initial set of cookies: "); if (cookies.isEmpty()) { System.out.println("none"); } else { for (int i = 0; i < cookies.size(); i++) { System.out.println("- " + cookies.get(i).toString()); } } } List<NameValuePair> nvps = new ArrayList<NameValuePair>(); nvps.add(new BasicNameValuePair("usernameInput", usern)); nvps.add(new BasicNameValuePair("passwordInput", passwu)); nvps.add(new BasicNameValuePair("domainInput", domain)); //nvps.add(new BasicNameValuePair("j_username", domain + "\\" + usern)); //nvps.add(new BasicNameValuePair("j_password", ipAddr + ";" + passwu)); if (entity != null && verbose) { System.out.println("Responce content length: " + entity.getContentLength()); } //System.out.println(EntityUtils.toString(entity)); httppost.setEntity(new UrlEncodedFormEntity(nvps, HTTP.UTF_8)); HttpResponse afterPostResponse = client.execute(httppost, context); HttpEntity afterPostEntity = afterPostResponse.getEntity(); cookies = cookieStore.getCookies(); if (entity != null && verbose) { System.out.println("----------------------------------------------"); System.out.println(afterPostResponse.getStatusLine()); System.out.println("Responce content length: " + afterPostEntity.getContentLength()); System.out.print("After POST set of cookies: "); if (cookies.isEmpty()) { System.out.println("none"); } else { for (int i = 0; i < cookies.size(); i++) { System.out.println("- " + cookies.get(i).toString()); } } } System.out.println(EntityUtils.toString(afterPostEntity)); EntityUtils.consume(entity); EntityUtils.consume(afterPostEntity); } finally { client.getConnectionManager().shutdown(); } }
From source file:org.eclipse.skalli.commons.HttpUtils.java
public static void consumeQuietly(HttpResponse response) { if (response != null) { try {/* w ww .ja v a2 s. c o m*/ EntityUtils.consume(response.getEntity()); } catch (IOException e) { LOG.warn("I/O Exception when trying to consume an HTTP response", e); //$NON-NLS-1$ } } }