Example usage for java.net URL openConnection

List of usage examples for java.net URL openConnection

Introduction

In this page you can find the example usage for java.net URL openConnection.

Prototype

public URLConnection openConnection() throws java.io.IOException 

Source Link

Document

Returns a java.net.URLConnection URLConnection instance that represents a connection to the remote object referred to by the URL .

Usage

From source file:com.pp.dcasajus.forkpoint.GcmSender.java

public static void main(String[] args) {
    if (args.length < 1 || args.length > 2 || args[0] == null) {
        System.err.println("usage: ./gradlew run -Pmsg=\"MESSAGE\" [-Pto=\"DEVICE_TOKEN\"]");
        System.err.println("");
        System.err.println(//from   ww  w .j ava  2 s . co m
                "Specify a test message to broadcast via GCM. If a device's GCM registration token is\n"
                        + "specified, the message will only be sent to that device. Otherwise, the message \n"
                        + "will be sent to all devices subscribed to the \"global\" topic.");
        System.err.println("");
        System.err.println(
                "Example (Broadcast):\n" + "On Windows:   .\\gradlew.bat run -Pmsg=\"<Your_Message>\"\n"
                        + "On Linux/Mac: ./gradlew run -Pmsg=\"<Your_Message>\"");
        System.err.println("");
        System.err.println("Example (Unicast):\n"
                + "On Windows:   .\\gradlew.bat run -Pmsg=\"<Your_Message>\" -Pto=\"<Your_Token>\"\n"
                + "On Linux/Mac: ./gradlew run -Pmsg=\"<Your_Message>\" -Pto=\"<Your_Token>\"");
        System.exit(1);
    }
    try {
        // Prepare JSON containing the GCM message content. What to send and where to send.
        JSONObject jGcmData = new JSONObject();
        JSONObject jData = new JSONObject();
        jData.put("message", args[0].trim());
        // Where to send GCM message.
        if (args.length > 1 && args[1] != null) {
            jGcmData.put("to", args[1].trim());
        } else {
            jGcmData.put("to", "/topics/global");
        }
        // What to send in GCM message.
        jGcmData.put("data", jData);

        // Create connection to send GCM Message request.
        URL url = new URL("https://android.googleapis.com/gcm/send");
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestProperty("Authorization", "key=" + API_KEY);
        conn.setRequestProperty("Content-Type", "application/json");
        conn.setRequestMethod("POST");
        conn.setDoOutput(true);

        // Send GCM message content.
        OutputStream outputStream = conn.getOutputStream();
        outputStream.write(jGcmData.toString().getBytes());

        // Read GCM response.
        InputStream inputStream = conn.getInputStream();
        String resp = IOUtils.toString(inputStream);
        System.out.println(resp);
        System.out.println("Check your device/emulator for notification or logcat for "
                + "confirmation of the receipt of the GCM message.");
    } catch (IOException e) {
        System.out.println("Unable to send GCM message.");
        System.out.println("Please ensure that API_KEY has been replaced by the server "
                + "API key, and that the device's registration token is correct (if specified).");
        e.printStackTrace();
    } catch (JSONException e) {
        e.printStackTrace();
    }
}

From source file:URLConnectionTest.java

public static void main(String[] args) {
    try {//from   w ww  . jav  a  2 s.c o  m
        String urlName;
        if (args.length > 0)
            urlName = args[0];
        else
            urlName = "http://java.sun.com";

        URL url = new URL(urlName);
        URLConnection connection = url.openConnection();

        // set username, password if specified on command line

        if (args.length > 2) {
            String username = args[1];
            String password = args[2];
            String input = username + ":" + password;
            String encoding = base64Encode(input);
            connection.setRequestProperty("Authorization", "Basic " + encoding);
        }

        connection.connect();

        // print header fields

        Map<String, List<String>> headers = connection.getHeaderFields();
        for (Map.Entry<String, List<String>> entry : headers.entrySet()) {
            String key = entry.getKey();
            for (String value : entry.getValue())
                System.out.println(key + ": " + value);
        }

        // print convenience functions

        System.out.println("----------");
        System.out.println("getContentType: " + connection.getContentType());
        System.out.println("getContentLength: " + connection.getContentLength());
        System.out.println("getContentEncoding: " + connection.getContentEncoding());
        System.out.println("getDate: " + connection.getDate());
        System.out.println("getExpiration: " + connection.getExpiration());
        System.out.println("getLastModifed: " + connection.getLastModified());
        System.out.println("----------");

        Scanner in = new Scanner(connection.getInputStream());

        // print first ten lines of contents

        for (int n = 1; in.hasNextLine() && n <= 10; n++)
            System.out.println(in.nextLine());
        if (in.hasNextLine())
            System.out.println(". . .");
    } catch (IOException e) {
        e.printStackTrace();
    }
}

From source file:com.jixstreet.temanusaha.gcm.GcmSender.java

public static void main(String[] args) {
    if (args.length < 1 || args.length > 2 || args[0] == null) {
        System.err.println("usage: ./gradlew run -Pmsg=\"MESSAGE\" [-Pto=\"DEVICE_TOKEN\"]");
        System.err.println("");
        System.err.println(/*from www .j a v  a2  s.  c o  m*/
                "Specify a test message to broadcast via GCM. If a device's GCM registration token is\n"
                        + "specified, the message will only be sent to that device. Otherwise, the message \n"
                        + "will be sent to all devices subscribed to the \"global\" topic.");
        System.err.println("");
        System.err.println(
                "Example (Broadcast):\n" + "On Windows:   .\\gradlew.bat run -Pmsg=\"<Your_Message>\"\n"
                        + "On Linux/Mac: ./gradlew run -Pmsg=\"<Your_Message>\"");
        System.err.println("");
        System.err.println("Example (Unicast):\n"
                + "On Windows:   .\\gradlew.bat run -Pmsg=\"<Your_Message>\" -Pto=\"<Your_Token>\"\n"
                + "On Linux/Mac: ./gradlew run -Pmsg=\"<Your_Message>\" -Pto=\"<Your_Token>\"");
        System.exit(1);
    }
    try {
        // Prepare JSON containing the GCM message content. What to send and where to send.
        JSONObject jGcmData = new JSONObject();
        JSONObject jData = new JSONObject();
        try {
            jData.put("message", args[0].trim());
        } catch (JSONException e) {
            e.printStackTrace();
        }
        // Where to send GCM message.
        if (args.length > 1 && args[1] != null) {
            try {
                jGcmData.put("to", args[1].trim());
            } catch (JSONException e) {
                e.printStackTrace();
            }
        } else {
            try {
                jGcmData.put("to", "/topics/global");
            } catch (JSONException e) {
                e.printStackTrace();
            }
        }
        // What to send in GCM message.
        try {
            JSONObject data = jGcmData.put("data", jData);
        } catch (JSONException e) {
            e.printStackTrace();
        }

        // Create connection to send GCM Message request.
        URL url = new URL("https://android.googleapis.com/gcm/send");
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestProperty("Authorization", "key=" + API_KEY);
        conn.setRequestProperty("Content-Type", "application/json");
        conn.setRequestMethod("POST");
        conn.setDoOutput(true);

        // Send GCM message content.
        OutputStream outputStream = conn.getOutputStream();
        outputStream.write(jGcmData.toString().getBytes());

        // Read GCM response.
        InputStream inputStream = conn.getInputStream();
        String resp = IOUtils.toString(inputStream);
        System.out.println(resp);
        System.out.println("Check your device/emulator for notification or logcat for "
                + "confirmation of the receipt of the GCM message.");
    } catch (IOException e) {
        System.out.println("Unable to send GCM message.");
        System.out.println("Please ensure that API_KEY has been replaced by the server "
                + "API key, and that the device's registration token is correct (if specified).");
        e.printStackTrace();
    }
}

From source file:com.newproject.ApacheHttp.java

public static void main(String[] args) throws Exception {
    CloseableHttpClient httpclient = HttpClients.createDefault();
    String jsonFilePath = "/Users/vikasmohandoss/Documents/Cloud/test.txt";
    String url = "http://www.sentiment140.com/api/bulkClassifyJson&appid=vm2446@columbia.edu";
    JSONParser jsonParser = new JSONParser();
    JSONObject jsonObject = new JSONObject();
    URL obj = new URL(url);
    HttpURLConnection con = (HttpURLConnection) obj.openConnection();
    try {/*  www.ja  va 2 s.  c  o  m*/
        FileReader fileReader = new FileReader(jsonFilePath);
        jsonObject = (JSONObject) jsonParser.parse(fileReader);
    } catch (FileNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    } catch (ParseException e) {
        e.printStackTrace();
    }
    System.out.println(jsonObject.toString());
    /*try {
    /*HttpGet httpGet = new HttpGet("http://httpbin.org/get");
    CloseableHttpResponse response1 = httpclient.execute(httpGet);
    // The underlying HTTP connection is still held by the response object
    // to allow the response content to be streamed directly from the network socket.
    // In order to ensure correct deallocation of system resources
    // the user MUST call CloseableHttpResponse#close() from a finally clause.
    // Please note that if response content is not fully consumed the underlying
    // connection cannot be safely re-used and will be shut down and discarded
    // by the connection manager.
    try {
        System.out.println(response1.getStatusLine());
        HttpEntity entity1 = response1.getEntity();
        // do something useful with the response body
        // and ensure it is fully consumed
        EntityUtils.consume(entity1);
    } finally {
        response1.close();
    }
    HttpPost httpPost = new HttpPost("http://httpbin.org/post");
    List <NameValuePair> nvps = new ArrayList <NameValuePair>();
    nvps.add(new BasicNameValuePair("username", "vip"));
    nvps.add(new BasicNameValuePair("password", "secret"));
    httpPost.setEntity(new UrlEncodedFormEntity(nvps));
    CloseableHttpResponse response2 = httpclient.execute(httpPost);
            
    try {
        System.out.println(response2.getStatusLine());
        HttpEntity entity2 = response2.getEntity();
        // do something useful with the response body
        // and ensure it is fully consumed
        EntityUtils.consume(entity2);
    } finally {
        response2.close();
    }
    } finally {
    httpclient.close();
    }*/
    try {
        HttpPost request = new HttpPost("http://www.sentiment140.com/api/bulkClassifyJson");
        StringEntity params = new StringEntity(jsonObject.toString());
        request.addHeader("content-type", "application/json");
        request.setEntity(params);
        HttpResponse response = httpclient.execute(request);
        System.out.println(response.toString());
        String result = EntityUtils.toString(response.getEntity());
        System.out.println(result);
        try {
            File file = new File("/Users/vikasmohandoss/Documents/Cloud/sentiment.txt");
            // if file doesnt exists, then create it
            if (!file.exists()) {
                file.createNewFile();
            }
            FileWriter fw = new FileWriter(file.getAbsoluteFile());
            BufferedWriter bw = new BufferedWriter(fw);
            bw.write(result);
            bw.close();
            System.out.println("Done");
        } catch (IOException e) {
            e.printStackTrace();
        }
        // handle response here...
    } catch (Exception ex) {
        // handle exception here
    } finally {
        httpclient.close();
    }
}

From source file:com.yahoo.athenz.example.ztoken.HttpExampleClient.java

public static void main(String[] args) throws MalformedURLException, IOException {

    // parse our command line to retrieve required input

    CommandLine cmd = parseCommandLine(args);

    String domainName = cmd.getOptionValue("domain");
    String serviceName = cmd.getOptionValue("service");
    String privateKeyPath = cmd.getOptionValue("pkey");
    String keyId = cmd.getOptionValue("keyid");
    String url = cmd.getOptionValue("url");
    String ztsUrl = cmd.getOptionValue("ztsurl");
    String providerDomain = cmd.getOptionValue("provider-domain");
    String providerRole = cmd.getOptionValue("provider-role");

    // we need to generate our principal credentials (ntoken). In
    // addition to the domain and service names, we need the
    // the service's private key and the key identifier - the
    // service with the corresponding public key must already be
    // registered in ZMS

    PrivateKey privateKey = Crypto.loadPrivateKey(new File(privateKeyPath));
    ServiceIdentityProvider identityProvider = new SimpleServiceIdentityProvider(domainName, serviceName,
            privateKey, keyId);/*from  w w w.j ava2  s.c o  m*/

    // now we need to retrieve a role token (ztoken) for accessing
    // the provider Athenz enabled service

    RoleToken roleToken = null;
    try (ZTSClient ztsClient = new ZTSClient(ztsUrl, domainName, serviceName, identityProvider)) {
        roleToken = ztsClient.getRoleToken(providerDomain, providerRole);
    }

    if (roleToken == null) {
        System.out.println(
                "Unable to retrieve role token for: " + providerRole + " in domain: " + providerDomain);
        System.exit(1);
    }

    URL obj = new URL(url);
    HttpURLConnection con = (HttpURLConnection) obj.openConnection();

    // set our Athenz credentials. The ZTSClient provides the header
    // name that we must use for authorization token while the role
    // token itself provides the token string (ztoken).

    System.out.println("Using RoleToken: " + roleToken.getToken());
    con.setRequestProperty(ZTSClient.getHeader(), roleToken.getToken());

    // now process our request

    int responseCode = con.getResponseCode();
    switch (responseCode) {
    case HttpURLConnection.HTTP_FORBIDDEN:
        System.out.println("Request was forbidden - not authorized: " + con.getResponseMessage());
        break;
    case HttpURLConnection.HTTP_OK:
        System.out.println("Successful response: ");
        try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()))) {
            String inputLine;
            while ((inputLine = in.readLine()) != null) {
                System.out.println(inputLine);
            }
        }
        break;
    default:
        System.out.println("Request failed - response status code: " + responseCode);
    }
}

From source file:SifterJava.java

public static void main(String[] args) throws Exception {
    /** Enter your sifter account and access key in the 
    ** string fields below *///from  ww  w  .  j a v  a2  s .  com

    // create URL object to SifterAPI
    String yourAccount = "your-account"; // enter your account name
    String yourDomain = yourAccount + ".sifterapp.com";
    URL sifter = new URL("https", yourDomain, "/api/projects");

    // open connectino to SifterAPI
    URLConnection sifterConnection = sifter.openConnection();

    // add Access Key to header request
    String yourAccessKey = "your-32char-sifterapi-access-key"; // enter your access key
    sifterConnection.setRequestProperty("X-Sifter-Token", yourAccessKey);

    // add Accept: application/json to header - also not necessary
    sifterConnection.addRequestProperty("Accept", "application/json");

    // URLconnection.connect() not necessary
    // getInputStream will connect

    // don't make a content handler, org.json reads a stream!

    // create buffer and open input stream
    BufferedReader in = new BufferedReader(new InputStreamReader(sifterConnection.getInputStream()));
    // don't readline, create stringbuilder, append, create string

    // construct json tokener from input stream or buffered reader
    JSONTokener x = new JSONTokener(in);

    // initialize "projects" JSONObject from string
    JSONObject projects = new JSONObject(x);

    // prettyprint "projects"
    System.out.println("************ projects ************");
    System.out.println(projects.toString(2));

    // array of projects
    JSONArray array = projects.getJSONArray("projects");
    int arrayLength = array.length();
    JSONObject[] p = new JSONObject[arrayLength];

    // projects
    for (int i = 0; i < arrayLength; i++) {
        p[i] = array.getJSONObject(i);
        System.out.println("************ project: " + (i + 1) + " ************");
        System.out.println(p[i].toString(2));
    }

    // check field names
    String[] checkNames = { "api_url", "archived", "api_issues_url", "milestones_url", "api_milestones_url",
            "api_categories_url", "issues_url", "name", "url", "api_people_url", "primary_company_name" };

    // project field names
    String[] fieldNames = JSONObject.getNames(p[0]);
    int numKeys = p[0].length();
    SifterProj[] proj = new SifterProj[arrayLength];

    for (int j = 0; j < arrayLength; j++) {
        proj[j] = new SifterProj(p[j].get("api_url").toString(), p[j].get("archived").toString(),
                p[j].get("api_issues_url").toString(), p[j].get("milestones_url").toString(),
                p[j].get("api_milestones_url").toString(), p[j].get("api_categories_url").toString(),
                p[j].get("issues_url").toString(), p[j].get("name").toString(), p[j].get("url").toString(),
                p[j].get("api_people_url").toString(), p[j].get("primary_company_name").toString());
        System.out.println("************ project: " + (j + 1) + " ************");
        for (int i = 0; i < numKeys; i++) {
            System.out.print(fieldNames[i]);
            System.out.print(" : ");
            System.out.println(p[j].get(fieldNames[i]));
        }
    }
}

From source file:OldExtractor.java

/**
 * @param args the command line arguments
 */// ww w.jav  a 2  s.c om
public static void main(String[] args) {
    // TODO code application logic here
    String bingUrl = "https://api.datamarket.azure.com/Bing/Search/Web?$top=10&$format=Atom&Query=%27gates%27";
    //Provide your account key here.
    String accountKey = "ghTYY7wD6LpyxUO9VRR7e1f98WFhHWYERMcw87aQTqQ";
    //  String accountKey = "xqbCjT87/MQz25JWdRzgMHdPkGYnOz77IYmP5FUIgC8";

    byte[] accountKeyBytes = Base64.encodeBase64((accountKey + ":" + accountKey).getBytes());
    String accountKeyEnc = new String(accountKeyBytes);

    try {
        URL url = new URL(bingUrl);
        URLConnection urlConnection = url.openConnection();

        urlConnection.setRequestProperty("Authorization", "Basic " + accountKeyEnc);
        InputStream inputStream = (InputStream) urlConnection.getContent();
        byte[] contentRaw = new byte[urlConnection.getContentLength()];
        inputStream.read(contentRaw);
        String content = new String(contentRaw);
        //System.out.println(content);
        try {
            File file = new File("Results.xml");
            FileWriter fileWriter = new FileWriter(file);
            fileWriter.write(content);
            //fileWriter.write("a test");
            fileWriter.flush();
            fileWriter.close();

        } catch (IOException e) {
            System.out.println(e);
        }

        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();

        try {
            //System.out.println("here");
            //Using factory get an instance of document builder
            DocumentBuilder db = dbf.newDocumentBuilder();

            //parse using builder to get DOM representation of the XML file
            Document dom = db.parse("Results.xml");
            Element docEle = (Element) dom.getDocumentElement();

            //get a nodelist of elements

            NodeList nl = docEle.getElementsByTagName("d:Url");
            if (nl != null && nl.getLength() > 0) {
                for (int i = 0; i < nl.getLength(); i++) {

                    //get the employee element
                    Element el = (Element) nl.item(i);
                    // System.out.println("here");
                    System.out.println(el.getTextContent());

                    //get the Employee object
                    //Employee e = getEmployee(el);

                    //add it to list
                    //myEmpls.add(e);
                }
            }
            NodeList n2 = docEle.getElementsByTagName("d:Title");
            if (n2 != null && n2.getLength() > 0) {
                for (int i = 0; i < n2.getLength(); i++) {

                    //get the employee element
                    Element e2 = (Element) n2.item(i);
                    // System.out.println("here");
                    System.out.println(e2.getTextContent());

                    //get the Employee object
                    //Employee e = getEmployee(el);

                    //add it to list
                    //myEmpls.add(e);
                }
            }

            NodeList n3 = docEle.getElementsByTagName("d:Description");
            if (n3 != null && n3.getLength() > 0) {
                for (int i = 0; i < n3.getLength(); i++) {

                    //get the employee element
                    Element e3 = (Element) n3.item(i);
                    // System.out.println("here");
                    System.out.println(e3.getTextContent());

                    //get the Employee object
                    //Employee e = getEmployee(el);

                    //add it to list
                    //myEmpls.add(e);
                }
            }

        } catch (SAXException se) {
            se.printStackTrace();
        } catch (ParserConfigurationException pe) {
            pe.printStackTrace();
        } catch (IOException ioe) {
            ioe.printStackTrace();
        }

    } catch (IOException e) {
        System.out.println(e);
    }

    //The content string is the xml/json output from Bing.

}

From source file:com.dict.crawl.NewsNationalGeographicCrawler.java

public static void main(String[] args) throws Exception {
    /*string,crawlPath??crawlPath,
      ????crawlPath/*w w w.  j  a va2 s.  c o  m*/
    */

    NewsNationalGeographicCrawler crawler = new NewsNationalGeographicCrawler("data/NewsNationalGeographic");
    crawler.setThreads(2);
    crawler.addSeed("http://ngm.nationalgeographic.com/");

    if (BaseCrawler.isNormalTime()) {

        crawler.addSeed("http://ngm.nationalgeographic.com/archives");
        crawler.addSeed("http://ngm.nationalgeographic.com/featurehub");
        //
        //}

        String jsonUrl = "http://news.nationalgeographic.com/bin/services/news/public/query/content.json?pageSize=20&page=0&contentTypes=news/components/pagetypes/article,news/components/pagetypes/simple-article,news/components/pagetypes/photo-gallery";

        URL urls = new URL(jsonUrl);
        HttpURLConnection urlConnection = (HttpURLConnection) urls.openConnection();
        InputStream is = urlConnection.getInputStream();
        Reader rd = new InputStreamReader(is, "utf-8");
        JsonArray json = new JsonParser().parse(rd).getAsJsonArray();
        for (JsonElement jOb : json) {
            String url = jOb.getAsJsonObject().get("page").getAsJsonObject().get("url").getAsString();
            if (url != null && !url.equals(""))
                crawler.addSeed(url);
        }

    }

    //        crawler.addSeed("http://news.nationalgeographic.com/2016/01/160118-mummies-world-bog-egypt-science/");
    //        List<Map<String, Object>> urls = crawler.jdbcTemplate.queryForList("SELECT id,title,url FROM parser_page where host like '%news.national%' ORDER by id desc;");
    //        for(int i = 0; i < urls.size(); i++) {
    //            String url = (String) urls.get(i).get("url");
    //            String title = (String) urls.get(i).get("title");
    ////            int id = (int) urls.get(i).get("id");
    //            crawler.addSeed(url);
    //        }

    //        Config
    Config.WAIT_THREAD_END_TIME = 1000 * 60 * 5;//???kill
    //        Config.TIMEOUT_CONNECT = 1000*10;
    //        Config.TIMEOUT_READ = 1000*30;
    Config.requestMaxInterval = 1000 * 60 * 20;//??-?>hung

    //requester??http??requester?http/socks?
    HttpRequesterImpl requester = (HttpRequesterImpl) crawler.getHttpRequester();
    AntiAntiSpiderHelper.defaultUserAgent(requester);

    //        requester.setUserAgent("Mozilla/5.0 (X11; Linux i686; rv:33.0) Gecko/20100101 Firefox/33.0");
    //        requester.setCookie("CNZZDATA1950488=cnzz_eid%3D739324831-1432460954-null%26ntime%3D1432460954; wdcid=44349d3f2aa96e51; vjuids=-53d395da8.14eca7eed44.0.f17be67e; CNZZDATA3473518=cnzz_eid%3D1882396923-1437965756-%26ntime%3D1440635510; pt_37a49e8b=uid=FuI4KYEfVz5xq7L4nzPd1w&nid=1&vid=r4AhSBmxisCiyeolr3V2Ow&vn=1&pvn=1&sact=1440639037916&to_flag=0&pl=t4NrgYqSK5M357L2nGEQCw*pt*1440639015734; _ga=GA1.3.1121158748.1437970841; __auc=c00a6ac114d85945f01d9c30128; CNZZDATA1975683=cnzz_eid%3D250014133-1432460541-null%26ntime%3D1440733997; CNZZDATA1254041250=2000695407-1442220871-%7C1442306691; pt_7f0a67e8=uid=6lmgYeZ3/jSObRMeK-t27A&nid=0&vid=lEKvEtZyZdd0UC264UyZnQ&vn=2&pvn=1&sact=1442306703728&to_flag=0&pl=7GB3sYS/PJDo1mY0qeu2cA*pt*1442306703728; 7NSx_98ef_saltkey=P05gN8zn; 7NSx_98ef_lastvisit=1444281282; IframeBodyHeight=256; NTVq_98ef_saltkey=j5PydYru; NTVq_98ef_lastvisit=1444282735; NTVq_98ef_atarget=1; NTVq_98ef_lastact=1444286377%09api.php%09js; 7NSx_98ef_sid=hZyDwc; __utmt=1; __utma=155578217.1121158748.1437970841.1443159326.1444285109.23; __utmb=155578217.57.10.1444285109; __utmc=155578217; __utmz=155578217.1439345650.3.2.utmcsr=travel.chinadaily.com.cn|utmccn=(referral)|utmcmd=referral|utmcct=/; CNZZDATA3089622=cnzz_eid%3D1722311508-1437912344-%26ntime%3D1444286009; wdlast=1444287704; vjlast=1437916393.1444285111.11; 7NSx_98ef_lastact=1444287477%09api.php%09chinadaily; pt_s_3bfec6ad=vt=1444287704638&cad=; pt_3bfec6ad=uid=bo87MAT/HC3hy12HDkBg1A&nid=0&vid=erwHQyFKxvwHXYc4-r6n-w&vn=28&pvn=2&sact=1444287708079&to_flag=0&pl=kkgvLoEHXsCD2gs4VJaWQg*pt*1444287704638; pt_t_3bfec6ad=?id=3bfec6ad.bo87MAT/HC3hy12HDkBg1A.erwHQyFKxvwHXYc4-r6n-w.kkgvLoEHXsCD2gs4VJaWQg.nZJ9Aj/bgfNDIKBXI5TwRQ&stat=167.132.1050.1076.1body%20div%3Aeq%288%29%20ul%3Aeq%280%29%20a%3Aeq%282%29.0.0.1595.3441.146.118&ptif=4");
    //?? Mozilla/5.0 (X11; Linux i686; rv:34.0) Gecko/20100101 Firefox/34.0
    //c requester.setProxy("
    /*
            
    //??
    RandomProxyGenerator proxyGenerator=new RandomProxyGenerator();
    proxyGenerator.addProxy("127.0.0.1",8080,Proxy.Type.SOCKS);
    requester.setProxyGenerator(proxyGenerator);
    */

    /*??*/
    //        crawler.setResumable(true);
    crawler.setResumable(false);

    crawler.start(2);
}

From source file:icevaluation.BingAPIAccess.java

public static void main(String[] args) {
    String searchText = "arts site:wikipedia.org";
    searchText = searchText.replaceAll(" ", "%20");
    // String accountKey="jTRIJt9d8DR2QT/Z3BJCAvY1BfoXj0zRYgSZ8deqHHo";
    String accountKey = "JfeJSA3x6CtsyVai0+KEP0A6CYEUBT8VWhZmm9CS738";

    byte[] accountKeyBytes = Base64.encodeBase64((accountKey + ":" + accountKey).getBytes());
    String accountKeyEnc = new String(accountKeyBytes);
    URL url;
    try {/*from   w ww .  j av  a  2 s  .c o m*/
        url = new URL("https://api.datamarket.azure.com/Bing/Search/v1/Composite?Sources=%27Web%27&Query=%27"
                + searchText + "%27&$format=JSON");
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestMethod("GET");
        conn.setRequestProperty("Authorization", "Basic " + accountKeyEnc);

        BufferedReader br = new BufferedReader(new InputStreamReader((conn.getInputStream())));
        StringBuilder sb = new StringBuilder();
        String output;
        System.out.println("Output from Server .... \n");
        //write json to string sb
        int c = 0;
        if ((output = br.readLine()) != null) {
            System.out.println("Output is: " + output);
            sb.append(output);
            c++;
            //System.out.println("C:"+c);

        }

        conn.disconnect();
        //find webtotal among output      
        int find = sb.indexOf("\"WebTotal\":\"");
        int startindex = find + 12;
        System.out.println("Find: " + find);

        int lastindex = sb.indexOf("\",\"WebOffset\"");

        System.out.println(sb.substring(startindex, lastindex));

    } catch (MalformedURLException e1) {
        e1.printStackTrace();
    } catch (IOException e) {

        e.printStackTrace();
    }

}

From source file:com.doculibre.constellio.lang.html.HtmlLangDetector.java

public static void main(String[] args) throws Exception {
    //java.net.URL url = new java.net.URL("http://www.gouv.qc.ca");
    java.net.URL url = new java.net.URL("http://andrew.triumf.ca/multilingual/samples/french.meta.html");
    java.net.URLConnection connection = url.openConnection();
    java.io.InputStream is = null;
    String content = null;//  ww  w . j  av a  2s  . c o  m
    try {
        is = connection.getInputStream();
        content = IOUtils.toString(is);
    } finally {
        IOUtils.closeQuietly(is);
    }

    HtmlLangDetector langDetector = new HtmlLangDetector();
    System.out.println(langDetector.getLang(content));
}