Example usage for org.apache.commons.httpclient HttpMethod getResponseBodyAsStream

List of usage examples for org.apache.commons.httpclient HttpMethod getResponseBodyAsStream

Introduction

In this page you can find the example usage for org.apache.commons.httpclient HttpMethod getResponseBodyAsStream.

Prototype

public abstract InputStream getResponseBodyAsStream() throws IOException;

Source Link

Usage

From source file:org.eclipsetrader.yahoo.internal.core.connector.BackfillConnector.java

private IOHLC[] backfill1DayHistory(IFeedIdentifier identifier) throws Exception {
    List<OHLC> list = new ArrayList<OHLC>();

    HttpMethod method = Util.get1DayHistoryFeedMethod(identifier);
    method.setFollowRedirects(true);/*from   ww  w.j  a va2  s.c o m*/

    HttpClient client = new HttpClient();
    Util.setupProxy(client, method.getURI().getHost());
    client.executeMethod(method);

    BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));
    read1DayBackfillStream(list, in);
    in.close();

    return list.toArray(new IOHLC[list.size()]);
}

From source file:org.eclipsetrader.yahoo.internal.core.connector.BackfillConnector.java

private IOHLC[] backfill5DayHistory(IFeedIdentifier identifier) throws Exception {
    List<OHLC> list = new ArrayList<OHLC>();

    HttpMethod method = Util.get5DayHistoryFeedMethod(identifier);
    method.setFollowRedirects(true);/*from  w w  w .j  a  v  a 2  s.  c om*/

    HttpClient client = new HttpClient();
    Util.setupProxy(client, method.getURI().getHost());
    client.executeMethod(method);

    BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));
    read1DayBackfillStream(list, in);
    in.close();

    return list.toArray(new IOHLC[list.size()]);
}

From source file:org.eclipsetrader.yahoo.internal.core.connector.BackfillConnector.java

@Override
public IDividend[] backfillDividends(IFeedIdentifier identifier, Date from, Date to) {
    List<IDividend> list = new ArrayList<IDividend>();

    try {/*w ww  .j a  v  a 2 s . c  o m*/
        HttpMethod method = Util.getDividendsHistoryMethod(identifier, from, to);
        method.setFollowRedirects(true);

        HttpClient client = new HttpClient();
        Util.setupProxy(client, method.getURI().getHost());
        client.executeMethod(method);

        BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));

        // The first line is the header, ignoring
        String inputLine = in.readLine();
        while ((inputLine = in.readLine()) != null) {
            if (inputLine.startsWith("<")) {
                continue;
            }

            try {
                Dividend dividend = parseDividendsResponseLine(inputLine);
                if (dividend != null) {
                    list.add(dividend);
                }
            } catch (ParseException e) {
                Status status = new Status(IStatus.ERROR, YahooActivator.PLUGIN_ID, 0,
                        "Error parsing data: " + inputLine, e);
                YahooActivator.log(status);
            }
        }

        in.close();

    } catch (Exception e) {
        Status status = new Status(IStatus.ERROR, YahooActivator.PLUGIN_ID, 0, "Error reading data", e);
        YahooActivator.log(status);
    }

    return list.toArray(new IDividend[list.size()]);
}

From source file:org.eclipsetrader.yahoo.internal.core.connector.SnapshotConnector.java

protected void fetchLatestSnapshot(HttpClient client, String[] symbols, boolean isStaleUpdate) {
    HttpMethod method = null;
    BufferedReader in = null;// www . jav  a  2s . c om
    String line = ""; //$NON-NLS-1$

    try {
        method = Util.getSnapshotFeedMethod(symbols);

        client.executeMethod(method);

        in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));
        while ((line = in.readLine()) != null) {
            processSnapshotData(line, isStaleUpdate);
        }

        FeedSubscription[] subscriptions;
        synchronized (symbolSubscriptions) {
            Collection<FeedSubscription> c = symbolSubscriptions.values();
            subscriptions = c.toArray(new FeedSubscription[c.size()]);
        }
        for (int i = 0; i < subscriptions.length; i++) {
            subscriptions[i].fireNotification();
        }

    } catch (Exception e) {
        Status status = new Status(IStatus.ERROR, YahooActivator.PLUGIN_ID, 0, "Error reading data", e);
        YahooActivator.log(status);
    } finally {
        try {
            if (in != null) {
                in.close();
            }
            if (method != null) {
                method.releaseConnection();
            }
        } catch (Exception e) {
            Status status = new Status(IStatus.WARNING, YahooActivator.PLUGIN_ID, 0,
                    "Connection wasn't closed cleanly", e);
            YahooActivator.log(status);
        }
    }
}

From source file:org.eclipsetrader.yahoo.internal.core.connector.StreamingConnector.java

@Override
public void run() {
    BufferedReader in = null;//ww w  . ja  v  a 2 s .  co  m
    char[] buffer = new char[512];

    try {
        HttpClient client = new HttpClient(new MultiThreadedHttpConnectionManager());
        client.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
        Util.setupProxy(client, Util.streamingFeedHost);

        HttpMethod method = null;

        while (!isStopping()) {
            // Check if the connection was not yet initialized or there are changed in the subscriptions.
            if (in == null || isSubscriptionsChanged()) {
                try {
                    if (method != null) {
                        method.releaseConnection();
                    }
                    if (in != null) {
                        in.close();
                    }
                } catch (Exception e) {
                    // We can't do anything at this time, ignore
                }

                String[] symbols;
                synchronized (symbolSubscriptions) {
                    Set<String> s = new HashSet<String>(symbolSubscriptions.keySet());
                    s.add("MSFT");
                    symbols = s.toArray(new String[s.size()]);
                    setSubscriptionsChanged(false);
                    if (symbols.length == 0) {
                        break;
                    }
                }
                method = Util.getStreamingFeedMethod(symbols);

                client.executeMethod(method);

                in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));

                line = new StringBuilder();
                script = new StringBuilder();
                inTag = false;
                inScript = false;

                fetchLatestSnapshot(client, symbols, false);
            }

            if (in.ready()) {
                int length = in.read(buffer);
                if (length == -1) {
                    in.close();
                    in = null;
                    continue;
                }
                processIncomingChars(buffer, length);
            } else {
                // Check stale data
                List<String> updateList = new ArrayList<String>();
                synchronized (symbolSubscriptions) {
                    long currentTime = System.currentTimeMillis();
                    for (FeedSubscription subscription : symbolSubscriptions.values()) {
                        long elapsedTime = currentTime - subscription.getIdentifierType().getLastUpdate();
                        if (elapsedTime >= 60000) {
                            updateList.add(subscription.getIdentifierType().getSymbol());
                            subscription.getIdentifierType().setLastUpdate(currentTime / 60000 * 60000);
                        }
                    }
                }
                if (updateList.size() != 0) {
                    fetchLatestSnapshot(client, updateList.toArray(new String[updateList.size()]), true);
                }
            }

            Thread.sleep(100);
        }
    } catch (Exception e) {
        Status status = new Status(IStatus.ERROR, YahooActivator.PLUGIN_ID, 0, "Error reading data", e);
        YahooActivator.log(status);
    } finally {
        try {
            if (in != null) {
                in.close();
            }
        } catch (Exception e) {
            // We can't do anything at this time, ignore
        }
    }
}

From source file:org.eclipsetrader.yahoojapan.internal.core.connector.BackfillConnector.java

private IOHLC[] backfillDailyHistory(IFeedIdentifier identifier, Date from, Date to) {
    List<OHLC> list = new ArrayList<OHLC>();

    Calendar c = Calendar.getInstance();
    c.setTime(from);/* w  w w.ja va 2  s.  c  om*/
    int firstYear = c.get(Calendar.YEAR);
    c.setTime(to);
    int lastYear = c.get(Calendar.YEAR);
    //limit 3 years
    if (firstYear < lastYear - 3) {
        firstYear = lastYear - 3;
    }

    HttpClient client = new HttpClient();
    for (int ys = firstYear; ys <= lastYear; ys++) {
        try {
            for (int page = 1; page < 21; page++) {
                HttpMethod method = Util.get1YearHistoryFeedMethod(identifier, ys, page);
                Util.setupProxy(client, method.getURI().getHost());
                client.executeMethod(method);

                BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));
                //                    readDailyBackfillStream(list, in);
                boolean hasNextPage = readBackfillStream(list, in);
                in.close();
                if (!hasNextPage) {
                    break;
                }
            }

        } catch (Exception e) {
            Status status = new Status(IStatus.ERROR, YahooJapanActivator.PLUGIN_ID, 0, "Error reading data",
                    e);
            YahooJapanActivator.log(status);
        }
    }

    Collections.sort(list, new Comparator<OHLC>() {

        @Override
        public int compare(OHLC o1, OHLC o2) {
            return o1.getDate().compareTo(o2.getDate());
        }
    });

    for (Iterator<OHLC> iter = list.iterator(); iter.hasNext();) {
        OHLC ohlc = iter.next();
        if (ohlc.getDate().before(from) || ohlc.getDate().after(to)) {
            iter.remove();
        }
    }

    return list.toArray(new IOHLC[list.size()]);
}

From source file:org.eclipsetrader.yahoojapan.internal.core.connector.BackfillConnector.java

@Override
public ISplit[] backfillSplits(IFeedIdentifier identifier, Date from, Date to) {
    List<ISplit> list = new ArrayList<ISplit>();

    System.out.println(from.toString());
    System.out.println(to.toString());
    Calendar c = Calendar.getInstance();
    c.setTime(to);//from w w w .j  a v  a 2  s  . com
    int lastYear = c.get(Calendar.YEAR);
    c.setTime(from);
    int firstYear = c.get(Calendar.YEAR);
    //limit 3 years
    if (firstYear < lastYear - 3) {
        firstYear = lastYear - 3;
        c.set(Calendar.YEAR, firstYear);
    }

    try {
        for (int page = 1; page < 21; page++) {
            HttpMethod method = Util.getDividendsHistoryMethod(identifier, to, c.getTime(), page);
            method.setFollowRedirects(true);

            HttpClient client = new HttpClient();
            Util.setupProxy(client, method.getURI().getHost());
            client.executeMethod(method);
            System.out.println(method.getURI().toString());

            BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));

            StringBuilder line = new StringBuilder();
            // The first line is the header, ignoring
            String inputLine = in.readLine();
            while ((inputLine = in.readLine()) != null) {
                //                   System.out.println("****" + inputLine);
                if (inputLine.indexOf("<!---->") >= 0) {
                    break;
                }
            }
            while ((inputLine = in.readLine()) != null) {
                //                   System.out.println("+++" + inputLine);
                if (inputLine.indexOf("<!--/-->") >= 0) {
                    String[] work = line.toString().split("</tr><tr><td>");
                    for (int i = 1; i < work.length; i++) {
                        try {
                            Split split = parseSplitsResponseLine(work[i]);
                            if (split != null) {
                                list.add(split);
                            }
                        } catch (ParseException e) {
                            Status status = new Status(IStatus.ERROR, YahooJapanActivator.PLUGIN_ID, 0,
                                    "Error parsing data: " + work[i], e);
                            YahooJapanActivator.log(status);
                        }
                    }
                    break;
                }

                if (inputLine.startsWith("</tr><tr><td>")) {
                    if (!inputLine.startsWith("</tr><tr><td>2")) {
                        line = new StringBuilder();
                    }
                    line.append(inputLine);
                } else if (inputLine.startsWith("</tr>")) {
                    line.append(inputLine);
                }
                if (inputLine.startsWith("<td>")) {
                    line.append(inputLine);
                }
            }
            boolean hasNextPage = false;
            if (inputLine.indexOf("<!---->") >= 0) {
                if (inputLine.indexOf("?") >= 0) {
                    hasNextPage = true;
                }
            } else {
                while ((inputLine = in.readLine()) != null) {
                    if (inputLine.indexOf("<!---->") >= 0) {
                        if (inputLine.indexOf("?") >= 0) {
                            hasNextPage = true;
                            break;
                        }
                    }
                }
            }

            in.close();
            if (!hasNextPage) {
                break;
            }
        }

    } catch (Exception e) {
        Status status = new Status(IStatus.ERROR, YahooJapanActivator.PLUGIN_ID, 0, "Error reading data", e);
        YahooJapanActivator.log(status);
    }

    return list.toArray(new ISplit[list.size()]);
}

From source file:org.eclipsetrader.yahoojapan.internal.core.connector.SnapshotConnector.java

protected void fetchLatestSnapshot(HttpClient client, String[] symbols, boolean isStaleUpdate) {
    HttpMethod method = null;
    BufferedReader in = null;//from  w  w  w .j a  v  a2 s . c  o m
    String line = ""; //$NON-NLS-1$

    try {
        for (int page = 1; page < 11; page++) {
            StringBuilder work = new StringBuilder();
            boolean isNextPage = false;

            method = Util.getSnapshotFeedMethod(symbols, page);
            method.setFollowRedirects(true);

            client.executeMethod(method);

            in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));
            while ((line = in.readLine()) != null) {
                if (line.indexOf("<!-- .pfListView -->") >= 0) {
                    break;
                }
            }
            while ((line = in.readLine()) != null) {
                if (line.indexOf("<!---->") >= 0) {
                    isNextPage = (line.indexOf("?</a>") >= 0);
                    break;
                }
                if (line.startsWith("<a href=\"http://stocks.finance.yahoo.co.jp/stocks/detail/?code=")) {
                    //CODE
                    //                      work.append(line.substring(63, 67));
                    //                      work.append(",");
                    work.append(line.substring(line.indexOf("<strong>") + 8, line.indexOf("</strong>")));
                    work.append(",");
                    //LAST, DATE
                    while ((line = in.readLine()) != null) {
                        if (line.equals("<td nowrap>")) {
                            break;
                        }
                    }
                    if ((line = in.readLine()) != null) {
                        if (line.indexOf("<td>") >= 0) {
                            work.append(line.substring(line.indexOf("<strong>") + 8, line.indexOf("</strong>"))
                                    .replaceAll(",", ""));
                            work.append(",");
                            String date = line.substring(0, line.indexOf("</td>"));
                            if (date.indexOf("/") >= 0) {
                                work.append(date);
                                work.append(",00:00,,");
                            } else {
                                Calendar c = Calendar.getInstance();
                                work.append(String.valueOf(c.get(Calendar.MONTH) + 1) + "/"
                                        + String.valueOf(c.get(Calendar.DAY_OF_MONTH)));
                                work.append(",");
                                work.append(date);
                                work.append(",,");
                            }
                        }
                    }
                    //OPEN
                    while ((line = in.readLine()) != null) {
                        if (line.equals("<td nowrap>")) {
                            break;
                        }
                    }
                    if ((line = in.readLine()) != null) {
                        if (line.indexOf("</td>") >= 0) {
                            work.append(line.substring(0, line.indexOf("</td>")).replaceAll(",", ""));
                        }
                    }
                    work.append(",");
                    //HIGH
                    while ((line = in.readLine()) != null) {
                        if (line.equals("<td nowrap>")) {
                            break;
                        }
                    }
                    if ((line = in.readLine()) != null) {
                        if (line.indexOf("</td>") >= 0) {
                            work.append(line.substring(0, line.indexOf("</td>")).replaceAll(",", ""));
                        }
                    }
                    work.append(",");
                    //LOW
                    while ((line = in.readLine()) != null) {
                        if (line.equals("<td nowrap>")) {
                            break;
                        }
                    }
                    if ((line = in.readLine()) != null) {
                        if (line.indexOf("</td>") >= 0) {
                            work.append(line.substring(0, line.indexOf("</td>")).replaceAll(",", ""));
                        }
                    }
                    work.append(",,,,,,,@");
                    processSnapshotData(work.toString(), isStaleUpdate);
                    work = new StringBuilder();
                }
            }

            FeedSubscription[] subscriptions;
            synchronized (symbolSubscriptions) {
                Collection<FeedSubscription> c = symbolSubscriptions.values();
                subscriptions = c.toArray(new FeedSubscription[c.size()]);
            }
            for (int i = 0; i < subscriptions.length; i++) {
                subscriptions[i].fireNotification();
            }

            if (!isNextPage) {
                break;
            }
        }

    } catch (Exception e) {
        Status status = new Status(IStatus.ERROR, YahooJapanActivator.PLUGIN_ID, 0, "Error reading data", e);
        YahooJapanActivator.log(status);
    } finally {
        try {
            if (in != null) {
                in.close();
            }
            if (method != null) {
                method.releaseConnection();
            }
        } catch (Exception e) {
            Status status = new Status(IStatus.WARNING, YahooJapanActivator.PLUGIN_ID, 0,
                    "Connection wasn't closed cleanly", e);
            YahooJapanActivator.log(status);
        }
    }
}

From source file:org.eclipsetrader.yahoojapan.internal.core.connector.SnapshotConnector.java

protected void fetchLatestSnapshot1(HttpClient client, String[] symbols, boolean isStaleUpdate) {
    HttpMethod method = null;
    BufferedReader in = null;/* w  ww.j  a va  2 s. c o m*/
    String line = ""; //$NON-NLS-1$

    try {
        StringBuilder work = new StringBuilder();

        method = Util.getSnapshotFeedMethod(symbols, 1);
        method.setFollowRedirects(true);

        client.executeMethod(method);

        in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));
        while ((line = in.readLine()) != null) {
            if (line.indexOf("<!-- NEW STOCKS DETAIL -->") >= 0) {
                break;
            }
        }
        //CODE
        while ((line = in.readLine()) != null) {
            if (line.indexOf("<dl class=\"stocksInfo\">") >= 0) {
                break;
            }
        }
        if ((line = in.readLine()) != null) {
            if (line.length() > 8) {
                work.append(line.substring(4, 8));
            }
        }
        work.append(",");
        //DATE
        String date = "";
        while ((line = in.readLine()) != null) {
            if (line.indexOf("<dd class=\"yjSb real\">") >= 0) {
                break;
            }
        }
        if (line.length() > 33) {
            date = line.substring(28, line.indexOf("</span>"));
        }
        //LAST
        while ((line = in.readLine()) != null) {
            if (line.indexOf("<td class=\"stoksPrice\">") >= 0) {
                break;
            }
        }
        if (line.length() > 23) {
            work.append(line.substring(line.indexOf("<td class=\"stoksPrice\">") + 23, line.indexOf("</td>"))
                    .replaceAll(",", ""));
        }
        work.append(",");
        if (date.indexOf("/") >= 0) {
            work.append(date);
            work.append(",00:00,,");
        } else {
            Calendar c = Calendar.getInstance();
            work.append(String.valueOf(c.get(Calendar.MONTH) + 1) + "/"
                    + String.valueOf(c.get(Calendar.DAY_OF_MONTH)));
            work.append(",");
            work.append(date);
            work.append(",,");
        }
        //YESTERDAY LAST
        while ((line = in.readLine()) != null) {
            if (line.indexOf("<div class=\"innerDate\">") >= 0) {
                break;
            }
        }
        while ((line = in.readLine()) != null) {
            if (line.indexOf("<strong>") >= 0) {
                break;
            }
        }
        //OPEN
        while ((line = in.readLine()) != null) {
            if (line.indexOf("<strong>") >= 0) {
                break;
            }
        }
        if (line.length() > 16) {
            work.append(line.substring(line.indexOf("<strong>") + 8, line.indexOf("</strong>")).replaceAll(",",
                    ""));
        }
        work.append(",");
        //HIGH
        while ((line = in.readLine()) != null) {
            if (line.indexOf("<strong>") >= 0) {
                break;
            }
        }
        if (line.length() > 16) {
            work.append(line.substring(line.indexOf("<strong>") + 8, line.indexOf("</strong>")).replaceAll(",",
                    ""));
        }
        work.append(",");
        //LOW
        while ((line = in.readLine()) != null) {
            if (line.indexOf("<strong>") >= 0) {
                break;
            }
        }
        if (line.length() > 16) {
            work.append(line.substring(line.indexOf("<strong>") + 8, line.indexOf("</strong>")).replaceAll(",",
                    ""));
        }
        work.append(",");
        //VOLUME
        while ((line = in.readLine()) != null) {
            if (line.indexOf("<strong>") >= 0) {
                break;
            }
        }
        if (line.length() > 16) {
            work.append(line.substring(line.indexOf("<strong>") + 8, line.indexOf("</strong>")).replaceAll(",",
                    ""));
        }
        work.append(",,,,,,@");

        processSnapshotData(work.toString(), isStaleUpdate);

        FeedSubscription[] subscriptions;
        synchronized (symbolSubscriptions) {
            Collection<FeedSubscription> c = symbolSubscriptions.values();
            subscriptions = c.toArray(new FeedSubscription[c.size()]);
        }
        for (int i = 0; i < subscriptions.length; i++) {
            subscriptions[i].fireNotification();
        }

    } catch (Exception e) {
        Status status = new Status(IStatus.ERROR, YahooJapanActivator.PLUGIN_ID, 0, "Error reading data", e);
        YahooJapanActivator.log(status);
    } finally {
        try {
            if (in != null) {
                in.close();
            }
            if (method != null) {
                method.releaseConnection();
            }
        } catch (Exception e) {
            Status status = new Status(IStatus.WARNING, YahooJapanActivator.PLUGIN_ID, 0,
                    "Connection wasn't closed cleanly", e);
            YahooJapanActivator.log(status);
        }
    }
}

From source file:org.eclipsetrader.yahoojapanfx.internal.core.connector.BackfillConnector.java

@Override
public IOHLC[] backfillHistory(IFeedIdentifier identifier, Date from, Date to, TimeSpan timeSpan) {
    List<OHLC> list = new ArrayList<OHLC>();

    String period = "";
    if (timeSpan.getUnits() == TimeSpan.Units.Days) {
        period = "1d";
    } else if (timeSpan.getUnits() == TimeSpan.Units.Minutes) {
        period = String.valueOf(timeSpan.getLength()) + "m";
    }/*from w w w .ja v a2  s  . co m*/

    HttpClient client = new HttpClient();
    try {
        HttpMethod method = Util.getPrepareHistoryFeedMethod(identifier);
        Util.setupProxy(client, method.getURI().getHost());

        client.executeMethod(method);

        BufferedReader in;
        if ((method.getResponseHeader("Content-Encoding") != null)
                && (method.getResponseHeader("Content-Encoding").getValue().equals("gzip"))) {
            in = new BufferedReader(
                    new InputStreamReader(new GZIPInputStream(method.getResponseBodyAsStream())));
        } else {
            in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));
        }
        String inputLine;
        String c = "";
        while ((inputLine = in.readLine()) != null) {
            if (inputLine.indexOf("YJFIN.c =") >= 0) {
                c = inputLine.substring(inputLine.indexOf("YJFIN.c =") + 11, inputLine.indexOf("\";"));
            }
        }
        in.close();

        method = Util.getHistoryFeedMethod(identifier, period, c);

        client.executeMethod(method);

        if ((method.getResponseHeader("Content-Encoding") != null)
                && (method.getResponseHeader("Content-Encoding").getValue().equals("gzip"))) {
            in = new BufferedReader(
                    new InputStreamReader(new GZIPInputStream(method.getResponseBodyAsStream())));
        } else {
            in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));
        }
        if (timeSpan.getUnits() == TimeSpan.Units.Days) {
            readHistoryStream(in, list);
        } else {
            readIntradayStream(in, list);
        }
        in.close();
    } catch (Exception e) {
        Status status = new Status(IStatus.ERROR, Activator.PLUGIN_ID, 0, "Error reading data", e); //$NON-NLS-1$
        Activator.log(status);
    }

    Collections.sort(list, new Comparator<OHLC>() {

        @Override
        public int compare(OHLC o1, OHLC o2) {
            return o1.getDate().compareTo(o2.getDate());
        }
    });

    for (Iterator<OHLC> iter = list.iterator(); iter.hasNext();) {
        OHLC ohlc = iter.next();
        if (ohlc.getDate().before(from) || ohlc.getDate().after(to)) {
            iter.remove();
        }
    }

    return list.toArray(new IOHLC[list.size()]);
}