List of usage examples for org.apache.commons.httpclient HttpMethod getResponseBodyAsStream
public abstract InputStream getResponseBodyAsStream() throws IOException;
From source file:org.eclipsetrader.yahoo.internal.core.connector.BackfillConnector.java
private IOHLC[] backfill1DayHistory(IFeedIdentifier identifier) throws Exception { List<OHLC> list = new ArrayList<OHLC>(); HttpMethod method = Util.get1DayHistoryFeedMethod(identifier); method.setFollowRedirects(true);/*from ww w.j a va2 s.c o m*/ HttpClient client = new HttpClient(); Util.setupProxy(client, method.getURI().getHost()); client.executeMethod(method); BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); read1DayBackfillStream(list, in); in.close(); return list.toArray(new IOHLC[list.size()]); }
From source file:org.eclipsetrader.yahoo.internal.core.connector.BackfillConnector.java
private IOHLC[] backfill5DayHistory(IFeedIdentifier identifier) throws Exception { List<OHLC> list = new ArrayList<OHLC>(); HttpMethod method = Util.get5DayHistoryFeedMethod(identifier); method.setFollowRedirects(true);/*from w w w .j a v a 2 s. c om*/ HttpClient client = new HttpClient(); Util.setupProxy(client, method.getURI().getHost()); client.executeMethod(method); BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); read1DayBackfillStream(list, in); in.close(); return list.toArray(new IOHLC[list.size()]); }
From source file:org.eclipsetrader.yahoo.internal.core.connector.BackfillConnector.java
@Override public IDividend[] backfillDividends(IFeedIdentifier identifier, Date from, Date to) { List<IDividend> list = new ArrayList<IDividend>(); try {/*w ww .j a v a 2 s . c o m*/ HttpMethod method = Util.getDividendsHistoryMethod(identifier, from, to); method.setFollowRedirects(true); HttpClient client = new HttpClient(); Util.setupProxy(client, method.getURI().getHost()); client.executeMethod(method); BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); // The first line is the header, ignoring String inputLine = in.readLine(); while ((inputLine = in.readLine()) != null) { if (inputLine.startsWith("<")) { continue; } try { Dividend dividend = parseDividendsResponseLine(inputLine); if (dividend != null) { list.add(dividend); } } catch (ParseException e) { Status status = new Status(IStatus.ERROR, YahooActivator.PLUGIN_ID, 0, "Error parsing data: " + inputLine, e); YahooActivator.log(status); } } in.close(); } catch (Exception e) { Status status = new Status(IStatus.ERROR, YahooActivator.PLUGIN_ID, 0, "Error reading data", e); YahooActivator.log(status); } return list.toArray(new IDividend[list.size()]); }
From source file:org.eclipsetrader.yahoo.internal.core.connector.SnapshotConnector.java
protected void fetchLatestSnapshot(HttpClient client, String[] symbols, boolean isStaleUpdate) { HttpMethod method = null; BufferedReader in = null;// www . jav a 2s . c om String line = ""; //$NON-NLS-1$ try { method = Util.getSnapshotFeedMethod(symbols); client.executeMethod(method); in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); while ((line = in.readLine()) != null) { processSnapshotData(line, isStaleUpdate); } FeedSubscription[] subscriptions; synchronized (symbolSubscriptions) { Collection<FeedSubscription> c = symbolSubscriptions.values(); subscriptions = c.toArray(new FeedSubscription[c.size()]); } for (int i = 0; i < subscriptions.length; i++) { subscriptions[i].fireNotification(); } } catch (Exception e) { Status status = new Status(IStatus.ERROR, YahooActivator.PLUGIN_ID, 0, "Error reading data", e); YahooActivator.log(status); } finally { try { if (in != null) { in.close(); } if (method != null) { method.releaseConnection(); } } catch (Exception e) { Status status = new Status(IStatus.WARNING, YahooActivator.PLUGIN_ID, 0, "Connection wasn't closed cleanly", e); YahooActivator.log(status); } } }
From source file:org.eclipsetrader.yahoo.internal.core.connector.StreamingConnector.java
@Override public void run() { BufferedReader in = null;//ww w . ja v a 2 s . co m char[] buffer = new char[512]; try { HttpClient client = new HttpClient(new MultiThreadedHttpConnectionManager()); client.getHttpConnectionManager().getParams().setConnectionTimeout(5000); Util.setupProxy(client, Util.streamingFeedHost); HttpMethod method = null; while (!isStopping()) { // Check if the connection was not yet initialized or there are changed in the subscriptions. if (in == null || isSubscriptionsChanged()) { try { if (method != null) { method.releaseConnection(); } if (in != null) { in.close(); } } catch (Exception e) { // We can't do anything at this time, ignore } String[] symbols; synchronized (symbolSubscriptions) { Set<String> s = new HashSet<String>(symbolSubscriptions.keySet()); s.add("MSFT"); symbols = s.toArray(new String[s.size()]); setSubscriptionsChanged(false); if (symbols.length == 0) { break; } } method = Util.getStreamingFeedMethod(symbols); client.executeMethod(method); in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); line = new StringBuilder(); script = new StringBuilder(); inTag = false; inScript = false; fetchLatestSnapshot(client, symbols, false); } if (in.ready()) { int length = in.read(buffer); if (length == -1) { in.close(); in = null; continue; } processIncomingChars(buffer, length); } else { // Check stale data List<String> updateList = new ArrayList<String>(); synchronized (symbolSubscriptions) { long currentTime = System.currentTimeMillis(); for (FeedSubscription subscription : symbolSubscriptions.values()) { long elapsedTime = currentTime - subscription.getIdentifierType().getLastUpdate(); if (elapsedTime >= 60000) { updateList.add(subscription.getIdentifierType().getSymbol()); subscription.getIdentifierType().setLastUpdate(currentTime / 60000 * 60000); } } } if (updateList.size() != 0) { fetchLatestSnapshot(client, updateList.toArray(new String[updateList.size()]), true); } } Thread.sleep(100); } } catch (Exception e) { Status status = new Status(IStatus.ERROR, YahooActivator.PLUGIN_ID, 0, "Error reading data", e); YahooActivator.log(status); } finally { try { if (in != null) { in.close(); } } catch (Exception e) { // We can't do anything at this time, ignore } } }
From source file:org.eclipsetrader.yahoojapan.internal.core.connector.BackfillConnector.java
private IOHLC[] backfillDailyHistory(IFeedIdentifier identifier, Date from, Date to) { List<OHLC> list = new ArrayList<OHLC>(); Calendar c = Calendar.getInstance(); c.setTime(from);/* w w w.ja va 2 s. c om*/ int firstYear = c.get(Calendar.YEAR); c.setTime(to); int lastYear = c.get(Calendar.YEAR); //limit 3 years if (firstYear < lastYear - 3) { firstYear = lastYear - 3; } HttpClient client = new HttpClient(); for (int ys = firstYear; ys <= lastYear; ys++) { try { for (int page = 1; page < 21; page++) { HttpMethod method = Util.get1YearHistoryFeedMethod(identifier, ys, page); Util.setupProxy(client, method.getURI().getHost()); client.executeMethod(method); BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); // readDailyBackfillStream(list, in); boolean hasNextPage = readBackfillStream(list, in); in.close(); if (!hasNextPage) { break; } } } catch (Exception e) { Status status = new Status(IStatus.ERROR, YahooJapanActivator.PLUGIN_ID, 0, "Error reading data", e); YahooJapanActivator.log(status); } } Collections.sort(list, new Comparator<OHLC>() { @Override public int compare(OHLC o1, OHLC o2) { return o1.getDate().compareTo(o2.getDate()); } }); for (Iterator<OHLC> iter = list.iterator(); iter.hasNext();) { OHLC ohlc = iter.next(); if (ohlc.getDate().before(from) || ohlc.getDate().after(to)) { iter.remove(); } } return list.toArray(new IOHLC[list.size()]); }
From source file:org.eclipsetrader.yahoojapan.internal.core.connector.BackfillConnector.java
@Override public ISplit[] backfillSplits(IFeedIdentifier identifier, Date from, Date to) { List<ISplit> list = new ArrayList<ISplit>(); System.out.println(from.toString()); System.out.println(to.toString()); Calendar c = Calendar.getInstance(); c.setTime(to);//from w w w .j a v a 2 s . com int lastYear = c.get(Calendar.YEAR); c.setTime(from); int firstYear = c.get(Calendar.YEAR); //limit 3 years if (firstYear < lastYear - 3) { firstYear = lastYear - 3; c.set(Calendar.YEAR, firstYear); } try { for (int page = 1; page < 21; page++) { HttpMethod method = Util.getDividendsHistoryMethod(identifier, to, c.getTime(), page); method.setFollowRedirects(true); HttpClient client = new HttpClient(); Util.setupProxy(client, method.getURI().getHost()); client.executeMethod(method); System.out.println(method.getURI().toString()); BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); StringBuilder line = new StringBuilder(); // The first line is the header, ignoring String inputLine = in.readLine(); while ((inputLine = in.readLine()) != null) { // System.out.println("****" + inputLine); if (inputLine.indexOf("<!---->") >= 0) { break; } } while ((inputLine = in.readLine()) != null) { // System.out.println("+++" + inputLine); if (inputLine.indexOf("<!--/-->") >= 0) { String[] work = line.toString().split("</tr><tr><td>"); for (int i = 1; i < work.length; i++) { try { Split split = parseSplitsResponseLine(work[i]); if (split != null) { list.add(split); } } catch (ParseException e) { Status status = new Status(IStatus.ERROR, YahooJapanActivator.PLUGIN_ID, 0, "Error parsing data: " + work[i], e); YahooJapanActivator.log(status); } } break; } if (inputLine.startsWith("</tr><tr><td>")) { if (!inputLine.startsWith("</tr><tr><td>2")) { line = new StringBuilder(); } line.append(inputLine); } else if (inputLine.startsWith("</tr>")) { line.append(inputLine); } if (inputLine.startsWith("<td>")) { line.append(inputLine); } } boolean hasNextPage = false; if (inputLine.indexOf("<!---->") >= 0) { if (inputLine.indexOf("?") >= 0) { hasNextPage = true; } } else { while ((inputLine = in.readLine()) != null) { if (inputLine.indexOf("<!---->") >= 0) { if (inputLine.indexOf("?") >= 0) { hasNextPage = true; break; } } } } in.close(); if (!hasNextPage) { break; } } } catch (Exception e) { Status status = new Status(IStatus.ERROR, YahooJapanActivator.PLUGIN_ID, 0, "Error reading data", e); YahooJapanActivator.log(status); } return list.toArray(new ISplit[list.size()]); }
From source file:org.eclipsetrader.yahoojapan.internal.core.connector.SnapshotConnector.java
protected void fetchLatestSnapshot(HttpClient client, String[] symbols, boolean isStaleUpdate) { HttpMethod method = null; BufferedReader in = null;//from w w w .j a v a2 s . c o m String line = ""; //$NON-NLS-1$ try { for (int page = 1; page < 11; page++) { StringBuilder work = new StringBuilder(); boolean isNextPage = false; method = Util.getSnapshotFeedMethod(symbols, page); method.setFollowRedirects(true); client.executeMethod(method); in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); while ((line = in.readLine()) != null) { if (line.indexOf("<!-- .pfListView -->") >= 0) { break; } } while ((line = in.readLine()) != null) { if (line.indexOf("<!---->") >= 0) { isNextPage = (line.indexOf("?</a>") >= 0); break; } if (line.startsWith("<a href=\"http://stocks.finance.yahoo.co.jp/stocks/detail/?code=")) { //CODE // work.append(line.substring(63, 67)); // work.append(","); work.append(line.substring(line.indexOf("<strong>") + 8, line.indexOf("</strong>"))); work.append(","); //LAST, DATE while ((line = in.readLine()) != null) { if (line.equals("<td nowrap>")) { break; } } if ((line = in.readLine()) != null) { if (line.indexOf("<td>") >= 0) { work.append(line.substring(line.indexOf("<strong>") + 8, line.indexOf("</strong>")) .replaceAll(",", "")); work.append(","); String date = line.substring(0, line.indexOf("</td>")); if (date.indexOf("/") >= 0) { work.append(date); work.append(",00:00,,"); } else { Calendar c = Calendar.getInstance(); work.append(String.valueOf(c.get(Calendar.MONTH) + 1) + "/" + String.valueOf(c.get(Calendar.DAY_OF_MONTH))); work.append(","); work.append(date); work.append(",,"); } } } //OPEN while ((line = in.readLine()) != null) { if (line.equals("<td nowrap>")) { break; } } if ((line = in.readLine()) != null) { if (line.indexOf("</td>") >= 0) { work.append(line.substring(0, line.indexOf("</td>")).replaceAll(",", "")); } } work.append(","); //HIGH while ((line = in.readLine()) != null) { if (line.equals("<td nowrap>")) { break; } } if ((line = in.readLine()) != null) { if (line.indexOf("</td>") >= 0) { work.append(line.substring(0, line.indexOf("</td>")).replaceAll(",", "")); } } work.append(","); //LOW while ((line = in.readLine()) != null) { if (line.equals("<td nowrap>")) { break; } } if ((line = in.readLine()) != null) { if (line.indexOf("</td>") >= 0) { work.append(line.substring(0, line.indexOf("</td>")).replaceAll(",", "")); } } work.append(",,,,,,,@"); processSnapshotData(work.toString(), isStaleUpdate); work = new StringBuilder(); } } FeedSubscription[] subscriptions; synchronized (symbolSubscriptions) { Collection<FeedSubscription> c = symbolSubscriptions.values(); subscriptions = c.toArray(new FeedSubscription[c.size()]); } for (int i = 0; i < subscriptions.length; i++) { subscriptions[i].fireNotification(); } if (!isNextPage) { break; } } } catch (Exception e) { Status status = new Status(IStatus.ERROR, YahooJapanActivator.PLUGIN_ID, 0, "Error reading data", e); YahooJapanActivator.log(status); } finally { try { if (in != null) { in.close(); } if (method != null) { method.releaseConnection(); } } catch (Exception e) { Status status = new Status(IStatus.WARNING, YahooJapanActivator.PLUGIN_ID, 0, "Connection wasn't closed cleanly", e); YahooJapanActivator.log(status); } } }
From source file:org.eclipsetrader.yahoojapan.internal.core.connector.SnapshotConnector.java
protected void fetchLatestSnapshot1(HttpClient client, String[] symbols, boolean isStaleUpdate) { HttpMethod method = null; BufferedReader in = null;/* w ww.j a va 2 s. c o m*/ String line = ""; //$NON-NLS-1$ try { StringBuilder work = new StringBuilder(); method = Util.getSnapshotFeedMethod(symbols, 1); method.setFollowRedirects(true); client.executeMethod(method); in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); while ((line = in.readLine()) != null) { if (line.indexOf("<!-- NEW STOCKS DETAIL -->") >= 0) { break; } } //CODE while ((line = in.readLine()) != null) { if (line.indexOf("<dl class=\"stocksInfo\">") >= 0) { break; } } if ((line = in.readLine()) != null) { if (line.length() > 8) { work.append(line.substring(4, 8)); } } work.append(","); //DATE String date = ""; while ((line = in.readLine()) != null) { if (line.indexOf("<dd class=\"yjSb real\">") >= 0) { break; } } if (line.length() > 33) { date = line.substring(28, line.indexOf("</span>")); } //LAST while ((line = in.readLine()) != null) { if (line.indexOf("<td class=\"stoksPrice\">") >= 0) { break; } } if (line.length() > 23) { work.append(line.substring(line.indexOf("<td class=\"stoksPrice\">") + 23, line.indexOf("</td>")) .replaceAll(",", "")); } work.append(","); if (date.indexOf("/") >= 0) { work.append(date); work.append(",00:00,,"); } else { Calendar c = Calendar.getInstance(); work.append(String.valueOf(c.get(Calendar.MONTH) + 1) + "/" + String.valueOf(c.get(Calendar.DAY_OF_MONTH))); work.append(","); work.append(date); work.append(",,"); } //YESTERDAY LAST while ((line = in.readLine()) != null) { if (line.indexOf("<div class=\"innerDate\">") >= 0) { break; } } while ((line = in.readLine()) != null) { if (line.indexOf("<strong>") >= 0) { break; } } //OPEN while ((line = in.readLine()) != null) { if (line.indexOf("<strong>") >= 0) { break; } } if (line.length() > 16) { work.append(line.substring(line.indexOf("<strong>") + 8, line.indexOf("</strong>")).replaceAll(",", "")); } work.append(","); //HIGH while ((line = in.readLine()) != null) { if (line.indexOf("<strong>") >= 0) { break; } } if (line.length() > 16) { work.append(line.substring(line.indexOf("<strong>") + 8, line.indexOf("</strong>")).replaceAll(",", "")); } work.append(","); //LOW while ((line = in.readLine()) != null) { if (line.indexOf("<strong>") >= 0) { break; } } if (line.length() > 16) { work.append(line.substring(line.indexOf("<strong>") + 8, line.indexOf("</strong>")).replaceAll(",", "")); } work.append(","); //VOLUME while ((line = in.readLine()) != null) { if (line.indexOf("<strong>") >= 0) { break; } } if (line.length() > 16) { work.append(line.substring(line.indexOf("<strong>") + 8, line.indexOf("</strong>")).replaceAll(",", "")); } work.append(",,,,,,@"); processSnapshotData(work.toString(), isStaleUpdate); FeedSubscription[] subscriptions; synchronized (symbolSubscriptions) { Collection<FeedSubscription> c = symbolSubscriptions.values(); subscriptions = c.toArray(new FeedSubscription[c.size()]); } for (int i = 0; i < subscriptions.length; i++) { subscriptions[i].fireNotification(); } } catch (Exception e) { Status status = new Status(IStatus.ERROR, YahooJapanActivator.PLUGIN_ID, 0, "Error reading data", e); YahooJapanActivator.log(status); } finally { try { if (in != null) { in.close(); } if (method != null) { method.releaseConnection(); } } catch (Exception e) { Status status = new Status(IStatus.WARNING, YahooJapanActivator.PLUGIN_ID, 0, "Connection wasn't closed cleanly", e); YahooJapanActivator.log(status); } } }
From source file:org.eclipsetrader.yahoojapanfx.internal.core.connector.BackfillConnector.java
@Override public IOHLC[] backfillHistory(IFeedIdentifier identifier, Date from, Date to, TimeSpan timeSpan) { List<OHLC> list = new ArrayList<OHLC>(); String period = ""; if (timeSpan.getUnits() == TimeSpan.Units.Days) { period = "1d"; } else if (timeSpan.getUnits() == TimeSpan.Units.Minutes) { period = String.valueOf(timeSpan.getLength()) + "m"; }/*from w w w .ja v a2 s . co m*/ HttpClient client = new HttpClient(); try { HttpMethod method = Util.getPrepareHistoryFeedMethod(identifier); Util.setupProxy(client, method.getURI().getHost()); client.executeMethod(method); BufferedReader in; if ((method.getResponseHeader("Content-Encoding") != null) && (method.getResponseHeader("Content-Encoding").getValue().equals("gzip"))) { in = new BufferedReader( new InputStreamReader(new GZIPInputStream(method.getResponseBodyAsStream()))); } else { in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); } String inputLine; String c = ""; while ((inputLine = in.readLine()) != null) { if (inputLine.indexOf("YJFIN.c =") >= 0) { c = inputLine.substring(inputLine.indexOf("YJFIN.c =") + 11, inputLine.indexOf("\";")); } } in.close(); method = Util.getHistoryFeedMethod(identifier, period, c); client.executeMethod(method); if ((method.getResponseHeader("Content-Encoding") != null) && (method.getResponseHeader("Content-Encoding").getValue().equals("gzip"))) { in = new BufferedReader( new InputStreamReader(new GZIPInputStream(method.getResponseBodyAsStream()))); } else { in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); } if (timeSpan.getUnits() == TimeSpan.Units.Days) { readHistoryStream(in, list); } else { readIntradayStream(in, list); } in.close(); } catch (Exception e) { Status status = new Status(IStatus.ERROR, Activator.PLUGIN_ID, 0, "Error reading data", e); //$NON-NLS-1$ Activator.log(status); } Collections.sort(list, new Comparator<OHLC>() { @Override public int compare(OHLC o1, OHLC o2) { return o1.getDate().compareTo(o2.getDate()); } }); for (Iterator<OHLC> iter = list.iterator(); iter.hasNext();) { OHLC ohlc = iter.next(); if (ohlc.getDate().before(from) || ohlc.getDate().after(to)) { iter.remove(); } } return list.toArray(new IOHLC[list.size()]); }