List of usage examples for twitter4j TwitterStream shutdown
TwitterStream shutdown();
From source file:StreamToFiles.java
License:Apache License
public static void main(String[] args) throws TwitterException, FileNotFoundException { String oAuthFileName = "mjlauKeys.auth"; outFile = args[0];/*from w ww . j a va 2 s .c o m*/ ConfigurationBuilder cb = Utils.createConfigurationBuilder(new File(oAuthFileName)); if (cb == null) { throw new IllegalArgumentException("Configuration File Issues"); } TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance(); StatusListener listener = new StatusListener() { long counter = 0; PrintWriter outputWriter = new PrintWriter(new File(outFile)); @Override public void onStatus(Status status) { // System.out.println("@" + status.getUser().getScreenName() // + " - " + status.getText()); if (status.getText().contains("#")) { Utils.sanitizeAndWriteTweet(status.getText(), outputWriter); counter++; if (counter % 1000 == 0) { System.out.println("done with " + counter); } if (counter >= NUM_TWEETS) { twitterStream.cleanUp(); twitterStream.shutdown(); outputWriter.close(); } } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { // System.out.println("Got a status deletion notice id:" // + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // System.out.println("Got track limitation notice:" // + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { // System.out.println("Got scrub_geo event userId:" + userId // + " upToStatusId:" + upToStatusId); } @Override public void onStallWarning(StallWarning warning) { // System.out.println("Got stall warning:" + warning); } @Override public void onException(Exception ex) { ex.printStackTrace(); } }; twitterStream.addListener(listener); twitterStream.sample("en"); }
From source file:at.aictopic1.twitter.AICStream.java
public void stopStream() { TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.shutdown(); }
From source file:be.ugent.tiwi.sleroux.newsrec.twittertest.StreamReaderService.java
public void readTwitterFeed() { TwitterStream stream = TwitterStreamBuilderUtil.getStream(); StatusListener listener = new StatusListener() { @Override/*from w ww . j a va 2 s . c o m*/ public void onException(Exception e) { } @Override public void onTrackLimitationNotice(int n) { } @Override public void onStatus(Status status) { if (status.getLang().equals("en")) { System.out.println(status.getText()); } } @Override public void onStallWarning(StallWarning arg0) { } @Override public void onScrubGeo(long arg0, long arg1) { } @Override public void onDeletionNotice(StatusDeletionNotice arg0) { } }; stream.addListener(listener); FilterQuery f = new FilterQuery(); f.language(new String[] { "en" }); f.follow(new long[] { 816653 }); stream.filter(f); try { Thread.sleep(60000); } catch (InterruptedException ex) { Logger.getLogger(StreamReaderService.class.getName()).log(Level.SEVERE, null, ex); } stream.shutdown(); }
From source file:birdseye.Sample.java
License:Apache License
public List<TweetData> execute(String[] args) throws TwitterException { final List<TweetData> statuses = new ArrayList(); ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setOAuthAccessToken("14538839-3MX2UoCEUaA6u95iWoYweTKRbhBjqEVuK1SPbCjDV"); cb.setOAuthAccessTokenSecret("nox7eYyOJpyiDiISHRDou90bGkHKasuw1IMqqJUZMaAbj"); TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance(); StatusListener listener = new StatusListener() { public void onStatus(Status status) { String user = status.getUser().getScreenName(); String content = status.getText(); TweetData newTweet = new TweetData(user, content); statuses.add(newTweet);//from www. j av a2 s.c o m System.out.println(statuses.size() + ":" + status.getText()); if (statuses.size() > 15) { synchronized (lock) { lock.notify(); } System.out.println("unlocked"); } } public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } public void onTrackLimitationNotice(int numberOfLimitedStatuses) { System.out.println("Got track limitation notice:" + numberOfLimitedStatuses); } public void onScrubGeo(long userId, long upToStatusId) { System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } public void onException(Exception ex) { ex.printStackTrace(); } @Override public void onStallWarning(StallWarning sw) { System.out.println(sw.getMessage()); } }; FilterQuery fq = new FilterQuery(); String[] keywords = args; fq.track(keywords); twitterStream.addListener(listener); twitterStream.filter(fq); try { synchronized (lock) { lock.wait(); } } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.out.println("returning statuses"); twitterStream.shutdown(); return statuses; }
From source file:com.accumulobook.designs.graph.TwitterGraph.java
License:Apache License
public static void ingest(final String[] args, final BatchWriter writer, int seconds) throws Exception { String token = args[0];/*from w ww. ja va 2 s .co m*/ String tokenSecret = args[1]; String consumerKey = args[2]; String consumerSecret = args[3]; TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addListener(new TGStatusListener(writer)); AccessToken accessToken = new AccessToken(token, tokenSecret); twitterStream.setOAuthConsumer(consumerKey, consumerSecret); twitterStream.setOAuthAccessToken(accessToken); System.out.println("processing twitter stream ..."); twitterStream.sample(); Thread.sleep(seconds * 1000); System.out.println("shutting down twitter stream."); twitterStream.shutdown(); writer.flush(); }
From source file:com.babatunde.twittergoogle.Utility.java
public void postToGoogle(PlusDomains s, String id, String hashtag) { try {//from w ww . ja va 2s . c om final PlusDomains serve = s; final String circleID = id; listener = new StatusListener() { @Override public void onStatus(Status status) { String msg = status.getUser().getName() + " - " + "@" + status.getUser().getScreenName() + " - " + status.getText(); System.out.println(msg); //Create a list of ACL entries if (serve != null && (!circleID.isEmpty() || (circleID != null))) { PlusDomainsAclentryResource resource = new PlusDomainsAclentryResource(); resource.setType("domain").setType("circle").setId(circleID); //Get Emails of people in the circle. List<PlusDomainsAclentryResource> aclEntries = new ArrayList<PlusDomainsAclentryResource>(); aclEntries.add(resource); Acl acl = new Acl(); acl.setItems(aclEntries); acl.setDomainRestricted(true); // Required, this does the domain restriction // Create a new activity object Activity activity = new Activity() .setObject(new Activity.PlusDomainsObject().setOriginalContent(msg)).setAccess(acl); try { // Execute the API request, which calls `activities.insert` for the logged in user activity = serve.activities().insert("me", activity).execute(); } catch (IOException ex) { Logger.getLogger(Utility.class.getName()).log(Level.SEVERE, null, ex); } } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { System.out.println("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } @Override public void onException(Exception ex) { ex.printStackTrace(); } }; } catch (Exception e) { } TwitterStream twitterStream = new TwitterStreamFactory(configuration).getInstance(); twitterStream.addListener(listener); String str[] = { hashtag }; twitterStream.shutdown(); try { Thread.sleep(1000); } catch (InterruptedException ex) { Logger.getLogger(Utility.class.getName()).log(Level.SEVERE, null, ex); } twitterStream.filter(new FilterQuery().track(str)); }
From source file:com.waves_rsp.ikb4stream.datasource.twitter.TwitterProducerConnector.java
License:Open Source License
/** * Listen tweets from twitter with a bounding box and load them with the data producer object * * @param dataProducer {@link IDataProducer} contains the data queue * @see TwitterProducerConnector#confBuilder * @see TwitterProducerConnector#boundingBox *//*from www. j a v a 2 s .co m*/ @Override public void load(IDataProducer dataProducer) { Objects.requireNonNull(dataProducer); TwitterStream twitterStream = null; try { TwitterStreamListener streamListener = new TwitterStreamListener(dataProducer); twitterStream = new TwitterStreamFactory(confBuilder.build()).getInstance(); FilterQuery filterQuery = new FilterQuery(); filterQuery.locations(boundingBox); twitterStream.addListener(streamListener); twitterStream.filter(filterQuery); Thread.currentThread().join(); } catch (IllegalArgumentException | IllegalStateException err) { LOGGER.error("Error loading : " + err.getMessage()); throw new IllegalStateException(err.getMessage()); } catch (InterruptedException e) { LOGGER.info("Close twitter"); Thread.currentThread().interrupt(); } finally { if (twitterStream != null) { twitterStream.cleanUp(); twitterStream.shutdown(); } Thread.currentThread().interrupt(); } }
From source file:de.jetwick.tw.TweetProducerViaStream.java
License:Apache License
@Override public void run() { TwitterStream stream = null;// w w w.ja v a2 s . co m TwitterStream oldStream = null; // we cannot detect frequency of all terms but detect + remove high frequent disturbers Map<String, Integer> termFreq = new LinkedHashMap<String, Integer>(); while (true) { try { // stream only LESS FREQUENT tags! leave popular tags only for search Collection<String> input = initTags(termFreq); termFreq.clear(); if (input.isEmpty()) { logger.error("No less frequent tags found! Frequency limit:" + tweetsPerSecLimit); if (!myWait(10)) break; continue; } int counter = 0; logger.info("Starting over with " + input.size() + " tags. indexed tweets:" + counter + " tweetsPerSecLimit:" + tweetsPerSecLimit + " " + input); if (stream != null) oldStream = stream; // use a separate collection here to let the listener release when doing garbage collection // (the listener which is added in the streamingTwitter method) BlockingQueue<JTweet> queue = new LinkedBlockingQueue<JTweet>(1000); stream = twSearch.streamingTwitter(input, queue); // shutdown old stream if (oldStream != null) { oldStream.shutdown(); // oldStream.cleanUp(); } long start = System.currentTimeMillis(); while (true) { JTweet tw = queue.take(); String matchingTerm = null; String txt = tw.getLowerCaseText(); for (String term : input) { if (txt.contains(term)) { matchingTerm = term; break; } } resultTweets.put(tw.setFeedSource("from stream:" + matchingTerm)); Integer integ = termFreq.put(matchingTerm, 1); if (integ != null) termFreq.put(matchingTerm, integ + 1); counter++; // UPDATE tags after a while if ((System.currentTimeMillis() - start) > newStreamInterval) break; } } catch (Exception ex) { logger.error("!! Error while getting tweets via streaming API. Waiting and trying again.", ex); if (!myWait(60 * 5)) break; } } logger.info(getName() + " finished"); }
From source file:druid.examples.twitter.TwitterSpritzerFirehoseFactory.java
License:Open Source License
@Override public Firehose connect() throws IOException { final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { @Override//from ww w . j a v a 2s . c om public void onConnect() { log.info("Connected_to_Twitter"); } @Override public void onDisconnect() { log.info("Disconnect_from_Twitter"); } /** * called before thread gets cleaned up */ @Override public void onCleanUp() { log.info("Cleanup_twitter_stream"); } }; // ConnectionLifeCycleListener final TwitterStream twitterStream; final StatusListener statusListener; final int QUEUE_SIZE = 2000; /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread. */ final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE); final LinkedList<String> dimensions = new LinkedList<String>(); final long startMsec = System.currentTimeMillis(); dimensions.add("htags"); dimensions.add("lang"); dimensions.add("utc_offset"); // // set up Twitter Spritzer // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { // time to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { boolean success = queue.offer(status, 15L, TimeUnit.SECONDS); if (!success) { log.warn("queue too slow!"); } } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // This notice will be sent each time a limited stream becomes unlimited. // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity. log.warn("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onException(Exception ex) { ex.printStackTrace(); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } }; twitterStream.addListener(statusListener); twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); return new Firehose() { private final Runnable doNothingRunnable = new Runnable() { public void run() { } }; private long rowCount = 0L; private boolean waitIfmax = (maxEventCount < 0L); private final Map<String, Object> theMap = new HashMap<String, Object>(2); // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper(); private boolean maxTimeReached() { if (maxRunMinutes <= 0) { return false; } else { return (System.currentTimeMillis() - startMsec) / 60000L >= maxRunMinutes; } } private boolean maxCountReached() { return maxEventCount >= 0 && rowCount >= maxEventCount; } @Override public boolean hasMore() { if (maxCountReached() || maxTimeReached()) { return waitIfmax; } else { return true; } } @Override public InputRow nextRow() { // Interrupted to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } // all done? if (maxCountReached() || maxTimeReached()) { if (waitIfmax) { // sleep a long time instead of terminating try { log.info("reached limit, sleeping a long time..."); sleep(2000000000L); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { // allow this event through, and the next hasMore() call will be false } } if (++rowCount % 1000 == 0) { log.info("nextRow() has returned %,d InputRows", rowCount); } Status status; try { status = queue.take(); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } HashtagEntity[] hts = status.getHashtagEntities(); if (hts != null && hts.length > 0) { List<String> hashTags = Lists.newArrayListWithExpectedSize(hts.length); for (HashtagEntity ht : hts) { hashTags.add(ht.getText()); } theMap.put("htags", Arrays.asList(hashTags.get(0))); } long retweetCount = status.getRetweetCount(); theMap.put("retweet_count", retweetCount); User user = status.getUser(); if (user != null) { theMap.put("follower_count", user.getFollowersCount()); theMap.put("friends_count", user.getFriendsCount()); theMap.put("lang", user.getLang()); theMap.put("utc_offset", user.getUtcOffset()); // resolution in seconds, -1 if not available? theMap.put("statuses_count", user.getStatusesCount()); } return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap); } @Override public Runnable commit() { // ephemera in, ephemera out. return doNothingRunnable; // reuse the same object each time } @Override public void close() throws IOException { log.info("CLOSE twitterstream"); twitterStream.shutdown(); // invokes twitterStream.cleanUp() } }; }
From source file:io.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java
License:Apache License
@Override public Firehose connect(InputRowParser parser) throws IOException { final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { @Override/*from ww w . j a v a2s . c o m*/ public void onConnect() { log.info("Connected_to_Twitter"); } @Override public void onDisconnect() { log.info("Disconnect_from_Twitter"); } /** * called before thread gets cleaned up */ @Override public void onCleanUp() { log.info("Cleanup_twitter_stream"); } }; // ConnectionLifeCycleListener final TwitterStream twitterStream; final StatusListener statusListener; final int QUEUE_SIZE = 2000; /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread. */ final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE); final long startMsec = System.currentTimeMillis(); // // set up Twitter Spritzer // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { // time to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { boolean success = queue.offer(status, 15L, TimeUnit.SECONDS); if (!success) { log.warn("queue too slow!"); } } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // This notice will be sent each time a limited stream becomes unlimited. // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity. log.warn("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onException(Exception ex) { ex.printStackTrace(); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } }; twitterStream.addListener(statusListener); twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); return new Firehose() { private final Runnable doNothingRunnable = new Runnable() { public void run() { } }; private long rowCount = 0L; private boolean waitIfmax = (getMaxEventCount() < 0L); private final Map<String, Object> theMap = new TreeMap<>(); // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper(); private boolean maxTimeReached() { if (getMaxRunMinutes() <= 0) { return false; } else { return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes(); } } private boolean maxCountReached() { return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount(); } @Override public boolean hasMore() { if (maxCountReached() || maxTimeReached()) { return waitIfmax; } else { return true; } } @Override public InputRow nextRow() { // Interrupted to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } // all done? if (maxCountReached() || maxTimeReached()) { if (waitIfmax) { // sleep a long time instead of terminating try { log.info("reached limit, sleeping a long time..."); sleep(2000000000L); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { // allow this event through, and the next hasMore() call will be false } } if (++rowCount % 1000 == 0) { log.info("nextRow() has returned %,d InputRows", rowCount); } Status status; try { status = queue.take(); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } theMap.clear(); HashtagEntity[] hts = status.getHashtagEntities(); String text = status.getText(); theMap.put("text", (null == text) ? "" : text); theMap.put("htags", (hts.length > 0) ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() { @Nullable @Override public String apply(HashtagEntity input) { return input.getText(); } }) : ImmutableList.<String>of()); long[] lcontrobutors = status.getContributors(); List<String> contributors = new ArrayList<>(); for (long contrib : lcontrobutors) { contributors.add(String.format("%d", contrib)); } theMap.put("contributors", contributors); GeoLocation geoLocation = status.getGeoLocation(); if (null != geoLocation) { double lat = status.getGeoLocation().getLatitude(); double lon = status.getGeoLocation().getLongitude(); theMap.put("lat", lat); theMap.put("lon", lon); } else { theMap.put("lat", null); theMap.put("lon", null); } if (status.getSource() != null) { Matcher m = sourcePattern.matcher(status.getSource()); theMap.put("source", m.find() ? m.group(1) : status.getSource()); } theMap.put("retweet", status.isRetweet()); if (status.isRetweet()) { Status original = status.getRetweetedStatus(); theMap.put("retweet_count", original.getRetweetCount()); User originator = original.getUser(); theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : ""); theMap.put("originator_follower_count", originator != null ? originator.getFollowersCount() : ""); theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : ""); theMap.put("originator_verified", originator != null ? originator.isVerified() : ""); } User user = status.getUser(); final boolean hasUser = (null != user); theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0); theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0); theMap.put("lang", hasUser ? user.getLang() : ""); theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available? theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0); theMap.put("user_id", hasUser ? String.format("%d", user.getId()) : ""); theMap.put("screen_name", hasUser ? user.getScreenName() : ""); theMap.put("location", hasUser ? user.getLocation() : ""); theMap.put("verified", hasUser ? user.isVerified() : ""); theMap.put("ts", status.getCreatedAt().getTime()); List<String> dimensions = Lists.newArrayList(theMap.keySet()); return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap); } @Override public Runnable commit() { // ephemera in, ephemera out. return doNothingRunnable; // reuse the same object each time } @Override public void close() throws IOException { log.info("CLOSE twitterstream"); twitterStream.shutdown(); // invokes twitterStream.cleanUp() } }; }