List of usage examples for twitter4j HttpResponseCode NOT_FOUND
int NOT_FOUND
To view the source code for twitter4j HttpResponseCode NOT_FOUND.
Click Source Link
From source file:com.avishkar.NewGetFollowersIDs.java
License:Apache License
private static List<Long> getUserFollwers(long twitterUserId) throws UnknownHostException, TwitterException, InterruptedException { Twitter twitter = new TwitterFactory(AccessTokenUtil.getConfig()).getInstance(); List<Long> filteredUserListOnFollowerCount = new ArrayList<Long>(); final Gson gson = new Gson(); try {//from w ww.ja v a 2 s . c o m checkRateLimit("/followers/ids", twitter); System.out.println("Listing followers's Follower for ID:" + twitterUserId + System.lineSeparator()); IDs followerIDs = twitter.getFollowersIDs(twitterUserId, -1); JsonObject followerJSON = new JsonObject(); followerJSON.addProperty("id", twitterUserId); followerJSON.addProperty("followers", gson.toJson(followerIDs.getIDs())); DBAccess.insert(gson.toJson(followerJSON)); System.out.println("Current Followers Fetched Size:" + followerIDs.getIDs().length); // Filtering for influential user if (followerIDs.getIDs().length > 1000) { System.out.println("User assumed as Influential"); return null; } int from = 0; int to = 99; int limit = checkRateLimit("/users/lookup", twitter); while (from < followerIDs.getIDs().length) { if (to > followerIDs.getIDs().length) to = followerIDs.getIDs().length - 1; if (limit == 0) checkRateLimit("/users/lookup", twitter); ResponseList<User> followers = twitter .lookupUsers(Arrays.copyOfRange(followerIDs.getIDs(), from, to)); System.out.println("Recieved User count:" + followers.size()); for (User user : followers) { DBAccess.insertUser(gson.toJson(user)); if (user.getFollowersCount() < 1000) { // if(user.getStatusesCount()>0 && // user.getStatus()!=null && // sevenDaysAgo.after(user.getStatus().getCreatedAt())) filteredUserListOnFollowerCount.add(user.getId()); getStatuses(user.getScreenName(), twitter); } else System.out.println("User " + user.getScreenName() + " is pruned for Influential or over subscription." + " Follower count:" + user.getFollowersCount() + " Friends Count:" + user.getFriendsCount()); } from += 100; to += 100; limit--; } } catch (TwitterException te) { if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println("Encountered locked profile. Skipping " + twitterUserId); return null; // log something here } te.printStackTrace(); System.out.println("Failed to get followers' Follower: " + te.getMessage()); // System.exit(-1); } return filteredUserListOnFollowerCount; }
From source file:friendsandfollowers.DBFollowersIDs.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check arguments that passed in if ((args == null) || (args.length == 0)) { System.err.println("2 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'OUTPUT: /output/path/'"); System.err.println("Second: (int) Number of ids to fetch. " + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Third (optional): 'screen_name / user_id_str'"); System.err.println("If 3rd argument not provided then provide" + " Twitter users through database."); System.exit(-1);// w w w.j a v a2 s .c om } MysqlDB DB = new MysqlDB(); AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/followers/ids"; String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; try { IDS_TO_FETCH_INT = Integer.parseInt(args[1]); } catch (NumberFormatException e) { System.err.println("Argument" + args[1] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } String targetedUser = ""; if (args.length == 3) { try { targetedUser = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an String."); System.exit(-1); } } try { TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); IDs ids; System.out.println("Listing followers ids."); // if targetedUser not provided by argument, then look into database. if (StringUtils.isEmpty(targetedUser)) { String selectQuery = "SELECT * FROM `followers_parent` WHERE " + "`targeteduser` != '' AND " + "`nextcursor` != '0' AND " + "`nextcursor` != '2'"; ResultSet results = DB.selectQ(selectQuery); int numRows = DB.numRows(results); if (numRows < 1) { System.err.println("No User in database to get followersIDS"); System.exit(-1); } OUTERMOST: while (results.next()) { int followers_parent_id = results.getInt("id"); targetedUser = results.getString("targeteduser"); long cursor = results.getLong("nextcursor"); System.out.println("Targeted User: " + targetedUser); int idsLoopCounter = 0; int totalIDs = 0; // put idsJSON in a file PrintWriter writer = new PrintWriter(OutputDirPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFollowersIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, // or if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Followers Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println( "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } // update cursor in "followers_parent" String fieldValues = "`nextcursor` = 2"; String where = "id = " + followers_parent_id; DB.Update("`followers_parent`", fieldValues, where); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth " + "Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); System.out.println(); // update cursor in "followers_parent" String fieldValues = "`nextcursor` = " + cursor; String where = "id = " + followers_parent_id; DB.Update("`followers_parent`", fieldValues, where); } // loop through every result found in db } else { // Second Argument Set, so we are here. System.out.println("screen_name / user_id_str passed by argument"); int idsLoopCounter = 0; int totalIDs = 0; // put idsJSON in a file PrintWriter writer = new PrintWriter( OutputDirPath + "/" + targetedUser + "_ids_" + helpers.getUnixTimeStamp(), "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); long cursor = -1; do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFollowersIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, or if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Followers Get Exception: " + te.getMessage()); } System.exit(-1); } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reach then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); System.out.println(); } } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get followers' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); }
From source file:friendsandfollowers.DBFriendsIDs.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check arguments that passed in if ((args == null) || (args.length == 0)) { System.err.println("2 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'OUTPUT: /output/path/'"); System.err.println("Second: (int) Number of ids to fetch. " + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Third (optional): 'screen_name / user_id_str'"); System.err.println("If 3rd argument not provided then provide" + " Twitter users through database."); System.exit(-1);/*from w w w . java 2s. c om*/ } MysqlDB DB = new MysqlDB(); AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/friends/ids"; String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; try { IDS_TO_FETCH_INT = Integer.parseInt(args[1]); } catch (NumberFormatException e) { System.err.println("Argument" + args[1] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } String targetedUser = ""; if (args.length == 3) { try { targetedUser = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an String."); System.exit(-1); } } try { TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); IDs ids; System.out.println("Listing friends ids."); // if targetedUser not provided by argument, then look into database. if (StringUtils.isEmpty(targetedUser)) { String selectQuery = "SELECT * FROM `followings_parent` WHERE " + "`targeteduser` != '' AND " + "`nextcursor` != '0' AND " + "`nextcursor` != '2'"; ResultSet results = DB.selectQ(selectQuery); int numRows = DB.numRows(results); if (numRows < 1) { System.err.println("No User in database to get friendsIDS"); System.exit(-1); } OUTERMOST: while (results.next()) { int following_parent_id = results.getInt("id"); targetedUser = results.getString("targeteduser"); long cursor = results.getLong("nextcursor"); System.out.println("Targeted User: " + targetedUser); int idsLoopCounter = 0; int totalIDs = 0; // put idsJSON in a file PrintWriter writer = new PrintWriter(OutputDirPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFriendsIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, // or if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Friends Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println( "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } // update cursor in "followings_parent" String fieldValues = "`nextcursor` = 2"; String where = "id = " + following_parent_id; DB.Update("`followings_parent`", fieldValues, where); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user. RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth " + "Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); System.out.println(); // update cursor in "followings_parent" String fieldValues = "`nextcursor` = " + cursor; String where = "id = " + following_parent_id; DB.Update("`followings_parent`", fieldValues, where); } // loop through every result found in db } else { // Second Argument Sets, so we are here. System.out.println("screen_name / user_id_str " + "passed by argument"); int idsLoopCounter = 0; int totalIDs = 0; // put idsJSON in a file PrintWriter writer = new PrintWriter( OutputDirPath + "/" + targetedUser + "_ids_" + helpers.getUnixTimeStamp(), "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); long cursor = -1; do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFriendsIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, // or if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Friends Get Exception: " + te.getMessage()); } System.exit(-1); } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reach then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); System.out.println(); } } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get friends' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); }
From source file:friendsandfollowers.FilesThreaderFollowersIDsParser.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check how many arguments were passed in if ((args == null) || (args.length < 5)) { System.err.println("5 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'INPUT: DB or /input/path/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) This Job Number"); System.err.println("Fifth: (int) Number of seconds to pause"); System.err.println("Sixth: (int) Number of ids to fetch" + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 1 3 75000"); System.exit(-1);//ww w . j a v a2 s .co m } // TODO documentation for command line AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/followers/ids"; String inputPath = null; try { inputPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument " + args[0] + " must be an String."); System.exit(-1); } String outputPath = null; try { outputPath = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument " + args[1] + " must be an String."); System.exit(-1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument " + args[2] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument " + args[3] + " must be an integer."); System.exit(1); } int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; if (args.length == 6) { try { IDS_TO_FETCH_INT = Integer.parseInt(args[5]); } catch (NumberFormatException e) { System.err.println("Argument" + args[5] + " must be an integer."); System.exit(-1); } } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); try { int TotalWorkLoad = 0; ArrayList<String> allFiles = null; try { final File folder = new File(inputPath); allFiles = helpers.listFilesForSingleFolder(folder); TotalWorkLoad = allFiles.size(); } catch (Exception e) { System.err.println("Input folder is not exists: " + e.getMessage()); System.exit(-1); } System.out.println("Total Workload is: " + TotalWorkLoad); if (TotalWorkLoad < 1) { System.err.println("No screen names file exists in: " + inputPath); System.exit(-1); } if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load. Please reduce 'Number of jobs' to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; int chunkSizeToGet = (JOB_NO + 1) * chunkSize; System.out.println("My Share is " + chunkSize); System.out.println(); // Load OAuh User TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println(); IDs ids; System.out.println("Going to get followers ids."); // to write output in a file System.out.flush(); if (JOB_NO + 1 == TOTAL_JOBS) { chunkSizeToGet = TotalWorkLoad; } List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet); for (String myFile : myFilesShare) { System.out.println("Going to parse file: " + myFile); try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) { String line; OUTERMOST: while ((line = br.readLine()) != null) { // process the line. System.out.println("Going to get followers ids of Screen-name / user_id: " + line); System.out.println(); String targetedUser = line.trim(); // tmp long cursor = -1; int idsLoopCounter = 0; int totalIDs = 0; PrintWriter writer = new PrintWriter(outputPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFollowersIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, or // if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Followers Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println( "New Loaded OAuth User " + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); } // while get records from single file } // read my single file catch (IOException e) { System.err.println("Failed to read lines from " + myFile); } // to write output in a file System.out.flush(); } // all my files share } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get followers' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }
From source file:friendsandfollowers.FilesThreaderFriendsIDsParser.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check how many arguments were passed in if ((args == null) || (args.length < 5)) { System.err.println("5 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'INPUT: DB or /input/path/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) This Job Number"); System.err.println("Fifth: (int) Number of seconds to pause"); System.err.println("Sixth: (int) Number of ids to fetch" + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 1 3 75000"); System.exit(-1);// www . j a v a2 s. com } // TODO documentation for command line AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/friends/ids"; String inputPath = null; try { inputPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument " + args[0] + " must be an String."); System.exit(-1); } String outputPath = null; try { outputPath = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument " + args[1] + " must be an String."); System.exit(-1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument " + args[2] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument " + args[3] + " must be an integer."); System.exit(1); } int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; if (args.length == 6) { try { IDS_TO_FETCH_INT = Integer.parseInt(args[5]); } catch (NumberFormatException e) { System.err.println("Argument" + args[5] + " must be an integer."); System.exit(-1); } } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); try { int TotalWorkLoad = 0; ArrayList<String> allFiles = null; try { final File folder = new File(inputPath); allFiles = helpers.listFilesForSingleFolder(folder); TotalWorkLoad = allFiles.size(); } catch (Exception e) { System.err.println("Input folder is not exists: " + e.getMessage()); System.exit(-1); } System.out.println("Total Workload is: " + TotalWorkLoad); if (TotalWorkLoad < 1) { System.err.println("No screen names file exists in: " + inputPath); System.exit(-1); } if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load. Please reduce 'Number of jobs' to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; int chunkSizeToGet = (JOB_NO + 1) * chunkSize; System.out.println("My Share is " + chunkSize); System.out.println(); // Load OAuh User TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println(); IDs ids; System.out.println("Going to get friends ids."); // to write output in a file System.out.flush(); if (JOB_NO + 1 == TOTAL_JOBS) { chunkSizeToGet = TotalWorkLoad; } List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet); for (String myFile : myFilesShare) { System.out.println("Going to parse file: " + myFile); try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) { String line; OUTERMOST: while ((line = br.readLine()) != null) { // process the line. System.out.println("Going to get friends ids of Screen-name / user_id: " + line); System.out.println(); String targetedUser = line.trim(); // tmp long cursor = -1; int idsLoopCounter = 0; int totalIDs = 0; PrintWriter writer = new PrintWriter(outputPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFriendsIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, or // if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Friends Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println( "New Loaded OAuth User " + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); } // while get records from single file } // read my single file catch (IOException e) { System.err.println("Failed to read lines from " + myFile); } // to write output in a file System.out.flush(); } // all my files share } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get friends' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }
From source file:timeline.CmdUserTimeline.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException { // Check how many arguments were passed in if ((args == null) || (args.length != 7)) { System.err.println("Please provide command as following."); System.err.println("java -cp twitter4j-multi-oauth-0.3.jar " + "timeline.CmdUserTimeline consumer_key consumer_secret" + " user_token user_secret output_path " + "screen_name_or_userid " + "number_of_tweets_to_get_max_is_3200"); System.exit(-1);/*from ww w . j av a2 s . c o m*/ } AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/statuses/user_timeline"; String consumer_key = null; try { consumer_key = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } String consumer_secret = null; try { consumer_secret = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an String."); System.exit(-1); } String user_token = null; try { user_token = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an String."); System.exit(-1); } String user_secret = null; try { user_secret = StringEscapeUtils.escapeJava(args[3]); } catch (Exception e) { System.err.println("Argument" + args[3] + " must be an String."); System.exit(-1); } String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[4]); } catch (Exception e) { System.err.println("Argument" + args[4] + " must be an String."); System.exit(-1); } String targetedUser = ""; try { targetedUser = StringEscapeUtils.escapeJava(args[5]); } catch (Exception e) { System.err.println("Argument" + args[5] + " must be an String."); System.exit(-1); } int NUMBER_OF_TWEETS = 0; try { NUMBER_OF_TWEETS = Integer.parseInt(args[6]); } catch (Exception e) { System.err.println("Argument" + args[6] + " must be an integer."); System.exit(-1); } try { TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, consumer_key, consumer_secret, user_token, user_secret); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls - 2; int RemainingCallsCounter = 0; System.out.println("Remianing Calls: " + RemainingCalls); // screen_name / user_id provided by arguments System.out.println("Trying to create output directory"); String filesPath = OutputDirPath + "/"; File theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath); System.err.println(se.getMessage()); System.exit(-1); } } String fileName = filesPath + targetedUser; PrintWriter writer = new PrintWriter(fileName, "UTF-8"); // Call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); List<Status> statuses = new ArrayList<>(); int size = statuses.size(); int pageno = 1; int totalTweets = 0; boolean tweetCounterReached = false; System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS); while (true) { try { Paging page = new Paging(pageno++, 200); if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } else { statuses.addAll(twitter.getUserTimeline(targetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } // If user's total tweet are less than 195 then no next call if (size == 0) { if (totalTweets < 195) { break; } } // If user's all tweets parsed then exit if (totalTweets == size) { break; } size = totalTweets; statuses.clear(); } catch (TwitterException e) { // e.printStackTrace(); // do not throw if user has protected tweets, or if they // deleted their account if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED || e.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Tweets Get Exception: " + e.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { System.out.println("No more remianing calls"); } if (totalTweets < 1) { writer.close(); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } break; } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { System.out.println("No more remianing calls"); break; } } // while get tweets writer.close(); if (totalTweets > 0) { System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets); } else { // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } } catch (TwitterException te) { // te.printStackTrace(); System.out.println("Failed to get followers' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); }
From source file:timeline.DBNonThreader.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException { // Check how many arguments were passed in if ((args == null) || (args.length == 0)) { System.err.println("First: 'OUTPUT_PATH' is mendatory."); System.err.println("Second: (int) Number of Tweets to get. Max 3200"); System.err.println("Third: 'screen_name / id_str'" + " is optional."); System.err.println("If 3rd argument not provided then provide" + " Twitter users through database."); System.exit(-1);/* w ww .ja va2 s .c o m*/ } MysqlDB DB = new MysqlDB(); AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/statuses/user_timeline"; String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } int NUMBER_OF_TWEETS = 3200; try { NUMBER_OF_TWEETS = Integer.parseInt(args[1]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an integer."); System.exit(-1); } String targetedUser = ""; if (args.length == 3) { try { targetedUser = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an String."); System.exit(-1); } } try { TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls - 2; int RemainingCallsCounter = 0; System.out.println("First Time Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println("User's Tweets"); // If targetedUser not provided by argument, then look into // database. if (StringUtils.isEmpty(targetedUser)) { String selectQuery = "SELECT `id`,`targeteduser` FROM " + "`twitter_users` WHERE " + "`tweets_dumped_all` = 0"; ResultSet results = DB.selectQ(selectQuery); int numRows = DB.numRows(results); if (numRows < 1) { System.err.println("No User in database to get Tweets"); System.exit(-1); } System.out.println("Trying to create output directory"); String filesPath = OutputDirPath + "/"; File theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath); System.err.println(se.getMessage()); System.exit(-1); } } OUTERMOST: while (results.next()) { int targetedUserID = results.getInt("id"); targetedUser = results.getString("targeteduser"); System.out.println("Targeted User: " + targetedUser); String fileName = filesPath + "/" + targetedUser; PrintWriter writer = new PrintWriter(fileName, "UTF-8"); // Call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); List<Status> statuses = new ArrayList<>(); int size = statuses.size(); int pageno = 1; int totalTweets = 0; boolean tweetCounterReached = false; System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS); while (true) { try { Paging page = new Paging(pageno++, 200); if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } else { statuses.addAll(twitter.getUserTimeline(targetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } // If user's total tweet are less than 195 // then no next call if (size == 0) { if (totalTweets < 195) { break; } } // If user's all tweets parsed // then proceed to next user if (totalTweets == size) { break; } size = totalTweets; statuses.clear(); } catch (TwitterException e) { // e.printStackTrace(); // do not throw if user has protected tweets, // or if they deleted their account if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED || e.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Tweets Get Exception: " + e.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println( "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (totalTweets < 1) { writer.close(); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } // Update stats String fieldValues = "`tweets_dumped_all` = 2"; String where = "id = " + targetedUserID; DB.Update("`twitter_users`", fieldValues, where); continue OUTERMOST; } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } // while get tweets writer.close(); if (totalTweets > 0) { System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets); // Update stats String fieldValues = "`tweets_dumped_all` = 1, " + " `tweets_all_count` = " + totalTweets; String where = "id = " + targetedUserID; DB.Update("`twitter_users`", fieldValues, where); } else { // Update stats String fieldValues = "`tweets_dumped_all` = 2"; String where = "id = " + targetedUserID; DB.Update("`twitter_users`", fieldValues, where); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } // while get users from database } else { // screen_name / user_id provided by arguments System.out.println("screen_name / user_id provided by arguments"); System.out.println("Trying to create output directory"); String filesPath = OutputDirPath + "/"; File theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath); System.err.println(se.getMessage()); System.exit(-1); } } System.out.println("Targeted User: " + targetedUser); String fileName = filesPath + "/" + targetedUser; PrintWriter writer = new PrintWriter(fileName, "UTF-8"); // Call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); List<Status> statuses = new ArrayList<>(); int size = statuses.size(); int pageno = 1; int totalTweets = 0; boolean tweetCounterReached = false; System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS); while (true) { try { Paging page = new Paging(pageno++, 200); if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } else { statuses.addAll(twitter.getUserTimeline(targetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } // If user's total tweet are less than 195 then no next call if (size == 0) { if (totalTweets < 195) { break; } } // If user's all tweets parsed then proceed to next user if (totalTweets == size) { break; } size = totalTweets; statuses.clear(); } catch (TwitterException e) { // e.printStackTrace(); // do not throw if user has protected tweets, or if they deleted their account if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED || e.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Tweets Get Exception: " + e.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (totalTweets < 1) { writer.close(); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } // while get tweets writer.close(); if (totalTweets > 0) { System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets); } else { // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } } // screen_name / user_id provided by arguments } catch (TwitterException te) { // te.printStackTrace(); System.out.println("Failed to get followers' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); }
From source file:timeline.DBThreaderParser.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException { // Check how many arguments were passed in if ((args == null) || (args.length < 5)) { System.err.println("5 Parameters are required to launch a Job."); System.err.println("First: String 'OUTPUT_PATH'"); System.err.println("Second: (int) Total Number Of Jobs"); System.err.println("Third: (int) This Job Number"); System.err.println("Fourth: (int) Seconds to pause between next launch"); System.err.println("Fifth: (int) Number of Tweets to get. Max 3200"); System.err.println("Example: fileName.class /output/path 10 1 2 3200"); System.exit(-1);/*from w ww. jav a 2 s . co m*/ } String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an String."); System.exit(1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[1]); } catch (NumberFormatException e) { System.err.println("Argument" + args[1] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument" + args[2] + " must be an integer."); System.exit(1); } int NUMBER_OF_TWEETS = 0; try { NUMBER_OF_TWEETS = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(1); } MysqlDB DB = new MysqlDB(); AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/statuses/user_timeline"; try { String totalRowsQuery = "SELECT count( * ) as wLoad FROM `twitter_users`" + " WHERE `tweets_dumped_all` = 0"; ResultSet totalRowsResults = DB.selectQ(totalRowsQuery); int TotalWorkLoad = 0; while (totalRowsResults.next()) { TotalWorkLoad = totalRowsResults.getInt("wLoad"); } if (TotalWorkLoad < 1) { System.err.println("No User in database to get Tweets"); System.exit(-1); } // Free memmory totalRowsResults = null; if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load Please reduce Number of jobs to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; String selectQuery = "SELECT `id`,`targeteduser` FROM " + "`twitter_users` WHERE " + "`tweets_dumped_all` = 0 LIMIT " + offSet + "," + chunkSize; ResultSet results = DB.selectQ(selectQuery); int numRows = DB.numRows(results); if (numRows < 1) { System.err.println( "No User in database to get Tweets" + " with offset " + offSet + " and limit " + chunkSize); System.exit(-1); } /** * wait before launching actual job */ int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(-1); } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); /** * wait before launching actual job */ TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls - 2; int RemainingCallsCounter = 0; System.out.println("First Time Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println("User's Tweets"); System.out.println("Trying to create output directory"); String filesPath = OutputDirPath + "/"; File theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath); System.err.println(se.getMessage()); System.exit(-1); } } System.out.flush(); OUTERMOST: while (results.next()) { int targetedUserID = results.getInt("id"); String targetedUser = results.getString("targeteduser"); System.out.println("Targeted User: " + targetedUser); // Create User file to push tweets in it String fileName = filesPath + "/" + targetedUser; PrintWriter writer = new PrintWriter(fileName, "UTF-8"); // Call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); List<Status> statuses = new ArrayList<>(); int size = statuses.size(); int pageno = 1; int totalTweets = 0; boolean tweetCounterReached = false; System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS); while (true) { try { Paging page = new Paging(pageno++, 200); if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } else { statuses.addAll(twitter.getUserTimeline(targetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } // If user's total tweet are less // than 195 then no next call if (size == 0) { if (totalTweets < 195) { break; } } // If user's all tweets parsed // then proceed to next user if (totalTweets == size) { break; } size = totalTweets; statuses.clear(); } catch (TwitterException e) { // e.printStackTrace(); // do not throw if user has protected tweets, // or if they deleted their account if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED || e.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Tweets Get Exception: " + e.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (totalTweets < 1) { writer.close(); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } // Update stats String fieldValues = "`tweets_dumped_all` = 2"; String where = "id = " + targetedUserID; DB.Update("`twitter_users`", fieldValues, where); continue OUTERMOST; } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } // while get tweets writer.close(); if (totalTweets > 0) { System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets); // Update stats String fieldValues = "`tweets_dumped_all` = 1, " + " `tweets_all_count` = " + totalTweets; String where = "id = " + targetedUserID; DB.Update("`twitter_users`", fieldValues, where); } else { // Update stats String fieldValues = "`tweets_dumped_all` = 2, " + " `tweets_all_count` = " + totalTweets; String where = "id = " + targetedUserID; DB.Update("`twitter_users`", fieldValues, where); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } System.out.flush(); } // while get users from database } catch (TwitterException te) { // te.printStackTrace(); System.out.println("Failed to get tweets: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }
From source file:timeline.FilesThreaderParser.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException { // Check how many arguments were passed in if ((args == null) || (args.length < 7)) { System.err.println("7 Parameters are required to launch a Job."); System.err.println("First: String 'INPUT: " + "/path/to/perline/screen_names/files/'"); System.err.println("Second: String 'OUTPUT_PATH'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) This Job Number"); System.err.println("Fifth: (int) Seconds to pause between " + "next launch"); System.err.println("Sixth: (int) Number of Tweets to get. Max 3200"); System.err.println("Name of current threader."); System.err.println("Example: fileName.class /output/path " + "10 2 3200 pool-1-thread-1"); System.exit(-1);//w w w . ja va2 s.com } String inputPath = null; try { inputPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument " + args[0] + " must be an String."); System.exit(-1); } String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an String."); System.exit(1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument" + args[2] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(1); } int NUMBER_OF_TWEETS = 0; try { NUMBER_OF_TWEETS = Integer.parseInt(args[5]); } catch (NumberFormatException e) { System.err.println("Argument" + args[5] + " must be an integer."); System.exit(1); } String ThreadName = null; try { ThreadName = StringEscapeUtils.escapeJava(args[6]); } catch (Exception e) { System.err.println("Argument" + args[6] + " must be an String."); System.exit(1); } AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/statuses/user_timeline"; try { int TotalWorkLoad = 0; ArrayList<String> allFiles = null; try { final File folder = new File(inputPath); allFiles = helpers.listFilesForSingleFolder(folder); TotalWorkLoad = allFiles.size(); } catch (Exception e) { System.err.println("Input folder is not exists: " + e.getMessage()); System.exit(-1); } System.out.println("Total Workload is: " + TotalWorkLoad); if (TotalWorkLoad < 1) { System.err.println("No input file exists in: " + inputPath); System.exit(-1); } if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load. Please reduce 'Number of jobs' to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; int chunkSizeToGet = (JOB_NO + 1) * chunkSize; System.out.println("My Share is " + chunkSize); System.out.println("My offSet is " + offSet); System.out.println("My chunkSizeToGet is " + chunkSizeToGet); System.out.println(); /** * wait before launching actual job */ int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); /** * wait before launching actual job */ TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls - 2; int RemainingCallsCounter = 0; System.out.println("First Time Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println("User's Tweets"); System.out.println("Trying to create output directory"); int dirCounter = 1; int fileCounter = 0; String filesPath = OutputDirPath + "/" + ThreadName + "-tweets-" + dirCounter; File theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath + "/" + ThreadName + "-tweets-" + dirCounter); System.err.println(se.getMessage()); System.exit(-1); } } if (JOB_NO + 1 == TOTAL_JOBS) { chunkSizeToGet = TotalWorkLoad; } // to write output in a file System.out.flush(); List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet); for (String myFile : myFilesShare) { System.out.println("Going to parse file: " + myFile); System.out.println(); try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) { String line; OUTERMOST: while ((line = br.readLine()) != null) { if (fileCounter >= 30000) { dirCounter++; fileCounter = 0; filesPath = OutputDirPath + "/" + ThreadName + "-tweets-" + dirCounter; theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath + "/" + ThreadName + "-tweets-" + dirCounter); System.err.println(se.getMessage()); System.exit(-1); } } } System.out.println("Going to get tweets of " + "Screen-name / user_id: " + line); String targetedUser = line.trim(); System.out.println("Targeted User: " + targetedUser); // Create User file to push tweets in it String fileName = filesPath + "/" + targetedUser; PrintWriter writer = new PrintWriter(fileName, "UTF-8"); // Call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); List<Status> statuses = new ArrayList<>(); int size = statuses.size(); int pageno = 1; int totalTweets = 0; boolean tweetCounterReached = false; System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS); System.out.println(); while (true) { try { Paging page = new Paging(pageno++, 200); if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } else { statuses.addAll(twitter.getUserTimeline(targetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } // If user's total tweet are less // than 195 then no next call if (size == 0) { if (totalTweets < 195) { break; } } // If user's all tweets parsed // then proceed to next user if (totalTweets == size) { break; } size = totalTweets; statuses.clear(); } catch (TwitterException e) { // e.printStackTrace(); // do not throw if user has protected tweets, // or if they deleted their account if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED || e.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Tweets Get Exception: " + e.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println( "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (totalTweets < 1) { writer.close(); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } continue OUTERMOST; } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } // while get tweets writer.close(); if (totalTweets > 0) { fileCounter++; System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets); } else { // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } // while get users } // read my single file catch (IOException e) { System.err.println("Failed to read lines from " + myFile); } File currentFile = new File(inputPath + "/" + myFile); currentFile.delete(); // to write output in a file System.out.flush(); } // all my files share } catch (TwitterException te) { // te.printStackTrace(); System.out.println("Failed to get tweets: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }