List of usage examples for org.apache.commons.lang3 StringEscapeUtils escapeJava
public static final String escapeJava(final String input)
Escapes the characters in a String using Java String rules.
Deals correctly with quotes and control-chars (tab, backslash, cr, ff, etc.)
So a tab becomes the characters '\\' and 't' .
The only difference between Java strings and JavaScript strings is that in JavaScript, a single quote and forward-slash (/) are escaped.
Example:
input string: He didn't say, "Stop!"Usage
From source file:friendsandfollowers.FilesThreaderFriendsIDs.java
public static void main(String[] args) throws InterruptedException { // Check how many arguments were passed in if ((args == null) || (args.length < 4)) { System.err.println("4 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'INPUT: /path/to/files/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) Number of seconds to pause"); System.err/*from ww w . ja v a 2 s .c o m*/ .println("Fifth: (int) Number of ids to fetch. " + "Provide number which increment by 5000 eg " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 2 75000"); System.exit(-1); } // to write output in a file ThreadPrintStream.replaceSystemOut(); try { INPUT = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } try { OUTPUT = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an String."); System.exit(-1); } try { TOTAL_JOBS_STR = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an integer."); System.exit(-1); } try { PAUSE_STR = StringEscapeUtils.escapeJava(args[3]); } catch (Exception e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(-1); } if (args.length == 5) { try { IDS_TO_FETCH = StringEscapeUtils.escapeJava(args[4]); } catch (Exception e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } } try { PAUSE = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(-1); } try { TOTAL_JOBS = Integer.parseInt(TOTAL_JOBS_STR); } catch (NumberFormatException e) { System.err.println("Argument" + TOTAL_JOBS_STR + " must be an integer."); System.exit(-1); } System.out.println("Going to launch jobs. " + "Please see logs files to track jobs."); ExecutorService threadPool = Executors.newFixedThreadPool(TOTAL_JOBS); for (int i = 0; i < TOTAL_JOBS; i++) { final String JOB_NO = Integer.toString(i); threadPool.submit(new Runnable() { public void run() { try { // Creating a text file where System.out.println() // will send its output for this thread. String name = Thread.currentThread().getName(); FileOutputStream fos = null; try { fos = new FileOutputStream(name + "-logs.txt"); } catch (Exception e) { System.err.println(e.getMessage()); System.exit(0); } // Create a PrintStream that will write to the new file. PrintStream stream = new PrintStream(new BufferedOutputStream(fos)); // Install the PrintStream to be used as // System.out for this thread. ((ThreadPrintStream) System.out).setThreadOut(stream); // Output three messages to System.out. System.out.println(name); System.out.println(); System.out.println(); FilesThreaderFriendsIDs.execTask(INPUT, OUTPUT, TOTAL_JOBS_STR, JOB_NO, PAUSE_STR, IDS_TO_FETCH); } catch (IOException | ClassNotFoundException | SQLException | JSONException e) { // e.printStackTrace(); System.out.println(e.getMessage()); } } }); helpers.pause(PAUSE); } threadPool.shutdown(); }From source file:friendsandfollowers.FilesThreaderFollowersIDs.java
public static void main(String[] args) throws InterruptedException { // Check how many arguments were passed in if ((args == null) || (args.length < 4)) { System.err.println("4 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'INPUT: /path/to/files/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) Number of seconds to pause"); System.err/*www. j av a2 s . c o m*/ .println("Fifth: (int) Number of ids to fetch. " + "Provide number which increment by 5000 eg " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 2 75000"); System.exit(-1); } // to write output in a file ThreadPrintStream.replaceSystemOut(); try { INPUT = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } try { OUTPUT = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an String."); System.exit(-1); } try { TOTAL_JOBS_STR = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an integer."); System.exit(-1); } try { PAUSE_STR = StringEscapeUtils.escapeJava(args[3]); } catch (Exception e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(-1); } if (args.length == 5) { try { IDS_TO_FETCH = StringEscapeUtils.escapeJava(args[4]); } catch (Exception e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } } try { PAUSE = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(-1); } try { TOTAL_JOBS = Integer.parseInt(TOTAL_JOBS_STR); } catch (NumberFormatException e) { System.err.println("Argument" + TOTAL_JOBS_STR + " must be an integer."); System.exit(-1); } System.out.println("Going to launch jobs. " + "Please see logs files to track jobs."); ExecutorService threadPool = Executors.newFixedThreadPool(TOTAL_JOBS); for (int i = 0; i < TOTAL_JOBS; i++) { final String JOB_NO = Integer.toString(i); threadPool.submit(new Runnable() { public void run() { try { // Creating a text file where System.out.println() // will send its output for this thread. String name = Thread.currentThread().getName(); FileOutputStream fos = null; try { fos = new FileOutputStream(name + "-logs.txt"); } catch (Exception e) { System.err.println(e.getMessage()); System.exit(0); } // Create a PrintStream that will write to the new file. PrintStream stream = new PrintStream(new BufferedOutputStream(fos)); // Install the PrintStream to be used as // System.out for this thread. ((ThreadPrintStream) System.out).setThreadOut(stream); // Output three messages to System.out. System.out.println(name); System.out.println(); System.out.println(); FilesThreaderFollowersIDs.execTask(INPUT, OUTPUT, TOTAL_JOBS_STR, JOB_NO, PAUSE_STR, IDS_TO_FETCH); } catch (IOException | ClassNotFoundException | SQLException | JSONException e) { // e.printStackTrace(); System.out.println(e.getMessage()); } } }); helpers.pause(PAUSE); } threadPool.shutdown(); }From source file:friendsandfollowers.DBFollowersIDs.java
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check arguments that passed in if ((args == null) || (args.length == 0)) { System.err.println("2 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'OUTPUT: /output/path/'"); System.err.println("Second: (int) Number of ids to fetch. " + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Third (optional): 'screen_name / user_id_str'"); System.err.println("If 3rd argument not provided then provide" + " Twitter users through database."); System.exit(-1);//from ww w .j a v a2s . com } MysqlDB DB = new MysqlDB(); AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/followers/ids"; String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; try { IDS_TO_FETCH_INT = Integer.parseInt(args[1]); } catch (NumberFormatException e) { System.err.println("Argument" + args[1] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } String targetedUser = ""; if (args.length == 3) { try { targetedUser = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an String."); System.exit(-1); } } try { TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); IDs ids; System.out.println("Listing followers ids."); // if targetedUser not provided by argument, then look into database. if (StringUtils.isEmpty(targetedUser)) { String selectQuery = "SELECT * FROM `followers_parent` WHERE " + "`targeteduser` != '' AND " + "`nextcursor` != '0' AND " + "`nextcursor` != '2'"; ResultSet results = DB.selectQ(selectQuery); int numRows = DB.numRows(results); if (numRows < 1) { System.err.println("No User in database to get followersIDS"); System.exit(-1); } OUTERMOST: while (results.next()) { int followers_parent_id = results.getInt("id"); targetedUser = results.getString("targeteduser"); long cursor = results.getLong("nextcursor"); System.out.println("Targeted User: " + targetedUser); int idsLoopCounter = 0; int totalIDs = 0; // put idsJSON in a file PrintWriter writer = new PrintWriter(OutputDirPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFollowersIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, // or if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Followers Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println( "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } // update cursor in "followers_parent" String fieldValues = "`nextcursor` = 2"; String where = "id = " + followers_parent_id; DB.Update("`followers_parent`", fieldValues, where); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth " + "Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); System.out.println(); // update cursor in "followers_parent" String fieldValues = "`nextcursor` = " + cursor; String where = "id = " + followers_parent_id; DB.Update("`followers_parent`", fieldValues, where); } // loop through every result found in db } else { // Second Argument Set, so we are here. System.out.println("screen_name / user_id_str passed by argument"); int idsLoopCounter = 0; int totalIDs = 0; // put idsJSON in a file PrintWriter writer = new PrintWriter( OutputDirPath + "/" + targetedUser + "_ids_" + helpers.getUnixTimeStamp(), "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); long cursor = -1; do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFollowersIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, or if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Followers Get Exception: " + te.getMessage()); } System.exit(-1); } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reach then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); System.out.println(); } } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get followers' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); }From source file:friendsandfollowers.DBFriendsIDs.java
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check arguments that passed in if ((args == null) || (args.length == 0)) { System.err.println("2 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'OUTPUT: /output/path/'"); System.err.println("Second: (int) Number of ids to fetch. " + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Third (optional): 'screen_name / user_id_str'"); System.err.println("If 3rd argument not provided then provide" + " Twitter users through database."); System.exit(-1);/*from ww w . ja va2 s . com*/ } MysqlDB DB = new MysqlDB(); AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/friends/ids"; String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; try { IDS_TO_FETCH_INT = Integer.parseInt(args[1]); } catch (NumberFormatException e) { System.err.println("Argument" + args[1] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } String targetedUser = ""; if (args.length == 3) { try { targetedUser = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an String."); System.exit(-1); } } try { TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); IDs ids; System.out.println("Listing friends ids."); // if targetedUser not provided by argument, then look into database. if (StringUtils.isEmpty(targetedUser)) { String selectQuery = "SELECT * FROM `followings_parent` WHERE " + "`targeteduser` != '' AND " + "`nextcursor` != '0' AND " + "`nextcursor` != '2'"; ResultSet results = DB.selectQ(selectQuery); int numRows = DB.numRows(results); if (numRows < 1) { System.err.println("No User in database to get friendsIDS"); System.exit(-1); } OUTERMOST: while (results.next()) { int following_parent_id = results.getInt("id"); targetedUser = results.getString("targeteduser"); long cursor = results.getLong("nextcursor"); System.out.println("Targeted User: " + targetedUser); int idsLoopCounter = 0; int totalIDs = 0; // put idsJSON in a file PrintWriter writer = new PrintWriter(OutputDirPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFriendsIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, // or if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Friends Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println( "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } // update cursor in "followings_parent" String fieldValues = "`nextcursor` = 2"; String where = "id = " + following_parent_id; DB.Update("`followings_parent`", fieldValues, where); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user. RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth " + "Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); System.out.println(); // update cursor in "followings_parent" String fieldValues = "`nextcursor` = " + cursor; String where = "id = " + following_parent_id; DB.Update("`followings_parent`", fieldValues, where); } // loop through every result found in db } else { // Second Argument Sets, so we are here. System.out.println("screen_name / user_id_str " + "passed by argument"); int idsLoopCounter = 0; int totalIDs = 0; // put idsJSON in a file PrintWriter writer = new PrintWriter( OutputDirPath + "/" + targetedUser + "_ids_" + helpers.getUnixTimeStamp(), "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); long cursor = -1; do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFriendsIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, // or if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Friends Get Exception: " + te.getMessage()); } System.exit(-1); } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reach then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); System.out.println(); } } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get friends' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); }From source file:friendsandfollowers.FilesThreaderFriendsIDsParser.java
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check how many arguments were passed in if ((args == null) || (args.length < 5)) { System.err.println("5 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'INPUT: DB or /input/path/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) This Job Number"); System.err.println("Fifth: (int) Number of seconds to pause"); System.err.println("Sixth: (int) Number of ids to fetch" + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 1 3 75000"); System.exit(-1);// w w w .j ava2 s. c o m } // TODO documentation for command line AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/friends/ids"; String inputPath = null; try { inputPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument " + args[0] + " must be an String."); System.exit(-1); } String outputPath = null; try { outputPath = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument " + args[1] + " must be an String."); System.exit(-1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument " + args[2] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument " + args[3] + " must be an integer."); System.exit(1); } int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; if (args.length == 6) { try { IDS_TO_FETCH_INT = Integer.parseInt(args[5]); } catch (NumberFormatException e) { System.err.println("Argument" + args[5] + " must be an integer."); System.exit(-1); } } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); try { int TotalWorkLoad = 0; ArrayList<String> allFiles = null; try { final File folder = new File(inputPath); allFiles = helpers.listFilesForSingleFolder(folder); TotalWorkLoad = allFiles.size(); } catch (Exception e) { System.err.println("Input folder is not exists: " + e.getMessage()); System.exit(-1); } System.out.println("Total Workload is: " + TotalWorkLoad); if (TotalWorkLoad < 1) { System.err.println("No screen names file exists in: " + inputPath); System.exit(-1); } if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load. Please reduce 'Number of jobs' to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; int chunkSizeToGet = (JOB_NO + 1) * chunkSize; System.out.println("My Share is " + chunkSize); System.out.println(); // Load OAuh User TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println(); IDs ids; System.out.println("Going to get friends ids."); // to write output in a file System.out.flush(); if (JOB_NO + 1 == TOTAL_JOBS) { chunkSizeToGet = TotalWorkLoad; } List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet); for (String myFile : myFilesShare) { System.out.println("Going to parse file: " + myFile); try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) { String line; OUTERMOST: while ((line = br.readLine()) != null) { // process the line. System.out.println("Going to get friends ids of Screen-name / user_id: " + line); System.out.println(); String targetedUser = line.trim(); // tmp long cursor = -1; int idsLoopCounter = 0; int totalIDs = 0; PrintWriter writer = new PrintWriter(outputPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFriendsIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, or // if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Friends Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println( "New Loaded OAuth User " + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); } // while get records from single file } // read my single file catch (IOException e) { System.err.println("Failed to read lines from " + myFile); } // to write output in a file System.out.flush(); } // all my files share } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get friends' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }From source file:friendsandfollowers.FilesThreaderFollowersIDsParser.java
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check how many arguments were passed in if ((args == null) || (args.length < 5)) { System.err.println("5 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'INPUT: DB or /input/path/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) This Job Number"); System.err.println("Fifth: (int) Number of seconds to pause"); System.err.println("Sixth: (int) Number of ids to fetch" + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 1 3 75000"); System.exit(-1);//from www .j a v a 2 s . co m } // TODO documentation for command line AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/followers/ids"; String inputPath = null; try { inputPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument " + args[0] + " must be an String."); System.exit(-1); } String outputPath = null; try { outputPath = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument " + args[1] + " must be an String."); System.exit(-1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument " + args[2] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument " + args[3] + " must be an integer."); System.exit(1); } int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; if (args.length == 6) { try { IDS_TO_FETCH_INT = Integer.parseInt(args[5]); } catch (NumberFormatException e) { System.err.println("Argument" + args[5] + " must be an integer."); System.exit(-1); } } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); try { int TotalWorkLoad = 0; ArrayList<String> allFiles = null; try { final File folder = new File(inputPath); allFiles = helpers.listFilesForSingleFolder(folder); TotalWorkLoad = allFiles.size(); } catch (Exception e) { System.err.println("Input folder is not exists: " + e.getMessage()); System.exit(-1); } System.out.println("Total Workload is: " + TotalWorkLoad); if (TotalWorkLoad < 1) { System.err.println("No screen names file exists in: " + inputPath); System.exit(-1); } if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load. Please reduce 'Number of jobs' to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; int chunkSizeToGet = (JOB_NO + 1) * chunkSize; System.out.println("My Share is " + chunkSize); System.out.println(); // Load OAuh User TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println(); IDs ids; System.out.println("Going to get followers ids."); // to write output in a file System.out.flush(); if (JOB_NO + 1 == TOTAL_JOBS) { chunkSizeToGet = TotalWorkLoad; } List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet); for (String myFile : myFilesShare) { System.out.println("Going to parse file: " + myFile); try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) { String line; OUTERMOST: while ((line = br.readLine()) != null) { // process the line. System.out.println("Going to get followers ids of Screen-name / user_id: " + line); System.out.println(); String targetedUser = line.trim(); // tmp long cursor = -1; int idsLoopCounter = 0; int totalIDs = 0; PrintWriter writer = new PrintWriter(outputPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFollowersIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, or // if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Followers Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println( "New Loaded OAuth User " + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); } // while get records from single file } // read my single file catch (IOException e) { System.err.println("Failed to read lines from " + myFile); } // to write output in a file System.out.flush(); } // all my files share } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get followers' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }From source file:cht.Parser.java
public static void main(String[] args) throws IOException { // TODO get from google drive boolean isUnicode = false; boolean isRemoveInputFileOnComplete = false; int rowNum;// w w w . j av a 2 s . com int colNum; Gson gson = new GsonBuilder().setPrettyPrinting().create(); Properties prop = new Properties(); try { prop.load(new FileInputStream("config.txt")); } catch (IOException ex) { ex.printStackTrace(); } String inputFilePath = prop.getProperty("inputFile"); String outputDirectory = prop.getProperty("outputDirectory"); System.out.println(outputDirectory); // optional String unicode = prop.getProperty("unicode"); String removeInputFileOnComplete = prop.getProperty("removeInputFileOnComplete"); inputFilePath = inputFilePath.trim(); outputDirectory = outputDirectory.trim(); if (unicode != null) { isUnicode = Boolean.parseBoolean(unicode.trim()); } if (removeInputFileOnComplete != null) { isRemoveInputFileOnComplete = Boolean.parseBoolean(removeInputFileOnComplete.trim()); } Writer out = null; FileInputStream in = null; final String newLine = System.getProperty("line.separator").toString(); final String separator = File.separator; try { in = new FileInputStream(inputFilePath); Workbook workbook = new XSSFWorkbook(in); Sheet sheet = workbook.getSheetAt(0); rowNum = sheet.getLastRowNum() + 1; colNum = sheet.getRow(0).getPhysicalNumberOfCells(); for (int j = 1; j < colNum; ++j) { String outputFilename = sheet.getRow(0).getCell(j).getStringCellValue(); // guess directory int slash = outputFilename.indexOf('/'); if (slash != -1) { // has directory outputFilename = outputFilename.substring(0, slash) + separator + outputFilename.substring(slash + 1); } String outputPath = FilenameUtils.concat(outputDirectory, outputFilename); System.out.println("--Writing " + outputPath); out = new OutputStreamWriter(new FileOutputStream(outputPath), "UTF-8"); TreeMap<String, Object> map = new TreeMap<String, Object>(); for (int i = 1; i < rowNum; i++) { try { String key = sheet.getRow(i).getCell(0).getStringCellValue(); //String value = ""; Cell tmp = sheet.getRow(i).getCell(j); if (tmp != null) { // not empty string! value = sheet.getRow(i).getCell(j).getStringCellValue(); } if (!key.equals("") && !key.startsWith("#") && !key.startsWith(".")) { value = isUnicode ? StringEscapeUtils.escapeJava(value) : value; int firstdot = key.indexOf("."); String keyName, keyAttribute; if (firstdot > 0) {// a.b.c.d keyName = key.substring(0, firstdot); // a keyAttribute = key.substring(firstdot + 1); // b.c.d TreeMap oldhash = null; Object old = null; if (map.get(keyName) != null) { old = map.get(keyName); if (old instanceof TreeMap == false) { System.out.println("different type of key:" + key); continue; } oldhash = (TreeMap) old; } else { oldhash = new TreeMap(); } int firstdot2 = keyAttribute.indexOf("."); String rootName, childName; if (firstdot2 > 0) {// c, d.f --> d, f rootName = keyAttribute.substring(0, firstdot2); childName = keyAttribute.substring(firstdot2 + 1); } else {// c, d -> d, null rootName = keyAttribute; childName = null; } TreeMap<String, Object> object = myPut(oldhash, rootName, childName); map.put(keyName, object); } else {// c, d -> d, null keyName = key; keyAttribute = null; // simple string mode map.put(key, value); } } } catch (Exception e) { // just ingore empty rows } } String json = gson.toJson(map); // output json out.write(json + newLine); out.close(); } in.close(); System.out.println("\n---Complete!---"); System.out.println("Read input file from " + inputFilePath); System.out.println(colNum - 1 + " output files ate generated at " + outputDirectory); System.out.println(rowNum + " records are generated for each output file."); System.out.println("output file is ecoded as unicode? " + (isUnicode ? "yes" : "no")); if (isRemoveInputFileOnComplete) { File input = new File(inputFilePath); input.deleteOnExit(); System.out.println("Deleted " + inputFilePath); } } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { if (in != null) { in.close(); } } }From source file:com.thoughtworks.go.util.StringUtil.java
public static String quoteJavascriptString(String s) { return "\"" + StringEscapeUtils.escapeJava(s) + "\""; }From source file:magixcel.Main.java
private static void setup() { Scanner s = new Scanner(System.in); boolean doContinue = false; // Intro printLogo();//from w w w . j av a 2s.com System.out.println("Welcome to MagiXcel"); System.out.println("Current encoding is " + Charset.defaultCharset()); // Table name while (!doContinue) { System.out.println("Type your desired table name:"); if (s.hasNextLine()) { Table.setTableName(s.nextLine()); } doContinue = isThisOk(s); } doContinue = false; // File path String path = ""; while (!doContinue) { System.out.println("Enter FULL path of .csv or .txt file you wish to convert"); if (s.hasNextLine()) { path = StringEscapeUtils.escapeJava(s.nextLine()); } FileMan.setPath(path); if (FileMan.currentPathIsValid()) { doContinue = isThisOk(s); } else { System.out.println("ERROR: Invalid Path"); doContinue = false; } } doContinue = false; // File setup FileMan.getLinesFromFile(); Table.setColumnNames(FileMan.takeFirstRow()); FileMan.addRowsToTable(); // Choose output mode System.out.println("Choose output mode:"); OutputFormat.printOutputOptionsNumbered(); int selectedOption = -1; while (!doContinue) { if (s.hasNextLine()) { selectedOption = Integer.parseInt(s.nextLine()); } doContinue = OutputFormat.chooseOption(selectedOption, s); } doContinue = false; OutputFormat.format(); FileMan.writeToFile(); System.out.println("Thank you for using MagiXcel!"); System.out.println("Press Enter to exit"); s.nextLine(); s.close(); }From source file:de.wbuecke.codec.JavaInput.java
@Override public String encode(String plaintext) { return StringEscapeUtils.escapeJava(plaintext); }