List of usage examples for java.io PrintWriter println
public void println(Object x)
From source file:com.medicaid.mmis.util.DataLoader.java
/** * The main function, imports the files given as arguments. * //from w w w. j av a 2 s . c o m * @param args the file names * @throws IOException for read/write errors * @throws PortalServiceException for any other errors */ public static void main(String[] args) throws IOException, PortalServiceException { if (args.length != 2) { System.out.println("2 file path arguments are required."); return; } PropertyConfigurator.configure("log4j.properties"); logger = Logger.getLogger(DataLoader.class); LookupServiceBean lookupBean = new LookupServiceBean(); EntityManagerFactory emf = Persistence.createEntityManagerFactory("cms-data-load"); EntityManager em = emf.createEntityManager(); lookupBean.setEm(em); DataLoader loader = new DataLoader(); loader.setLookup(lookupBean); SequenceGeneratorBean sequence = new SequenceGeneratorBean(); sequence.setEm(em); ProviderEnrollmentServiceBean enrollmentBean = new ProviderEnrollmentServiceBean(); enrollmentBean.setEm(em); enrollmentBean.setSequence(sequence); enrollmentBean.setLookupService(lookupBean); loader.setEnrollmentService(enrollmentBean); long processId = sequence.getNextValue("PROCESS_ID"); System.out.println("Started process id " + processId); BufferedReader br = null; PrintWriter accepted = null; PrintWriter rejected = null; try { System.out.println("Processing file 1..."); File success = new File("accepted_1_" + processId + ".txt"); File failure = new File("rejected_1_" + processId + ".txt"); success.createNewFile(); failure.createNewFile(); accepted = new PrintWriter(success); rejected = new PrintWriter(failure); br = new BufferedReader(new FileReader(args[0])); String line = null; int total = 0; int errors = 0; while ((line = br.readLine()) != null) { total++; try { em.getTransaction().begin(); loader.readProviderFile(new ByteArrayInputStream(line.getBytes())); em.getTransaction().commit(); accepted.println(line); logger.info("Commit row " + total); } catch (PortalServiceException e) { rejected.println(line); em.getTransaction().rollback(); errors++; logger.error("Rollback row " + total + " :" + e.getMessage()); } } accepted.flush(); accepted.close(); rejected.flush(); rejected.close(); br.close(); System.out.println("Total records read: " + total); System.out.println("Total rejected: " + errors); System.out.println("Processing file 2..."); success = new File("accepted_2_" + processId + ".txt"); failure = new File("rejected_2_" + processId + ".txt"); success.createNewFile(); failure.createNewFile(); accepted = new PrintWriter(success); rejected = new PrintWriter(failure); br = new BufferedReader(new FileReader(args[1])); line = null; total = 0; errors = 0; while ((line = br.readLine()) != null) { total++; try { em.getTransaction().begin(); Map<String, OwnershipInformation> owners = loader .readWS000EXT2OWNBEN(new ByteArrayInputStream(line.getBytes())); for (Map.Entry<String, OwnershipInformation> entry : owners.entrySet()) { enrollmentBean.addBeneficialOwners(entry.getKey(), entry.getValue()); } em.getTransaction().commit(); accepted.println(line); logger.info("Commit row " + total); } catch (PortalServiceException e) { rejected.println(line); em.getTransaction().rollback(); errors++; logger.error("Rollback row " + total + " :" + e.getMessage()); } } accepted.flush(); rejected.flush(); System.out.println("Total records read: " + total); System.out.println("Total rejected: " + errors); } finally { if (br != null) { br.close(); } if (accepted != null) { accepted.close(); } if (rejected != null) { rejected.close(); } } }
From source file:id3Crawler.java
public static void main(String[] args) throws IOException { //Input for the directory to be searched. System.out.println("Please enter a directory: "); Scanner scanner = new Scanner(System.in); String input = scanner.nextLine(); //Start a timer to calculate runtime startTime = System.currentTimeMillis(); System.out.println("Starting scan..."); //Files for output PrintWriter pw1 = new PrintWriter(new FileWriter("C:/Users/gbob3_000/Desktop/id3CrawlerOutput/Song.txt")); PrintWriter pw2 = new PrintWriter(new FileWriter("C:/Users/gbob3_000/Desktop/id3CrawlerOutput/Artist.txt")); PrintWriter pw3 = new PrintWriter(new FileWriter("C:/Users/gbob3_000/Desktop/id3CrawlerOutput/Album.txt")); PrintWriter pw4 = new PrintWriter( new FileWriter("C:/Users/gbob3_000/Desktop/id3CrawlerOutput/PerformedBy.txt")); PrintWriter pw5 = new PrintWriter( new FileWriter("C:/Users/gbob3_000/Desktop/id3CrawlerOutput/TrackOf.txt")); PrintWriter pw6 = new PrintWriter( new FileWriter("C:/Users/gbob3_000/Desktop/id3CrawlerOutput/CreatedBy.txt")); //This is used for creating IDs for artists, songs, albums. int idCounter = 0; //This is used to prevent duplicate artists String previousArtist = " "; String currentArtist;/*from www . j a v a 2s . c o m*/ int artistID = 0; //This is used to prevent duplicate albums String previousAlbum = " "; String currentAlbum; int albumID = 0; //This array holds valid extensions to iterate through String[] extensions = new String[] { "mp3" }; //iterate through all files in a directory Iterator<File> it = FileUtils.iterateFiles(new File(input), extensions, true); while (it.hasNext()) { //open the next file File file = it.next(); //instantiate an mp3file object with the opened file MP3 song = GetMP3(file); //pass the song through SongInfo and return the required information SongInfo info = new SongInfo(song); //This is used to prevent duplicate artists/albums currentArtist = info.getArtistInfo(); currentAlbum = info.getAlbumInfo(); //Append the song information to the end of a text file pw1.println(idCounter + "\t" + info.getTitleInfo()); //This prevents duplicates of artists if (!(currentArtist.equals(previousArtist))) { pw2.println(idCounter + "\t" + info.getArtistInfo()); previousArtist = currentArtist; artistID = idCounter; } //This prevents duplicates of albums if (!(currentAlbum.equals(previousAlbum))) { pw3.println(idCounter + "\t" + info.getAlbumInfo()); previousAlbum = currentAlbum; albumID = idCounter; //This formats the IDs for a "CreatedBy" relationship table pw6.println(artistID + "\t" + albumID); } //This formats the IDs for a "PerformedBy" relationship table pw4.println(idCounter + "\t" + artistID); //This formats the IDs for a "TrackOf" relationship table pw5.println(idCounter + "\t" + albumID); idCounter++; songCounter++; } scanner.close(); pw1.close(); pw2.close(); pw3.close(); pw4.close(); pw5.close(); pw6.close(); System.out.println("Scan took " + ((System.currentTimeMillis() - startTime) / 1000.0) + " seconds to scan " + songCounter + " items!"); }
From source file:com.galois.fiveui.HeadlessRunner.java
/** * @param args list of headless run description filenames * @throws IOException/*from w w w . j a va 2s .com*/ * @throws URISyntaxException * @throws ParseException */ @SuppressWarnings("static-access") public static void main(final String[] args) throws IOException, URISyntaxException, ParseException { // Setup command line options Options options = new Options(); Option help = new Option("h", "print this help message"); Option output = OptionBuilder.withArgName("outfile").hasArg().withDescription("write output to file") .create("o"); Option report = OptionBuilder.withArgName("report directory").hasArg() .withDescription("write HTML reports to given directory").create("r"); options.addOption(output); options.addOption(report); options.addOption("v", false, "verbose output"); options.addOption("vv", false, "VERY verbose output"); options.addOption(help); // Parse command line options CommandLineParser parser = new GnuParser(); CommandLine cmd = null; try { cmd = parser.parse(options, args); } catch (ParseException e) { System.err.println("Command line option parsing failed. Reason: " + e.getMessage()); System.exit(1); } // Display help if requested if (cmd.hasOption("h")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("headless <input file 1> [<input file 2> ...]", options); System.exit(1); } // Set logging levels BasicConfigurator.configure(); Logger fiveuiLogger = Logger.getLogger("com.galois.fiveui"); Logger rootLogger = Logger.getRootLogger(); if (cmd.hasOption("v")) { fiveuiLogger.setLevel(Level.DEBUG); rootLogger.setLevel(Level.ERROR); } else if (cmd.hasOption("vv")) { fiveuiLogger.setLevel(Level.DEBUG); rootLogger.setLevel(Level.DEBUG); } else { fiveuiLogger.setLevel(Level.ERROR); rootLogger.setLevel(Level.ERROR); } // Setup output file if requested PrintWriter outStream = null; if (cmd.hasOption("o")) { String outfile = cmd.getOptionValue("o"); try { outStream = new PrintWriter(new BufferedWriter(new FileWriter(outfile))); } catch (IOException e) { System.err.println("Could not open outfile for writing: " + cmd.getOptionValue("outfile")); System.exit(1); } } else { outStream = new PrintWriter(new BufferedWriter(new PrintWriter(System.out))); } // Setup HTML reports directory before the major work happens in case we // have to throw an exception. PrintWriter summaryFile = null; PrintWriter byURLFile = null; PrintWriter byRuleFile = null; if (cmd.hasOption("r")) { String repDir = cmd.getOptionValue("r"); try { File file = new File(repDir); if (!file.exists()) { file.mkdir(); logger.info("report directory created: " + repDir); } else { logger.info("report directory already exists!"); } summaryFile = new PrintWriter(new FileWriter(repDir + File.separator + "summary.html")); byURLFile = new PrintWriter(new FileWriter(repDir + File.separator + "byURL.html")); byRuleFile = new PrintWriter(new FileWriter(repDir + File.separator + "byRule.html")); } catch (IOException e) { System.err.println("could not open report directory / files for writing"); System.exit(1); } } // Major work: process input files ImmutableList<Result> results = null; for (String in : cmd.getArgs()) { HeadlessRunDescription descr = HeadlessRunDescription.parse(in); logger.debug("invoking headless run..."); BatchRunner runner = new BatchRunner(); results = runner.runHeadless(descr); logger.debug("runHeadless returned " + results.size() + " results"); // write results to the output stream as we go for (Result result : results) { outStream.println(result.toString()); } outStream.flush(); } outStream.close(); // Write report files if requested if (cmd.hasOption("r") && results != null) { Reporter kermit = new Reporter(results); summaryFile.write(kermit.getSummary()); summaryFile.close(); byURLFile.write(kermit.getByURL()); byURLFile.close(); byRuleFile.write(kermit.getByRule()); byRuleFile.close(); } }
From source file:whitebox.stats.Kriging.java
public static void main(String[] args) { try {/*from w ww. j a v a2 s . co m*/ //ChartPanel(createChart(createDataset())); Kriging k = new Kriging(); k.readPointFile("/Users/johnlindsay/Documents/Data/Krigging Test Data/test.shp", "Z"); k.ConsiderNugget = false; k.LagSize = 50; k.NumberOfLags = 100; k.Anisotropic = true; k.Tolerance = Math.PI / 4; k.BandWidth = 5000; PrintWriter pw = new PrintWriter("/Users/johnlindsay/Documents/Data/Krigging Test Data/test.txt"); for (int i = 0; i < 13; i++) { k.Angle = Math.PI / 12.0 * i; Variogram var = k.getSemivariogram(SemivariogramType.SPHERICAL, 1, k.NumberOfLags, true, true); pw.println(k.Angle + "," + var.Range + "," + var.Sill + "," + var.Nugget); pw.flush(); System.out.println((i + 1) + " of 12"); } System.out.println("Done!"); } catch (Exception e) { } // k.ConsiderNugget = false; // k.Points = k.ReadPointFile( // "G:\\Optimized Sensory Network\\PALS\\20120607\\test.shp","Z"); // k.LagSize = 2000; // k.Anisotropic = false; // Variogram var = k.SemiVariogram(SemiVariogramType.Spherical, 0.27, 50,false, true); // // //var.Range = 4160.672768; // //var.Sill = 1835.571948; // // k.resolution = 914; // k.BMinX = 588450 + k.resolution/2; // k.BMaxX = 601246 - k.resolution/2; // k.BMinY = 5474650 + k.resolution/2; // k.BMaxY = 5545942 - k.resolution/2; // // List<KrigingPoint> outPnts = k.calcInterpolationPoints() ; // outPnts = k.InterpolatePoints(var, outPnts, 5); // k.BuildRaster("G:\\Optimized Sensory Network\\PALS\\20120607\\Pnts60.dep", outPnts,false); // k.BuildRaster("G:\\Optimized Sensory Network\\PALS\\20120607\\PntsVar60.dep", outPnts,true); // // k.DrawSemiVariogram(k.bins, var); //k.calcBinSurface(SemiVariogramType.Spherical, 1, 99,false); //k.DrawSemiVariogramSurface(k.LagSize*(k.NumberOfLags), false); // k.LagSize = 2500; // k.Anisotropic = false; //// k.Angle = Math.PI*3/4; //// k.Tolerance = Math.PI/4; //// k.BandWidth = 3*k.LagSize; // for (int i = 0; i < 50; i++) { // k.Points = k.ReadPointFile("G:\\Optimized Sensory Network\\PALS\\Pals Shapefiles\\PALS_TA_20120607_HiAlt_v100.shp","h"); // k.Points = k.RandomizePoints(k.Points, 200); // Variogram var = k.SemiVariogram(SemiVariogramType.Spherical, 1, 25,false); // k.resolution = 900; // List<KrigingPoint> pnts = k.calcInterpolationPoints(); // pnts = k.InterpolatePoints(var, pnts, 5); // k.BuildRaster("G:\\Optimized Sensory Network\\PALS\\Pals Shapefiles\\PALS_TA_20120607_HiAlt_v100"+ i +".dep", pnts); // } //k.DrawSemiVariogram(k.bins, var); //k.calcBinSurface(SemiVariogramType.Spherical, 0.27, 25,false); //k.DrawSemiVariogramSurface(k.LagSize*(k.NumberOfLags),false); //k.Points = k.ReadPointFile("G:\\Papers\\AGU 2013\\Sample\\Sample.shp","V"); //k.Points = k.ReadPointFile("G:\\Papers\\AGU 2013\\WakerLake\\WakerLake.shp","V"); // String s= new String(); // PrintWriter pw = null ; // try { // pw = new PrintWriter("G:\\test.txt"); // } catch (FileNotFoundException ex) { // Logger.getLogger(Kriging.class.getName()).log(Level.SEVERE, null, ex); // } // // for (int i = 0; i < 500; i++) { // Kriging k = new Kriging(); // k.ConsiderNugget = false; // k.LagSize = 50; // k.Anisotropic = true; // k.Angle = Math.PI*3/4; // k.Tolerance = Math.PI/4; // k.BandWidth = 3*k.LagSize; // k.Points = k.ReadPointFile("G:\\Optimized Sensory Network\\PALS\\AGU\\SV_Test.shp","v"); // k.Points = k.RandomizePoints(k.Points, 500); // Variogram var = k.SemiVariogram(SemiVariogramType.Exponential, 0.27, 99,true); // s = var.Range + " , " + var.Sill; // // // pw.println(s); // pw.flush(); // k = null; // } // pw.close();; //k.resolution = 2.5; //k.DrawSemiVariogram(k.bins, var); //List<point> pnts = k.calcInterpolationPoints(); // var.Range = 50; // var.Sill = 104843.2; // var.Type = SemiVariogramType.Exponential; // // //pnts = k.InterpolatePoints(var, pnts,10); //k.BuildRaster("G:\\Papers\\AGU 2013\\WakerLake\\WakerLakeOut15.dep", pnts); //k.calcBinSurface(SemiVariogramType.Exponential, 0.27, 99,false); //k.DrawSemiVariogramSurface(k.LagSize*(k.NumberOfLags),true); //Kriging.point p = k.point(65, 137, 0); // List<Kriging.point> pnts = new ArrayList(); // pnts.add(p); }
From source file:microbiosima.SelectiveMicrobiosima.java
/** * @param args/* ww w. j ava 2 s . c o m*/ * the command line arguments * @throws java.io.FileNotFoundException * @throws java.io.UnsupportedEncodingException */ public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException { int populationSize = 500;//Integer.parseInt(parameters[1]); int microSize = 1000;//Integer.parseInt(parameters[2]); int numberOfSpecies = 150;//Integer.parseInt(parameters[3]); int numberOfGeneration = 10000; int Ngene = 10; int numberOfObservation = 100; int numberOfReplication = 10; double Ngenepm = 5; double pctEnv = 0; double pctPool = 0; double msCoeff = 1; double hsCoeff = 1; boolean HMS_or_TMS = true; Options options = new Options(); Option help = new Option("h", "help", false, "print this message"); Option version = new Option("v", "version", false, "print the version information and exit"); options.addOption(help); options.addOption(version); options.addOption(Option.builder("o").longOpt("obs").hasArg().argName("OBS") .desc("Number generation for observation [default: 100]").build()); options.addOption(Option.builder("r").longOpt("rep").hasArg().argName("REP") .desc("Number of replication [default: 1]").build()); Builder C = Option.builder("c").longOpt("config").numberOfArgs(6).argName("Pop Micro Spec Gen") .desc("Four Parameters in the following orders: " + "(1) population size, (2) microbe size, (3) number of species, (4) number of generation, (5) number of total traits, (6)number of traits per microbe" + " [default: 500 1000 150 10000 10 5]"); options.addOption(C.build()); HelpFormatter formatter = new HelpFormatter(); String syntax = "microbiosima pctEnv pctPool"; String header = "\nSimulates the evolutionary and ecological dynamics of microbiomes within a population of hosts.\n\n" + "required arguments:\n" + " pctEnv Percentage of environmental acquisition\n" + " pctPool Percentage of pooled environmental component\n" + " msCoeff Parameter related to microbe selection strength\n" + " hsCoeff Parameter related to host selection strength\n" + " HMS_or_TMS String HMS or TMS to specify host-mediated or trait-mediated microbe selection\n" + "\noptional arguments:\n"; String footer = "\n"; formatter.setWidth(80); CommandLineParser parser = new DefaultParser(); CommandLine cmd = null; try { cmd = parser.parse(options, args); String[] pct_config = cmd.getArgs(); if (cmd.hasOption("h") || args.length == 0) { formatter.printHelp(syntax, header, options, footer, true); System.exit(0); } if (cmd.hasOption("v")) { System.out.println("Microbiosima " + VERSION); System.exit(0); } if (pct_config.length != 5) { System.out.println( "ERROR! Required exactly five argumennts for pct_env, pct_pool, msCoeff, hsCoeff and HMS_or_TMS. It got " + pct_config.length + ": " + Arrays.toString(pct_config)); formatter.printHelp(syntax, header, options, footer, true); System.exit(3); } else { pctEnv = Double.parseDouble(pct_config[0]); pctPool = Double.parseDouble(pct_config[1]); msCoeff = Double.parseDouble(pct_config[2]); hsCoeff = Double.parseDouble(pct_config[3]); if (pct_config[4].equals("HMS")) HMS_or_TMS = true; if (pct_config[4].equals("TMS")) HMS_or_TMS = false; if (pctEnv < 0 || pctEnv > 1) { System.out.println( "ERROR: pctEnv (Percentage of environmental acquisition) must be between 0 and 1 (pctEnv=" + pctEnv + ")! EXIT"); System.exit(3); } if (pctPool < 0 || pctPool > 1) { System.out.println( "ERROR: pctPool (Percentage of pooled environmental component must) must be between 0 and 1 (pctPool=" + pctPool + ")! EXIT"); System.exit(3); } if (msCoeff < 1) { System.out.println( "ERROR: msCoeff (parameter related to microbe selection strength) must be not less than 1 (msCoeff=" + msCoeff + ")! EXIT"); System.exit(3); } if (hsCoeff < 1) { System.out.println( "ERROR: hsCoeff (parameter related to host selection strength) must be not less than 1 (hsCoeff=" + hsCoeff + ")! EXIT"); System.exit(3); } if (!(pct_config[4].equals("HMS") || pct_config[4].equals("TMS"))) { System.out.println( "ERROR: HMS_or_TMS (parameter specifying host-mediated or trait-mediated selection) must be either 'HMS' or 'TMS' (HMS_or_TMS=" + pct_config[4] + ")! EXIT"); System.exit(3); } } if (cmd.hasOption("config")) { String[] configs = cmd.getOptionValues("config"); populationSize = Integer.parseInt(configs[0]); microSize = Integer.parseInt(configs[1]); numberOfSpecies = Integer.parseInt(configs[2]); numberOfGeneration = Integer.parseInt(configs[3]); Ngene = Integer.parseInt(configs[4]); Ngenepm = Double.parseDouble(configs[5]); if (Ngenepm > Ngene) { System.out.println( "ERROR: number of traits per microbe must not be greater than number of total traits! EXIT"); System.exit(3); } } if (cmd.hasOption("obs")) { numberOfObservation = Integer.parseInt(cmd.getOptionValue("obs")); } if (cmd.hasOption("rep")) { numberOfReplication = Integer.parseInt(cmd.getOptionValue("rep")); } } catch (ParseException e) { e.printStackTrace(); System.exit(3); } StringBuilder sb = new StringBuilder(); sb.append("Configuration Summary:").append("\n\tPopulation size: ").append(populationSize) .append("\n\tMicrobe size: ").append(microSize).append("\n\tNumber of species: ") .append(numberOfSpecies).append("\n\tNumber of generation: ").append(numberOfGeneration) .append("\n\tNumber generation for observation: ").append(numberOfObservation) .append("\n\tNumber of replication: ").append(numberOfReplication) .append("\n\tNumber of total traits: ").append(Ngene).append("\n\tNumber of traits per microbe: ") .append(Ngenepm).append("\n"); System.out.println(sb.toString()); double[] environment = new double[numberOfSpecies]; for (int i = 0; i < numberOfSpecies; i++) { environment[i] = 1 / (double) numberOfSpecies; } int[] fitnessToHost = new int[Ngene]; int[] fitnessToMicrobe = new int[Ngene]; for (int rep = 0; rep < numberOfReplication; rep++) { String prefix = "" + (rep + 1) + "_"; String sufix; if (HMS_or_TMS) sufix = "_E" + pctEnv + "_P" + pctPool + "_HS" + hsCoeff + "_HMS" + msCoeff + ".txt"; else sufix = "_E" + pctEnv + "_P" + pctPool + "_HS" + hsCoeff + "_TMS" + msCoeff + ".txt"; System.out.println("Output 5 result files in the format of: " + prefix + "[****]" + sufix); try { PrintWriter file1 = new PrintWriter( new BufferedWriter(new FileWriter(prefix + "gamma_diversity" + sufix))); PrintWriter file2 = new PrintWriter( new BufferedWriter(new FileWriter(prefix + "alpha_diversity" + sufix))); PrintWriter file3 = new PrintWriter( new BufferedWriter(new FileWriter(prefix + "beta_diversity" + sufix))); PrintWriter file4 = new PrintWriter(new BufferedWriter(new FileWriter(prefix + "sum" + sufix))); PrintWriter file5 = new PrintWriter( new BufferedWriter(new FileWriter(prefix + "inter_generation_distance" + sufix))); PrintWriter file6 = new PrintWriter( new BufferedWriter(new FileWriter(prefix + "environment_population_distance" + sufix))); PrintWriter file7 = new PrintWriter( new BufferedWriter(new FileWriter(prefix + "host_fitness" + sufix))); PrintWriter file8 = new PrintWriter( new BufferedWriter(new FileWriter(prefix + "cos_theta" + sufix))); PrintWriter file9 = new PrintWriter( new BufferedWriter(new FileWriter(prefix + "host_fitness_distribution" + sufix))); PrintWriter file10 = new PrintWriter( new BufferedWriter(new FileWriter(prefix + "microbiome_fitness_distribution" + sufix))); PrintWriter file11 = new PrintWriter( new BufferedWriter(new FileWriter(prefix + "bacteria_contents" + sufix))); PrintWriter file12 = new PrintWriter( new BufferedWriter(new FileWriter(prefix + "individual_bacteria_contents" + sufix))); for (int i = 0; i < Ngene; i++) { fitnessToMicrobe[i] = MathUtil.getNextInt(2) - 1; fitnessToHost[i] = MathUtil.getNextInt(2) - 1; } MathUtil.setSeed(rep % numberOfReplication); SelectiveSpeciesRegistry ssr = new SelectiveSpeciesRegistry(numberOfSpecies, Ngene, Ngenepm, msCoeff, fitnessToHost, fitnessToMicrobe); MathUtil.setSeed(); SelectivePopulation population = new SelectivePopulation(microSize, environment, populationSize, pctEnv, pctPool, 0, 0, ssr, hsCoeff, HMS_or_TMS); while (population.getNumberOfGeneration() < numberOfGeneration) { population.sumSpecies(); if (population.getNumberOfGeneration() % numberOfObservation == 0) { //file1.print(population.gammaDiversity(false)); //file2.print(population.alphaDiversity(false)); //file1.print("\t"); //file2.print("\t"); file1.println(population.gammaDiversity(true)); file2.println(population.alphaDiversity(true)); //file3.print(population.betaDiversity(true)); //file3.print("\t"); file3.println(population.BrayCurtis(true)); file4.println(population.printOut()); file5.println(population.interGenerationDistance()); file6.println(population.environmentPopulationDistance()); file7.print(population.averageHostFitness()); file7.print("\t"); file7.println(population.varianceHostFitness()); file8.println(population.cosOfMH()); file9.println(population.printOutHFitness()); file10.println(population.printOutMFitness()); file11.println(population.printBacteriaContents()); } population.getNextGen(); } for (SelectiveIndividual host : population.getIndividuals()) { file12.println(host.printBacteriaContents()); } file1.close(); file2.close(); file3.close(); file4.close(); file5.close(); file6.close(); file7.close(); file8.close(); file9.close(); file10.close(); file11.close(); file12.close(); } catch (IOException e) { e.printStackTrace(); } } }
From source file:friendsandfollowers.DBFollowersIDs.java
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check arguments that passed in if ((args == null) || (args.length == 0)) { System.err.println("2 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'OUTPUT: /output/path/'"); System.err.println("Second: (int) Number of ids to fetch. " + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Third (optional): 'screen_name / user_id_str'"); System.err.println("If 3rd argument not provided then provide" + " Twitter users through database."); System.exit(-1);/*from w w w . jav a2 s . co m*/ } MysqlDB DB = new MysqlDB(); AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/followers/ids"; String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; try { IDS_TO_FETCH_INT = Integer.parseInt(args[1]); } catch (NumberFormatException e) { System.err.println("Argument" + args[1] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } String targetedUser = ""; if (args.length == 3) { try { targetedUser = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an String."); System.exit(-1); } } try { TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); IDs ids; System.out.println("Listing followers ids."); // if targetedUser not provided by argument, then look into database. if (StringUtils.isEmpty(targetedUser)) { String selectQuery = "SELECT * FROM `followers_parent` WHERE " + "`targeteduser` != '' AND " + "`nextcursor` != '0' AND " + "`nextcursor` != '2'"; ResultSet results = DB.selectQ(selectQuery); int numRows = DB.numRows(results); if (numRows < 1) { System.err.println("No User in database to get followersIDS"); System.exit(-1); } OUTERMOST: while (results.next()) { int followers_parent_id = results.getInt("id"); targetedUser = results.getString("targeteduser"); long cursor = results.getLong("nextcursor"); System.out.println("Targeted User: " + targetedUser); int idsLoopCounter = 0; int totalIDs = 0; // put idsJSON in a file PrintWriter writer = new PrintWriter(OutputDirPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFollowersIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, // or if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Followers Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println( "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } // update cursor in "followers_parent" String fieldValues = "`nextcursor` = 2"; String where = "id = " + followers_parent_id; DB.Update("`followers_parent`", fieldValues, where); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth " + "Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); System.out.println(); // update cursor in "followers_parent" String fieldValues = "`nextcursor` = " + cursor; String where = "id = " + followers_parent_id; DB.Update("`followers_parent`", fieldValues, where); } // loop through every result found in db } else { // Second Argument Set, so we are here. System.out.println("screen_name / user_id_str passed by argument"); int idsLoopCounter = 0; int totalIDs = 0; // put idsJSON in a file PrintWriter writer = new PrintWriter( OutputDirPath + "/" + targetedUser + "_ids_" + helpers.getUnixTimeStamp(), "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); long cursor = -1; do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFollowersIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, or if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Followers Get Exception: " + te.getMessage()); } System.exit(-1); } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reach then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); System.out.println(); } } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get followers' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); }
From source file:friendsandfollowers.DBFriendsIDs.java
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check arguments that passed in if ((args == null) || (args.length == 0)) { System.err.println("2 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'OUTPUT: /output/path/'"); System.err.println("Second: (int) Number of ids to fetch. " + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Third (optional): 'screen_name / user_id_str'"); System.err.println("If 3rd argument not provided then provide" + " Twitter users through database."); System.exit(-1);// w ww . j ava 2s .co m } MysqlDB DB = new MysqlDB(); AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/friends/ids"; String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; try { IDS_TO_FETCH_INT = Integer.parseInt(args[1]); } catch (NumberFormatException e) { System.err.println("Argument" + args[1] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } String targetedUser = ""; if (args.length == 3) { try { targetedUser = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an String."); System.exit(-1); } } try { TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); IDs ids; System.out.println("Listing friends ids."); // if targetedUser not provided by argument, then look into database. if (StringUtils.isEmpty(targetedUser)) { String selectQuery = "SELECT * FROM `followings_parent` WHERE " + "`targeteduser` != '' AND " + "`nextcursor` != '0' AND " + "`nextcursor` != '2'"; ResultSet results = DB.selectQ(selectQuery); int numRows = DB.numRows(results); if (numRows < 1) { System.err.println("No User in database to get friendsIDS"); System.exit(-1); } OUTERMOST: while (results.next()) { int following_parent_id = results.getInt("id"); targetedUser = results.getString("targeteduser"); long cursor = results.getLong("nextcursor"); System.out.println("Targeted User: " + targetedUser); int idsLoopCounter = 0; int totalIDs = 0; // put idsJSON in a file PrintWriter writer = new PrintWriter(OutputDirPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFriendsIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, // or if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Friends Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println( "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } // update cursor in "followings_parent" String fieldValues = "`nextcursor` = 2"; String where = "id = " + following_parent_id; DB.Update("`followings_parent`", fieldValues, where); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user. RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth " + "Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); System.out.println(); // update cursor in "followings_parent" String fieldValues = "`nextcursor` = " + cursor; String where = "id = " + following_parent_id; DB.Update("`followings_parent`", fieldValues, where); } // loop through every result found in db } else { // Second Argument Sets, so we are here. System.out.println("screen_name / user_id_str " + "passed by argument"); int idsLoopCounter = 0; int totalIDs = 0; // put idsJSON in a file PrintWriter writer = new PrintWriter( OutputDirPath + "/" + targetedUser + "_ids_" + helpers.getUnixTimeStamp(), "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); long cursor = -1; do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFriendsIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, // or if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Friends Get Exception: " + te.getMessage()); } System.exit(-1); } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reach then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); System.out.println(); } } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get friends' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); }
From source file:friendsandfollowers.FilesThreaderFriendsIDsParser.java
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check how many arguments were passed in if ((args == null) || (args.length < 5)) { System.err.println("5 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'INPUT: DB or /input/path/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) This Job Number"); System.err.println("Fifth: (int) Number of seconds to pause"); System.err.println("Sixth: (int) Number of ids to fetch" + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 1 3 75000"); System.exit(-1);/*from ww w .j a va 2s . co m*/ } // TODO documentation for command line AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/friends/ids"; String inputPath = null; try { inputPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument " + args[0] + " must be an String."); System.exit(-1); } String outputPath = null; try { outputPath = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument " + args[1] + " must be an String."); System.exit(-1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument " + args[2] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument " + args[3] + " must be an integer."); System.exit(1); } int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; if (args.length == 6) { try { IDS_TO_FETCH_INT = Integer.parseInt(args[5]); } catch (NumberFormatException e) { System.err.println("Argument" + args[5] + " must be an integer."); System.exit(-1); } } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); try { int TotalWorkLoad = 0; ArrayList<String> allFiles = null; try { final File folder = new File(inputPath); allFiles = helpers.listFilesForSingleFolder(folder); TotalWorkLoad = allFiles.size(); } catch (Exception e) { System.err.println("Input folder is not exists: " + e.getMessage()); System.exit(-1); } System.out.println("Total Workload is: " + TotalWorkLoad); if (TotalWorkLoad < 1) { System.err.println("No screen names file exists in: " + inputPath); System.exit(-1); } if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load. Please reduce 'Number of jobs' to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; int chunkSizeToGet = (JOB_NO + 1) * chunkSize; System.out.println("My Share is " + chunkSize); System.out.println(); // Load OAuh User TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println(); IDs ids; System.out.println("Going to get friends ids."); // to write output in a file System.out.flush(); if (JOB_NO + 1 == TOTAL_JOBS) { chunkSizeToGet = TotalWorkLoad; } List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet); for (String myFile : myFilesShare) { System.out.println("Going to parse file: " + myFile); try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) { String line; OUTERMOST: while ((line = br.readLine()) != null) { // process the line. System.out.println("Going to get friends ids of Screen-name / user_id: " + line); System.out.println(); String targetedUser = line.trim(); // tmp long cursor = -1; int idsLoopCounter = 0; int totalIDs = 0; PrintWriter writer = new PrintWriter(outputPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFriendsIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, or // if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Friends Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println( "New Loaded OAuth User " + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); } // while get records from single file } // read my single file catch (IOException e) { System.err.println("Failed to read lines from " + myFile); } // to write output in a file System.out.flush(); } // all my files share } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get friends' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }
From source file:friendsandfollowers.FilesThreaderFollowersIDsParser.java
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check how many arguments were passed in if ((args == null) || (args.length < 5)) { System.err.println("5 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'INPUT: DB or /input/path/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) This Job Number"); System.err.println("Fifth: (int) Number of seconds to pause"); System.err.println("Sixth: (int) Number of ids to fetch" + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 1 3 75000"); System.exit(-1);/*from w ww .j a v a 2s.com*/ } // TODO documentation for command line AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/followers/ids"; String inputPath = null; try { inputPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument " + args[0] + " must be an String."); System.exit(-1); } String outputPath = null; try { outputPath = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument " + args[1] + " must be an String."); System.exit(-1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument " + args[2] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument " + args[3] + " must be an integer."); System.exit(1); } int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; if (args.length == 6) { try { IDS_TO_FETCH_INT = Integer.parseInt(args[5]); } catch (NumberFormatException e) { System.err.println("Argument" + args[5] + " must be an integer."); System.exit(-1); } } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); try { int TotalWorkLoad = 0; ArrayList<String> allFiles = null; try { final File folder = new File(inputPath); allFiles = helpers.listFilesForSingleFolder(folder); TotalWorkLoad = allFiles.size(); } catch (Exception e) { System.err.println("Input folder is not exists: " + e.getMessage()); System.exit(-1); } System.out.println("Total Workload is: " + TotalWorkLoad); if (TotalWorkLoad < 1) { System.err.println("No screen names file exists in: " + inputPath); System.exit(-1); } if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load. Please reduce 'Number of jobs' to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; int chunkSizeToGet = (JOB_NO + 1) * chunkSize; System.out.println("My Share is " + chunkSize); System.out.println(); // Load OAuh User TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println(); IDs ids; System.out.println("Going to get followers ids."); // to write output in a file System.out.flush(); if (JOB_NO + 1 == TOTAL_JOBS) { chunkSizeToGet = TotalWorkLoad; } List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet); for (String myFile : myFilesShare) { System.out.println("Going to parse file: " + myFile); try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) { String line; OUTERMOST: while ((line = br.readLine()) != null) { // process the line. System.out.println("Going to get followers ids of Screen-name / user_id: " + line); System.out.println(); String targetedUser = line.trim(); // tmp long cursor = -1; int idsLoopCounter = 0; int totalIDs = 0; PrintWriter writer = new PrintWriter(outputPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFollowersIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, or // if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Followers Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println( "New Loaded OAuth User " + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); } // while get records from single file } // read my single file catch (IOException e) { System.err.println("Failed to read lines from " + myFile); } // to write output in a file System.out.flush(); } // all my files share } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get followers' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }
From source file:Main.java
public static void writeToFile(File file, String content) { try {/* w ww .java 2 s . c om*/ PrintWriter out = new PrintWriter(file); out.println(content); out.close(); } catch (FileNotFoundException e) { e.printStackTrace(); } }