List of usage examples for java.lang String trim
public String trim()
From source file:module.entities.UsernameChecker.CheckOpengovUsernames.java
/** * @param args the command line arguments *///from ww w .java 2 s. c o m public static void main(String[] args) throws SQLException, IOException { // args = new String[1]; // args[0] = "searchConf.txt"; Date d = new Date(); long milTime = d.getTime(); long execStart = System.nanoTime(); Timestamp startTime = new Timestamp(milTime); long lStartTime; long lEndTime = 0; int status_id = 1; JSONObject obj = new JSONObject(); if (args.length != 1) { System.out.println("None or too many argument parameters where defined! " + "\nPlease provide ONLY the configuration file name as the only argument."); } else { try { configFile = args[0]; initLexicons(); Database.init(); lStartTime = System.currentTimeMillis(); System.out.println("Opengov username identification process started at: " + startTime); usernameCheckerId = Database.LogUsernameChecker(lStartTime); TreeMap<Integer, String> OpenGovUsernames = Database.GetOpenGovUsers(); HashSet<ReportEntry> report_names = new HashSet<>(); if (OpenGovUsernames.size() > 0) { for (int userID : OpenGovUsernames.keySet()) { String DBusername = Normalizer .normalize(OpenGovUsernames.get(userID).toUpperCase(locale), Normalizer.Form.NFD) .replaceAll("\\p{M}", ""); String username = ""; int type; String[] splitUsername = DBusername.split(" "); if (checkNameInLexicons(splitUsername)) { for (String splText : splitUsername) { username += splText + " "; } type = 1; } else if (checkOrgInLexicons(splitUsername)) { for (String splText : splitUsername) { username += splText + " "; } type = 2; } else { username = DBusername; type = -1; } ReportEntry cerEntry = new ReportEntry(userID, username.trim(), type); report_names.add(cerEntry); } status_id = 2; obj.put("message", "Opengov username checker finished with no errors"); obj.put("details", ""); Database.UpdateOpengovUsersReportName(report_names); lEndTime = System.currentTimeMillis(); } else { status_id = 2; obj.put("message", "Opengov username checker finished with no errors"); obj.put("details", "No usernames needed to be checked"); lEndTime = System.currentTimeMillis(); } } catch (Exception ex) { System.err.println(ex.getMessage()); status_id = 3; obj.put("message", "Opengov username checker encountered an error"); obj.put("details", ex.getMessage().toString()); lEndTime = System.currentTimeMillis(); } } long execEnd = System.nanoTime(); long executionTime = (execEnd - execStart); System.out.println("Total process time: " + (((executionTime / 1000000) / 1000) / 60) + " minutes."); Database.UpdateLogUsernameChecker(lEndTime, status_id, usernameCheckerId, obj); Database.closeConnection(); }
From source file:net.iiit.siel.analysis.lang.LanguageIdentifier.java
/** * The main method./*from w w w . j a v a 2 s . c o m*/ * * @param args the arguments */ public static void main(String args[]) { String usage = "Usage: LanguageIdentifier " + "[-identifyrows filename maxlines] " + "[-identifyfile charset filename] " + "[-identifyfileset charset files] " + "[-identifytext text] " + "[-identifyurl url]"; int command = 0; final int IDFILE = 1; final int IDTEXT = 2; final int IDURL = 3; final int IDFILESET = 4; final int IDROWS = 5; Vector fileset = new Vector(); String filename = ""; String charset = ""; String url = ""; String text = ""; int max = 0; // TODO niket writing test args here.. /* args = new String[2]; args[0] = "-identifyurl"; args[1] = "file:/home1/niket/TamilSamplePage.html"; //args[2] = "/home1/niket/nutch-clia/input.txt"; */ // TODO niket end here if (args.length == 0) { System.err.println(usage); System.exit(-1); } for (int i = 0; i < args.length; i++) { // parse command line if (args[i].equals("-identifyfile")) { command = IDFILE; charset = args[++i]; filename = args[++i]; } if (args[i].equals("-identifyurl")) { command = IDURL; filename = args[++i]; } if (args[i].equals("-identifyrows")) { command = IDROWS; filename = args[++i]; max = Integer.parseInt(args[++i]); } if (args[i].equals("-identifytext")) { command = IDTEXT; for (i++; i < args.length - 1; i++) text += args[i] + " "; } if (args[i].equals("-identifyfileset")) { command = IDFILESET; charset = args[++i]; for (i++; i < args.length; i++) { File[] files = null; File f = new File(args[i]); if (f.isDirectory()) { files = f.listFiles(); } else { files = new File[] { f }; } for (int j = 0; j < files.length; j++) { fileset.add(files[j].getAbsolutePath()); } } } } Configuration conf = NutchConfiguration.create(); String lang = null; LanguageIdentifier idfr = new LanguageIdentifier(conf); File f; FileInputStream fis; try { switch (command) { case IDTEXT: lang = idfr.identify(text); System.out.println("Lang :" + lang); break; case IDFILE: f = new File(filename); fis = new FileInputStream(f); lang = idfr.identify(fis, charset); fis.close(); break; case IDURL: lang = LangIdentifierUtility.IdentifyLangFromURLDirectly(filename); /* * our url identifier is confused or couldn't identify lang from * URL */ if (lang == null || lang.equalsIgnoreCase("en")) { System.out.println("Ambuguity in identifying language from URL"); } else { System.out.println("Lang was identified(using URL) as: " + lang); } break; case IDROWS: f = new File(filename); BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(f))); String line; while (max > 0 && (line = br.readLine()) != null) { line = line.trim(); if (line.length() > 2) { max--; lang = idfr.identify(line); System.out.println("R=" + lang + ":" + line); } } br.close(); System.exit(0); break; case IDFILESET: /* * used for benchs for (int j=128; j<=524288; j*=2) { long start * = System.currentTimeMillis(); idfr.analyzeLength = j; */ System.out.println("FILESET"); Iterator i = fileset.iterator(); while (i.hasNext()) { try { filename = (String) i.next(); f = new File(filename); fis = new FileInputStream(f); lang = idfr.identify(fis, charset); fis.close(); } catch (Exception e) { System.out.println(e); } System.out.println(filename + " was identified as " + lang); } /* * used for benchs System.out.println(j + "/" + * (System.currentTimeMillis()-start)); } */ System.exit(0); break; } } catch (Exception e) { System.out.println(e); System.out.println("lang could not be identified properly"); e.printStackTrace(); } System.out.println("text was identified as " + lang); /* * DONOT delete the next few lines, they should be enabled, when a lang. * mapping map needs to be generated. TODO this is for printing * the hashMapRangeLangIDTable only * * idfr.langMarkerObject.printHashmapTableWithFormatting(); * * System.out * .println("\n\n\n Printing english text contents in this file:\n"); * System.out.println(idfr.langMarkerObject.getLangCharacters( * LanguageIdentifierConstants.LangShortNames.ENGLISH * .langShortName()).toString()); * * System.out * .println("\n\n\n Printing telugu text contents in this file:\n"); * System.out.println(idfr.langMarkerObject.getLangCharacters( * LanguageIdentifierConstants.LangShortNames.TELUGU * .langShortName()).toString()); * * System.out * .println("\n\n\n Printing unknown text contents in this file:\n"); * System.out.println(idfr.langMarkerObject.getLangCharacters( * LanguageIdentifierConstants.LangShortNames.UNKNOWN_LANG * .langShortName()).toString()); */ }
From source file:net.yacy.yacy.java
/** * Main-method which is started by java. Checks for special arguments or * starts up the application.//from w w w .jav a2 s .com * * @param args * Given arguments from the command line. */ public static void main(String args[]) { try { // check assertion status //ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true); boolean assertionenabled = false; assert (assertionenabled = true) == true; // compare to true to remove warning: "Possible accidental assignement" if (assertionenabled) System.out.println("Asserts are enabled"); // check memory amount System.gc(); final long startupMemFree = MemoryControl.free(); final long startupMemTotal = MemoryControl.total(); // maybe go into headless awt mode: we have three cases depending on OS and one exception: // windows : better do not go into headless mode // mac : go into headless mode because an application is shown in gui which may not be wanted // linux : go into headless mode because this does not need any head operation // exception : if the -gui option is used then do not go into headless mode since that uses a gui boolean headless = true; if (OS.isWindows) headless = false; if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) headless = false; System.setProperty("java.awt.headless", headless ? "true" : "false"); String s = ""; for (final String a : args) s += a + " "; yacyRelease.startParameter = s.trim(); File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/')); File dataRoot = applicationRoot; //System.out.println("args.length=" + args.length); //System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]"); if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-startup") || args[0].equals("-start"))) { // normal start-up of yacy if (args.length > 1) { if (args[1].startsWith(File.separator)) { /* data root folder provided as an absolute path */ dataRoot = new File(args[1]); } else { /* data root folder provided as a path relative to the user home folder */ dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]); } } preReadSavedConfigandInit(dataRoot); startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false); } else if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) { // start-up of yacy with gui if (args.length > 1) { if (args[1].startsWith(File.separator)) { /* data root folder provided as an absolute path */ dataRoot = new File(args[1]); } else { /* data root folder provided as a path relative to the user home folder */ dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]); } } preReadSavedConfigandInit(dataRoot); startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, true); } else if ((args.length >= 1) && ((args[0].toLowerCase(Locale.ROOT).equals("-shutdown")) || (args[0].equals("-stop")))) { // normal shutdown of yacy if (args.length == 2) applicationRoot = new File(args[1]); shutdown(applicationRoot); } else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-update"))) { // aut-update yacy if (args.length == 2) applicationRoot = new File(args[1]); update(applicationRoot); } else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-version"))) { // show yacy version System.out.println(copyright); } else if ((args.length > 1) && (args[0].toLowerCase(Locale.ROOT).equals("-config"))) { // set config parameter. Special handling of adminAccount=user:pwd (generates md5 encoded password) // on Windows parameter should be enclosed in doublequotes to accept = sign (e.g. -config "port=8090" "port.ssl=8043") File f = new File(dataRoot, "DATA/SETTINGS/"); if (!f.exists()) { mkdirsIfNeseccary(f); } else { if (new File(dataRoot, "DATA/yacy.running").exists()) { System.out.println("please restart YaCy"); } } // use serverSwitch to read config properties (including init values from yacy.init serverSwitch ss = new serverSwitch(dataRoot, applicationRoot, "defaults/yacy.init", "DATA/SETTINGS/yacy.conf"); for (int icnt = 1; icnt < args.length; icnt++) { String cfg = args[icnt]; int pos = cfg.indexOf('='); if (pos > 0) { String cmd = cfg.substring(0, pos); String val = cfg.substring(pos + 1); if (!val.isEmpty()) { if (cmd.equalsIgnoreCase(SwitchboardConstants.ADMIN_ACCOUNT)) { // special command to set adminusername and md5-pwd int cpos = val.indexOf(':'); //format adminAccount=adminname:adminpwd if (cpos >= 0) { String username = val.substring(0, cpos); String pwdtxt = val.substring(cpos + 1); if (!username.isEmpty()) { ss.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, username); System.out.println("Set property " + SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME + " = " + username); } else { username = ss.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"); } ss.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "MD5:" + Digest.encodeMD5Hex(username + ":" + ss.getConfig(SwitchboardConstants.ADMIN_REALM, "YaCy") + ":" + pwdtxt)); System.out.println("Set property " + SwitchboardConstants.ADMIN_ACCOUNT_B64MD5 + " = " + ss.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); } } else { ss.setConfig(cmd, val); System.out.println("Set property " + cmd + " = " + val); } } } else { System.out.println( "skip parameter " + cfg + " (equal sign missing, put parameter in doublequotes)"); } System.out.println(); } } else { if (args.length == 1) { applicationRoot = new File(args[0]); } preReadSavedConfigandInit(dataRoot); startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false); } } finally { ConcurrentLog.shutdown(); } }
From source file:com.cws.esolutions.security.main.PasswordUtility.java
public static void main(final String[] args) { final String methodName = PasswordUtility.CNAME + "#main(final String[] args)"; if (DEBUG) {/* ww w .j a va 2 s .c o m*/ DEBUGGER.debug("Value: {}", methodName); } if (args.length == 0) { HelpFormatter usage = new HelpFormatter(); usage.printHelp(PasswordUtility.CNAME, options, true); System.exit(1); } BufferedReader bReader = null; BufferedWriter bWriter = null; try { // load service config first !! SecurityServiceInitializer.initializeService(PasswordUtility.SEC_CONFIG, PasswordUtility.LOG_CONFIG, false); if (DEBUG) { DEBUGGER.debug("Options options: {}", options); for (String arg : args) { DEBUGGER.debug("Value: {}", arg); } } CommandLineParser parser = new PosixParser(); CommandLine commandLine = parser.parse(options, args); if (DEBUG) { DEBUGGER.debug("CommandLineParser parser: {}", parser); DEBUGGER.debug("CommandLine commandLine: {}", commandLine); DEBUGGER.debug("CommandLine commandLine.getOptions(): {}", (Object[]) commandLine.getOptions()); DEBUGGER.debug("CommandLine commandLine.getArgList(): {}", commandLine.getArgList()); } final SecurityConfigurationData secConfigData = PasswordUtility.svcBean.getConfigData(); final SecurityConfig secConfig = secConfigData.getSecurityConfig(); final PasswordRepositoryConfig repoConfig = secConfigData.getPasswordRepo(); final SystemConfig systemConfig = secConfigData.getSystemConfig(); if (DEBUG) { DEBUGGER.debug("SecurityConfigurationData secConfig: {}", secConfigData); DEBUGGER.debug("SecurityConfig secConfig: {}", secConfig); DEBUGGER.debug("RepositoryConfig secConfig: {}", repoConfig); DEBUGGER.debug("SystemConfig systemConfig: {}", systemConfig); } if (commandLine.hasOption("encrypt")) { if ((StringUtils.isBlank(repoConfig.getPasswordFile())) || (StringUtils.isBlank(repoConfig.getSaltFile()))) { System.err.println("The password/salt files are not configured. Entries will not be stored!"); } File passwordFile = FileUtils.getFile(repoConfig.getPasswordFile()); File saltFile = FileUtils.getFile(repoConfig.getSaltFile()); if (DEBUG) { DEBUGGER.debug("File passwordFile: {}", passwordFile); DEBUGGER.debug("File saltFile: {}", saltFile); } final String entryName = commandLine.getOptionValue("entry"); final String username = commandLine.getOptionValue("username"); final String password = commandLine.getOptionValue("password"); final String salt = RandomStringUtils.randomAlphanumeric(secConfig.getSaltLength()); if (DEBUG) { DEBUGGER.debug("String entryName: {}", entryName); DEBUGGER.debug("String username: {}", username); DEBUGGER.debug("String password: {}", password); DEBUGGER.debug("String salt: {}", salt); } final String encodedSalt = PasswordUtils.base64Encode(salt); final String encodedUserName = PasswordUtils.base64Encode(username); final String encryptedPassword = PasswordUtils.encryptText(password, salt, secConfig.getSecretAlgorithm(), secConfig.getIterations(), secConfig.getKeyBits(), secConfig.getEncryptionAlgorithm(), secConfig.getEncryptionInstance(), systemConfig.getEncoding()); final String encodedPassword = PasswordUtils.base64Encode(encryptedPassword); if (DEBUG) { DEBUGGER.debug("String encodedSalt: {}", encodedSalt); DEBUGGER.debug("String encodedUserName: {}", encodedUserName); DEBUGGER.debug("String encodedPassword: {}", encodedPassword); } if (commandLine.hasOption("store")) { try { new File(passwordFile.getParent()).mkdirs(); new File(saltFile.getParent()).mkdirs(); boolean saltFileExists = (saltFile.exists()) ? true : saltFile.createNewFile(); if (DEBUG) { DEBUGGER.debug("saltFileExists: {}", saltFileExists); } // write the salt out first if (!(saltFileExists)) { throw new IOException("Unable to create salt file"); } boolean passwordFileExists = (passwordFile.exists()) ? true : passwordFile.createNewFile(); if (!(passwordFileExists)) { throw new IOException("Unable to create password file"); } if (commandLine.hasOption("replace")) { File[] files = new File[] { saltFile, passwordFile }; if (DEBUG) { DEBUGGER.debug("File[] files: {}", (Object) files); } for (File file : files) { if (DEBUG) { DEBUGGER.debug("File: {}", file); } String currentLine = null; File tmpFile = new File(FileUtils.getTempDirectory() + "/" + "tmpFile"); if (DEBUG) { DEBUGGER.debug("File tmpFile: {}", tmpFile); } bReader = new BufferedReader(new FileReader(file)); bWriter = new BufferedWriter(new FileWriter(tmpFile)); while ((currentLine = bReader.readLine()) != null) { if (!(StringUtils.equals(currentLine.trim().split(",")[0], entryName))) { bWriter.write(currentLine + System.getProperty("line.separator")); bWriter.flush(); } } bWriter.close(); FileUtils.deleteQuietly(file); FileUtils.copyFile(tmpFile, file); FileUtils.deleteQuietly(tmpFile); } } FileUtils.writeStringToFile(saltFile, entryName + "," + encodedUserName + "," + encodedSalt + System.getProperty("line.separator"), true); FileUtils.writeStringToFile(passwordFile, entryName + "," + encodedUserName + "," + encodedPassword + System.getProperty("line.separator"), true); } catch (IOException iox) { ERROR_RECORDER.error(iox.getMessage(), iox); } } System.out.println("Entry Name " + entryName + " stored."); } if (commandLine.hasOption("decrypt")) { String saltEntryName = null; String saltEntryValue = null; String decryptedPassword = null; String passwordEntryName = null; if ((StringUtils.isEmpty(commandLine.getOptionValue("entry")) && (StringUtils.isEmpty(commandLine.getOptionValue("username"))))) { throw new ParseException("No entry or username was provided to decrypt."); } if (StringUtils.isEmpty(commandLine.getOptionValue("username"))) { throw new ParseException("no entry provided to decrypt"); } String entryName = commandLine.getOptionValue("entry"); String username = commandLine.getOptionValue("username"); if (DEBUG) { DEBUGGER.debug("String entryName: {}", entryName); DEBUGGER.debug("String username: {}", username); } File passwordFile = FileUtils.getFile(repoConfig.getPasswordFile()); File saltFile = FileUtils.getFile(repoConfig.getSaltFile()); if (DEBUG) { DEBUGGER.debug("File passwordFile: {}", passwordFile); DEBUGGER.debug("File saltFile: {}", saltFile); } if ((!(saltFile.canRead())) || (!(passwordFile.canRead()))) { throw new IOException( "Unable to read configured password/salt file. Please check configuration and/or permissions."); } for (String lineEntry : FileUtils.readLines(saltFile, systemConfig.getEncoding())) { saltEntryName = lineEntry.split(",")[0]; if (DEBUG) { DEBUGGER.debug("String saltEntryName: {}", saltEntryName); } if (StringUtils.equals(saltEntryName, entryName)) { saltEntryValue = PasswordUtils.base64Decode(lineEntry.split(",")[2]); break; } } if (StringUtils.isEmpty(saltEntryValue)) { throw new SecurityException("No entries were found that matched the provided information"); } for (String lineEntry : FileUtils.readLines(passwordFile, systemConfig.getEncoding())) { passwordEntryName = lineEntry.split(",")[0]; if (DEBUG) { DEBUGGER.debug("String passwordEntryName: {}", passwordEntryName); } if (StringUtils.equals(passwordEntryName, saltEntryName)) { String decodedPassword = PasswordUtils.base64Decode(lineEntry.split(",")[2]); decryptedPassword = PasswordUtils.decryptText(decodedPassword, saltEntryValue, secConfig.getSecretAlgorithm(), secConfig.getIterations(), secConfig.getKeyBits(), secConfig.getEncryptionAlgorithm(), secConfig.getEncryptionInstance(), systemConfig.getEncoding()); break; } } if (StringUtils.isEmpty(decryptedPassword)) { throw new SecurityException("No entries were found that matched the provided information"); } System.out.println(decryptedPassword); } else if (commandLine.hasOption("encode")) { System.out.println(PasswordUtils.base64Encode((String) commandLine.getArgList().get(0))); } else if (commandLine.hasOption("decode")) { System.out.println(PasswordUtils.base64Decode((String) commandLine.getArgList().get(0))); } } catch (IOException iox) { ERROR_RECORDER.error(iox.getMessage(), iox); System.err.println("An error occurred during processing: " + iox.getMessage()); System.exit(1); } catch (ParseException px) { ERROR_RECORDER.error(px.getMessage(), px); System.err.println("An error occurred during processing: " + px.getMessage()); System.exit(1); } catch (SecurityException sx) { ERROR_RECORDER.error(sx.getMessage(), sx); System.err.println("An error occurred during processing: " + sx.getMessage()); System.exit(1); } catch (SecurityServiceException ssx) { ERROR_RECORDER.error(ssx.getMessage(), ssx); System.exit(1); } finally { try { if (bReader != null) { bReader.close(); } if (bWriter != null) { bReader.close(); } } catch (IOException iox) { } } System.exit(0); }
From source file:com.vmware.photon.controller.common.auth.AuthOIDCRegistrar.java
public static int main(String[] args) { Options options = new Options(); options.addOption(USERNAME_ARG, true, "Lightwave user name"); options.addOption(PASSWORD_ARG, true, "Password"); options.addOption(TARGET_ARG, true, "Registration Hostname or IPAddress"); // Possible // load-balancer // address options.addOption(MANAGEMENT_UI_REG_FILE_ARG, true, "Management UI Registration Path"); options.addOption(SWAGGER_UI_REG_FILE_ARG, true, "Swagger UI Registration Path"); options.addOption(HELP_ARG, false, "Help"); try {/*from www . ja v a 2s .c o m*/ String username = null; String password = null; String registrationAddress = null; String mgmtUiRegPath = null; String swaggerUiRegPath = null; CommandLineParser parser = new DefaultParser(); CommandLine cmd = null; cmd = parser.parse(options, args); if (cmd.hasOption(HELP_ARG)) { showUsage(options); return 0; } if (cmd.hasOption(USERNAME_ARG)) { username = cmd.getOptionValue(USERNAME_ARG); } if (cmd.hasOption(PASSWORD_ARG)) { password = cmd.getOptionValue(PASSWORD_ARG); } if (cmd.hasOption(TARGET_ARG)) { registrationAddress = cmd.getOptionValue(TARGET_ARG); } if (cmd.hasOption(MANAGEMENT_UI_REG_FILE_ARG)) { mgmtUiRegPath = cmd.getOptionValue(MANAGEMENT_UI_REG_FILE_ARG); } if (cmd.hasOption(SWAGGER_UI_REG_FILE_ARG)) { swaggerUiRegPath = cmd.getOptionValue(SWAGGER_UI_REG_FILE_ARG); } if (username == null || username.trim().isEmpty()) { throw new UsageException("Error: username is not specified"); } if (password == null) { char[] passwd = System.console().readPassword("Password:"); password = new String(passwd); } DomainInfo domainInfo = DomainInfo.build(); AuthOIDCRegistrar registrar = new AuthOIDCRegistrar(domainInfo); registrar.register(registrationAddress, username, password, mgmtUiRegPath, swaggerUiRegPath); return 0; } catch (ParseException e) { System.err.println(e.getMessage()); return ERROR_PARSE_EXCEPTION; } catch (UsageException e) { System.err.println(e.getMessage()); showUsage(options); return ERROR_USAGE_EXCEPTION; } catch (AuthException e) { System.err.println(e.getMessage()); return ERROR_AUTH_EXCEPTION; } }
From source file:module.entities.NameFinder.RegexNameFinder.java
/** * @param args the command line arguments *//* www .jav a 2 s .co m*/ public static void main(String[] args) throws SQLException, IOException { if (args.length == 1) { Config.configFile = args[0]; } long lStartTime = System.currentTimeMillis(); Timestamp startTime = new Timestamp(lStartTime); System.out.println("Regex Name Finder process started at: " + startTime); DB.initPostgres(); regexerId = DB.LogRegexFinder(lStartTime); initLexicons(); JSONObject obj = new JSONObject(); TreeMap<Integer, String> consultations = DB.getDemocracitConsultationBody(); Document doc; int count = 0; TreeMap<Integer, String> consFoundNames = new TreeMap<>(); TreeMap<Integer, String> consFoundRoles = new TreeMap<>(); for (int consId : consultations.keySet()) { String consBody = consultations.get(consId); String signName = "", roleName = ""; doc = Jsoup.parse(consBody); Elements allPars = new Elements(); Elements paragraphs = doc.select("p"); for (Element par : paragraphs) { if (par.html().contains("<br>")) { String out = "<p>" + par.html().replaceAll("<br>", "</p><p>") + "</p>"; Document internal_doc = Jsoup.parse(out); Elements subparagraphs = internal_doc.select("p"); allPars.addAll(subparagraphs); } else { allPars.add(par); } // System.out.println(formatedText); } String signature = getSignatureFromParagraphs(allPars); // System.out.println(signature); if (signature.contains("#")) { String[] sign_tokens = signature.split("#"); signName = sign_tokens[0]; if (sign_tokens.length > 1) { roleName = sign_tokens[1]; } consFoundNames.put(consId, signName.trim()); consFoundRoles.put(consId, roleName.trim()); count++; } else { System.err.println("--" + consId); } // } DB.insertDemocracitConsultationMinister(consFoundNames, consFoundRoles); TreeMap<Integer, String> consultationsCompletedText = DB.getDemocracitCompletedConsultationBody(); Document doc2; TreeMap<Integer, String> complConsFoundNames = new TreeMap<>(); int count2 = 0; for (int consId : consultationsCompletedText.keySet()) { String consBody = consultationsCompletedText.get(consId); String signName = "", roleName = ""; doc2 = Jsoup.parse(consBody); // if (doc.text().contains("<br>")) { // doc.text().replaceAll("(<[Bb][Rr]>)+", "<p>"); // } Elements allPars = new Elements(); Elements paragraphs = doc2.select("p"); for (Element par : paragraphs) { if (par.html().contains("<br>")) { String out = "<p>" + par.html().replaceAll("<br>", "</p><p>") + "</p>"; Document internal_doc = Jsoup.parse(out); Elements subparagraphs = internal_doc.select("p"); allPars.addAll(subparagraphs); } else { allPars.add(par); } } String signature = getSignatureFromParagraphs(allPars); if (signature.contains("#")) { String[] sign_tokens = signature.split("#"); signName = sign_tokens[0]; if (sign_tokens.length > 1) { roleName = sign_tokens[1]; } consFoundNames.put(consId, signName.trim()); consFoundRoles.put(consId, roleName.trim()); // System.out.println(consId); // System.out.println(signName.trim()); // System.out.println("***************"); count2++; } else { System.err.println("++" + consId); } } DB.insertDemocracitConsultationMinister(complConsFoundNames, consFoundRoles); long lEndTime = System.currentTimeMillis(); System.out.println("Regex Name Finder process finished at: " + startTime); obj.put("message", "Regex Name Finder finished with no errors"); obj.put("details", ""); DB.UpdateLogRegexFinder(lEndTime, regexerId, obj); DB.close(); }
From source file:imp.lstm.main.Driver.java
public static void main(String[] args) throws FileNotFoundException, IOException, ConfigurationException, InvalidParametersException { FileBasedConfigurationBuilder<PropertiesConfiguration> builder = new FileBasedConfigurationBuilder<>( PropertiesConfiguration.class).configure( new Parameters().properties().setFileName(args[0]).setThrowExceptionOnMissing(true) .setListDelimiterHandler(new DefaultListDelimiterHandler(';')) .setIncludesAllowed(false)); Configuration config = builder.getConfiguration(); String inputSongPath = config.getString("input_song"); String outputFolderPath = config.getString("output_folder"); String autoEncoderParamsPath = config.getString("auto_encoder_params"); String nameGeneratorParamsPath = config.getString("name_generator_params"); String queueFolderPath = config.getString("queue_folder"); String referenceQueuePath = config.getString("reference_queue", "nil"); String inputCorpusFolder = config.getString("input_corpus_folder"); boolean shouldWriteQueue = config.getBoolean("should_write_generated_queue"); boolean frankensteinTest = config.getBoolean("queue_tests_frankenstein"); boolean interpolateTest = config.getBoolean("queue_tests_interpolation"); boolean iterateOverCorpus = config.getBoolean("iterate_over_corpus", false); boolean shouldGenerateSongTitle = config.getBoolean("generate_song_title"); boolean shouldGenerateSong = config.getBoolean("generate_leadsheet"); LogTimer.initStartTime(); //start our logging timer to keep track of our execution time LogTimer.log("Creating name generator..."); //here is just silly code for generating name based on an LSTM lol $wag LSTM lstm = new LSTM(); FullyConnectedLayer fullLayer = new FullyConnectedLayer(Operations.None); Loadable titleNetLoader = new Loadable() { @Override/*from ww w . ja v a 2 s . c o m*/ public boolean load(INDArray array, String path) { String car = pathCar(path); String cdr = pathCdr(path); switch (car) { case "full": return fullLayer.load(array, cdr); case "lstm": return lstm.load(array, cdr); default: return false; } } }; LogTimer.log("Packing name generator from files..."); (new NetworkConnectomeLoader()).load(nameGeneratorParamsPath, titleNetLoader); String characterString = " !\"'[],-.01245679:?ABCDEFGHIJKLMNOPQRSTUVWYZabcdefghijklmnopqrstuvwxyz"; //Initialization LogTimer.log("Creating autoencoder..."); int inputSize = 34; int outputSize = EncodingParameters.noteEncoder.getNoteLength(); int featureVectorSize = 100; ProductCompressingAutoencoder autoencoder = new ProductCompressingAutoencoder(24, 48, 84 + 1, false); //create our network int numInterpolationDivisions = 5; //"pack" the network from weights and biases file directory LogTimer.log("Packing autoencoder from files"); (new NetworkConnectomeLoader()).load(autoEncoderParamsPath, autoencoder); File[] songFiles; if (iterateOverCorpus) { songFiles = new File(inputCorpusFolder).listFiles(); } else { songFiles = new File[] { new File(inputSongPath) }; } for (File inputFile : songFiles) { (new NetworkConnectomeLoader()).refresh(autoEncoderParamsPath, autoencoder, "initialstate"); String songTitle; if (shouldGenerateSong) { Random rand = new Random(); AVector charOut = Vector.createLength(characterString.length()); GroupedSoftMaxSampler sampler = new GroupedSoftMaxSampler( new Group[] { new Group(0, characterString.length(), true) }); songTitle = ""; for (int i = 0; i < 50; i++) { charOut = fullLayer.forward(lstm.step(charOut)); charOut = sampler.filter(charOut); int charIndex = 0; for (; charIndex < charOut.length(); charIndex++) { if (charOut.get(charIndex) == 1.0) { break; } } songTitle += characterString.substring(charIndex, charIndex + 1); } songTitle = songTitle.trim(); LogTimer.log("Generated song name: " + songTitle); } else { songTitle = "The Song We Never Name"; } LogTimer.log("Reading file..."); LeadSheetDataSequence inputSequence = LeadSheetIO.readLeadSheet(inputFile); //read our leadsheet to get a data vessel as retrieved in rbm-provisor LeadSheetDataSequence outputSequence = inputSequence.copy(); outputSequence.clearMelody(); if (interpolateTest) { LeadSheetDataSequence additionalOutput = outputSequence.copy(); for (int i = 0; i < numInterpolationDivisions; i++) { outputSequence.concat(additionalOutput.copy()); } } LeadSheetDataSequence decoderInputSequence = outputSequence.copy(); LogTimer.startLog("Encoding data..."); //TradingTimer.initStart(); //start our trading timer to keep track our our generation versus realtime play while (inputSequence.hasNext()) { //iterate through time steps in input data //TradingTimer.waitForNextTimedInput(); autoencoder.encodeStep(inputSequence.retrieve()); //feed the resultant input vector into the network if (advanceDecoding) { //if we are using advance decoding (we start decoding as soon as we can) if (autoencoder.canDecode()) { //if queue has enough data to decode from outputSequence.pushStep(null, null, autoencoder.decodeStep(decoderInputSequence.retrieve())); //take sampled data for a timestep from autoencoder //TradingTimer.logTimestep(); //log our time to TradingTimer so we can know how far ahead of realtime we are } } } LogTimer.endLog(); if (shouldWriteQueue) { String queueFilePath = queueFolderPath + java.io.File.separator + inputFile.getName().replace(".ls", ".q"); FragmentedNeuralQueue currQueue = autoencoder.getQueue(); currQueue.writeToFile(queueFilePath); LogTimer.log("Wrote queue " + inputFile.getName().replace(".ls", ".q") + " to file..."); } if (shouldGenerateSong) { if (interpolateTest) { FragmentedNeuralQueue refQueue = new FragmentedNeuralQueue(); refQueue.initFromFile(referenceQueuePath); FragmentedNeuralQueue currQueue = autoencoder.getQueue(); //currQueue.writeToFile(queueFilePath); autoencoder.setQueue(currQueue.copy()); while (autoencoder.hasDataStepsLeft()) { //we are done encoding all time steps, so just finish decoding!{ outputSequence.pushStep(null, null, autoencoder.decodeStep(decoderInputSequence.retrieve())); //take sampled data for a timestep from autoencoder //TradingTimer.logTimestep(); //log our time to TradingTimer so we can know how far ahead of realtime we are } for (int i = 1; i <= numInterpolationDivisions; i++) { System.out.println("Starting interpolation " + ((1.0 / numInterpolationDivisions) * (i))); (new NetworkConnectomeLoader()).refresh(autoEncoderParamsPath, autoencoder, "initialstate"); FragmentedNeuralQueue currCopy = currQueue.copy(); currCopy.basicInterpolate(refQueue, (1.0 / numInterpolationDivisions) * (i)); autoencoder.setQueue(currCopy); int timeStep = 0; while (autoencoder.hasDataStepsLeft()) { //we are done encoding all time steps, so just finish decoding!{ System.out.println("interpolation " + i + " step " + ++timeStep); outputSequence.pushStep(null, null, autoencoder.decodeStep(decoderInputSequence.retrieve())); //take sampled data for a timestep from autoencoder //TradingTimer.logTimestep(); //log our time to TradingTimer so we can know how far ahead of realtime we are } } } if (frankensteinTest) { LogTimer.startLog("Loading queues"); File queueFolder = new File(queueFolderPath); int numComponents = config.getInt("frankenstein_num_components", 5); int numCombinations = config.getInt("frankenstein_num_combinations", 6); double interpolationMagnitude = config.getDouble("frankenstein_magnitude", 2.0); if (queueFolder.isDirectory()) { File[] queueFiles = queueFolder.listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.contains(".q"); } }); List<File> fileList = new ArrayList<>(); for (File file : queueFiles) { fileList.add(file); } Collections.shuffle(fileList); int numSelectedFiles = (numComponents > queueFiles.length) ? queueFiles.length : numComponents; for (int i = 0; i < queueFiles.length - numSelectedFiles; i++) { fileList.remove(fileList.size() - 1); } List<FragmentedNeuralQueue> queuePopulation = new ArrayList<>(fileList.size()); songTitle += " - a mix of "; for (File file : fileList) { FragmentedNeuralQueue newQueue = new FragmentedNeuralQueue(); newQueue.initFromFile(file.getPath()); queuePopulation.add(newQueue); songTitle += file.getName().replaceAll(".ls", "") + ", "; } LogTimer.endLog(); LeadSheetDataSequence additionalOutput = outputSequence.copy(); for (int i = 1; i < numCombinations; i++) { outputSequence.concat(additionalOutput.copy()); } decoderInputSequence = outputSequence.copy(); FragmentedNeuralQueue origQueue = autoencoder.getQueue(); for (int i = 0; i < numCombinations; i++) { LogTimer.startLog("Performing queue interpolation..."); AVector combinationStrengths = Vector.createLength(queuePopulation.size()); Random vectorRand = new Random(i); for (int j = 0; j < combinationStrengths.length(); j++) { combinationStrengths.set(j, vectorRand.nextDouble()); } combinationStrengths.divide(combinationStrengths.elementSum()); FragmentedNeuralQueue currQueue = origQueue.copy(); for (int k = 0; k < combinationStrengths.length(); k++) { currQueue.basicInterpolate(queuePopulation.get(k), combinationStrengths.get(k) * interpolationMagnitude); } LogTimer.endLog(); autoencoder.setQueue(currQueue); LogTimer.startLog("Refreshing autoencoder state..."); (new NetworkConnectomeLoader()).refresh(autoEncoderParamsPath, autoencoder, "initialstate"); LogTimer.endLog(); LogTimer.startLog("Decoding segment..."); while (autoencoder.hasDataStepsLeft()) { //we are done encoding all time steps, so just finish decoding!{ outputSequence.pushStep(null, null, autoencoder.decodeStep(decoderInputSequence.retrieve())); //take sampled data for a timestep from autoencoder //TradingTimer.logTimestep(); //log our time to TradingTimer so we can know how far ahead of realtime we are } LogTimer.endLog(); } } } while (autoencoder.hasDataStepsLeft()) { //we are done encoding all time steps, so just finish decoding!{ outputSequence.pushStep(null, null, autoencoder.decodeStep(decoderInputSequence.retrieve())); //take sampled data for a timestep from autoencoder //TradingTimer.logTimestep(); //log our time to TradingTimer so we can know how far ahead of realtime we are } LogTimer.log("Writing file..."); String outputFilename = outputFolderPath + java.io.File.separator + inputFile.getName().replace(".ls", "_Output"); //we'll write our generated file with the same name plus "_Output" LeadSheetIO.writeLeadSheet(outputSequence, outputFilename, songTitle); System.out.println(outputFilename); } else { autoencoder.setQueue(new FragmentedNeuralQueue()); } } LogTimer.log("Process finished"); //Done! }
From source file:io.apicurio.studio.tools.release.ReleaseTool.java
/** * Main method.// w w w. j av a2 s .co m * @param args */ public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption("n", "release-name", true, "The name of the new release."); options.addOption("p", "prerelease", false, "Indicate that this is a pre-release."); options.addOption("t", "release-tag", true, "The tag name of the new release."); options.addOption("o", "previous-tag", true, "The tag name of the previous release."); options.addOption("g", "github-pat", true, "The GitHub PAT (for authentication/authorization)."); options.addOption("a", "artifact", true, "The binary release artifact (full path)."); options.addOption("d", "output-directory", true, "Where to store output file(s)."); CommandLineParser parser = new DefaultParser(); CommandLine cmd = parser.parse(options, args); if (!cmd.hasOption("n") || !cmd.hasOption("t") || !cmd.hasOption("o") || !cmd.hasOption("g") || !cmd.hasOption("a")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("release-studio", options); System.exit(1); } // Arguments (command line) String releaseName = cmd.getOptionValue("n"); boolean isPrerelease = cmd.hasOption("p"); String releaseTag = cmd.getOptionValue("t"); String oldReleaseTag = cmd.getOptionValue("o"); String githubPAT = cmd.getOptionValue("g"); String artifact = cmd.getOptionValue("a"); File outputDir = new File(""); if (cmd.hasOption("d")) { outputDir = new File(cmd.getOptionValue("d")); if (!outputDir.exists()) { outputDir.mkdirs(); } } File releaseArtifactFile = new File(artifact); File releaseArtifactSigFile = new File(artifact + ".asc"); String releaseArtifact = releaseArtifactFile.getName(); String releaseArtifactSig = releaseArtifactSigFile.getName(); if (!releaseArtifactFile.isFile()) { System.err.println("Missing file: " + releaseArtifactFile.getAbsolutePath()); System.exit(1); } if (!releaseArtifactSigFile.isFile()) { System.err.println("Missing file: " + releaseArtifactSigFile.getAbsolutePath()); System.exit(1); } System.out.println("========================================="); System.out.println("Creating Release: " + releaseTag); System.out.println("Previous Release: " + oldReleaseTag); System.out.println(" Name: " + releaseName); System.out.println(" Artifact: " + releaseArtifact); System.out.println(" Pre-Release: " + isPrerelease); System.out.println("========================================="); String releaseNotes = ""; // Step #1 - Generate Release Notes // * Grab info about the previous release (extract publish date) // * Query all Issues for ones closed since that date // * Generate Release Notes from the resulting Issues try { System.out.println("Getting info about release " + oldReleaseTag); HttpResponse<JsonNode> response = Unirest .get("https://api.github.com/repos/apicurio/apicurio-studio/releases/tags/v" + oldReleaseTag) .header("Accept", "application/json").header("Authorization", "token " + githubPAT).asJson(); if (response.getStatus() != 200) { throw new Exception("Failed to get old release info: " + response.getStatusText()); } JsonNode body = response.getBody(); String publishedDate = body.getObject().getString("published_at"); if (publishedDate == null) { throw new Exception("Could not find Published Date for previous release " + oldReleaseTag); } System.out.println("Release " + oldReleaseTag + " was published on " + publishedDate); List<JSONObject> issues = getIssuesForRelease(publishedDate, githubPAT); System.out.println("Found " + issues.size() + " issues closed in release " + releaseTag); System.out.println("Generating Release Notes"); releaseNotes = generateReleaseNotes(releaseName, releaseTag, issues); System.out.println("------------ Release Notes --------------"); System.out.println(releaseNotes); System.out.println("-----------------------------------------"); } catch (Exception e) { e.printStackTrace(); System.exit(1); } String assetUploadUrl = null; // Step #2 - Create a GitHub Release try { System.out.println("\nCreating GitHub Release " + releaseTag); JSONObject body = new JSONObject(); body.put("tag_name", "v" + releaseTag); body.put("name", releaseName); body.put("body", releaseNotes); body.put("prerelease", isPrerelease); HttpResponse<JsonNode> response = Unirest .post("https://api.github.com/repos/apicurio/apicurio-studio/releases") .header("Accept", "application/json").header("Content-Type", "application/json") .header("Authorization", "token " + githubPAT).body(body).asJson(); if (response.getStatus() != 201) { throw new Exception("Failed to create release in GitHub: " + response.getStatusText()); } assetUploadUrl = response.getBody().getObject().getString("upload_url"); if (assetUploadUrl == null || assetUploadUrl.trim().isEmpty()) { throw new Exception("Failed to get Asset Upload URL for newly created release!"); } } catch (Exception e) { e.printStackTrace(); System.exit(1); } // Step #3 - Upload Release Artifact (zip file) System.out.println("\nUploading Quickstart Artifact: " + releaseArtifact); try { String artifactUploadUrl = createUploadUrl(assetUploadUrl, releaseArtifact); byte[] artifactData = loadArtifactData(releaseArtifactFile); System.out.println("Uploading artifact asset: " + artifactUploadUrl); HttpResponse<JsonNode> response = Unirest.post(artifactUploadUrl).header("Accept", "application/json") .header("Content-Type", "application/zip").header("Authorization", "token " + githubPAT) .body(artifactData).asJson(); if (response.getStatus() != 201) { throw new Exception("Failed to upload asset: " + releaseArtifact, new Exception(response.getStatus() + "::" + response.getStatusText())); } Thread.sleep(1000); artifactUploadUrl = createUploadUrl(assetUploadUrl, releaseArtifactSig); artifactData = loadArtifactData(releaseArtifactSigFile); System.out.println("Uploading artifact asset: " + artifactUploadUrl); response = Unirest.post(artifactUploadUrl).header("Accept", "application/json") .header("Content-Type", "text/plain").header("Authorization", "token " + githubPAT) .body(artifactData).asJson(); if (response.getStatus() != 201) { throw new Exception("Failed to upload asset: " + releaseArtifactSig, new Exception(response.getStatus() + "::" + response.getStatusText())); } } catch (Exception e) { e.printStackTrace(); System.exit(1); } Thread.sleep(1000); // Step #4 - Download Latest Release JSON for inclusion in the project web site try { System.out.println("Getting info about the release."); HttpResponse<JsonNode> response = Unirest .get("https://api.github.com/repos/apicurio/apicurio-studio/releases/latest") .header("Accept", "application/json").asJson(); if (response.getStatus() != 200) { throw new Exception("Failed to get release info: " + response.getStatusText()); } JsonNode body = response.getBody(); String publishedDate = body.getObject().getString("published_at"); if (publishedDate == null) { throw new Exception("Could not find Published Date for release."); } String fname = publishedDate.replace(':', '-'); File outFile = new File(outputDir, fname + ".json"); System.out.println("Writing latest release info to: " + outFile.getAbsolutePath()); String output = body.getObject().toString(4); try (FileOutputStream fos = new FileOutputStream(outFile)) { fos.write(output.getBytes("UTF-8")); fos.flush(); } System.out.println("Release info successfully written."); } catch (Exception e) { e.printStackTrace(); System.exit(1); } System.out.println("========================================="); System.out.println("All Done!"); System.out.println("========================================="); }
From source file:io.yucca.lucene.IndexUtility.java
public static void main(String[] args) { CommandLineParser parser = new PosixParser(); Options options = new Options(); initOptions(options);//from ww w. java 2 s.co m try { final CommandLine line = parser.parse(options, args); if (line.hasOption('h')) { usage(options); } if (line.hasOption('s')) { String value = line.getOptionValue('s'); sourceIndexDirectory = new File(value); if (sourceIndexDirectory.exists() == false) { log.error("Index source directory: {} does not exist!", sourceIndexDirectory); System.exit(1); } } else { usage(options); System.exit(1); } if (line.hasOption('d')) { String value = line.getOptionValue('d'); destIndexDirectory = new File(value); if (destIndexDirectory.exists() == true) { log.error("Index destination directory: {} already exist", destIndexDirectory); System.exit(1); } } else { usage(options); System.exit(1); } if (line.hasOption('v')) { try { String value = line.getOptionValue('v'); version = Version.parseLeniently(value); } catch (Exception e) { log.error("Unrecognized index version, exiting"); usage(options); System.exit(1); } } if (line.hasOption('r')) { String value = line.getOptionValue('r'); String[] fields = value.trim().split(" *, *"); if (fields == null || fields.length == 0) { log.error("No fields were given, exiting"); usage(options); System.exit(1); } (new FieldRemover()).removeFields(sourceIndexDirectory, destIndexDirectory, fields, version); System.exit(0); } } catch (IndexUtilityException e) { log.error("Failed to work on index:", e); System.exit(1); } catch (MissingOptionException e) { log.error("Mandatory options is missing!"); usage(options); System.exit(1); } catch (ParseException e) { log.error("Failed to parse commandline options!"); usage(options); System.exit(1); } }
From source file:CourserankConnector.java
public static void main(String[] args) throws Exception { /////////////////////////////////////// //Tagger init //MaxentTagger tagger = new MaxentTagger("models/english-left3words-distsim.tagger"); /////w ww. ja v a 2 s.c o m //CLIENT INITIALIZATION ImportData importCourse = new ImportData(); HttpClient httpclient = new DefaultHttpClient(); httpclient = WebClientDevWrapper.wrapClient(httpclient); try { /* httpclient.getCredentialsProvider().setCredentials( new AuthScope(null, -1), new UsernamePasswordCredentials("eadrian", "eactresp1")); */ ////////////////////////////////////////////////// //Get Course Bulletin Departments page List<Course> courses = new ArrayList<Course>(); HttpGet httpget = new HttpGet("http://explorecourses.stanford.edu"); System.out.println("executing request" + httpget.getRequestLine()); HttpResponse response = httpclient.execute(httpget); HttpEntity entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); String bulletinpage = ""; //STORE RETURNED HTML TO BULLETINPAGE if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { bulletinpage += line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); /////////////////////////////////////////////////////////////////////////////// //Login to Courserank httpget = new HttpGet("https://courserank.com/stanford/main"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); String page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page += line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); //////////////////////////////////////////////////// //POST REQUEST LOGIN HttpPost post = new HttpPost("https://www.courserank.com/stanford/main"); List<NameValuePair> pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("RT", "")); pairs.add(new BasicNameValuePair("action", "login")); pairs.add(new BasicNameValuePair("password", "trespass")); pairs.add(new BasicNameValuePair("username", "eaconte@stanford.edu")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); HttpResponse resp = httpclient.execute(post); HttpEntity ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /////////////////////////////////////////////////// //THIS STEP MAY NOT BE NEEDED BUT GETS MAIN PROFILE PAGE HttpGet gethome = new HttpGet("https://www.courserank.com/stanford/home"); System.out.println("executing request" + gethome.getRequestLine()); HttpResponse gresp = httpclient.execute(gethome); HttpEntity gent = gresp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + gent.getContentLength()); InputStream i = gent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } ///////////////////////////////////////////////////////////////////////////////// //Parse Bulletin String results = getToken(bulletinpage, "RESULTS HEADER", "Additional Searches"); String[] depts = results.split("href"); //SPLIT FOR EACH DEPARTMENT LINK, ITERATE boolean ready = false; for (int i = 1; i < depts.length; i++) { //EXTRACT LINK, DEPARTMENT NAME AND ABBREVIATION String dept = new String(depts[i]); String abbr = getToken(dept, "(", ")"); String name = getToken(dept, ">", "("); name.trim(); //System.out.println(tagger.tagString(name)); String link = getToken(dept, "=\"", "\">"); System.out.println(name + " : " + abbr + " : " + link); System.out.println("======================================================================"); if (i <= 10 || i >= 127) //values to keep it to undergraduate courses. Excludes law, med, business, overseas continue; /*if (i<=46) continue; */ //Start at BIOHOP /*if (abbr.equals("INTNLREL")) ready = true; if (!ready) continue;*/ //Construct department course search URL //Then request page String URL = "http://explorecourses.stanford.edu/" + link + "&filter-term-Autumn=on&filter-term-Winter=on&filter-term-Spring=on"; httpget = new HttpGet(URL); //System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); //ystem.out.println("----------------------------------------"); //System.out.println(response.getStatusLine()); String rpage = ""; if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream in = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String line; while ((line = br.readLine()) != null) { rpage += line; //System.out.println(line); } br.close(); in.close(); } EntityUtils.consume(entity); //Process results page List<Course> deptCourses = new ArrayList<Course>(); List<Course> result = processResultPage(rpage); deptCourses.addAll(result); //While there are more result pages, keep going boolean more = (!(result.size() == 0) && (result.get((result.size() - 1)).courseNumber < 299)); boolean morepages = anotherPage(rpage); while (morepages && more) { URL = nextURL(URL); httpget = new HttpGet(URL); //System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); //System.out.println("----------------------------------------"); //System.out.println(response.getStatusLine()); rpage = ""; if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream in = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String line; while ((line = br.readLine()) != null) { rpage += line; //System.out.println(line); } br.close(); in.close(); } EntityUtils.consume(entity); morepages = anotherPage(rpage); result = processResultPage(rpage); deptCourses.addAll(result); more = (!(result.size() == 0) && (result.get((result.size() - 1)).courseNumber < 299)); /*String mores = more? "yes": "no"; String pagess = morepages?"yes":"no"; System.out.println("more: "+mores+" morepages: "+pagess); System.out.println("more");*/ } //Get course ratings for all department courses via courserank deptCourses = getRatings(httpclient, abbr, deptCourses); for (int j = 0; j < deptCourses.size(); j++) { Course c = deptCourses.get(j); System.out.println("" + c.title + " : " + c.rating); c.tags = name; c.code = c.code.trim(); c.department = name; c.deptAB = abbr; c.writeToDatabase(); //System.out.println(tagger.tagString(c.title)); } } if (!page.equals("")) return; /////////////////////////////////////////////////// //Get Course Bulletin Department courses /* httpget = new HttpGet("https://courserank.com/stanford/main"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page +=line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); //////////////////////////////////////////////////// //POST REQUEST LOGIN HttpPost post = new HttpPost("https://www.courserank.com/stanford/main"); List<NameValuePair> pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("RT", "")); pairs.add(new BasicNameValuePair("action", "login")); pairs.add(new BasicNameValuePair("password", "trespass")); pairs.add(new BasicNameValuePair("username", "eaconte@stanford.edu")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); HttpResponse resp = httpclient.execute(post); HttpEntity ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /////////////////////////////////////////////////// //THIS STEP MAY NOT BE NEEDED BUT GETS MAIN PROFILE PAGE HttpGet gethome = new HttpGet("https://www.courserank.com/stanford/home"); System.out.println("executing request" + gethome.getRequestLine()); HttpResponse gresp = httpclient.execute(gethome); HttpEntity gent = gresp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + gent.getContentLength()); InputStream i = gent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } //////////////////////////////////////// //GETS FIRST PAGE OF RESULTS EntityUtils.consume(gent); post = new HttpPost("https://www.courserank.com/stanford/search_results"); pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("filter_term_currentYear", "on")); pairs.add(new BasicNameValuePair("query", "")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); resp = httpclient.execute(post); ent = resp.getEntity(); System.out.println("----------------------------------------"); String rpage = ""; if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { rpage += line; System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); //////////////////////////////////////////////////// //PARSE FIRST PAGE OF RESULTS //int index = rpage.indexOf("div class=\"searchItem"); String []classSplit = rpage.split("div class=\"searchItem"); for (int i=1; i<classSplit.length; i++) { String str = classSplit[i]; //ID String CID = getToken(str, "course?id=","\">"); // CODE String CODE = getToken(str,"class=\"code\">" ,":</"); //TITLE String NAME = getToken(str, "class=\"title\">","</"); //DESCRIP String DES = getToken(str, "class=\"description\">","</"); //TERM String TERM = getToken(str, "Terms:", "|"); //UNITS String UNITS = getToken(str, "Units:", "<br/>"); //WORKLOAD String WLOAD = getToken(str, "Workload:", "|"); //GER String GER = getToken(str, "GERs:", "</d"); //RATING int searchIndex = 0; float rating = 0; while (true) { int ratingIndex = str.indexOf("large_Full", searchIndex); if (ratingIndex ==-1) { int halfratingIndex = str.indexOf("large_Half", searchIndex); if (halfratingIndex == -1) break; else rating += .5; break; } searchIndex = ratingIndex+1; rating++; } String RATING = ""+rating; //GRADE String GRADE = getToken(str, "div class=\"unofficialGrade\">", "</"); if (GRADE.equals("NOT FOUND")) { GRADE = getToken(str, "div class=\"officialGrade\">", "</"); } //REVIEWS String REVIEWS = getToken(str, "class=\"ratings\">", " ratings"); System.out.println(""+CODE+" : "+NAME + " : "+CID); System.out.println("----------------------------------------"); System.out.println("Term: "+TERM+" Units: "+UNITS+ " Workload: "+WLOAD + " Grade: "+ GRADE); System.out.println("Rating: "+RATING+ " Reviews: "+REVIEWS); System.out.println("=========================================="); System.out.println(DES); System.out.println("=========================================="); } /////////////////////////////////////////////////// //GETS SECOND PAGE OF RESULTS post = new HttpPost("https://www.courserank.com/stanford/search_results"); pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("filter_term_currentYear", "on")); pairs.add(new BasicNameValuePair("page", "2")); pairs.add(new BasicNameValuePair("query", "")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); resp = httpclient.execute(post); ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /* httpget = new HttpGet("https://github.com/"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page +=line; System.out.println(line); } br.close(); i.close(); }*/ EntityUtils.consume(entity); } finally { // When HttpClient instance is no longer needed, // shut down the connection manager to ensure // immediate deallocation of all system resources httpclient.getConnectionManager().shutdown(); } }