List of usage examples for java.lang String split
public String[] split(String regex)
From source file:io.anserini.index.IndexTweets.java
@SuppressWarnings("static-access") public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption(new Option(HELP_OPTION, "show help")); options.addOption(new Option(OPTIMIZE_OPTION, "merge indexes into a single segment")); options.addOption(new Option(STORE_TERM_VECTORS_OPTION, "store term vectors")); options.addOption(OptionBuilder.withArgName("dir").hasArg().withDescription("source collection directory") .create(COLLECTION_OPTION)); options.addOption(/* w ww .j av a 2s.c o m*/ OptionBuilder.withArgName("dir").hasArg().withDescription("index location").create(INDEX_OPTION)); options.addOption(OptionBuilder.withArgName("file").hasArg().withDescription("file with deleted tweetids") .create(DELETES_OPTION)); options.addOption(OptionBuilder.withArgName("id").hasArg().withDescription("max id").create(MAX_ID_OPTION)); CommandLine cmdline = null; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); System.exit(-1); } if (cmdline.hasOption(HELP_OPTION) || !cmdline.hasOption(COLLECTION_OPTION) || !cmdline.hasOption(INDEX_OPTION)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(IndexTweets.class.getName(), options); System.exit(-1); } String collectionPath = cmdline.getOptionValue(COLLECTION_OPTION); String indexPath = cmdline.getOptionValue(INDEX_OPTION); final FieldType textOptions = new FieldType(); textOptions.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); textOptions.setStored(true); textOptions.setTokenized(true); if (cmdline.hasOption(STORE_TERM_VECTORS_OPTION)) { textOptions.setStoreTermVectors(true); } LOG.info("collection: " + collectionPath); LOG.info("index: " + indexPath); LongOpenHashSet deletes = null; if (cmdline.hasOption(DELETES_OPTION)) { deletes = new LongOpenHashSet(); File deletesFile = new File(cmdline.getOptionValue(DELETES_OPTION)); if (!deletesFile.exists()) { System.err.println("Error: " + deletesFile + " does not exist!"); System.exit(-1); } LOG.info("Reading deletes from " + deletesFile); FileInputStream fin = new FileInputStream(deletesFile); byte[] ignoreBytes = new byte[2]; fin.read(ignoreBytes); // "B", "Z" bytes from commandline tools BufferedReader br = new BufferedReader(new InputStreamReader(new CBZip2InputStream(fin))); String s; while ((s = br.readLine()) != null) { if (s.contains("\t")) { deletes.add(Long.parseLong(s.split("\t")[0])); } else { deletes.add(Long.parseLong(s)); } } br.close(); fin.close(); LOG.info("Read " + deletes.size() + " tweetids from deletes file."); } long maxId = Long.MAX_VALUE; if (cmdline.hasOption(MAX_ID_OPTION)) { maxId = Long.parseLong(cmdline.getOptionValue(MAX_ID_OPTION)); LOG.info("index: " + maxId); } long startTime = System.currentTimeMillis(); File file = new File(collectionPath); if (!file.exists()) { System.err.println("Error: " + file + " does not exist!"); System.exit(-1); } StatusStream stream = new JsonStatusCorpusReader(file); Directory dir = FSDirectory.open(Paths.get(indexPath)); final IndexWriterConfig config = new IndexWriterConfig(ANALYZER); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); IndexWriter writer = new IndexWriter(dir, config); int cnt = 0; Status status; try { while ((status = stream.next()) != null) { if (status.getText() == null) { continue; } // Skip deletes tweetids. if (deletes != null && deletes.contains(status.getId())) { continue; } if (status.getId() > maxId) { continue; } cnt++; Document doc = new Document(); doc.add(new LongPoint(StatusField.ID.name, status.getId())); doc.add(new StoredField(StatusField.ID.name, status.getId())); doc.add(new LongPoint(StatusField.EPOCH.name, status.getEpoch())); doc.add(new StoredField(StatusField.EPOCH.name, status.getEpoch())); doc.add(new TextField(StatusField.SCREEN_NAME.name, status.getScreenname(), Store.YES)); doc.add(new Field(StatusField.TEXT.name, status.getText(), textOptions)); doc.add(new IntPoint(StatusField.FRIENDS_COUNT.name, status.getFollowersCount())); doc.add(new StoredField(StatusField.FRIENDS_COUNT.name, status.getFollowersCount())); doc.add(new IntPoint(StatusField.FOLLOWERS_COUNT.name, status.getFriendsCount())); doc.add(new StoredField(StatusField.FOLLOWERS_COUNT.name, status.getFriendsCount())); doc.add(new IntPoint(StatusField.STATUSES_COUNT.name, status.getStatusesCount())); doc.add(new StoredField(StatusField.STATUSES_COUNT.name, status.getStatusesCount())); long inReplyToStatusId = status.getInReplyToStatusId(); if (inReplyToStatusId > 0) { doc.add(new LongPoint(StatusField.IN_REPLY_TO_STATUS_ID.name, inReplyToStatusId)); doc.add(new StoredField(StatusField.IN_REPLY_TO_STATUS_ID.name, inReplyToStatusId)); doc.add(new LongPoint(StatusField.IN_REPLY_TO_USER_ID.name, status.getInReplyToUserId())); doc.add(new StoredField(StatusField.IN_REPLY_TO_USER_ID.name, status.getInReplyToUserId())); } String lang = status.getLang(); if (!lang.equals("unknown")) { doc.add(new TextField(StatusField.LANG.name, status.getLang(), Store.YES)); } long retweetStatusId = status.getRetweetedStatusId(); if (retweetStatusId > 0) { doc.add(new LongPoint(StatusField.RETWEETED_STATUS_ID.name, retweetStatusId)); doc.add(new StoredField(StatusField.RETWEETED_STATUS_ID.name, retweetStatusId)); doc.add(new LongPoint(StatusField.RETWEETED_USER_ID.name, status.getRetweetedUserId())); doc.add(new StoredField(StatusField.RETWEETED_USER_ID.name, status.getRetweetedUserId())); doc.add(new IntPoint(StatusField.RETWEET_COUNT.name, status.getRetweetCount())); doc.add(new StoredField(StatusField.RETWEET_COUNT.name, status.getRetweetCount())); if (status.getRetweetCount() < 0 || status.getRetweetedStatusId() < 0) { LOG.warn("Error parsing retweet fields of " + status.getId()); } } writer.addDocument(doc); if (cnt % 100000 == 0) { LOG.info(cnt + " statuses indexed"); } } LOG.info(String.format("Total of %s statuses added", cnt)); if (cmdline.hasOption(OPTIMIZE_OPTION)) { LOG.info("Merging segments..."); writer.forceMerge(1); LOG.info("Done!"); } LOG.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms"); } catch (Exception e) { e.printStackTrace(); } finally { writer.close(); dir.close(); stream.close(); } }
From source file:module.entities.UsernameChecker.CheckOpengovUsernames.java
/** * @param args the command line arguments *//*from w ww. ja v a2 s . c o m*/ public static void main(String[] args) throws SQLException, IOException { // args = new String[1]; // args[0] = "searchConf.txt"; Date d = new Date(); long milTime = d.getTime(); long execStart = System.nanoTime(); Timestamp startTime = new Timestamp(milTime); long lStartTime; long lEndTime = 0; int status_id = 1; JSONObject obj = new JSONObject(); if (args.length != 1) { System.out.println("None or too many argument parameters where defined! " + "\nPlease provide ONLY the configuration file name as the only argument."); } else { try { configFile = args[0]; initLexicons(); Database.init(); lStartTime = System.currentTimeMillis(); System.out.println("Opengov username identification process started at: " + startTime); usernameCheckerId = Database.LogUsernameChecker(lStartTime); TreeMap<Integer, String> OpenGovUsernames = Database.GetOpenGovUsers(); HashSet<ReportEntry> report_names = new HashSet<>(); if (OpenGovUsernames.size() > 0) { for (int userID : OpenGovUsernames.keySet()) { String DBusername = Normalizer .normalize(OpenGovUsernames.get(userID).toUpperCase(locale), Normalizer.Form.NFD) .replaceAll("\\p{M}", ""); String username = ""; int type; String[] splitUsername = DBusername.split(" "); if (checkNameInLexicons(splitUsername)) { for (String splText : splitUsername) { username += splText + " "; } type = 1; } else if (checkOrgInLexicons(splitUsername)) { for (String splText : splitUsername) { username += splText + " "; } type = 2; } else { username = DBusername; type = -1; } ReportEntry cerEntry = new ReportEntry(userID, username.trim(), type); report_names.add(cerEntry); } status_id = 2; obj.put("message", "Opengov username checker finished with no errors"); obj.put("details", ""); Database.UpdateOpengovUsersReportName(report_names); lEndTime = System.currentTimeMillis(); } else { status_id = 2; obj.put("message", "Opengov username checker finished with no errors"); obj.put("details", "No usernames needed to be checked"); lEndTime = System.currentTimeMillis(); } } catch (Exception ex) { System.err.println(ex.getMessage()); status_id = 3; obj.put("message", "Opengov username checker encountered an error"); obj.put("details", ex.getMessage().toString()); lEndTime = System.currentTimeMillis(); } } long execEnd = System.nanoTime(); long executionTime = (execEnd - execStart); System.out.println("Total process time: " + (((executionTime / 1000000) / 1000) / 60) + " minutes."); Database.UpdateLogUsernameChecker(lEndTime, status_id, usernameCheckerId, obj); Database.closeConnection(); }
From source file:org.kuali.student.git.importer.ConvertBuildTagBranchesToGitTags.java
/** * @param args// w ww . j ava 2 s.c o m */ public static void main(String[] args) { if (args.length < 3 || args.length > 6) { System.err.println("USAGE: <git repository> <bare> <ref mode> [<ref prefix> <username> <password>]"); System.err.println("\t<bare> : 0 (false) or 1 (true)"); System.err.println("\t<ref mode> : local or name of remote"); System.err.println("\t<ref prefix> : refs/heads (default) or say refs/remotes/origin (test clone)"); System.exit(-1); } boolean bare = false; if (args[1].trim().equals("1")) { bare = true; } String remoteName = args[2].trim(); String refPrefix = Constants.R_HEADS; if (args.length == 4) refPrefix = args[3].trim(); String userName = null; String password = null; if (args.length == 5) userName = args[4].trim(); if (args.length == 6) password = args[5].trim(); try { Repository repo = GitRepositoryUtils.buildFileRepository(new File(args[0]).getAbsoluteFile(), false, bare); Git git = new Git(repo); ObjectInserter objectInserter = repo.newObjectInserter(); Collection<Ref> repositoryHeads = repo.getRefDatabase().getRefs(refPrefix).values(); RevWalk rw = new RevWalk(repo); Map<String, ObjectId> tagNameToTagId = new HashMap<>(); Map<String, Ref> tagNameToRef = new HashMap<>(); for (Ref ref : repositoryHeads) { String branchName = ref.getName().substring(refPrefix.length() + 1); if (branchName.contains("tag") && branchName.contains("builds")) { String branchParts[] = branchName.split("_"); int buildsIndex = ArrayUtils.indexOf(branchParts, "builds"); String moduleName = StringUtils.join(branchParts, "_", buildsIndex + 1, branchParts.length); RevCommit commit = rw.parseCommit(ref.getObjectId()); ObjectId tag = GitRefUtils.insertTag(moduleName, commit, objectInserter); tagNameToTagId.put(moduleName, tag); tagNameToRef.put(moduleName, ref); } } BatchRefUpdate batch = repo.getRefDatabase().newBatchUpdate(); List<RefSpec> branchesToDelete = new ArrayList<>(); for (Entry<String, ObjectId> entry : tagNameToTagId.entrySet()) { String tagName = entry.getKey(); // create the reference to the tag object batch.addCommand( new ReceiveCommand(null, entry.getValue(), Constants.R_TAGS + tagName, Type.CREATE)); // delete the original branch object Ref branch = tagNameToRef.get(entry.getKey()); if (remoteName.equals("local")) { batch.addCommand(new ReceiveCommand(branch.getObjectId(), null, branch.getName(), Type.DELETE)); } else { String adjustedBranchName = branch.getName().substring(refPrefix.length() + 1); branchesToDelete.add(new RefSpec(":" + Constants.R_HEADS + adjustedBranchName)); } } // create the tags batch.execute(rw, new TextProgressMonitor()); if (!remoteName.equals("local")) { // push the tag to the remote right now PushCommand pushCommand = git.push().setRemote(remoteName).setPushTags() .setProgressMonitor(new TextProgressMonitor()); if (userName != null) pushCommand.setCredentialsProvider(new UsernamePasswordCredentialsProvider(userName, password)); Iterable<PushResult> results = pushCommand.call(); for (PushResult pushResult : results) { if (!pushResult.equals(Result.NEW)) { log.warn("failed to push tag " + pushResult.getMessages()); } } // delete the branches from the remote results = git.push().setRemote(remoteName).setRefSpecs(branchesToDelete) .setProgressMonitor(new TextProgressMonitor()).call(); log.info(""); } // Result result = GitRefUtils.createTagReference(repo, moduleName, tag); // // if (!result.equals(Result.NEW)) { // log.warn("failed to create tag {} for branch {}", moduleName, branchName); // continue; // } // // if (deleteMode) { // result = GitRefUtils.deleteRef(repo, ref); // // if (!result.equals(Result.NEW)) { // log.warn("failed to delete branch {}", branchName); // continue; // } objectInserter.release(); rw.release(); } catch (Exception e) { log.error("unexpected Exception ", e); } }
From source file:akori.AKORI.java
public static void main(String[] args) throws IOException, InterruptedException { System.out.println("esto es AKORI"); URL = "http://www.mbauchile.cl"; PATH = "E:\\NetBeansProjects\\AKORI\\"; NAME = "mbauchile.png"; // Extrar DOM tree Document doc = Jsoup.connect(URL).timeout(0).get(); // The Firefox driver supports javascript WebDriver driver = new FirefoxDriver(); driver.manage().window().maximize(); System.out.println(driver.manage().window().getSize().toString()); System.out.println(driver.manage().window().getPosition().toString()); int xmax = driver.manage().window().getSize().width; int ymax = driver.manage().window().getSize().height; // Go to the URL page driver.get(URL);// w w w . j av a 2 s. com File screen = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE); FileUtils.copyFile(screen, new File(PATH + NAME)); BufferedImage img = ImageIO.read(new File(PATH + NAME)); //Graphics2D graph = img.createGraphics(); BufferedImage img1 = new BufferedImage(xmax, ymax, BufferedImage.TYPE_INT_ARGB); Graphics2D graph1 = img.createGraphics(); double[][] matrix = new double[ymax][xmax]; BufferedReader in = new BufferedReader(new FileReader("et.txt")); String linea; double max = 0; graph1.drawImage(img, 0, 0, null); HashMap<String, Integer> lista = new HashMap<String, Integer>(); int count = 0; for (int i = 0; (linea = in.readLine()) != null && i < 10000; ++i) { String[] datos = linea.split(","); int x = (int) Double.parseDouble(datos[0]); int y = (int) Double.parseDouble(datos[2]); long time = Double.valueOf(datos[4]).longValue(); if (x >= xmax || y >= ymax) continue; if (time < 691215) continue; if (time > 705648) break; if (lista.containsKey(x + "," + y)) lista.put(x + "," + y, lista.get(x + "," + y) + 1); else lista.put(x + "," + y, 1); ++count; } System.out.println(count); in.close(); Iterator iter = lista.entrySet().iterator(); Map.Entry e; for (String key : lista.keySet()) { Integer i = lista.get(key); if (max < i) max = i; } System.out.println(max); max = 0; while (iter.hasNext()) { e = (Map.Entry) iter.next(); String xy = (String) e.getKey(); String[] datos = xy.split(","); int x = Integer.parseInt(datos[0]); int y = Integer.parseInt(datos[1]); matrix[y][x] += (int) e.getValue(); double aux; if ((aux = normalMatrix(matrix, y, x, ((int) e.getValue()) * 4)) > max) { max = aux; } //normalMatrix(matrix,x,y,20); if (matrix[y][x] > max) max = matrix[y][x]; } int A, R, G, B, n; for (int i = 0; i < xmax; ++i) { for (int j = 0; j < ymax; ++j) { if (matrix[j][i] != 0) { n = (int) Math.round(matrix[j][i] * 100 / max); R = Math.round((255 * n) / 100); G = Math.round((255 * (100 - n)) / 100); B = 0; A = Math.round((255 * n) / 100); ; if (R > 255) R = 255; if (R < 0) R = 0; if (G > 255) G = 255; if (G < 0) G = 0; if (R < 50) A = 0; graph1.setColor(new Color(R, G, B, A)); graph1.fillOval(i, j, 1, 1); } } } //graph1.dispose(); ImageIO.write(img, "png", new File("example.png")); System.out.println(max); graph1.setColor(Color.RED); // Extraer elementos Elements e1 = doc.body().getAllElements(); int i = 1; ArrayList<String> tags = new ArrayList<String>(); for (Element temp : e1) { if (tags.indexOf(temp.tagName()) == -1) { tags.add(temp.tagName()); List<WebElement> query = driver.findElements(By.tagName(temp.tagName())); for (WebElement temp1 : query) { Point po = temp1.getLocation(); Dimension d = temp1.getSize(); if (d.width <= 0 || d.height <= 0 || po.x < 0 || po.y < 0) continue; System.out.println(i + " " + temp.nodeName()); System.out.println(" x: " + po.x + " y: " + po.y); System.out.println(" width: " + d.width + " height: " + d.height); graph1.draw(new Rectangle(po.x, po.y, d.width, d.height)); ++i; } } } graph1.dispose(); ImageIO.write(img, "png", new File(PATH + NAME)); driver.quit(); }
From source file:it.anyplace.sync.client.Main.java
public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption("C", "set-config", true, "set config file for s-client"); options.addOption("c", "config", false, "dump config"); options.addOption("sp", "set-peers", true, "set peer, or comma-separated list of peers"); options.addOption("q", "query", true, "query directory server for device id"); options.addOption("d", "discovery", true, "discovery local network for device id"); options.addOption("p", "pull", true, "pull file from network"); options.addOption("P", "push", true, "push file to network"); options.addOption("o", "output", true, "set output file/directory"); options.addOption("i", "input", true, "set input file/directory"); options.addOption("lp", "list-peers", false, "list peer addresses"); options.addOption("a", "address", true, "use this peer addresses"); options.addOption("L", "list-remote", false, "list folder (root) content from network"); options.addOption("I", "list-info", false, "dump folder info from network"); options.addOption("li", "list-info", false, "list folder info from local db"); // options.addOption("l", "list-local", false, "list folder content from local (saved) index"); options.addOption("s", "search", true, "search local index for <term>"); options.addOption("D", "delete", true, "push delete to network"); options.addOption("M", "mkdir", true, "push directory create to network"); options.addOption("h", "help", false, "print help"); CommandLineParser parser = new DefaultParser(); CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("h")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("s-client", options); return;/*from ww w . jav a2s. co m*/ } File configFile = cmd.hasOption("C") ? new File(cmd.getOptionValue("C")) : new File(System.getProperty("user.home"), ".s-client.properties"); logger.info("using config file = {}", configFile); ConfigurationService configuration = ConfigurationService.newLoader().loadFrom(configFile); FileUtils.cleanDirectory(configuration.getTemp()); KeystoreHandler.newLoader().loadAndStore(configuration); if (cmd.hasOption("c")) { logger.info("configuration =\n{}", configuration.newWriter().dumpToString()); } else { logger.trace("configuration =\n{}", configuration.newWriter().dumpToString()); } logger.debug("{}", configuration.getStorageInfo().dumpAvailableSpace()); if (cmd.hasOption("sp")) { List<String> peers = Lists.newArrayList(Lists.transform( Arrays.<String>asList(cmd.getOptionValue("sp").split(",")), new Function<String, String>() { @Override public String apply(String input) { return input.trim(); } })); logger.info("set peers = {}", peers); configuration.edit().setPeers(Collections.<DeviceInfo>emptyList()); for (String peer : peers) { KeystoreHandler.validateDeviceId(peer); configuration.edit().addPeers(new DeviceInfo(peer, null)); } configuration.edit().persistNow(); } if (cmd.hasOption("q")) { String deviceId = cmd.getOptionValue("q"); logger.info("query device id = {}", deviceId); List<DeviceAddress> deviceAddresses = new GlobalDiscoveryHandler(configuration).query(deviceId); logger.info("server response = {}", deviceAddresses); } if (cmd.hasOption("d")) { String deviceId = cmd.getOptionValue("d"); logger.info("discovery device id = {}", deviceId); List<DeviceAddress> deviceAddresses = new LocalDiscorveryHandler(configuration).queryAndClose(deviceId); logger.info("local response = {}", deviceAddresses); } if (cmd.hasOption("p")) { String path = cmd.getOptionValue("p"); logger.info("file path = {}", path); String folder = path.split(":")[0]; path = path.split(":")[1]; try (SyncthingClient client = new SyncthingClient(configuration); BlockExchangeConnectionHandler connectionHandler = client.connectToBestPeer()) { InputStream inputStream = client.pullFile(connectionHandler, folder, path).waitForComplete() .getInputStream(); String fileName = client.getIndexHandler().getFileInfoByPath(folder, path).getFileName(); File file; if (cmd.hasOption("o")) { File param = new File(cmd.getOptionValue("o")); file = param.isDirectory() ? new File(param, fileName) : param; } else { file = new File(fileName); } FileUtils.copyInputStreamToFile(inputStream, file); logger.info("saved file to = {}", file.getAbsolutePath()); } } if (cmd.hasOption("P")) { String path = cmd.getOptionValue("P"); File file = new File(cmd.getOptionValue("i")); checkArgument(!path.startsWith("/")); //TODO check path syntax logger.info("file path = {}", path); String folder = path.split(":")[0]; path = path.split(":")[1]; try (SyncthingClient client = new SyncthingClient(configuration); BlockPusher.FileUploadObserver fileUploadObserver = client.pushFile(new FileInputStream(file), folder, path)) { while (!fileUploadObserver.isCompleted()) { fileUploadObserver.waitForProgressUpdate(); logger.debug("upload progress {}", fileUploadObserver.getProgressMessage()); } logger.info("uploaded file to network"); } } if (cmd.hasOption("D")) { String path = cmd.getOptionValue("D"); String folder = path.split(":")[0]; path = path.split(":")[1]; logger.info("delete path = {}", path); try (SyncthingClient client = new SyncthingClient(configuration); IndexEditObserver observer = client.pushDelete(folder, path)) { observer.waitForComplete(); logger.info("deleted path"); } } if (cmd.hasOption("M")) { String path = cmd.getOptionValue("M"); String folder = path.split(":")[0]; path = path.split(":")[1]; logger.info("dir path = {}", path); try (SyncthingClient client = new SyncthingClient(configuration); IndexEditObserver observer = client.pushDir(folder, path)) { observer.waitForComplete(); logger.info("uploaded dir to network"); } } if (cmd.hasOption("L")) { try (SyncthingClient client = new SyncthingClient(configuration)) { client.waitForRemoteIndexAquired(); for (String folder : client.getIndexHandler().getFolderList()) { try (IndexBrowser indexBrowser = client.getIndexHandler().newIndexBrowserBuilder() .setFolder(folder).build()) { logger.info("list folder = {}", indexBrowser.getFolder()); for (FileInfo fileInfo : indexBrowser.listFiles()) { logger.info("\t\t{} {} {}", fileInfo.getType().name().substring(0, 1), fileInfo.getPath(), fileInfo.describeSize()); } } } } } if (cmd.hasOption("I")) { try (SyncthingClient client = new SyncthingClient(configuration)) { if (cmd.hasOption("a")) { String deviceId = cmd.getOptionValue("a").substring(0, 63), address = cmd.getOptionValue("a").substring(64); try (BlockExchangeConnectionHandler connection = client.getConnection( DeviceAddress.newBuilder().setDeviceId(deviceId).setAddress(address).build())) { client.getIndexHandler().waitForRemoteIndexAquired(connection); } } else { client.waitForRemoteIndexAquired(); } String folderInfo = ""; for (String folder : client.getIndexHandler().getFolderList()) { folderInfo += "\n\t\tfolder info : " + client.getIndexHandler().getFolderInfo(folder); folderInfo += "\n\t\tfolder stats : " + client.getIndexHandler().newFolderBrowser().getFolderStats(folder).dumpInfo() + "\n"; } logger.info("folders:\n{}\n", folderInfo); } } if (cmd.hasOption("li")) { try (SyncthingClient client = new SyncthingClient(configuration)) { String folderInfo = ""; for (String folder : client.getIndexHandler().getFolderList()) { folderInfo += "\n\t\tfolder info : " + client.getIndexHandler().getFolderInfo(folder); folderInfo += "\n\t\tfolder stats : " + client.getIndexHandler().newFolderBrowser().getFolderStats(folder).dumpInfo() + "\n"; } logger.info("folders:\n{}\n", folderInfo); } } if (cmd.hasOption("lp")) { try (SyncthingClient client = new SyncthingClient(configuration); DeviceAddressSupplier deviceAddressSupplier = client.getDiscoveryHandler() .newDeviceAddressSupplier()) { String deviceAddressesStr = ""; for (DeviceAddress deviceAddress : Lists.newArrayList(deviceAddressSupplier)) { deviceAddressesStr += "\n\t\t" + deviceAddress.getDeviceId() + " : " + deviceAddress.getAddress(); } logger.info("device addresses:\n{}\n", deviceAddressesStr); } } if (cmd.hasOption("s")) { String term = cmd.getOptionValue("s"); try (SyncthingClient client = new SyncthingClient(configuration); IndexFinder indexFinder = client.getIndexHandler().newIndexFinderBuilder().build()) { client.waitForRemoteIndexAquired(); logger.info("search term = '{}'", term); IndexFinder.SearchCompletedEvent event = indexFinder.doSearch(term); if (event.hasGoodResults()) { logger.info("search results for term = '{}' :", term); for (FileInfo fileInfo : event.getResultList()) { logger.info("\t\t{} {} {}", fileInfo.getType().name().substring(0, 1), fileInfo.getPath(), fileInfo.describeSize()); } } else if (event.hasTooManyResults()) { logger.info("too many results found for term = '{}'", term); } else { logger.info("no result found for term = '{}'", term); } } } // if (cmd.hasOption("l")) { // String indexDump = new IndexHandler(configuration).dumpIndex(); // logger.info("index dump = \n\n{}\n", indexDump); // } IOUtils.closeQuietly(configuration); }
From source file:mlbench.pagerank.PagerankNaive.java
@SuppressWarnings({ "rawtypes", "unchecked" })
public static void main(String[] args) throws IOException, InterruptedException {
try {//from w ww . j a v a 2s . c o m
parseArgs(args);
HashMap<String, String> conf = new HashMap<String, String>();
initConf(conf);
MPI_D.Init(args, MPI_D.Mode.Common, conf);
JobConf jobConf = new JobConf(confPath);
if (MPI_D.COMM_BIPARTITE_O != null) {
// O communicator
int rank = MPI_D.Comm_rank(MPI_D.COMM_BIPARTITE_O);
int size = MPI_D.Comm_size(MPI_D.COMM_BIPARTITE_O);
if (rank == 0) {
LOG.info(PagerankNaive.class.getSimpleName() + " O start.");
}
FileSplit[] inputs1 = DataMPIUtil.HDFSDataLocalLocator.getTaskInputs(MPI_D.COMM_BIPARTITE_O,
jobConf, edgeDir, rank);
FileSplit[] inputs2 = DataMPIUtil.HDFSDataLocalLocator.getTaskInputs(MPI_D.COMM_BIPARTITE_O,
jobConf, vecDir, rank);
FileSplit[] inputs = (FileSplit[]) ArrayUtils.addAll(inputs2, inputs1);
for (int i = 0; i < inputs.length; i++) {
FileSplit fsplit = inputs[i];
LineRecordReader kvrr = new LineRecordReader(jobConf, fsplit);
LongWritable key = kvrr.createKey();
Text value = kvrr.createValue();
{
IntWritable k = new IntWritable();
Text v = new Text();
while (kvrr.next(key, value)) {
String line_text = value.toString();
// ignore comments in edge file
if (line_text.startsWith("#"))
continue;
final String[] line = line_text.split("\t");
if (line.length < 2)
continue;
// vector : ROWID VALUE('vNNNN')
if (line[1].charAt(0) == 'v') {
k.set(Integer.parseInt(line[0]));
v.set(line[1]);
MPI_D.Send(k, v);
} else {
/*
* In other matrix-vector multiplication, we
* output (dst, src) here However, In PageRank,
* the matrix-vector computation formula is M^T
* * v. Therefore, we output (src,dst) here.
*/
int src_id = Integer.parseInt(line[0]);
int dst_id = Integer.parseInt(line[1]);
k.set(src_id);
v.set(line[1]);
MPI_D.Send(k, v);
if (make_symmetric == 1) {
k.set(dst_id);
v.set(line[0]);
MPI_D.Send(k, v);
}
}
}
}
}
} else if (MPI_D.COMM_BIPARTITE_A != null) {
// A communicator
int rank = MPI_D.Comm_rank(MPI_D.COMM_BIPARTITE_A);
if (rank == 0) {
LOG.info(PagerankNaive.class.getSimpleName() + " A start.");
}
HadoopWriter<IntWritable, Text> outrw = HadoopIOUtil.getNewWriter(jobConf, outDir,
IntWritable.class, Text.class, TextOutputFormat.class, null, rank, MPI_D.COMM_BIPARTITE_A);
IntWritable oldKey = null;
int i;
double cur_rank = 0;
ArrayList<Integer> dst_nodes_list = new ArrayList<Integer>();
Object[] keyValue = MPI_D.Recv();
while (keyValue != null) {
IntWritable key = (IntWritable) keyValue[0];
Text value = (Text) keyValue[1];
if (oldKey == null) {
oldKey = key;
}
// A new key arrives
if (!key.equals(oldKey)) {
outrw.write(oldKey, new Text("s" + cur_rank));
int outdeg = dst_nodes_list.size();
if (outdeg > 0) {
cur_rank = cur_rank / (double) outdeg;
}
for (i = 0; i < outdeg; i++) {
outrw.write(new IntWritable(dst_nodes_list.get(i)), new Text("v" + cur_rank));
}
oldKey = key;
cur_rank = 0;
dst_nodes_list = new ArrayList<Integer>();
}
// common record
String line_text = value.toString();
final String[] line = line_text.split("\t");
if (line.length == 1) {
if (line_text.charAt(0) == 'v') { // vector : VALUE
cur_rank = Double.parseDouble(line_text.substring(1));
} else { // edge : ROWID
dst_nodes_list.add(Integer.parseInt(line[0]));
}
}
keyValue = MPI_D.Recv();
}
// write the left part
if (cur_rank != 0) {
outrw.write(oldKey, new Text("s" + cur_rank));
int outdeg = dst_nodes_list.size();
if (outdeg > 0) {
cur_rank = cur_rank / (double) outdeg;
}
for (i = 0; i < outdeg; i++) {
outrw.write(new IntWritable(dst_nodes_list.get(i)), new Text("v" + cur_rank));
}
}
outrw.close();
}
MPI_D.Finalize();
} catch (MPI_D_Exception e) {
e.printStackTrace();
}
}
From source file:implementation.java
public static void main(String[] args) throws IOException { // Network Variables KUSOCKET ClientSocket = new KUSOCKET(); ENDPOINT ClientAddress = new ENDPOINT("0.0.0.0", 0); ENDPOINT BroadcastAddress = new ENDPOINT("255.255.255.255", TokenAccess.SERVER_PORT_NUMBER); ENDPOINT AnyAddress = new ENDPOINT("0.0.0.0", 0); ENDPOINT ServerAddress = new ENDPOINT(); MESSAGE OutgoingMessage = new MESSAGE(); MESSAGE IncomingMessage = new MESSAGE(); // Output variables int CommentLength = TokenAccess.COMMENT_LENGTH; String Comment = new String(); // Protocol variables Timer firstnode = new Timer(); Timer tokentimer = new Timer(); Timer tokenlost = new Timer(); Timer exit = new Timer(); int DialogueNumber = 0; int numRetries = 0; Integer addressIP[] = new Integer[2]; int trigger = 0; String lanAddress = "192.168.1.67"; // Create socket and await connection /////////////////////////////////////// // -------------------------------------------------------------------------- // FINITE STATE MACHINE // -------------------------------------------------------------------------- System.err.println("FSM Started ===================" + '\n'); // FSM variables TokenAccess.STATES state = TokenAccess.STATES.STATE_INITIAL; // Initial STATE TokenAccess.STATES lastState = state; // Last STATE boolean bContinueEventWait = false; boolean bContinueStateLoop = true; String last = null;//from w w w . ja va2 s.c o m ClientSocket.CreateUDPSocket(ClientAddress); while (bContinueStateLoop) { switch (state) { case STATE_INITIAL: firstnode.Start(TokenAccess.firstnode); state = TokenAccess.STATES.STATE_STARTED; bContinueEventWait = false; // Stop Events loop break; case STATE_STARTED: ClientSocket.MakeConnection(AnyAddress); if (firstnode.isExpired()) { tokentimer.Start(TokenAccess.tokentimer); System.out.println("implementation.main()"); // adding local host ip to array String t = InetAddress.getLocalHost().getHostAddress(); String[] split = t.split("\\."); addressIP[0] = Integer.parseInt(split[3]); System.out.println(split[3]); state = TokenAccess.STATES.STATE_TOKENED; bContinueEventWait = false;// Stop Events loop } boolean isMessageQueued = false; isMessageQueued = ClientSocket.RetrieveQueuedMessage(TokenAccess.READ_INTERVAL, IncomingMessage, ServerAddress); if (isMessageQueued && (IncomingMessage.Type == TokenAccess.PDU_CONNECT)) { System.out.println("ring exists"); //pdu_connect used instead of ring exisits state = TokenAccess.STATES.STATE_tokenLess; bContinueEventWait = false; } case STATE_tokenLess: ClientSocket.MakeConnection(AnyAddress); while (state == TokenAccess.STATES.STATE_tokenLess) { // Instructs the socket to accept // building ip address message OutgoingMessage.BuildMessage(++DialogueNumber, TokenAccess.PDU_CONNECT, InetAddress.getLocalHost().getHostAddress(), InetAddress.getLocalHost().getHostAddress().length()); //broadcasting ip address ClientSocket.DeliverMessage(OutgoingMessage, BroadcastAddress); // System.out.println("broadcasting address"); isMessageQueued = ClientSocket.RetrieveQueuedMessage(TokenAccess.READ_INTERVAL, IncomingMessage, ServerAddress); //will trigger if used on a 192 network i didnt know how to use regex for any number and a dot if (isMessageQueued == true && (IncomingMessage.Type == TokenAccess.PDU_CLOSE)) { System.out.println("pdu close"); String ipaddress = IncomingMessage.Buffer; for (int n : addressIP) { if (addressIP[n] == Integer.parseInt(ipaddress)) { String a; a = Integer.toString(addressIP[n]); OutgoingMessage.BuildMessage(++DialogueNumber, TokenAccess.PDU_ACK, InetAddress.getLocalHost().getHostAddress(), InetAddress.getLocalHost().getHostAddress().length()); ENDPOINT exitAddress = new ENDPOINT(a, TokenAccess.SERVER_PORT_NUMBER); ClientSocket.DeliverMessage(OutgoingMessage, exitAddress); addressIP = ArrayUtils.removeElement(addressIP, n); } } } if (last != null && tokenlost.bRunning == false) { tokenlost.Start(TokenAccess.tokenlost); last = null; System.out.println("token lost timer started"); } if (isMessageQueued == true && (IncomingMessage.Type == TokenAccess.CMD_CONNECT)) { System.out.println("token lost timer stopped"); tokenlost.Stop(); } if (tokenlost.isExpired()) { System.out.println("token lost"); for (int n = 0; n < addressIP.length; n++) { String ipaddress = InetAddress.getLocalHost().toString(); int lastdot = ipaddress.lastIndexOf(".") + 1; ipaddress = ipaddress.substring(lastdot, ipaddress.length()); System.out.println(n); if (addressIP[n] == null) { } else { if (addressIP[n] == Integer.parseInt(ipaddress)) { String a; a = Integer.toString(addressIP[n]); System.out.println("variable a " + a); OutgoingMessage.BuildMessage(++DialogueNumber, TokenAccess.PDU_ACK, InetAddress.getLocalHost().getHostAddress(), InetAddress.getLocalHost().getHostAddress().length()); ENDPOINT exitAddress = new ENDPOINT(lanAddress, TokenAccess.SERVER_PORT_NUMBER); ClientSocket.DeliverMessage(OutgoingMessage, exitAddress); OutgoingMessage.BuildMessage(++DialogueNumber, TokenAccess.PDU_TOKEN, InetAddress.getLocalHost().getHostAddress(), InetAddress.getLocalHost().getHostAddress().length()); String b = Integer.toString(addressIP[n]); b = lanAddress; System.out.println(b); ENDPOINT exitAddresss = new ENDPOINT(b, TokenAccess.SERVER_PORT_NUMBER); ClientSocket.DeliverMessage(OutgoingMessage, exitAddresss); } } } } if (exit.isExpired()) { OutgoingMessage.BuildMessage(++DialogueNumber, TokenAccess.PDU_CLOSE, InetAddress.getLocalHost().getHostAddress(), InetAddress.getLocalHost().getHostAddress().length()); exit.Start(30); } if (isMessageQueued && (IncomingMessage.Type == TokenAccess.PDU_TOKEN)) { tokenlost.Stop(); System.out.println(IncomingMessage.Type); state = TokenAccess.STATES.STATE_TOKENED; tokentimer.Start(TokenAccess.tokentimer); break; } } case STATE_TOKENED: while (state == TokenAccess.STATES.STATE_TOKENED) { //send data OutgoingMessage.BuildMessage(0, 0, "ALIVE", CommentLength); ClientSocket.DeliverMessage(OutgoingMessage, BroadcastAddress); tokenlost.Stop(); while (trigger == 0) { System.out.println("TOKENED"); trigger++; } if (tokentimer.isExpired()) { trigger = 0; System.out.println("token expired"); String ipaddress = InetAddress.getLocalHost().toString(); // System.out.println( "System name : "+ipaddress); for (int n = 0; n < addressIP.length; n++) { // System.out.println(n); int lastdot = ipaddress.lastIndexOf(".") + 1; ipaddress = ipaddress.substring(lastdot, ipaddress.length()); // System.out.println("IP address: "+ipaddress); int address1 = Integer.parseInt(ipaddress); System.out.println("made it to line 214"); System.out.println(n); System.out.println(addressIP[0]); if (addressIP[n] == address1) { String a; a = Integer.toString(addressIP[n]); System.out.println("variable a = " + a); String ip = InetAddress.getLocalHost().getHostAddress(); System.out.println("line 226"); System.out.println(ip); OutgoingMessage.BuildMessage(++DialogueNumber, TokenAccess.PDU_TOKEN, InetAddress.getLocalHost().getHostAddress(), InetAddress.getLocalHost().getHostAddress().length()); String b; System.out.println("line 222"); if (n + 1 >= addressIP.length) { b = (ip); // System.out.println(b); } else { b = ip; } System.out.println(b); //b = InetAddress.getLocalHost().toString().substring(0, lastdot); // int slash = b.indexOf("/"); System.out.println(b); ENDPOINT exitAddresss = new ENDPOINT(b, TokenAccess.SERVER_PORT_NUMBER); ClientSocket.DeliverMessage(OutgoingMessage, exitAddresss); System.out.println("exiting tokened state"); state = TokenAccess.STATES.STATE_tokenLess; last = "1"; } break; } } } case STATE_EXIT: while (state == TokenAccess.STATES.STATE_EXIT) { System.exit(0); } } } }
From source file:edu.msu.cme.rdp.kmer.KmerFilter.java
public static void main(String[] args) throws Exception { final KmerTrie kmerTrie; final SeqReader queryReader; final SequenceType querySeqType; final File queryFile; final KmerStartsWriter out; final boolean translQuery; final int wordSize; final int translTable; final boolean alignedSeqs; final List<String> refLabels = new ArrayList(); final int maxThreads; try {/*from w w w.ja va2s . c o m*/ CommandLine cmdLine = new PosixParser().parse(options, args); args = cmdLine.getArgs(); if (args.length < 3) { throw new Exception("Unexpected number of arguments"); } if (cmdLine.hasOption("out")) { out = new KmerStartsWriter(cmdLine.getOptionValue("out")); } else { out = new KmerStartsWriter(System.out); } if (cmdLine.hasOption("aligned")) { alignedSeqs = true; } else { alignedSeqs = false; } if (cmdLine.hasOption("transl-table")) { translTable = Integer.valueOf(cmdLine.getOptionValue("transl-table")); } else { translTable = 11; } if (cmdLine.hasOption("threads")) { maxThreads = Integer.valueOf(cmdLine.getOptionValue("threads")); } else { maxThreads = Runtime.getRuntime().availableProcessors(); } queryFile = new File(args[1]); wordSize = Integer.valueOf(args[0]); SequenceType refSeqType = null; querySeqType = SeqUtils.guessSequenceType(queryFile); queryReader = new SequenceReader(queryFile); if (querySeqType == SequenceType.Protein) { throw new Exception("Expected nucl query sequences"); } refSeqType = SeqUtils .guessSequenceType(new File(args[2].contains("=") ? args[2].split("=")[1] : args[2])); translQuery = refSeqType == SequenceType.Protein; if (translQuery && wordSize % 3 != 0) { throw new Exception("Word size must be a multiple of 3 for nucl ref seqs"); } int trieWordSize; if (translQuery) { trieWordSize = wordSize / 3; } else { trieWordSize = wordSize; } kmerTrie = new KmerTrie(trieWordSize, translQuery); for (int index = 2; index < args.length; index++) { String refName; String refFileName = args[index]; if (refFileName.contains("=")) { String[] lexemes = refFileName.split("="); refName = lexemes[0]; refFileName = lexemes[1]; } else { String tmpName = new File(refFileName).getName(); if (tmpName.contains(".")) { refName = tmpName.substring(0, tmpName.lastIndexOf(".")); } else { refName = tmpName; } } File refFile = new File(refFileName); if (refSeqType != SeqUtils.guessSequenceType(refFile)) { throw new Exception( "Reference file " + refFile + " contains " + SeqUtils.guessFileFormat(refFile) + " sequences but expected " + refSeqType + " sequences"); } SequenceReader seqReader = new SequenceReader(refFile); Sequence seq; while ((seq = seqReader.readNextSequence()) != null) { if (seq.getSeqName().startsWith("#")) { continue; } if (alignedSeqs) { kmerTrie.addModelSequence(seq, refLabels.size()); } else { kmerTrie.addSequence(seq, refLabels.size()); } } seqReader.close(); refLabels.add(refName); } } catch (Exception e) { new HelpFormatter().printHelp("KmerSearch <word_size> <query_file> [name=]<ref_file> ...", options); System.err.println(e.getMessage()); e.printStackTrace(); System.exit(1); throw new RuntimeException("Stupid jvm"); //While this will never get thrown it is required to make sure javac doesn't get confused about uninitialized variables } long startTime = System.currentTimeMillis(); long seqCount = 0; final int maxTasks = 25000; /* * if (args.length == 4) { maxThreads = Integer.valueOf(args[3]); } else { */ //} System.err.println("Starting kmer mapping at " + new Date()); System.err.println("* Number of threads: " + maxThreads); System.err.println("* References: " + refLabels); System.err.println("* Reads file: " + queryFile); System.err.println("* Kmer length: " + kmerTrie.getWordSize()); final AtomicInteger processed = new AtomicInteger(); final AtomicInteger outstandingTasks = new AtomicInteger(); ExecutorService service = Executors.newFixedThreadPool(maxThreads); Sequence querySeq; while ((querySeq = queryReader.readNextSequence()) != null) { seqCount++; String seqString = querySeq.getSeqString(); if (seqString.length() < 3) { System.err.println("Sequence " + querySeq.getSeqName() + "'s length is less than 3"); continue; } final Sequence threadSeq = querySeq; Runnable r = new Runnable() { public void run() { try { processSeq(threadSeq, refLabels, kmerTrie, out, wordSize, translQuery, translTable, false); processSeq(threadSeq, refLabels, kmerTrie, out, wordSize, translQuery, translTable, true); } catch (IOException e) { throw new RuntimeException(e); } processed.incrementAndGet(); outstandingTasks.decrementAndGet(); } }; outstandingTasks.incrementAndGet(); service.submit(r); while (outstandingTasks.get() >= maxTasks) ; if ((processed.get() + 1) % 1000000 == 0) { System.err.println("Processed " + processed + " sequences in " + (System.currentTimeMillis() - startTime) + " ms"); } } service.shutdown(); service.awaitTermination(1, TimeUnit.DAYS); System.err.println("Finished Processed " + processed + " sequences in " + (System.currentTimeMillis() - startTime) + " ms"); out.close(); }
From source file:net.sourceforge.doddle_owl.data.JpnWordNetDic.java
public static void main(String[] args) throws Exception { JpnWordNetDic.initJPNWNDic();/*from www .ja v a 2 s . c om*/ String id1 = "08675967-n"; // String id1 = "JPNWN_ROOT"; Concept c = JpnWordNetDic.getConcept(id1); System.out.println(c); Set<String> idSet = new HashSet<String>(); BufferedReader reader = new BufferedReader( new InputStreamReader(new FileInputStream(DODDLEConstants.JPWN_HOME + "tree.data"), "UTF-8")); while (reader.ready()) { String line = reader.readLine(); if (line.indexOf(id1) != -1) { String id = line.split("\t\\|")[0]; idSet.add(id); } } System.out.println(idSet); for (String id : idSet) { c = JpnWordNetDic.getConcept(id); System.out.println(c); } }
From source file:com.twentyn.chemicalClassifier.Runner.java
public static void main(String[] args) throws Exception { BufferedReader reader = new BufferedReader(new FileReader(args[0])); BufferedWriter writer = new BufferedWriter(new FileWriter(args[1])); try {/* ww w. j a v a 2s . c o m*/ Oscar oscar = new Oscar(); String line = null; /* NOTE: this is exactly the wrong way to write a TSV reader. Caveat emptor. * See http://tburette.github.io/blog/2014/05/25/so-you-want-to-write-your-own-CSV-code/ * and then use org.apache.commons.csv.CSVParser instead. */ while ((line = reader.readLine()) != null) { // TSV means split on tabs! Nothing else will do. List<String> fields = Arrays.asList(line.split("\t")); // Choke if our invariants aren't satisfied. We expect ever line to have a name and an InChI. if (fields.size() != 2) { throw new RuntimeException( String.format("Found malformed line (all lines must have two fields: %s", line)); } String name = fields.get(1); List<ResolvedNamedEntity> entities = oscar.findAndResolveNamedEntities(name); System.out.println("**********"); System.out.println("Name: " + name); List<String> outputFields = new ArrayList<>(fields.size() + 1); outputFields.addAll(fields); if (entities.size() == 0) { System.out.println("No match"); outputFields.add("noMatch"); } else if (entities.size() == 1) { ResolvedNamedEntity entity = entities.get(0); NamedEntity ne = entity.getNamedEntity(); if (ne.getStart() != 0 || ne.getEnd() != name.length()) { System.out.println("Partial match"); printEntity(entity); outputFields.add("partialMatch"); } else { System.out.println("Exact match"); printEntity(entity); outputFields.add("exactMatch"); List<ChemicalStructure> structures = entity.getChemicalStructures(FormatType.STD_INCHI); for (ChemicalStructure s : structures) { outputFields.add(s.getValue()); } } } else { // Multiple matches found! System.out.println("Multiple matches"); for (ResolvedNamedEntity e : entities) { printEntity(e); } outputFields.add("multipleMatches"); } writer.write(String.join("\t", outputFields)); writer.newLine(); } } finally { writer.flush(); writer.close(); } }