List of usage examples for java.io IOException IOException
public IOException(Throwable cause)
From source file:com.opensearchserver.affinities.Main.java
public static void main(String[] args) throws IOException, ParseException { Logger.getLogger("").setLevel(Level.WARNING); Options options = new Options(); options.addOption("h", "help", false, "print this message"); options.addOption("d", "datadir", true, "Data directory"); options.addOption("p", "port", true, "TCP port"); CommandLineParser parser = new GnuParser(); CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("java -jar target/oss-affinities.jar", options); return;/*from www.j a va 2 s . c o m*/ } int port = cmd.hasOption("p") ? Integer.parseInt(cmd.getOptionValue("p")) : 9092; File dataDir = new File(System.getProperty("user.home"), "opensearchserver_affinities"); if (cmd.hasOption("d")) dataDir = new File(cmd.getOptionValue("d")); if (!dataDir.exists()) throw new IOException("The data directory does not exists: " + dataDir); if (!dataDir.isDirectory()) throw new IOException("The data directory path is not a directory: " + dataDir); AffinityList.load(dataDir); UndertowJaxrsServer server = new UndertowJaxrsServer() .start(Undertow.builder().addHttpListener(port, "0.0.0.0")); server.deploy(Main.class); }
From source file:com.marklogic.client.example.util.Bootstrapper.java
/** * Command-line invocation./*from w ww . ja va 2 s. c om*/ * @param args command-line arguments specifying the configuration and REST server */ public static void main(String[] args) throws ClientProtocolException, IOException, FactoryConfigurationError { Properties properties = new Properties(); for (int i = 0; i < args.length; i++) { String name = args[i]; if (name.startsWith("-") && name.length() > 1 && ++i < args.length) { name = name.substring(1); if ("properties".equals(name)) { InputStream propsStream = Bootstrapper.class.getClassLoader().getResourceAsStream(name); if (propsStream == null) throw new IOException("Could not read bootstrapper properties"); Properties props = new Properties(); props.load(propsStream); props.putAll(properties); properties = props; } else { properties.put(name, args[i]); } } else { System.err.println("invalid argument: " + name); System.err.println(getUsage()); System.exit(1); } } String invalid = joinList(listInvalidKeys(properties)); if (invalid != null && invalid.length() > 0) { System.err.println("invalid arguments: " + invalid); System.err.println(getUsage()); System.exit(1); } // TODO: catch invalid argument exceptions and provide feedback new Bootstrapper().makeServer(properties); System.out.println("Created " + properties.getProperty("restserver") + " server on " + properties.getProperty("restport") + " port for " + properties.getProperty("restdb") + " database"); }
From source file:de.tudarmstadt.ukp.experiments.argumentation.convincingness.sampling.Step4MTurkOutputCollector.java
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { String inputDirWithArgumentPairs = args[0]; File[] resultFiles;//ww w .j av a 2 s. c o m if (args[1].contains("*")) { File path = new File(args[1]); File directory = path.getParentFile(); String regex = path.getName().replaceAll("\\*", ""); List<File> files = new ArrayList<>(FileUtils.listFiles(directory, new String[] { regex }, false)); resultFiles = new File[files.size()]; for (int i = 0; i < files.size(); i++) { resultFiles[i] = files.get(i); } } else { // result file is a comma-separated list of CSV files from MTurk String[] split = args[1].split(","); resultFiles = new File[split.length]; for (int i = 0; i < split.length; i++) { resultFiles[i] = new File(split[i]); } } File outputDir = new File(args[2]); if (!outputDir.exists()) { if (!outputDir.mkdirs()) { throw new IOException("Cannot create directory " + outputDir); } } // error if output folder not empty to prevent any confusion by mixing files if (!FileUtils.listFiles(outputDir, null, false).isEmpty()) { throw new IllegalArgumentException("Output dir " + outputDir + " is not empty"); } // collected assignments with empty reason for rejections Set<String> assignmentsWithEmptyReason = new HashSet<>(); // parse with first line as header MTurkOutputReader mTurkOutputReader = new MTurkOutputReader(resultFiles); Collection<File> files = FileUtils.listFiles(new File(inputDirWithArgumentPairs), new String[] { "xml" }, false); if (files.isEmpty()) { throw new IOException("No xml files found in " + inputDirWithArgumentPairs); } // statistics: how many hits with how many assignments ; hit ID / assignments Map<String, Map<String, Integer>> assignmentsPerHits = new HashMap<>(); // collect accept/reject statistics for (Map<String, String> record : mTurkOutputReader) { boolean wasRejected = "Rejected".equals(record.get("assignmentstatus")); String hitID = record.get("hitid"); String hitTypeId = record.get("hittypeid"); if (!wasRejected) { // update statistics if (!assignmentsPerHits.containsKey(hitTypeId)) { assignmentsPerHits.put(hitTypeId, new HashMap<String, Integer>()); } if (!assignmentsPerHits.get(hitTypeId).containsKey(hitID)) { assignmentsPerHits.get(hitTypeId).put(hitID, 0); } assignmentsPerHits.get(hitTypeId).put(hitID, assignmentsPerHits.get(hitTypeId).get(hitID) + 1); } } // statistics: how many hits with how many assignments ; hit ID / assignments Map<String, Integer> approvedAssignmentsPerHit = new HashMap<>(); Map<String, Integer> rejectedAssignmentsPerHit = new HashMap<>(); // collect accept/reject statistics for (Map<String, String> record : mTurkOutputReader) { boolean approved = "Approved".equals(record.get("assignmentstatus")); boolean rejected = "Rejected".equals(record.get("assignmentstatus")); String hitID = record.get("hitid"); if (approved) { // update statistics if (!approvedAssignmentsPerHit.containsKey(hitID)) { approvedAssignmentsPerHit.put(hitID, 0); } approvedAssignmentsPerHit.put(hitID, approvedAssignmentsPerHit.get(hitID) + 1); } else if (rejected) { // update statistics if (!rejectedAssignmentsPerHit.containsKey(hitID)) { rejectedAssignmentsPerHit.put(hitID, 0); } rejectedAssignmentsPerHit.put(hitID, rejectedAssignmentsPerHit.get(hitID) + 1); } else { throw new IllegalStateException( "Unknown state: " + record.get("assignmentstatus") + " HITID: " + hitID); } } // System.out.println("Approved: " + approvedAssignmentsPerHit); // System.out.println("Rejected: " + rejectedAssignmentsPerHit); System.out.println("Approved (values): " + new HashSet<>(approvedAssignmentsPerHit.values())); System.out.println("Rejected (values): " + new HashSet<>(rejectedAssignmentsPerHit.values())); // rejection statistics int totalRejected = 0; for (Map.Entry<String, Integer> rejectionEntry : rejectedAssignmentsPerHit.entrySet()) { totalRejected += rejectionEntry.getValue(); } System.out.println("Total rejections: " + totalRejected); /* // generate .success files for adding more annotations for (File resultFile : resultFiles) { String hitTypeID = mTurkOutputReader.getHitTypeIdForFile().get(resultFile); // assignments for that hittypeid (= file) Map<String, Integer> assignments = assignmentsPerHits.get(hitTypeID); prepareUpdateHITsFiles(assignments, hitTypeID, resultFile); } */ int totalSavedPairs = 0; // load all previously prepared argument pairs for (File file : files) { List<ArgumentPair> argumentPairs = (List<ArgumentPair>) XStreamTools.getXStream().fromXML(file); List<AnnotatedArgumentPair> annotatedArgumentPairs = new ArrayList<>(); for (ArgumentPair argumentPair : argumentPairs) { AnnotatedArgumentPair annotatedArgumentPair = new AnnotatedArgumentPair(argumentPair); // is there such an answer? String key = "Answer." + argumentPair.getId(); // iterate only if there is such column to save time if (mTurkOutputReader.getColumnNames().contains(key)) { // now find the results for (Map<String, String> record : mTurkOutputReader) { if (record.containsKey(key)) { // extract the values AnnotatedArgumentPair.MTurkAssignment assignment = new AnnotatedArgumentPair.MTurkAssignment(); boolean wasRejected = "Rejected".equals(record.get("assignmentstatus")); // only non-rejected (if required) if (!wasRejected) { String hitID = record.get("hitid"); String workerID = record.get("workerid"); String assignmentId = record.get("assignmentid"); try { assignment.setAssignmentAcceptTime( DATE_FORMAT.parse(record.get("assignmentaccepttime"))); assignment.setAssignmentSubmitTime( DATE_FORMAT.parse(record.get("assignmentsubmittime"))); assignment.setHitComment(record.get("Answer.feedback")); assignment.setHitID(hitID); assignment.setTurkID(workerID); assignment.setAssignmentId(assignmentId); // and answer specific fields String valueRaw = record.get(key); // so far the label has had format aXXX_aYYY_a1, aXXX_aYYY_a2, or aXXX_aYYY_equal // strip now only true label String label = valueRaw.split("_")[2]; assignment.setValue(label); String reason = record.get(key + "_reason"); // missing reason if (reason == null) { assignmentsWithEmptyReason.add(assignmentId); } else { assignment.setReason(reason); // get worker's stance String stanceRaw = record.get(key + "_stance"); if (stanceRaw != null) { // parse stance String stance = stanceRaw.split("_stance_")[1]; assignment.setWorkerStance(stance); } // we take maximal 5 assignments Collections.sort(annotatedArgumentPair.mTurkAssignments, new Comparator<AnnotatedArgumentPair.MTurkAssignment>() { @Override public int compare(AnnotatedArgumentPair.MTurkAssignment o1, AnnotatedArgumentPair.MTurkAssignment o2) { return o1.getAssignmentAcceptTime() .compareTo(o2.getAssignmentAcceptTime()); } }); if (annotatedArgumentPair.mTurkAssignments .size() < MAXIMUM_ASSIGNMENTS_PER_HIT) { annotatedArgumentPair.mTurkAssignments.add(assignment); } } } catch (IllegalArgumentException | NullPointerException ex) { System.err.println("Malformed annotations for HIT " + hitID + ", worker " + workerID + ", assignment " + assignmentId + "; " + ex.getMessage() + ", full record: " + record); } } } } } // and if there are some annotations, add it to the result set if (!annotatedArgumentPair.mTurkAssignments.isEmpty()) { annotatedArgumentPairs.add(annotatedArgumentPair); } } if (!annotatedArgumentPairs.isEmpty()) { File outputFile = new File(outputDir, file.getName()); XStreamTools.toXML(annotatedArgumentPairs, outputFile); System.out.println("Saved " + annotatedArgumentPairs.size() + " annotated pairs to " + outputFile); totalSavedPairs += annotatedArgumentPairs.size(); } } System.out.println("Total saved " + totalSavedPairs + " pairs"); // print assignments with empty reasons if (!assignmentsWithEmptyReason.isEmpty()) { System.out.println( "== Assignments with empty reason:\nassignmentIdToReject\tassignmentIdToRejectComment"); for (String assignmentId : assignmentsWithEmptyReason) { System.out.println( assignmentId + "\t\"Dear worker, you did not fill the required field with a reason.\""); } } }
From source file:com.marklogic.client.tutorial.util.Bootstrapper.java
/** * Command-line invocation.// www.j a v a 2 s .c om * @param args command-line arguments specifying the configuration and REST server */ public static void main(String[] args) throws ClientProtocolException, IOException, XMLStreamException, FactoryConfigurationError { Properties properties = new Properties(); for (int i = 0; i < args.length; i++) { String name = args[i]; if (name.startsWith("-") && name.length() > 1 && ++i < args.length) { name = name.substring(1); if ("properties".equals(name)) { InputStream propsStream = Bootstrapper.class.getClassLoader().getResourceAsStream(name); if (propsStream == null) throw new IOException("Could not read bootstrapper properties"); Properties props = new Properties(); props.load(propsStream); props.putAll(properties); properties = props; } else { properties.put(name, args[i]); } } else { System.err.println("invalid argument: " + name); System.err.println(getUsage()); System.exit(1); } } String invalid = joinList(listInvalidKeys(properties)); if (invalid != null && invalid.length() > 0) { System.err.println("invalid arguments: " + invalid); System.err.println(getUsage()); System.exit(1); } new Bootstrapper().makeServer(properties); System.out.println("Created " + properties.getProperty("restserver") + " server on " + properties.getProperty("restport") + " port for " + properties.getProperty("restdb") + " database"); }
From source file:de.tudarmstadt.ukp.argumentation.data.roomfordebate.DataFetcher.java
public static void main(String[] args) throws Exception { File crawledPagesFolder = new File(args[0]); if (!crawledPagesFolder.exists()) { crawledPagesFolder.mkdirs();//from w ww.j a va 2 s . c om } File outputFolder = new File(args[1]); if (!outputFolder.exists()) { outputFolder.mkdirs(); } // read links from text file final String urlsResourceName = "roomfordebate-urls.txt"; InputStream urlsStream = DataFetcher.class.getClassLoader().getResourceAsStream(urlsResourceName); if (urlsStream == null) { throw new IOException("Cannot find resource " + urlsResourceName + " on the classpath"); } // read list of urls List<String> urls = new ArrayList<>(); LineIterator iterator = IOUtils.lineIterator(urlsStream, "utf-8"); while (iterator.hasNext()) { // ignore commented url (line starts with #) String line = iterator.nextLine(); if (!line.startsWith("#") && !line.trim().isEmpty()) { urls.add(line.trim()); } } // download all crawlPages(urls, crawledPagesFolder); List<File> files = new ArrayList<>(FileUtils.listFiles(crawledPagesFolder, null, false)); Collections.sort(files, new Comparator<File>() { @Override public int compare(File o1, File o2) { return o1.getName().compareTo(o2.getName()); } }); int idCounter = 0; for (File file : files) { NYTimesCommentsScraper commentsScraper = new NYTimesCommentsScraper(); NYTimesArticleExtractor extractor = new NYTimesArticleExtractor(); String html = FileUtils.readFileToString(file, "utf-8"); idCounter++; File outputFileArticle = new File(outputFolder, String.format("Cx%03d.txt", idCounter)); File outputFileComments = new File(outputFolder, String.format("Dx%03d.txt", idCounter)); try { List<Comment> comments = commentsScraper.extractComments(html); Article article = extractor.extractArticle(html); saveArticleToText(article, outputFileArticle); System.out.println("Saved to " + outputFileArticle); saveCommentsToText(comments, outputFileComments, article); System.out.println("Saved to " + outputFileComments); } catch (IOException ex) { System.err.println(file.getName() + "\n" + ex.getMessage()); } } }
From source file:de.hpi.fgis.hdrs.tools.Loader.java
public static void main(String[] args) throws IOException { if (2 > args.length) { System.out.println(usage); System.out.println(options); System.exit(1);/* w w w . j a v a 2 s.c o m*/ } if (0 > args[0].indexOf(':')) { args[0] += ":" + Configuration.DEFAULT_RPC_PORT; } Configuration conf = Configuration.create(); Client client = new Client(conf, args[0]); File[] files; String options = ""; if (args[1].startsWith("-")) { options = args[1]; if (3 > args.length) { System.out.println(usage); System.exit(1); } if (0 < options.indexOf('d')) { File dir = new File(args[2]); if (!dir.isDirectory()) { throw new IOException("Directory does not exist."); } files = dir.listFiles(); } else { files = new File[] { new File(args[2]) }; } } else { files = new File[] { new File(args[1]) }; } boolean quiet = 0 < options.indexOf('q'); boolean context = 0 < options.indexOf('c'); boolean bench = 0 < options.indexOf('b'); List<BenchSample> benchSamples = null; if (bench) { benchSamples = new ArrayList<BenchSample>(); } long timeStalled = 0; long timeRouterUpdate = 0; long abortedTransactions = 0; long nBytesTotal = 0; long nTriplesTotal = 0; long timeTotal = 0; for (int i = 0; i < files.length; ++i) { Closeable source = null; TripleScanner scanner = null; try { if (0 < options.indexOf('t')) { TripleFileReader reader = new TripleFileReader(files[i]); reader.open(); scanner = reader.getScanner(); source = reader; } else if (0 < options.indexOf('z')) { GZIPInputStream stream = new GZIPInputStream(new FileInputStream(files[i])); BTCParser parser = new BTCParser(); parser.setSkipContext(!context); scanner = new StreamScanner(stream, parser); source = stream; } else { BTCParser parser = new BTCParser(); parser.setSkipContext(!context); FileSource file = new FileSource(files[i], parser); scanner = file.getScanner(); source = file; } } catch (IOException ioe) { System.out.println("Error: Couldn't open " + files[i] + ". See log for details."); LOG.error("Error: Couldn't open " + files[i] + ":", ioe); continue; } long nBytes = 0; long nTriples = 0; long time = System.currentTimeMillis(); TripleOutputStream out = client.getOutputStream(); while (scanner.next()) { Triple t = scanner.pop(); out.add(t); nBytes += t.serializedSize(); nTriples++; if (!quiet && 0 == (nTriples % (16 * 1024))) { System.out.print(String.format("\rloading... %d triples (%.2f MB, %.2f MB/s)", nTriples, LogFormatUtil.MB(nBytes), LogFormatUtil.MBperSec(nBytes, System.currentTimeMillis() - time))); } } out.close(); time = System.currentTimeMillis() - time; scanner.close(); source.close(); if (!quiet) { System.out.print("\r"); } System.out.println(String.format("%s: %d triples (%.2f MB) loaded " + "in %.2f seconds (%.2f MB/s)", files[i], nTriples, LogFormatUtil.MB(nBytes), time / 1000.0, LogFormatUtil.MBperSec(nBytes, time))); nBytesTotal += nBytes; nTriplesTotal += nTriples; timeTotal += time; timeStalled += out.getTimeStalled(); timeRouterUpdate += out.getTimeRouterUpdate(); abortedTransactions += out.getAbortedTransactions(); if (bench) { benchSamples.add(new BenchSample(time, nTriples, nBytes)); } } client.close(); if (0 == nTriplesTotal) { System.out.println("No triples loaded."); return; } System.out.println( String.format("Done loading. Totals: %d triples (%.2f MB) loaded " + "in %.2f seconds (%.2f MB/s)", nTriplesTotal, LogFormatUtil.MB(nBytesTotal), timeTotal / 1000.0, LogFormatUtil.MBperSec(nBytesTotal, timeTotal))); System.out.println(String.format( " Client stats. Stalled: %.2f s RouterUpdate: %.2f s" + " AbortedTransactions: %d", timeStalled / 1000.0, timeRouterUpdate / 1000.0, abortedTransactions)); if (bench) { System.out.println(); System.out.println("Benchmark Samples:"); System.out.println("time\tsum T\tsum MB\tMB/s"); System.out.println(String.format("%.2f\t%d\t%.2f\t%.2f", 0f, 0, 0f, 0f)); long time = 0, nTriples = 0, nBytes = 0; for (BenchSample sample : benchSamples) { time += sample.time; nTriples += sample.nTriples; nBytes += sample.nBytes; System.out.println(String.format("%.2f\t%d\t%.2f\t%.2f", time / 1000.0, nTriples, LogFormatUtil.MB(nBytes), LogFormatUtil.MBperSec(sample.nBytes, sample.time))); } } }
From source file:de.cwclan.cwsa.serverendpoint.main.ServerEndpoint.java
/** * @param args the command line arguments */// w ww . ja v a 2 s . c o m public static void main(String[] args) { Options options = new Options(); options.addOption(OptionBuilder.withArgName("file").hasArg().withDescription( "Used to enter path of configfile. Default file is endpoint.properties. NOTE: If the file is empty or does not exsist, a default config is created.") .create("config")); options.addOption("h", "help", false, "displays this page"); CommandLineParser parser = new PosixParser(); Properties properties = new Properties(); try { /* * parse default config shipped with jar */ CommandLine cmd = parser.parse(options, args); /* * load default configuration */ InputStream in = ServerEndpoint.class.getResourceAsStream("/endpoint.properties"); if (in == null) { throw new IOException("Unable to load default config from JAR. This should not happen."); } properties.load(in); in.close(); log.debug("Loaded default config base: {}", properties.toString()); if (cmd.hasOption("help")) { printHelp(options); System.exit(0); } /* * parse cutom config if exists, otherwise create default cfg */ if (cmd.hasOption("config")) { File file = new File(cmd.getOptionValue("config", "endpoint.properties")); if (file.exists() && file.canRead() && file.isFile()) { in = new FileInputStream(file); properties.load(in); log.debug("Loaded custom config from {}: {}", file.getAbsoluteFile(), properties); } else { log.warn("Config file does not exsist. A default file will be created."); } FileWriter out = new FileWriter(file); properties.store(out, "Warning, this file is recreated on every startup to merge missing parameters."); } /* * create and start endpoint */ log.info("Config read successfull. Values are: {}", properties); ServerEndpoint endpoint = new ServerEndpoint(properties); Runtime.getRuntime().addShutdownHook(endpoint.getShutdownHook()); endpoint.start(); } catch (IOException ex) { log.error("Error while reading config.", ex); } catch (ParseException ex) { log.error("Error while parsing commandline options: {}", ex.getMessage()); printHelp(options); System.exit(1); } }
From source file:edu.usc.goffish.gofs.tools.GoFSFormat.java
public static void main(String[] args) throws IOException { if (args.length < REQUIRED_ARGS) { PrintUsageAndQuit(null);//from w w w .j a v a 2 s . c o m } if (args.length == 1 && args[0].equals("-help")) { PrintUsageAndQuit(null); } Path executableDirectory; try { executableDirectory = Paths .get(GoFSFormat.class.getProtectionDomain().getCodeSource().getLocation().toURI()).getParent(); } catch (URISyntaxException e) { throw new RuntimeException("Unexpected error retrieving executable location", e); } Path configPath = executableDirectory.resolve(DEFAULT_CONFIG).normalize(); boolean copyBinaries = false; // parse optional arguments int i = 0; OptArgLoop: for (i = 0; i < args.length - REQUIRED_ARGS; i++) { switch (args[i]) { case "-config": i++; try { configPath = Paths.get(args[i]); } catch (InvalidPathException e) { PrintUsageAndQuit("Config file - " + e.getMessage()); } break; case "-copyBinaries": copyBinaries = true; break; default: break OptArgLoop; } } if (args.length - i < REQUIRED_ARGS) { PrintUsageAndQuit(null); } // finished parsing args if (i < args.length) { PrintUsageAndQuit("Unrecognized argument \"" + args[i] + "\""); } // parse config System.out.println("Parsing config..."); PropertiesConfiguration config = new PropertiesConfiguration(); config.setDelimiterParsingDisabled(true); try { config.load(Files.newInputStream(configPath)); } catch (ConfigurationException e) { throw new IOException(e); } // retrieve data nodes ArrayList<URI> dataNodes; { String[] dataNodesArray = config.getStringArray(GOFS_DATANODES_KEY); if (dataNodesArray.length == 0) { throw new ConversionException("Config must contain key " + GOFS_DATANODES_KEY); } dataNodes = new ArrayList<>(dataNodesArray.length); if (dataNodesArray.length == 0) { throw new ConversionException("Config key " + GOFS_DATANODES_KEY + " has invalid format - must define at least one data node"); } try { for (String node : dataNodesArray) { URI dataNodeURI = new URI(node); if (!"file".equalsIgnoreCase(dataNodeURI.getScheme())) { throw new ConversionException("config key " + GOFS_DATANODES_KEY + " value \"" + dataNodeURI + "\" has invalid format - data node urls must have 'file' scheme"); } else if (dataNodeURI.getPath() == null || dataNodeURI.getPath().isEmpty()) { throw new ConversionException("config key " + GOFS_DATANODES_KEY + " value \"" + dataNodeURI + "\" has invalid format - data node urls must have an absolute path specified"); } // ensure uri ends with a slash, so we know it is a directory if (!dataNodeURI.getPath().endsWith("/")) { dataNodeURI = dataNodeURI.resolve(dataNodeURI.getPath() + "/"); } dataNodes.add(dataNodeURI); } } catch (URISyntaxException e) { throw new ConversionException( "Config key " + GOFS_DATANODES_KEY + " has invalid format - " + e.getMessage()); } } // validate serializer type Class<? extends ISliceSerializer> serializerType; { String serializerTypeName = config.getString(GOFS_SERIALIZER_KEY); if (serializerTypeName == null) { throw new ConversionException("Config must contain key " + GOFS_SERIALIZER_KEY); } try { serializerType = SliceSerializerProvider.loadSliceSerializerType(serializerTypeName); } catch (ReflectiveOperationException e) { throw new ConversionException( "Config key " + GOFS_SERIALIZER_KEY + " has invalid format - " + e.getMessage()); } } // retrieve name node IInternalNameNode nameNode; try { nameNode = NameNodeProvider.loadNameNodeFromConfig(config, GOFS_NAMENODE_TYPE_KEY, GOFS_NAMENODE_LOCATION_KEY); } catch (ReflectiveOperationException e) { throw new RuntimeException("Unable to load name node", e); } System.out.println("Contacting name node..."); // validate name node if (!nameNode.isAvailable()) { throw new IOException("Name node at " + nameNode.getURI() + " is not available"); } System.out.println("Contacting data nodes..."); // validate data nodes for (URI dataNode : dataNodes) { // only attempt ssh if host exists if (dataNode.getHost() != null) { try { SSHHelper.SSH(dataNode, "true"); } catch (IOException e) { throw new IOException("Data node at " + dataNode + " is not available", e); } } } // create temporary directory Path workingDir = Files.createTempDirectory("gofs_format"); try { // create deploy directory Path deployDirectory = Files.createDirectory(workingDir.resolve(DATANODE_DIR_NAME)); // create empty slice directory Files.createDirectory(deployDirectory.resolve(DataNode.DATANODE_SLICE_DIR)); // copy binaries if (copyBinaries) { System.out.println("Copying binaries..."); FileUtils.copyDirectory(executableDirectory.toFile(), deployDirectory.resolve(executableDirectory.getFileName()).toFile()); } // write config file Path dataNodeConfigFile = deployDirectory.resolve(DataNode.DATANODE_CONFIG); try { // create config for every data node and scp deploy folder into place for (URI dataNodeParent : dataNodes) { URI dataNode = dataNodeParent.resolve(DATANODE_DIR_NAME); PropertiesConfiguration datanode_config = new PropertiesConfiguration(); datanode_config.setDelimiterParsingDisabled(true); datanode_config.setProperty(DataNode.DATANODE_INSTALLED_KEY, true); datanode_config.setProperty(DataNode.DATANODE_NAMENODE_TYPE_KEY, config.getString(GOFS_NAMENODE_TYPE_KEY)); datanode_config.setProperty(DataNode.DATANODE_NAMENODE_LOCATION_KEY, config.getString(GOFS_NAMENODE_LOCATION_KEY)); datanode_config.setProperty(DataNode.DATANODE_LOCALHOSTURI_KEY, dataNode.toString()); try { datanode_config.save(Files.newOutputStream(dataNodeConfigFile)); } catch (ConfigurationException e) { throw new IOException(e); } System.out.println("Formatting data node " + dataNode.toString() + "..."); // scp everything into place on the data node SCPHelper.SCP(deployDirectory, dataNodeParent); // update name node nameNode.addDataNode(dataNode); } // update name node nameNode.setSerializer(serializerType); } catch (Exception e) { System.out.println( "ERROR: data node formatting interrupted - name node and data nodes are in an inconsistent state and require clean up"); throw e; } System.out.println("GoFS format complete"); } finally { FileUtils.deleteQuietly(workingDir.toFile()); } }
From source file:edu.msu.cme.rdp.taxatree.TreeBuilder.java
public static void main(String[] args) throws IOException { if (args.length != 3) { System.err.println("USAGE: TreeBuilder <idmapping> <merges.bin> <newick_out>"); return;//from ww w. j a v a 2 s.com } IdMapping<Integer> idMapping = IdMapping.fromFile(new File(args[0])); DataInputStream mergeStream = new DataInputStream(new BufferedInputStream(new FileInputStream(args[1]))); TaxonHolder lastMerged = null; int taxid = 0; final Map<Integer, Double> distMap = new HashMap(); Map<Integer, TaxonHolder> taxonMap = new HashMap(); try { while (true) { if (mergeStream.readBoolean()) { // Singleton int cid = mergeStream.readInt(); int intId = mergeStream.readInt(); TaxonHolder<Taxon> holder; List<String> seqids = idMapping.getIds(intId); if (seqids.size() == 1) { holder = new TaxonHolder(new Taxon(taxid++, seqids.get(0), "")); } else { holder = new TaxonHolder(new Taxon(taxid++, "", "")); for (String seqid : seqids) { int id = taxid++; distMap.put(id, 0.0); TaxonHolder th = new TaxonHolder(new Taxon(id, seqid, "")); th.setParent(holder); holder.addChild(th); } } lastMerged = holder; taxonMap.put(cid, holder); } else { int ci = mergeStream.readInt(); int cj = mergeStream.readInt(); int ck = mergeStream.readInt(); double dist = (double) mergeStream.readInt() / DistanceCalculator.MULTIPLIER; TaxonHolder holder = new TaxonHolder(new Taxon(taxid++, "", "")); taxonMap.put(ck, holder); holder.addChild(taxonMap.get(ci)); taxonMap.get(ci).setParent(holder); distMap.put(ci, dist); holder.addChild(taxonMap.get(cj)); taxonMap.get(cj).setParent(holder); distMap.put(cj, dist); lastMerged = holder; } } } catch (EOFException e) { } if (lastMerged == null) { throw new IOException("No merges in file"); } PrintStream newickTreeOut = new PrintStream(new File(args[2])); NewickPrintVisitor visitor = new NewickPrintVisitor(newickTreeOut, false, new NewickDistanceFactory() { public float getDistance(int i) { return distMap.get(i).floatValue(); } }); lastMerged.biDirectionDepthFirst(visitor); newickTreeOut.close(); }
From source file:es.upm.oeg.tools.quality.ldsniffer.cmd.LDSnifferApp.java
public static void main(String[] args) { HelpFormatter help = new HelpFormatter(); String header = "Assess a list of Linked Data resources using Linked Data Quality Model."; String footer = "Please report issues at https://github.com/nandana/ld-sniffer"; try {/*from ww w . ja v a2 s. co m*/ CommandLine line = parseArguments(args); if (line.hasOption("help")) { help.printHelp("LDSnifferApp", header, OPTIONS, footer, true); System.exit(0); } evaluationTimeout = Integer.parseInt(line.getOptionValue("t", "10")); if (line.hasOption("md")) { includeMetricDefinitions = true; } if (line.hasOption("rdf")) { rdfOutput = true; } logger.info("URL List: " + line.getOptionValue("ul")); logger.info("TDB Path: " + line.getOptionValue("tdb")); logger.info("Metrics Path: " + line.getOptionValue("ml")); logger.info("Include Metric definitions: " + line.getOptionValue("ml")); logger.info("RDF output: " + line.getOptionValue("rdf")); logger.info("Timeout (mins): " + evaluationTimeout); if (line.hasOption("ml")) { Path path = Paths.get(line.getOptionValue("ml")); if (!Files.exists(path)) { throw new IOException(path.toAbsolutePath().toString() + " : File doesn't exit."); } } //Set the TDB path String tdbDirectory; if (line.hasOption("tdb")) { tdbDirectory = line.getOptionValue("tdb"); } else { Path tempPath = Files.createTempDirectory("tdb_"); tdbDirectory = tempPath.toAbsolutePath().toString(); } // Create the URL list for the evaluation if (!line.hasOption("ul") && !line.hasOption("url")) { System.out.println("One of the following parameters are required: url or urlList "); help.printHelp("LDSnifferApp", header, OPTIONS, footer, true); System.exit(0); } else if (line.hasOption("ul") && line.hasOption("url")) { System.out.println("You have to specify either url or urlList, not both."); help.printHelp("LDSnifferApp", header, OPTIONS, footer, true); System.exit(0); } List<String> urlList = null; if (line.hasOption("ul")) { Path path = Paths.get(line.getOptionValue("ul")); logger.info("Path : " + path.toAbsolutePath().toString()); logger.info("Path exits : " + Files.exists(path)); urlList = Files.readAllLines(path, Charset.defaultCharset()); } else if (line.hasOption("url")) { urlList = new ArrayList<>(); urlList.add(line.getOptionValue("url")); } Executor executor = new Executor(tdbDirectory, urlList); executor.execute(); } catch (MissingOptionException e) { help.printHelp("LDSnifferApp", header, OPTIONS, footer, true); logger.error("Missing arguments. Reason: " + e.getMessage(), e); System.exit(1); } catch (ParseException e) { logger.error("Parsing failed. Reason: " + e.getMessage(), e); System.exit(1); } catch (IOException e) { logger.error("Execution failed. Reason: " + e.getMessage(), e); System.exit(1); } }