List of usage examples for java.io File mkdirs
public boolean mkdirs()
From source file:com.linkedin.pinotdruidbenchmark.DruidResponseTime.java
public static void main(String[] args) throws Exception { if (args.length != 4 && args.length != 5) { System.err.println(// w w w . j a v a 2 s . com "4 or 5 arguments required: QUERY_DIR, RESOURCE_URL, WARM_UP_ROUNDS, TEST_ROUNDS, RESULT_DIR (optional)."); return; } File queryDir = new File(args[0]); String resourceUrl = args[1]; int warmUpRounds = Integer.parseInt(args[2]); int testRounds = Integer.parseInt(args[3]); File resultDir; if (args.length == 4) { resultDir = null; } else { resultDir = new File(args[4]); if (!resultDir.exists()) { if (!resultDir.mkdirs()) { throw new RuntimeException("Failed to create result directory: " + resultDir); } } } File[] queryFiles = queryDir.listFiles(); assert queryFiles != null; Arrays.sort(queryFiles); try (CloseableHttpClient httpClient = HttpClients.createDefault()) { HttpPost httpPost = new HttpPost(resourceUrl); httpPost.addHeader("content-type", "application/json"); for (File queryFile : queryFiles) { StringBuilder stringBuilder = new StringBuilder(); try (BufferedReader bufferedReader = new BufferedReader(new FileReader(queryFile))) { int length; while ((length = bufferedReader.read(CHAR_BUFFER)) > 0) { stringBuilder.append(new String(CHAR_BUFFER, 0, length)); } } String query = stringBuilder.toString(); httpPost.setEntity(new StringEntity(query)); System.out.println( "--------------------------------------------------------------------------------"); System.out.println("Running query: " + query); System.out.println( "--------------------------------------------------------------------------------"); // Warm-up Rounds System.out.println("Run " + warmUpRounds + " times to warm up..."); for (int i = 0; i < warmUpRounds; i++) { CloseableHttpResponse httpResponse = httpClient.execute(httpPost); httpResponse.close(); System.out.print('*'); } System.out.println(); // Test Rounds System.out.println("Run " + testRounds + " times to get response time statistics..."); long[] responseTimes = new long[testRounds]; long totalResponseTime = 0L; for (int i = 0; i < testRounds; i++) { long startTime = System.currentTimeMillis(); CloseableHttpResponse httpResponse = httpClient.execute(httpPost); httpResponse.close(); long responseTime = System.currentTimeMillis() - startTime; responseTimes[i] = responseTime; totalResponseTime += responseTime; System.out.print(responseTime + "ms "); } System.out.println(); // Store result. if (resultDir != null) { File resultFile = new File(resultDir, queryFile.getName() + ".result"); CloseableHttpResponse httpResponse = httpClient.execute(httpPost); try (BufferedInputStream bufferedInputStream = new BufferedInputStream( httpResponse.getEntity().getContent()); BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(resultFile))) { int length; while ((length = bufferedInputStream.read(BYTE_BUFFER)) > 0) { bufferedWriter.write(new String(BYTE_BUFFER, 0, length)); } } httpResponse.close(); } // Process response times. double averageResponseTime = (double) totalResponseTime / testRounds; double temp = 0; for (long responseTime : responseTimes) { temp += (responseTime - averageResponseTime) * (responseTime - averageResponseTime); } double standardDeviation = Math.sqrt(temp / testRounds); System.out.println("Average response time: " + averageResponseTime + "ms"); System.out.println("Standard deviation: " + standardDeviation); } } }
From source file:MakeDirectories.java
public static void main(String[] args) { if (args.length < 1) usage();// w w w .j a va 2 s .c o m if (args[0].equals("-r")) { if (args.length != 3) usage(); File old = new File(args[1]), rname = new File(args[2]); old.renameTo(rname); fileData(old); fileData(rname); return; // Exit main } int count = 0; boolean del = false; if (args[0].equals("-d")) { count++; del = true; } count--; while (++count < args.length) { File f = new File(args[count]); if (f.exists()) { System.out.println(f + " exists"); if (del) { System.out.println("deleting..." + f); f.delete(); } } else { // Doesn't exist if (!del) { f.mkdirs(); System.out.println("created " + f); } } fileData(f); } }
From source file:edu.oregonstate.eecs.mcplan.ml.KernelPrincipalComponentsAnalysis.java
public static void main(final String[] args) throws FileNotFoundException { final File root = new File("test/KernelPrincipalComponentsAnalysis"); root.mkdirs(); final int seed = 42; final int N = 30; final RandomGenerator rng = new MersenneTwister(seed); final ArrayList<RealVector> data = new ArrayList<RealVector>(); final ArrayList<RealVector> shuffled = new ArrayList<RealVector>(); // final double[][] covariance = new double[][] { {1.0, 0.0}, // {0.0, 1.0} }; // final MultivariateNormalDistribution p = new MultivariateNormalDistribution( // rng, new double[] { 0.0, 0.0 }, covariance ); // final MultivariateNormalDistribution q = new MultivariateNormalDistribution( // rng, new double[] { 10.0, 0.0 }, covariance ); ///* w ww .j a va 2 s. c o m*/ // for( int i = 0; i < N; ++i ) { // data.add( new ArrayRealVector( p.sample() ) ); // data.add( new ArrayRealVector( q.sample() ) ); // } // Fn.shuffle( rng, data ); final double sigma = 0.01; final double[][] covariance = new double[][] { { sigma, 0.0 }, { 0.0, sigma } }; final MultivariateNormalDistribution p = new MultivariateNormalDistribution(rng, new double[] { 0.0, 0.0 }, covariance); for (final double r : new double[] { 1.0, 3.0, 5.0 }) { for (int i = 0; i < N; ++i) { final double theta = i * 2 * Math.PI / N; final double[] noise = p.sample(); final RealVector v = new ArrayRealVector( new double[] { r * Math.cos(theta) + noise[0], r * Math.sin(theta) + noise[1] }); data.add(v); shuffled.add(v); } } Fn.shuffle(rng, shuffled); final Csv.Writer data_writer = new Csv.Writer(new PrintStream(new File(root, "data.csv"))); for (final RealVector v : data) { for (int i = 0; i < v.getDimension(); ++i) { data_writer.cell(v.getEntry(i)); } data_writer.newline(); } data_writer.close(); System.out.println("[Training]"); final int Ncomponents = 2; final KernelPrincipalComponentsAnalysis<RealVector> kpca = new KernelPrincipalComponentsAnalysis<RealVector>( shuffled, new RadialBasisFunctionKernel(0.5), 1e-6); System.out.println("[Finished]"); for (int i = 0; i < Ncomponents; ++i) { System.out.println(kpca.eigenvectors.get(i)); } System.out.println("Transformed data:"); final KernelPrincipalComponentsAnalysis.Transformer<RealVector> transformer = kpca .makeTransformer(Ncomponents); final Csv.Writer transformed_writer = new Csv.Writer(new PrintStream(new File(root, "transformed.csv"))); for (final RealVector u : data) { final RealVector v = transformer.transform(u); System.out.println(v); for (int i = 0; i < v.getDimension(); ++i) { transformed_writer.cell(v.getEntry(i)); } transformed_writer.newline(); } transformed_writer.close(); }
From source file:de.tudarmstadt.ukp.experiments.argumentation.convincingness.sampling.Step5GoldLabelEstimator.java
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { String inputDir = args[0];/*ww w.j ava 2s.c o m*/ File outputDir = new File(args[1]); if (!outputDir.exists()) { outputDir.mkdirs(); } // we will process only a subset first List<AnnotatedArgumentPair> allArgumentPairs = new ArrayList<>(); Collection<File> files = IOHelper.listXmlFiles(new File(inputDir)); for (File file : files) { allArgumentPairs.addAll((List<AnnotatedArgumentPair>) XStreamTools.getXStream().fromXML(file)); } // collect turkers and csv List<String> turkerIDs = extractAndSortTurkerIDs(allArgumentPairs); String preparedCSV = prepareCSV(allArgumentPairs, turkerIDs); // save CSV and run MACE Path tmpDir = Files.createTempDirectory("mace"); File maceInputFile = new File(tmpDir.toFile(), "input.csv"); FileUtils.writeStringToFile(maceInputFile, preparedCSV, "utf-8"); File outputPredictions = new File(tmpDir.toFile(), "predictions.txt"); File outputCompetence = new File(tmpDir.toFile(), "competence.txt"); // run MACE MACE.main(new String[] { "--iterations", "500", "--threshold", String.valueOf(MACE_THRESHOLD), "--restarts", "50", "--outputPredictions", outputPredictions.getAbsolutePath(), "--outputCompetence", outputCompetence.getAbsolutePath(), maceInputFile.getAbsolutePath() }); // read back the predictions and competence List<String> predictions = FileUtils.readLines(outputPredictions, "utf-8"); // check the output if (predictions.size() != allArgumentPairs.size()) { throw new IllegalStateException("Wrong size of the predicted file; expected " + allArgumentPairs.size() + " lines but was " + predictions.size()); } String competenceRaw = FileUtils.readFileToString(outputCompetence, "utf-8"); String[] competence = competenceRaw.split("\t"); if (competence.length != turkerIDs.size()) { throw new IllegalStateException( "Expected " + turkerIDs.size() + " competence number, got " + competence.length); } // rank turkers by competence Map<String, Double> turkerIDCompetenceMap = new TreeMap<>(); for (int i = 0; i < turkerIDs.size(); i++) { turkerIDCompetenceMap.put(turkerIDs.get(i), Double.valueOf(competence[i])); } // sort by value descending Map<String, Double> sortedCompetences = IOHelper.sortByValue(turkerIDCompetenceMap, false); System.out.println("Sorted turker competences: " + sortedCompetences); // assign the gold label and competence for (int i = 0; i < allArgumentPairs.size(); i++) { AnnotatedArgumentPair annotatedArgumentPair = allArgumentPairs.get(i); String goldLabel = predictions.get(i).trim(); // might be empty if (!goldLabel.isEmpty()) { // so far the gold label has format aXXX_aYYY_a1, aXXX_aYYY_a2, or aXXX_aYYY_equal // strip now only the gold label annotatedArgumentPair.setGoldLabel(goldLabel); } // update turker competence for (AnnotatedArgumentPair.MTurkAssignment assignment : annotatedArgumentPair.mTurkAssignments) { String turkID = assignment.getTurkID(); int turkRank = getTurkerRank(turkID, sortedCompetences); assignment.setTurkRank(turkRank); double turkCompetence = turkerIDCompetenceMap.get(turkID); assignment.setTurkCompetence(turkCompetence); } } // now sort the data back according to their original file name Map<String, List<AnnotatedArgumentPair>> fileNameAnnotatedPairsMap = new HashMap<>(); for (AnnotatedArgumentPair argumentPair : allArgumentPairs) { String fileName = IOHelper.createFileName(argumentPair.getDebateMetaData(), argumentPair.getArg1().getStance()); if (!fileNameAnnotatedPairsMap.containsKey(fileName)) { fileNameAnnotatedPairsMap.put(fileName, new ArrayList<AnnotatedArgumentPair>()); } fileNameAnnotatedPairsMap.get(fileName).add(argumentPair); } // and save them to the output file for (Map.Entry<String, List<AnnotatedArgumentPair>> entry : fileNameAnnotatedPairsMap.entrySet()) { String fileName = entry.getKey(); List<AnnotatedArgumentPair> argumentPairs = entry.getValue(); File outputFile = new File(outputDir, fileName); // and save all sampled pairs into a XML file XStreamTools.toXML(argumentPairs, outputFile); System.out.println("Saved " + argumentPairs.size() + " pairs to " + outputFile); } }
From source file:com.rtl.http.Upload.java
public static void main(String[] args) throws Exception { if (args.length != 4) { // Jxj001 20150603 d:\rpt_rtl_0001.txt d:\confDir logger.error("??4,dataTypedataVersionfilePathconfDir"); // System.out.println("??3,dataTypedataVersionfilePath"); return;/*from w w w. ja va 2 s. co m*/ } else { Upload upload = new Upload(); upload.readInfo(args[3]); Date date = new Date(); DateFormat format1 = new SimpleDateFormat("yyyyMMdd"); String todayDir = format1.format(date); File dir = new File(logDir + File.separator + todayDir); if (dir != null && !dir.exists()) { dir.mkdirs(); } // SimpleLayout layout = new SimpleLayout(); PatternLayout layout = new PatternLayout("%d %-5p %c - %m%n"); FileAppender appender = null; try { appender = new FileAppender(layout, dir.getPath() + File.separator + todayDir + "client.log", true); } catch (Exception e) { logger.error(""); } logger.addAppender(appender); logger.setLevel(Level.INFO); InputStream fileIn; dataType = args[0]; dataVersion = args[1]; filePath = args[2]; logger.info("dataType=" + dataType); logger.info("dataVersion=" + dataVersion); logger.info("filePath=" + filePath); try { File file = new File(filePath); if (!file.exists()) { // System.out.println("?:"+filePath); logger.error("?:" + filePath); return; } fileIn = new FileInputStream(filePath); String responseStr = send(upload.getJson(), fileIn, url); if (responseStr != null) { String[] values = responseStr.split(","); if ("ok".equals(values[0])) { System.out.println("0"); logger.info("ok"); } else { System.out.println("1"); logger.info(" A" + values[1]); } } else { System.out.println("1"); logger.info(" B????"); } } catch (Exception e) { System.out.println("1"); logger.error(" C" + e.getMessage()); } } logger.info("??"); // System.out.println("??"); }
From source file:com.betfair.cougar.test.socket.app.SocketCompatibilityTestingApp.java
public static void main(String[] args) throws Exception { Parser parser = new PosixParser(); Options options = new Options(); options.addOption("r", "repo", true, "Repository type to search: local|central"); options.addOption("c", "client-concurrency", true, "Max threads to allow each client tester to run tests, defaults to 10"); options.addOption("t", "test-concurrency", true, "Max client testers to run concurrently, defaults to 5"); options.addOption("m", "max-time", true, "Max time (in minutes) to allow tests to complete, defaults to 10"); options.addOption("v", "version", false, "Print version and exit"); options.addOption("h", "help", false, "This help text"); CommandLine commandLine = parser.parse(options, args); if (commandLine.hasOption("h")) { System.out.println(options); System.exit(0);//from w ww. j ava2s . c om } if (commandLine.hasOption("v")) { System.out.println("How the hell should I know?"); System.exit(0); } // 1. Find all testers in given repos List<RepoSearcher> repoSearchers = new ArrayList<>(); for (String repo : commandLine.getOptionValues("r")) { if ("local".equals(repo.toLowerCase())) { repoSearchers.add(new LocalRepoSearcher()); } else if ("central".equals(repo.toLowerCase())) { repoSearchers.add(new CentralRepoSearcher()); } else { System.err.println("Unrecognized repo: " + repo); System.err.println(options); System.exit(1); } } int clientConcurrency = 10; if (commandLine.hasOption("c")) { try { clientConcurrency = Integer.parseInt(commandLine.getOptionValue("c")); } catch (NumberFormatException nfe) { System.err.println( "client-concurrency is not a valid integer: '" + commandLine.getOptionValue("c") + "'"); System.exit(1); } } int testConcurrency = 5; if (commandLine.hasOption("t")) { try { testConcurrency = Integer.parseInt(commandLine.getOptionValue("t")); } catch (NumberFormatException nfe) { System.err.println( "test-concurrency is not a valid integer: '" + commandLine.getOptionValue("t") + "'"); System.exit(1); } } int maxMinutes = 10; if (commandLine.hasOption("m")) { try { maxMinutes = Integer.parseInt(commandLine.getOptionValue("m")); } catch (NumberFormatException nfe) { System.err.println("max-time is not a valid integer: '" + commandLine.getOptionValue("m") + "'"); System.exit(1); } } Properties clientProps = new Properties(); clientProps.setProperty("client.concurrency", String.valueOf(clientConcurrency)); File baseRunDir = new File(System.getProperty("user.dir") + "/run"); baseRunDir.mkdirs(); File tmpDir = new File(baseRunDir, "jars"); tmpDir.mkdirs(); List<ServerRunner> serverRunners = new ArrayList<>(); List<ClientRunner> clientRunners = new ArrayList<>(); for (RepoSearcher searcher : repoSearchers) { List<File> jars = searcher.findAndCache(tmpDir); for (File f : jars) { ServerRunner serverRunner = new ServerRunner(f, baseRunDir); System.out.println("Found tester: " + serverRunner.getVersion()); serverRunners.add(serverRunner); clientRunners.add(new ClientRunner(f, baseRunDir, clientProps)); } } // 2. Start servers and collect ports System.out.println(); System.out.println("Starting " + serverRunners.size() + " servers..."); for (ServerRunner server : serverRunners) { server.startServer(); } System.out.println(); List<TestCombo> tests = new ArrayList<>(serverRunners.size() * clientRunners.size()); for (ServerRunner server : serverRunners) { for (ClientRunner client : clientRunners) { tests.add(new TestCombo(server, client)); } } System.out.println("Enqueued " + tests.size() + " test combos to run..."); long startTime = System.currentTimeMillis(); // 3. Run every client against every server, collecting results BlockingQueue<Runnable> workQueue = new ArrayBlockingQueue(serverRunners.size() * clientRunners.size()); ThreadPoolExecutor service = new ThreadPoolExecutor(testConcurrency, testConcurrency, 5000, TimeUnit.MILLISECONDS, workQueue); service.prestartAllCoreThreads(); workQueue.addAll(tests); while (!workQueue.isEmpty()) { Thread.sleep(1000); } service.shutdown(); service.awaitTermination(maxMinutes, TimeUnit.MINUTES); long endTime = System.currentTimeMillis(); long totalTimeSecs = Math.round((endTime - startTime) / 1000.0); for (ServerRunner server : serverRunners) { server.shutdownServer(); } System.out.println(); System.out.println("======="); System.out.println("Results"); System.out.println("-------"); // print a summary int totalTests = 0; int totalSuccess = 0; for (TestCombo combo : tests) { String clientVer = combo.getClientVersion(); String serverVer = combo.getServerVersion(); String results = combo.getClientResults(); ObjectMapper mapper = new ObjectMapper(new JsonFactory()); JsonNode node = mapper.reader().readTree(results); JsonNode resultsArray = node.get("results"); int numTests = resultsArray.size(); int numSuccess = 0; for (int i = 0; i < numTests; i++) { if ("success".equals(resultsArray.get(i).get("result").asText())) { numSuccess++; } } totalSuccess += numSuccess; totalTests += numTests; System.out.println(clientVer + "/" + serverVer + ": " + numSuccess + "/" + numTests + " succeeded - took " + String.format("%2f", combo.getRunningTime()) + " seconds"); } System.out.println("-------"); System.out.println( "Overall: " + totalSuccess + "/" + totalTests + " succeeded - took " + totalTimeSecs + " seconds"); FileWriter out = new FileWriter("results.json"); PrintWriter pw = new PrintWriter(out); // 4. Output full results pw.println("{\n \"results\": ["); for (TestCombo combo : tests) { combo.emitResults(pw, " "); } pw.println(" ],"); pw.println(" \"servers\": ["); for (ServerRunner server : serverRunners) { server.emitInfo(pw, " "); } pw.println(" ],"); pw.close(); }
From source file:acmi.l2.clientmod.l2_version_switcher.Main.java
public static void main(String[] args) { if (args.length != 3 && args.length != 4) { System.out.println("USAGE: l2_version_switcher.jar host game version <--splash> <filter>"); System.out.println("EXAMPLE: l2_version_switcher.jar " + L2.NCWEST_HOST + " " + L2.NCWEST_GAME + " 1 \"system\\*\""); System.out.println(/*from w w w . java 2 s .c o m*/ " l2_version_switcher.jar " + L2.PLAYNC_TEST_HOST + " " + L2.PLAYNC_TEST_GAME + " 48"); System.exit(0); } List<String> argsList = new ArrayList<>(Arrays.asList(args)); String host = argsList.get(0); String game = argsList.get(1); int version = Integer.parseInt(argsList.get(2)); Helper helper = new Helper(host, game, version); boolean available = false; try { available = helper.isAvailable(); } catch (IOException e) { System.err.print(e.getClass().getSimpleName()); if (e.getMessage() != null) { System.err.print(": " + e.getMessage()); } System.err.println(); } System.out.println(String.format("Version %d available: %b", version, available)); if (!available) { System.exit(0); } List<FileInfo> fileInfoList = null; try { fileInfoList = helper.getFileInfoList(); } catch (IOException e) { System.err.println("Couldn\'t get file info map"); System.exit(1); } boolean splash = argsList.remove("--splash"); if (splash) { Optional<FileInfo> splashObj = fileInfoList.stream() .filter(fi -> fi.getPath().contains("sp_32b_01.bmp")).findAny(); if (splashObj.isPresent()) { try (InputStream is = new FilterInputStream( Util.getUnzipStream(helper.getDownloadStream(splashObj.get().getPath()))) { @Override public int read() throws IOException { int b = super.read(); if (b >= 0) b ^= 0x36; return b; } @Override public int read(byte[] b, int off, int len) throws IOException { int r = super.read(b, off, len); if (r >= 0) { for (int i = 0; i < r; i++) b[off + i] ^= 0x36; } return r; } }) { new DataInputStream(is).readFully(new byte[28]); BufferedImage bi = ImageIO.read(is); JFrame frame = new JFrame("Lineage 2 [" + version + "] " + splashObj.get().getPath()); frame.setContentPane(new JComponent() { { setPreferredSize(new Dimension(bi.getWidth(), bi.getHeight())); } @Override protected void paintComponent(Graphics g) { g.drawImage(bi, 0, 0, null); } }); frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE); frame.pack(); frame.setLocationRelativeTo(null); frame.setVisible(true); } catch (IOException e) { e.printStackTrace(); } } else { System.out.println("Splash not found"); } return; } String filter = argsList.size() > 3 ? separatorsToSystem(argsList.get(3)) : null; File l2Folder = new File(System.getProperty("user.dir")); List<FileInfo> toUpdate = fileInfoList.parallelStream().filter(fi -> { String filePath = separatorsToSystem(fi.getPath()); if (filter != null && !wildcardMatch(filePath, filter, IOCase.INSENSITIVE)) return false; File file = new File(l2Folder, filePath); try { if (file.exists() && file.length() == fi.getSize() && Util.hashEquals(file, fi.getHash())) { System.out.println(filePath + ": OK"); return false; } } catch (IOException e) { System.out.println(filePath + ": couldn't check hash: " + e); return true; } System.out.println(filePath + ": need update"); return true; }).collect(Collectors.toList()); List<String> errors = Collections.synchronizedList(new ArrayList<>()); ExecutorService executor = Executors.newFixedThreadPool(16); CompletableFuture[] tasks = toUpdate.stream().map(fi -> CompletableFuture.runAsync(() -> { String filePath = separatorsToSystem(fi.getPath()); File file = new File(l2Folder, filePath); File folder = file.getParentFile(); if (!folder.exists()) { if (!folder.mkdirs()) { errors.add(filePath + ": couldn't create parent dir"); return; } } try (InputStream input = Util .getUnzipStream(new BufferedInputStream(helper.getDownloadStream(fi.getPath()))); OutputStream output = new BufferedOutputStream(new FileOutputStream(file))) { byte[] buffer = new byte[Math.min(fi.getSize(), 1 << 24)]; int pos = 0; int r; while ((r = input.read(buffer, pos, buffer.length - pos)) >= 0) { pos += r; if (pos == buffer.length) { output.write(buffer, 0, pos); pos = 0; } } if (pos != 0) { output.write(buffer, 0, pos); } System.out.println(filePath + ": OK"); } catch (IOException e) { String msg = filePath + ": FAIL: " + e.getClass().getSimpleName(); if (e.getMessage() != null) { msg += ": " + e.getMessage(); } errors.add(msg); } }, executor)).toArray(CompletableFuture[]::new); CompletableFuture.allOf(tasks).thenRun(() -> { for (String err : errors) System.err.println(err); executor.shutdown(); }); }
From source file:csv.parser.CSVParser.java
@SuppressWarnings("resource") public static void main(String[] args) throws Exception { String file_to_parse;// w w w. java 2s .c o m String[] val_array; file_to_parse = "./input/E-library-data-3.csv"; //Build reader instance //Read CSV file CSVReader reader = new CSVReader(new FileReader(file_to_parse), ';', '"', 1); //Read all rows at once List<String[]> allRows = reader.readAll(); // Read CSV line by line and use the string array as you want for (String[] row : allRows) { for (int i = 0; i < row.length; i++) { //Removing all newlines, tabs and '&' characters(invalid XML character) row[i] = row[i].replaceAll("(\\r|\\n|\\r\\n)+", " "); row[i] = row[i].replaceAll(System.getProperty("line.separator"), "; "); row[i] = row[i].replaceAll("&", "and"); } System.out.println(Arrays.toString(row)); } //Get the input fields List<String[]> map = getMap(); String[] field; //Numbering for folders, folderNum is incremented for each new file long folderNum; folderNum = 0; for (String[] row : allRows) { //Creating new folder File file1 = new File("./output//newdir//folder" + folderNum + ""); file1.mkdirs(); //Creating content file PrintWriter writer_content = new PrintWriter("./output//newdir//folder" + folderNum + "//contents", "UTF-16"); //Creating metadata_lrmi.xml PrintWriter writer_lrmi = new PrintWriter( "./output//newdir//folder" + folderNum + "//metadata_lrmi.xml", "UTF-16"); //Creating content.xml PrintWriter writer = new PrintWriter("./output//newdir//folder" + folderNum + "//content.xml", "UTF-16"); writer.println("<?xml version=\"1.0\" encoding=\"utf-16\" standalone=\"no\"?>"); writer.println("<dublin_core schema=\"dc\">"); writer_lrmi.println("<?xml version=\"1.0\" encoding=\"utf-16\" standalone=\"no\"?>"); writer_lrmi.println("<dublin_core shema=\"lrmi\">"); for (int i = 0; i < row.length; i++) { //After snooping data, we have to change these setting for each new csv file, as the data fileds are many times mismatched //These if-else statements take care of mismatched steps. if (i == 43) { continue; } else if (i == 43) { field = map.get(42); } else if (i == 44) { field = map.get(43); } else if (i == 45 || i == 46) { continue; } else { field = map.get(i); } //Separate multiple values val_array = parseVal(row[i]); // if (val_array.length == 0) { // continue; // } PrintWriter useWriter = writer; if (field[0].equals("lrmi")) { useWriter = writer_lrmi; } switch (field.length) { case 2: writeXML(useWriter, field[1], "", val_array); break; case 3: writeXML(useWriter, field[1], field[2], val_array); break; default: } } folderNum++; writer.println("</dublin_core>"); writer_lrmi.println("</dublin_core>"); writer.close(); writer_lrmi.close(); writer_content.close(); } }
From source file:ie.pars.bnc.preprocess.MainBNCProcess.java
/** * Main method use for processing BNC text. Claws PoSs are replaced by * Stanford CoreNLP results for consistency with the rest of data! Currently * the structure of BNC is mainly discarded, only paragraphs and sentences * * @param sugare//from www .j ava2 s .co m * @throws IOException * @throws ArchiveException * @throws Exception */ public static void main(String[] sugare) throws IOException, ArchiveException, Exception { pathInput = sugare[0]; pathOutput = sugare[1]; letter = sugare[2]; filesProcesed = new TreeSet(); File folder = new File(pathOutput); if (folder.exists()) { for (File f : folder.listFiles()) { if (f.isFile()) { String pfile = f.getName().split("\\.")[0]; filesProcesed.add(pfile); } } } else { folder.mkdirs(); } getZippedFile(); }
From source file:com.cohesionforce.dis.ConvertCSV.java
/** * The main method of this converter. It could be more robust, but it is * intended to be a test tool./* ww w . j a va2 s . c o m*/ */ public static void main(String[] args) { HelpFormatter formatter = new HelpFormatter(); // create Options object Options options = new Options(); options.addOption("i", true, "CSV Log file"); options.addOption("o", true, "Ouput directory for Avro files"); options.addOption("c", true, "Compression Codec <snappy,deflate,none> Default is snappy"); options.addOption("p", false, "Write to Parquet format"); CommandLineParser parser = new DefaultParser(); CommandLine cmd; String inputFile = null; String outputFile = null; try { cmd = parser.parse(options, args); inputFile = cmd.getOptionValue("i"); outputFile = cmd.getOptionValue("o"); if (inputFile == null || outputFile == null) { formatter.printHelp("usage: ConvertCSV", options); return; } if (cmd.hasOption("p")) { LogWriter.setParquet(true); } if (cmd.hasOption("c")) { LogWriter.setCodec(cmd.getOptionValue("c")); } } catch (ParseException exc) { System.err.println("Problem with command line parameters: " + exc.getMessage()); return; } // Make sure the output directory exists File outputDir = new File(outputFile); outputDir.mkdirs(); // Start the logger thread listening final ConvertCSV converter = new ConvertCSV(); if (cmd.hasOption("p")) { converter.setExtension("parquet"); } converter.setOutputDirectory(outputFile); converter.setInputFile(inputFile); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { System.out.println("Shutting down DIS Logger"); synchronized (converter) { converter.shutdown(); } } }); converter.run(); converter.shutdown(); }