List of usage examples for org.apache.hadoop.fs FileSystem deleteOnExit
Set deleteOnExit
To view the source code for org.apache.hadoop.fs FileSystem deleteOnExit.
Click Source Link
From source file:org.apache.mahout.clustering.spectral.VectorCache.java
License:Apache License
/** * @param key SequenceFile key/*ww w . j a va2 s . co m*/ * @param vector Vector to save, to be wrapped as VectorWritable */ public static void save(Writable key, Vector vector, Path output, Configuration conf, boolean overwritePath, boolean deleteOnExit) throws IOException { FileSystem fs = FileSystem.get(output.toUri(), conf); output = fs.makeQualified(output); if (overwritePath) { HadoopUtil.delete(conf, output); } // set the cache DistributedCache.setCacheFiles(new URI[] { output.toUri() }, conf); // set up the writer SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, output, IntWritable.class, VectorWritable.class); try { writer.append(key, new VectorWritable(vector)); } finally { Closeables.close(writer, false); } if (deleteOnExit) { fs.deleteOnExit(output); } }
From source file:org.apache.mahout.math.hadoop.stochasticsvd.SSVDCli.java
License:Apache License
@Override public int run(String[] args) throws Exception { addInputOption();/* www . ja v a 2 s .c o m*/ addOutputOption(); addOption("rank", "k", "decomposition rank", true); addOption("oversampling", "p", "oversampling", String.valueOf(15)); addOption("blockHeight", "r", "Y block height (must be > (k+p))", String.valueOf(10000)); addOption("outerProdBlockHeight", "oh", "block height of outer products during multiplication, increase for sparse inputs", String.valueOf(30000)); addOption("abtBlockHeight", "abth", "block height of Y_i in ABtJob during AB' multiplication, increase for extremely sparse inputs", String.valueOf(200000)); addOption("minSplitSize", "s", "minimum split size", String.valueOf(-1)); addOption("computeU", "U", "compute U (true/false)", String.valueOf(true)); addOption("uHalfSigma", "uhs", "Compute U * Sigma^0.5", String.valueOf(false)); addOption("uSigma", "us", "Compute U * Sigma", String.valueOf(false)); addOption("computeV", "V", "compute V (true/false)", String.valueOf(true)); addOption("vHalfSigma", "vhs", "compute V * Sigma^0.5", String.valueOf(false)); addOption("reduceTasks", "t", "number of reduce tasks (where applicable)", true); addOption("powerIter", "q", "number of additional power iterations (0..2 is good)", String.valueOf(0)); addOption("broadcast", "br", "whether use distributed cache to broadcast matrices wherever possible", String.valueOf(true)); addOption("pca", "pca", "run in pca mode: compute column-wise mean and subtract from input", String.valueOf(false)); addOption("pcaOffset", "xi", "path(glob) of external pca mean (optional, dont compute, use external mean"); addOption(DefaultOptionCreator.overwriteOption().create()); Map<String, List<String>> pargs = parseArguments(args); if (pargs == null) { return -1; } int k = Integer.parseInt(getOption("rank")); int p = Integer.parseInt(getOption("oversampling")); int r = Integer.parseInt(getOption("blockHeight")); int h = Integer.parseInt(getOption("outerProdBlockHeight")); int abh = Integer.parseInt(getOption("abtBlockHeight")); int q = Integer.parseInt(getOption("powerIter")); int minSplitSize = Integer.parseInt(getOption("minSplitSize")); boolean computeU = Boolean.parseBoolean(getOption("computeU")); boolean computeV = Boolean.parseBoolean(getOption("computeV")); boolean cUHalfSigma = Boolean.parseBoolean(getOption("uHalfSigma")); boolean cUSigma = Boolean.parseBoolean(getOption("uSigma")); boolean cVHalfSigma = Boolean.parseBoolean(getOption("vHalfSigma")); int reduceTasks = Integer.parseInt(getOption("reduceTasks")); boolean broadcast = Boolean.parseBoolean(getOption("broadcast")); String xiPathStr = getOption("pcaOffset"); Path xiPath = xiPathStr == null ? null : new Path(xiPathStr); boolean pca = Boolean.parseBoolean(getOption("pca")) || xiPath != null; boolean overwrite = hasOption(DefaultOptionCreator.OVERWRITE_OPTION); Configuration conf = getConf(); if (conf == null) { throw new IOException("No Hadoop configuration present"); } Path[] inputPaths = { getInputPath() }; Path tempPath = getTempPath(); FileSystem fs = FileSystem.get(getTempPath().toUri(), conf); // housekeeping if (overwrite) { // clear the output path HadoopUtil.delete(getConf(), getOutputPath()); // clear the temp path HadoopUtil.delete(getConf(), getTempPath()); } fs.mkdirs(getOutputPath()); // MAHOUT-817 if (pca && xiPath == null) { xiPath = new Path(tempPath, "xi"); if (overwrite) { fs.delete(xiPath, true); } MatrixColumnMeansJob.run(conf, inputPaths[0], xiPath); } SSVDSolver solver = new SSVDSolver(conf, inputPaths, new Path(tempPath, "ssvd"), r, k, p, reduceTasks); solver.setMinSplitSize(minSplitSize); solver.setComputeU(computeU); solver.setComputeV(computeV); solver.setcUHalfSigma(cUHalfSigma); solver.setcVHalfSigma(cVHalfSigma); solver.setcUSigma(cUSigma); solver.setOuterBlockHeight(h); solver.setAbtBlockHeight(abh); solver.setQ(q); solver.setBroadcast(broadcast); solver.setOverwrite(overwrite); if (xiPath != null) { solver.setPcaMeanPath(new Path(xiPath, "part-*")); } solver.run(); Vector svalues = solver.getSingularValues().viewPart(0, k); SSVDHelper.saveVector(svalues, getOutputPath("sigma"), conf); if (computeU && !fs.rename(new Path(solver.getUPath()), getOutputPath())) { throw new IOException("Unable to move U results to the output path."); } if (cUHalfSigma && !fs.rename(new Path(solver.getuHalfSigmaPath()), getOutputPath())) { throw new IOException("Unable to move U*Sigma^0.5 results to the output path."); } if (cUSigma && !fs.rename(new Path(solver.getuSigmaPath()), getOutputPath())) { throw new IOException("Unable to move U*Sigma results to the output path."); } if (computeV && !fs.rename(new Path(solver.getVPath()), getOutputPath())) { throw new IOException("Unable to move V results to the output path."); } if (cVHalfSigma && !fs.rename(new Path(solver.getvHalfSigmaPath()), getOutputPath())) { throw new IOException("Unable to move V*Sigma^0.5 results to the output path."); } // Delete the temp path on exit fs.deleteOnExit(getTempPath()); return 0; }
From source file:org.apache.phoenix.mapreduce.MultiHfileOutputFormat.java
License:Apache License
/** * Configure <code>job</code> with a TotalOrderPartitioner, partitioning against * <code>splitPoints</code>. Cleans up the partitions file after job exists. *//*from ww w . j a v a 2s.com*/ static void configurePartitioner(Job job, Set<TableRowkeyPair> tablesStartKeys) throws IOException { Configuration conf = job.getConfiguration(); // create the partitions file Path partitionsPath = new Path(conf.get("hadoop.tmp.dir"), "partitions_" + UUID.randomUUID()); FileSystem fs = partitionsPath.getFileSystem(conf); fs.makeQualified(partitionsPath); writePartitions(conf, partitionsPath, tablesStartKeys); fs.deleteOnExit(partitionsPath); // configure job to use it job.setPartitionerClass(TotalOrderPartitioner.class); TotalOrderPartitioner.setPartitionFile(conf, partitionsPath); }
From source file:org.apache.pirk.test.distributed.testsuite.DistTestSuite.java
License:Apache License
@SuppressWarnings("unused") public static List<QueryResponseJSON> performQuery(String queryType, ArrayList<String> selectors, FileSystem fs, boolean isSpark, int numThreads, boolean isStreaming) throws Exception { logger.info("performQuery: "); String queryInputDir = SystemConfiguration.getProperty(DistributedTestDriver.PIR_QUERY_INPUT_DIR); String outputFile = SystemConfiguration.getProperty(DistributedTestDriver.OUTPUT_DIRECTORY_PROPERTY); fs.delete(new Path(outputFile), true); // Ensure old output does not exist. SystemConfiguration.setProperty("pir.queryInput", queryInputDir); SystemConfiguration.setProperty("pir.outputFile", outputFile); SystemConfiguration.setProperty("pir.numReduceTasks", "1"); SystemConfiguration.setProperty("pir.stopListFile", SystemConfiguration.getProperty(DistributedTestDriver.PIR_STOPLIST_FILE)); // Create the temp result file File fileFinalResults = File.createTempFile("finalResultsFile", ".txt"); fileFinalResults.deleteOnExit();//from w ww.j a v a 2 s .c o m logger.info("fileFinalResults = " + fileFinalResults.getAbsolutePath()); boolean embedSelector = SystemConfiguration.getBooleanProperty("pirTest.embedSelector", false); boolean useExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useExpLookupTable", false); boolean useHDFSExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false); // Set the necessary objects QueryInfo queryInfo = new QueryInfo(BaseTests.queryIdentifier, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize, queryType, useExpLookupTable, embedSelector, useHDFSExpLookupTable); Paillier paillier = new Paillier(BaseTests.paillierBitSize, BaseTests.certainty); // Perform the encryption logger.info("Performing encryption of the selectors - forming encrypted query vectors:"); EncryptQuery encryptQuery = new EncryptQuery(queryInfo, selectors, paillier); Querier querier = encryptQuery.encrypt(numThreads); logger.info("Completed encryption of the selectors - completed formation of the encrypted query vectors:"); // Write the Query object to a file Path queryInputDirPath = new Path(queryInputDir); new HadoopFileSystemStore(fs).store(queryInputDirPath, querier.getQuery()); fs.deleteOnExit(queryInputDirPath); // Grab the original data and query schema properties to reset upon completion String dataSchemaProp = SystemConfiguration.getProperty("data.schemas"); String querySchemaProp = SystemConfiguration.getProperty("query.schemas"); // Get the correct input format class name JSONInputFormatBase jFormat = new JSONInputFormatBase(); String jsonBaseInputFormatString = jFormat.getClass().getName(); SystemConfiguration.setProperty("pir.baseInputFormat", jsonBaseInputFormatString); // Submitting the tool for encrypted query logger.info("Performing encrypted query:"); if (isSpark) { logger.info("spark.home = " + SystemConfiguration.getProperty("spark.home")); // Build args String inputFormat = SystemConfiguration.getProperty("pir.dataInputFormat"); logger.info("inputFormat = " + inputFormat); ArrayList<String> args = new ArrayList<>(); if (isStreaming) { logger.info("platform = sparkstreaming"); args.add("-" + ResponderProps.PLATFORM + "=sparkstreaming"); args.add("-" + ResponderProps.BATCHSECONDS + "=" + SystemConfiguration.getProperty("pir.sparkstreaming.batchSeconds", "30")); args.add("-" + ResponderProps.WINDOWLENGTH + "=" + SystemConfiguration.getProperty("pir.sparkstreaming.windowLength", "60")); args.add("-" + ResponderProps.MAXBATCHES + "=" + SystemConfiguration.getProperty("pir.sparkstreaming.maxBatches", "-1")); args.add("-" + ResponderProps.STOPGRACEFULLY + "=" + SystemConfiguration.getProperty("spark.streaming.stopGracefullyOnShutdown", "false")); args.add("-" + ResponderProps.NUMDATAPARTITIONS + "=" + SystemConfiguration.getProperty("pir.numDataPartitions", "3")); args.add("-" + ResponderProps.USEQUEUESTREAM + "=" + SystemConfiguration.getProperty("pir.sparkstreaming.useQueueStream", "false")); } else { logger.info("platform = spark"); args.add("-" + ResponderProps.PLATFORM + "=spark"); } args.add("-" + ResponderProps.DATAINPUTFORMAT + "=" + inputFormat); args.add("-" + ResponderProps.QUERYINPUT + "=" + SystemConfiguration.getProperty("pir.queryInput")); args.add("-" + ResponderProps.OUTPUTFILE + "=" + SystemConfiguration.getProperty("pir.outputFile")); args.add("-" + ResponderProps.STOPLISTFILE + "=" + SystemConfiguration.getProperty("pir.stopListFile")); args.add("-" + ResponderProps.USELOCALCACHE + "=" + SystemConfiguration.getProperty("pir.useLocalCache", "true")); args.add("-" + ResponderProps.LIMITHITSPERSELECTOR + "=" + SystemConfiguration.getProperty("pir.limitHitsPerSelector", "false")); args.add("-" + ResponderProps.MAXHITSPERSELECTOR + "=" + SystemConfiguration.getProperty("pir.maxHitsPerSelector", "1000")); args.add("-" + ResponderProps.QUERYSCHEMAS + "=" + Inputs.HDFS_QUERY_FILES); args.add("-" + ResponderProps.DATASCHEMAS + "=" + Inputs.DATA_SCHEMA_FILE_HDFS); args.add("-" + ResponderProps.NUMEXPLOOKUPPARTS + "=" + SystemConfiguration.getProperty("pir.numExpLookupPartitions", "100")); args.add("-" + ResponderProps.USEMODEXPJOIN + "=" + SystemConfiguration.getProperty("pir.useModExpJoin", "false")); args.add("-" + ResponderProps.NUMCOLMULTPARTITIONS + "=" + SystemConfiguration.getProperty("pir.numColMultPartitions", "20")); args.add("-" + ResponderProps.COLMULTREDUCEBYKEY + "=" + SystemConfiguration.getProperty("pir.colMultReduceByKey", "false")); if (inputFormat.equals(InputFormatConst.BASE_FORMAT)) { args.add("-" + ResponderProps.INPUTDATA + "=" + SystemConfiguration.getProperty("pir.inputData")); args.add("-" + ResponderProps.BASEQUERY + "=" + SystemConfiguration.getProperty("pir.baseQuery")); args.add("-" + ResponderProps.BASEINPUTFORMAT + "=" + SystemConfiguration.getProperty("pir.baseInputFormat")); } else if (inputFormat.equals(InputFormatConst.ES)) { args.add("-" + ResponderProps.ESQUERY + "=" + SystemConfiguration.getProperty("pir.esQuery")); args.add("-" + ResponderProps.ESRESOURCE + "=" + SystemConfiguration.getProperty("pir.esResource")); args.add("-" + ResponderProps.ESNODES + "=" + SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_NODES_PROPERTY)); args.add("-" + ResponderProps.ESPORT + "=" + SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_PORT_PROPERTY)); } for (String arg : args) { logger.info("arg = " + arg); } // Run spark application Process sLauncher = new SparkLauncher().setAppResource(SystemConfiguration.getProperty("jarFile")) .setSparkHome(SystemConfiguration.getProperty("spark.home")) .setMainClass("org.apache.pirk.responder.wideskies.ResponderDriver") .addAppArgs(args.toArray(new String[args.size()])).setMaster("yarn-cluster") .setConf(SparkLauncher.EXECUTOR_MEMORY, "2g").setConf(SparkLauncher.DRIVER_MEMORY, "2g") .setConf(SparkLauncher.EXECUTOR_CORES, "1").launch(); sLauncher.waitFor(); } else { SystemConfiguration.setProperty("data.schemas", Inputs.DATA_SCHEMA_FILE_HDFS); SystemConfiguration.setProperty("query.schemas", Inputs.HDFS_QUERY_FILES); ComputeResponseTool responseTool = new ComputeResponseTool(); ToolRunner.run(responseTool, new String[] {}); } logger.info("Completed encrypted query"); // Perform decryption // Reconstruct the necessary objects from the files logger.info("Performing decryption; writing final results file"); if (isStreaming) { outputFile = outputFile + "_0"; // currently only processing one batch for testing } logger.info("Pulling results from outputFile = " + outputFile); Response response = new HadoopFileSystemStore(fs).recall(outputFile, Response.class); // Perform decryption and output the result file DecryptResponse decryptResponse = new DecryptResponse(response, querier); QueryResultsWriter.writeResultFile(fileFinalResults, decryptResponse.decrypt(numThreads)); logger.info("Completed performing decryption and writing final results file"); // Read in results logger.info("Reading in and checking results"); List<QueryResponseJSON> results = TestUtils.readResultsFile(fileFinalResults); // Reset data and query schema properties SystemConfiguration.setProperty("data.schemas", dataSchemaProp); SystemConfiguration.setProperty("query.schemas", querySchemaProp); // Clean up output dir in hdfs fs.delete(new Path(outputFile), true); return results; }
From source file:org.apache.pirk.test.utils.Inputs.java
License:Apache License
/** * Create the test data schema file/*from w w w.jav a 2 s .c o m*/ */ private static void createDataSchema(FileSystem fs, boolean hdfs) throws IOException { // Create a temporary file for the test schema, set in the properties File file = File.createTempFile(DATA_SCHEMA_FILE_LOCALFS, ".xml"); file.deleteOnExit(); logger.info("file = " + file.toString()); SystemConfiguration.setProperty("data.schemas", file.toString()); // If we are performing distributed testing, write both the local and hdfs files OutputStreamWriter osw = null; if (hdfs) { Path filePath = new Path(DATA_SCHEMA_FILE_HDFS); fs.deleteOnExit(filePath); osw = new OutputStreamWriter(fs.create(filePath, true)); logger.info("hdfs: filePath = " + filePath.toString()); } // Write to the file try { DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); Document doc = dBuilder.newDocument(); // root element Element rootElement = doc.createElement("schema"); doc.appendChild(rootElement); // Add the schemaName Element schemaNameElement = doc.createElement("schemaName"); schemaNameElement.appendChild(doc.createTextNode(TEST_DATA_SCHEMA_NAME)); rootElement.appendChild(schemaNameElement); String primitiveTypePartitionerName = PrimitiveTypePartitioner.class.getName(); String ipPartitionerName = IPDataPartitioner.class.getName(); String datePartitioner = ISO8601DatePartitioner.class.getName(); // date TestUtils.addElement(doc, rootElement, DATE, PrimitiveTypePartitioner.STRING, "false", datePartitioner); // qname TestUtils.addElement(doc, rootElement, QNAME, PrimitiveTypePartitioner.STRING, "false", primitiveTypePartitionerName); // src_ip TestUtils.addElement(doc, rootElement, SRCIP, PrimitiveTypePartitioner.STRING, "false", ipPartitionerName); // dest_ip TestUtils.addElement(doc, rootElement, DSTIP, PrimitiveTypePartitioner.STRING, "false", ipPartitionerName); // qtype TestUtils.addElement(doc, rootElement, QTYPE, PrimitiveTypePartitioner.SHORT, "true", primitiveTypePartitionerName); // rcode TestUtils.addElement(doc, rootElement, RCODE, PrimitiveTypePartitioner.INT, "false", primitiveTypePartitionerName); // ip TestUtils.addElement(doc, rootElement, IPS, PrimitiveTypePartitioner.STRING, "true", ipPartitionerName); // Write to a xml file - both localFS and hdfs TransformerFactory transformerFactory = TransformerFactory.newInstance(); Transformer transformer = transformerFactory.newTransformer(); DOMSource source = new DOMSource(doc); // LocalFS StreamResult resultLocalFS = new StreamResult(file); transformer.transform(source, resultLocalFS); if (hdfs) { StreamResult resultHDFS = new StreamResult(osw); transformer.transform(source, resultHDFS); } // Output for testing StreamResult consoleResult = new StreamResult(System.out); transformer.transform(source, consoleResult); System.out.println(); if (osw != null) { osw.close(); } } catch (Exception e) { e.printStackTrace(); } }
From source file:org.apache.pirk.test.utils.TestUtils.java
License:Apache License
/** * Creates the test query schema file/*from w w w . java 2 s. c o m*/ */ public static void createQuerySchema(String schemaFile, String querySchemaName, String dataSchemaNameInput, String selectorNameInput, List<String> elementNames, List<String> filterNames, String filter, boolean append, FileSystem fs, boolean hdfs, HashMap<String, String> additionalFields) throws IOException { logger.info("createQuerySchema: querySchemaName = " + querySchemaName); // Create a temporary file for the test schema, set in the properties String fileName; File file = null; OutputStreamWriter osw = null; if (hdfs) { Path filePath = new Path(schemaFile); fs.deleteOnExit(filePath); fileName = filePath.toString(); osw = new OutputStreamWriter(fs.create(filePath, true)); logger.info("hdfs: filePath = " + fileName); } else { file = File.createTempFile(schemaFile, ".xml"); file.deleteOnExit(); fileName = file.toString(); logger.info("localFS: file = " + file.toString()); } if (append) { String currentSchemas = SystemConfiguration.getProperty("query.schemas", ""); if (currentSchemas.equals("") || currentSchemas.equals("none")) { SystemConfiguration.setProperty("query.schemas", fileName); } else { SystemConfiguration.setProperty("query.schemas", SystemConfiguration.getProperty("query.schemas", "") + "," + fileName); } } logger.info("query.schemas = " + SystemConfiguration.getProperty("query.schemas")); // Write to the file try { DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); Document doc = dBuilder.newDocument(); // root element Element rootElement = doc.createElement("schema"); doc.appendChild(rootElement); // Add the schemaName Element schemaNameElement = doc.createElement("schemaName"); schemaNameElement.appendChild(doc.createTextNode(querySchemaName)); rootElement.appendChild(schemaNameElement); // Add the dataSchemaName Element dataSchemaNameElement = doc.createElement("dataSchemaName"); dataSchemaNameElement.appendChild(doc.createTextNode(dataSchemaNameInput)); rootElement.appendChild(dataSchemaNameElement); // Add the selectorName Element selectorNameElement = doc.createElement("selectorName"); selectorNameElement.appendChild(doc.createTextNode(selectorNameInput)); rootElement.appendChild(selectorNameElement); // Add the elementNames Element elements = doc.createElement("elements"); rootElement.appendChild(elements); for (String elementName : elementNames) { logger.info("elementName = " + elementName); Element name = doc.createElement("name"); name.appendChild(doc.createTextNode(elementName)); elements.appendChild(name); } // Add the filter if (filter != null) { Element filterElement = doc.createElement("filter"); filterElement.appendChild(doc.createTextNode(filter)); rootElement.appendChild(filterElement); // Add the filterNames Element filterNamesElement = doc.createElement("filterNames"); rootElement.appendChild(filterNamesElement); for (String filterName : filterNames) { logger.info("filterName = " + filterName); Element name = doc.createElement("name"); name.appendChild(doc.createTextNode(filterName)); filterNamesElement.appendChild(name); } } // Add the additionalFields if (additionalFields != null) { Element additionalElement = doc.createElement("additional"); rootElement.appendChild(additionalElement); // Add the key,value pairs for (String key : additionalFields.keySet()) { logger.info("Creating field element with key = " + key + " and value = " + additionalFields.get(key)); Element fieldElement = doc.createElement("field"); additionalElement.appendChild(fieldElement); Element keyElement = doc.createElement("key"); keyElement.appendChild(doc.createTextNode(key)); fieldElement.appendChild(keyElement); Element valueElement = doc.createElement("value"); valueElement.appendChild(doc.createTextNode(additionalFields.get(key))); fieldElement.appendChild(valueElement); } } // Write to a xml file TransformerFactory transformerFactory = TransformerFactory.newInstance(); Transformer transformer = transformerFactory.newTransformer(); DOMSource source = new DOMSource(doc); StreamResult result; if (hdfs) { result = new StreamResult(osw); } else { result = new StreamResult(file); } transformer.transform(source, result); // Output for testing StreamResult consoleResult = new StreamResult(System.out); transformer.transform(source, consoleResult); System.out.println(); if (osw != null) { osw.close(); } } catch (Exception e) { e.printStackTrace(); } }
From source file:org.apache.pirk.utils.HDFS.java
License:Apache License
public static void writeFile(Collection<String> elements, FileSystem fs, String path, boolean deleteOnExit) { Path filePath = new Path(path); try {//www . j av a 2 s . c o m // create writer BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(filePath, true))); // write each element on a new line for (String element : elements) { bw.write(element); bw.newLine(); } bw.close(); // delete file once the filesystem is closed if (deleteOnExit) { fs.deleteOnExit(filePath); } } catch (IOException e) { e.printStackTrace(); } }
From source file:org.apache.pirk.utils.HDFS.java
License:Apache License
public static void writeFile(List<JSONObject> elements, FileSystem fs, String path, boolean deleteOnExit) { Path filePath = new Path(path); try {//www . j av a2s .co m // create writer BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(filePath, true))); // write each element on a new line for (JSONObject element : elements) { bw.write(element.toString()); bw.newLine(); } bw.close(); // delete file once the filesystem is closed if (deleteOnExit) { fs.deleteOnExit(filePath); } } catch (IOException e) { e.printStackTrace(); } }
From source file:org.apache.pirk.utils.HDFS.java
License:Apache License
public static void writeFile(List<BigInteger> elements, FileSystem fs, Path path, boolean deleteOnExit) { try {//from w ww . ja v a 2 s. c om // create writer BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(path, true))); // write each element on a new line for (BigInteger element : elements) { bw.write(element.toString()); bw.newLine(); } bw.close(); // delete file once the filesystem is closed if (deleteOnExit) { fs.deleteOnExit(path); } } catch (IOException e) { e.printStackTrace(); } }
From source file:org.apache.pirk.utils.HDFS.java
License:Apache License
public static void writeFile(Map<String, Integer> sortedMap, FileSystem fs, String path, boolean deleteOnExit) { Path filePath = new Path(path); try {// w w w . ja v a 2s .co m // create writer BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(filePath, true))); // write each element on a new line for (Entry<String, Integer> entry : sortedMap.entrySet()) { bw.write(entry.getKey() + "," + entry.getValue()); bw.newLine(); } bw.close(); // delete file once the filesystem is closed if (deleteOnExit) { fs.deleteOnExit(filePath); } } catch (IOException e) { e.printStackTrace(); } }