List of usage examples for org.apache.hadoop.fs FileSystem create
public FSDataOutputStream create(Path f, short replication) throws IOException
From source file:com.ibm.bi.dml.runtime.util.MapReduceTool.java
License:Open Source License
/** * /* w w w . j a v a 2 s .c o m*/ * @param filename * @param overwrite * @return * @throws IOException */ public static FSDataOutputStream getHDFSDataOutputStream(String filename, boolean overwrite) throws IOException { FileSystem fs = FileSystem.get(_rJob); Path path = new Path(filename); return fs.create(path, overwrite); }
From source file:com.ibm.bi.dml.test.integration.functions.transform.ScalingTest.java
License:Open Source License
private void generateSpecFile(int cols, String specFile) throws IOException, Exception { final String NAME = "name"; final String METHOD = "method"; final String SCALE_METHOD_Z = "z-score"; final String SCALE_METHOD_M = "mean-subtraction"; JSONObject outputSpec = new JSONObject(); JSONArray scaleSpec = new JSONArray(); for (int colID = 1; colID <= cols; colID++) { JSONObject obj = new JSONObject(); obj.put(NAME, "V" + colID); if (colID <= cols / 2) obj.put(METHOD, SCALE_METHOD_M); else/*from w w w.j a v a 2s . c o m*/ obj.put(METHOD, SCALE_METHOD_Z); scaleSpec.add(obj); } outputSpec.put(TX_METHOD.SCALE.toString(), scaleSpec); FileSystem fs = FileSystem.get(TestUtils.conf); BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(specFile), true))); out.write(outputSpec.toString()); out.close(); }
From source file:com.ibm.bi.dml.test.integration.functions.transform.ScalingTest.java
License:Open Source License
private void generateFrameMTD(String datafile) throws IllegalArgumentException, IOException, Exception { JSONObject mtd = new JSONObject(); mtd.put("data_type", "frame"); mtd.put("format", "csv"); mtd.put("header", false); FileSystem fs = FileSystem.get(TestUtils.conf); BufferedWriter out = new BufferedWriter( new OutputStreamWriter(fs.create(new Path(datafile + ".mtd"), true))); out.write(mtd.toString());/*from ww w.j a va 2s .com*/ out.close(); }
From source file:com.ibm.bi.dml.test.utils.TestUtils.java
License:Open Source License
public static void writeCSVTestMatrix(String file, double[][] matrix) { try {/* ww w . j ava 2s . c o m*/ //create outputstream to HDFS / FS and writer DataOutputStream out = null; FileSystem fs = FileSystem.get(conf); out = fs.create(new Path(file), true); BufferedWriter pw = new BufferedWriter(new OutputStreamWriter(out)); //writer actual matrix StringBuilder sb = new StringBuilder(); for (int i = 0; i < matrix.length; i++) { sb.setLength(0); if (matrix[i][0] != 0) sb.append(matrix[i][0]); for (int j = 1; j < matrix[i].length; j++) { sb.append(","); if (matrix[i][j] == 0) continue; sb.append(matrix[i][j]); } sb.append('\n'); pw.append(sb.toString()); } //close writer and streams pw.close(); out.close(); } catch (IOException e) { fail("unable to write (csv) test matrix (" + file + "): " + e.getMessage()); } }
From source file:com.ibm.bi.dml.test.utils.TestUtils.java
License:Open Source License
/** * <p>// w ww.ja v a 2 s . c o m * Writes a matrix to a file using the text format. * </p> * * @param file * file name * @param matrix * matrix * @param isR * when true, writes a R matrix to disk * */ public static void writeTestMatrix(String file, double[][] matrix, boolean isR) { try { //create outputstream to HDFS / FS and writer DataOutputStream out = null; if (!isR) { FileSystem fs = FileSystem.get(conf); out = fs.create(new Path(file), true); } else { out = new DataOutputStream(new FileOutputStream(file)); } BufferedWriter pw = new BufferedWriter(new OutputStreamWriter(out)); //write header if (isR) { /** add R header */ pw.append("%%MatrixMarket matrix coordinate real general\n"); pw.append("" + matrix.length + " " + matrix[0].length + " " + matrix.length * matrix[0].length + "\n"); } //writer actual matrix StringBuilder sb = new StringBuilder(); boolean emptyOutput = true; for (int i = 0; i < matrix.length; i++) { for (int j = 0; j < matrix[i].length; j++) { if (matrix[i][j] == 0) continue; sb.append(i + 1); sb.append(' '); sb.append(j + 1); sb.append(' '); sb.append(matrix[i][j]); sb.append('\n'); pw.append(sb.toString()); sb.setLength(0); emptyOutput = false; } } //writer dummy entry if empty if (emptyOutput) pw.append("1 1 " + matrix[0][0]); //close writer and streams pw.close(); out.close(); } catch (IOException e) { fail("unable to write test matrix (" + file + "): " + e.getMessage()); } }
From source file:com.ibm.bi.dml.yarn.DMLAppMaster.java
License:Open Source License
/** * //from www . j ava 2 s. c o m * @param msg */ private void writeMessageToHDFSWorkingDir(String msg) { //construct working directory (consistent with client) DMLConfig conf = ConfigurationManager.getConfig(); String hdfsWD = DMLAppMasterUtils.constructHDFSWorkingDir(conf, _appId); Path msgPath = new Path(hdfsWD, DMLYarnClient.DML_STOPMSG_NAME); //write given message to hdfs try { FileSystem fs = FileSystem.get(_conf); FSDataOutputStream fout = fs.create(msgPath, true); fout.writeBytes(msg); fout.close(); LOG.debug("Stop message written to HDFS file: " + msgPath); } catch (Exception ex) { LOG.error("Failed to write stop message to HDFS file: " + msgPath, ex); } }
From source file:com.ibm.bi.dml.yarn.DMLYarnClient.java
License:Open Source License
/** * //from w w w . j a v a 2s. co m * @param appId * @throws ParseException * @throws IOException * @throws DMLRuntimeException * @throws InterruptedException */ @SuppressWarnings("deprecation") private void copyResourcesToHdfsWorkingDir(YarnConfiguration yconf, String hdfsWD) throws ParseException, IOException, DMLRuntimeException, InterruptedException { FileSystem fs = FileSystem.get(yconf); //create working directory MapReduceTool.createDirIfNotExistOnHDFS(hdfsWD, DMLConfig.DEFAULT_SHARED_DIR_PERMISSION); //serialize the dml config to HDFS file //NOTE: we do not modify and ship the absolute scratch space path of the current user //because this might result in permission issues if the app master is run with a different user //(runtime plan migration during resource reoptimizations now needs to use qualified names //for shipping/reading intermediates) TODO modify resource reoptimizer on prototype integration. Path confPath = new Path(hdfsWD, DML_CONFIG_NAME); FSDataOutputStream fout = fs.create(confPath, true); //_dmlConfig.makeQualifiedScratchSpacePath(); fout.writeBytes(_dmlConfig.serializeDMLConfig() + "\n"); fout.close(); _hdfsDMLConfig = confPath.makeQualified(fs).toString(); LOG.debug("DML config written to HDFS file: " + _hdfsDMLConfig + ""); //serialize the dml script to HDFS file Path scriptPath = new Path(hdfsWD, DML_SCRIPT_NAME); FSDataOutputStream fout2 = fs.create(scriptPath, true); fout2.writeBytes(_dmlScript); fout2.close(); _hdfsDMLScript = scriptPath.makeQualified(fs).toString(); LOG.debug("DML script written to HDFS file: " + _hdfsDMLScript + ""); // copy local jar file to HDFS (try to get the original jar filename) String fname = getLocalJarFileNameFromEnvConst(); if (fname == null) { //get location of unpacked jar classes and repackage (if required) String lclassFile = DMLYarnClient.class.getProtectionDomain().getCodeSource().getLocation().getPath() .toString(); File flclassFile = new File(lclassFile); if (!flclassFile.isDirectory()) //called w/ jar fname = lclassFile; else //called w/ unpacked jar (need to be repackaged) fname = createJar(lclassFile); } Path srcPath = new Path(fname); Path dstPath = new Path(hdfsWD, srcPath.getName()); FileUtil.copy(FileSystem.getLocal(yconf), srcPath, fs, dstPath, false, true, yconf); _hdfsJarFile = dstPath.makeQualified(fs).toString(); LOG.debug( "Jar file copied from local file: " + srcPath.toString() + " to HDFS file: " + dstPath.toString()); }
From source file:com.ibm.jaql.lang.expr.system.RUtil.java
License:Apache License
/** * Function that puts a local file into HDFS. * @param localPath//from ww w.j av a 2 s .c o m * @param hdfsPath * @return */ public static boolean saveToHDFS(String localPath, String hdfsPath) { try { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); int bufferSize = 4 * 1024; byte[] buffer = new byte[bufferSize]; InputStream input = new BufferedInputStream(new FileInputStream(localPath), bufferSize); Path outputPath = new Path(hdfsPath); if (fs.exists(outputPath)) { if (!fs.isFile(outputPath)) { throw new IOException("Output path is a directory that already exists."); } LOG.info("Output path" + outputPath + " already exists. Overwriting it."); } FSDataOutputStream output = fs.create(outputPath, true); int numBytesRead; while ((numBytesRead = input.read(buffer)) > 0) { output.write(buffer, 0, numBytesRead); } input.close(); output.close(); return true; } catch (IOException e) { LOG.info("Error in writing file to HDFS.", e); return false; } }
From source file:com.ibm.stocator.fs.swift2d.systemtests.SwiftTestUtils.java
License:Open Source License
/** * Write the text to a file, returning the converted byte array * for use in validating the round trip//from www . j a va2s . c o m * @param fs filesystem * @param path path of file * @param text text to write * @param overwrite should the operation overwrite any existing file? * @return the read bytes * @throws IOException on IO problems */ public static byte[] writeTextFile(FileSystem fs, Path path, String text, boolean overwrite) throws IOException { FSDataOutputStream stream = fs.create(path, overwrite); byte[] bytes = new byte[0]; if (text != null) { bytes = toAsciiByteArray(text); stream.write(bytes); } stream.close(); return bytes; }
From source file:com.jbw.tar.sf.TarOutputFormat.java
@Override public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); String extension = ".tar"; Path file = getDefaultWorkFile(context, extension); FileSystem fs = file.getFileSystem(conf); OutputStream fileOut = fs.create(file, false); //tar?/* www .ja v a2 s . c om*/ return new TarOutputWriter<>(fileOut); }