List of usage examples for org.apache.hadoop.fs FileSystem mkdirs
public boolean mkdirs(Path f) throws IOException
From source file:com.ibm.crail.hdfs.tools.HdfsIOBenchmark.java
License:Apache License
void createFile() throws Exception, InterruptedException { System.out.println("create file async hdfs, path " + path + ", size " + size + ", loop " + loop); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); int repfactor = 4; for (int k = 0; k < repfactor; k++) { LinkedBlockingQueue<Path> pathQueue = new LinkedBlockingQueue<Path>(); fs.mkdirs(path); for (int i = 0; i < loop * size; i++) { String name = "" + i; Path f = new Path(path, name); pathQueue.add(f);/*from w w w .j av a 2 s . c o m*/ } LinkedBlockingQueue<FSDataOutputStream> streamQueue = new LinkedBlockingQueue<FSDataOutputStream>(); long start = System.currentTimeMillis(); for (int i = 0; i < size; i++) { //single operation == loop for (int j = 0; j < loop; j++) { Path path = pathQueue.poll(); fs.create(path).close(); } } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)); double latency = executionTime * 1000.0 / ((double) size); System.out.println("execution time [ms] " + executionTime); System.out.println("latency [us] " + latency); while (!streamQueue.isEmpty()) { FSDataOutputStream stream = streamQueue.poll(); stream.close(); } if (k < repfactor - 1) { fs.delete(path, true); Thread.sleep(2000); } } fs.close(); }
From source file:com.ibm.jaql.io.hadoop.CompositeOutputAdapter.java
License:Apache License
@Override public void checkOutputSpecs(FileSystem ignored, JobConf conf) throws IOException { for (int i = 0; i < outputs.length; i++) { outputs[i].checkOutputSpecs(ignored, subconfs[i]); // HACK: Hadoop 0.18 has hacks that specialize FileOutputFormat handling. In particular, // the temporary directory is created by the Task or LocalJobRunner; they also promote // the temporary files to the parent upon completion. We create the temporary file here, // if it doesn't already exist. On Path outputPath = FileOutputFormat.getOutputPath(subconfs[i]); if (outputPath != null) { final String TEMP_DIR_NAME = "_temporary"; // MRConstants isn't public... Path jobTmpDir = new Path(outputPath, TEMP_DIR_NAME); // MRConstants.TEMP_DIR_NAME FileSystem fs = jobTmpDir.getFileSystem(subconfs[i]); if (!fs.exists(jobTmpDir)) { fs.mkdirs(jobTmpDir); }/* ww w . j a v a 2 s.c o m*/ } } }
From source file:com.ibm.jaql.io.hadoop.DirectFileOutputCommiter.java
License:Apache License
@Override public void setupJob(JobContext context) throws IOException { // Create the path to the file, if needed. JobConf conf = context.getJobConf(); Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath != null) { Path tmpDir = outputPath.getParent(); FileSystem fileSys = outputPath.getFileSystem(conf); if (!fileSys.mkdirs(outputPath.getParent())) { throw new IOException("Mkdirs failed to create " + tmpDir.toString()); }// ww w . jav a2 s. c om } }
From source file:com.iflytek.spider.crawl.CrawlDb.java
License:Apache License
public static void install(Job job, Path crawlDb) throws IOException { Path newCrawlDb = FileOutputFormat.getOutputPath(job); FileSystem fs = FileSystem.get(job.getConfiguration()); Path old = new Path(crawlDb, "old"); Path current = new Path(crawlDb, CURRENT_NAME); if (fs.exists(current)) { if (fs.exists(old)) fs.delete(old, true);/*from w ww. j a v a 2s . c o m*/ fs.rename(current, old); } fs.mkdirs(crawlDb); fs.rename(newCrawlDb, current); if (fs.exists(old)) fs.delete(old, true); Path lock = new Path(crawlDb, LOCK_NAME); LockUtil.removeLockFile(fs, lock); }
From source file:com.iflytek.spider.util.LockUtil.java
License:Apache License
/** * Create a lock file.//from w w w . j a va2 s . co m * @param fs filesystem * @param lockFile name of the lock file * @param accept if true, and the target file exists, consider it valid. If false * and the target file exists, throw an IOException. * @throws IOException if accept is false, and the target file already exists, * or if it's a directory. */ public static void createLockFile(FileSystem fs, Path lockFile, boolean accept) throws IOException { if (fs.exists(lockFile)) { if (!accept) throw new IOException("lock file " + lockFile + " already exists."); if (fs.getFileStatus(lockFile).isDir()) throw new IOException("lock file " + lockFile + " already exists and is a directory."); // do nothing - the file already exists. } else { // make sure parents exist fs.mkdirs(lockFile.getParent()); fs.createNewFile(lockFile); } }
From source file:com.ikanow.infinit.e.core.mapreduce.HadoopJobRunner.java
License:Open Source License
private void bringTempOutputToFront(CustomMapReduceJobPojo cmr) throws IOException, SAXException, ParserConfigurationException { // Get the names: Configuration config = HadoopUtils.getConfiguration(prop_custom); FileSystem fs = FileSystem.get(config); Path pathTmp = HadoopUtils.getPathForJob(cmr, config, true); Path pathFinal = HadoopUtils.getPathForJob(cmr, config, false); // OK don't do anything if pathTmp doesn't exist... if (fs.exists(pathTmp)) { // If the final path exists, delete it if (!fs.exists(pathFinal)) { // create it, which guarantees the parent path also exists //(otherwise the rename fails sigh) fs.mkdirs(pathFinal); }/* w ww . j ava 2s.c o m*/ fs.delete(pathFinal, true); fs.rename(pathTmp, pathFinal); } }
From source file:com.ikanow.infinit.e.processing.custom.utils.InfiniteHadoopUtils.java
License:Open Source License
public static void bringTempOutputToFront(CustomMapReduceJobPojo cmr, PropertiesManager prop_custom) throws IOException, SAXException, ParserConfigurationException { // Get the names: Configuration config = HadoopUtils.getConfiguration(prop_custom); FileSystem fs = FileSystem.get(config); Path pathTmp = HadoopUtils.getPathForJob(cmr, config, true); Path pathFinal = HadoopUtils.getPathForJob(cmr, config, false); // OK don't do anything if pathTmp doesn't exist... if (fs.exists(pathTmp)) { // If the final path exists, delete it if (!fs.exists(pathFinal)) { // create it, which guarantees the parent path also exists //(otherwise the rename fails sigh) fs.mkdirs(pathFinal); }/* ww w . j av a2 s. co m*/ fs.delete(pathFinal, true); fs.rename(pathTmp, pathFinal); } }
From source file:com.ikanow.infinit.e.processing.custom.utils.InfiniteHadoopUtils.java
License:Open Source License
public static Path cacheLocalFile(String localPath, String localName, Configuration config) throws IOException { FileSystem fs = FileSystem.get(config); Path toDir = new Path("cache"); Path destFile = new Path("cache/" + localName); File fromFile = new File(localPath + "/" + localName); if (!fromFile.exists()) { throw new IOException("Source file does not exist: " + fromFile.toString()); }// ww w. ja v a2 s . com boolean needToCopyFile = true; if (!fs.exists(toDir)) { // (ie relative to WD) fs.mkdirs(toDir); } else { // Now check if the file already exists if (fs.exists(destFile)) { FileStatus fsStat = fs.getFileStatus(destFile); if ((fsStat.getLen() == fromFile.length()) && (fromFile.lastModified() <= fsStat.getModificationTime())) { needToCopyFile = false; } } } if (needToCopyFile) { fs.copyFromLocalFile(false, true, new Path(localPath + "/" + localName), destFile); } return new Path(fs.getFileStatus(destFile).getPath().toUri().getPath()); // (apparently the path has to be in absolute format without even the hdfs:// at the front?!) }
From source file:com.indeed.imhotep.builder.tsv.TsvConverter.java
License:Apache License
/** * * @return true if upload succeeded// www. j a v a2 s .c o m */ private static boolean uploadShard(String localShardDir, String shardName, String indexName, Path finalIndexPath, FileSystem finalFS, boolean qaMode) { final Path finalIndexDirPath = new Path(finalIndexPath, indexName); final Path finalShardPath = new Path(finalIndexDirPath, shardName + ".sqar"); try { if (!finalFS.exists(finalIndexDirPath)) { finalFS.mkdirs(finalIndexDirPath); if (qaMode) { makeWorldWritable(finalFS, finalIndexDirPath); } } if (finalFS.exists(finalShardPath)) { log.info("File already exists. HDFS upload aborted."); return true; } final String scheme = finalFS.getUri().getScheme(); if (scheme.equals("hdfs")) { /* * upload to temp file then rename, * to avoid having other systems see a partial file */ final String tmpUploadShardName = indexName + "-" + shardName; final Path tempUploadPath = new Path(new Path("/tmp/"), tmpUploadShardName + ".sqar"); final File shardDir = new File(localShardDir, shardName); final SquallArchiveWriter writer = new SquallArchiveWriter(finalFS, tempUploadPath, true, SquallArchiveCompressor.GZIP); writer.batchAppendDirectory(shardDir); writer.commit(); finalFS.rename(tempUploadPath, finalShardPath); } else if (scheme.equals("s3n")) { /* * s3 files are only visible after the upload is complete, * so no need to use a temp file */ final File shardDir = new File(localShardDir, shardName); final SquallArchiveWriter writer = new SquallArchiveWriter(finalFS, finalShardPath, true, SquallArchiveCompressor.GZIP); writer.batchAppendDirectory(shardDir); writer.commit(); } } catch (IOException e) { log.error(e); return false; } if (qaMode) { try { // try to set permissions on the uploaded file makeWorldWritable(finalFS, finalShardPath); } catch (Exception e) { log.warn("Failed to set permissions on the uploaded file " + finalShardPath); } } return true; }
From source file:com.inmobi.conduit.AbstractService.java
License:Apache License
protected void publishMissingPaths(FileSystem fs, String destDir, long commitTime, String categoryName) throws Exception { Long prevRuntime = new Long(-1); if (!prevRuntimeForCategory.containsKey(categoryName)) { LOG.debug("Calculating Previous Runtime from Directory Listing"); prevRuntime = getPreviousRuntime(fs, destDir, categoryName); } else {// w w w . ja v a 2 s. c o m LOG.debug("Reading Previous Runtime from Cache"); prevRuntime = prevRuntimeForCategory.get(categoryName); } if (prevRuntime != -1) { if (isMissingPaths(commitTime, prevRuntime)) { LOG.debug("Previous Runtime: [" + getLogDateString(prevRuntime) + "]"); Set<Path> pathsToBeRegistered = null; String tableName = null; if (isStreamHCatEnabled(categoryName)) { tableName = getTableName(categoryName); pathsToBeRegistered = pathsToBeregisteredPerTable.get(tableName); } while (isMissingPaths(commitTime, prevRuntime)) { String missingPath = Cluster.getDestDir(destDir, categoryName, prevRuntime); Path missingDir = new Path(missingPath); if (!fs.exists(missingDir)) { LOG.info("Creating Missing Directory [" + missingDir + "]"); fs.mkdirs(missingDir); if (isStreamHCatEnabled(categoryName)) { synchronized (pathsToBeRegistered) { pathsToBeRegistered.add(missingDir); } } ConduitMetrics.updateSWGuage(getServiceType(), EMPTYDIR_CREATE, categoryName, 1); } prevRuntime += MILLISECONDS_IN_MINUTE; } if (isStreamHCatEnabled(categoryName)) { pathsToBeregisteredPerTable.put(tableName, pathsToBeRegistered); } } } // prevRuntimeForCategory map is updated with commitTime, // even if prevRuntime is -1, since service did run at this point prevRuntimeForCategory.put(categoryName, commitTime); }