List of usage examples for org.apache.hadoop.fs FileSystem exists
public boolean exists(Path f) throws IOException
From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.NFS4Handler.java
License:Apache License
/** * * @param stateID/*from w ww. j av a 2s . com*/ * @param fs * @param fileHandle * @param overwrite * @return * @throws NFS4Exception * @throws IOException */ public synchronized FSDataOutputStream forWrite(StateID stateID, FileSystem fs, FileHandle fileHandle, boolean overwrite) throws NFS4Exception, IOException { FileHolder fileHolder = mFileHandleMap.get(fileHandle); if (fileHolder != null) { OpenFile<FSDataOutputStream> file = fileHolder.getFSDataOutputStream(); if (file != null) { if (file.isOwnedBy(stateID)) { return file.get(); } throw new NFS4Exception(NFS4ERR_FILE_OPEN); } Path path = new Path(fileHolder.getPath()); boolean exists = fs.exists(path); // If overwrite = false, fs.create throws IOException which // is useless. In case of IOE do we always return EXIST? // doesn't seem to make sense. As such, I am mitigating the issue // even if there is a known race between the exists and create if (!overwrite && exists) { // append to a file // We used to be NFS4ERR_EXIST here but the linux client behaved rather // oddly. // It would open the fily with overwrite=true but then send the data // which // was to be appended at offset 0 throw new NFS4Exception(NFS4ERR_PERM, "File Exists and overwrite = false", true); } if (path.getParent() != null) { // TODO bad perms will fail with IOException, perhaps we should check // that file can be created before trying to so we can return the // correct error perm denied } if (exists && fs.getFileStatus(path).isDir()) { throw new NFS4Exception(NFS4ERR_ISDIR); } FSDataOutputStream out = fs.create(path, overwrite); this.incrementMetric("FILES_OPENED_WRITE", 1); fileHolder.setFSDataOutputStream(stateID, out); return out; } throw new NFS4Exception(NFS4ERR_STALE); }
From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.state.HDFSState.java
License:Apache License
/** * Files which are in the process of being created need to exist from a NFS * perspective even if they do not exist from an HDFS perspective. This * method intercepts requests for files that are open and calls * FileSystem.exists for other files./*from ww w .j ava 2 s. co m*/ * * @param path * @return true if the file exists or is open for write * @throws IOException */ public synchronized boolean fileExists(FileSystem fs, Path path) throws IOException { FileHandle fileHandle = getOrCreateFileHandle(path); HDFSFile hdfsFile = mOpenFilesMap.get(fileHandle); if ((hdfsFile != null) && hdfsFile.isOpenForWrite()) { return true; } return fs.exists(path); }
From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.state.HDFSState.java
License:Apache License
/** * Open a file handle for write/* w w w . ja v a 2 s .c o m*/ * @param stateID * @param fileHandle * @param overwrite * @throws NFS4Exception * @throws IOException */ public synchronized HDFSOutputStream openForWrite(FileSystem fs, StateID stateID, FileHandle fileHandle, boolean overwrite) throws NFS4Exception, IOException { HDFSFile hdsfsFile = mOpenFilesMap.get(fileHandle); if (hdsfsFile != null) { OpenResource<HDFSOutputStream> file = hdsfsFile.getHDFSOutputStreamForWrite(); if (file != null) { if (file.isOwnedBy(stateID)) { return file.get(); } throw new NFS4Exception(NFS4ERR_FILE_OPEN); } } INode inode = mFileHandleINodeMap.getINodeByFileHandle(fileHandle); if (inode == null) { throw new NFS4Exception(NFS4ERR_STALE); } Path path = new Path(inode.getPath()); boolean exists = fs.exists(path); // If overwrite = false, fs.create throws IOException which // is useless. In case of IOE do we always return EXIST? // doesn't seem to make sense. As such, I am mitigating the issue // even if there is a known race between the exists and create if (!overwrite && exists) { // append to a file // We used to be NFS4ERR_EXIST here but the linux client behaved // rather oddly. It would open the file with overwrite=true but // then send the data which was to be appended at offset 0 throw new NFS4Exception(NFS4ERR_PERM, "File Exists and overwrite = false"); } if (path.getParent() != null) { // TODO bad perms will fail with IOException, perhaps we should check // that file can be created before trying to so we can return the // correct error perm denied // check(user, groups, status, access); } if (exists && fs.getFileStatus(path).isDir()) { throw new NFS4Exception(NFS4ERR_ISDIR); } HDFSOutputStream out = new HDFSOutputStream(fs.create(path, overwrite), path.toString(), fileHandle); mMetrics.incrementMetric(FILES_OPENED_WRITE, 1); if (hdsfsFile == null) { hdsfsFile = new HDFSFile(fileHandle, inode.getPath(), inode.getNumber()); mOpenFilesMap.put(fileHandle, hdsfsFile); } hdsfsFile.setHDFSOutputStream(stateID, out); return out; }
From source file:com.cloudera.HioBench.java
License:Apache License
public static void main(String[] args) throws Exception { options = new Options(); final Configuration conf = new Configuration(); if (options.dumpConf) { Configuration.dumpConfiguration(conf, new PrintWriter(System.out)); }/* w ww .ja va 2 s . co m*/ final FileSystem fs = FileSystem.get(new URI(options.hdfsUri), conf); fs.setVerifyChecksum(!options.skipChecksum); if (!fs.exists(options.filePath)) { System.out.println("no file at " + options.filePath + "; writing " + "new file now with length " + options.nGigsInFile + " gigs..."); writeFile(fs); System.out.println("done."); } else if (fs.getLength(options.filePath) != options.nBytesInFile) { System.out.println("existing file " + options.filename + " has length " + fs.getLength(options.filePath) + ", but we wanted length " + options.nBytesInFile + ". Re-creating."); writeFile(fs); System.out.println("done."); } else { System.out.println( "using existing file at " + options.filePath + " of length " + options.nGigsInFile + " gigs."); } long nanoStart = System.nanoTime(); WorkerThread threads[] = new WorkerThread[options.nThreads]; for (int i = 0; i < options.nThreads; i++) { threads[i] = new WorkerThread(i == 0, fs, WorkerThread.createBenchReader(options, i)); } for (int i = 0; i < options.nThreads; i++) { threads[i].start(); } for (int i = 0; i < options.nThreads; i++) { threads[i].join(); } for (int i = 0; i < options.nThreads; i++) { Throwable t = threads[i].getException(); if (t != null) { System.err.println("there were exceptions. Aborting."); System.exit(1); } } long nanoEnd = System.nanoTime(); fs.close(); long totalIo = options.nThreads; totalIo *= options.nBytesToRead; float nanoDiff = nanoEnd - nanoStart; float seconds = nanoDiff / 1000000000; System.out.println(String.format("Using %d threads, read %s in %f seconds", options.nThreads, prettyPrintByteSize(totalIo), seconds)); float rate = totalIo / seconds; System.out.println("Average rate was " + prettyPrintByteSize(rate) + "/s"); }
From source file:com.cloudera.hive.scd.SQLUpdater.java
License:Open Source License
private List<String> loadUpdateStatements(InputSplit split, JobConf jc) throws IOException { long currentSCDTime = asSCDTime(jc.get("scd.time", ""), System.currentTimeMillis()); List<String> stmts = Lists.newArrayList(); if (split instanceof FileSplit) { Path base = ((FileSplit) split).getPath(); FileSystem fs = base.getFileSystem(jc); Path updates = new Path(base.getParent(), ".updates"); if (fs.exists(updates)) { stmts.addAll(readLines(fs, updates, currentSCDTime)); }//from w w w. j a v a2 s .c o m } return stmts; }
From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java
License:Open Source License
private void testRename() throws Exception { FileSystem fs = FileSystem.get(getHadoopConf()); Path path = new Path(getHadoopTestDir(), "foo"); fs.mkdirs(path);/*www . jav a 2 s. c o m*/ fs.close(); Configuration conf = new Configuration(); conf.set("fs.http.impl", HoopFileSystem.class.getName()); fs = FileSystem.get(getJettyURL().toURI(), conf); Path oldPath = new Path(path.toUri().getPath()); Path newPath = new Path(path.getParent(), "bar"); fs.rename(oldPath, newPath); fs.close(); fs = FileSystem.get(getHadoopConf()); Assert.assertFalse(fs.exists(oldPath)); Assert.assertTrue(fs.exists(newPath)); fs.close(); }
From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java
License:Open Source License
private void testDelete() throws Exception { Path foo = new Path(getHadoopTestDir(), "foo"); Path bar = new Path(getHadoopTestDir(), "bar"); Path foe = new Path(getHadoopTestDir(), "foe"); FileSystem fs = FileSystem.get(getHadoopConf()); fs.mkdirs(foo);/*from w ww . j a v a2s . c o m*/ fs.mkdirs(new Path(bar, "a")); fs.mkdirs(foe); Configuration conf = new Configuration(); conf.set("fs.http.impl", HoopFileSystem.class.getName()); FileSystem hoopFs = FileSystem.get(getJettyURL().toURI(), conf); Assert.assertTrue(hoopFs.delete(new Path(foo.toUri().getPath()), false)); Assert.assertFalse(fs.exists(foo)); try { hoopFs.delete(new Path(bar.toUri().getPath()), false); Assert.fail(); } catch (IOException ex) { } catch (Exception ex) { Assert.fail(); } Assert.assertTrue(fs.exists(bar)); Assert.assertTrue(hoopFs.delete(new Path(bar.toUri().getPath()), true)); Assert.assertFalse(fs.exists(bar)); Assert.assertTrue(fs.exists(foe)); Assert.assertTrue(hoopFs.delete(foe)); Assert.assertFalse(fs.exists(foe)); hoopFs.close(); fs.close(); }
From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java
License:Open Source License
private void testMkdirs() throws Exception { Path path = new Path(getHadoopTestDir(), "foo"); Configuration conf = new Configuration(); conf.set("fs.http.impl", HoopFileSystem.class.getName()); FileSystem fs = FileSystem.get(getJettyURL().toURI(), conf); fs.mkdirs(path);//from www. java 2s. co m fs.close(); fs = FileSystem.get(getHadoopConf()); Assert.assertTrue(fs.exists(path)); fs.close(); }
From source file:com.cloudera.impala.catalog.HdfsTable.java
License:Apache License
/** * Create HdfsPartition objects corresponding to 'partitions'. * * If there are no partitions in the Hive metadata, a single partition is added with no * partition keys.//from w ww. j a va 2 s . c o m * * For files that have not been changed, reuses file descriptors from oldFileDescMap. * * TODO: If any partition fails to load, the entire table will fail to load. Instead, * we should consider skipping partitions that cannot be loaded and raise a warning * whenever the table is accessed. */ private void loadPartitions(List<org.apache.hadoop.hive.metastore.api.Partition> msPartitions, org.apache.hadoop.hive.metastore.api.Table msTbl, Map<String, List<FileDescriptor>> oldFileDescMap) throws IOException, CatalogException { resetPartitionMd(); partitions_.clear(); hdfsBaseDir_ = msTbl.getSd().getLocation(); // Map of filesystem to parent path to a list of new/modified // FileDescriptors. FileDescriptors in this Map will have their block location // information (re)loaded. This is used to speed up the incremental refresh of a // table's metadata by skipping unmodified, previously loaded FileDescriptors. Map<FsKey, Map<String, List<FileDescriptor>>> fileDescsToLoad = Maps.newHashMap(); // INSERT statements need to refer to this if they try to write to new partitions // Scans don't refer to this because by definition all partitions they refer to // exist. addDefaultPartition(msTbl.getSd()); Long cacheDirectiveId = HdfsCachingUtil.getCacheDirIdFromParams(msTbl.getParameters()); isMarkedCached_ = cacheDirectiveId != null; if (msTbl.getPartitionKeysSize() == 0) { Preconditions.checkArgument(msPartitions == null || msPartitions.isEmpty()); // This table has no partition key, which means it has no declared partitions. // We model partitions slightly differently to Hive - every file must exist in a // partition, so add a single partition with no keys which will get all the // files in the table's root directory. HdfsPartition part = createPartition(msTbl.getSd(), null, oldFileDescMap, fileDescsToLoad); addPartition(part); if (isMarkedCached_) part.markCached(); Path location = new Path(hdfsBaseDir_); FileSystem fs = location.getFileSystem(CONF); if (fs.exists(location)) { accessLevel_ = getAvailableAccessLevel(fs, location); } } else { for (org.apache.hadoop.hive.metastore.api.Partition msPartition : msPartitions) { HdfsPartition partition = createPartition(msPartition.getSd(), msPartition, oldFileDescMap, fileDescsToLoad); addPartition(partition); // If the partition is null, its HDFS path does not exist, and it was not added to // this table's partition list. Skip the partition. if (partition == null) continue; if (msPartition.getParameters() != null) ; { partition.setNumRows(getRowCount(msPartition.getParameters())); } if (!TAccessLevelUtil.impliesWriteAccess(partition.getAccessLevel())) { // TODO: READ_ONLY isn't exactly correct because the it's possible the // partition does not have READ permissions either. When we start checking // whether we can READ from a table, this should be updated to set the // table's access level to the "lowest" effective level across all // partitions. That is, if one partition has READ_ONLY and another has // WRITE_ONLY the table's access level should be NONE. accessLevel_ = TAccessLevel.READ_ONLY; } } } loadBlockMd(fileDescsToLoad); }
From source file:com.cloudera.impala.catalog.HdfsTable.java
License:Apache License
/** * Gets the AccessLevel that is available for Impala for this table based on the * permissions Impala has on the given path. If the path does not exist, recurses up the * path until a existing parent directory is found, and inherit access permissions from * that.//from w w w . j a v a2s .c o m */ private TAccessLevel getAvailableAccessLevel(FileSystem fs, Path location) throws IOException { FsPermissionChecker permissionChecker = FsPermissionChecker.getInstance(); while (location != null) { if (fs.exists(location)) { FsPermissionChecker.Permissions perms = permissionChecker.getPermissions(fs, location); if (perms.canReadAndWrite()) { return TAccessLevel.READ_WRITE; } else if (perms.canRead()) { LOG.debug(String.format("Impala does not have WRITE access to '%s' in table: %s", location, getFullName())); return TAccessLevel.READ_ONLY; } else if (perms.canWrite()) { LOG.debug(String.format("Impala does not have READ access to '%s' in table: %s", location, getFullName())); return TAccessLevel.WRITE_ONLY; } LOG.debug(String.format("Impala does not have READ or WRITE access to " + "'%s' in table: %s", location, getFullName())); return TAccessLevel.NONE; } location = location.getParent(); } // Should never get here. Preconditions.checkNotNull(location, "Error: no path ancestor exists"); return TAccessLevel.NONE; }