Example usage for org.apache.hadoop.fs Path getParent

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getParent.

Prototype

public Path getParent() 

Source Link

Document

Returns the parent of a path or null if at root.

Usage

From source file:com.ceph.rados.fs.hdfs.RadosFileSystem.java

License:Apache License

@Override
public boolean rename(Path src, Path dst) throws IOException {
    Path absoluteSrc = makeAbsolute(src);
    final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
    INode srcINode = store.retrieveINode(absoluteSrc);
    boolean debugEnabled = LOG.isDebugEnabled();
    if (srcINode == null) {
        // src path doesn't exist
        if (debugEnabled) {
            LOG.debug(debugPreamble + "returning false as src does not exist");
        }//  w  w  w .  j  a  va  2s .co m
        return false;
    }

    Path absoluteDst = makeAbsolute(dst);

    //validate the parent dir of the destination
    Path dstParent = absoluteDst.getParent();
    if (dstParent != null) {
        //if the dst parent is not root, make sure it exists
        INode dstParentINode = store.retrieveINode(dstParent);
        if (dstParentINode == null) {
            // dst parent doesn't exist
            if (debugEnabled) {
                LOG.debug(debugPreamble + "returning false as dst parent does not exist");
            }
            return false;
        }
        if (dstParentINode.isFile()) {
            // dst parent exists but is a file
            if (debugEnabled) {
                LOG.debug(debugPreamble + "returning false as dst parent exists and is a file");
            }
            return false;
        }
    }

    //get status of source
    boolean srcIsFile = srcINode.isFile();

    INode dstINode = store.retrieveINode(absoluteDst);
    boolean destExists = dstINode != null;
    boolean destIsDir = destExists && !dstINode.isFile();
    if (srcIsFile) {

        //source is a simple file
        if (destExists) {
            if (destIsDir) {
                //outcome #1 dest exists and is dir -filename to subdir of dest
                if (debugEnabled) {
                    LOG.debug(debugPreamble + "copying src file under dest dir to " + absoluteDst);
                }
                absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
            } else {
                //outcome #2 dest it's a file: fail iff different from src
                boolean renamingOnToSelf = absoluteSrc.equals(absoluteDst);
                if (debugEnabled) {
                    LOG.debug(debugPreamble + "copying file onto file, outcome is " + renamingOnToSelf);
                }
                return renamingOnToSelf;
            }
        } else {
            // #3 dest does not exist: use dest as path for rename
            if (debugEnabled) {
                LOG.debug(debugPreamble + "copying file onto file");
            }
        }
    } else {
        //here the source exists and is a directory
        // outcomes (given we know the parent dir exists if we get this far)
        // #1 destination is a file: fail
        // #2 destination is a directory: create a new dir under that one
        // #3 destination doesn't exist: create a new dir with that name
        // #3 and #4 are only allowed if the dest path is not == or under src

        if (destExists) {
            if (!destIsDir) {
                // #1 destination is a file: fail
                if (debugEnabled) {
                    LOG.debug(debugPreamble + "returning false as src is a directory, but not dest");
                }
                return false;
            } else {
                // the destination dir exists
                // destination for rename becomes a subdir of the target name
                absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
                if (debugEnabled) {
                    LOG.debug(debugPreamble + "copying src dir under dest dir to " + absoluteDst);
                }
            }
        }
        //the final destination directory is now know, so validate it for
        //illegal moves

        if (absoluteSrc.equals(absoluteDst)) {
            //you can't rename a directory onto itself
            if (debugEnabled) {
                LOG.debug(debugPreamble + "Dest==source && isDir -failing");
            }
            return false;
        }
        if (absoluteDst.toString().startsWith(absoluteSrc.toString() + "/")) {
            //you can't move a directory under itself
            if (debugEnabled) {
                LOG.debug(debugPreamble + "dst is equal to or under src dir -failing");
            }
            return false;
        }
    }
    //here the dest path is set up -so rename
    return renameRecursive(absoluteSrc, absoluteDst);
}

---

From source file:com.cloudera.CacheTool.java

License:Apache License

public static void createFile(FileSystem fs, Path fileName, long fileLen) throws IOException {
    int bufferLen = 1024;
    assert bufferLen > 0;
    if (!fs.mkdirs(fileName.getParent())) {
        throw new IOException("Mkdirs failed to create " + fileName.getParent().toString());
    }//ww w . jav a2s  .co  m
    FSDataOutputStream out = null;
    try {
        out = fs.create(fileName, true,
                fs.getConf().getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), (short) 1,
                fs.getDefaultBlockSize(fileName));
        if (fileLen > 0) {
            byte[] toWrite = new byte[bufferLen];
            Random rb = new Random(0);
            long bytesToWrite = fileLen;
            while (bytesToWrite > 0) {
                rb.nextBytes(toWrite);
                int bytesToWriteNext = (bufferLen < bytesToWrite) ? bufferLen : (int) bytesToWrite;

                out.write(toWrite, 0, bytesToWriteNext);
                bytesToWrite -= bytesToWriteNext;
            }
        }
    } finally {
        if (out != null) {
            out.close();
        }
    }
}

---

From source file:com.cloudera.cdk.data.filesystem.FileSystemDatasetWriter.java

License:Apache License

public FileSystemDatasetWriter(FileSystem fileSystem, Path path, Schema schema, boolean enableCompression) {

    this.fileSystem = fileSystem;
    this.path = path;
    this.pathTmp = new Path(path.getParent(), "." + path.getName() + ".tmp");
    this.schema = schema;
    this.enableCompression = enableCompression;
    this.state = ReaderWriterState.NEW;
}

---

From source file:com.cloudera.cdk.data.filesystem.FileSystemView.java

License:Apache License

private static boolean cleanlyDelete(FileSystem fs, Path root, Path dir) {
    try {/*from   w  w  w . j  ava  2 s  . co m*/
        boolean deleted = false;
        if (dir.isAbsolute()) {
            deleted = fs.delete(dir, true /* include any files */ );
        } else {
            // the path should be treated as relative to the root path
            Path absolute = new Path(root, dir);
            deleted = fs.delete(absolute, true /* include any files */ );
            // iterate up to the root, removing empty directories
            for (Path current = absolute.getParent(); !current.equals(root)
                    && !current.isRoot(); current = current.getParent()) {
                final FileStatus[] stats = fs.listStatus(current);
                if (stats == null || stats.length == 0) {
                    // dir is empty and should be removed
                    deleted = fs.delete(current, true) || deleted;
                } else {
                    // all parent directories will be non-empty
                    break;
                }
            }
        }
        return deleted;
    } catch (IOException ex) {
        throw new DatasetIOException("Could not cleanly delete path:" + dir, ex);
    }
}

---

From source file:com.cloudera.cdk.data.filesystem.ParquetFileSystemDatasetWriter.java

License:Apache License

public ParquetFileSystemDatasetWriter(FileSystem fileSystem, Path path, Schema schema,
        boolean enableCompression) {
    this.fileSystem = fileSystem;
    this.path = path;
    this.pathTmp = new Path(path.getParent(), "." + path.getName() + ".tmp");
    this.schema = schema;
    this.enableCompression = enableCompression;
    this.state = ReaderWriterState.NEW;
}

---

From source file:com.cloudera.cdk.data.filesystem.PathConversion.java

License:Apache License

public StorageKey toKey(Path fromPath, StorageKey storage) {
    final List<FieldPartitioner> partitioners = storage.getPartitionStrategy().getFieldPartitioners();
    final List<Object> values = Lists.newArrayList(new Object[partitioners.size()]);

    Path currentPath = fromPath;
    int index = partitioners.size() - 1;
    while (currentPath != null && index >= 0) {
        values.set(index,//ww w.jav a2s  . co m
                valueForDirname((FieldPartitioner<?, ?>) partitioners.get(index), currentPath.getName()));

        // update
        currentPath = currentPath.getParent();
        index -= 1;
    }

    storage.replaceValues(values);
    return storage;
}

---

From source file:com.cloudera.cdk.morphline.hadoop.core.DownloadHdfsFileTest.java

License:Apache License

@Test
public void testBasic() throws IOException {
    String msg = "hello world";

    // setup: copy a file to HDFS to prepare inputFile    
    Path inputFile = fileSystem.makeQualified(new Path(testDirectory, fileName));
    FSDataOutputStream out = fileSystem.create(inputFile);
    IOUtils.copyBytes(new ByteArrayInputStream(msg.getBytes(Charsets.UTF_8)), out, fileSystem.getConf());
    out.close();/*from   w  w w  .  j av a2 s . c o  m*/

    File cwd = Files.createTempDir().getAbsoluteFile();
    if (isDir) {
        dst = new File(cwd, testDirectory.getName() + "/" + inputFile.getName());
        inputFile = inputFile.getParent();
    } else {
        dst = new File(cwd, inputFile.getName());
    }
    Assert.assertFalse(dst.exists());
    new File(cwd, fileName).mkdirs(); // will be auto deleted!
    Files.write("wrong msg", new File(new File(cwd, fileName), fileName), Charsets.UTF_8); // will be auto deleted!

    Command morphline = createMorphline("test-morphlines/testDownloadHdfsFile", inputFile, cwd);
    Assert.assertTrue(morphline.process(new Record()));
    Assert.assertEquals(msg, Files.toString(dst, Charsets.UTF_8));
    if (isDir) {
        FileUtil.fullyDelete(dst.getParentFile());
    } else {
        FileUtil.fullyDelete(dst);
    }
    Assert.assertTrue(fileSystem.exists(inputFile));
    Assert.assertTrue(FileUtil.fullyDelete(cwd));

    // verify that subsequent calls with same inputFile won't copy the file again (to prevent races)
    morphline = createMorphline("test-morphlines/downloadHdfsFile", inputFile, cwd);
    Assert.assertTrue(morphline.process(new Record()));
    Assert.assertFalse(dst.exists());
    Assert.assertTrue(morphline.process(new Record()));
    Assert.assertFalse(dst.exists());
    Assert.assertFalse(cwd.exists());

    Assert.assertTrue(fileSystem.delete(inputFile, true));

    try {
        morphline = createMorphline("test-morphlines/downloadHdfsFile", new Path("nonExistingInputFile"), cwd);
        Assert.fail("failed to detect non-existing input file");
    } catch (MorphlineCompilationException e) {
        Assert.assertTrue(e.getCause() instanceof FileNotFoundException);
    }
    Assert.assertFalse(dst.exists());
}

---

From source file:com.cloudera.data.filesystem.ParquetFileSystemDatasetWriter.java

License:Apache License

public ParquetFileSystemDatasetWriter(FileSystem fileSystem, Path path, Schema schema) {
    this.fileSystem = fileSystem;
    this.path = path;
    this.pathTmp = new Path(path.getParent(), "." + path.getName() + ".tmp");
    this.schema = schema;
    this.state = ReaderWriterState.NEW;
}

---

From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.BaseClient.java

License:Apache License

protected FileHandle lookup(Path path) throws NFS4Exception {
    Path parent;/* w w w  .j ava 2 s .  c  o  m*/
    LOGGER.info("Lookup on " + path);
    if (path.equals(ROOT)) {
        parent = path;
    } else {
        parent = path.getParent();
    }
    FileHandle parentFileHandle = mPathFileHandleMap.get(parent);
    if (parentFileHandle == null) {
        parentFileHandle = lookup(parent);
    }

    if (parent.equals(path)) {
        return parentFileHandle;
    }

    CompoundRequest compoundRequest = newRequest();
    List<OperationRequest> operations = Lists.newArrayList();
    PUTFHRequest putFhRequest = new PUTFHRequest();
    putFhRequest.setFileHandle(parentFileHandle);
    operations.add(putFhRequest);
    LOOKUPRequest lookupRequest = new LOOKUPRequest();
    lookupRequest.setName(path.getName());
    operations.add(lookupRequest);

    operations.add(new GETFHRequest());
    operations.add(newGETATTRRequest());

    compoundRequest.setOperations(operations);

    List<OperationResponse> operationResponses = getResult(compoundRequest);

    getResponse(operationResponses.remove(0), PUTFHResponse.class);
    getResponse(operationResponses.remove(0), LOOKUPResponse.class);
    GETFHResponse getFHResponse = getResponse(operationResponses.remove(0), GETFHResponse.class);
    FileHandle fileHandle = getFHResponse.getFileHandle();
    mPathFileHandleMap.put(path, fileHandle);
    mFileHandlePathMap.put(fileHandle, path);
    GETATTRResponse getAttrResponse = getResponse(operationResponses.remove(0), GETATTRResponse.class);
    mFileHandleAttributeMap.put(fileHandle, getAttrResponse.getAttrValues());
    return fileHandle;
}

---

From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.BaseClient.java

License:Apache License

public OutputStream forWrite(final Path path) throws Exception {
    setClientIDIfUnset();//from w w  w  . jav a2s .  c o  m

    final FileHandle parentFileHandle = checkNotNull(lookup(path.getParent()));
    final StateID stateID = checkNotNull(
            doOpen(parentFileHandle, path.getName(), NFS4_OPEN4_SHARE_ACCESS_WRITE, NFS4_OPEN4_CREATE));
    final FileHandle fileHandle = checkNotNull(mPathFileHandleMap.get(path));

    return new OutputStream() {

        protected long fileOffset = 0L;

        @Override
        public void write(int b) throws IOException {
            CompoundRequest compoundRequest = newRequest();
            List<OperationRequest> operations = Lists.newArrayList();
            PUTFHRequest putFhRequest = new PUTFHRequest();
            putFhRequest.setFileHandle(fileHandle);
            operations.add(putFhRequest);

            WRITERequest writeRequest = new WRITERequest();
            byte[] data = new byte[1];
            data[0] = (byte) b;
            writeRequest.setData(data, 0, data.length);
            writeRequest.setOffset(fileOffset);
            writeRequest.setStable(NFS4_COMMIT_UNSTABLE4);
            writeRequest.setStateID(stateID);

            operations.add(writeRequest);

            compoundRequest.setOperations(operations);
            List<OperationResponse> operationResponses;
            try {
                operationResponses = getResult(compoundRequest);
            } catch (NFS4Exception e) {
                throw new RuntimeException(e);
            }
            getResponse(operationResponses.remove(0), PUTFHResponse.class);

            WRITEResponse writeResponse = getResponse(operationResponses.remove(0), WRITEResponse.class);
            if (writeResponse.getCount() != data.length) {
                throw new IOException("Write failed: " + writeResponse.getCount());
            }
            fileOffset++;
        }

        @Override
        public void close() throws IOException {

            CompoundRequest compoundRequest = newRequest();
            List<OperationRequest> operations = Lists.newArrayList();
            PUTFHRequest putFhRequest = new PUTFHRequest();
            putFhRequest.setFileHandle(fileHandle);
            operations.add(putFhRequest);

            COMMITRequest commitRequest = new COMMITRequest();
            commitRequest.setCount(0);
            commitRequest.setOffset(0);
            operations.add(commitRequest);

            CLOSERequest closeRequest = new CLOSERequest();
            closeRequest.setSeqID(stateID.getSeqID() + 1);
            closeRequest.setStateID(stateID);
            operations.add(closeRequest);

            compoundRequest.setOperations(operations);
            List<OperationResponse> operationResponses;
            try {
                operationResponses = getResult(compoundRequest);
            } catch (NFS4Exception e) {
                throw new RuntimeException(e);
            }
            getResponse(operationResponses.remove(0), PUTFHResponse.class);

            getResponse(operationResponses.remove(0), COMMITResponse.class);

            CLOSEResponse closeResponse = getResponse(operationResponses.remove(0), CLOSEResponse.class);

            mFileHandleStateID.put(fileHandle, closeResponse.getStateID());
        }
    };
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »