Example usage for org.apache.hadoop.fs Path toString

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toString.

Prototype

@Override
    public String toString() 

Source Link

Usage

From source file:com.cloudera.crunch.impl.mr.run.CrunchInputs.java

License:Apache License

public static void addInputPath(Job job, Path path, Class<? extends InputFormat> inputFormatClass,
        int nodeIndex) {
    Configuration conf = job.getConfiguration();
    String inputs = JOINER.join(inputFormatClass.getName(), nodeIndex, path.toString());
    String existing = conf.get(RuntimeParameters.MULTI_INPUTS);
    conf.set(RuntimeParameters.MULTI_INPUTS, existing == null ? inputs : existing + RECORD_SEP + inputs);
}

---

From source file:com.cloudera.crunch.io.text.TextFileSource.java

License:Open Source License

private static boolean isBZip2(Path path) {
    String strPath = path.toString();
    return strPath.endsWith(".bz") || strPath.endsWith(".bz2");
}

---

From source file:com.cloudera.crunch.util.DistCache.java

License:Open Source License

public static Object read(Configuration conf, Path path) throws IOException {
    URI target = null;//  ww  w .ja  va2s  . c o m
    for (URI uri : DistributedCache.getCacheFiles(conf)) {
        if (uri.toString().equals(path.toString())) {
            target = uri;
            break;
        }
    }
    Object value = null;
    if (target != null) {
        Path targetPath = new Path(target.toString());
        ObjectInputStream ois = new ObjectInputStream(targetPath.getFileSystem(conf).open(targetPath));
        try {
            value = ois.readObject();
        } catch (ClassNotFoundException e) {
            throw new CrunchRuntimeException(e);
        }
        ois.close();
    }
    return value;
}

---

From source file:com.cloudera.fts.spark.format.RawFileRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {

    Configuration conf = context.getConfiguration();
    FileSplit split = (FileSplit) inputSplit;
    Path path = split.getPath();
    FileSystem fs = path.getFileSystem(conf);
    fileIn = fs.open(path);/*  www.  ja v  a2 s .c  om*/
    key = new Text(path.toString());
    finished = false;
}

---

From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.state.HDFSState.java

License:Apache License

/**
 * Open a file handle for write/*ww w .ja v  a 2s  .co  m*/
 * @param stateID
 * @param fileHandle
 * @param overwrite
 * @throws NFS4Exception
 * @throws IOException
 */
public synchronized HDFSOutputStream openForWrite(FileSystem fs, StateID stateID, FileHandle fileHandle,
        boolean overwrite) throws NFS4Exception, IOException {
    HDFSFile hdsfsFile = mOpenFilesMap.get(fileHandle);
    if (hdsfsFile != null) {
        OpenResource<HDFSOutputStream> file = hdsfsFile.getHDFSOutputStreamForWrite();
        if (file != null) {
            if (file.isOwnedBy(stateID)) {
                return file.get();
            }
            throw new NFS4Exception(NFS4ERR_FILE_OPEN);
        }
    }
    INode inode = mFileHandleINodeMap.getINodeByFileHandle(fileHandle);
    if (inode == null) {
        throw new NFS4Exception(NFS4ERR_STALE);
    }
    Path path = new Path(inode.getPath());
    boolean exists = fs.exists(path);
    // If overwrite = false, fs.create throws IOException which
    // is useless. In case of IOE do we always return EXIST?
    // doesn't seem to make sense. As such, I am mitigating the issue
    // even if there is a known race between the exists and create
    if (!overwrite && exists) {
        // append to a file
        // We used to be NFS4ERR_EXIST here but the linux client behaved
        // rather oddly. It would open the file with overwrite=true but
        // then send the data which was to be appended at offset 0
        throw new NFS4Exception(NFS4ERR_PERM, "File Exists and overwrite = false");
    }
    if (path.getParent() != null) {
        // TODO bad perms will fail with IOException, perhaps we should check
        // that file can be created before trying to so we can return the
        // correct error perm denied
        // check(user, groups, status, access);
    }
    if (exists && fs.getFileStatus(path).isDir()) {
        throw new NFS4Exception(NFS4ERR_ISDIR);
    }
    HDFSOutputStream out = new HDFSOutputStream(fs.create(path, overwrite), path.toString(), fileHandle);
    mMetrics.incrementMetric(FILES_OPENED_WRITE, 1);
    if (hdsfsFile == null) {
        hdsfsFile = new HDFSFile(fileHandle, inode.getPath(), inode.getNumber());
        mOpenFilesMap.put(fileHandle, hdsfsFile);
    }
    hdsfsFile.setHDFSOutputStream(stateID, out);
    return out;
}

---

From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.TestWithClient.java

License:Apache License

public void doReadDir(BaseClient client) throws IOException, InterruptedException, NFS4Exception {
    /*/*w ww.ja v  a  2 s.  co m*/
     * traverse through a directory that does not change often and ensure it
     * checks out the same as through the native api
     */
    Path rootPath = new Path("/");
    Path etcPath = new Path(TestUtils.tmpDirPathForTest);
    compareFileStatusFile(client.getFileStatus(rootPath));
    ImmutableList<Path> paths = client.listPath(new Path(rootPath, etcPath));
    File etcFile = new File(rootPath.toString(), etcPath.toString());
    assertEquals(etcFile.list().length, paths.size());
    for (Path path : paths) {
        LOGGER.debug("checking file => " + path.getName());
        compareFileStatusFile(client.getFileStatus(path));
    }

    client.shutdown();
}

---

From source file:com.cloudera.hoop.client.fs.HoopFileSystem.java

License:Open Source License

/**
 * Renames Path src to Path dst.  Can take place on local fs
 * or remote DFS.//ww  w .jav  a 2 s  .  c om
 */
@Override
public boolean rename(Path src, Path dst) throws IOException {
    Map<String, String> params = new HashMap<String, String>();
    params.put("op", "rename");
    params.put("to", dst.toString());
    HttpURLConnection conn = getConnection("PUT", params, src);
    validateResponse(conn, HttpURLConnection.HTTP_OK);
    JSONObject json = (JSONObject) jsonParse(conn);
    return (Boolean) json.get("rename");
}

---

From source file:com.cloudera.hoop.fs.FSHomeDir.java

License:Open Source License

/**
 * Executes the filesystem operation./*from   www . jav a2s  .c  o  m*/
 *
 * @param fs filesystem instance to use.
 * @return a JSON object with the user home directory.
 * @throws IOException thrown if an IO error occured.
 */
@Override
@SuppressWarnings("unchecked")
public JSONObject execute(FileSystem fs) throws IOException {
    Path homeDir = fs.getHomeDirectory();
    JSONObject json = new JSONObject();
    homeDir = FSUtils.convertPathToHoop(homeDir, HoopServer.get().getBaseUrl());
    json.put("homeDir", homeDir.toString());
    return json;
}

---

From source file:com.cloudera.impala.common.FileSystemUtil.java

License:Apache License

/**
 * Moves (renames) the given file to a new location (either another directory or a
 * file. If renameIfAlreadyExists is true, no error will be thrown if a file with the
 * same name already exists in the destination location. Instead, a UUID will be
 * appended to the base file name, preserving the the existing file extension.
 * If renameIfAlreadyExists is false, an IOException will be thrown if there is a
 * file name conflict.//  w ww  . j a  va  2 s .  c  om
 */
public static void moveFile(Path sourceFile, Path dest, boolean renameIfAlreadyExists) throws IOException {
    FileSystem fs = dest.getFileSystem(CONF);

    Path destFile = fs.isDirectory(dest) ? new Path(dest, sourceFile.getName()) : dest;
    // If a file with the same name does not already exist in the destination location
    // then use the same file name. Otherwise, generate a unique file name.
    if (renameIfAlreadyExists && fs.exists(destFile)) {
        Path destDir = fs.isDirectory(dest) ? dest : dest.getParent();
        destFile = new Path(destDir, appendToBaseFileName(destFile.getName(), UUID.randomUUID().toString()));
    }
    LOG.debug(String.format("Moving '%s' to '%s'", sourceFile.toString(), destFile.toString()));
    // Move (rename) the file.
    fs.rename(sourceFile, destFile);
}

---

From source file:com.cloudera.impala.planner.PlannerTestBase.java

License:Apache License

/**
 * Construct a string representation of the scan ranges for this request.
 *//* w ww . j a v  a  2 s  . co  m*/
private StringBuilder printScanRangeLocations(TQueryExecRequest execRequest) {
    StringBuilder result = new StringBuilder();
    if (execRequest.per_node_scan_ranges == null) {
        return result;
    }
    for (Map.Entry<Integer, List<TScanRangeLocations>> entry : execRequest.per_node_scan_ranges.entrySet()) {
        result.append("NODE " + entry.getKey().toString() + ":\n");
        if (entry.getValue() == null) {
            continue;
        }

        for (TScanRangeLocations locations : entry.getValue()) {
            // print scan range
            result.append("  ");
            if (locations.scan_range.isSetHdfs_file_split()) {
                THdfsFileSplit split = locations.scan_range.getHdfs_file_split();
                THdfsTable table = findTable(entry.getKey());
                THdfsPartition partition = table.getPartitions().get(split.partition_id);
                THdfsPartitionLocation location = partition.getLocation();
                String file_location = location.getSuffix();
                if (location.prefix_index != -1) {
                    file_location = table.getPartition_prefixes().get(location.prefix_index) + file_location;
                }
                Path filePath = new Path(file_location, split.file_name);
                filePath = cleanseFilePath(filePath);
                result.append("HDFS SPLIT " + filePath.toString() + " " + Long.toString(split.offset) + ":"
                        + Long.toString(split.length));
            }
            if (locations.scan_range.isSetHbase_key_range()) {
                THBaseKeyRange keyRange = locations.scan_range.getHbase_key_range();
                Integer hostIdx = locations.locations.get(0).host_idx;
                TNetworkAddress networkAddress = execRequest.getHost_list().get(hostIdx);
                result.append("HBASE KEYRANGE ");
                result.append("port=" + networkAddress.port + " ");
                if (keyRange.isSetStartKey()) {
                    result.append(HBaseScanNode.printKey(keyRange.getStartKey().getBytes()));
                } else {
                    result.append("<unbounded>");
                }
                result.append(":");
                if (keyRange.isSetStopKey()) {
                    result.append(HBaseScanNode.printKey(keyRange.getStopKey().getBytes()));
                } else {
                    result.append("<unbounded>");
                }
            }

            if (locations.scan_range.isSetKudu_key_range()) {
                TKuduKeyRange kr = locations.scan_range.getKudu_key_range();
                Integer hostIdx = locations.locations.get(0).host_idx;
                TNetworkAddress networkAddress = execRequest.getHost_list().get(hostIdx);
                result.append("KUDU KEYRANGE ");
                // TODO Enable the lines below once we have better testing for
                //      non-local key-ranges
                //result.append("host=" + networkAddress.hostname + ":" +
                //    networkAddress.port + " ");
                result.append(Arrays.toString(kr.getRange_start_key()));
                result.append(":");
                result.append(Arrays.toString(kr.getRange_stop_key()));
            }

            result.append("\n");
        }
    }
    return result;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »