Example usage for org.apache.hadoop.fs Path SEPARATOR

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path SEPARATOR.

Prototype

String SEPARATOR

To view the source code for org.apache.hadoop.fs Path SEPARATOR.

Click Source Link

Document

The directory separator, a slash.

Usage

From source file:com.datatorrent.stram.client.StatsAgent.java

License:Apache License

public String getStatsDirectory(String appId) {
    String appPath = stramAgent.getAppPath(appId);
    if (appPath == null) {
        return null;
    }//from  w  w  w  .  j  av a 2  s .c  o m
    return appPath + Path.SEPARATOR + "stats";
}

---

From source file:com.datatorrent.stram.FSRecoveryHandler.java

License:Apache License

@Override
public void save(Object state) throws IOException {

    if (fs.exists(snapshotBackupPath)) {
        throw new IllegalStateException("Found previous backup " + snapshotBackupPath);
    }/*from   w ww. j  a va  2 s .  co  m*/

    if (fs.exists(snapshotPath)) {
        LOG.debug("Backup {} to {}", snapshotPath, snapshotBackupPath);
        fs.rename(snapshotPath, snapshotBackupPath);
    }

    LOG.debug("Writing checkpoint to {}", snapshotPath);
    final FSDataOutputStream fsOutputStream = fs.create(snapshotPath);
    try {
        ObjectOutputStream oos = new ObjectOutputStream(fsOutputStream);
        try {
            oos.writeObject(state);
        } finally {
            oos.close();
        }
    } finally {
        fsOutputStream.close();
    }
    // remove snapshot backup
    if (fs.exists(snapshotBackupPath) && !fs.delete(snapshotBackupPath, false)) {
        throw new IOException("Failed to remove " + snapshotBackupPath);
    }

    // remove log backup
    Path logBackup = new Path(basedir + Path.SEPARATOR + FILE_LOG_BACKUP);
    if (fs.exists(logBackup) && !fs.delete(logBackup, false)) {
        throw new IOException("Failed to remove " + logBackup);
    }

}

---

From source file:com.facebook.hiveio.output.HiveApiOutputFormat.java

License:Apache License

/**
 * Initialize passed in profile ID with Configuration and output description
 * passed in./*from w ww.  jav  a  2s  . com*/
 * @param conf Configuration to use
 * @param outputDesc HiveOutputDescription
 * @param profileId Profile to use
 * @throws TException Hive Metastore issues
 */
public static void initProfile(Configuration conf, HiveOutputDescription outputDesc, String profileId)
        throws TException {
    String dbName = outputDesc.getTableDesc().getDatabaseName();
    String tableName = outputDesc.getTableDesc().getTableName();

    ThriftHiveMetastore.Iface client = outputDesc.metastoreClient(conf);

    Table table = client.get_table(dbName, tableName);
    sanityCheck(table, outputDesc);

    OutputInfo outputInfo = new OutputInfo(table);

    String partitionPiece;
    if (outputInfo.hasPartitionInfo()) {
        partitionPiece = HiveUtils.computePartitionPath(outputInfo.getPartitionInfo(),
                outputDesc.getPartitionValues());
    } else {
        partitionPiece = "_temp";
    }
    String partitionPath = outputInfo.getTableRoot() + Path.SEPARATOR + partitionPiece;

    outputInfo.setPartitionPath(partitionPath);
    HadoopUtils.setOutputDir(conf, partitionPath);

    if (outputInfo.hasPartitionInfo()) {
        outputInfo.setFinalOutputPath(outputInfo.getPartitionPath());
    } else {
        outputInfo.setFinalOutputPath(table.getSd().getLocation());
    }

    HiveTableSchema tableSchema = HiveTableSchemaImpl.fromTable(conf, table);
    HiveTableSchemas.put(conf, profileId, tableSchema);

    OutputConf outputConf = new OutputConf(conf, profileId);
    outputConf.writeOutputDescription(outputDesc);
    outputConf.writeOutputTableInfo(outputInfo);

    LOG.info("initProfile '{}' using {}", profileId, outputDesc);
}

---

From source file:com.github.sakserv.minicluster.oozie.sharelib.util.OozieShareLibUtil.java

License:Apache License

public void createShareLib() {

    if (!oozieShareLibCreate) {
        LOG.info("OOZIE: Share Lib Create Disabled... skipping");
    } else {//from ww w  .  j av a 2  s  .c  om

        final String fullOozieTarFilePath = shareLibCacheDir + Path.SEPARATOR + getOozieTarFileName();

        try {

            // Get and extract the oozie release
            getOozieTarFileFromRepo();
            String oozieExtractTempDir = extractOozieTarFileToTempDir(new File(fullOozieTarFilePath));

            // Extract the sharelib tarball to a temp dir
            String fullOozieShareLibTarFilePath = oozieExtractTempDir + Path.SEPARATOR + "oozie-"
                    + getOozieVersionFromOozieTarFileName() + Path.SEPARATOR + "oozie-sharelib-"
                    + getOozieVersionFromOozieTarFileName() + ".tar.gz";
            String oozieShareLibExtractTempDir = extractOozieShareLibTarFileToTempDir(
                    new File(fullOozieShareLibTarFilePath));

            // Copy the sharelib into HDFS
            Path destPath = new Path(oozieHdfsShareLibDir + Path.SEPARATOR + "oozie" + Path.SEPARATOR
                    + SHARE_LIB_PREFIX + getTimestampDirectory());
            LOG.info("OOZIE: Writing share lib contents to: {}", destPath);
            hdfsFileSystem.copyFromLocalFile(false, new Path(new File(oozieShareLibExtractTempDir).toURI()),
                    destPath);

            if (purgeLocalShareLibCache) {
                FileUtils.deleteDirectory(new File(shareLibCacheDir));
            }

        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

---

From source file:com.github.sakserv.minicluster.oozie.sharelib.util.OozieShareLibUtil.java

License:Apache License

public void getOozieTarFileFromRepo() throws IOException {

    final String fullOozieTarFilePath = shareLibCacheDir + Path.SEPARATOR + getOozieTarFileName();

    if (purgeLocalShareLibCache) {
        FileUtils.deleteDirectory(new File(shareLibCacheDir));
    }// w w  w  .j a  va 2  s. c om

    if (new File(fullOozieTarFilePath).exists()) {
        LOG.info("OOZIE: Found Oozie tarball in cache, skipping download: {}", fullOozieTarFilePath);
    } else {
        downloadOozieTarFileToLocalCacheDir();
    }
}

---

From source file:com.github.sakserv.minicluster.oozie.sharelib.util.OozieShareLibUtil.java

License:Apache License

public void downloadOozieTarFileToLocalCacheDir() throws IOException {
    final String fullOozieTarFilePath = shareLibCacheDir + Path.SEPARATOR + getOozieTarFileName();
    HttpUtils.downloadFileWithProgress(getOozieTarFileUrl(), fullOozieTarFilePath);
}

---

From source file:com.github.sakserv.minicluster.oozie.util.OozieShareLibUtil.java

License:Apache License

public void createShareLib() {

    if (!oozieShareLibCreate) {
        LOG.info("OOZIE: Share Lib Create Disabled... skipping");
    } else {/*from  w  w w.j  av a  2 s  .  c  om*/

        final String fullOozieTarFilePath = shareLibCacheDir + Path.SEPARATOR + getOozieTarFileName();

        try {

            // Get and extract the oozie release
            getOozieTarFileFromRepo();
            String oozieExtractTempDir = extractOozieTarFileToTempDir(new File(fullOozieTarFilePath));

            // Extract the sharelib tarball to a temp dir
            String fullOozieShareLibTarFilePath = oozieExtractTempDir + Path.SEPARATOR + "oozie-"
                    + getOozieVersionFromOozieTarFileName() + Path.SEPARATOR + "oozie-sharelib-"
                    + getOozieVersionFromOozieTarFileName() + ".tar.gz";
            String oozieShareLibExtractTempDir = extractOozieShareLibTarFileToTempDir(
                    new File(fullOozieShareLibTarFilePath));

            // Copy the sharelib into HDFS
            Path destPath = new Path(
                    oozieHdfsShareLibDir + Path.SEPARATOR + SHARE_LIB_PREFIX + getTimestampDirectory());
            LOG.info("OOZIE: Writing share lib contents to: {}", destPath);
            hdfsFileSystem.copyFromLocalFile(false, new Path(new File(oozieShareLibExtractTempDir).toURI()),
                    destPath);

            if (purgeLocalShareLibCache) {
                FileUtils.deleteDirectory(new File(shareLibCacheDir));
            }

        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

---

From source file:com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS.java

License:Open Source License

/**
 * Follow HDFS conventions except allow for ':' in paths.
 *//*  w w  w. j a va2s .  c  o m*/
@Override
public boolean isValidName(String src) {
    StringTokenizer tokens = new StringTokenizer(src, Path.SEPARATOR);
    while (tokens.hasMoreTokens()) {
        String element = tokens.nextToken();
        if (element.equals("..") || element.equals(".")) {
            return false;
        }
    }
    return true;
}

---

From source file:com.ibm.stocator.fs.common.ObjectStoreGlobber.java

License:Open Source License

/**
 * Translate an absolute path into a list of path components. We merge double
 * slashes into a single slash here. POSIX root path, i.e. '/', does not get
 * an entry in the list.//from w  w  w .  jav  a2  s  .  c  om
 */
private static List<String> getPathComponents(String path) throws IOException {
    ArrayList<String> ret = new ArrayList<String>();
    for (String component : path.split(Path.SEPARATOR)) {
        if (!component.isEmpty()) {
            ret.add(component);
        }
    }
    return ret;
}

---

From source file:com.ibm.stocator.fs.common.ObjectStoreGlobber.java

License:Open Source License

public FileStatus[] glob() throws IOException {
    // First we get the scheme and authority of the pattern that was passed
    // in./*w  w  w. j a  va2 s.c  o  m*/
    LOG.debug("Welcome to glob : " + pathPattern.toString());
    String scheme = schemeFromPath(pathPattern);
    String authority = authorityFromPath(pathPattern);

    // Next we strip off everything except the pathname itself, and expand all
    // globs. Expansion is a process which turns "grouping" clauses,
    // expressed as brackets, into separate path patterns.
    String pathPatternString = pathPattern.toUri().getPath();
    List<String> flattenedPatterns = ObjectStoreGlobExpander.expand(pathPatternString);

    LOG.debug("expanded : " + pathPatternString);
    // Now loop over all flattened patterns. In every case, we'll be trying to
    // match them to entries in the filesystem.
    ArrayList<FileStatus> results = new ArrayList<FileStatus>(flattenedPatterns.size());
    boolean sawWildcard = false;
    for (String flatPattern : flattenedPatterns) {
        LOG.debug("pattern from list: " + flatPattern);
        Path absPattern = new Path(flatPattern.isEmpty() ? Path.CUR_DIR : flatPattern);
        List<String> components = getPathComponents(absPattern.toUri().getPath());
        ArrayList<FileStatus> candidates = new ArrayList<FileStatus>(1);
        FileStatus rootPlaceholder = new FileStatus(0, true, 0, 0, 0,
                new Path(scheme, authority, Path.SEPARATOR));
        LOG.debug("Going to add candidate: " + rootPlaceholder.getPath().toString());
        candidates.add(rootPlaceholder);
        String cmpCombined = "";
        ObjectStoreGlobFilter globFilter = null;
        for (int componentIdx = 0; componentIdx < components.size() && !sawWildcard; componentIdx++) {
            globFilter = new ObjectStoreGlobFilter(components.get(componentIdx));
            if (globFilter.hasPattern()) {
                sawWildcard = true;
            } else {
                cmpCombined = cmpCombined + "/" + components.get(componentIdx);
            }
        }
        String component = unescapePathComponent(cmpCombined);
        if (component != null && component.length() > 0) {
            for (FileStatus candidate : candidates) {
                candidate.setPath(new Path(candidate.getPath(), component));
            }
        } else {
            globFilter = new ObjectStoreGlobFilter(components.get(0));
        }
        ArrayList<FileStatus> newCandidates = new ArrayList<FileStatus>(candidates.size());
        for (FileStatus candidate : candidates) {
            if (globFilter.hasPattern()) {
                FileStatus[] children = listStatus(candidate.getPath());
                if (children.length == 1) {
                    if (!getFileStatus(candidate.getPath()).isDirectory()) {
                        continue;
                    }
                }
                for (FileStatus child : children) {
                    if (globFilter.accept(child.getPath())) {
                        newCandidates.add(child);
                    }
                }
            } else {
                FileStatus childStatus = null;
                childStatus = getFileStatus(new Path(candidate.getPath(), component));
                if (childStatus != null) {
                    newCandidates.add(childStatus);
                }
            }
        }
        candidates = newCandidates;
        for (FileStatus status : candidates) {
            if (status == rootPlaceholder) {
                status = getFileStatus(rootPlaceholder.getPath());
                if (status == null) {
                    continue;
                }
            }
            if (filter.accept(status.getPath())) {
                results.add(status);
            }
        }
    }
    if (!sawWildcard && results.isEmpty() && (flattenedPatterns.size() <= 1)) {
        return null;
    }
    return results.toArray(new FileStatus[0]);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »