Example usage for org.apache.hadoop.fs Path getParent

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getParent.

Prototype

public Path getParent()

Source Link

Document

Returns the parent of a path or null if at root.

Usage

From source file:org.apache.falcon.converter.OozieProcessMapper.java

License:Apache License

private void decoratePIGAction(Cluster cluster, Process process, Workflow processWorkflow, PIG pigAction,
        Path parentWfPath) throws FalconException {
    Path userWfPath = getUserWorkflowPath(cluster, parentWfPath.getParent());
    pigAction.setScript("${nameNode}" + userWfPath.toString());

    addPrepareDeleteOutputPath(process, pigAction);

    final List<String> paramList = pigAction.getParam();
    addInputFeedsAsParams(paramList, process, cluster, EngineType.PIG.name().toLowerCase());
    addOutputFeedsAsParams(paramList, process, cluster);

    propagateProcessProperties(pigAction, process);

    Storage.TYPE storageType = getStorageType(cluster, process);
    if (Storage.TYPE.TABLE == storageType) {
        // adds hive-site.xml in pig classpath
        setupHiveConfiguration(cluster, parentWfPath, ""); // DO NOT ADD PREFIX!!!
        pigAction.getFile().add("${wf:appPath()}/conf/hive-site.xml");
    }//  ww w.  j  av a2 s .c  o m

    addArchiveForCustomJars(cluster, processWorkflow, pigAction.getArchive(),
            getUserLibPath(cluster, parentWfPath.getParent()));
}

From source file:org.apache.falcon.converter.OozieProcessMapper.java

License:Apache License

private void decorateHiveAction(Cluster cluster, Process process, Workflow processWorkflow, ACTION wfAction,
        Path parentWfPath) throws FalconException {

    JAXBElement<org.apache.falcon.oozie.hive.ACTION> actionJaxbElement = unMarshalHiveAction(wfAction);
    org.apache.falcon.oozie.hive.ACTION hiveAction = actionJaxbElement.getValue();

    Path userWfPath = getUserWorkflowPath(cluster, parentWfPath.getParent());
    hiveAction.setScript("${nameNode}" + userWfPath.toString());

    addPrepareDeleteOutputPath(process, hiveAction);

    final List<String> paramList = hiveAction.getParam();
    addInputFeedsAsParams(paramList, process, cluster, EngineType.HIVE.name().toLowerCase());
    addOutputFeedsAsParams(paramList, process, cluster);

    propagateProcessProperties(hiveAction, process);

    setupHiveConfiguration(cluster, parentWfPath, "falcon-");

    addArchiveForCustomJars(cluster, processWorkflow, hiveAction.getArchive(),
            getUserLibPath(cluster, parentWfPath.getParent()));

    marshalHiveAction(wfAction, actionJaxbElement);
}

From source file:org.apache.falcon.converter.OozieProcessMapperTest.java

License:Apache License

private void prepare(Process process) throws IOException {
    Path wf = new Path(process.getWorkflow().getPath());
    fs.mkdirs(wf.getParent());
    fs.create(wf).close();/*w w w  .j  av  a2  s  .com*/
}

From source file:org.apache.falcon.entity.FileSystemStorage.java

License:Apache License

private void deleteInstance(FileSystem fs, Path path, Path feedBasePath) throws IOException {
    if (fs.delete(path, true)) {
        LOG.info("Deleted instance: {}", path);
    } else {/*  w w w  .  j a  v a 2 s.co m*/
        throw new IOException("Unable to delete instance: " + path);
    }
    deleteParentIfEmpty(fs, path.getParent(), feedBasePath);
}

From source file:org.apache.falcon.entity.FileSystemStorage.java

License:Apache License

private void deleteParentIfEmpty(FileSystem fs, Path parent, Path feedBasePath) throws IOException {
    if (feedBasePath.equals(parent)) {
        LOG.info("Not deleting feed base path: {}", parent);
    } else {/*  w ww . ja v  a 2s .  c om*/
        FileStatus[] files = fs.listStatus(parent);
        if (files != null && files.length == 0) {
            LOG.info("Parent path: {} is empty, deleting path", parent);
            if (fs.delete(parent, true)) {
                LOG.info("Deleted empty dir: {}", parent);
            } else {
                throw new IOException("Unable to delete parent path:" + parent);
            }
            deleteParentIfEmpty(fs, parent.getParent(), feedBasePath);
        }
    }
}

From source file:org.apache.falcon.entity.ProcessHelper.java

License:Apache License

public static Path getUserWorkflowPath(Process process, org.apache.falcon.entity.v0.cluster.Cluster cluster,
        Path buildPath) throws FalconException {
    try {/* w  ww  .  j a  v  a  2s  .  co m*/
        FileSystem fs = HadoopClientFactory.get()
                .createProxiedFileSystem(ClusterHelper.getConfiguration(cluster));
        Path wfPath = new Path(process.getWorkflow().getPath());
        if (fs.isFile(wfPath)) {
            return new Path(buildPath.getParent(), EntityUtil.PROCESS_USER_DIR + "/" + wfPath.getName());
        } else {
            return new Path(buildPath.getParent(), EntityUtil.PROCESS_USER_DIR);
        }
    } catch (IOException e) {
        throw new FalconException("Failed to get workflow path", e);
    }
}

From source file:org.apache.falcon.service.SharedLibraryHostingService.java

License:Apache License

private void pushExtensionArtifactsToCluster(final Cluster cluster, final FileSystem clusterFs)
        throws FalconException {
    if (!Services.get().isRegistered(ExtensionService.SERVICE_NAME)) {
        LOG.info("ExtensionService not registered, return");
        return;//from w w  w  .  ja  v a  2s . co m
    }

    ExtensionStore store = ExtensionStore.get();
    if (!store.isExtensionStoreInitialized()) {
        LOG.info(
                "Extension store not initialized by Extension service. Make sure Extension service is added in "
                        + "start up properties");
        return;
    }

    final String filterPath = "/apps/falcon/extensions/mirroring/";
    Path extensionStorePath = store.getExtensionStorePath();
    LOG.info("extensionStorePath :{}", extensionStorePath);
    FileSystem falconFileSystem = HadoopClientFactory.get().createFalconFileSystem(extensionStorePath.toUri());
    String nameNode = StringUtils
            .removeEnd(falconFileSystem.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY), File.separator);

    String clusterStorageUrl = StringUtils.removeEnd(ClusterHelper.getStorageUrl(cluster), File.separator);

    // If default fs for Falcon server is same as cluster fs abort copy
    if (nameNode.equalsIgnoreCase(clusterStorageUrl)) {
        LOG.info("clusterStorageUrl :{} same return", clusterStorageUrl);
        return;
    }

    try {
        RemoteIterator<LocatedFileStatus> fileStatusListIterator = falconFileSystem
                .listFiles(extensionStorePath, true);

        while (fileStatusListIterator.hasNext()) {
            LocatedFileStatus srcfileStatus = fileStatusListIterator.next();
            Path filePath = Path.getPathWithoutSchemeAndAuthority(srcfileStatus.getPath());

            if (filePath != null && filePath.toString().startsWith(filterPath)) {
                /* HiveDR uses filter path as store path in DRStatusStore, so skip it. Copy only the extension
                 artifacts */
                continue;
            }

            if (srcfileStatus.isDirectory()) {
                if (!clusterFs.exists(filePath)) {
                    HadoopClientFactory.mkdirs(clusterFs, filePath, srcfileStatus.getPermission());
                }
            } else {
                if (clusterFs.exists(filePath)) {
                    FileStatus targetfstat = clusterFs.getFileStatus(filePath);
                    if (targetfstat.getLen() == srcfileStatus.getLen()) {
                        continue;
                    }
                }

                Path parentPath = filePath.getParent();
                if (!clusterFs.exists(parentPath)) {
                    FsPermission dirPerm = falconFileSystem.getFileStatus(parentPath).getPermission();
                    HadoopClientFactory.mkdirs(clusterFs, parentPath, dirPerm);
                }

                FileUtil.copy(falconFileSystem, srcfileStatus, clusterFs, filePath, false, true,
                        falconFileSystem.getConf());
                FileUtil.chmod(clusterFs.makeQualified(filePath).toString(),
                        srcfileStatus.getPermission().toString());
            }
        }
    } catch (IOException | InterruptedException e) {
        throw new FalconException("Failed to copy extension artifacts to cluster" + cluster.getName(), e);
    }
}

From source file:org.apache.flink.streaming.connectors.fs.bucketing.BucketingSink.java

License:Apache License

private Path getPendingPathFor(Path path) {
    return new Path(path.getParent(), pendingPrefix + path.getName()).suffix(pendingSuffix);
}

From source file:org.apache.flink.streaming.connectors.fs.bucketing.BucketingSink.java

License:Apache License

private Path getInProgressPathFor(Path path) {
    return new Path(path.getParent(), inProgressPrefix + path.getName()).suffix(inProgressSuffix);
}

From source file:org.apache.flink.streaming.connectors.fs.bucketing.BucketingSink.java

License:Apache License

private Path getValidLengthPathFor(Path path) {
    return new Path(path.getParent(), validLengthPrefix + path.getName()).suffix(validLengthSuffix);
}