List of usage examples for org.apache.hadoop.fs Path depth
public int depth()
From source file:org.apache.gobblin.service.modules.core.GitConfigMonitor.java
License:Apache License
/** * check whether the file has the proper naming and hierarchy * @param configFilePath the relative path from the repo root * @return false if the file does not conform *///from w ww . ja v a 2 s.c o m private boolean checkConfigFilePath(String configFilePath) { // The config needs to stored at configDir/flowGroup/flowName.(pull|job|json|conf) Path configFile = new Path(configFilePath); String fileExtension = Files.getFileExtension(configFile.getName()); if (configFile.depth() != CONFIG_FILE_DEPTH || !configFile.getParent().getParent().getName().equals(folderName) || !(PullFileLoader.DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS.contains(fileExtension) || PullFileLoader.DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS.contains(fileExtension))) { log.warn("Changed file does not conform to directory structure and file name format, skipping: " + configFilePath); return false; } return true; }
From source file:org.apache.gobblin.service.modules.core.GitFlowGraphMonitor.java
License:Apache License
/** * Add an element (i.e., a {@link DataNode}, or a {@link FlowEdge} to * the {@link FlowGraph} for an added, updated or modified node or edge file. * @param change/*from w w w . j a v a 2s. c o m*/ */ @Override public void addChange(DiffEntry change) { Path path = new Path(change.getNewPath()); if (path.depth() == NODE_FILE_DEPTH) { addDataNode(change); } else if (path.depth() == EDGE_FILE_DEPTH) { addFlowEdge(change); } }
From source file:org.apache.gobblin.service.modules.core.GitFlowGraphMonitor.java
License:Apache License
/** * Remove an element (i.e. either a {@link DataNode} or a {@link FlowEdge} from the {@link FlowGraph} for * a renamed or deleted {@link DataNode} or {@link FlowEdge} file. * @param change//from w ww . java2 s. c o m */ @Override public void removeChange(DiffEntry change) { Path path = new Path(change.getOldPath()); if (path.depth() == NODE_FILE_DEPTH) { removeDataNode(change); } else if (path.depth() == EDGE_FILE_DEPTH) { removeFlowEdge(change); } }
From source file:org.apache.gobblin.service.modules.core.GitFlowGraphMonitor.java
License:Apache License
/** * check whether the file has the proper naming and hierarchy * @param file the relative path from the repo root * @return false if the file does not conform *//*from w ww.j av a 2 s . com*/ private boolean checkFilePath(String file, int depth) { // The file is either a node file or an edge file and needs to be stored at either: // flowGraphDir/nodeName/nodeName.properties (if it is a node file), or // flowGraphDir/nodeName/nodeName/edgeName.properties (if it is an edge file) Path filePath = new Path(file); String fileExtension = Files.getFileExtension(filePath.getName()); if (filePath.depth() != depth || !checkFileLevelRelativeToRoot(filePath, depth) || !(this.javaPropsExtensions.contains(fileExtension))) { log.warn("Changed file does not conform to directory structure and file name format, skipping: " + filePath); return false; } return true; }
From source file:org.apache.impala.common.FileSystemUtil.java
License:Apache License
/** * Returns true if Path 'p' is a descendant of Path 'parent', false otherwise. * This function relies on Path.equals() which requires paths to have the same * schema and authority to compare equal. So both 'p' and 'parent' should either * be qualified or unqualified paths for this function to behave as expected. *//* ww w . j a v a2 s. c o m*/ public static boolean isDescendantPath(Path p, Path parent) { if (p == null || parent == null) return false; while (!p.isRoot() && p.depth() != parent.depth()) p = p.getParent(); if (p.isRoot()) return false; boolean result = p.equals(parent); if (!result && LOG.isTraceEnabled()) { // Add a message to the log if 'p' and 'parent' have inconsistent qualification. URI pUri = p.toUri(); URI parentUri = parent.toUri(); boolean sameScheme = Objects.equal(pUri.getScheme(), parentUri.getScheme()); boolean sameAuthority = Objects.equal(pUri.getAuthority(), parentUri.getAuthority()); if (!sameScheme || !sameAuthority) { LOG.trace("Inconsistent schema or authority for paths: " + p.toString() + " " + parent.toString()); } } return result; }
From source file:org.apache.nifi.processors.hadoop.AbstractHadoopProcessor.java
License:Apache License
/** * Returns the relative path of the child that does not include the filename or the root path. * * @param root// w ww. j av a 2s .c om * the path to relativize from * @param child * the path to relativize * @return the relative path */ public static String getPathDifference(final Path root, final Path child) { final int depthDiff = child.depth() - root.depth(); if (depthDiff <= 1) { return "".intern(); } String lastRoot = root.getName(); Path childsParent = child.getParent(); final StringBuilder builder = new StringBuilder(); builder.append(childsParent.getName()); for (int i = (depthDiff - 3); i >= 0; i--) { childsParent = childsParent.getParent(); String name = childsParent.getName(); if (name.equals(lastRoot) && childsParent.toString().endsWith(root.toString())) { break; } builder.insert(0, Path.SEPARATOR).insert(0, name); } return builder.toString(); }
From source file:org.apache.zeppelin.submarine.commons.SubmarineUtils.java
License:Apache License
public static HashMap propertiesToJinjaParams(Properties properties, SubmarineJob submarineJob, boolean outLog) throws IOException { StringBuffer sbMessage = new StringBuffer(); String noteId = submarineJob.getNoteId(); // Check user-set job variables String machinelearingDistributed = getProperty(properties, MACHINELEARNING_DISTRIBUTED_ENABLE, outLog, sbMessage);//from ww w .j ava2 s. c o m String inputPath = getProperty(properties, INPUT_PATH, outLog, sbMessage); String checkPointPath = getProperty(properties, CHECKPOINT_PATH, outLog, sbMessage); String psLaunchCmd = ""; if (StringUtils.equals(machinelearingDistributed, "true")) { psLaunchCmd = getProperty(properties, PS_LAUNCH_CMD, outLog, sbMessage); } String workerLaunchCmd = getProperty(properties, WORKER_LAUNCH_CMD, outLog, sbMessage); // Check interpretere set Properties String submarineHadoopHome; submarineHadoopHome = getProperty(properties, SUBMARINE_HADOOP_HOME, outLog, sbMessage); File file = new File(submarineHadoopHome); if (!file.exists()) { sbMessage.append(SUBMARINE_HADOOP_HOME + ": " + submarineHadoopHome + " is not a valid file path!\n"); } String submarineJar = getProperty(properties, HADOOP_YARN_SUBMARINE_JAR, outLog, sbMessage); file = new File(submarineJar); if (!file.exists()) { sbMessage.append(HADOOP_YARN_SUBMARINE_JAR + ":" + submarineJar + " is not a valid file path!\n"); } String submarineYarnQueue = getProperty(properties, SUBMARINE_YARN_QUEUE, outLog, sbMessage); String containerNetwork = getProperty(properties, DOCKER_CONTAINER_NETWORK, outLog, sbMessage); String parameterServicesImage = getProperty(properties, TF_PARAMETER_SERVICES_DOCKER_IMAGE, outLog, sbMessage); String parameterServicesNum = getProperty(properties, TF_PARAMETER_SERVICES_NUM, outLog, sbMessage); String parameterServicesGpu = getProperty(properties, TF_PARAMETER_SERVICES_GPU, outLog, sbMessage); String parameterServicesCpu = getProperty(properties, TF_PARAMETER_SERVICES_CPU, outLog, sbMessage); String parameterServicesMemory = getProperty(properties, TF_PARAMETER_SERVICES_MEMORY, outLog, sbMessage); String workerServicesImage = getProperty(properties, TF_WORKER_SERVICES_DOCKER_IMAGE, outLog, sbMessage); String workerServicesNum = getProperty(properties, TF_WORKER_SERVICES_NUM, outLog, sbMessage); String workerServicesGpu = getProperty(properties, TF_WORKER_SERVICES_GPU, outLog, sbMessage); String workerServicesCpu = getProperty(properties, TF_WORKER_SERVICES_CPU, outLog, sbMessage); String workerServicesMemory = getProperty(properties, TF_WORKER_SERVICES_MEMORY, outLog, sbMessage); String algorithmUploadPath = getProperty(properties, SUBMARINE_ALGORITHM_HDFS_PATH, outLog, sbMessage); String submarineHadoopKeytab = getProperty(properties, SUBMARINE_HADOOP_KEYTAB, outLog, sbMessage); file = new File(submarineHadoopKeytab); if (!file.exists()) { sbMessage .append(SUBMARINE_HADOOP_KEYTAB + ":" + submarineHadoopKeytab + " is not a valid file path!\n"); } String submarineHadoopPrincipal = getProperty(properties, SUBMARINE_HADOOP_PRINCIPAL, outLog, sbMessage); String dockerHadoopHdfsHome = getProperty(properties, DOCKER_HADOOP_HDFS_HOME, outLog, sbMessage); String dockerJavaHome = getProperty(properties, DOCKER_JAVA_HOME, outLog, sbMessage); String intpLaunchMode = getProperty(properties, INTERPRETER_LAUNCH_MODE, outLog, sbMessage); if (StringUtils.isEmpty(intpLaunchMode)) { intpLaunchMode = "local"; // default } String tensorboardEnable = getProperty(properties, TF_TENSORBOARD_ENABLE, outLog, sbMessage); if (StringUtils.isEmpty(tensorboardEnable)) { tensorboardEnable = "false"; // default } // check String tensorboardCheckpoint = getProperty(properties, TF_CHECKPOINT_PATH, outLog, sbMessage); if (StringUtils.equals(tensorboardEnable, "true") && StringUtils.isEmpty(tensorboardCheckpoint)) { sbMessage.append("Tensorboard checkpoint path cannot be empty!\n"); } String userTensorboardCheckpoint = submarineJob.getUserTensorboardPath(); Path chkpntPath = new Path(userTensorboardCheckpoint); if (chkpntPath.depth() <= 3) { sbMessage.append("Checkpoint path depth must be greater than 3!\n"); } String sumbarineHadoopConfDir = getProperty(properties, SUBMARINE_HADOOP_CONF_DIR, outLog, sbMessage); String dockerContainerTimezone = getProperty(properties, DOCKER_CONTAINER_TIME_ZONE, outLog, sbMessage); String notePath = algorithmUploadPath + File.separator + noteId; List<String> arrayHdfsFiles = new ArrayList<>(); List<Path> hdfsFiles = submarineJob.getHdfsClient().list(new Path(notePath + "/*")); if (hdfsFiles.size() == 0) { sbMessage .append("EXECUTE_SUBMARINE_ERROR: The " + notePath + " file directory was is empty in HDFS!\n"); } else { if (outLog) { StringBuffer sbCommitFiles = new StringBuffer(); sbCommitFiles.append("INFO: You commit total of " + hdfsFiles.size() + " algorithm files.\n"); for (int i = 0; i < hdfsFiles.size(); i++) { String filePath = hdfsFiles.get(i).toUri().toString(); arrayHdfsFiles.add(filePath); sbCommitFiles.append("INFO: [" + hdfsFiles.get(i).getName() + "] -> " + filePath + "\n"); } submarineJob.getSubmarineUI().outputLog("Execution information", sbCommitFiles.toString()); } } // Found null variable, throw exception if (!StringUtils.isEmpty(sbMessage.toString()) && outLog) { throw new RuntimeException(sbMessage.toString()); } // Save user-set variables and interpreter configuration parameters String jobName = SubmarineUtils.getJobName(submarineJob.getUserName(), submarineJob.getNoteId()); HashMap<String, Object> mapParams = new HashMap(); mapParams.put(unifyKey(INTERPRETER_LAUNCH_MODE), intpLaunchMode); mapParams.put(unifyKey(SUBMARINE_HADOOP_HOME), submarineHadoopHome); mapParams.put(unifyKey(SUBMARINE_HADOOP_CONF_DIR), sumbarineHadoopConfDir); mapParams.put(unifyKey(DOCKER_HADOOP_HDFS_HOME), dockerHadoopHdfsHome); mapParams.put(unifyKey(DOCKER_JAVA_HOME), dockerJavaHome); mapParams.put(unifyKey(DOCKER_CONTAINER_TIME_ZONE), dockerContainerTimezone); mapParams.put(unifyKey(HADOOP_YARN_SUBMARINE_JAR), submarineJar); mapParams.put(unifyKey(JOB_NAME), jobName); mapParams.put(unifyKey(DOCKER_CONTAINER_NETWORK), containerNetwork); mapParams.put(unifyKey(SUBMARINE_YARN_QUEUE), submarineYarnQueue); mapParams.put(unifyKey(SUBMARINE_HADOOP_KEYTAB), submarineHadoopKeytab); mapParams.put(unifyKey(SUBMARINE_HADOOP_PRINCIPAL), submarineHadoopPrincipal); mapParams.put(unifyKey(MACHINELEARNING_DISTRIBUTED_ENABLE), machinelearingDistributed); mapParams.put(unifyKey(SUBMARINE_ALGORITHM_HDFS_PATH), notePath); mapParams.put(unifyKey(SUBMARINE_ALGORITHM_HDFS_FILES), arrayHdfsFiles); mapParams.put(unifyKey(INPUT_PATH), inputPath); mapParams.put(unifyKey(CHECKPOINT_PATH), checkPointPath); mapParams.put(unifyKey(PS_LAUNCH_CMD), psLaunchCmd); mapParams.put(unifyKey(WORKER_LAUNCH_CMD), workerLaunchCmd); mapParams.put(unifyKey(TF_PARAMETER_SERVICES_DOCKER_IMAGE), parameterServicesImage); mapParams.put(unifyKey(TF_PARAMETER_SERVICES_NUM), parameterServicesNum); mapParams.put(unifyKey(TF_PARAMETER_SERVICES_GPU), parameterServicesGpu); mapParams.put(unifyKey(TF_PARAMETER_SERVICES_CPU), parameterServicesCpu); mapParams.put(unifyKey(TF_PARAMETER_SERVICES_MEMORY), parameterServicesMemory); mapParams.put(unifyKey(TF_WORKER_SERVICES_DOCKER_IMAGE), workerServicesImage); mapParams.put(unifyKey(TF_WORKER_SERVICES_NUM), workerServicesNum); mapParams.put(unifyKey(TF_WORKER_SERVICES_GPU), workerServicesGpu); mapParams.put(unifyKey(TF_WORKER_SERVICES_CPU), workerServicesCpu); mapParams.put(unifyKey(TF_WORKER_SERVICES_MEMORY), workerServicesMemory); mapParams.put(unifyKey(TF_TENSORBOARD_ENABLE), tensorboardEnable); mapParams.put(unifyKey(TF_CHECKPOINT_PATH), userTensorboardCheckpoint); return mapParams; }
From source file:org.apache.zeppelin.submarine.job.SubmarineJob.java
License:Apache License
public void cleanJobDefaultCheckpointPath() { String jobCheckpointPath = getJobDefaultCheckpointPath(); Path notePath = new Path(jobCheckpointPath); if (notePath.depth() <= 3) { submarineUI.outputLog("ERROR", "Checkpoint path depth must be greater than 3"); return;// w w w.j av a 2 s .c o m } try { String message = "Clean up the checkpoint directory: " + jobCheckpointPath; submarineUI.outputLog("", message); hdfsClient.delete(notePath); } catch (IOException e) { LOGGER.error(e.getMessage(), e); } }
From source file:org.icgc.dcc.submission.sftp.fs.BaseDirectoryHdfsSshFile.java
License:Open Source License
@Override public HdfsSshFile getChild(Path filePath) { try {/* w w w . j a v a 2 s .co m*/ switch (filePath.depth()) { case 0: return this; case 1: return new FileHdfsSshFile(context, this, filePath.getName()); } } catch (Exception e) { return handleException(HdfsSshFile.class, e); } return handleException(HdfsSshFile.class, "Invalid file path: %s%s", getAbsolutePath(), filePath.toString()); }
From source file:org.icgc.dcc.submission.sftp.fs.HdfsFileSystemView.java
License:Open Source License
/** * Returns the appropriate file system abstraction for the specified {@code file} path. * // w w w . j av a 2 s . com * @param file - the path to the file to get * @return the {@link SshFile} for the provided file path */ @Override public SshFile getFile(String file) { try { Path filePath = getFilePath(file); RootHdfsSshFile root = new RootHdfsSshFile(context, subject); switch (filePath.depth()) { case 0: return root; case 1: return getSubmissionDirectory(file, filePath, root); case 2: return getSubmissionFile(file, filePath, root); default: throw new FileNotFoundException("Invalid file path: " + file); } } catch (Exception e) { return handleException(SshFile.class, e); } }