List of usage examples for org.apache.hadoop.fs FileStatus getModificationTime
public long getModificationTime()
From source file:gobblin.source.extractor.extract.google.GoogleDriveFsHelper.java
License:Apache License
@Override public long getFileMTime(String fileId) throws FileBasedHelperException { Preconditions.checkNotNull(fileId, "fileId is required"); Path p = new Path(fileId); try {/*from w w w .j a v a2 s . c o m*/ FileStatus status = fileSystem.getFileStatus(p); return status.getModificationTime(); } catch (IOException e) { throw new FileBasedHelperException( "Failed to retrieve getModificationTime on path: " + p + " , fileId: " + fileId, e); } }
From source file:gobblin.util.commit.DeleteFileCommitStep.java
License:Apache License
/** * Checks whether existing file in filesystem is the expected file (compares length and modificaiton time). *//*www . j a v a 2 s . co m*/ private boolean existsAndIsExpectedFile(FileStatus status) throws IOException { if (!getFS().exists(status.getPath())) { return false; } FileStatus currentFileStatus = getFS().getFileStatus(status.getPath()); if (currentFileStatus.getLen() != status.getLen() || currentFileStatus.getModificationTime() > status.getModificationTime()) { return false; } return true; }
From source file:gobblin.util.filesystem.InstrumentedFileSystemUtils.java
License:Apache License
/** * Replace the scheme of the input {@link FileStatus} if it matches the string to replace. *///from www . jav a2 s . c o m public static FileStatus replaceScheme(FileStatus st, String replace, String replacement) { if (replace != null && replace.equals(replacement)) { return st; } try { return new FileStatus(st.getLen(), st.isDir(), st.getReplication(), st.getBlockSize(), st.getModificationTime(), st.getAccessTime(), st.getPermission(), st.getOwner(), st.getGroup(), st.isSymlink() ? st.getSymlink() : null, replaceScheme(st.getPath(), replace, replacement)); } catch (IOException ioe) { throw new RuntimeException(ioe); } }
From source file:gobblin.yarn.YarnHelixUtils.java
License:Apache License
/** * Add a file as a Yarn {@link org.apache.hadoop.yarn.api.records.LocalResource}. * * @param fs a {@link FileSystem} instance * @param destFilePath the destination file path * @param resourceType the {@link org.apache.hadoop.yarn.api.records.LocalResourceType} of the file * @param resourceMap a {@link Map} of file names to their corresponding * {@link org.apache.hadoop.yarn.api.records.LocalResource}s * @throws IOException if there's something wrong adding the file as a * {@link org.apache.hadoop.yarn.api.records.LocalResource} *//*from w w w . j a v a2 s . co m*/ public static void addFileAsLocalResource(FileSystem fs, Path destFilePath, LocalResourceType resourceType, Map<String, LocalResource> resourceMap) throws IOException { LocalResource fileResource = Records.newRecord(LocalResource.class); FileStatus fileStatus = fs.getFileStatus(destFilePath); fileResource.setResource(ConverterUtils.getYarnUrlFromPath(destFilePath)); fileResource.setSize(fileStatus.getLen()); fileResource.setTimestamp(fileStatus.getModificationTime()); fileResource.setType(resourceType); fileResource.setVisibility(LocalResourceVisibility.APPLICATION); resourceMap.put(destFilePath.getName(), fileResource); }
From source file:hadoop.yarn.distributedshell.DshellClient.java
License:Apache License
/** * Main run function for the client/* w w w.ja v a 2s .c om*/ * * @return true if application completed successfully * @throws IOException * @throws YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask // if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource // manager int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); if (amVCores > maxVCores) { LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); appContext.setApplicationName(appName); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of // the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(), localResources, null); } // The shell script has to be made available on the final container(s) // where it will be executed. // To do this, we need to first copy into the filesystem that is visible // to the yarn framework. // We do not need to set this as a local resource for the application // master as the application master does not need it. String hdfsShellScriptLocation = ""; long hdfsShellScriptLen = 0; long hdfsShellScriptTimestamp = 0; if (!shellScriptPath.isEmpty()) { Path shellSrc = new Path(shellScriptPath); String shellPathSuffix = appName + "/" + appId.toString() + "/" + SCRIPT_PATH; Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix); fs.copyFromLocalFile(false, true, shellSrc, shellDst); hdfsShellScriptLocation = shellDst.toUri().toString(); FileStatus shellFileStatus = fs.getFileStatus(shellDst); hdfsShellScriptLen = shellFileStatus.getLen(); hdfsShellScriptTimestamp = shellFileStatus.getModificationTime(); } if (!shellCommand.isEmpty()) { addToLocalResources(fs, null, shellCommandPath, appId.toString(), localResources, shellCommand); } if (shellArgs.length > 0) { addToLocalResources(fs, null, shellArgsPath, appId.toString(), localResources, StringUtils.join(shellArgs, " ")); } // Set local resource info into app master container launch context amContainer.setLocalResources(localResources); // Set the necessary security tokens as needed // amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application // master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // put location of shell script into env // using the env info, the application master will create the correct // local resource for the // eventual containers that will be launched to execute the shell // scripts env.put(DshellDSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation); env.put(DshellDSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp)); env.put(DshellDSConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen)); // ========================================jar? if (containerJarPaths.length != 0) { for (int i = 0; i < containerJarPaths.length; i++) { String hdfsJarLocation = ""; String[] jarNameSplit = containerJarPaths[i].split("/"); String jarName = jarNameSplit[jarNameSplit.length - 1]; long hdfsJarLen = 0; long hdfsJarTimestamp = 0; if (!containerJarPaths[i].isEmpty()) { Path jarSrc = new Path(containerJarPaths[i]); String jarPathSuffix = appName + "/" + appId.toString() + "/" + jarName; Path jarDst = new Path(fs.getHomeDirectory(), jarPathSuffix); fs.copyFromLocalFile(false, true, jarSrc, jarDst); hdfsJarLocation = jarDst.toUri().toString(); FileStatus jarFileStatus = fs.getFileStatus(jarDst); hdfsJarLen = jarFileStatus.getLen(); hdfsJarTimestamp = jarFileStatus.getModificationTime(); env.put(DshellDSConstants.DISTRIBUTEDJARLOCATION + i, hdfsJarLocation); env.put(DshellDSConstants.DISTRIBUTEDJARTIMESTAMP + i, Long.toString(hdfsJarTimestamp)); env.put(DshellDSConstants.DISTRIBUTEDJARLEN + i, Long.toString(hdfsJarLen)); } } } // ========================================jar? // ========================================archive? if (containerArchivePaths.length != 0) { for (int i = 0; i < containerArchivePaths.length; i++) { String hdfsArchiveLocation = ""; String[] archiveNameSplit = containerArchivePaths[i].split("/"); String archiveName = archiveNameSplit[archiveNameSplit.length - 1]; long hdfsArchiveLen = 0; long hdfsArchiveTimestamp = 0; if (!containerArchivePaths[i].isEmpty()) { Path archiveSrc = new Path(containerArchivePaths[i]); String archivePathSuffix = appName + "/" + appId.toString() + "/" + archiveName; Path archiveDst = new Path(fs.getHomeDirectory(), archivePathSuffix); fs.copyFromLocalFile(false, true, archiveSrc, archiveDst); hdfsArchiveLocation = archiveDst.toUri().toString(); FileStatus archiveFileStatus = fs.getFileStatus(archiveDst); hdfsArchiveLen = archiveFileStatus.getLen(); hdfsArchiveTimestamp = archiveFileStatus.getModificationTime(); env.put(DshellDSConstants.DISTRIBUTEDARCHIVELOCATION + i, hdfsArchiveLocation); env.put(DshellDSConstants.DISTRIBUTEDARCHIVETIMESTAMP + i, Long.toString(hdfsArchiveTimestamp)); env.put(DshellDSConstants.DISTRIBUTEDARCHIVELEN + i, Long.toString(hdfsArchiveLen)); } } } // ========================================archive? // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); classPathEnv.append(c.trim()); } classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); amContainer.setEnvironment(env); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add("--container_memory " + String.valueOf(containerMemory)); vargs.add("--container_vcores " + String.valueOf(containerVirtualCores)); vargs.add("--num_containers " + String.valueOf(numContainers)); vargs.add("--priority " + String.valueOf(shellCmdPriority)); for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } if (debugFlag) { vargs.add("--debug"); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); amContainer.setCommands(commands); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMemory); capability.setVirtualCores(amVCores); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = // applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on // success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // TODO // Try submitting the same request again // app submission failure? // Monitor the application return monitorApplication(appId); }
From source file:hdfs.HdfsDirectory.java
License:Apache License
public long fileModified(String name) throws IOException { FileStatus fileStatus = getFileSystem().getFileStatus(new Path(hdfsDirPath, name)); return fileStatus.getModificationTime(); }
From source file:hdfs.jsr203.attribute.HadoopFileAttributeView.java
License:Apache License
Object attribute(AttrID id, FileStatus hfas) { switch (id) { case accessTime: return hfas.getAccessTime(); case blockSize: return hfas.getBlockSize(); case group://ww w. jav a2 s. c o m return hfas.getGroup(); case len: return hfas.getLen(); case modificationTime: return hfas.getModificationTime(); case owner: return hfas.getOwner(); case replication: return hfas.getReplication(); case isDirectory: return hfas.isDirectory(); // TODO enable encryption //case isEncrypted: // return hfas.isEncrypted(); case isFile: return hfas.isFile(); case isSymLink: return hfas.isSymlink(); } return null; }
From source file:hdfs.jsr203.HadoopFileSystem.java
License:Apache License
public void setTimes(byte[] bs, FileTime mtime, FileTime atime, FileTime ctime) throws IOException { org.apache.hadoop.fs.Path hp = new HadoopPath(this, bs).getRawResolvedPath(); long mtime_millis = 0; long atime_millis = 0; // Get actual value if (mtime == null || atime == null) { FileStatus stat = this.fs.getFileStatus(hp); mtime_millis = stat.getModificationTime(); atime_millis = stat.getAccessTime(); }/*from w ww.j a va 2 s . c om*/ if (mtime != null) { mtime_millis = mtime.toMillis(); } if (atime != null) { atime_millis = atime.toMillis(); } this.fs.setTimes(hp, mtime_millis, atime_millis); }
From source file:husky.client.HuskyYarnClient.java
License:Apache License
private Pair<String, LocalResource> constructLocalResource(String name, String path, LocalResourceType type) throws IOException { LOG.info("To copy " + name + "(" + path + ") from local file system"); Path resourcePath = new Path(path); if (path.startsWith("hdfs://")) { FileStatus fileStatus = mFileSystem.getFileStatus(resourcePath); if (!fileStatus.isFile()) { throw new RuntimeException("Only files can be provided as local resources."); }// www . j av a2 s.c om } else { File file = new File(path); if (!file.exists()) { throw new RuntimeException("File not exist: " + path); } if (!file.isFile()) { throw new RuntimeException("Only files can be provided as local resources."); } } // if the file is not on hdfs, upload it to hdfs first. if (!path.startsWith("hdfs://")) { Path src = resourcePath; String newPath = mLocalResourceHDFSPaths + '/' + mAppName + '/' + mAppId + '/' + name; resourcePath = new Path(newPath); mFileSystem.copyFromLocalFile(false, true, src, resourcePath); LOG.info("Upload " + path + " to " + newPath); path = newPath; } FileStatus fileStatus = mFileSystem.getFileStatus(resourcePath); LocalResource resource = Records.newRecord(LocalResource.class); resource.setType(type); resource.setVisibility(LocalResourceVisibility.APPLICATION); resource.setResource(ConverterUtils.getYarnUrlFromPath(resourcePath)); resource.setTimestamp(fileStatus.getModificationTime()); resource.setSize(fileStatus.getLen()); return new Pair<String, LocalResource>(path, resource); }
From source file:husky.server.HuskyRMCallbackHandler.java
License:Apache License
private LocalResource constructLocalResource(String name, String path, LocalResourceType type) throws IOException { LOG.info("To copy " + name + " from " + path); if (!path.startsWith("hdfs://")) { throw new RuntimeException( "Local resources provided to application master must be already located on HDFS."); }/*from ww w .j a v a 2 s . com*/ FileStatus dsfStaus = mAppMaster.getFileSystem().getFileStatus(new Path(path)); LocalResource resource = Records.newRecord(LocalResource.class); resource.setType(type); resource.setVisibility(LocalResourceVisibility.APPLICATION); resource.setResource(ConverterUtils.getYarnUrlFromPath(new Path(path))); resource.setTimestamp(dsfStaus.getModificationTime()); resource.setSize(dsfStaus.getLen()); return resource; }