Example usage for org.apache.hadoop.fs Path getName

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getName.

Prototype

public String getName() 

Source Link

Document

Returns the final component of this path.

Usage

From source file:com.datatorrent.lib.io.fs.FileSplitter.java

License:Open Source License

/**
 * Creates file metadata and populates no. of blocks in the metadata.
 *
 * @param fPath file-path//  w  ww.  j  av  a  2  s.co m
 * @return file-metadata
 * @throws IOException
 */
protected FileMetadata buildFileMetadata(String fPath) throws IOException {
    currentFile = fPath;
    Path path = new Path(fPath);

    FileMetadata fileMetadata = readEntity();
    fileMetadata.setFileName(path.getName());

    FileStatus status = fs.getFileStatus(path);
    int noOfBlocks = (int) ((status.getLen() / blockSize) + (((status.getLen() % blockSize) == 0) ? 0 : 1));
    if (fileMetadata.getDataOffset() >= status.getLen()) {
        noOfBlocks = 0;
    }
    fileMetadata.setFileLength(status.getLen());
    fileMetadata.setNumberOfBlocks(noOfBlocks);
    populateBlockIds(fileMetadata);
    return fileMetadata;
}

---

From source file:com.datatorrent.lib.io.fs.FileStitcher.java

License:Apache License

protected void mergeBlocks(T stitchedFileMetaData) throws IOException {
    //when writing to tmp files there can be vagrant tmp files which we have to clean
    final Path dst = new Path(filePath, stitchedFileMetaData.getStitchedFileRelativePath());
    PathFilter tempFileFilter = new PathFilter() {
        @Override/*from   w  w w .ja  v a2s.c  o  m*/
        public boolean accept(Path path) {
            return path.getName().startsWith(dst.getName()) && path.getName().endsWith(PART_FILE_EXTENTION);
        }
    };
    if (outputFS.exists(dst.getParent())) {
        FileStatus[] statuses = outputFS.listStatus(dst.getParent(), tempFileFilter);
        for (FileStatus status : statuses) {
            String statusName = status.getPath().getName();
            LOG.debug("deleting vagrant file {}", statusName);
            outputFS.delete(status.getPath(), true);
        }
    }
    tempOutFilePath = new Path(filePath, stitchedFileMetaData.getStitchedFileRelativePath() + '.'
            + System.currentTimeMillis() + PART_FILE_EXTENTION);
    try {
        writeTempOutputFile(stitchedFileMetaData);
        moveToFinalFile(stitchedFileMetaData);
    } catch (BlockNotFoundException e) {
        LOG.warn("Block file {} not found. Assuming recovery mode for file {}. ", e.getBlockPath(),
                stitchedFileMetaData.getStitchedFileRelativePath());
        //Remove temp output file
        outputFS.delete(tempOutFilePath, false);
    }
}

---

From source file:com.datatorrent.lib.io.jms.FSPsuedoTransactionableStore.java

License:Open Source License

@Override
public void storeCommittedWindowId(String appId, int operatorId, long windowId) {
    Path recoveryPath = getOperatorRecoveryPath(appId, operatorId);
    Path windowPath = getOperatorWindowRecoveryPath(appId, operatorId, windowId);
    String windowString = Long.toString(windowId);

    try {//  w  w w.  jav a2  s  .  c  o m
        fs.create(windowPath);
        FileStatus[] windowFiles = fs.listStatus(recoveryPath);

        for (FileStatus fileStatus : windowFiles) {
            Path tempPath = fileStatus.getPath();
            if (!tempPath.getName().equals(windowString)) {
                fs.delete(tempPath, true);
            }
        }
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}

---

From source file:com.datatorrent.stram.cli.ApexCli.java

License:Apache License

private File copyToLocal(String[] files) throws IOException {
    File tmpDir = new File(System.getProperty("java.io.tmpdir") + "/datatorrent/"
            + ManagementFactory.getRuntimeMXBean().getName());
    tmpDir.mkdirs();/*  w w  w.  j  av a 2s . com*/
    for (int i = 0; i < files.length; i++) {
        try {
            URI uri = new URI(files[i]);
            String scheme = uri.getScheme();
            if (scheme == null || scheme.equals("file")) {
                files[i] = uri.getPath();
            } else {
                try (FileSystem tmpFs = FileSystem.newInstance(uri, conf)) {
                    Path srcPath = new Path(uri.getPath());
                    Path dstPath = new Path(tmpDir.getAbsolutePath(), String.valueOf(i) + srcPath.getName());
                    tmpFs.copyToLocalFile(srcPath, dstPath);
                    files[i] = dstPath.toUri().getPath();
                }
            }
        } catch (URISyntaxException ex) {
            throw new RuntimeException(ex);
        }
    }

    return tmpDir;
}

---

From source file:com.datatorrent.stram.cli.DTCli.java

License:Apache License

private File copyToLocal(String[] files) throws IOException {
    File tmpDir = new File("/tmp/datatorrent/" + ManagementFactory.getRuntimeMXBean().getName());
    tmpDir.mkdirs();//from w  w w  . ja  va 2s.  co  m
    for (int i = 0; i < files.length; i++) {
        try {
            URI uri = new URI(files[i]);
            String scheme = uri.getScheme();
            if (scheme == null || scheme.equals("file")) {
                files[i] = uri.getPath();
            } else {
                FileSystem tmpFs = FileSystem.newInstance(uri, conf);
                try {
                    Path srcPath = new Path(uri.getPath());
                    Path dstPath = new Path(tmpDir.getAbsolutePath(), String.valueOf(i) + srcPath.getName());
                    tmpFs.copyToLocalFile(srcPath, dstPath);
                    files[i] = dstPath.toUri().getPath();
                } finally {
                    tmpFs.close();
                }
            }
        } catch (URISyntaxException ex) {
            throw new RuntimeException(ex);
        }
    }

    return tmpDir;
}

---

From source file:com.datatorrent.stram.client.StramAppLauncher.java

License:Apache License

public StramAppLauncher(FileSystem fs, Path path, Configuration conf) throws Exception {
    File jarsDir = new File(StramClientUtils.getUserDTDirectory(), "jars");
    jarsDir.mkdirs();//from  ww  w  .  ja  v a2 s  . co m
    File localJarFile = new File(jarsDir, path.getName());
    this.fs = fs;
    fs.copyToLocalFile(path, new Path(localJarFile.getAbsolutePath()));
    this.jarFile = localJarFile;
    this.propertiesBuilder = new LogicalPlanConfiguration(conf);
    init(this.jarFile.getName());
}

---

From source file:com.datatorrent.stram.client.StramAppLauncher.java

License:Apache License

private void processLibJars(String libjars, Set<URL> clUrls) throws Exception {
    for (String libjar : libjars.split(",")) {
        // if hadoop file system, copy from hadoop file system to local
        URI uri = new URI(libjar);
        String scheme = uri.getScheme();
        if (scheme == null) {
            // expand wildcards
            DirectoryScanner scanner = new DirectoryScanner();
            scanner.setIncludes(new String[] { libjar });
            scanner.scan();/*from w  w  w  .ja  v a  2  s .c o  m*/
            String[] files = scanner.getIncludedFiles();
            for (String file : files) {
                clUrls.add(new URL("file:" + file));
            }
        } else if (scheme.equals("file")) {
            clUrls.add(new URL(libjar));
        } else {
            if (fs != null) {
                Path path = new Path(libjar);
                File dependencyJarsDir = new File(StramClientUtils.getUserDTDirectory(), "dependencyJars");
                dependencyJarsDir.mkdirs();
                File localJarFile = new File(dependencyJarsDir, path.getName());
                fs.copyToLocalFile(path, new Path(localJarFile.getAbsolutePath()));
                clUrls.add(new URL("file:" + localJarFile.getAbsolutePath()));
            } else {
                throw new NotImplementedException(
                        "Jar file needs to be from Hadoop File System also in order for the dependency jars to be in Hadoop File System");
            }
        }
    }
}

---

From source file:com.datatorrent.stram.client.StramAppLauncher.java

License:Apache License

/**
 * Submit application to the cluster and return the app id.
 * Sets the context class loader for application dependencies.
 *
 * @param appConfig//from w  ww.j a v a  2  s  . c o m
 * @return ApplicationId
 * @throws Exception
 */
public ApplicationId launchApp(AppFactory appConfig) throws Exception {
    loadDependencies();
    Configuration conf = propertiesBuilder.conf;
    conf.setEnum(StreamingApplication.ENVIRONMENT, StreamingApplication.Environment.CLUSTER);
    LogicalPlan dag = appConfig.createApp(propertiesBuilder);
    String hdfsTokenMaxLifeTime = conf.get(StramClientUtils.HDFS_TOKEN_MAX_LIFE_TIME);
    if (hdfsTokenMaxLifeTime != null && hdfsTokenMaxLifeTime.trim().length() > 0) {
        dag.setAttribute(LogicalPlan.HDFS_TOKEN_LIFE_TIME, Long.parseLong(hdfsTokenMaxLifeTime));
    }
    String rmTokenMaxLifeTime = conf.get(StramClientUtils.RM_TOKEN_MAX_LIFE_TIME);
    if (rmTokenMaxLifeTime != null && rmTokenMaxLifeTime.trim().length() > 0) {
        dag.setAttribute(LogicalPlan.RM_TOKEN_LIFE_TIME, Long.parseLong(rmTokenMaxLifeTime));
    }
    if (conf.get(StramClientUtils.KEY_TAB_FILE) != null) {
        dag.setAttribute(LogicalPlan.KEY_TAB_FILE, conf.get(StramClientUtils.KEY_TAB_FILE));
    } else if (conf.get(StramUserLogin.DT_AUTH_KEYTAB) != null) {
        Path localKeyTabPath = new Path(conf.get(StramUserLogin.DT_AUTH_KEYTAB));
        FileSystem fs = StramClientUtils.newFileSystemInstance(conf);
        try {
            Path destPath = new Path(StramClientUtils.getDTDFSRootDir(fs, conf), localKeyTabPath.getName());
            if (!fs.exists(destPath)) {
                fs.copyFromLocalFile(false, false, localKeyTabPath, destPath);
            }
            dag.setAttribute(LogicalPlan.KEY_TAB_FILE, destPath.toString());
        } finally {
            fs.close();
        }
    }
    String tokenRefreshFactor = conf.get(StramClientUtils.TOKEN_ANTICIPATORY_REFRESH_FACTOR);
    if (tokenRefreshFactor != null && tokenRefreshFactor.trim().length() > 0) {
        dag.setAttribute(LogicalPlan.TOKEN_REFRESH_ANTICIPATORY_FACTOR, Double.parseDouble(tokenRefreshFactor));
    }
    StramClient client = new StramClient(conf, dag);
    try {
        client.start();
        LinkedHashSet<String> libjars = Sets.newLinkedHashSet();
        String libjarsCsv = conf.get(LIBJARS_CONF_KEY_NAME);
        if (libjarsCsv != null) {
            String[] jars = StringUtils.splitByWholeSeparator(libjarsCsv, StramClient.LIB_JARS_SEP);
            libjars.addAll(Arrays.asList(jars));
        }
        if (deployJars != null) {
            for (File deployJar : deployJars) {
                libjars.add(deployJar.getAbsolutePath());
            }
        }

        client.setResources(libjars);
        client.setFiles(conf.get(FILES_CONF_KEY_NAME));
        client.setArchives(conf.get(ARCHIVES_CONF_KEY_NAME));
        client.setOriginalAppId(conf.get(ORIGINAL_APP_ID));
        client.setQueueName(conf.get(QUEUE_NAME));
        client.startApplication();
        return client.getApplicationReport().getApplicationId();
    } finally {
        client.stop();
    }
}

---

From source file:com.datatorrent.stram.LaunchContainerRunnable.java

License:Apache License

public static void addFilesToLocalResources(LocalResourceType type, String commaSeparatedFileNames,
        Map<String, LocalResource> localResources, FileSystem fs) throws IOException {
    String[] files = StringUtils.splitByWholeSeparator(commaSeparatedFileNames, StramClient.LIB_JARS_SEP);
    for (String file : files) {
        Path dst = new Path(file);
        // Create a local resource to point to the destination jar path
        FileStatus destStatus = fs.getFileStatus(dst);
        LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
        // Set the type of resource - file or archive
        amJarRsrc.setType(type);//www . j  a v a  2 s  .c o  m
        // Set visibility of the resource
        // Setting to most private option
        amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
        // Set the resource to be copied over
        amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
        // Set timestamp and length of file so that the framework
        // can do basic sanity checks for the local resource
        // after it has been copied over to ensure it is the same
        // resource the client intended to use with the application
        amJarRsrc.setTimestamp(destStatus.getModificationTime());
        amJarRsrc.setSize(destStatus.getLen());
        localResources.put(dst.getName(), amJarRsrc);
    }
}

---

From source file:com.datatorrent.stram.StramClient.java

License:Apache License

private String copyFromLocal(FileSystem fs, Path basePath, String[] files) throws IOException {
    StringBuilder csv = new StringBuilder(files.length * (basePath.toString().length() + 16));
    for (String localFile : files) {
        Path src = new Path(localFile);
        String filename = src.getName();
        Path dst = new Path(basePath, filename);
        URI localFileURI = null;/*from  www.ja  v a 2s  . co  m*/
        try {
            localFileURI = new URI(localFile);
        } catch (URISyntaxException e) {
            throw new IOException(e);
        }
        if (localFileURI.getScheme() == null || localFileURI.getScheme().startsWith("file")) {
            LOG.info("Copy {} from local filesystem to {}", localFile, dst);
            fs.copyFromLocalFile(false, true, src, dst);
        } else {
            LOG.info("Copy {} from DFS to {}", localFile, dst);
            FileUtil.copy(fs, src, fs, dst, false, true, conf);
        }
        if (csv.length() > 0) {
            csv.append(LIB_JARS_SEP);
        }
        csv.append(dst.toString());
    }
    return csv.toString();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »