Example usage for org.apache.hadoop.fs Path getName

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getName.

Prototype

public String getName() 

Source Link

Document

Returns the final component of this path.

Usage

From source file:com.google.GsHdfs.java

License:Open Source License

public void copyHdfsToGs(String hdfsFn, String gsFn) throws Exception {
    Path srcPath = new Path(hdfsFn);
    if (hdfs.isFile(srcPath)) {
        FSDataInputStream src = hdfs.open(srcPath);
        Process gsutil = Runtime.getRuntime().exec(new String[] { "gsutil", "cp", "-", gsFn });
        OutputStream dst = gsutil.getOutputStream();
        System.out.println(hdfsFn + " -> " + gsFn);
        doCopy(src, dst, hdfsFn);/*ww w. j  ava  2s .  c o m*/
    } else {
        // Recurse
        for (FileStatus file : hdfs.listStatus(srcPath)) {
            Path path = file.getPath();
            copyHdfsToGs(path.toString(), gsFn + "/" + path.getName());
        }
    }
}

---

From source file:com.grantingersoll.intell.clustering.KMeansClusteringEngine.java

License:Apache License

private static Map<Integer, List<String>> readPoints(Path pointsPathDir, Configuration conf)
        throws IOException {
    Map<Integer, List<String>> result = new TreeMap<Integer, List<String>>();

    FileSystem fs = pointsPathDir.getFileSystem(conf);
    FileStatus[] children = fs.listStatus(pointsPathDir, new PathFilter() {
        public boolean accept(Path path) {
            String name = path.getName();
            return !(name.endsWith(".crc") || name.startsWith("_"));
        }/*from   w  ww.  jav a2 s .  c  o m*/
    });

    for (FileStatus file : children) {
        Path path = file.getPath();
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
        try {
            IntWritable key = reader.getKeyClass().asSubclass(IntWritable.class).newInstance();
            WeightedVectorWritable value = reader.getValueClass().asSubclass(WeightedVectorWritable.class)
                    .newInstance();
            while (reader.next(key, value)) {
                //key is the clusterId, value is a list of points
                //String clusterId = value.toString();
                List<String> pointList = result.get(key.get());
                if (pointList == null) {
                    pointList = new ArrayList<String>();
                    result.put(key.get(), pointList);
                }
                //We know we are dealing with named vectors, b/c we generated from the id field
                String name = ((NamedVector) value.getVector()).getName();
                pointList.add(name);
                //value = reader.getValueClass().asSubclass(WeightedVectorWritable.class).newInstance();
            }
        } catch (InstantiationException e) {
            log.error("Exception", e);
        } catch (IllegalAccessException e) {
            log.error("Exception", e);
        }
    }

    return result;
}

---

From source file:com.gruter.hadoop.customShell.CustomShell.java

License:Apache License

/**
 * Fetch all files that match the file pattern <i>srcf</i> and display
 * their content on stdout. //from  www .  ja  va2 s .c  o m
 * @param srcf: a file pattern specifying source files
 * @exception: IOException
 * @see org.apache.hadoop.fs.FileSystem.globStatus 
 */
void cat(final String src, boolean verifyChecksum) throws IOException {
    //cat behavior in Linux
    //  [~/1207]$ ls ?.txt
    //  x.txt  z.txt
    //  [~/1207]$ cat x.txt y.txt z.txt
    //  xxx
    //  cat: y.txt: No such file or directory
    //  zzz

    Path srcPattern = new Path(src);
    new DelayedExceptionThrowing() {
        @Override
        void process(Path p, FileSystem srcFs) throws IOException {
            FSDataInputStream in = srcFs.open(p);
            /**
             * snappy
             */
            if (isSnappy(p.getName())) {
                printToStdout(getSnappyCodec().createInputStream(in));
            } else {
                printToStdout(in);
            }
            /**
             * end
             */
        }
    }.globAndProcess(srcPattern, getSrcFileSystem(srcPattern, verifyChecksum));
}

---

From source file:com.gruter.hadoop.customShell.CustomShell.java

License:Apache License

private InputStream forMagic(Path p, FileSystem srcFs) throws IOException {
    FSDataInputStream i = srcFs.open(p);
    switch (i.readShort()) {
    case 0x1f8b: // RFC 1952
        i.seek(0);//from w  w  w .  j ava  2 s  .  c  o  m
        return new GZIPInputStream(i);
    case 0x5345: // 'S' 'E'
        if (i.readByte() == 'Q') {
            i.close();
            return new TextRecordInputStream(srcFs.getFileStatus(p));
        }
        break;
    }
    i.seek(0);
    /**
     * snappy
     */
    if (isSnappy(p.getName())) {
        return getSnappyCodec().createInputStream(i);
    }
    /**
     * end
     */
    return i;
}

---

From source file:com.hazelcast.yarn.HazelcastYarnClient.java

License:Open Source License

private ApplicationId submitApplication(YarnClient yarnClient) throws Exception {
    YarnClientApplication app = yarnClient.createApplication();

    Path appJar = YarnUtil.createHDFSFile(this.fs, this.pathToAppJar,
            this.properties.hazelcastWorkDir() + File.separator + YarnUtil.JAR_NAME);

    ContainerLaunchContext amContainer = getContainerLaunchContext();

    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setEnv(appMasterEnv, this.conf);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = getCapability();
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();

    appContext.setApplicationName(appName);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);/*w ww . java2s. c om*/
    appContext.setQueue("default");

    ApplicationId appId = appContext.getApplicationId();
    LocalResource appMasterJar = YarnUtil.createFileResource(appJar, this.fs, LocalResourceType.FILE);
    amContainer.setLocalResources(Collections.singletonMap(appJar.getName(), appMasterJar));
    yarnClient.submitApplication(appContext);
    LOG.log(Level.INFO, "Submitted application. Application id: {0}", appId);
    return appId;
}

---

From source file:com.hdfs.concat.clean.Clean.java

License:Apache License

public int cleanup(Path p) {
    try {//w  ww. j  a  va  2s .  c o  m
        if (fs.isFile(p)) {
            if (conf.get(TARGET_EXPR) != null) {
                if (p.getName().matches(conf.get(TARGET_EXPR))) {
                    warnOrDelete(p);
                }
            }
            if (conf.get(CUTTOFF_MILLIS) != null) {
                if (fs.getFileStatus(p).getModificationTime() < cutoff) {
                    warnOrDelete(p);
                }
            }
        }

        if (fs.isDirectory(p)) {
            for (FileStatus stat : fs.listStatus(p)) {
                cleanup(stat.getPath());
            }
            if (fs.listStatus(p).length == 0) {
                if (conf.get(TARGET_EXPR) != null) {
                    if (p.getName().matches(conf.get(TARGET_EXPR))) {
                        warnOrDelete(p);
                    }
                }
                if (conf.get(CUTTOFF_MILLIS) != null) {
                    if (fs.getFileStatus(p).getModificationTime() < cutoff) {
                        warnOrDelete(p);
                    }
                }
            }
        }
    } catch (IOException e) {
        System.out.println("exception " + e);
        return 7;
    }
    return 0;
}

---

From source file:com.hdfs.concat.crush.Crush.java

License:Apache License

/**
 * Returns the output from {@link CrushReducer}. Each reducer writes out a mapping of source files to crush output file.
 *///  ww w. ja  v  a 2  s .  co m
private List<FileStatus> getOutputMappings() throws IOException {
    FileStatus[] files = fs.listStatus(outDir, new PathFilter() {
        Matcher matcher = Pattern.compile("part-\\d+").matcher("dummy");

        @Override
        public boolean accept(Path path) {
            matcher.reset(path.getName());

            return matcher.matches();
        }
    });

    return asList(files);
}

---

From source file:com.hdfs.concat.crush.Crush.java

License:Apache License

/**
 * Renames the source file to the destination file, taking into consideration that compression codes can mangle file names. Also
 * ensures that the parent directory of the destination exists.
 *
 * @param src/* w  w w  . j  av a  2s.c o m*/
 *          The path to the file to copy.
 * @param destDir
 *          The dir to which the file must be copied
 * @param fileName
 *          The new name of the file or null to keep the original file name
 *
 * @throws IOException
 */
private void rename(Path src, Path destDir, String fileName) throws IOException {
    fs.mkdirs(destDir);

    if (null != codecExtension && !fs.exists(src)) {
        /*
         * Try mangling the name like a codec would and invoke rename. Let execoptions bubble up.
         */
        src = new Path(src + codecExtension);
    }

    Path dest;

    if (null == fileName) {
        dest = new Path(destDir, src.getName());
    } else {
        dest = new Path(destDir, fileName);
    }

    fs.rename(src, dest);

    print(Verbosity.VERBOSE, format("\n  %s => %s", src, dest));
}

---

From source file:com.hdfstoftp.main.HdfsToFtp.java

/**
 * ?//from w w  w .  ja  v a2 s. c  o m
 * 
 * @param srcFS
 *            
 * @param src
 *            ?
 * @param dst
 *            
 * @param queryStr
 *            
 * @param deleteSource
 *            ??
 * @param overwrite
 *            ????
 * @return boolean
 * @throws Exception
 */
private static boolean copyFromHDFSToFTP(Config config) throws Exception {
    // ?hdfs
    Configuration conf = new Configuration();
    FileSystem srcFS = FileSystem.get(conf);
    long start = System.currentTimeMillis();
    boolean isRename = config.isRenameUploaded();
    int retryTimes = config.getRetryTimes();
    // ?
    String dstPath = config.getDestDir();
    Path src = new Path(config.getSouceDir());
    FileStatus fileStatus = srcFS.getFileStatus(src);
    String subDir = null;
    if (fileStatus.isDirectory()) {// 
        if (isRename) {// ??rename
            subDir = Config.RENAME_DIR;
            srcFS.mkdirs(new Path(fileStatus.getPath(), subDir));
        }
        int threadNum = config.getThreadNum();
        // 
        ExecutorService threadPool = Executors.newFixedThreadPool(threadNum);
        // ?ftp
        FTPClientPool ftpPool = new FTPClientPool(threadNum, new FtpClientFactory(config.getFTPClientConfig()));
        FTPClient ftpClient = ftpPool.borrowObject();
        // ?
        ftpClient.makeDirectory(dstPath);
        ftpPool.returnObject(ftpClient);
        // ??
        FileStatus contents[] = srcFS.listStatus(src);
        long beginFilter = 0;
        long endFileter = 0;

        if (config.getCommandLine().hasOption("d") || config.getCommandLine().hasOption("h")
                || config.getCommandLine().hasOption("t")) {// ?"["
            beginFilter = System.currentTimeMillis();
            Long[] timeRange = parseTimeRange(config.getCommandLine());
            contents = getNewContents(timeRange, contents);
            endFileter = System.currentTimeMillis();
        }
        // ?
        if (config.getCommandLine().hasOption("r")) {// "["??
            beginFilter = System.currentTimeMillis();
            contents = getFilterContents(config.getCommandLine().getOptionValue("r").trim(), contents);
            endFileter = System.currentTimeMillis();
        }
        logger.info("total file count:" + contents.length);
        Map<String, String> fileNameMap = null;
        long beginSkip = 0;
        long endSkip = 0;
        boolean overwrite = true;
        if (config.getCommandLine().hasOption("o")) {
            overwrite = "true".equals(config.getCommandLine().getOptionValue("o").trim());
        }
        if (!overwrite) {// ?????
            beginSkip = System.currentTimeMillis();
            fileNameMap = getFileNameMap(dstPath, ftpPool);
            endSkip = System.currentTimeMillis();
        }
        int skiped = 0;

        List<Future<?>> futureList = new ArrayList<Future<?>>();
        for (int i = 0; i < contents.length; i++) {
            if (!overwrite && fileNameMap.containsKey(contents[i].getPath().getName())) {
                // 
                skiped++;
                Log.info("skiped filename:" + contents[i].getPath().getName());
                continue;
            }
            if (contents[i].isDirectory()) {
                continue;
            }
            // ???
            Future<?> future = threadPool.submit(new UploadFileTask(srcFS, contents[i].getPath(),
                    new Path(dstPath, contents[i].getPath().getName()), ftpPool, false, isRename, subDir,
                    retryTimes));
            futureList.add(future);
        }
        int transfered = 0;
        int failed = 0;
        for (Future<?> future : futureList) {
            Boolean computeResult = (Boolean) future.get();
            if (computeResult) {
                transfered++;
                if (transfered % 50 == 0 || transfered == contents.length) {
                    logger.info("have transfered:" + transfered + " files");
                }
            } else {
                failed++;
                logger.error("failed transter:" + failed + " files");
            }
        }
        // 
        threadPool.shutdown();
        // FTPCient
        ftpPool.close();
        // ****************
        logger.info("filter time:" + (endFileter - beginFilter) + " ms");
        if (!overwrite) {
            logger.info("skip time:" + (endSkip - beginSkip) + " ms");
        }
        logger.info("total file count:" + contents.length);
        logger.info("total transtered: " + transfered + ",total failed:" + failed + ",total skiped:" + skiped);

    } else {// 

        BufferedReader reader = null;
        FtpClientFactory facotry = new FtpClientFactory(config.getFTPClientConfig());
        FTPClient ftpClient = null;
        InputStream in = null;
        try {
            Path path = fileStatus.getPath();
            if (!path.getName().contains("log")) {

            }
            reader = new BufferedReader(new FileReader(new File(path.toUri().getPath())));
            String str = null;

            ftpClient = facotry.makeObject();

            while ((str = reader.readLine()) != null) {
                String[] feilds = str.split("&");
                Path filePath = null;
                if (feilds.length == 2 && feilds[1] != "") {
                    filePath = new Path(feilds[1]);
                    in = srcFS.open(filePath);
                    boolean result = ftpClient.storeFile(dstPath, in);
                    System.out.println(ftpClient.getReplyCode());
                    if (result) {
                        logger.info(filePath.toString());
                    } else {
                        logger_failed.info(filePath.toString());
                    }
                } else {
                    continue;
                }

            }
        } catch (Exception e) {
            e.printStackTrace();

        } finally {
            in.close();
            reader.close();
            facotry.destroyObject(ftpClient);
        }

    }
    long end = System.currentTimeMillis();
    logger.info("finished transfer,total time:" + (end - start) / 1000 + "s");
    return true;
}

---

From source file:com.hortonworks.minicluster.InJvmContainerExecutor.java

License:Apache License

/**
 * YARN provides ability to pass resources (e.g., classpath) through
 * {@link LocalResource}s which allows user to provision all the resources
 * required to run the app. This method will extract those resources as a
 * {@link Set} of {@link URL}s so they are used when {@link ClassLoader} for a
 * container is created.//w w  w.  j a  v  a  2s . c  o m
 *
 * This is done primarily as a convenience for applications that rely on
 * automatic classpath propagation (e.g., pull everything from my dev
 * classpath) instead of manual.
 *
 * @param container
 * @return
 */
private Set<URL> filterAndBuildUserClasspath(Container container) {
    if (logger.isDebugEnabled()) {
        logger.debug("Building additional classpath for the container: " + container);
    }
    Set<URL> additionalClassPathUrls = new HashSet<URL>();
    Set<Path> userClassPath = this.extractUserProvidedClassPathEntries(container);

    for (Path resourcePath : userClassPath) {
        String resourceName = resourcePath.getName();
        if (logger.isDebugEnabled()) {
            logger.debug("\t adding " + resourceName + " to the classpath");
        }
        try {
            additionalClassPathUrls.add(resourcePath.toUri().toURL());
        } catch (Exception e) {
            throw new IllegalArgumentException(e);
        }
    }
    return additionalClassPathUrls;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »