Example usage for org.apache.hadoop.fs Path getFileSystem

List of usage examples for org.apache.hadoop.fs Path getFileSystem

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. getFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getFileSystem.

Prototype

public FileSystem getFileSystem(Configuration conf) throws IOException

Source Link

Document

Return the FileSystem that owns this Path.

Usage

From source file:com.alexholmes.hdfsslurper.WorkerThread.java

License:Apache License

private void process(FileStatus srcFileStatus) throws IOException, InterruptedException {

    Path stagingFile = null;// w  w w.j  av a  2 s  . c om
    FileSystem destFs = null;
    String filenameBatchidDelimiter = config.getFileNameBatchIdDelimiter();

    try {
        FileSystem srcFs = srcFileStatus.getPath().getFileSystem(config.getConfig());

        // run a script which can change the name of the file as well as
        // write out a new version of the file
        //
        if (config.getWorkScript() != null) {
            Path newSrcFile = stageSource(srcFileStatus);
            srcFileStatus = srcFileStatus.getPath().getFileSystem(config.getConfig()).getFileStatus(newSrcFile);
        }

        Path srcFile = srcFileStatus.getPath();

        // get the target HDFS file
        //
        Path destFile = getHdfsTargetPath(srcFileStatus);

        if (config.getCodec() != null) {
            String ext = config.getCodec().getDefaultExtension();
            if (!destFile.getName().endsWith(ext)) {
                destFile = new Path(destFile.toString() + ext);
            }
        }

        destFs = destFile.getFileSystem(config.getConfig());

        // get the staging HDFS file
        //
        stagingFile = fileSystemManager.getStagingFile(srcFileStatus, destFile);
        String batchId = srcFile.toString().substring(
                srcFile.toString().lastIndexOf(filenameBatchidDelimiter) + 1, srcFile.toString().length());

        log.info("event#Copying source file '" + srcFile + "' to staging destination '" + stagingFile + "'"
                + "$batchId#" + batchId);

        // if the directory of the target file doesn't exist, attempt to
        // create it
        //
        Path destParentDir = destFile.getParent();
        if (!destFs.exists(destParentDir)) {
            log.info("event#Attempting creation of target directory: " + destParentDir.toUri());
            if (!destFs.mkdirs(destParentDir)) {
                throw new IOException("event#Failed to create target directory: " + destParentDir.toUri());
            }
        }

        // if the staging directory doesn't exist, attempt to create it
        //
        Path destStagingParentDir = stagingFile.getParent();
        if (!destFs.exists(destStagingParentDir)) {
            log.info("event#Attempting creation of staging directory: " + destStagingParentDir.toUri());
            if (!destFs.mkdirs(destStagingParentDir)) {
                throw new IOException("event#Failed to create staging directory: " + destParentDir.toUri());
            }
        }

        // copy the file
        //
        InputStream is = null;
        OutputStream os = null;
        CRC32 crc = new CRC32();
        try {
            is = new BufferedInputStream(srcFs.open(srcFile));
            if (config.isVerify()) {
                is = new CheckedInputStream(is, crc);
            }
            os = destFs.create(stagingFile);

            if (config.getCodec() != null) {
                os = config.getCodec().createOutputStream(os);
            }

            IOUtils.copyBytes(is, os, 4096, false);
        } finally {
            IOUtils.closeStream(is);
            IOUtils.closeStream(os);
        }

        long srcFileSize = srcFs.getFileStatus(srcFile).getLen();
        long destFileSize = destFs.getFileStatus(stagingFile).getLen();
        if (config.getCodec() == null && srcFileSize != destFileSize) {
            throw new IOException(
                    "event#File sizes don't match, source = " + srcFileSize + ", dest = " + destFileSize);
        }

        log.info("event#Local file size = " + srcFileSize + ", HDFS file size = " + destFileSize + "$batchId#"
                + batchId);

        if (config.isVerify()) {
            verify(stagingFile, crc.getValue());
        }

        if (destFs.exists(destFile)) {
            destFs.delete(destFile, false);
        }

        log.info("event#Moving staging file '" + stagingFile + "' to destination '" + destFile + "'"
                + "$batchId#" + batchId);
        if (!destFs.rename(stagingFile, destFile)) {
            throw new IOException("event#Failed to rename file");
        }

        if (config.isCreateLzopIndex() && destFile.getName().endsWith(lzopExt)) {
            Path lzoIndexPath = new Path(destFile.toString() + LzoIndex.LZO_INDEX_SUFFIX);
            if (destFs.exists(lzoIndexPath)) {
                log.info("event#Deleting index file as it already exists");
                destFs.delete(lzoIndexPath, false);
            }
            indexer.index(destFile);
        }

        fileSystemManager.fileCopyComplete(srcFileStatus);

    } catch (Throwable t) {
        log.error("event#Caught exception working on file " + srcFileStatus.getPath(), t);

        // delete the staging file if it still exists
        //
        try {
            if (destFs != null && destFs.exists(stagingFile)) {
                destFs.delete(stagingFile, false);
            }
        } catch (Throwable t2) {
            log.error("event#Failed to delete staging file " + stagingFile, t2);
        }

        fileSystemManager.fileCopyError(srcFileStatus);
    }

}

From source file:com.alexholmes.hdfsslurper.WorkerThread.java

License:Apache License

private long hdfsFileCRC32(Path path) throws IOException {
    InputStream in = null;//from  w w  w . java2s  .c o  m
    CRC32 crc = new CRC32();
    try {
        InputStream is = new BufferedInputStream(path.getFileSystem(config.getConfig()).open(path));
        if (config.getCodec() != null) {
            is = config.getCodec().createInputStream(is);
        }
        in = new CheckedInputStream(is, crc);
        org.apache.commons.io.IOUtils.copy(in, new NullOutputStream());
    } finally {
        org.apache.commons.io.IOUtils.closeQuietly(in);
    }
    return crc.getValue();
}

From source file:com.alibaba.jstorm.hdfs.blobstore.HdfsBlobStoreImpl.java

License:Apache License

public HdfsBlobStoreImpl(Path path, Map<String, Object> conf, Configuration hconf) throws IOException {
    LOG.info("Blob store based in {}", path);
    _fullPath = path;/*from  www .  ja  v  a2 s  . co  m*/
    _hadoopConf = hconf;

    String hdfsHostName = (String) conf.get(Config.BLOBSTORE_HDFS_HOSTNAME);
    Integer hdfsPort = JStormUtils.parseInt(conf.get(Config.BLOBSTORE_HDFS_PORT));
    String defaultFS = (String) conf.get(Config.BLOBSTORE_HDFS_DEFAULT_FS);
    if ((hdfsHostName == null || hdfsPort == null) && defaultFS == null) {
        throw new RuntimeException(
                "<blobstore.hdfs.hostname, blobstore.hdfs.port> and blobstore.hdfs.defaultFS "
                        + "is empty. You must specify an HDFS location! ");
    }
    if (defaultFS == null) {
        defaultFS = String.format("hdfs://%s:%d", hdfsHostName, hdfsPort);
    }
    LOG.info("HDFS blob store, using defaultFS: {}", defaultFS);
    _hadoopConf.set("fs.defaultFS", defaultFS);

    String keyPrefix = "blobstore.hdfs.";
    for (Map.Entry<String, Object> confEntry : conf.entrySet()) {
        String key = confEntry.getKey();
        Object value = confEntry.getValue();
        if (key.startsWith(keyPrefix) && value != null) {
            key = key.substring(keyPrefix.length(), key.length());
            LOG.info("adding \"{}={}\" to hadoop conf", key, value);
            _hadoopConf.set(key, value.toString());
        }
    }

    _fs = path.getFileSystem(_hadoopConf);

    if (!_fs.exists(_fullPath)) {
        FsPermission perms = new FsPermission(BLOBSTORE_DIR_PERMISSION);
        boolean success = false;
        try {
            success = _fs.mkdirs(_fullPath, perms);
        } catch (IOException e) {
            LOG.error("fs mkdir ", e);
        }
        if (!success) {
            throw new IOException("Error creating blobstore directory: " + _fullPath);
        }
    }

    Object shouldCleanup = conf.get(Config.BLOBSTORE_CLEANUP_ENABLE);
    if (JStormUtils.parseBoolean(shouldCleanup, false)) {
        LOG.debug("Starting hdfs blobstore cleaner");
        _cleanup = new TimerTask() {
            @Override
            public void run() {
                try {
                    fullCleanup(FULL_CLEANUP_FREQ);
                } catch (IOException e) {
                    LOG.error("Error trying to cleanup", e);
                }
            }
        };
        timer.scheduleAtFixedRate(_cleanup, 0, FULL_CLEANUP_FREQ);
    }
}

From source file:com.alibaba.jstorm.hdfs.blobstore.HdfsBlobStoreImpl.java

License:Apache License

/**
 * Check if the key exists in the blob store.
 *
 * @param key the key to check for/*from  w  w  w  .jav  a2 s  .  co m*/
 * @return true if it exists else false.
 */
public boolean exists(String key) {
    Path dir = getKeyDir(key);
    boolean res = false;
    try {
        _fs = dir.getFileSystem(_hadoopConf);
        res = _fs.exists(dir);
    } catch (IOException e) {
        LOG.warn("Exception checking for exists on: " + key);
    }
    return res;
}

From source file:com.asakusafw.bulkloader.collector.ExportFileSendTest.java

License:Apache License

@SuppressWarnings("unchecked")
private File prepareInput(String path) throws IOException {
    File result = folder.newFile();
    Path p = new Path(new File(path).toURI());
    FileSystem fs = p.getFileSystem(new Configuration());
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, fs.getConf());
    try {//from   ww  w  .ja va 2  s  .  com
        Writable buffer = (Writable) reader.getValueClass().newInstance();
        ModelOutput<Writable> output = (ModelOutput<Writable>) TemporaryStorage.openOutput(fs.getConf(),
                reader.getValueClass(), new BufferedOutputStream(new FileOutputStream(result)));
        try {
            while (reader.next(NullWritable.get(), buffer)) {
                output.write(buffer);
            }
        } finally {
            output.close();
        }
    } catch (Exception e) {
        throw new AssertionError(e);
    } finally {
        reader.close();
    }
    return result;
}

From source file:com.asakusafw.cleaner.main.HDFSCleaner.java

License:Apache License

/**
 * HDFSCleaner???/*ww  w  .  j a  va 2s. co m*/
 * @param args 
 * @return 
 */
protected int execute(String[] args) {
    String[] prop = new String[1];
    String mode = null;
    String user = null;
    FileSystem fs = null;

    if (args.length > 0) {
        mode = args[0];
    }
    if (args.length > 1) {
        user = args[1];
    }
    if (args.length > 2) {
        prop[0] = args[2];
    }

    // ??
    if (args.length != 3) {
        System.err.println("ERROR????? ?" + args.length
                + " " + mode + " ??" + user
                + " " + prop[0]);
        Log.log(CLASS, MessageIdConst.HCLN_PARAMCHECK_ERROR, "?", args.length, new Date(), mode,
                prop[0]);
        return Constants.EXIT_CODE_ERROR;
    }

    try {
        // ??
        if (!CleanerInitializer.initDFSCleaner(prop)) {
            Log.log(CLASS, MessageIdConst.HCLN_INIT_ERROR, new Date(), mode, prop[0]);
            return Constants.EXIT_CODE_ERROR;
        }

        // 
        Log.log(CLASS, MessageIdConst.HCLN_START, new Date(), mode, prop[0]);

        // ?
        boolean recursive = false;
        if (Constants.CLEAN_MODE_NOMAL.equals(mode)) {
            recursive = false;
        } else if (Constants.CLEAN_MODE_RECURSIVE.equals(mode)) {
            recursive = true;
        } else {
            Log.log(CLASS, MessageIdConst.HCLN_PARAMCHECK_ERROR, "", mode, new Date(), mode,
                    prop[0]);
            return Constants.EXIT_CODE_ERROR;
        }

        // HDFS??
        DFSCleanerBean[] bean = null;
        try {
            bean = getCleanLocalPath(user);
        } catch (CleanerSystemException e) {
            Log.log(e.getCause(), e.getClazz(), e.getMessageId(), e.getMessageArgs());
            return Constants.EXIT_CODE_ERROR;
        }

        // ???
        int keepDate = getHDFSFileKeepDate();

        boolean cleanResult = true;
        Date now = new Date();
        for (int i = 0; i < bean.length; i++) {
            try {
                // 
                Path cleanDir = bean[i].getCleanDir();
                // ?
                try {
                    Configuration conf = getConf();
                    fs = cleanDir.getFileSystem(conf);
                    if (fs == null) {
                        Log.log(CLASS, MessageIdConst.HCLN_CLEN_DIR_ERROR,
                                "Path.getFileSystem??null", cleanDir.toString());
                        cleanResult = false;
                        continue;
                    }
                } catch (IOException e) {
                    Log.log(e, CLASS, MessageIdConst.HCLN_CLEN_DIR_ERROR,
                            "HDFS????", cleanDir.toString());
                    cleanResult = false;
                    continue;
                }

                boolean target = bean[i].hasExecutionId();
                String pattern = bean[i].getPattern();
                Log.log(CLASS, MessageIdConst.HCLN_CLEN_FILE, cleanDir.toString(), pattern, keepDate, mode,
                        target, now);
                if (cleanDir(fs, cleanDir, target, pattern, keepDate, now, recursive)) {
                    Log.log(CLASS, MessageIdConst.HCLN_CLEN_DIR_SUCCESS, cleanDir.toString(), keepDate, mode);
                } else {
                    Log.log(CLASS, MessageIdConst.HCLN_CLEN_DIR_FAIL, cleanDir.toString(), keepDate, mode);
                    cleanResult = false;
                }
            } catch (CleanerSystemException e) {
                Log.log(e.getCause(), e.getClazz(), e.getMessageId(), e.getMessageArgs());
                cleanResult = false;
            } finally {
                if (fs != null) {
                    // CHECKSTYLE:OFF EmptyBlockCheck
                    try {
                        fs.close();
                    } catch (IOException ignored) {
                        // ignored
                    }
                    // CHECKSTYLE:ON EmptyBlockCheck
                }
            }
        }

        // 
        if (cleanResult) {
            Log.log(CLASS, MessageIdConst.HCLN_EXIT_SUCCESS, new Date(), mode, prop[0]);
            return Constants.EXIT_CODE_SUCCESS;
        } else {
            Log.log(CLASS, MessageIdConst.HCLN_EXIT_WARNING, new Date(), mode, prop[0]);
            return Constants.EXIT_CODE_WARNING;
        }
    } catch (RuntimeException e) {
        try {
            Log.log(e, CLASS, MessageIdConst.HCLN_EXCEPRION, new Date(), mode, prop[0]);
            return Constants.EXIT_CODE_ERROR;
        } catch (Exception e1) {
            System.err.print("HDFSCleaner????????");
            e1.printStackTrace();
            return Constants.EXIT_CODE_ERROR;
        }
    }
}

From source file:com.asakusafw.compiler.fileio.HadoopFileIoProcessorTest.java

License:Apache License

private <T> ModelOutput<T> openOutput(Class<T> aClass, Location location) throws IOException {
    Configuration conf = tester.configuration();
    Path path = new Path(location.toPath('/'));
    FileSystem fs = path.getFileSystem(conf);
    SequenceFile.Writer writer = SequenceFile.createWriter(conf,
            SequenceFile.Writer.file(fs.makeQualified(path)), SequenceFile.Writer.keyClass(NullWritable.class),
            SequenceFile.Writer.valueClass(aClass));
    return new SequenceFileModelOutput<>(writer);
}

From source file:com.asakusafw.compiler.fileio.HadoopFileIoProcessorTest.java

License:Apache License

private List<Ex1> getList(Class<? extends FileExporterDescription> exporter) {
    try {//from  www. j av  a2s  . com
        FileExporterDescription instance = exporter.newInstance();
        Path path = new Path(Location.fromPath(instance.getPathPrefix(), '/').toString());
        FileSystem fs = path.getFileSystem(tester.configuration());
        FileStatus[] statuses = fs.globStatus(path);
        List<Ex1> results = new ArrayList<>();
        for (FileStatus status : statuses) {
            try (SequenceFile.Reader reader = new SequenceFile.Reader(tester.configuration(),
                    SequenceFile.Reader.file(fs.makeQualified(status.getPath())))) {
                Ex1 model = new Ex1();
                while (reader.next(NullWritable.get(), model)) {
                    Ex1 copy = new Ex1();
                    copy.copyFrom(model);
                    results.add(copy);
                }
            }
        }
        Collections.sort(results, new Comparator<Ex1>() {
            @Override
            public int compare(Ex1 o1, Ex1 o2) {
                return o1.getSidOption().compareTo(o2.getSidOption());
            }
        });
        return results;
    } catch (Exception e) {
        throw new AssertionError(e);
    }
}

From source file:com.asakusafw.compiler.util.tester.HadoopDriver.java

License:Apache License

private void copyFromHadoop(Location location, File targetDirectory) throws IOException {
    targetDirectory.mkdirs();// w  w w. java2  s  .  co  m
    logger.info("copy {} to {}", location, targetDirectory);

    Path path = new Path(location.toPath('/'));
    FileSystem fs = path.getFileSystem(configuration);
    FileStatus[] list = fs.globStatus(path);
    if (list == null) {
        throw new IOException(
                MessageFormat.format("Failed to fs -get: source={0}, destination={1}", path, targetDirectory));
    }
    for (FileStatus status : list) {
        Path p = status.getPath();
        try {
            fs.copyToLocalFile(p, new Path(new File(targetDirectory, p.getName()).toURI()));
        } catch (IOException e) {
            throw new IOException(
                    MessageFormat.format("Failed to fs -get: source={0}, destination={1}", p, targetDirectory),
                    e);
        }
    }
}

From source file:com.asakusafw.compiler.util.tester.HadoopDriver.java

License:Apache License

/**
 * Cleans up the temporary working area.
 * @throws IOException if failed to clean up
 *//* w  ww. j  a v  a 2  s . c om*/
public void clean() throws IOException {
    logger.info("clean user directory");
    Path path = new Path(toPath().toPath('/'));
    FileSystem fs = path.getFileSystem(configuration);
    try {
        if (fs.exists(path)) {
            fs.delete(path, true);
        }
    } catch (IOException e) {
        logger.info(MessageFormat.format("Failed to fs -rmr {0}", toPath()), e);
    }
}