Example usage for org.apache.hadoop.fs FileSystem exists

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem exists.

Prototype

public boolean exists(Path f) throws IOException

Source Link

Document

Check if a path exists.

Usage

From source file:com.asakusafw.dag.runtime.directio.TransactionManager.java

License:Apache License

boolean isCommitted() throws IOException {
    Path commitMark = getCommitMarkPath();
    FileSystem fs = commitMark.getFileSystem(configuration);
    return fs.exists(commitMark);
}

From source file:com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditor.java

License:Apache License

private TransactionInfo toInfoObject(FileStatus stat) throws IOException {
    assert stat != null;
    Path path = stat.getPath();/*from w w  w  .j av a  2  s . co  m*/
    String executionId = HadoopDataSourceUtil.getTransactionInfoExecutionId(path);
    long timestamp = stat.getModificationTime();
    List<String> comment = new ArrayList<>();
    Path commitMarkPath = HadoopDataSourceUtil.getCommitMarkPath(getConf(), executionId);
    FileSystem fs = path.getFileSystem(getConf());
    boolean committed = fs.exists(commitMarkPath);
    try (FSDataInputStream input = fs.open(path);
            Scanner scanner = new Scanner(new InputStreamReader(input, HadoopDataSourceUtil.COMMENT_CHARSET))) {
        while (scanner.hasNextLine()) {
            comment.add(scanner.nextLine());
        }
    } catch (IOException e) {
        comment.add(e.toString());
    }

    return new TransactionInfo(executionId, timestamp, committed, comment);
}

From source file:com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditor.java

License:Apache License

private boolean doApply(String executionId) throws IOException, InterruptedException {
    assert executionId != null;
    Path transactionInfo = HadoopDataSourceUtil.getTransactionInfoPath(getConf(), executionId);
    Path commitMark = HadoopDataSourceUtil.getCommitMarkPath(getConf(), executionId);
    FileSystem fs = commitMark.getFileSystem(getConf());
    if (fs.exists(transactionInfo) == false) {
        return false;
    }/* ww  w . j a va  2  s.  co  m*/
    boolean succeed = true;
    if (fs.exists(commitMark) == false) {
        // FIXME cleanup
        return false;
    }
    DirectDataSourceRepository repo = getRepository();
    for (String containerPath : repo.getContainerPaths()) {
        String datasourceId = repo.getRelatedId(containerPath);
        try {
            DirectDataSource datasource = repo.getRelatedDataSource(containerPath);
            OutputTransactionContext context = HadoopDataSourceUtil.createContext(executionId, datasourceId);
            datasource.commitTransactionOutput(context);
            datasource.cleanupTransactionOutput(context);
        } catch (IOException e) {
            succeed = false;
            LOG.error(MessageFormat.format("Failed to apply transaction (datastoreId={0}, executionId={1})",
                    datasourceId, executionId));
        }
    }
    if (succeed) {
        LOG.info(MessageFormat.format("Deleting commit mark (executionId={0}, path={1})", executionId,
                commitMark));
        try {
            if (fs.delete(commitMark, true) == false) {
                LOG.warn(MessageFormat.format("Failed to delete commit mark (executionId={0}, path={1})",
                        executionId, commitMark));
            } else if (fs.delete(transactionInfo, true) == false) {
                LOG.warn(MessageFormat.format("Failed to delete transaction info (executionId={0}, path={1})",
                        executionId, transactionInfo));
            }
        } catch (FileNotFoundException e) {
            LOG.warn(MessageFormat.format("Failed to delete commit mark (executionId={0}, path={1})",
                    executionId, commitMark), e);
        }
        return true;
    } else {
        throw new IOException(MessageFormat.format("Failed to apply this transaction (executionId={0});"
                + " if you want to ignore this transaction, please abort this.", executionId));
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditor.java

License:Apache License

private boolean doAbort(String executionId) throws IOException, InterruptedException {
    assert executionId != null;
    Path transactionInfo = HadoopDataSourceUtil.getTransactionInfoPath(getConf(), executionId);
    Path commitMark = HadoopDataSourceUtil.getCommitMarkPath(getConf(), executionId);
    FileSystem fs = commitMark.getFileSystem(getConf());
    if (fs.exists(transactionInfo) == false) {
        return false;
    }//from w w  w  . j a va2  s  .co m
    boolean succeed = true;
    if (fs.exists(commitMark)) {
        LOG.info(MessageFormat.format("Deleting commit mark (executionId={0}, path={1})", executionId,
                commitMark));
        if (fs.delete(commitMark, true) == false) {
            succeed = false;
            LOG.warn(MessageFormat.format("Failed to delete commit mark (executionId={0}, path={1})",
                    executionId, commitMark));
        }
    }
    DirectDataSourceRepository repo = getRepository();
    for (String containerPath : repo.getContainerPaths()) {
        String datasourceId = repo.getRelatedId(containerPath);
        try {
            DirectDataSource datasource = repo.getRelatedDataSource(containerPath);
            OutputTransactionContext context = HadoopDataSourceUtil.createContext(executionId, datasourceId);
            datasource.cleanupTransactionOutput(context);
        } catch (IOException e) {
            succeed = false;
            LOG.error(MessageFormat.format("Failed to abort transaction (datastoreId={0}, executionId={1})",
                    datasourceId, executionId));
        }
    }
    if (succeed) {
        LOG.info(MessageFormat.format("Deleting transaction info (executionId={0}, path={1})", executionId,
                commitMark));
        try {
            if (fs.delete(transactionInfo, true) == false) {
                LOG.warn(MessageFormat.format("Failed to delete transaction info (executionId={0}, path={1})",
                        executionId, transactionInfo));
            }
        } catch (FileNotFoundException e) {
            LOG.warn(MessageFormat.format("Failed to delete transaction info (executionId={0}, path={1})",
                    executionId, commitMark), e);
        }
        return true;
    } else {
        throw new IOException(MessageFormat.format(
                "Failed to abort this transaction (executionId={0});"
                        + " if you want to ignore this transaction, please delete {1} manually.",
                executionId, transactionInfo));
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCore.java

License:Apache License

@Override
public void cleanupTransactionOutput(OutputTransactionContext context)
        throws IOException, InterruptedException {
    FileSystem fs = profile.getFileSystem();
    Path path = getTemporaryOutput(context);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Delete temporary area (id={0}, path={1})", //$NON-NLS-1$
                profile.getId(), path));
    }//from w w w . ja  va2 s  . co m
    try {
        if (fs.exists(path)) {
            if (fs.delete(path, true) == false) {
                LOG.warn(MessageFormat.format("Failed to delete temporary area (id={0}, path={0})",
                        profile.getId(), path));
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Temporary area is not found (may be not used): {0}", //$NON-NLS-1$
                        path));
            }
        }
    } catch (FileNotFoundException e) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("Temporary area is not found (may be not used): {0}", //$NON-NLS-1$
                    path));
        }
    }
}

From source file:com.asakusafw.runtime.stage.launcher.LauncherOptionsParser.java

License:Apache License

private List<Path> consumeLibraryPaths(LinkedList<String> rest) throws IOException {
    List<String> names = consumeLibraryNames(rest);
    if (names.isEmpty()) {
        return Collections.emptyList();
    }/*from   ww w .jav a  2s.  c  o  m*/
    List<Path> results = new ArrayList<>();
    LocalFileSystem local = FileSystem.getLocal(configuration);
    for (String name : names) {
        Path path = new Path(name);
        FileSystem fs;
        if (path.toUri().getScheme() == null) {
            fs = local;
        } else {
            fs = path.getFileSystem(configuration);
        }
        path = fs.makeQualified(path);
        if (fs.exists(path) == false) {
            throw new FileNotFoundException(path.toString());
        }
        results.add(path);
    }
    return results;
}

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private Path doResolve(Path sourcePath) throws IOException, InterruptedException {
    assert sourcePath.isAbsolute();
    FileSystem fs = sourcePath.getFileSystem(configuration);
    if (fs.exists(sourcePath) == false) {
        throw new FileNotFoundException(sourcePath.toString());
    }/*from  w  w  w . j a va 2  s  .co  m*/
    long sourceChecksum = computeChecksum(fs, sourcePath);
    Path cachePath = computeCachePath(sourcePath);
    Path cacheChecksumPath = computeCacheChecksumPath(cachePath);

    IOException firstException = null;
    RetryObject retry = retryStrategy
            .newInstance(MessageFormat.format("preparing cache ({0} -> {1})", sourcePath, cachePath));
    do {
        try (LockObject<? super Path> lock = lockProvider.tryLock(cachePath)) {
            // TODO reduce lock scope?
            if (lock == null) {
                continue;
            }
            if (isCached(cachePath, cacheChecksumPath, sourceChecksum)) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("cache hit: {0} -> {1}", //$NON-NLS-1$
                            sourcePath, cachePath));
                }
                // just returns cached file
            } else {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("cache miss: {0} -> {1}", //$NON-NLS-1$
                            sourcePath, cachePath));
                }
                updateCache(sourcePath, sourceChecksum, cachePath, cacheChecksumPath);
            }
            return cachePath;
        } catch (IOException e) {
            LOG.warn(MessageFormat.format("Failed to prepare cache: {0} -> {1}", sourcePath, cachePath), e);
            if (firstException == null) {
                firstException = e;
            }
        }
    } while (retry.waitForNextAttempt());
    if (firstException == null) {
        throw new IOException(MessageFormat.format("Failed to acquire a lock for remote cache file: {0} ({1})",
                sourcePath, cachePath));
    }
    throw firstException;
}

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private boolean isCached(Path cacheFilePath, Path cacheChecksumPath, long checksum) throws IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("checking remote cache: {0}", //$NON-NLS-1$
                cacheFilePath));/* w  w  w .  ja  va2  s . c  o  m*/
    }
    FileSystem fs = cacheChecksumPath.getFileSystem(configuration);
    if (fs.exists(cacheChecksumPath) == false || fs.exists(cacheFilePath) == false) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("remote cache is not found: {0}", //$NON-NLS-1$
                    cacheFilePath));
        }
        return false;
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("reading remote cache checksum: {0}", //$NON-NLS-1$
                    cacheFilePath));
        }
        long other;
        try (FSDataInputStream input = fs.open(cacheChecksumPath)) {
            other = input.readLong();
        }
        return checksum == other;
    }
}

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private void delete(FileSystem fs, Path path) throws IOException {
    if (checkBeforeDelete && fs.exists(path) == false) {
        return;/*  w w  w  .j a  va  2 s.  c  om*/
    }
    fs.delete(path, false);
}

From source file:com.asakusafw.testdriver.testing.moderator.MockImporterPreparator.java

License:Apache License

@Override
public void truncate(MockImporterDescription description, TestContext context) throws IOException {
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    Path target = fs.makeQualified(new Path(description.getDirectory()));
    if (fs.exists(target)) {
        fs.delete(target, true);//  w ww  .  j  a  va  2 s . c  o m
    }
}