Example usage for com.amazonaws.services.s3 AmazonS3 shutdown

Introduction

In this page you can find the example usage for com.amazonaws.services.s3 AmazonS3 shutdown.

Prototype

void shutdown();

Source Link

Document

Shuts down this client object, releasing any resources that might be held open.

Usage

From source file:com.thinkbiganalytics.kylo.catalog.aws.S3FileSystemProvider.java

License:Apache License

@Nonnull
@Override//from  w  w  w. j  ava 2 s .co m
public List<DataSetFile> listFiles(@Nonnull final Path path, @Nonnull final Configuration conf) {
    // Determine the credentials
    final AmazonS3 s3;
    final URI uri = path.toUri();

    if ("s3".equalsIgnoreCase(uri.getScheme()) || "s3bfs".equalsIgnoreCase(uri.getScheme())
            || "s3n".equalsIgnoreCase(uri.getScheme())) {
        s3 = createS3Client(uri, conf);
    } else if ("s3a".equalsIgnoreCase(uri.getScheme())) {
        final Class<? extends S3ClientFactory> s3ClientFactoryClass = conf.getClass(
                Constants.S3_CLIENT_FACTORY_IMPL, Constants.DEFAULT_S3_CLIENT_FACTORY_IMPL,
                S3ClientFactory.class);
        try {
            s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf).createS3Client(uri);
        } catch (final IOException e) {
            throw new IllegalArgumentException("Unable to create S3 client: " + e, e);
        }
    } else {
        log.debug("Scheme {} not supported for S3 path: {}", uri.getScheme(), path);
        throw new CatalogException("catalog.fs.s3.invalidScheme", uri.getScheme());
    }

    // Fetch the list of buckets
    try {
        return s3.listBuckets().stream().map(bucket -> {
            final DataSetFile file = new DataSetFile();
            file.setName(bucket.getName());
            file.setDirectory(true);
            file.setModificationTime(bucket.getCreationDate().getTime());
            file.setPath(uri.getScheme() + "://" + bucket.getName() + "/");
            return file;
        }).collect(Collectors.toList());
    } finally {
        s3.shutdown();
    }
}

---

From source file:org.apache.flink.streaming.tests.util.s3.S3UtilProgram.java

License:Apache License

private static void numberOfLinesInFile(ParameterTool params) {
    final String bucket = params.getRequired("bucket");
    final String s3file = params.getRequired("s3file");
    AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient();
    System.out.print(S3QueryUtil.queryFile(s3client, bucket, s3file, countQuery));
    s3client.shutdown();
}

---

From source file:org.apache.flink.streaming.tests.util.s3.S3UtilProgram.java

License:Apache License

private static void numberOfLinesInFilesWithFullAndNamePrefix(ParameterTool params) {
    final String bucket = params.getRequired("bucket");
    final String s3prefix = params.getRequired("s3prefix");
    final String s3filePrefix = params.get("s3filePrefix", "");
    int parallelism = params.getInt("parallelism", 10);

    List<String> files = listByFullPathPrefix(bucket, s3prefix);

    ExecutorService executor = Executors.newFixedThreadPool(parallelism);
    AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient();
    List<CompletableFuture<Integer>> requests = submitLineCountingRequestsForFilesAsync(executor, s3client,
            bucket, files, s3filePrefix);
    int count = waitAndComputeTotalLineCountResult(requests);

    executor.shutdownNow();//  w  ww.ja  v  a 2  s.co m
    s3client.shutdown();
    System.out.print(count);
}

---