Example usage for com.amazonaws.services.s3.model GetObjectRequest GetObjectRequest

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model GetObjectRequest GetObjectRequest.

Prototype

public GetObjectRequest(String bucketName, String key) 

Source Link

Document

Constructs a new GetObjectRequest with all the required parameters.

Usage

From source file:org.apache.nifi.processors.aws.s3.FetchS3Object.java

License:Apache License

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();//from   w  w  w . j av a 2s  .co  m
    if (flowFile == null) {
        return;
    }

    final long startNanos = System.nanoTime();
    final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue();
    final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue();
    final String versionId = context.getProperty(VERSION_ID).evaluateAttributeExpressions(flowFile).getValue();

    final AmazonS3 client = getClient();
    final GetObjectRequest request;
    if (versionId == null) {
        request = new GetObjectRequest(bucket, key);
    } else {
        request = new GetObjectRequest(bucket, key, versionId);
    }

    final Map<String, String> attributes = new HashMap<>();
    try (final S3Object s3Object = client.getObject(request)) {
        flowFile = session.importFrom(s3Object.getObjectContent(), flowFile);
        attributes.put("s3.bucket", s3Object.getBucketName());

        final ObjectMetadata metadata = s3Object.getObjectMetadata();
        if (metadata.getContentDisposition() != null) {
            final String fullyQualified = metadata.getContentDisposition();
            final int lastSlash = fullyQualified.lastIndexOf("/");
            if (lastSlash > -1 && lastSlash < fullyQualified.length() - 1) {
                attributes.put(CoreAttributes.PATH.key(), fullyQualified.substring(0, lastSlash));
                attributes.put(CoreAttributes.ABSOLUTE_PATH.key(), fullyQualified);
                attributes.put(CoreAttributes.FILENAME.key(), fullyQualified.substring(lastSlash + 1));
            } else {
                attributes.put(CoreAttributes.FILENAME.key(), metadata.getContentDisposition());
            }
        }
        if (metadata.getContentMD5() != null) {
            attributes.put("hash.value", metadata.getContentMD5());
            attributes.put("hash.algorithm", "MD5");
        }
        if (metadata.getContentType() != null) {
            attributes.put(CoreAttributes.MIME_TYPE.key(), metadata.getContentType());
        }
        if (metadata.getETag() != null) {
            attributes.put("s3.etag", metadata.getETag());
        }
        if (metadata.getExpirationTime() != null) {
            attributes.put("s3.expirationTime", String.valueOf(metadata.getExpirationTime().getTime()));
        }
        if (metadata.getExpirationTimeRuleId() != null) {
            attributes.put("s3.expirationTimeRuleId", metadata.getExpirationTimeRuleId());
        }
        if (metadata.getUserMetadata() != null) {
            attributes.putAll(metadata.getUserMetadata());
        }
        if (metadata.getSSEAlgorithm() != null) {
            attributes.put("s3.sseAlgorithm", metadata.getSSEAlgorithm());
        }
        if (metadata.getVersionId() != null) {
            attributes.put("s3.version", metadata.getVersionId());
        }
    } catch (final IOException | AmazonClientException ioe) {
        getLogger().error("Failed to retrieve S3 Object for {}; routing to failure",
                new Object[] { flowFile, ioe });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }

    if (!attributes.isEmpty()) {
        flowFile = session.putAllAttributes(flowFile, attributes);
    }

    session.transfer(flowFile, REL_SUCCESS);
    final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    getLogger().info("Successfully retrieved S3 Object for {} in {} millis; routing to success",
            new Object[] { flowFile, transferMillis });
    session.getProvenanceReporter().fetch(flowFile, "http://" + bucket + ".amazonaws.com/" + key,
            transferMillis);
}

---

From source file:org.apache.oodt.cas.filemgr.datatransfer.S3DataTransferer.java

License:Apache License

@Override
public void retrieveProduct(Product product, File directory) throws DataTransferException, IOException {
    for (Reference ref : product.getProductReferences()) {
        GetObjectRequest request = new GetObjectRequest(bucketName,
                stripProtocol(ref.getDataStoreReference(), true));
        S3Object file = s3Client.getObject(request);
        stageFile(file, ref, directory);
    }/*from   w  w w  .  j a  v  a2 s  . c o  m*/
}

---

From source file:org.apache.usergrid.apm.service.CrashUtil.java

License:Apache License

public static String getCrashSummary(String fullAppName, String s3CrashFileName) {

    DeploymentConfig config = DeploymentConfig.geDeploymentConfig();
    AWSCredentials credentials = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey());
    AmazonS3Client s3Client = new AmazonS3Client(credentials);
    String s3FullFileName = AWSUtil.formS3CrashFileUrl(fullAppName, s3CrashFileName);

    log.info("Crash file bucket " + config.getS3LogBucket() + " and file name : " + s3FullFileName);

    GetObjectRequest objectRequest = new GetObjectRequest(config.getS3LogBucket(), s3FullFileName);
    String crashSummary = null;/*from   w w w  . j a v a 2  s.c o  m*/
    try {
        S3Object s3Object = s3Client.getObject(objectRequest);
        InputStream is = s3Object.getObjectContent();
        String fileContents = IOUtils.toString(is);

        CrashLogParser parser = null;
        if (fileContents != null) {
            if (s3CrashFileName.endsWith(".crash")) {
                parser = new iOSCrashLogParser();
                if (parser.parseCrashLog(fileContents))
                    crashSummary = parser.getCrashSummary();
                else {
                    log.error("problem parsing ios crash file for app " + fullAppName + " file: "
                            + s3CrashFileName);
                    crashSummary = "Not able to get summary for iOS crash log";
                }
            } else if (s3CrashFileName.endsWith(".stacktrace")) {
                parser = new AndroidCrashLogParser();
                if (parser.parseCrashLog(fileContents))
                    crashSummary = parser.getCrashSummary();
                else {
                    log.error("problem parsing Android crash file for app " + fullAppName + " file: "
                            + s3CrashFileName);
                    crashSummary = "Not able to get summary for Android crash log";
                }
            }
        }
    } catch (AmazonServiceException e1) {
        e1.printStackTrace();
        log.error("Promblem downloading crash file from S3 for " + s3FullFileName, e1);
    } catch (Exception e) {
        e.printStackTrace();
        log.error("Promblem downloading crash file from S3 for S3 for " + s3FullFileName, e);
    }

    log.info("Crash summary " + crashSummary);
    if (crashSummary != null && crashSummary.length() > 250) {
        crashSummary = crashSummary.substring(0, 249);
    }
    return crashSummary;
}

---

From source file:org.apache.zeppelin.notebook.repo.OldS3NotebookRepo.java

License:Apache License

private Note getNote(String key) throws IOException {
    S3Object s3object;// w  w  w.  ja v  a2s  .  co  m
    try {
        s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to retrieve object from S3: " + ace, ace);
    }

    try (InputStream ins = s3object.getObjectContent()) {
        String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
        return Note.fromJson(json);
    }
}

---

From source file:org.apache.zeppelin.notebook.repo.S3NotebookRepo.java

License:Apache License

@Override
public Note get(String noteId, String notePath, AuthenticationInfo subject) throws IOException {
    S3Object s3object;/*from www  .  j  av a  2  s.  co m*/
    try {
        s3object = s3client.getObject(
                new GetObjectRequest(bucketName, rootFolder + "/" + buildNoteFileName(noteId, notePath)));
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to retrieve object from S3: " + ace, ace);
    }
    try (InputStream ins = s3object.getObjectContent()) {
        String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
        return Note.fromJson(json);
    }
}

---

From source file:org.applicationMigrator.migrationclient.FileTransferClient.java

License:Apache License

public void downloadFiles(String outputFilesPaths[])
        throws IOException, InterruptedException, ClassNotFoundException {
    if (outputFilesPaths == null) {
        return;//  www.  j  a v  a 2s  . c  om
    }
    AWSCredentials awsCredentials = currentUser.getAwsCredentials();
    Environment.getExternalStorageDirectory().setReadable(true, false);
    Environment.getExternalStorageDirectory().setWritable(true, false);

    for (String outputFilePath : outputFilesPaths) {
        String fileName = getFileName(outputFilePath);
        String userName = currentUser.getUserNameString();
        String keyNameString = userName + "/" + applicationName + "/" + fileName;
        AmazonS3 s3client = new AmazonS3Client(awsCredentials);
        S3Object s3Object = s3client.getObject(new GetObjectRequest(BUCKET_NAME, keyNameString));
        try {
            writeObjectToFile(s3Object, outputFilePath);
            File outputFile = new File(outputFilePath);
            outputFile.setReadable(true, false);
            outputFile.setWritable(true, false);

        } catch (Exception ignored) {
        }
    }
}

---

From source file:org.applicationMigrator.serverAgent.ServerAgentFileTransferClient.java

License:Apache License

public void downloadFiles(String outputFilesPaths[], String credentialsFilePath)
        throws IOException, InterruptedException {
    if (outputFilesPaths == null) {
        return;/*from ww  w .j av  a2  s.co m*/
    }
    AWSCredentials awsCredentials = getCredentials(credentialsFilePath);
    Environment.getExternalStorageDirectory().setReadable(true, false);
    Environment.getExternalStorageDirectory().setWritable(true, false);

    for (String outputFilePath : outputFilesPaths) {
        String fileName = getFileName(outputFilePath);
        String keyNameString = userName + "/" + applicationName + "/" + fileName;
        AmazonS3 s3client = new AmazonS3Client(awsCredentials);
        S3Object s3Object = s3client.getObject(new GetObjectRequest(BUCKET_NAME, keyNameString));
        try {
            writeObjectToFile(s3Object, outputFilePath);
            File outputFile = new File(outputFilePath);
            outputFile.setReadable(true, false);
            outputFile.setWritable(true, false);

        } catch (Exception e) {
            /*File erroneousFile = new File(outputFilePath);
            if (erroneousFile.exists())
               erroneousFile.delete();*/
        }
    }
}

---

From source file:org.boriken.s3fileuploader.S3SampleRefactored.java

License:Open Source License

public static void downloadFile(AmazonS3 s3, String bucketName, String key) throws IOException {
    /*// w w  w .j ava2 s .  c  o  m
     * Download an object - When you download an object, you get all of
     * the object's metadata and a stream from which to read the contents.
     * It's important to read the contents of the stream as quickly as
     * possibly since the data is streamed directly from Amazon S3 and your
     * network connection will remain open until you read all the data or
     * close the input stream.
     *
     * GetObjectRequest also supports several other options, including
     * conditional downloading of objects based on modification times,
     * ETags, and selectively downloading a range of an object.
     */
    System.out.println("Downloading an object");
    S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
    System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
    displayTextInputStream(object.getObjectContent());
}

---

From source file:org.broadleafcommerce.vendor.amazon.s3.S3FileServiceProvider.java

License:Apache License

@Override
public File getResource(String name, FileApplicationType fileApplicationType) {
    final S3Configuration s3config = s3ConfigurationService.lookupS3Configuration();
    final String resourceName = buildResourceName(s3config, name);
    final File returnFile = blFileService.getLocalResource(resourceName);
    final String s3Uri = String.format("s3://%s/%s", s3config.getDefaultBucketName(), resourceName);

    OutputStream outputStream = null;
    InputStream inputStream = null;

    try {/*  w  ww. j  a va 2s . c o  m*/
        final AmazonS3Client s3 = getAmazonS3Client(s3config);
        final S3Object object = s3.getObject(
                new GetObjectRequest(s3config.getDefaultBucketName(), buildResourceName(s3config, name)));

        if (LOG.isTraceEnabled()) {
            LOG.trace("retrieving " + s3Uri);
        }
        inputStream = object.getObjectContent();

        if (!returnFile.getParentFile().exists()) {
            if (!returnFile.getParentFile().mkdirs()) {
                // Other thread could have created - check one more time.
                if (!returnFile.getParentFile().exists()) {
                    throw new RuntimeException("Unable to create parent directories for file: " + name);
                }
            }
        }
        outputStream = new FileOutputStream(returnFile);
        int read = 0;
        byte[] bytes = new byte[1024];

        while ((read = inputStream.read(bytes)) != -1) {
            outputStream.write(bytes, 0, read);
        }
    } catch (IOException ioe) {
        throw new RuntimeException(String.format("Error writing %s to local file system at %s", s3Uri,
                returnFile.getAbsolutePath()), ioe);
    } catch (AmazonS3Exception s3Exception) {
        LOG.error(String.format("%s for %s; name = %s, resourceName = %s, returnFile = %s",
                s3Exception.getErrorCode(), s3Uri, name, resourceName, returnFile.getAbsolutePath()));

        if ("NoSuchKey".equals(s3Exception.getErrorCode())) {
            //return new File("this/path/should/not/exist/" + UUID.randomUUID());
            return null;
        } else {
            throw s3Exception;
        }
    } finally {
        if (inputStream != null) {
            try {
                inputStream.close();
            } catch (IOException e) {
                throw new RuntimeException("Error closing input stream while writing s3 file to file system",
                        e);
            }
        }
        if (outputStream != null) {
            try {
                outputStream.close();
            } catch (IOException e) {
                throw new RuntimeException("Error closing output stream while writing s3 file to file system",
                        e);
            }

        }
    }
    return returnFile;
}

---

From source file:org.caboclo.clients.AmazonClient.java

License:Open Source License

@Override
public MultiPartDownload initializeDownload(File file, RemoteFile remoteFile) throws IOException {

    long start = 0;
    long end = Constants.CHUNK_DOWNLOAD_SIZE - 1;
    String child = remoteFile.getPath();

    GetObjectRequest rangeObjectRequest = new GetObjectRequest(getBucketName(), child);

    rangeObjectRequest.setRange(start, end);

    S3Object obj = s3.getObject(rangeObjectRequest);

    BufferedInputStream bis = new BufferedInputStream(obj.getObjectContent());

    writeStreamToFile(bis, file);//  ww  w.j  a  v  a2  s  . c om

    MultiPartDownload mpd = new MultiPartDownload(file, remoteFile, end);

    return mpd;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• »