List of usage examples for com.amazonaws.services.s3.model ObjectTagging ObjectTagging
public ObjectTagging(List<Tag> tagSet)
From source file:com.streamsets.pipeline.stage.executor.s3.AmazonS3Executor.java
License:Apache License
private void changeExistingObject(Record record, ELVars variables, String bucket, String objectPath) throws OnRecordErrorException { // Tag application if (!config.taskConfig.tags.isEmpty()) { List<Tag> newTags = new ArrayList<>(); // Evaluate each tag separately for (Map.Entry<String, String> entry : config.taskConfig.tags.entrySet()) { newTags.add(new Tag(evaluate(record, "tags", variables, entry.getKey()), evaluate(record, "tags", variables, entry.getValue()))); }//w w w. j av a 2 s .c o m // Apply all tags at once config.s3Config.getS3Client() .setObjectTagging(new SetObjectTaggingRequest(bucket, objectPath, new ObjectTagging(newTags))); Events.FILE_CHANGED.create(getContext()).with("object_key", objectPath).createAndSend(); } }
From source file:com.universal.storage.UniversalS3Storage.java
License:Open Source License
/** * This method uploads a file with a length greater than PART_SIZE (5Mb). * /* w w w . j a v a 2s . co m*/ * @param file to be stored within the storage. * @param path is the path for this new file within the root. * @throws UniversalIOException when a specific IO error occurs. */ private void uploadFile(File file, String path) throws UniversalIOException { // Create a list of UploadPartResponse objects. You get one of these // for each part upload. List<PartETag> partETags = new ArrayList<PartETag>(); // Step 1: Initialize. InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(this.settings.getRoot(), file.getName()); InitiateMultipartUploadResult initResponse = this.s3client.initiateMultipartUpload(initRequest); long contentLength = file.length(); long partSize = PART_SIZE; // Set part size to 5 MB. ObjectMetadata objectMetadata = new ObjectMetadata(); if (this.settings.getEncryption()) { objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); } List<Tag> tags = new ArrayList<Tag>(); for (String key : this.settings.getTags().keySet()) { tags.add(new Tag(key, this.settings.getTags().get(key))); } try { this.triggerOnStoreFileListeners(); // Step 2: Upload parts. long filePosition = 0; for (int i = 1; filePosition < contentLength; i++) { // Last part can be less than 5 MB. Adjust part size. partSize = Math.min(partSize, (contentLength - filePosition)); // Create request to upload a part. UploadPartRequest uploadRequest = new UploadPartRequest() .withBucketName(this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path))) .withKey(file.getName()).withUploadId(initResponse.getUploadId()).withPartNumber(i) .withFileOffset(filePosition).withFile(file).withObjectMetadata(objectMetadata) .withPartSize(partSize); // Upload part and add response to our list. partETags.add(this.s3client.uploadPart(uploadRequest).getPartETag()); filePosition += partSize; } // Step 3: Complete. CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest( this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path)), file.getName(), initResponse.getUploadId(), partETags); CompleteMultipartUploadResult result = this.s3client.completeMultipartUpload(compRequest); StorageClass storageClass = getStorageClass(); if (storageClass != StorageClass.Standard) { CopyObjectRequest copyObjectRequest = new CopyObjectRequest(this.settings.getRoot(), file.getName(), this.settings.getRoot(), file.getName()).withStorageClass(storageClass); this.s3client.copyObject(copyObjectRequest); } if (!tags.isEmpty()) { this.s3client.setObjectTagging(new SetObjectTaggingRequest(this.settings.getRoot(), file.getName(), new ObjectTagging(tags))); } this.triggerOnFileStoredListeners(new UniversalStorageData(file.getName(), PREFIX_S3_URL + (this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path))) + "/" + file.getName(), result.getVersionId(), this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path)))); } catch (Exception e) { this.s3client.abortMultipartUpload(new AbortMultipartUploadRequest(this.settings.getRoot(), file.getName(), initResponse.getUploadId())); UniversalIOException error = new UniversalIOException(e.getMessage()); this.triggerOnErrorListeners(error); throw error; } }
From source file:com.universal.storage.UniversalS3Storage.java
License:Open Source License
/** * This method uploads a file with a length lesser than PART_SIZE (5Mb). * /*from w w w . j av a 2 s . c o m*/ * @param file to be stored within the storage. * @param path is the path for this new file within the root. * @throws UniversalIOException when a specific IO error occurs. */ private void uploadTinyFile(File file, String path) throws UniversalIOException { try { ObjectMetadata objectMetadata = new ObjectMetadata(); if (this.settings.getEncryption()) { objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); } List<Tag> tags = new ArrayList<Tag>(); for (String key : this.settings.getTags().keySet()) { tags.add(new Tag(key, this.settings.getTags().get(key))); } PutObjectRequest request = new PutObjectRequest( this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path)), file.getName(), file); request.setMetadata(objectMetadata); request.setTagging(new ObjectTagging(tags)); request.setStorageClass(getStorageClass()); this.triggerOnStoreFileListeners(); PutObjectResult result = this.s3client.putObject(request); this.triggerOnFileStoredListeners(new UniversalStorageData(file.getName(), PREFIX_S3_URL + (this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path))) + "/" + file.getName(), result.getVersionId(), this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path)))); } catch (Exception e) { UniversalIOException error = new UniversalIOException(e.getMessage()); this.triggerOnErrorListeners(error); throw error; } }
From source file:org.apache.nifi.processors.aws.s3.TagS3Object.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) { FlowFile flowFile = session.get();//www .ja va 2 s . c o m if (flowFile == null) { return; } final long startNanos = System.nanoTime(); final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue(); final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue(); final String newTagKey = context.getProperty(TAG_KEY).evaluateAttributeExpressions(flowFile).getValue(); final String newTagVal = context.getProperty(TAG_VALUE).evaluateAttributeExpressions(flowFile).getValue(); if (StringUtils.isBlank(bucket)) { failFlowWithBlankEvaluatedProperty(session, flowFile, BUCKET); return; } if (StringUtils.isBlank(key)) { failFlowWithBlankEvaluatedProperty(session, flowFile, KEY); return; } if (StringUtils.isBlank(newTagKey)) { failFlowWithBlankEvaluatedProperty(session, flowFile, TAG_KEY); return; } if (StringUtils.isBlank(newTagVal)) { failFlowWithBlankEvaluatedProperty(session, flowFile, TAG_VALUE); return; } final String version = context.getProperty(VERSION_ID).evaluateAttributeExpressions(flowFile).getValue(); final AmazonS3 s3 = getClient(); SetObjectTaggingRequest r; List<Tag> tags = new ArrayList<>(); try { if (context.getProperty(APPEND_TAG).asBoolean()) { final GetObjectTaggingRequest gr = new GetObjectTaggingRequest(bucket, key); GetObjectTaggingResult res = s3.getObjectTagging(gr); // preserve tags on S3 object, but filter out existing tag keys that match the one we're setting tags = res.getTagSet().stream().filter(t -> !t.getKey().equals(newTagKey)) .collect(Collectors.toList()); } tags.add(new Tag(newTagKey, newTagVal)); if (StringUtils.isBlank(version)) { r = new SetObjectTaggingRequest(bucket, key, new ObjectTagging(tags)); } else { r = new SetObjectTaggingRequest(bucket, key, version, new ObjectTagging(tags)); } s3.setObjectTagging(r); } catch (final AmazonServiceException ase) { getLogger().error("Failed to tag S3 Object for {}; routing to failure", new Object[] { flowFile, ase }); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); return; } flowFile = setTagAttributes(session, flowFile, tags); session.transfer(flowFile, REL_SUCCESS); final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); getLogger().info("Successfully tagged S3 Object for {} in {} millis; routing to success", new Object[] { flowFile, transferMillis }); }
From source file:org.finra.herd.dao.impl.S3DaoImpl.java
License:Apache License
private void tagVersionsHelper(final S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto, final S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto, final List<S3VersionSummary> s3VersionSummaries, final Tag tag) { // Initialize an S3 version for the error message in the catch block. S3VersionSummary currentS3VersionSummary = s3VersionSummaries.get(0); // Amazon S3 client to access S3 objects. AmazonS3Client s3Client = null;/* w w w. j a v a 2 s .com*/ // Amazon S3 client for S3 object tagging. AmazonS3Client s3ObjectTaggerClient = null; try { // Create an S3 client to access S3 objects. s3Client = getAmazonS3(s3FileTransferRequestParamsDto); // Create an S3 client for S3 object tagging. s3ObjectTaggerClient = getAmazonS3(s3ObjectTaggerParamsDto); // Create a get object tagging request. GetObjectTaggingRequest getObjectTaggingRequest = new GetObjectTaggingRequest( s3FileTransferRequestParamsDto.getS3BucketName(), null, null); // Create a set object tagging request. SetObjectTaggingRequest setObjectTaggingRequest = new SetObjectTaggingRequest( s3FileTransferRequestParamsDto.getS3BucketName(), null, null, null); for (S3VersionSummary s3VersionSummary : s3VersionSummaries) { // Set the current S3 version summary. currentS3VersionSummary = s3VersionSummary; // Retrieve the current tagging information for the S3 version. getObjectTaggingRequest.setKey(s3VersionSummary.getKey()); getObjectTaggingRequest.setVersionId(s3VersionSummary.getVersionId()); GetObjectTaggingResult getObjectTaggingResult = s3Operations .getObjectTagging(getObjectTaggingRequest, s3Client); // Update the list of tags to include the specified S3 object tag. List<Tag> updatedTags = new ArrayList<>(); updatedTags.add(tag); if (CollectionUtils.isNotEmpty(getObjectTaggingResult.getTagSet())) { for (Tag currentTag : getObjectTaggingResult.getTagSet()) { if (!StringUtils.equals(tag.getKey(), currentTag.getKey())) { updatedTags.add(currentTag); } } } // Update tagging information for the S3 version. setObjectTaggingRequest.setKey(s3VersionSummary.getKey()); setObjectTaggingRequest.setVersionId(s3VersionSummary.getVersionId()); setObjectTaggingRequest.setTagging(new ObjectTagging(updatedTags)); s3Operations.setObjectTagging(setObjectTaggingRequest, s3ObjectTaggerClient); } } catch (Exception e) { throw new IllegalStateException(String.format( "Failed to tag S3 object with \"%s\" key and \"%s\" version id in \"%s\" bucket. Reason: %s", currentS3VersionSummary.getKey(), currentS3VersionSummary.getVersionId(), s3FileTransferRequestParamsDto.getS3BucketName(), e.getMessage()), e); } finally { if (s3Client != null) { s3Client.shutdown(); } if (s3ObjectTaggerClient != null) { s3ObjectTaggerClient.shutdown(); } } }