Example usage for com.amazonaws.services.s3.model Tag Tag

List of usage examples for com.amazonaws.services.s3.model Tag Tag

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. Tag
  6. Tag

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model Tag Tag.

Prototype

public Tag(String key, String value)

Source Link

Document

Constructs an instance of this object.

Usage

From source file:com.streamsets.pipeline.stage.executor.s3.AmazonS3Executor.java

License:Apache License

private void changeExistingObject(Record record, ELVars variables, String bucket, String objectPath)
        throws OnRecordErrorException {
    // Tag application
    if (!config.taskConfig.tags.isEmpty()) {
        List<Tag> newTags = new ArrayList<>();

        // Evaluate each tag separately
        for (Map.Entry<String, String> entry : config.taskConfig.tags.entrySet()) {
            newTags.add(new Tag(evaluate(record, "tags", variables, entry.getKey()),
                    evaluate(record, "tags", variables, entry.getValue())));
        }//from ww  w  .j av a2  s  .  c om

        // Apply all tags at once
        config.s3Config.getS3Client()
                .setObjectTagging(new SetObjectTaggingRequest(bucket, objectPath, new ObjectTagging(newTags)));

        Events.FILE_CHANGED.create(getContext()).with("object_key", objectPath).createAndSend();
    }
}

From source file:com.universal.storage.UniversalS3Storage.java

License:Open Source License

/**
 * This method uploads a file with a length greater than PART_SIZE (5Mb).
 * /*  w  w  w . java  2  s  .  co  m*/
 * @param file to be stored within the storage.
 * @param path is the path for this new file within the root.
 * @throws UniversalIOException when a specific IO error occurs.
 */
private void uploadFile(File file, String path) throws UniversalIOException {
    // Create a list of UploadPartResponse objects. You get one of these
    // for each part upload.
    List<PartETag> partETags = new ArrayList<PartETag>();

    // Step 1: Initialize.
    InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(this.settings.getRoot(),
            file.getName());
    InitiateMultipartUploadResult initResponse = this.s3client.initiateMultipartUpload(initRequest);

    long contentLength = file.length();
    long partSize = PART_SIZE; // Set part size to 5 MB.

    ObjectMetadata objectMetadata = new ObjectMetadata();
    if (this.settings.getEncryption()) {
        objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
    }

    List<Tag> tags = new ArrayList<Tag>();
    for (String key : this.settings.getTags().keySet()) {
        tags.add(new Tag(key, this.settings.getTags().get(key)));
    }

    try {
        this.triggerOnStoreFileListeners();
        // Step 2: Upload parts.
        long filePosition = 0;
        for (int i = 1; filePosition < contentLength; i++) {
            // Last part can be less than 5 MB. Adjust part size.
            partSize = Math.min(partSize, (contentLength - filePosition));

            // Create request to upload a part.
            UploadPartRequest uploadRequest = new UploadPartRequest()
                    .withBucketName(this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path)))
                    .withKey(file.getName()).withUploadId(initResponse.getUploadId()).withPartNumber(i)
                    .withFileOffset(filePosition).withFile(file).withObjectMetadata(objectMetadata)
                    .withPartSize(partSize);

            // Upload part and add response to our list.
            partETags.add(this.s3client.uploadPart(uploadRequest).getPartETag());

            filePosition += partSize;
        }

        // Step 3: Complete.
        CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(
                this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path)), file.getName(),
                initResponse.getUploadId(), partETags);

        CompleteMultipartUploadResult result = this.s3client.completeMultipartUpload(compRequest);

        StorageClass storageClass = getStorageClass();
        if (storageClass != StorageClass.Standard) {
            CopyObjectRequest copyObjectRequest = new CopyObjectRequest(this.settings.getRoot(), file.getName(),
                    this.settings.getRoot(), file.getName()).withStorageClass(storageClass);

            this.s3client.copyObject(copyObjectRequest);
        }

        if (!tags.isEmpty()) {
            this.s3client.setObjectTagging(new SetObjectTaggingRequest(this.settings.getRoot(), file.getName(),
                    new ObjectTagging(tags)));
        }

        this.triggerOnFileStoredListeners(new UniversalStorageData(file.getName(),
                PREFIX_S3_URL + (this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path))) + "/"
                        + file.getName(),
                result.getVersionId(), this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path))));
    } catch (Exception e) {
        this.s3client.abortMultipartUpload(new AbortMultipartUploadRequest(this.settings.getRoot(),
                file.getName(), initResponse.getUploadId()));

        UniversalIOException error = new UniversalIOException(e.getMessage());
        this.triggerOnErrorListeners(error);
        throw error;
    }
}

From source file:com.universal.storage.UniversalS3Storage.java

License:Open Source License

/**
 * This method uploads a file with a length lesser than PART_SIZE (5Mb).
 * //w  ww.ja v a2  s  .  c o  m
 * @param file to be stored within the storage.
 * @param path is the path for this new file within the root.
 * @throws UniversalIOException when a specific IO error occurs.
 */
private void uploadTinyFile(File file, String path) throws UniversalIOException {
    try {
        ObjectMetadata objectMetadata = new ObjectMetadata();
        if (this.settings.getEncryption()) {
            objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
        }

        List<Tag> tags = new ArrayList<Tag>();
        for (String key : this.settings.getTags().keySet()) {
            tags.add(new Tag(key, this.settings.getTags().get(key)));
        }

        PutObjectRequest request = new PutObjectRequest(
                this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path)), file.getName(), file);
        request.setMetadata(objectMetadata);
        request.setTagging(new ObjectTagging(tags));
        request.setStorageClass(getStorageClass());
        this.triggerOnStoreFileListeners();

        PutObjectResult result = this.s3client.putObject(request);

        this.triggerOnFileStoredListeners(new UniversalStorageData(file.getName(),
                PREFIX_S3_URL + (this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path))) + "/"
                        + file.getName(),
                result.getVersionId(), this.settings.getRoot() + ("".equals(path) ? "" : ("/" + path))));
    } catch (Exception e) {
        UniversalIOException error = new UniversalIOException(e.getMessage());
        this.triggerOnErrorListeners(error);
        throw error;
    }
}

From source file:org.apache.nifi.processors.aws.s3.TagS3Object.java

License:Apache License

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();//www  . ja va 2 s .  c o  m
    if (flowFile == null) {
        return;
    }

    final long startNanos = System.nanoTime();

    final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue();
    final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue();
    final String newTagKey = context.getProperty(TAG_KEY).evaluateAttributeExpressions(flowFile).getValue();
    final String newTagVal = context.getProperty(TAG_VALUE).evaluateAttributeExpressions(flowFile).getValue();

    if (StringUtils.isBlank(bucket)) {
        failFlowWithBlankEvaluatedProperty(session, flowFile, BUCKET);
        return;
    }

    if (StringUtils.isBlank(key)) {
        failFlowWithBlankEvaluatedProperty(session, flowFile, KEY);
        return;
    }

    if (StringUtils.isBlank(newTagKey)) {
        failFlowWithBlankEvaluatedProperty(session, flowFile, TAG_KEY);
        return;
    }

    if (StringUtils.isBlank(newTagVal)) {
        failFlowWithBlankEvaluatedProperty(session, flowFile, TAG_VALUE);
        return;
    }

    final String version = context.getProperty(VERSION_ID).evaluateAttributeExpressions(flowFile).getValue();

    final AmazonS3 s3 = getClient();

    SetObjectTaggingRequest r;
    List<Tag> tags = new ArrayList<>();

    try {
        if (context.getProperty(APPEND_TAG).asBoolean()) {
            final GetObjectTaggingRequest gr = new GetObjectTaggingRequest(bucket, key);
            GetObjectTaggingResult res = s3.getObjectTagging(gr);

            // preserve tags on S3 object, but filter out existing tag keys that match the one we're setting
            tags = res.getTagSet().stream().filter(t -> !t.getKey().equals(newTagKey))
                    .collect(Collectors.toList());
        }

        tags.add(new Tag(newTagKey, newTagVal));

        if (StringUtils.isBlank(version)) {
            r = new SetObjectTaggingRequest(bucket, key, new ObjectTagging(tags));
        } else {
            r = new SetObjectTaggingRequest(bucket, key, version, new ObjectTagging(tags));
        }
        s3.setObjectTagging(r);
    } catch (final AmazonServiceException ase) {
        getLogger().error("Failed to tag S3 Object for {}; routing to failure", new Object[] { flowFile, ase });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }

    flowFile = setTagAttributes(session, flowFile, tags);

    session.transfer(flowFile, REL_SUCCESS);
    final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    getLogger().info("Successfully tagged S3 Object for {} in {} millis; routing to success",
            new Object[] { flowFile, transferMillis });
}

From source file:org.finra.herd.service.impl.BusinessObjectDataInitiateDestroyHelperServiceImpl.java

License:Apache License

/**
 * Executes S3 specific steps required for initiation of a business object data destroy.
 *
 * @param businessObjectDataDestroyDto the DTO that holds various parameters needed to initiate a business object data destroy
 *///from  www .j  a  v a 2 s .  co  m
void executeS3SpecificStepsImpl(BusinessObjectDataDestroyDto businessObjectDataDestroyDto) {
    // Create an S3 file transfer parameters DTO to access the S3 bucket.
    // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = storageHelper
            .getS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3Endpoint(businessObjectDataDestroyDto.getS3Endpoint());
    s3FileTransferRequestParamsDto.setS3BucketName(businessObjectDataDestroyDto.getS3BucketName());
    s3FileTransferRequestParamsDto
            .setS3KeyPrefix(StringUtils.appendIfMissing(businessObjectDataDestroyDto.getS3KeyPrefix(), "/"));

    // Create an S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = storageHelper
            .getS3FileTransferRequestParamsDtoByRole(businessObjectDataDestroyDto.getS3ObjectTaggerRoleArn(),
                    businessObjectDataDestroyDto.getS3ObjectTaggerRoleSessionName());
    s3ObjectTaggerParamsDto.setS3Endpoint(businessObjectDataDestroyDto.getS3Endpoint());

    // Get all S3 objects matching the S3 key prefix from the S3 bucket.
    List<S3VersionSummary> s3VersionSummaries = s3Service.listVersions(s3FileTransferRequestParamsDto);

    // Tag the S3 objects to initiate the deletion.
    s3Service.tagVersions(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, s3VersionSummaries,
            new Tag(businessObjectDataDestroyDto.getS3ObjectTagKey(),
                    businessObjectDataDestroyDto.getS3ObjectTagValue()));
}

From source file:org.finra.herd.service.impl.StoragePolicyProcessorHelperServiceImpl.java

License:Apache License

/**
 * Executes a storage policy transition as per specified storage policy selection.
 *
 * @param storagePolicyTransitionParamsDto the storage policy transition DTO that contains parameters needed to perform a storage policy transition
 *///from w  w  w  .  j  a  v  a  2  s .c  o  m
protected void executeStoragePolicyTransitionImpl(
        StoragePolicyTransitionParamsDto storagePolicyTransitionParamsDto) {
    // Create an S3 file transfer parameters DTO to access the S3 bucket.
    // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = storageHelper
            .getS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3Endpoint(storagePolicyTransitionParamsDto.getS3Endpoint());
    s3FileTransferRequestParamsDto.setS3BucketName(storagePolicyTransitionParamsDto.getS3BucketName());
    s3FileTransferRequestParamsDto.setS3KeyPrefix(
            StringUtils.appendIfMissing(storagePolicyTransitionParamsDto.getS3KeyPrefix(), "/"));

    // Create an S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = storageHelper
            .getS3FileTransferRequestParamsDtoByRole(
                    storagePolicyTransitionParamsDto.getS3ObjectTaggerRoleArn(),
                    storagePolicyTransitionParamsDto.getS3ObjectTaggerRoleSessionName());
    s3ObjectTaggerParamsDto.setS3Endpoint(storagePolicyTransitionParamsDto.getS3Endpoint());

    // Get actual S3 files by selecting all S3 keys matching the S3 key prefix form the S3 bucket.
    // When listing S3 files, we ignore 0 byte objects that represent S3 directories.
    List<S3ObjectSummary> actualS3FilesWithoutZeroByteDirectoryMarkers = s3Service
            .listDirectory(s3FileTransferRequestParamsDto, true);

    // Validate existence of the S3 files.
    storageFileHelper.validateRegisteredS3Files(storagePolicyTransitionParamsDto.getStorageFiles(),
            actualS3FilesWithoutZeroByteDirectoryMarkers, storagePolicyTransitionParamsDto.getStorageName(),
            storagePolicyTransitionParamsDto.getBusinessObjectDataKey());

    // Get actual S3 files by selecting all S3 keys matching the S3 key prefix form the S3 bucket.
    // This time, we do not ignore 0 byte objects that represent S3 directories.
    List<S3ObjectSummary> actualS3Files = s3Service.listDirectory(s3FileTransferRequestParamsDto, false);

    // Tag the S3 objects to initiate the archiving.
    s3Service.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, actualS3Files,
            new Tag(storagePolicyTransitionParamsDto.getS3ObjectTagKey(),
                    storagePolicyTransitionParamsDto.getS3ObjectTagValue()));
}