List of usage examples for com.amazonaws.services.s3 AmazonS3URI AmazonS3URI
public AmazonS3URI(final URI uri)
From source file:br.com.ingenieux.mojo.cloudformation.PushStackMojo.java
License:Apache License
@Override protected Object executeInternal() throws Exception { shouldFailIfMissingStack(failIfMissing); if (!templateLocation.exists() && !templateLocation.isFile()) { getLog().warn("File not found (or not a file): " + templateLocation.getPath() + ". Skipping."); return null; }/*from ww w .j av a2s. c o m*/ if (isNotBlank(s3Url)) { getLog().info("Uploading file " + this.templateLocation + " to location " + this.s3Url); s3Client = new BeanstalkerS3Client(getAWSCredentials(), getClientConfiguration(), getRegion()); s3Client.setMultipartUpload(false); this.destinationS3Uri = new AmazonS3URI(s3Url); uploadContents(templateLocation, destinationS3Uri); } else { templateBody = IOUtils.toString(new FileInputStream(this.templateLocation)); } { ValidateTemplateResult validateTemplateResult = validateTemplate(); if (!validateTemplateResult.getParameters().isEmpty()) { Set<String> existingParameterNames = this.parameters.stream().map(x -> x.getParameterKey()) .collect(Collectors.toSet()); Set<String> requiredParameterNames = validateTemplateResult.getParameters().stream() .map(x -> x.getParameterKey()).collect(Collectors.toSet()); for (String requiredParameter : requiredParameterNames) { if (!existingParameterNames.contains(requiredParameter)) { getLog().warn("Missing required parameter name: " + requiredParameter); getLog().warn("If its an update, will reuse previous value"); } this.parameters.add(new com.amazonaws.services.cloudformation.model.Parameter() .withParameterKey(requiredParameter).withUsePreviousValue(true)); } } } WaitForStackCommand.WaitForStackContext ctx = null; Object result = null; if (null == stackSummary) { getLog().info("Must Create Stack"); CreateStackResult createStackResult; result = createStackResult = createStack(); ctx = new WaitForStackCommand.WaitForStackContext(createStackResult.getStackId(), getService(), this::info, 30, asList(StackStatus.CREATE_COMPLETE)); } else { getLog().info("Must Update Stack"); UpdateStackResult updateStackResult; result = updateStackResult = updateStack(); if (null != result) { ctx = new WaitForStackCommand.WaitForStackContext(updateStackResult.getStackId(), getService(), this::info, 30, asList(StackStatus.UPDATE_COMPLETE)); } } if (null != ctx) new WaitForStackCommand(ctx).execute(); return result; }
From source file:com.facebook.presto.kinesis.s3config.S3TableConfigClient.java
License:Apache License
/** * Call S3 to get the most recent object list. * * This is an object list request to AWS in the given "directory". * * @return// w ww . j a v a 2 s .c o m */ protected List<S3ObjectSummary> getObjectSummaries() { AmazonS3Client s3client = this.clientManager.getS3Client(); AmazonS3URI directoryURI = new AmazonS3URI(this.bucketUrl); ArrayList<S3ObjectSummary> returnList = new ArrayList<S3ObjectSummary>(); try { log.info("Getting the listing of objects in the S3 table config directory: bucket %s prefix %s :", directoryURI.getBucket(), directoryURI.getKey()); ListObjectsRequest req = new ListObjectsRequest().withBucketName(directoryURI.getBucket()) .withPrefix(directoryURI.getKey() + "/").withDelimiter("/").withMaxKeys(25); ObjectListing result; do { result = s3client.listObjects(req); returnList.addAll(result.getObjectSummaries()); req.setMarker(result.getNextMarker()); } while (result.isTruncated()); log.info("Completed getting S3 object listing."); } catch (AmazonServiceException ase) { StringBuilder sb = new StringBuilder(); sb.append("Caught an AmazonServiceException, which means your request made it "); sb.append("to Amazon S3, but was rejected with an error response for some reason.\n"); sb.append("Error Message: " + ase.getMessage()); sb.append("HTTP Status Code: " + ase.getStatusCode()); sb.append("AWS Error Code: " + ase.getErrorCode()); sb.append("Error Type: " + ase.getErrorType()); sb.append("Request ID: " + ase.getRequestId()); log.error(sb.toString(), ase); } catch (AmazonClientException ace) { StringBuilder sb = new StringBuilder(); sb.append("Caught an AmazonClientException, " + "which means the client encountered " + "an internal error while trying to communicate" + " with S3, " + "such as not being able to access the network."); sb.append("Error Message: " + ace.getMessage()); log.error(sb.toString(), ace); } return returnList; }
From source file:com.facebook.presto.kinesis.s3config.S3TableConfigClient.java
License:Apache License
/** * Connect to S3 directory to look for new or updated table definitions and then * update the map./*w w w . j av a2s . com*/ */ protected void updateTablesFromS3() { long now = System.currentTimeMillis(); List<S3ObjectSummary> objectList = this.getObjectSummaries(); AmazonS3Client s3client = this.clientManager.getS3Client(); AmazonS3URI directoryURI = new AmazonS3URI(this.bucketUrl); // Build map of "deltas" which in the end contains new definitions and deleted tables HashMap<String, KinesisStreamDescription> deltasMap = new HashMap<String, KinesisStreamDescription>(); internalMapLock.readLock().lock(); try { Iterator<String> keysIter = this.internalMap.keySet().iterator(); while (keysIter.hasNext()) { deltasMap.put(keysIter.next(), dummyStreamDesc); } } finally { internalMapLock.readLock().unlock(); } for (S3ObjectSummary objInfo : objectList) { if (!deltasMap.containsKey(objInfo.getKey()) || objInfo.getLastModified().getTime() >= this.lastCheck) { // New or updated file, so we must read from AWS try { if (objInfo.getKey().endsWith("/")) { continue; } log.info("Getting : %s - %s", objInfo.getBucketName(), objInfo.getKey()); S3Object object = s3client .getObject(new GetObjectRequest(objInfo.getBucketName(), objInfo.getKey())); StringBuilder resultStr = new StringBuilder(""); try (BufferedReader reader = new BufferedReader( new InputStreamReader(object.getObjectContent()))) { boolean hasMore = true; while (hasMore) { String line = reader.readLine(); if (line != null) { resultStr.append(line); } else { hasMore = false; } } KinesisStreamDescription table = streamDescriptionCodec.fromJson(resultStr.toString()); deltasMap.put(objInfo.getKey(), table); log.info("Put table description into the map from %s : %s.%s", objInfo.getKey(), table.getSchemaName(), table.getTableName()); } catch (IOException iox) { log.error("Problem reading input stream from object.", iox); } catch (IllegalArgumentException iax) { // Note: this gets thrown by airlift json library when the input is malformed. log.error("Invalid JSON table description.", iax); } } catch (AmazonServiceException ase) { StringBuilder sb = new StringBuilder(); sb.append("Caught an AmazonServiceException, which means your request made it "); sb.append("to Amazon S3, but was rejected with an error response for some reason.\n"); sb.append("Error Message: " + ase.getMessage()); sb.append("HTTP Status Code: " + ase.getStatusCode()); sb.append("AWS Error Code: " + ase.getErrorCode()); sb.append("Error Type: " + ase.getErrorType()); sb.append("Request ID: " + ase.getRequestId()); log.error(sb.toString(), ase); } catch (AmazonClientException ace) { StringBuilder sb = new StringBuilder(); sb.append("Caught an AmazonClientException, " + "which means the client encountered " + "an internal error while trying to communicate" + " with S3, " + "such as not being able to access the network."); sb.append("Error Message: " + ace.getMessage()); log.error(sb.toString(), ace); } } else if (deltasMap.containsKey(objInfo.getKey())) { deltasMap.remove(objInfo.getKey()); } } // end loop through object descriptions // Deltas: key pointing to dummy means delete, key pointing to other object means update. // This approach lets us delete and update while shortening the locked critical section. Iterator<Map.Entry<String, KinesisStreamDescription>> deltasIter = deltasMap.entrySet().iterator(); internalMapLock.writeLock().lock(); try { while (deltasIter.hasNext()) { Map.Entry<String, KinesisStreamDescription> entry = deltasIter.next(); if (entry.getValue().getTableName().equals("__DUMMY__")) { this.internalMap.remove(entry.getKey()); } else { this.internalMap.put(entry.getKey(), entry.getValue()); } } } finally { internalMapLock.writeLock().unlock(); } log.info("Completed updating table definitions from S3."); this.lastCheck = now; return; }
From source file:com.netflix.genie.common.internal.aws.s3.S3ProtocolResolver.java
License:Apache License
/** * {@inheritDoc}//from w w w . j a va 2s . com */ @Override public Resource resolve(final String location, final ResourceLoader resourceLoader) { log.debug("Attempting to resolve if {} is a S3 resource or not", location); final AmazonS3URI s3URI; try { s3URI = new AmazonS3URI(location); } catch (final IllegalArgumentException iae) { log.debug("{} is not a valid S3 resource (Error message: {}).", location, iae.getMessage()); return null; } final AmazonS3 client = this.s3ClientFactory.getClient(s3URI); log.debug("{} is a valid S3 resource.", location); // TODO: This implementation from Spring Cloud AWS always wraps the passed in client with a proxy that follows // redirects. I'm not sure if we want that or not. Probably ok for now but maybe revisit later? return new SimpleStorageResource(client, s3URI.getBucket(), s3URI.getKey(), this.s3TaskExecutor, s3URI.getVersionId()); }
From source file:com.netflix.genie.common.internal.services.impl.S3JobArchiverImpl.java
License:Apache License
/** * {@inheritDoc}/* w ww .j a v a 2s .co m*/ */ @Override public boolean archiveDirectory(@NotNull final Path directory, @NotNull final URI target) throws JobArchiveException { final String uriString = target.toString(); final AmazonS3URI s3URI; try { s3URI = new AmazonS3URI(target); } catch (final IllegalArgumentException iae) { log.debug("{} is not a valid S3 URI", uriString); return false; } final String directoryString = directory.toString(); log.debug("{} is a valid S3 location. Proceeding to archive {} to location: {}", uriString, directoryString, uriString); try { final TransferManager transferManager = this.s3ClientFactory.getTransferManager(s3URI); final MultipleFileUpload upload = transferManager.uploadDirectory(s3URI.getBucket(), s3URI.getKey(), directory.toFile(), true); upload.waitForCompletion(); return true; } catch (final Exception e) { log.error("Error archiving to S3 location: {} ", uriString, e); throw new JobArchiveException("Error archiving " + directoryString, e); } }
From source file:com.netflix.imflibrary.app.IMFTrackFileReader.java
License:Apache License
public static void main(String[] args) throws IOException { if (args.length != 2) { logger.error(usage());//from w ww . j a v a 2 s . c om throw new IllegalArgumentException("Invalid parameters"); } ResourceByteRangeProvider resourceByteRangeProvider; String fileName; if (args[0].startsWith("s3://")) { AmazonS3URI uri = new AmazonS3URI(args[0]); resourceByteRangeProvider = new S3ByteRangeProvider(uri); fileName = new File(uri.getKey()).getName(); } else { File inputFile = new File(args[0]); if (!inputFile.exists()) { logger.error(String.format("File %s does not exist", inputFile.getAbsolutePath())); System.exit(-1); } resourceByteRangeProvider = new FileByteRangeProvider(inputFile); fileName = inputFile.getName(); } File workingDirectory = new File(args[1]); IMFTrackFileReader imfTrackFileReader = null; IMFTrackFileCPLBuilder imfTrackFileCPLBuilder = null; IMFErrorLogger imfErrorLogger = new IMFErrorLoggerImpl(); try { imfTrackFileReader = new IMFTrackFileReader(workingDirectory, resourceByteRangeProvider); imfTrackFileCPLBuilder = new IMFTrackFileCPLBuilder(workingDirectory, fileName, resourceByteRangeProvider); } catch (IMFException | MXFException e) { if (e instanceof IMFException) { IMFException imfException = (IMFException) e; imfErrorLogger.addAllErrors(imfException.getErrors()); } else if (e instanceof MXFException) { MXFException mxfException = (MXFException) e; imfErrorLogger.addAllErrors(mxfException.getErrors()); } imfErrorLogger.addAllErrors(imfErrorLogger.getErrors()); } Set<HeaderPartition.EssenceTypeEnum> supportedEssenceComponentTypes = new HashSet<>(); supportedEssenceComponentTypes.add(HeaderPartition.EssenceTypeEnum.MainImageEssence); supportedEssenceComponentTypes.add(HeaderPartition.EssenceTypeEnum.MainAudioEssence); supportedEssenceComponentTypes.add(HeaderPartition.EssenceTypeEnum.MarkerEssence); if (imfTrackFileReader != null && imfTrackFileCPLBuilder != null && supportedEssenceComponentTypes.contains(imfTrackFileReader.getEssenceType(imfErrorLogger))) { try { HeaderPartition headerPartition = imfTrackFileReader.headerPartition.getHeaderPartitionOP1A() .getHeaderPartition(); List<InterchangeObject.InterchangeObjectBO> subDescriptors = headerPartition.getSubDescriptors(); for (InterchangeObject.InterchangeObjectBO subDescriptor : subDescriptors) { if (subDescriptor instanceof PHDRMetaDataTrackSubDescriptor.PHDRMetaDataTrackSubDescriptorBO) { logger.info("Found a PHDRMetaDataTrackSubDescriptor with instanceID: " + UUIDHelper .fromUUID(UUID.nameUUIDFromBytes(subDescriptor.getInstanceUID().getUID()))); } } for (InterchangeObject.InterchangeObjectBO essenceDescriptor : imfTrackFileReader .getEssenceDescriptors(imfErrorLogger)) { /* create dom */ DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); Document document = docBuilder.newDocument(); /*Output file containing the RegXML representation of the EssenceDescriptor*/ KLVPacket.Header essenceDescriptorHeader = essenceDescriptor.getHeader(); List<KLVPacket.Header> subDescriptorHeaders = imfTrackFileReader .getSubDescriptorKLVHeader(essenceDescriptor, imfErrorLogger); File outputFile = imfTrackFileCPLBuilder.getEssenceDescriptorAsXMLFile(document, essenceDescriptorHeader, subDescriptorHeaders, imfErrorLogger); logger.info(String.format( "The EssenceDescriptor in the IMFTrackFile has been written to a XML document at the following location %s", outputFile.getAbsolutePath())); } } catch (ParserConfigurationException | TransformerException e) { throw new MXFException(e); } } List<ErrorLogger.ErrorObject> errors = imfErrorLogger.getErrors(); if (errors.size() > 0) { long warningCount = errors.stream() .filter(e -> e.getErrorLevel().equals(IMFErrorLogger.IMFErrors.ErrorLevels.WARNING)).count(); logger.info(String.format("IMFTrackFile has %d errors and %d warnings", errors.size() - warningCount, warningCount)); for (ErrorLogger.ErrorObject errorObject : errors) { if (errorObject.getErrorLevel() != IMFErrorLogger.IMFErrors.ErrorLevels.WARNING) { logger.error(errorObject.toString()); } else if (errorObject.getErrorLevel() == IMFErrorLogger.IMFErrors.ErrorLevels.WARNING) { logger.warn(errorObject.toString()); } } } else { /*if(imfTrackFileReader != null && imfTrackFileCPLBuilder != null) { logger.info(String.format("%n %s", imfTrackFileReader.toString())); }*/ logger.info("No errors were detected in the IMFTrackFile"); } }
From source file:com.nextdoor.bender.Bender.java
License:Apache License
protected static void invokeS3Handler(String source_file) throws HandlerException { /*/* ww w . j a va 2s.c om*/ * https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html * https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html */ String awsRegion = "us-east-1"; String eventName = "s3:ObjectCreated:Put"; String eventSource = "aws:s3"; String eventVersion = "2.0"; String s3ConfigurationId = "cli-id"; String s3SchemaVersion = "1.0"; S3BucketEntity s3BucketEntity = null; S3ObjectEntity s3ObjectEntity = null; /* * Make sure the URL was submitted properly * * Split the s3://bucket/object path into an S3BucketEntity and S3ObjectEntity object */ try { AmazonS3URI s3URI = new AmazonS3URI(source_file); s3BucketEntity = new S3BucketEntity(s3URI.getBucket(), null, null); s3ObjectEntity = new S3ObjectEntity(s3URI.getKey(), 1L, null, null); } catch (IllegalArgumentException e) { logger.error("Invalid source_file URL supplied (" + source_file + "): " + e); System.exit(1); } /* * Override the AWS Region if its supplied */ if (System.getenv("AWS_REGION") != null) { awsRegion = System.getenv("AWS_REGION"); } /* * Set the arrival timestamp as the function run time. */ DateTime eventTime = new DateTime().toDateTime(); /* * Generate our context/handler objects.. we'll be populating them shortly. */ TestContext ctx = getContext(); S3Handler handler = new S3Handler(); /* * Create a S3EventNotification event */ S3Entity s3Entity = new S3Entity(s3ConfigurationId, s3BucketEntity, s3ObjectEntity, s3SchemaVersion); S3EventNotificationRecord rec = new S3EventNotificationRecord(awsRegion, eventName, eventSource, eventTime.toString(), eventVersion, null, null, s3Entity, null); List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(2); notifications.add(rec); S3EventNotification s3event = new S3EventNotification(notifications); /* * Invoke handler */ handler.handler(s3event, ctx); handler.shutdown(); }
From source file:com.nextdoor.bender.handler.BaseHandler.java
License:Apache License
/** * Loads @{link com.nextdoor.bender.config.Configuration} from a resource file and initializes * classes./*from w ww .j a va 2s . c o m*/ * * @param ctx function context as specified when function is invoked by lambda. * @throws HandlerException error while loading the @{link * com.nextdoor.bender.config.Configuration}. */ public void init(Context ctx) throws HandlerException { /* * Function alias is the last part of the Function ARN */ String alias = null; String[] tokens = ctx.getInvokedFunctionArn().split(":"); if (tokens.length == 7) { alias = "$LATEST"; } else if (tokens.length == 8) { alias = tokens[7]; } BenderLayout.ALIAS = alias; BenderLayout.VERSION = ctx.getFunctionVersion(); /* * Create a new monitor and then get a static copy of it */ monitor = Monitor.getInstance(); monitor.addTag("functionName", ctx.getFunctionName()); monitor.addTag("functionVersion", alias); String configFile; /* * TODO: Replace this to always use env vars. Code was written prior to lambda env vars * existing. */ if (System.getenv("BENDER_CONFIG") != null) { configFile = System.getenv("BENDER_CONFIG"); } else if (CONFIG_FILE == null) { configFile = "/config/" + alias; } else { configFile = CONFIG_FILE; } logger.info(String.format("Bender Initializing (config: %s)", configFile)); try { if (configFile.startsWith("s3://")) { config = BenderConfig.load(s3ClientFactory, new AmazonS3URI(configFile)); } else if (configFile.startsWith("file://")) { File file = new File(configFile.replaceFirst("file://", "")); String string = FileUtils.readFileToString(file); config = BenderConfig.load(configFile, string); } else { config = BenderConfig.load(configFile); } } catch (ConfigurationException | IOException e) { throw new HandlerException("Error loading configuration: " + e.getMessage(), e); } HandlerResources handlerResources; try { handlerResources = new HandlerResources(config); } catch (ClassNotFoundException e) { throw new HandlerException("Unable to load resource: " + e.getMessage(), e); } /* * Add user tags */ monitor.addTags(config.getHandlerConfig().getMetricTags()); /* * Add Lambda function tags. These will override duplicate user tags. */ if (config.getHandlerConfig().getIncludeFunctionTags()) { AWSLambda lambda = this.lambdaClientFactory.newInstance(); ListTagsResult res = lambda.listTags(new ListTagsRequest().withResource(ctx.getInvokedFunctionArn())); monitor.addTagsMap(res.getTags()); } /* * Register reporters */ monitor.addReporters(handlerResources.getReporters()); /* * Init other things */ wrapper = handlerResources.getWrapperFactory().newInstance(); ser = handlerResources.getSerializerProcessor(); setIpcService(new IpcSenderService(handlerResources.getTransportFactory())); sources = new ArrayList<Source>(handlerResources.getSources().values()); queueSize = config.getHandlerConfig().getQueueSize(); initialized = true; }
From source file:com.nextdoor.bender.operations.geo.GeoIpOperationFactory.java
License:Apache License
@Override public void setConf(AbstractConfig config) { this.config = (GeoIpOperationConfig) config; AmazonS3Client client = this.s3Factory.newInstance(); AmazonS3URI uri = new AmazonS3URI(this.config.getGeoLiteDb()); GetObjectRequest req = new GetObjectRequest(uri.getBucket(), uri.getKey()); S3Object obj = client.getObject(req); try {// w w w. j a v a2 s . c o m this.databaseReader = new DatabaseReader.Builder(obj.getObjectContent()).withCache(new CHMCache()) .build(); } catch (IOException e) { throw new ConfigurationException("Unable to read " + this.config.getGeoLiteDb(), e); } }
From source file:com.qubole.presto.kinesis.s3config.S3TableConfigClient.java
License:Apache License
/** * Connect to S3 directory to look for new or updated table definitions and then * update the map./*from ww w . ja va2 s .c o m*/ */ protected void updateTablesFromS3() { long now = System.currentTimeMillis(); List<S3ObjectSummary> objectList = this.getObjectSummaries(); AmazonS3Client s3client = this.clientManager.getS3Client(); AmazonS3URI directoryURI = new AmazonS3URI(this.bucketUrl); for (S3ObjectSummary objInfo : objectList) { if (!this.internalMap.containsKey(objInfo.getKey()) || objInfo.getLastModified().getTime() >= this.lastCheck) { // New or updated file, so we must read from AWS try { if (objInfo.getKey().endsWith("/")) { continue; } log.info("Getting : %s - %s", objInfo.getBucketName(), objInfo.getKey()); S3Object object = s3client .getObject(new GetObjectRequest(objInfo.getBucketName(), objInfo.getKey())); StringBuilder resultStr = new StringBuilder(""); try (BufferedReader reader = new BufferedReader( new InputStreamReader(object.getObjectContent()))) { boolean hasMore = true; while (hasMore) { String line = reader.readLine(); if (line != null) { resultStr.append(line); } else { hasMore = false; } } KinesisStreamDescription table = streamDescriptionCodec.fromJson(resultStr.toString()); internalMap.put(objInfo.getKey(), table); log.info("Put table description into the map from %s", objInfo.getKey()); } catch (IOException iox) { log.error("Problem reading input stream from object.", iox); } } catch (AmazonServiceException ase) { StringBuilder sb = new StringBuilder(); sb.append("Caught an AmazonServiceException, which means your request made it "); sb.append("to Amazon S3, but was rejected with an error response for some reason.\n"); sb.append("Error Message: " + ase.getMessage()); sb.append("HTTP Status Code: " + ase.getStatusCode()); sb.append("AWS Error Code: " + ase.getErrorCode()); sb.append("Error Type: " + ase.getErrorType()); sb.append("Request ID: " + ase.getRequestId()); log.error(sb.toString(), ase); } catch (AmazonClientException ace) { StringBuilder sb = new StringBuilder(); sb.append("Caught an AmazonClientException, " + "which means the client encountered " + "an internal error while trying to communicate" + " with S3, " + "such as not being able to access the network."); sb.append("Error Message: " + ace.getMessage()); log.error(sb.toString(), ace); } } } // end loop through object descriptions log.info("Completed updating table definitions from S3."); this.lastCheck = now; return; }