Java tutorial
/* * Copyright (c) 2014 Baidu.com, Inc. All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.baidubce.services.bos; import com.baidubce.AbstractBceClient; import com.baidubce.BceClientException; import com.baidubce.BceServiceException; import com.baidubce.auth.BceV1Signer; import com.baidubce.auth.SignOptions; import com.baidubce.auth.Signer; import com.baidubce.http.Headers; import com.baidubce.http.HttpMethodName; import com.baidubce.http.StatusCodes; import com.baidubce.http.handler.BceErrorResponseHandler; import com.baidubce.http.handler.BceJsonResponseHandler; import com.baidubce.http.handler.BceMetadataResponseHandler; import com.baidubce.http.handler.HttpResponseHandler; import com.baidubce.internal.*; import com.baidubce.model.AbstractBceRequest; import com.baidubce.model.User; import com.baidubce.services.bos.model.*; import com.baidubce.util.*; import com.baidubce.util.HttpUtils; import com.fasterxml.jackson.core.JsonGenerator; import com.google.common.collect.Lists; import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Hex; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.net.MalformedURLException; import java.net.URL; import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Map.Entry; import static com.google.common.base.Preconditions.checkNotNull; /** * Provides the client for accessing the Baidu Object Service. */ public class BosClient extends AbstractBceClient { private static Logger logger = LoggerFactory.getLogger(BosClient.class); /** * Responsible for handling httpResponses from all Bos service calls. */ private static final HttpResponseHandler[] bos_handlers = new HttpResponseHandler[] { new BceMetadataResponseHandler(), new BosMetadataResponseHandler(), new BceErrorResponseHandler(), new BosObjectResponseHandler(), new BceJsonResponseHandler() }; /** * Constructs a new client to invoke service methods on Bos. */ public BosClient() { this(new BosClientConfiguration()); } /** * Constructs a new Bos client using the client configuration to access Bos. * * @param clientConfiguration The bos client configuration options controlling how this client * connects to Bos (e.g. proxy settings, retry counts, etc). */ public BosClient(BosClientConfiguration clientConfiguration) { super(clientConfiguration, bos_handlers, true); } /** * Gets the current owner of the Bos account that the authenticated sender of the request is using. * * <p> * The caller <i>must</i> authenticate with a valid BCE Access Key ID that is registered with Bos. * * @return The account of the authenticated sender */ public User getBosAccountOwner() { return this.getBosAccountOwner(new GetBosAccountOwnerRequest()); } /** * Gets the current owner of the Bos account that the authenticated sender of the request is using. * * <p> * The caller <i>must</i> authenticate with a valid BCE Access Key ID that is registered with Bos. * * @param request This request containing the credentials for getting the account of the authenticated sender. * @return The account of the authenticated sender */ public User getBosAccountOwner(GetBosAccountOwnerRequest request) { checkNotNull(request, "request should not be null."); return this.invokeHttpClient(this.createRequest(request, HttpMethodName.GET), ListBucketsResponse.class) .getOwner(); } /** * Returns a list of all Bos buckets that the authenticated sender of the request owns. * * <p> * Users must authenticate with a valid BCE Access Key ID that is registered * with Bos. Anonymous requests cannot list buckets, and users cannot * list buckets that they did not create. * * @return All of the Bos buckets owned by the authenticated sender of the request. */ public ListBucketsResponse listBuckets() { return this.listBuckets(new ListBucketsRequest()); } /** * Returns a list of all Bos buckets that the authenticated sender of the request owns. * * <p> * Users must authenticate with a valid BCE Access Key ID that is registered * with Bos. Anonymous requests cannot list buckets, and users cannot * list buckets that they did not create. * * @param request The request containing all of the options related to the listing of buckets. * @return All of the Bos buckets owned by the authenticated sender of the request. */ public ListBucketsResponse listBuckets(ListBucketsRequest request) { checkNotNull(request, "request should not be null."); return this.invokeHttpClient(this.createRequest(request, HttpMethodName.GET), ListBucketsResponse.class); } /** * Creates a new Bos bucket with the specified name. * * @param bucketName The name of the bucket to create. * All buckets in Bos share a single namespace; ensure the bucket is given a unique name. * @return The newly created bucket. */ public CreateBucketResponse createBucket(String bucketName) { return this.createBucket(new CreateBucketRequest(bucketName)); } /** * Creates a new Bos bucket with the specified name. * * @param request The request object containing all options for creating a Bos bucket. * @return The newly created bucket. */ public CreateBucketResponse createBucket(CreateBucketRequest request) { checkNotNull(request, "request should not be null."); InternalRequest internalRequest = this.createRequest(request, HttpMethodName.PUT); this.setZeroContentLength(internalRequest); BosResponse response = this.invokeHttpClient(internalRequest, BosResponse.class); CreateBucketResponse result = new CreateBucketResponse(); result.setName(request.getBucketName()); result.setLocation(response.getMetadata().getLocation()); return result; } /** * Checks if the specified bucket exists. Bos buckets are named in a * global namespace; use this method to determine if a specified bucket name * already exists, and therefore can't be used to create a new bucket. * * <p> * If invalid security credentials are used to execute this method, the * client is not able to distinguish between bucket permission errors and * invalid credential errors, and this method could return an incorrect * result. * * @param bucketName The name of the bucket to check. * @return The value <code>true</code> if the specified bucket exists in Bos; * the value <code>false</code> if there is no bucket in Bos with that name. */ public boolean doesBucketExist(String bucketName) { return this.doesBucketExist(new DoesBucketExistRequest(bucketName)); } /** * Checks if the specified bucket exists. Bos buckets are named in a * global namespace; use this method to determine if a specified bucket name * already exists, and therefore can't be used to create a new bucket. * * <p> * If invalid security credentials are used to execute this method, the * client is not able to distinguish between bucket permission errors and * invalid credential errors, and this method could return an incorrect * result. * * @param request The request object containing all options for checking a Bos bucket. * @return The value <code>true</code> if the specified bucket exists in Bos; * the value <code>false</code> if there is no bucket in Bos with that name. */ public boolean doesBucketExist(DoesBucketExistRequest request) { checkNotNull(request, "request should not be null."); try { this.invokeHttpClient(this.createRequest(request, HttpMethodName.HEAD), BosResponse.class); return true; } catch (BceServiceException e) { // Forbidden means that the bucket exists. if (e.getStatusCode() == StatusCodes.FORBIDDEN) { return true; } if (e.getStatusCode() == StatusCodes.NOT_FOUND) { return false; } throw e; } } /** * Gets the ACL for the specified Bos bucket. * * <p> * Each bucket and object in Bos has an ACL that defines its access * control policy. When a request is made, Bos authenticates the * request using its standard authentication procedure and then checks the * ACL to verify the sender was granted access to the bucket or object. If * the sender is approved, the request proceeds. Otherwise, Bos * returns an error. * * @param bucketName The name of the bucket whose ACL is being retrieved. * @return The <code>GetBuckeetAclResponse</code> for the specified Bos bucket. */ public GetBucketAclResponse getBucketAcl(String bucketName) { return this.getBucketAcl(new GetBucketAclRequest(bucketName)); } /** * Gets the ACL for the specified Bos bucket. * * <p> * Each bucket and object in Bos has an ACL that defines its access * control policy. When a request is made, Bos authenticates the * request using its standard authentication procedure and then checks the * ACL to verify the sender was granted access to the bucket or object. If * the sender is approved, the request proceeds. Otherwise, Bos * returns an error. * * @param request The request containing the name of the bucket whose ACL is being retrieved. * @return The <code>GetBuckeetAclResponse</code> for the specified Bos bucket. */ public GetBucketAclResponse getBucketAcl(GetBucketAclRequest request) { checkNotNull(request, "request should not be null."); InternalRequest internalRequest = this.createRequest(request, HttpMethodName.GET); internalRequest.addParameter("acl", null); GetBucketAclResponse response = this.invokeHttpClient(internalRequest, GetBucketAclResponse.class); if (response.getVersion() > response.MAX_SUPPORTED_ACL_VERSION) { throw new BceClientException("Unsupported acl version."); } return response; } /** * Gets the Location for the specified Bos bucket. * * <p> * Each bucket and object in Bos has an Location that defines its location * * @param bucketName The name of the bucket whose Location is being retrieved. * @return The <code>GetBuckeetLocationResponse</code> for the specified Bos bucket. */ public GetBucketLocationResponse getBucketLocation(String bucketName) { return this.getBucketLocation(new GetBucketLocationRequest(bucketName)); } /** * Gets the Location for the specified Bos bucket. * * <p> * Each bucket and object in Bos has an Location that defines its location * * @param request The request containing the name of the bucket whose Location is being retrieved. * @return The <code>GetBuckeetLocationResponse</code> for the specified Bos bucket. */ public GetBucketLocationResponse getBucketLocation(GetBucketLocationRequest request) { checkNotNull(request, "request should not be null."); InternalRequest internalRequest = this.createRequest(request, HttpMethodName.GET); internalRequest.addParameter("location", null); GetBucketLocationResponse response = this.invokeHttpClient(internalRequest, GetBucketLocationResponse.class); return response; } /** * Sets the CannedAccessControlList for the specified Bos bucket using one of * the pre-configured <code>CannedAccessControlLists</code>. * * <p> * A <code>CannedAccessControlList</code> * provides a quick way to configure an object or bucket with commonly used * access control policies. * * @param bucketName The name of the bucket whose ACL is being set. * @param acl The pre-configured <code>CannedAccessControlLists</code> to set for the specified bucket. */ public void setBucketAcl(String bucketName, CannedAccessControlList acl) { this.setBucketAcl(new SetBucketAclRequest(bucketName, acl)); } /** * Sets the Acl for the specified Bos bucket. * * @param request The request object containing the bucket to modify and the ACL to set. */ public void setBucketAcl(SetBucketAclRequest request) { checkNotNull(request, "request should not be null."); InternalRequest internalRequest = this.createRequest(request, HttpMethodName.PUT); internalRequest.addParameter("acl", null); if (request.getCannedAcl() != null) { internalRequest.addHeader(Headers.BCE_ACL, request.getCannedAcl().toString()); this.setZeroContentLength(internalRequest); } else if (request.getAccessControlList() != null) { byte[] json = null; List<Grant> grants = request.getAccessControlList(); StringWriter writer = new StringWriter(); try { JsonGenerator jsonGenerator = JsonUtils.jsonGeneratorOf(writer); jsonGenerator.writeStartObject(); jsonGenerator.writeArrayFieldStart("accessControlList"); for (Grant grant : grants) { jsonGenerator.writeStartObject(); jsonGenerator.writeArrayFieldStart("grantee"); for (Grantee grantee : grant.getGrantee()) { jsonGenerator.writeStartObject(); jsonGenerator.writeStringField("id", grantee.getId()); jsonGenerator.writeEndObject(); } jsonGenerator.writeEndArray(); jsonGenerator.writeArrayFieldStart("permission"); for (Permission permission : grant.getPermission()) { jsonGenerator.writeString(permission.toString()); } jsonGenerator.writeEndArray(); jsonGenerator.writeEndObject(); } jsonGenerator.writeEndArray(); jsonGenerator.writeEndObject(); jsonGenerator.close(); } catch (IOException e) { throw new BceClientException("Fail to generate json", e); } try { json = writer.toString().getBytes(DEFAULT_ENCODING); } catch (UnsupportedEncodingException e) { throw new BceClientException("Fail to get UTF-8 bytes", e); } internalRequest.addHeader(Headers.CONTENT_LENGTH, String.valueOf(json.length)); internalRequest.addHeader(Headers.CONTENT_TYPE, "application/json"); internalRequest.setContent(RestartableInputStream.wrap(json)); } else { checkNotNull(null, "request.acl should not be null."); } this.invokeHttpClient(internalRequest, BosResponse.class); } /** * Deletes the specified bucket. All objects in the bucket must be deleted before the bucket itself * can be deleted. * * <p> * Only the owner of a bucket can delete it, regardless of the bucket's access control policy. * * @param bucketName The name of the bucket to delete. */ public void deleteBucket(String bucketName) { this.deleteBucket(new DeleteBucketRequest(bucketName)); } /** * Deletes the specified bucket. All objects in the bucket must be deleted before the bucket itself * can be deleted. * * <p> * Only the owner of a bucket can delete it, regardless of the bucket's access control policy. * * @param request The request object containing all options for deleting a Bos bucket. */ public void deleteBucket(DeleteBucketRequest request) { checkNotNull(request, "request should not be null."); this.invokeHttpClient(this.createRequest(request, HttpMethodName.DELETE), BosResponse.class); } /** * Returns a pre-signed URL for accessing a Bos resource. * * @param bucketName The name of the bucket containing the desired object. * @param key The key in the specified bucket under which the desired object is stored. * @param expirationInSeconds The expiration after which the returned pre-signed URL will expire. * @return A pre-signed URL which expires at the specified time, and can be * used to allow anyone to download the specified object from Bos, * without exposing the owner's Bce secret access key. */ public URL generatePresignedUrl(String bucketName, String key, int expirationInSeconds) { return this.generatePresignedUrl(bucketName, key, expirationInSeconds, HttpMethodName.GET); } /** * Returns a pre-signed URL for accessing a Bos resource. * * @param bucketName The name of the bucket containing the desired object. * @param key The key in the specified bucket under which the desired object is stored. * @param expirationInSeconds The expiration after which the returned pre-signed URL will expire. * @param method The HTTP method verb to use for this URL * @return A pre-signed URL which expires at the specified time, and can be * used to allow anyone to download the specified object from Bos, * without exposing the owner's Bce secret access key. */ public URL generatePresignedUrl(String bucketName, String key, int expirationInSeconds, HttpMethodName method) { GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, key, method); request.setExpiration(expirationInSeconds); return this.generatePresignedUrl(request); } /** * Returns a pre-signed URL for accessing a Bos resource. * * @param request The request object containing all the options for generating a * pre-signed URL (bucket name, key, expiration date, etc). * @return A pre-signed URL which expires at the specified time, and can be * used to allow anyone to download the specified object from Bos, * without exposing the owner's Bce secret access key. */ public URL generatePresignedUrl(GeneratePresignedUrlRequest request) { checkNotNull(request, "The request parameter must be specified when generating a pre-signed URL"); HttpMethodName httpMethod = HttpMethodName.valueOf(request.getMethod().toString()); // If the key starts with a slash character itself, the following method // will actually add another slash before the resource path to prevent // the HttpClient mistakenly treating the slash as a path delimiter. // For presigned request, we need to remember to remove this extra slash // before generating the URL. InternalRequest internalRequest = new InternalRequest(httpMethod, HttpUtils.appendUri(this.getEndpoint(), URL_PREFIX, request.getBucketName(), request.getKey())); internalRequest.setCredentials(request.getRequestCredentials()); SignOptions options = new SignOptions(); options.setExpirationInSeconds(request.getExpiration()); for (Entry<String, String> entry : request.getRequestHeaders().entrySet()) { if (entry.getValue() == null) { internalRequest.addHeader(entry.getKey(), ""); } else { internalRequest.addHeader(entry.getKey(), entry.getValue()); } } for (Entry<String, String> entry : request.getRequestParameters().entrySet()) { if (entry.getValue() == null) { internalRequest.addParameter(entry.getKey(), ""); } else { internalRequest.addParameter(entry.getKey(), entry.getValue()); } } if (request.getContentType() != null) { internalRequest.addHeader(Headers.CONTENT_TYPE, request.getContentType()); } if (request.getContentMd5() != null) { internalRequest.addHeader(Headers.CONTENT_MD5, request.getContentMd5()); } addResponseHeaderParameters(internalRequest, request.getResponseHeaders()); Signer signer = new BceV1Signer(); signer.sign(internalRequest, this.config.getCredentials(), options); // Remove the leading slash (if any) in the resource-path return convertRequestToUrl(internalRequest); } /** * Returns ListObjectsResponse containing a list of summary information about the objects in the specified buckets. * List results are <i>always</i> returned in lexicographic (alphabetical) order. * * @param bucketName The name of the Bos bucket to list. * @return ListObjectsResponse containing a listing of the objects in the specified bucket, along with any * other associated information, such as common prefixes (if a delimiter was specified), the original * request parameters, etc. */ public ListObjectsResponse listObjects(String bucketName) { return this.listObjects(new ListObjectsRequest(bucketName)); } /** * Returns ListObjectsResponse containing a list of summary information about the objects in the specified buckets. * List results are <i>always</i> returned in lexicographic (alphabetical) order. * * @param bucketName The name of the Bos bucket to list. * @param prefix An optional parameter restricting the response to keys beginning with the specified prefix. * Use prefixes to separate a bucket into different sets of keys, similar to how a file system * organizes files into directories. * @return ListObjectsResponse containing a listing of the objects in the specified bucket, along with any * other associated information, such as common prefixes (if a delimiter was specified), the original * request parameters, etc. */ public ListObjectsResponse listObjects(String bucketName, String prefix) { return this.listObjects(new ListObjectsRequest(bucketName, prefix)); } /** * Returns ListObjectsResponse containing a list of summary information about the objects in the specified buckets. * List results are <i>always</i> returned in lexicographic (alphabetical) order. * * @param request The request object containing all options for listing the objects in a specified bucket. * @return ListObjectsResponse containing a listing of the objects in the specified bucket, along with any * other associated information, such as common prefixes (if a delimiter was specified), the original * request parameters, etc. */ public ListObjectsResponse listObjects(ListObjectsRequest request) { checkNotNull(request, "request should not be null."); InternalRequest internalRequest = this.createRequest(request, HttpMethodName.GET); if (request.getPrefix() != null) { internalRequest.addParameter("prefix", request.getPrefix()); } if (request.getMarker() != null) { internalRequest.addParameter("marker", request.getMarker()); } if (request.getDelimiter() != null) { internalRequest.addParameter("delimiter", request.getDelimiter()); } if (request.getMaxKeys() >= 0) { internalRequest.addParameter("maxKeys", String.valueOf(request.getMaxKeys())); } ListObjectsResponse response = this.invokeHttpClient(internalRequest, ListObjectsResponse.class); response.setBucketName(request.getBucketName()); List<BosObjectSummary> contents = response.getContents(); for (BosObjectSummary object : contents) { object.setBucketName(request.getBucketName()); } return response; } /** * Provides an easy way to continue a truncated object listing and retrieve the next page of results. * * @param previousResponse The previous truncated <code>ListObjectsResponse</code>. If a non-truncated * <code>ListObjectsResponse</code> is passed in, an empty <code>ListObjectsResponse</code> * is returned without ever contacting Bos. * @return The next set of <code>ListObjectsResponse</code> results, beginning immediately * after the last result in the specified previous <code>ListObjectsResponse</code>. */ public ListObjectsResponse listNextBatchOfObjects(ListObjectsResponse previousResponse) { checkNotNull(previousResponse, "previousResponse should not be null."); if (!previousResponse.isTruncated()) { ListObjectsResponse emptyResponse = new ListObjectsResponse(); emptyResponse.setBucketName(previousResponse.getBucketName()); emptyResponse.setDelimiter(previousResponse.getDelimiter()); emptyResponse.setMarker(previousResponse.getNextMarker()); emptyResponse.setMaxKeys(previousResponse.getMaxKeys()); emptyResponse.setPrefix(previousResponse.getPrefix()); emptyResponse.setTruncated(false); return emptyResponse; } return this.listObjects(new ListObjectsRequest(previousResponse.getBucketName()) .withPrefix(previousResponse.getPrefix()).withMarker(previousResponse.getNextMarker()) .withDelimiter(previousResponse.getDelimiter()).withMaxKeys(previousResponse.getMaxKeys())); } /** * Gets the object stored in Bos under the specified bucket and key. * * @param bucketName The name of the bucket containing the desired object. * @param key The key under which the desired object is stored. * @return The object stored in Bos in the specified bucket and key. */ public BosObject getObject(String bucketName, String key) { return this.getObject(new GetObjectRequest(bucketName, key)); } /** * Gets the object metadata for the object stored in Bos under the specified bucket and key, * and saves the object contents to the specified file. * Returns <code>null</code> if the specified constraints weren't met. * * @param bucketName The name of the bucket containing the desired object. * @param key The key under which the desired object is stored. * @param destinationFile Indicates the file (which might already exist) * where to save the object content being downloading from Bos. * @return All Bos object metadata for the specified object. * Returns <code>null</code> if constraints were specified but not met. */ public ObjectMetadata getObject(String bucketName, String key, File destinationFile) { return this.getObject(new GetObjectRequest(bucketName, key), destinationFile); } /** * Gets the object stored in Bos under the specified bucket and key. * * @param request The request object containing all the options on how to download the object. * @return The object stored in Bos in the specified bucket and key. */ public BosObject getObject(GetObjectRequest request) { checkNotNull(request, "request should not be null."); InternalRequest internalRequest = this.createRequest(request, HttpMethodName.GET); long[] range = request.getRange(); if (range != null) { internalRequest.addHeader(Headers.RANGE, "bytes=" + range[0] + "-" + range[1]); } GetObjectResponse response = this.invokeHttpClient(internalRequest, GetObjectResponse.class); BosObject bosObject = response.getObject(); bosObject.setBucketName(request.getBucketName()); bosObject.setKey(request.getKey()); return bosObject; } /** * Gets the object metadata for the object stored in Bos under the specified bucket and key, * and saves the object contents to the specified file. * Returns <code>null</code> if the specified constraints weren't met. * * @param request The request object containing all the options on how to download the Bos object content. * @param destinationFile Indicates the file (which might already exist) where to save the object * content being downloading from Bos. * @return All Bos object metadata for the specified object. * Returns <code>null</code> if constraints were specified but not met. */ public ObjectMetadata getObject(GetObjectRequest request, File destinationFile) { checkNotNull(request, "request should not be null."); checkNotNull(destinationFile, "destinationFile should not be null."); BosObject bosObject = this.getObject(request); this.downloadObjectToFile(bosObject, destinationFile, request.getRange() == null); return bosObject.getObjectMetadata(); } /** * Gets the object content stored in Bos under the specified bucket and key. * * @param bucketName The name of the bucket containing the desired object. * @param key The key under which the desired object is stored. * @return The object content stored in Bos in the specified bucket and key. */ public byte[] getObjectContent(String bucketName, String key) { return this.getObjectContent(new GetObjectRequest(bucketName, key)); } /** * Gets the object content stored in Bos under the specified bucket and key. * * @param request The request object containing all the options on how to download the Bos object content. * @return The object content stored in Bos in the specified bucket and key. */ public byte[] getObjectContent(GetObjectRequest request) { BosObjectInputStream content = this.getObject(request).getObjectContent(); try { return IOUtils.toByteArray(content); } catch (IOException e) { try { content.close(); } catch (IOException e1) { // ignore, throw e not e1. } throw new BceClientException("Fail read object content", e); } finally { try { content.close(); } catch (IOException e) { // ignore } } } /** * Gets the metadata for the specified Bos object without actually fetching the object itself. * This is useful in obtaining only the object metadata, and avoids wasting bandwidth on fetching * the object data. * * <p> * The object metadata contains information such as content type, content disposition, etc., * as well as custom user metadata that can be associated with an object in Bos. * * @param bucketName The name of the bucket containing the object's whose metadata is being retrieved. * @param key The key of the object whose metadata is being retrieved. * @return All Bos object metadata for the specified object. */ public ObjectMetadata getObjectMetadata(String bucketName, String key) { return this.getObjectMetadata(new GetObjectMetadataRequest(bucketName, key)); } /** * Gets the metadata for the specified Bos object without actually fetching the object itself. * This is useful in obtaining only the object metadata, and avoids wasting bandwidth on fetching * the object data. * * <p> * The object metadata contains information such as content type, content disposition, etc., * as well as custom user metadata that can be associated with an object in Bos. * * @param request The request object specifying the bucket, key whose metadata is being retrieved. * @return All Bos object metadata for the specified object. */ public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest request) { checkNotNull(request, "request should not be null."); return this.invokeHttpClient(this.createRequest(request, HttpMethodName.HEAD), GetObjectResponse.class) .getObject().getObjectMetadata(); } /** * Uploads the specified file to Bos under the specified bucket and key name. * * @param bucketName The name of an existing bucket, to which you have Write permission. * @param key The key under which to store the specified file. * @param file The file containing the data to be uploaded to Bos. * @return A PutObjectResponse object containing the information returned by Bos for the newly created object. */ public PutObjectResponse putObject(String bucketName, String key, File file) { return this.putObject(new PutObjectRequest(bucketName, key, file)); } /** * Uploads the specified file and object metadata to Bos under the specified bucket and key name. * * @param bucketName The name of an existing bucket, to which you have Write permission. * @param key The key under which to store the specified file. * @param file The file containing the data to be uploaded to Bos. * @param metadata Additional metadata instructing Bos how to handle the uploaded data * (e.g. custom user metadata, hooks for specifying content type, etc.). * @return A PutObjectResponse object containing the information returned by Bos for the newly created object. */ public PutObjectResponse putObject(String bucketName, String key, File file, ObjectMetadata metadata) { return this.putObject(new PutObjectRequest(bucketName, key, file, metadata)); } /** * Uploads the specified string to Bos under the specified bucket and key name. * * @param bucketName The name of an existing bucket, to which you have Write permission. * @param key The key under which to store the specified file. * @param value The string containing the value to be uploaded to Bos. * @return A PutObjectResponse object containing the information returned by Bos for the newly created object. */ public PutObjectResponse putObject(String bucketName, String key, String value) { try { return this.putObject(bucketName, key, value.getBytes(DEFAULT_ENCODING), new ObjectMetadata()); } catch (UnsupportedEncodingException e) { throw new BceClientException("Fail to get bytes.", e); } } /** * Uploads the specified string and object metadata to Bos under the specified bucket and key name. * * @param bucketName The name of an existing bucket, to which you have Write permission. * @param key The key under which to store the specified file. * @param value The string containing the value to be uploaded to Bos. * @param metadata Additional metadata instructing Bos how to handle the uploaded data * (e.g. custom user metadata, hooks for specifying content type, etc.). * @return A PutObjectResponse object containing the information returned by Bos for the newly created object. */ public PutObjectResponse putObject(String bucketName, String key, String value, ObjectMetadata metadata) { try { return this.putObject(bucketName, key, value.getBytes(DEFAULT_ENCODING), metadata); } catch (UnsupportedEncodingException e) { throw new BceClientException("Fail to get bytes.", e); } } /** * Uploads the specified bytes to Bos under the specified bucket and key name. * * @param bucketName The name of an existing bucket, to which you have Write permission. * @param key The key under which to store the specified file. * @param value The bytes containing the value to be uploaded to Bos. * @return A PutObjectResponse object containing the information returned by Bos for the newly created object. */ public PutObjectResponse putObject(String bucketName, String key, byte[] value) { return this.putObject(bucketName, key, value, new ObjectMetadata()); } /** * Uploads the specified bytes and object metadata to Bos under the specified bucket and key name. * * @param bucketName The name of an existing bucket, to which you have Write permission. * @param key The key under which to store the specified file. * @param value The bytes containing the value to be uploaded to Bos. * @param metadata Additional metadata instructing Bos how to handle the uploaded data * (e.g. custom user metadata, hooks for specifying content type, etc.). * @return A PutObjectResponse object containing the information returned by Bos for the newly created object. */ public PutObjectResponse putObject(String bucketName, String key, byte[] value, ObjectMetadata metadata) { if (metadata.getContentLength() == -1) { metadata.setContentLength(value.length); } return this.putObject(new PutObjectRequest(bucketName, key, RestartableInputStream.wrap(value), metadata)); } /** * Uploads the specified input stream to Bos under the specified bucket and key name. * * @param bucketName The name of an existing bucket, to which you have Write permission. * @param key The key under which to store the specified file. * @param input The input stream containing the value to be uploaded to Bos. * @return A PutObjectResponse object containing the information returned by Bos for the newly created object. */ public PutObjectResponse putObject(String bucketName, String key, InputStream input) { return this.putObject(new PutObjectRequest(bucketName, key, input)); } /** * Uploads the specified input stream and object metadata to Bos under the specified bucket and key name. * * @param bucketName The name of an existing bucket, to which you have Write permission. * @param key The key under which to store the specified file. * @param input The input stream containing the value to be uploaded to Bos. * @param metadata Additional metadata instructing Bos how to handle the uploaded data * (e.g. custom user metadata, hooks for specifying content type, etc.). * @return A PutObjectResponse object containing the information returned by Bos for the newly created object. */ public PutObjectResponse putObject(String bucketName, String key, InputStream input, ObjectMetadata metadata) { return this.putObject(new PutObjectRequest(bucketName, key, input, metadata)); } /** * Uploads a new object to the specified Bos bucket. The <code>PutObjectRequest</code> contains all the * details of the request, including the bucket to upload to, the key the object will be uploaded under, * and the file or input stream containing the data to upload. * * @param request The request object containing all the parameters to upload a new object to Bos. * @return A PutObjectResponse object containing the information returned by Bos for the newly created object. */ public PutObjectResponse putObject(PutObjectRequest request) { checkNotNull(request, "request should not be null."); assertStringNotNullOrEmpty(request.getKey(), "object key should not be null or empty"); ObjectMetadata metadata = request.getObjectMetadata(); InputStream input = request.getInputStream(); InternalRequest internalRequest = this.createRequest(request, HttpMethodName.PUT); // If a file is specified for upload, we need to pull some additional information from it to auto-configure a // few options if (request.getFile() != null) { File file = request.getFile(); if (file.length() > 5 * 1024 * 1024 * 1024L) { BceServiceException bse = new BceServiceException( "Your proposed upload exceeds the maximum allowed " + "object size."); bse.setStatusCode(400); bse.setErrorCode("EntityTooLarge"); bse.setErrorType(BceServiceException.ErrorType.Client); throw bse; } // Always set the content length, even if it's already set metadata.setContentLength(file.length()); if (metadata.getContentType() == null) { metadata.setContentType(Mimetypes.getInstance().getMimetype(file)); } FileInputStream fileInputStream = null; try { fileInputStream = new FileInputStream(file); metadata.setBceContentSha256( new String(Hex.encodeHex(HashUtils.computeSha256Hash(fileInputStream)))); } catch (Exception e) { throw new BceClientException("Unable to calculate SHA-256 hash", e); } finally { try { if (fileInputStream != null) { fileInputStream.close(); } } catch (Exception e) { // ignore } } try { internalRequest.setContent(new RestartableFileInputStream(file)); } catch (FileNotFoundException e) { throw new BceClientException("Unable to find file to upload", e); } } else { checkNotNull(input, "Either file or inputStream should be set for PutObjectRequest."); if (metadata.getContentLength() < 0) { logger.warn("No content length specified for stream data. Trying to read them all into memory."); List<byte[]> data = this.readAll(input, metadata); internalRequest .setContent(new RestartableMultiByteArrayInputStream(data, metadata.getContentLength())); } else if (input instanceof RestartableInputStream) { internalRequest.setContent((RestartableInputStream) input); } else { internalRequest.setContent(this.wrapRestartableInputStream(input)); } } internalRequest.addHeader(Headers.CONTENT_LENGTH, String.valueOf(metadata.getContentLength())); populateRequestMetadata(internalRequest, metadata); BosResponse response; try { response = this.invokeHttpClient(internalRequest, BosResponse.class); } finally { try { internalRequest.getContent().close(); } catch (Exception e) { logger.warn("Fail to close input stream", e); } } PutObjectResponse result = new PutObjectResponse(); result.setETag(response.getMetadata().getETag()); return result; } /** * Copies a source object to a new destination in Bos. * * @param sourceBucketName The name of the bucket containing the source object to copy. * @param sourceKey The key in the source bucket under which the source object is stored. * @param destinationBucketName The name of the bucket in which the new object will be created. * This can be the same name as the source bucket's. * @param destinationKey The key in the destination bucket under which the new object will be created. * @return A CopyObjectResponse object containing the information returned by Bos for the newly created object. */ public CopyObjectResponse copyObject(String sourceBucketName, String sourceKey, String destinationBucketName, String destinationKey) { return this.copyObject( new CopyObjectRequest(sourceBucketName, sourceKey, destinationBucketName, destinationKey)); } /** * Copies a source object to a new destination in Bos. * * @param request The request object containing all the options for copying an Bos object. * @return A CopyObjectResponse object containing the information returned by Bos for the newly created object. */ public CopyObjectResponse copyObject(CopyObjectRequest request) { checkNotNull(request, "request should not be null."); assertStringNotNullOrEmpty(request.getSourceKey(), "object key should not be null or empty"); InternalRequest internalRequest = this.createRequest(request, HttpMethodName.PUT); String copySourceHeader = "/" + request.getSourceBucketName() + "/" + request.getSourceKey(); copySourceHeader = HttpUtils.normalizePath(copySourceHeader); internalRequest.addHeader(Headers.BCE_COPY_SOURCE, copySourceHeader); if (request.getETag() != null) { internalRequest.addHeader(Headers.BCE_COPY_SOURCE_IF_MATCH, "\"" + request.getETag() + "\""); } ObjectMetadata newObjectMetadata = request.getNewObjectMetadata(); if (newObjectMetadata != null) { internalRequest.addHeader(Headers.BCE_COPY_METADATA_DIRECTIVE, "replace"); populateRequestMetadata(internalRequest, newObjectMetadata); } else { internalRequest.addHeader(Headers.BCE_COPY_METADATA_DIRECTIVE, "copy"); } this.setZeroContentLength(internalRequest); return this.invokeHttpClient(internalRequest, CopyObjectResponse.class); } /** * Deletes the specified object in the specified bucket. * * @param bucketName The name of the Bos bucket containing the object to delete. * @param key The key of the object to delete. */ public void deleteObject(String bucketName, String key) { this.deleteObject(new DeleteObjectRequest(bucketName, key)); } /** * Deletes the specified object in the specified bucket. * * @param request The request object containing all options for deleting a Bos object. */ public void deleteObject(DeleteObjectRequest request) { checkNotNull(request, "request should not be null."); assertStringNotNullOrEmpty(request.getKey(), "object key should not be null or empty"); this.invokeHttpClient(this.createRequest(request, HttpMethodName.DELETE), BosResponse.class); } /** * Initiates a multipart upload and returns an InitiateMultipartUploadResponse * which contains an upload ID. This upload ID associates all the parts in * the specific upload and is used in each of your subsequent uploadPart requests. * You also include this upload ID in the final request to either complete, or abort the multipart * upload request. * * @param bucketName The name of the Bos bucket containing the object to initiate. * @param key The key of the object to initiate. * @return An InitiateMultipartUploadResponse from Bos. */ public InitiateMultipartUploadResponse initiateMultipartUpload(String bucketName, String key) { return this.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, key)); } /** * Initiates a multipart upload and returns an InitiateMultipartUploadResponse * which contains an upload ID. This upload ID associates all the parts in * the specific upload and is used in each of your subsequent uploadPart requests. * You also include this upload ID in the final request to either complete, or abort the multipart * upload request. * * @param request The InitiateMultipartUploadRequest object that specifies all the parameters of this operation. * @return An InitiateMultipartUploadResponse from Bos. */ public InitiateMultipartUploadResponse initiateMultipartUpload(InitiateMultipartUploadRequest request) { checkNotNull(request, "request should not be null."); InternalRequest internalRequest = this.createRequest(request, HttpMethodName.POST); internalRequest.addParameter("uploads", null); this.setZeroContentLength(internalRequest); if (request.getObjectMetadata() != null) { populateRequestMetadata(internalRequest, request.getObjectMetadata()); } return this.invokeHttpClient(internalRequest, InitiateMultipartUploadResponse.class); } /** * Uploads a part in a multipart upload. You must initiate a multipart * upload before you can upload any part. * * @param request The UploadPartRequest object that specifies all the parameters of this operation. * @return An UploadPartResponse from Bos containing the part number and ETag of the new part. */ public UploadPartResponse uploadPart(UploadPartRequest request) { checkNotNull(request, "request should not be null."); checkNotNull(request.getPartSize(), "partSize should not be null"); checkNotNull(request.getPartNumber(), "partNumber should not be null"); if (request.getPartSize() > 5 * 1024 * 1024 * 1024L) { throw new BceClientException( "PartNumber " + request.getPartNumber() + " : Part Size should not be more than 5GB."); } InternalRequest internalRequest = this.createRequest(request, HttpMethodName.PUT); internalRequest.addParameter("uploadId", request.getUploadId()); internalRequest.addParameter("partNumber", String.valueOf(request.getPartNumber())); internalRequest.addHeader(Headers.CONTENT_LENGTH, String.valueOf(request.getPartSize())); InputStream input = request.getInputStream(); MD5DigestCalculatingInputStream md5DigestStream = null; if (request.getMd5Digest() == null) { try { md5DigestStream = new MD5DigestCalculatingInputStream(input); input = md5DigestStream; } catch (NoSuchAlgorithmException e) { logger.warn("Unable to verify data integrity.", e); } } try { internalRequest.setContent(this.wrapRestartableInputStream(input)); BosResponse response = this.invokeHttpClient(internalRequest, BosResponse.class); if (md5DigestStream != null) { byte[] clientSideHash = md5DigestStream.getMd5Digest(); byte[] serverSideHash; try { serverSideHash = Hex.decodeHex(response.getMetadata().getETag().toCharArray()); } catch (Exception e) { throw new BceClientException("Unable to verify integrity of data upload.", e); } if (!Arrays.equals(clientSideHash, serverSideHash)) { throw new BceClientException("Unable to verify integrity of data upload. " + "Client calculated content hash didn't match hash calculated by " + "Baidu BOS. " + "You may need to delete the data stored in Baiddu BOS."); } } UploadPartResponse result = new UploadPartResponse(); result.setETag(response.getMetadata().getETag()); result.setPartNumber(request.getPartNumber()); return result; } finally { if (input != null) { try { input.close(); } catch (Exception e) { } } } } /** * Lists the parts that have been uploaded for a specific multipart upload. * * @param bucketName The name of the bucket containing the multipart upload whose parts are being listed. * @param key The key of the associated multipart upload whose parts are being listed. * @param uploadId The ID of the multipart upload whose parts are being listed. * @return Returns a ListPartsResponse from Bos. */ public ListPartsResponse listParts(String bucketName, String key, String uploadId) { return this.listParts(new ListPartsRequest(bucketName, key, uploadId)); } /** * Lists the parts that have been uploaded for a specific multipart upload. * * @param request The ListPartsRequest object that specifies all the parameters of this operation. * @return Returns a ListPartsResponse from Bos. */ public ListPartsResponse listParts(ListPartsRequest request) { checkNotNull(request, "request should not be null."); InternalRequest interrnalRequest = this.createRequest(request, HttpMethodName.GET); interrnalRequest.addParameter("uploadId", request.getUploadId()); int maxParts = request.getMaxParts(); if (maxParts >= 0) { interrnalRequest.addParameter("maxParts", String.valueOf(maxParts)); } interrnalRequest.addParameter("partNumberMarker", String.valueOf(request.getPartNumberMarker())); ListPartsResponse response = this.invokeHttpClient(interrnalRequest, ListPartsResponse.class); response.setBucketName(request.getBucketName()); return response; } /** * Completes a multipart upload by assembling previously uploaded parts. * * @param bucketName The name of the bucket containing the multipart upload to complete. * @param key The key of the multipart upload to complete. * @param uploadId The ID of the multipart upload to complete. * @param partETags The list of part numbers and ETags to use when completing the multipart upload. * @return A CompleteMultipartUploadResponse from Bos containing the ETag for * the new object composed of the individual parts. */ public CompleteMultipartUploadResponse completeMultipartUpload(String bucketName, String key, String uploadId, List<PartETag> partETags) { return this .completeMultipartUpload(new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags)); } /** * Completes a multipart upload by assembling previously uploaded parts. * * @param bucketName The name of the bucket containing the multipart upload to complete. * @param key The key of the multipart upload to complete. * @param uploadId The ID of the multipart upload to complete. * @param partETags The list of part numbers and ETags to use when completing the multipart upload. * @param metadata Additional metadata instructing Bos how to handle the uploaded data * (e.g. custom user metadata, hooks for specifying content type, etc.). * @return A CompleteMultipartUploadResponse from Bos containing the ETag for * the new object composed of the individual parts. */ public CompleteMultipartUploadResponse completeMultipartUpload(String bucketName, String key, String uploadId, List<PartETag> partETags, ObjectMetadata metadata) { return this.completeMultipartUpload( new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags, metadata)); } /** * Completes a multipart upload by assembling previously uploaded parts. * * @param request The CompleteMultipartUploadRequest object that specifies all the parameters of this operation. * @return A CompleteMultipartUploadResponse from Bos containing the ETag for * the new object composed of the individual parts. */ public CompleteMultipartUploadResponse completeMultipartUpload(CompleteMultipartUploadRequest request) { checkNotNull(request, "request should not be null."); InternalRequest internalRequest = this.createRequest(request, HttpMethodName.POST); internalRequest.addParameter("uploadId", request.getUploadId()); ObjectMetadata metadata = request.getObjectMetadata(); if (metadata != null) { populateRequestMetadata(internalRequest, metadata); } byte[] json = null; List<PartETag> partETags = request.getPartETags(); StringWriter writer = new StringWriter(); try { JsonGenerator jsonGenerator = JsonUtils.jsonGeneratorOf(writer); jsonGenerator.writeStartObject(); jsonGenerator.writeArrayFieldStart("parts"); for (PartETag partETag : partETags) { jsonGenerator.writeStartObject(); jsonGenerator.writeNumberField("partNumber", partETag.getPartNumber()); jsonGenerator.writeStringField("eTag", partETag.getETag()); jsonGenerator.writeEndObject(); } jsonGenerator.writeEndArray(); jsonGenerator.writeEndObject(); jsonGenerator.close(); } catch (IOException e) { throw new BceClientException("Fail to generate json", e); } try { json = writer.toString().getBytes(DEFAULT_ENCODING); } catch (UnsupportedEncodingException e) { throw new BceClientException("Fail to get UTF-8 bytes", e); } internalRequest.addHeader(Headers.CONTENT_LENGTH, String.valueOf(json.length)); internalRequest.addHeader(Headers.CONTENT_TYPE, "application/json"); internalRequest.setContent(RestartableInputStream.wrap(json)); CompleteMultipartUploadResponse response = this.invokeHttpClient(internalRequest, CompleteMultipartUploadResponse.class); response.setBucketName(request.getBucketName()); return response; } /** * Aborts a multipart upload. After a multipart upload is aborted, no * additional parts can be uploaded using that upload ID. The storage * consumed by any previously uploaded parts will be freed. However, if any * part uploads are currently in progress, those part uploads may or may not * succeed. As a result, it may be necessary to abort a given multipart * upload multiple times in order to completely free all storage consumed by * all parts. * * @param bucketName The name of the bucket containing the multipart upload to abort. * @param key The key of the multipart upload to abort. * @param uploadId The ID of the multipart upload to abort. */ public void abortMultipartUpload(String bucketName, String key, String uploadId) { this.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, key, uploadId)); } /** * Aborts a multipart upload. After a multipart upload is aborted, no * additional parts can be uploaded using that upload ID. The storage * consumed by any previously uploaded parts will be freed. However, if any * part uploads are currently in progress, those part uploads may or may not * succeed. As a result, it may be necessary to abort a given multipart * upload multiple times in order to completely free all storage consumed by * all parts. * * @param request The AbortMultipartUploadRequest object that specifies all the parameters of this operation. */ public void abortMultipartUpload(AbortMultipartUploadRequest request) { checkNotNull(request, "request should not be null."); InternalRequest internalRequest = this.createRequest(request, HttpMethodName.DELETE); internalRequest.addParameter("uploadId", request.getUploadId()); this.invokeHttpClient(internalRequest, BosResponse.class); } /** * Lists in-progress multipart uploads. An in-progress multipart upload is a multipart upload that has * been initiated, using the InitiateMultipartUpload request, but has not yet been completed or aborted. * * @param bucketName The name of the bucket containing the uploads to list. * @return A ListMultipartUploadsResponse from Bos. */ public ListMultipartUploadsResponse listMultipartUploads(String bucketName) { return this.listMultipartUploads(new ListMultipartUploadsRequest(bucketName)); } /** * Lists in-progress multipart uploads. An in-progress multipart upload is a multipart upload that has * been initiated, using the InitiateMultipartUpload request, but has not yet been completed or aborted. * * @param request The ListMultipartUploadsRequest object that specifies all the parameters of this operation. * @return A ListMultipartUploadsResponse from Bos. */ public ListMultipartUploadsResponse listMultipartUploads(ListMultipartUploadsRequest request) { checkNotNull(request, "request should not be null."); InternalRequest internalRequest = this.createRequest(request, HttpMethodName.GET); internalRequest.addParameter("uploads", null); String keyMarker = request.getKeyMarker(); if (keyMarker != null) { internalRequest.addParameter("keyMarker", keyMarker); } int maxUploads = request.getMaxUploads(); if (maxUploads >= 0) { internalRequest.addParameter("maxUploads", String.valueOf(maxUploads)); } String delimiter = request.getDelimiter(); if (delimiter != null) { internalRequest.addParameter("delimiter", delimiter); } String prefix = request.getPrefix(); if (prefix != null) { internalRequest.addParameter("prefix", prefix); } ListMultipartUploadsResponse response = this.invokeHttpClient(internalRequest, ListMultipartUploadsResponse.class); response.setBucketName(request.getBucketName()); return response; } /** * Populates the specified request object with the appropriate headers from the ObjectMetadata object. * * @param request The request to populate with headers. * @param metadata The metadata containing the header information to include in the request. */ private static void populateRequestMetadata(InternalRequest request, ObjectMetadata metadata) { if (metadata.getContentType() != null) { request.addHeader(Headers.CONTENT_TYPE, metadata.getContentType()); } if (metadata.getContentMd5() != null) { request.addHeader(Headers.CONTENT_MD5, metadata.getContentMd5()); } if (metadata.getContentEncoding() != null) { request.addHeader(Headers.CONTENT_ENCODING, metadata.getContentEncoding()); } if (metadata.getBceContentSha256() != null) { request.addHeader(Headers.BCE_CONTENT_SHA256, metadata.getBceContentSha256()); } if (metadata.getContentDisposition() != null) { request.addHeader(Headers.CONTENT_DISPOSITION, metadata.getContentDisposition()); } if (metadata.getETag() != null) { request.addHeader(Headers.ETAG, metadata.getETag()); } Map<String, String> userMetadata = metadata.getUserMetadata(); if (userMetadata != null) { for (Entry<String, String> entry : userMetadata.entrySet()) { String key = entry.getKey(); if (key == null) { continue; } String value = entry.getValue(); if (value == null) { value = ""; } if (key.length() + value.length() > 1024 * 32) { throw new BceClientException("MetadataTooLarge"); } request.addHeader(Headers.BCE_USER_METADATA_PREFIX + HttpUtils.normalize(key.trim()), HttpUtils.normalize(value)); } } } /** * Creates and initializes a new request object for the specified Bos resource. This method is responsible * for determining the right way to address resources. * * @param bceRequest The original request, as created by the user. * @param httpMethod The HTTP method to use when sending the request. * @return A new request object, populated with endpoint, resource path, ready for callers to populate * any additional headers or parameters, and execute. */ private InternalRequest createRequest(AbstractBceRequest bceRequest, HttpMethodName httpMethod) { String bucketName = null; String key = null; if (bceRequest instanceof GenericBucketRequest) { bucketName = ((GenericBucketRequest) bceRequest).getBucketName(); } if (bceRequest instanceof GenericObjectRequest) { key = ((GenericObjectRequest) bceRequest).getKey(); } InternalRequest request = new InternalRequest(httpMethod, HttpUtils.appendUri(this.getEndpoint(), URL_PREFIX, bucketName, key)); request.setCredentials(bceRequest.getRequestCredentials()); return request; } private void downloadObjectToFile(BosObject bosObject, File destinationFile, boolean verifyIntegrity) { // attempt to create the parent if it doesn't exist File parentDirectory = destinationFile.getParentFile(); if (parentDirectory != null && !parentDirectory.exists()) { parentDirectory.mkdirs(); } OutputStream outputStream = null; try { outputStream = new BufferedOutputStream(new FileOutputStream(destinationFile)); byte[] buffer = new byte[this.getStreamBufferSize()]; int bytesRead; while ((bytesRead = bosObject.getObjectContent().read(buffer)) > -1) { outputStream.write(buffer, 0, bytesRead); } } catch (IOException e) { try { bosObject.getObjectContent().close(); } catch (IOException abortException) { logger.warn("Couldn't abort stream", abortException); } throw new BceClientException("Unable to write to disk", e); } finally { try { if (outputStream != null) { outputStream.close(); } } catch (Exception e) { } try { bosObject.getObjectContent().close(); } catch (Exception e) { } } if (verifyIntegrity) { byte[] serverSideHash = null; byte[] clientSideHash = null; ObjectMetadata objectMetadata = bosObject.getObjectMetadata(); try { if (objectMetadata.getBceContentSha256() != null) { serverSideHash = Hex.decodeHex(objectMetadata.getBceContentSha256().toCharArray()); clientSideHash = HashUtils.computeSha256Hash(new FileInputStream(destinationFile)); } else if (objectMetadata.getContentMd5() != null) { serverSideHash = Base64.decodeBase64(objectMetadata.getContentMd5().getBytes(DEFAULT_ENCODING)); clientSideHash = HashUtils.computeMd5Hash(new FileInputStream(destinationFile)); } } catch (Exception e) { logger.warn("Unable to verify the integrity of the downloaded file", e); } if (serverSideHash != null && clientSideHash != null && !Arrays.equals(clientSideHash, serverSideHash)) { throw new BceClientException("Integrity verification failed! " + "Client calculated content hash didn't match hash from server. " + "The data stored in '" + destinationFile.getAbsolutePath() + "' may be corrupt."); } } } private List<byte[]> readAll(InputStream input, ObjectMetadata metadata) { List<byte[]> result = Lists.newArrayList(); int bufferSize = this.getStreamBufferSize(); long length = 0; for (;;) { byte[] buffer = new byte[bufferSize]; result.add(buffer); int off = 0; while (off < bufferSize) { int count; try { count = input.read(buffer, off, bufferSize - off); } catch (IOException e) { throw new BceClientException("Fail to read data.", e); } if (count < 0) { metadata.setContentLength(length); return result; } length += count; off += count; } } } private RestartableInputStream wrapRestartableInputStream(InputStream input) { if (input.markSupported()) { return new RestartableResettableInputStream(input); } else { return new RestartableNonResettableInputStream(input, this.getStreamBufferSize()); } } private void setZeroContentLength(InternalRequest req) { req.addHeader(Headers.CONTENT_LENGTH, String.valueOf(0)); } private int getStreamBufferSize() { return ((BosClientConfiguration) this.config).getStreamBufferSize(); } /** * Asserts that the specified parameter value is not <code>null</code> or <code>empty</code> and if it is, * throws an <code>IllegalArgumentException</code> with the specified error message. * * @param parameterValue The parameter value being checked. * @param errorMessage The error message to include in the IllegalArgumentException * if the specified parameter is null. */ private void assertStringNotNullOrEmpty(String parameterValue, String errorMessage) { if (parameterValue == null) { throw new IllegalArgumentException(errorMessage); } if (parameterValue.isEmpty()) { throw new IllegalArgumentException(errorMessage); } } /** * Adds response headers parameters to the request given, if non-null. * * @param request The request to add the response header parameters to. * @param responseHeaders The full set of response headers to add, or null for none. */ private void addResponseHeaderParameters(InternalRequest request, ResponseHeaderOverrides responseHeaders) { if (responseHeaders != null) { if (responseHeaders.getCacheControl() != null) { request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CACHE_CONTROL, responseHeaders.getCacheControl()); } if (responseHeaders.getContentDisposition() != null) { request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_DISPOSITION, responseHeaders.getContentDisposition()); } if (responseHeaders.getContentEncoding() != null) { request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_ENCODING, responseHeaders.getContentEncoding()); } if (responseHeaders.getContentLanguage() != null) { request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_LANGUAGE, responseHeaders.getContentLanguage()); } if (responseHeaders.getContentType() != null) { request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_TYPE, responseHeaders.getContentType()); } if (responseHeaders.getExpires() != null) { request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_EXPIRES, responseHeaders.getExpires()); } } } /** * Converts the specified request object into a URL, containing all the * specified parameters, the specified request endpoint, etc. * * @param request The request to convert into a URL. * @return A new URL representing the specified request. */ private URL convertRequestToUrl(InternalRequest request) { String resourcePath = HttpUtils.normalizePath(request.getUri().getPath()); // Removed the padding "/" that was already added into the request's resource path. if (resourcePath.startsWith("/")) { resourcePath = resourcePath.substring(1); } // Some http client libraries (e.g. Apache HttpClient) cannot handle // consecutive "/"s between URL authority and path components. // So we escape "////..." into "/%2F%2F%2F...", in the same way as how // we treat consecutive "/"s in AmazonS3Client#presignRequest(...) String urlPath = "/" + resourcePath; urlPath = urlPath.replaceAll("(?<=/)/", "%2F"); String urlString = this.config.getEndpoint() + urlPath; boolean firstParam = true; for (String param : request.getParameters().keySet()) { if (firstParam) { urlString += "?"; firstParam = false; } else { urlString += "&"; } String value = request.getParameters().get(param); urlString += param + "=" + HttpUtils.normalize(value); } String authorization = request.getHeaders().get(Headers.AUTHORIZATION); if (authorization != null) { if (firstParam) { urlString += "?"; } else { urlString += "&"; } urlString += "authorization" + "=" + HttpUtils.normalize(authorization); } try { return new URL(urlString); } catch (MalformedURLException e) { throw new BceClientException("Unable to convert request to well formed URL: " + e.getMessage(), e); } } }