Example usage for com.amazonaws.services.s3.model GetObjectRequest setRange

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model GetObjectRequest setRange.

Prototype

public void setRange(long start, long end) 

Source Link

Document

Sets the optional inclusive byte range within the desired object that will be downloaded by this request.

Usage

From source file:c3.ops.priam.backup.RangeReadInputStream.java

License:Apache License

public int read(final byte b[], final int off, final int len) throws IOException {
    //        logger.info(String.format("incoming buf req's size = %d, off = %d, len to read = %d, on file size %d, cur offset = %d path = %s",
    //                b.length, off, len, path.getSize(), offset, path.getRemotePath()));
    final long fileSize = path.getSize();
    if (fileSize > 0 && offset >= fileSize)
        return -1;
    final long firstByte = offset;
    long curEndByte = firstByte + len;
    curEndByte = curEndByte <= fileSize ? curEndByte : fileSize;

    //need to subtract one as the call to getRange is inclusive
    //meaning if you want to download the first 10 bytes of a file, request bytes 0..9
    final long endByte = curEndByte - 1;
    //        logger.info(String.format("start byte = %d, end byte = %d", firstByte, endByte));
    try {/*w w  w  .  ja  v a 2s . c o  m*/
        Integer cnt = new RetryableCallable<Integer>() {
            public Integer retriableCall() throws IOException {
                GetObjectRequest req = new GetObjectRequest(bucketName, path.getRemotePath());
                req.setRange(firstByte, endByte);
                S3ObjectInputStream is = null;
                try {
                    is = s3Client.getObject(req).getObjectContent();

                    byte[] readBuf = new byte[4092];
                    int rCnt;
                    int readTotal = 0;
                    int incomingOffet = off;
                    while ((rCnt = is.read(readBuf, 0, readBuf.length)) >= 0) {
                        System.arraycopy(readBuf, 0, b, incomingOffet, rCnt);
                        readTotal += rCnt;
                        incomingOffet += rCnt;
                        //                            logger.info("    local read cnt = " + rCnt + "Current Thread Name = "+Thread.currentThread().getName());
                    }
                    if (readTotal == 0 && rCnt == -1)
                        return -1;
                    offset += readTotal;
                    return Integer.valueOf(readTotal);
                } finally {
                    IOUtils.closeQuietly(is);
                }
            }
        }.call();
        //            logger.info("read cnt = " + cnt);
        return cnt.intValue();
    } catch (Exception e) {
        String msg = String.format("failed to read offset range %d-%d of file %s whose size is %d", firstByte,
                endByte, path.getRemotePath(), path.getSize());
        throw new IOException(msg, e);
    }
}

---

From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java

License:Open Source License

@Override
public GetObjectExtendedResponseType getObjectExtended(final GetObjectExtendedType request) throws S3Exception {
    User requestUser = getRequestUser(request);
    OsgInternalS3Client internalS3Client = null;

    Boolean getMetaData = request.getGetMetaData();
    Long byteRangeStart = request.getByteRangeStart();
    Long byteRangeEnd = request.getByteRangeEnd();
    Date ifModifiedSince = request.getIfModifiedSince();
    Date ifUnmodifiedSince = request.getIfUnmodifiedSince();
    String ifMatch = request.getIfMatch();
    String ifNoneMatch = request.getIfNoneMatch();

    GetObjectRequest getRequest = new GetObjectRequest(request.getBucket(), request.getKey());
    if (byteRangeStart == null) {
        byteRangeStart = 0L;//from  w w  w . ja  va  2  s  . c om
    }
    if (byteRangeEnd != null) {
        getRequest.setRange(byteRangeStart, byteRangeEnd);
    }
    if (getMetaData != null) {
        //Get object metadata
    }
    if (ifModifiedSince != null) {
        getRequest.setModifiedSinceConstraint(ifModifiedSince);
    }
    if (ifUnmodifiedSince != null) {
        getRequest.setUnmodifiedSinceConstraint(ifUnmodifiedSince);
    }
    if (ifMatch != null) {
        List matchList = new ArrayList();
        matchList.add(ifMatch);
        getRequest.setMatchingETagConstraints(matchList);
    }
    if (ifNoneMatch != null) {
        List nonMatchList = new ArrayList();
        nonMatchList.add(ifNoneMatch);
        getRequest.setNonmatchingETagConstraints(nonMatchList);
    }
    try {
        internalS3Client = getS3Client(requestUser);
        AmazonS3Client s3Client = internalS3Client.getS3Client();
        S3Object response = s3Client.getObject(getRequest);

        GetObjectExtendedResponseType reply = request.getReply();
        populateResponseMetadata(reply, response.getObjectMetadata());
        reply.setDataInputStream(response.getObjectContent());
        reply.setByteRangeStart(request.getByteRangeStart());
        reply.setByteRangeEnd(request.getByteRangeEnd());
        return reply;
    } catch (AmazonServiceException e) {
        LOG.debug("Error from backend", e);
        throw S3ExceptionMapper.fromAWSJavaSDK(e);
    }
}

---

From source file:com.ge.predix.sample.blobstore.repository.BlobstoreService.java

License:Apache License

/**
 * Get the Blob from the binded bucket//from  w  ww .  j av a2s . c o  m
 *
 * @param fileName String
 * @throws Exception
 */
public InputStream get(String fileName, String range) throws Exception {

    if (range != null && !range.isEmpty()) {
        String[] r = range.split(":");
        if (r.length != 2) {
            throw new Exception("Invalid range format");
        }

        try {
            long start = Long.parseLong(r[0]);
            long end = Long.parseLong(r[1]);

            GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucket, fileName);
            rangeObjectRequest.setRange(start, end);
            S3Object objectPortion = s3Client.getObject(rangeObjectRequest);

            InputStream objectData = objectPortion.getObjectContent();

            return objectData;

        } catch (NumberFormatException e) {
            throw new Exception("Invalid range specified ", e);
        }
    } else {
        try {
            S3Object object = s3Client.getObject(new GetObjectRequest(bucket, fileName));
            InputStream objectData = object.getObjectContent();

            return objectData;

        } catch (Exception e) {
            log.error("Exception Occurred in get(): " + e.getMessage());
            throw e;
        }
    }

}

---

From source file:com.treasure_data.td_import.source.S3Source.java

License:Apache License

@Override
public InputStream getInputStream() throws IOException {
    LOG.info(String.format("get s3 file by client %s: bucket=%s, key=%s", client, bucket, key));
    GetObjectRequest req = new GetObjectRequest(bucket, key);
    req.setRange(0, size);
    S3Object object = client.getObject(req);

    if (object != null) {
        return object.getObjectContent();
    } else {/*ww w  . j  a v  a  2 s  .  c o  m*/
        throw new IOException("s3 file is null.");
    }
}

---

From source file:dbs.Getfroms3.java

public static void main(String[] args) throws IOException {

    AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(Regions.AP_SOUTH_1)
            .withCredentials(new AWSStaticCredentialsProvider(credentials)).build();
    //AmazonS3 s3Client = new AmazonS3Client(new ProfileCredentialsProvider("inhack2hire28"));
    try {/*from  w  w  w  .  j  av a 2  s  .c o m*/
        System.out.println("Downloading an object");
        S3Object s3object = s3Client.getObject(new GetObjectRequest(bucketName, key));
        System.out.println("Content-Type: " + s3object.getObjectMetadata().getContentType());
        displayTextInputStream(s3object.getObjectContent());

        // Get a range of bytes from an object.

        GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucketName, key);
        rangeObjectRequest.setRange(0, 10);
        S3Object objectPortion = s3Client.getObject(rangeObjectRequest);

        System.out.println("Printing bytes retrieved.");
        displayTextInputStream(objectPortion.getObjectContent());

    } catch (AmazonServiceException ase) {
        System.out.println("Caught an AmazonServiceException, which" + " means your request made it "
                + "to Amazon S3, but was rejected with an error response" + " for some reason.");
        System.out.println("Error Message:    " + ase.getMessage());
        System.out.println("HTTP Status Code: " + ase.getStatusCode());
        System.out.println("AWS Error Code:   " + ase.getErrorCode());
        System.out.println("Error Type:       " + ase.getErrorType());
        System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        System.out.println("Caught an AmazonClientException, which means" + " the client encountered "
                + "an internal error while trying to " + "communicate with S3, "
                + "such as not being able to access the network.");
        System.out.println("Error Message: " + ace.getMessage());
    }
}

---

From source file:io.minio.awssdk.tests.S3TestUtils.java

License:Apache License

void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey, String expectedMD5, int start,
        int length) throws Exception, IOException {
    GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, keyName).withSSECustomerKey(sseKey);

    if (start >= 0 && length >= 0) {
        getObjectRequest.setRange(start, start + length - 1);
    }//from  w  w  w.  j av a  2s  .  co  m

    S3Object s3Object = s3Client.getObject(getObjectRequest);

    int size = 0;
    int c;

    S3ObjectInputStream input = s3Object.getObjectContent();

    ByteArrayOutputStream output = new ByteArrayOutputStream();
    String data = "";
    while ((c = input.read()) != -1) {
        output.write((byte) c);
        size++;
    }

    if (length >= 0 && size != length) {
        throw new Exception(
                "downloaded object has unexpected size, expected: " + length + ", received: " + size);
    }

    String calculatedMD5 = Utils.getBufferMD5(output.toByteArray());

    if (!expectedMD5.equals("") && !calculatedMD5.equals(expectedMD5)) {
        throw new Exception("downloaded object has unexpected md5sum, expected: " + expectedMD5 + ", found: "
                + calculatedMD5);

    }
}

---

From source file:jp.inc.forrest.aws.swf.AverageCalculatorActivitiesImpl.java

License:Open Source License

@Override
public int computeSumForChunk(String bucketName, String filename, int chunkNumber, int chunkSize)
        throws IOException {
    int sum = 0;/*from  w w  w  .  ja va 2  s  . com*/
    int from = chunkNumber * chunkSize;
    int to = from + chunkSize;

    int offset = chunkNumber * chunkSize * ROW_SIZE;
    int bytesToRead = chunkSize * ROW_SIZE;

    // Create a request to download content for computing the sum for this
    // chunk
    GetObjectRequest getRequest = new GetObjectRequest(bucketName, filename);
    getRequest.setRange(offset, offset + bytesToRead - 1);

    // Download content
    S3Object obj = storage.getObject(getRequest);
    InputStream inputStream = obj.getObjectContent();
    InputStreamReader inputStreamReader = null;
    BufferedReader reader = null;
    try {
        String line = null;
        inputStreamReader = new InputStreamReader(inputStream);
        reader = new BufferedReader(inputStreamReader);

        // Compute sum for downloaded content
        while ((line = reader.readLine()) != null) {
            sum += Integer.parseInt(line);
        }
    } finally {
        if (reader != null)
            reader.close();
        if (inputStreamReader != null)
            inputStreamReader.close();
        if (inputStream != null)
            inputStream.close();
    }

    System.out.printf("Sum from '%d' to '%d' is: '%d'\n", from + 1, to, sum);
    return sum;
}

---

From source file:org.apache.beam.sdk.io.aws.s3.S3ReadableSeekableByteChannel.java

License:Apache License

@Override
public int read(ByteBuffer destinationBuffer) throws IOException {
    if (!isOpen()) {
        throw new ClosedChannelException();
    }/*from  w w  w .j  a v  a  2 s. c  om*/
    if (!destinationBuffer.hasRemaining()) {
        return 0;
    }
    if (position == contentLength) {
        return -1;
    }

    if (s3Object == null) {
        GetObjectRequest request = new GetObjectRequest(path.getBucket(), path.getKey());
        request.setSSECustomerKey(options.getSSECustomerKey());
        if (position > 0) {
            request.setRange(position, contentLength);
        }
        try {
            s3Object = amazonS3.getObject(request);
        } catch (AmazonClientException e) {
            throw new IOException(e);
        }
        s3ObjectContentChannel = Channels
                .newChannel(new BufferedInputStream(s3Object.getObjectContent(), 1024 * 1024));
    }

    int totalBytesRead = 0;
    int bytesRead = 0;

    do {
        totalBytesRead += bytesRead;
        try {
            bytesRead = s3ObjectContentChannel.read(destinationBuffer);
        } catch (AmazonClientException e) {
            // TODO replace all catch AmazonServiceException with client exception
            throw new IOException(e);
        }
    } while (bytesRead > 0);

    position += totalBytesRead;
    return totalBytesRead;
}

---

From source file:org.apache.hadoop.fs.s3a.S3AInputStream.java

License:Apache License

private synchronized void reopen(long pos) throws IOException {
    if (wrappedStream != null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Aborting old stream to open at pos " + pos);
        }/*from www. j a va 2 s.  c o m*/
        wrappedStream.abort();
    }

    LOG.info("Actually opening file " + key + " at pos " + pos);

    GetObjectRequest request = new GetObjectRequest(bucket, key);
    request.setRange(pos, contentLength - 1);

    wrappedObject = client.getObject(request);
    wrappedStream = wrappedObject.getObjectContent();

    if (wrappedStream == null) {
        throw new IOException("Null IO stream");
    }

    this.pos = pos;
}

---

From source file:org.caboclo.clients.AmazonClient.java

License:Open Source License

@Override
public MultiPartDownload initializeDownload(File file, RemoteFile remoteFile) throws IOException {

    long start = 0;
    long end = Constants.CHUNK_DOWNLOAD_SIZE - 1;
    String child = remoteFile.getPath();

    GetObjectRequest rangeObjectRequest = new GetObjectRequest(getBucketName(), child);

    rangeObjectRequest.setRange(start, end);

    S3Object obj = s3.getObject(rangeObjectRequest);

    BufferedInputStream bis = new BufferedInputStream(obj.getObjectContent());

    writeStreamToFile(bis, file);/*from w  w  w .  ja  va  2  s  . c  o  m*/

    MultiPartDownload mpd = new MultiPartDownload(file, remoteFile, end);

    return mpd;
}

---

• «
• 1
• 2
• »