Example usage for com.amazonaws.services.s3.model PutObjectRequest setGeneralProgressListener

List of usage examples for com.amazonaws.services.s3.model PutObjectRequest setGeneralProgressListener

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. PutObjectRequest
  6. setGeneralProgressListener

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model PutObjectRequest setGeneralProgressListener.

Prototype

public void setGeneralProgressListener(ProgressListener progressListener)

Source Link

Document

Sets the optional progress listener for receiving updates about the progress of the request.

Usage

From source file:br.com.ingenieux.mojo.aws.util.BeanstalkerS3Client.java

License:Apache License

@Override
public PutObjectResult putObject(PutObjectRequest req) throws AmazonClientException, AmazonServiceException {
    if (!multipartUpload) {
        return super.putObject(req);
    }// ww  w .  java  2 s .  co  m

    final long contentLen = TransferManagerUtils.getContentLength(req);

    String tempFilename = req.getKey() + ".tmp";
    String origFilename = req.getKey();

    req.setKey(tempFilename);

    XProgressListener progressListener = new XProgressListener();

    req.setGeneralProgressListener(new ProgressListenerChain(progressListener));

    progressListener.setContentLen(contentLen);
    progressListener.setUpload(transferManager.upload(req));
    progressListener.setSilentUpload(silentUpload);

    try {
        progressListener.getUpload().waitForCompletion();
    } catch (InterruptedException e) {
        throw new AmazonClientException(e.getMessage(), e);
    }

    CopyObjectRequest copyReq = new CopyObjectRequest(req.getBucketName(), tempFilename, req.getBucketName(),
            origFilename);

    copyObject(copyReq);

    deleteObject(new DeleteObjectRequest(req.getBucketName(), tempFilename));

    return null;
}

From source file:com.streamsets.datacollector.lib.emr.S3Manager.java

License:Apache License

String uploadToS3(String name, File file) throws IOException {
    long start = System.currentTimeMillis();
    long fileLength = file.length() / (1000 * 1000);
    String bucket = getBucket(pipelineEmrConfigs.getS3StagingUri());
    String path = getPath(pipelineEmrConfigs.getS3StagingUri()) + "/" + pipelineId + "/" + uniquePrefix;
    String s3Uri = "s3://" + bucket + "/" + path + "/" + name;
    try {// w w  w  .j  a v  a  2s .  c o m

        // Upload
        PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, path + "/" + name, file);
        putObjectRequest.setGeneralProgressListener(new ProgressListener() {
            long counter;
            long tick = -1;

            @Override
            public void progressChanged(ProgressEvent progressEvent) {
                counter += progressEvent.getBytesTransferred();
                if (counter / (100 * 1000000) > tick) {
                    tick++;
                    LOG.debug("Uploading '{}' {}/{} MB, {} secs", s3Uri, counter / (1000 * 1000), fileLength,
                            (System.currentTimeMillis() - start) / 1000);
                }
            }
        });

        getS3TransferManager().upload(putObjectRequest).waitForCompletion();

        LOG.info("Uploaded file at: {}", s3Uri);
        return s3Uri;
    } catch (SdkBaseException | InterruptedException ex) {
        throw new IOException(ex);
    }
}

From source file:io.dockstore.common.FileProvisioning.java

License:Apache License

public void provisionOutputFile(FileInfo file, String cwlOutputPath) {
    File sourceFile = new File(cwlOutputPath);
    long inputSize = sourceFile.length();
    if (file.getUrl().startsWith("s3://")) {
        AmazonS3 s3Client = FileProvisioning.getAmazonS3Client(config);
        String trimmedPath = file.getUrl().replace("s3://", "");
        List<String> splitPathList = Lists.newArrayList(trimmedPath.split("/"));
        String bucketName = splitPathList.remove(0);

        PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, Joiner.on("/").join(splitPathList),
                sourceFile);/*w w  w.j  ava  2  s . c o m*/
        putObjectRequest.setGeneralProgressListener(new ProgressListener() {
            ProgressPrinter printer = new ProgressPrinter();
            long runningTotal = 0;

            @Override
            public void progressChanged(ProgressEvent progressEvent) {
                if (progressEvent.getEventType() == ProgressEventType.REQUEST_BYTE_TRANSFER_EVENT) {
                    runningTotal += progressEvent.getBytesTransferred();
                }
                printer.handleProgress(runningTotal, inputSize);
            }
        });
        try {
            s3Client.putObject(putObjectRequest);
        } finally {
            System.out.println();
        }
    } else {
        try {
            FileSystemManager fsManager;
            // trigger a copy from the URL to a local file path that's a UUID to avoid collision
            fsManager = VFS.getManager();
            // check for a local file path
            FileObject dest = fsManager.resolveFile(file.getUrl());
            FileObject src = fsManager.resolveFile(sourceFile.getAbsolutePath());
            copyFromInputStreamToOutputStream(src.getContent().getInputStream(), inputSize,
                    dest.getContent().getOutputStream());
        } catch (IOException e) {
            throw new RuntimeException("Could not provision output files", e);
        }
    }
}

From source file:msv_upload_tool.FXMLDocumentController.java

private void uploadObject() {

    final Long max = file.length();

    task = new Task<Void>() {
        @Override/*w ww .ja  va2 s.  c  o  m*/
        protected Void call() {

            boolean doLoop = true;
            long total = 0;

            while (doLoop) {

                lock.readLock().lock();

                try {
                    total = totalBytes;
                } finally {
                    lock.readLock().unlock();
                }

                updateProgress(total, max);
                if (total == max)
                    doLoop = false;

                try {
                    Thread.sleep(50); //1000 milliseconds is one second.
                } catch (InterruptedException ex) {
                    Thread.currentThread().interrupt();
                }

            }

            updateProgress(-1, max);

            this.succeeded();
            return null;

        }
    };

    uploadProgress.progressProperty().bind(task.progressProperty());
    task.setOnSucceeded(new EventHandler() {

        @Override
        public void handle(Event event) {

            label.setText("");

            label2.setText("");
            button.setDisable(true);
            button2.setDisable(false);

        }

    });

    Thread th = new Thread(task);

    th.setDaemon(true);

    //disable the buttons
    button.setDisable(true);
    button2.setDisable(true);

    th.start();

    String existingBucketName = "mstargeneralfiles";
    String keyName = "duh/" + file.getName();
    String filePath = file.getAbsolutePath();

    TransferManager tm = new TransferManager(new ProfileCredentialsProvider());

    // For more advanced uploads, you can create a request object 
    // and supply additional request parameters (ex: progress listeners,
    // canned ACLs, etc.)
    PutObjectRequest request = new PutObjectRequest(existingBucketName, keyName, new File(filePath));

    // You can ask the upload for its progress, or you can 
    // add a ProgressListener to your request to receive notifications 
    // when bytes are transferred.
    request.setGeneralProgressListener(new ProgressListener() {

        @Override
        public void progressChanged(ProgressEvent progressEvent) {

            System.out.println(progressEvent.toString());

            lock.writeLock().lock();

            try {
                totalBytes += progressEvent.getBytesTransferred();
            } finally {
                lock.writeLock().unlock();
            }

        }
    });

    // TransferManager processes all transfers asynchronously, 
    // so this call will return immediately.
    Upload upload = tm.upload(request);

}

From source file:org.apache.hadoop.fs.s3a.S3AFastOutputStream.java

License:Apache License

private void putObject() throws IOException {
    LOG.debug("Executing regular upload for bucket '{}' key '{}'", bucket, key);
    final ObjectMetadata om = createDefaultMetadata();
    final int size = buffer.size();
    om.setContentLength(size);//ww w.  j a  v a2s  . co  m
    final PutObjectRequest putObjectRequest = fs.newPutObjectRequest(key, om,
            new ByteArrayInputStream(buffer.toByteArray()));
    putObjectRequest.setGeneralProgressListener(progressListener);
    ListenableFuture<PutObjectResult> putObjectResult = executorService.submit(new Callable<PutObjectResult>() {
        @Override
        public PutObjectResult call() throws Exception {
            fs.incrementPutStartStatistics(size);
            return client.putObject(putObjectRequest);
        }
    });
    //wait for completion
    try {
        putObjectResult.get();
    } catch (InterruptedException ie) {
        LOG.warn("Interrupted object upload: {}", ie, ie);
        Thread.currentThread().interrupt();
    } catch (ExecutionException ee) {
        throw extractException("regular upload", key, ee);
    }
}

From source file:org.apache.hadoop.fs.s3r.S3RFastOutputStream.java

License:Apache License

private void putObject() throws IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Executing regular upload for bucket '{}' key '{}'", bucket, key);
    }/* w w w .j  a  v  a  2s . c o m*/
    final ObjectMetadata om = createDefaultMetadata();
    om.setContentLength(buffer.size());
    final PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key,
            new ByteArrayInputStream(buffer.toByteArray()), om);
    putObjectRequest.setCannedAcl(cannedACL);
    putObjectRequest.setGeneralProgressListener(progressListener);
    ListenableFuture<PutObjectResult> putObjectResult = executorService.submit(new Callable<PutObjectResult>() {
        @Override
        public PutObjectResult call() throws Exception {
            return client.putObject(putObjectRequest);
        }
    });
    //wait for completion
    try {
        putObjectResult.get();
    } catch (InterruptedException ie) {
        LOG.warn("Interrupted object upload:" + ie, ie);
        Thread.currentThread().interrupt();
    } catch (ExecutionException ee) {
        throw new IOException("Regular upload failed", ee.getCause());
    }
}

From source file:org.springfield.lou.servlet.LouServlet.java

License:Open Source License

private String handleFileUpload(HttpServletRequest request) {
    System.out.println("HANDLE FILE UPLOAD");
    try {//from  w  w  w  .  j a  va  2  s.  c om
        String targetid = request.getParameter("targetid");
        System.out.println("TARGETID UPLOAD=" + targetid);
        String screenid = request.getParameter("screenid");
        String cfilename = request.getParameter("cfilename");
        System.out.println("CFILENAME=" + cfilename);
        String cfilesize = request.getParameter("cfilesize");
        System.out.println("CFILESIZE=" + cfilesize);

        Html5ApplicationInterface app = null;
        String url = request.getRequestURI();
        int pos = url.indexOf("/domain/");
        if (pos != -1) {
            String tappname = url.substring(pos);
            app = ApplicationManager.instance().getApplication(tappname);
        }
        Screen eventscreen = app.getScreen(screenid);

        if (eventscreen == null)
            return null;

        String method = eventscreen.getModel()
                .getProperty("/screen['upload']/target['" + targetid + "']/method");
        System.out.println("METHOD=" + method);

        String destpath = eventscreen.getModel()
                .getProperty("/screen['upload']/target['" + targetid + "']/destpath");
        System.out.println("DESTPATH=" + destpath + " T=" + targetid);
        if (destpath == null || destpath.equals("")) {
            setUploadError(eventscreen, targetid, "destpath not set");
            return null;
        }

        String destname_prefix = eventscreen.getModel()
                .getProperty("/screen['upload']/target['" + targetid + "']/destname_prefix");
        if (destname_prefix == null || destname_prefix.equals("")) {
            setUploadError(eventscreen, targetid, "destname_prefix not set");
            return null;
        }

        String filetype = eventscreen.getModel()
                .getProperty("/screen['upload']/target['" + targetid + "']/filetype");
        if (filetype == null || filetype.equals("")) {
            setUploadError(eventscreen, targetid, "filetype not set");
            return null;
        }

        String fileext = eventscreen.getModel()
                .getProperty("/screen['upload']/target['" + targetid + "']/fileext");
        if (fileext == null || fileext.equals("")) {
            setUploadError(eventscreen, targetid, "fileext not set");
            return null;
        }

        String checkupload = eventscreen.getModel()
                .getProperty("/screen['upload']/target['" + targetid + "']/checkupload");
        if (checkupload == null || checkupload.equals("")) {
            setUploadError(eventscreen, targetid, "checkupload not set");
            return null;
        }

        String storagehost = eventscreen.getModel()
                .getProperty("/screen['upload']/target['" + targetid + "']/storagehost");
        if (storagehost == null || storagehost.equals("")) {
            setUploadError(eventscreen, targetid, "storagehost not set");
            return null;
        }

        String destname_type = eventscreen.getModel()
                .getProperty("/screen['upload']/target['" + targetid + "']/destname_type");
        if (destname_type == null || destname_type.equals("")) {
            setUploadError(eventscreen, targetid, "destname_type not set");
            return null;
        }

        String publicpath = eventscreen.getModel()
                .getProperty("/screen['upload']/target['" + targetid + "']/publicpath");
        if (publicpath == null || publicpath.equals("")) {
            setUploadError(eventscreen, targetid, "publicpath not set");
            return null;
        }

        // here we can check if its a valid upload based on filename and other specs and kill if needed, also map real extension 

        fileext = getValidExtension(fileext, cfilename);
        if (fileext == null)
            return null; // kill the request its not a valid format

        if (method.equals("s3amazon")) {
            String bucketname = eventscreen.getModel()
                    .getProperty("/screen['upload']/target['" + targetid + "']/bucketname");
            if (bucketname == null || bucketname.equals("")) {
                setUploadError(eventscreen, targetid, "bucketname not set");
                return null;
            }

            AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
                    .withCredentials(new EnvironmentVariableCredentialsProvider()).build();
            String filename = "unknown";
            int storageport = 22;

            if (destname_type.equals("epoch")) {
                filename = destpath + destname_prefix + "" + new Date().getTime();
            }

            String publicurl = publicpath + bucketname + "/" + filename + "." + fileext;

            FsPropertySet ps = new FsPropertySet(); // we will use this to send status reports back
            ps.setProperty("action", "start");
            ps.setProperty("progress", "0");
            ps.setProperty("cfilename", cfilename);
            ps.setProperty("url", publicurl);
            eventscreen.getModel().setProperties("/screen/upload/" + targetid, ps);

            try {
                InputStream inst = request.getInputStream();
                int read = 0;
                int readtotal = 0;
                int b;
                while ((b = inst.read()) != 44) {
                    // skip the base64 tagline, not sure how todo this better
                }

                Base64InputStream b64i = new Base64InputStream(inst);

                //System.out.println("Uploading a new object to S3 from a stream "+bucketname+"/"+filename+"."+fileext);

                ObjectMetadata metadata = new ObjectMetadata();
                metadata.setContentType(filetype + "/" + fileext);

                PutObjectRequest or = new PutObjectRequest(bucketname, filename + "." + fileext, b64i,
                        metadata);

                or.setGeneralProgressListener(new UploadProgressListener(eventscreen.getModel(), publicurl,
                        cfilename, cfilesize, targetid));
                s3Client.putObject(or);

            } catch (AmazonServiceException ase) {
                ase.printStackTrace();
            }
            ps.setProperty("action", "done");
            ps.setProperty("progress", "100");
            ps.setProperty("cfilename", cfilename);
            ps.setProperty("url", publicurl);

            eventscreen.getModel().setProperties("/screen/upload/" + targetid, ps);
            return bucketname + "/" + filename + "." + fileext;

        } else if (method.equals("scp")) {
            String pemfile = eventscreen.getModel()
                    .getProperty("/screen['upload']/target['" + targetid + "']/pemfile");
            if (destpath == null || destpath.equals("")) {
                setUploadError(eventscreen, targetid, "destpath not set");
                return null;
            }

            String storagename = eventscreen.getModel()
                    .getProperty("/screen['upload']/target['" + targetid + "']/storagename");
            if (storagename == null || storagehost.equals("")) {
                setUploadError(eventscreen, targetid, "storagename not set");
                return null;
            }

            String filename = "unknown";
            int storageport = 22;

            if (destname_type.equals("epoch")) {
                filename = destname_prefix + "" + new Date().getTime();
            }

            String publicurl = publicpath + filename + "." + fileext;

            FsPropertySet ps = new FsPropertySet(); // we will use this to send status reports back
            ps.setProperty("action", "start");
            ps.setProperty("progress", "0");
            ps.setProperty("url", publicurl);
            eventscreen.getModel().setProperties("/screen/upload/" + targetid, ps);

            JSch jsch = new JSch();
            jsch.addIdentity(pemfile);
            jsch.setConfig("StrictHostKeyChecking", "no");
            Session session = jsch.getSession(storagename, storagehost, storageport);
            session.connect();
            Channel channel = session.openChannel("sftp");

            channel.connect();
            ChannelSftp channelSftp = (ChannelSftp) channel;
            channelSftp.cd(destpath);

            InputStream inst = request.getInputStream();
            int read = 0;
            int readtotal = 0;
            int b;
            while ((b = inst.read()) != 44) {
                // skip the base64 tagline, not sure how todo this better
            }
            Base64InputStream b64i = new Base64InputStream(inst);

            channelSftp.put(b64i, filename + "." + fileext);

            ps.setProperty("action", "done");
            ps.setProperty("progress", "100");
            ps.setProperty("url", publicurl);
            eventscreen.getModel().setProperties("/screen/upload/" + targetid, ps);
            return filename + "." + fileext;
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
    return null;
}