Example usage for org.apache.commons.io IOUtils copyLarge

List of usage examples for org.apache.commons.io IOUtils copyLarge

  1. HOME
  2. Java
  3. org.apache.commons
  4. org.apache.commons.io.*
  5. IOUtils
  6. copyLarge

Introduction

In this page you can find the example usage for org.apache.commons.io IOUtils copyLarge.

Prototype

public static long copyLarge(Reader input, Writer output) throws IOException

Source Link

Document

Copy chars from a large (over 2GB) Reader to a Writer.

Usage

From source file:ch.cyberduck.core.openstack.SwiftLargeUploadWriteFeatureTest.java

@Test
public void testWriteZeroLength() throws Exception {
    final Host host = new Host(new SwiftProtocol(), "identity.api.rackspacecloud.com",
            new Credentials(System.getProperties().getProperty("rackspace.key"),
                    System.getProperties().getProperty("rackspace.secret")));
    final SwiftSession session = new SwiftSession(host);
    session.open(new DisabledHostKeyCallback());
    session.login(new DisabledPasswordStore(), new DisabledLoginCallback(), new DisabledCancelCallback());
    final SwiftRegionService regionService = new SwiftRegionService(session);
    final SwiftLargeUploadWriteFeature feature = new SwiftLargeUploadWriteFeature(session, regionService,
            new SwiftSegmentService(session, ".segments-test/"));
    final Path container = new Path("test.cyberduck.ch", EnumSet.of(Path.Type.directory, Path.Type.volume));
    final byte[] content = RandomUtils.nextBytes(0);
    final TransferStatus status = new TransferStatus();
    status.setLength(-1L);/* www  .  ja  va2 s.  c  om*/
    final Path file = new Path(container, new AlphanumericRandomStringService().random(),
            EnumSet.of(Path.Type.file));
    final HttpResponseOutputStream<List<StorageObject>> out = feature.write(file, status,
            new DisabledConnectionCallback());
    final ByteArrayInputStream in = new ByteArrayInputStream(content);
    assertEquals(content.length, IOUtils.copyLarge(in, out));
    in.close();
    out.close();
    assertNotNull(out.getStatus());
    assertTrue(new DefaultFindFeature(session).find(file));
    final byte[] compare = new byte[content.length];
    final InputStream stream = new SwiftReadFeature(session, regionService).read(file,
            new TransferStatus().length(content.length), new DisabledConnectionCallback());
    IOUtils.readFully(stream, compare);
    stream.close();
    assertArrayEquals(content, compare);
    new SwiftDeleteFeature(session).delete(Collections.singletonList(file), new DisabledLoginCallback(),
            new Delete.DisabledCallback());
}

From source file:com.manydesigns.elements.blobs.BlobManager.java

public Blob updateBlob(String code, InputStream sourceStream, String characterEncoding) throws IOException {
    ensureValidCode(code);/*  w  w  w  .  j av  a2s .c o  m*/

    Blob blob = loadBlob(code);
    File dataFile = blob.getDataFile();

    // copy the data
    long size = IOUtils.copyLarge(sourceStream, new FileOutputStream(dataFile));

    // set and save the metadata
    blob.setSize(size);
    blob.setCharacterEncoding(characterEncoding);
    blob.saveMetaProperties();

    return blob;
}

From source file:de.tudarmstadt.ukp.dkpro.core.api.datasets.internal.actions.Explode.java

private void extract7z(ActionDescription aAction, Path aCachedFile, Path aTarget)
        throws IOException, RarException {
    // We always extract archives into a subfolder. Figure out the name of the folder.
    String base = getBase(aCachedFile.getFileName().toString());

    Map<String, Object> cfg = aAction.getConfiguration();
    int strip = cfg.containsKey("strip") ? (int) cfg.get("strip") : 0;

    AntFileFilter filter = new AntFileFilter(coerceToList(cfg.get("includes")),
            coerceToList(cfg.get("excludes")));

    try (SevenZFile archive = new SevenZFile(aCachedFile.toFile())) {
        SevenZArchiveEntry entry = archive.getNextEntry();
        while (entry != null) {
            String name = stripLeadingFolders(entry.getName(), strip);

            if (name == null) {
                // Stripped to null - nothing left to extract - continue;
                continue;
            }/* w  w w  .  j  av  a2s.  co  m*/

            if (filter.accept(name)) {
                Path out = aTarget.resolve(base).resolve(name);
                if (entry.isDirectory()) {
                    Files.createDirectories(out);
                } else {
                    Files.createDirectories(out.getParent());
                    try (OutputStream os = Files.newOutputStream(out)) {
                        InputStream is = new SevenZEntryInputStream(archive, entry);
                        IOUtils.copyLarge(is, os);
                    }
                }
            }

            entry = archive.getNextEntry();
        }
    }
}

From source file:it.openutils.mgnlaws.magnolia.datastore.S3DataStore.java

/**
 * {@inheritDoc}/*from   www  .  j av a  2  s.  co  m*/
 */
public DataRecord addRecord(InputStream input) throws DataStoreException {
    File temporary = null;
    try {
        temporary = newTemporaryFile();
        DataIdentifier tempId = new DataIdentifier(temporary.getName());
        usesIdentifier(tempId);
        // Copy the stream to the temporary file and calculate the
        // stream length and the message digest of the stream
        MessageDigest digest = MessageDigest.getInstance(DIGEST);
        OutputStream output = new DigestOutputStream(new FileOutputStream(temporary), digest);
        try {
            IOUtils.copyLarge(input, output);
        } finally {
            IOUtils.closeQuietly(output);
        }
        DataIdentifier identifier = new DataIdentifier(digest.digest());
        // File file;
        String tmpKey = PREFIX_TMP + identifier.toString();
        String key = getKey(identifier);

        if (!objectExists(identifier)) {
            Upload upload = transferManager.upload(bucket, tmpKey, temporary);
            try {
                AmazonClientException e;
                if ((e = upload.waitForException()) != null && upload.getState() != TransferState.Completed) {
                    throw new DataStoreException("Error uploading file to s3", e);
                }
            } catch (InterruptedException e) {
                throw new DataStoreException("Upload interrupted", e);
            }
        }

        S3DataRecord record;
        synchronized (this) {
            // Check if the same record already exists, or
            // move the temporary file in place if needed
            usesIdentifier(identifier);
            if (!objectExists(identifier)) {
                amazonS3.copyObject(bucket, tmpKey, bucket, key);
                amazonS3.deleteObject(bucket, tmpKey);
            } else {
                // effettua un touch sul file
                touch(key);
            }
            record = new S3DataRecord(identifier, new S3LazyObject(amazonS3, bucket, key));
            if (useCache) {
                cache.put(new Element(identifier.toString(),
                        new CachedS3DataRecord(record, cacheDirectory, temporary)));
                // no need to remove file
                temporary = null;
            }
        }
        // this will also make sure that
        // tempId is not garbage collected until here
        inUse.remove(tempId);
        return record;
    } catch (NoSuchAlgorithmException e) {
        throw new DataStoreException(DIGEST + " not available", e);
    } catch (IOException e) {
        throw new DataStoreException("Could not add record", e);
    } finally {
        if (temporary != null) {
            temporary.delete();
        }
    }
}

From source file:com.dotmarketing.portlets.cmsmaintenance.ajax.IndexAjaxAction.java

public void restoreIndex(HttpServletRequest request, HttpServletResponse response)
        throws ServletException, IOException, DotDataException {
    try {/*from w  w w . ja  v a  2 s.  c o m*/
        FileItemFactory factory = new DiskFileItemFactory();
        ServletFileUpload upload = new ServletFileUpload(factory);
        List<FileItem> items = (List<FileItem>) upload.parseRequest(request);

        String indexToRestore = null;
        boolean clearBeforeRestore = false;
        String aliasToRestore = null;
        File ufile = null;
        boolean isFlash = false;
        for (FileItem it : items) {
            if (it.getFieldName().equalsIgnoreCase("indexToRestore")) {
                indexToRestore = it.getString().trim();
            } else if (it.getFieldName().equalsIgnoreCase("aliasToRestore")) {
                aliasToRestore = it.getString().trim();
            } else if (it.getFieldName().equalsIgnoreCase("uploadedfiles[]")
                    || it.getFieldName().equals("uploadedfile")
                    || it.getFieldName().equalsIgnoreCase("uploadedfileFlash")) {
                isFlash = it.getFieldName().equalsIgnoreCase("uploadedfileFlash");
                ufile = File.createTempFile("indexToRestore", "idx");
                InputStream in = it.getInputStream();
                FileOutputStream out = new FileOutputStream(ufile);
                IOUtils.copyLarge(in, out);
                IOUtils.closeQuietly(out);
                IOUtils.closeQuietly(in);
            } else if (it.getFieldName().equalsIgnoreCase("clearBeforeRestore")) {
                clearBeforeRestore = true;
            }
        }

        if (LicenseUtil.getLevel() >= 200) {
            if (UtilMethods.isSet(aliasToRestore)) {
                String indexName = APILocator.getESIndexAPI()
                        .getAliasToIndexMap(APILocator.getSiteSearchAPI().listIndices()).get(aliasToRestore);
                if (UtilMethods.isSet(indexName))
                    indexToRestore = indexName;
            } else if (!UtilMethods.isSet(indexToRestore)) {
                indexToRestore = APILocator.getIndiciesAPI().loadIndicies().site_search;
            }
        }

        if (ufile != null) {
            final boolean clear = clearBeforeRestore;
            final String index = indexToRestore;
            final File file = ufile;
            new Thread() {
                public void run() {
                    try {
                        if (clear)
                            APILocator.getESIndexAPI().clearIndex(index);
                        APILocator.getESIndexAPI().restoreIndex(file, index);
                        Logger.info(this, "finished restoring index " + index);
                    } catch (Exception ex) {
                        Logger.error(IndexAjaxAction.this, "Error restoring", ex);
                    }
                }
            }.start();

        }

        PrintWriter out = response.getWriter();
        if (isFlash) {
            out.println("response=ok");
        } else {
            response.setContentType("application/json");
            out.println("{\"response\":1}");
        }

    } catch (FileUploadException fue) {
        Logger.error(this, "Error uploading file", fue);
        throw new IOException(fue);
    }
}

From source file:de.fhg.iais.asc.xslt.binaries.download.Downloader.java

private static void writeStreamToFile(InputStream contentStream, File target) throws IOException {
    ParentDirectoryUtils.forceCreateParentDirectoryOf(target);

    AtomicFileOutputStream aos = null;/* www  . ja v a  2 s  .  co  m*/
    try {
        aos = new AtomicFileOutputStream(target);

        IOUtils.copyLarge(contentStream, aos);
        aos.closeAsCompleted();
    } finally {
        IOUtils.closeQuietly(aos);
    }
}

From source file:herddb.upgrade.ZIPUtils.java

public static List<File> unZip(InputStream fs, File outDir) throws IOException {

    try (ZipInputStream zipStream = new ZipInputStream(fs, StandardCharsets.UTF_8);) {
        ZipEntry entry = zipStream.getNextEntry();
        List<File> listFiles = new ArrayList<>();
        while (entry != null) {
            if (entry.isDirectory()) {
                entry = zipStream.getNextEntry();
                continue;
            }/* w  w  w . ja  v  a2 s  . c o m*/

            String normalized = normalizeFilenameForFileSystem(entry.getName());
            File outFile = new File(outDir, normalized);
            File parentDir = outFile.getParentFile();
            if (parentDir != null && !parentDir.isDirectory()) {
                Files.createDirectories(parentDir.toPath());
            }

            listFiles.add(outFile);
            try (FileOutputStream out = new FileOutputStream(outFile);
                    SimpleBufferedOutputStream oo = new SimpleBufferedOutputStream(out)) {
                IOUtils.copyLarge(zipStream, oo);
            }
            entry = zipStream.getNextEntry();

        }
        return listFiles;
    } catch (IllegalArgumentException ex) {
        throw new IOException(ex);
    }

}

From source file:com.joyent.manta.client.MantaClientIT.java

@Test
public final void canCreateStreamInOneThreadAndCloseInAnother() throws Exception {
    final String name = UUID.randomUUID().toString();
    final String path = testPathPrefix + name;

    try (InputStream in = new RandomInputStream(8000)) {
        mantaClient.put(path, in);//from  w  w  w  .j ava2 s  . co  m
    }

    File temp = File.createTempFile("object-" + name, ".data");
    FileUtils.forceDeleteOnExit(temp);

    FileOutputStream out = new FileOutputStream(temp);

    Callable<InputStream> callable = () -> mantaClient.getAsInputStream(path);

    ExecutorService service = Executors.newFixedThreadPool(1);
    InputStream in = service.submit(callable).get();

    try {
        IOUtils.copyLarge(in, out);
    } finally {
        in.close();
        out.close();
    }
}

From source file:com.liferay.sync.engine.document.library.handler.DownloadFileHandler.java

protected void copyFile(final SyncFile syncFile, Path filePath, InputStream inputStream, boolean append)
        throws Exception {

    OutputStream outputStream = null;

    Watcher watcher = WatcherManager.getWatcher(getSyncAccountId());

    try {/* w w w  .  j  a va  2s  .  c  o m*/
        Path tempFilePath = FileUtil.getTempFilePath(syncFile);

        boolean exists = FileUtil.exists(filePath);

        if (append) {
            outputStream = Files.newOutputStream(tempFilePath, StandardOpenOption.APPEND);

            IOUtils.copyLarge(inputStream, outputStream);
        } else {
            if (exists && (boolean) getParameterValue("patch")) {
                if (_logger.isDebugEnabled()) {
                    _logger.debug("Patching {}", syncFile.getFilePathName());
                }

                Files.copy(filePath, tempFilePath, StandardCopyOption.REPLACE_EXISTING);

                IODeltaUtil.patch(tempFilePath, inputStream);
            } else {
                Files.copy(inputStream, tempFilePath, StandardCopyOption.REPLACE_EXISTING);
            }
        }

        watcher.addDownloadedFilePathName(filePath.toString());

        if (GetterUtil.getBoolean(syncFile.getLocalExtraSettingValue("restoreEvent"))) {

            syncFile.unsetLocalExtraSetting("restoreEvent");

            syncFile.setUiEvent(SyncFile.UI_EVENT_RESTORED_REMOTE);
        } else if (exists) {
            syncFile.setUiEvent(SyncFile.UI_EVENT_DOWNLOADED_UPDATE);
        } else {
            syncFile.setUiEvent(SyncFile.UI_EVENT_DOWNLOADED_NEW);
        }

        FileKeyUtil.writeFileKey(tempFilePath, String.valueOf(syncFile.getSyncFileId()), false);

        FileUtil.setModifiedTime(tempFilePath, syncFile.getModifiedTime());

        if (MSOfficeFileUtil.isLegacyExcelFile(filePath)) {
            syncFile.setLocalExtraSetting("lastSavedDate", MSOfficeFileUtil.getLastSavedDate(tempFilePath));
        }

        Files.move(tempFilePath, filePath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);

        ExecutorService executorService = SyncEngine.getExecutorService();

        Runnable runnable = new Runnable() {

            @Override
            public void run() {
                IODeltaUtil.checksums(syncFile);

                syncFile.setState(SyncFile.STATE_SYNCED);

                SyncFileService.update(syncFile);
            }

        };

        executorService.execute(runnable);
    } catch (FileSystemException fse) {
        if (fse instanceof AccessDeniedException) {
            _logger.error(fse.getMessage(), fse);

            syncFile.setState(SyncFile.STATE_ERROR);
            syncFile.setUiEvent(SyncFile.UI_EVENT_ACCESS_DENIED_LOCAL);

            SyncFileService.update(syncFile);

            return;
        } else if (fse instanceof NoSuchFileException) {
            if (isEventCancelled()) {
                SyncFileService.deleteSyncFile(syncFile);

                return;
            }
        }

        watcher.removeDownloadedFilePathName(filePath.toString());

        String message = fse.getMessage();

        _logger.error(message, fse);

        syncFile.setState(SyncFile.STATE_ERROR);

        if (message.contains("File name too long")) {
            syncFile.setUiEvent(SyncFile.UI_EVENT_FILE_NAME_TOO_LONG);
        }

        SyncFileService.update(syncFile);
    } finally {
        StreamUtil.cleanUp(outputStream);
    }
}

From source file:de.quadrillenschule.azocamsyncd.ftpservice.FTPConnection.java

public LinkedList<AZoFTPFile> download(LinkedList<AZoFTPFile> afs, LocalStorage localStorage) {
    if (afs.size() <= 0) {
        return afs;
    }//w w  w.  j a v  a  2 s  .co  m
    LinkedList<AZoFTPFile> retval = new LinkedList<>();
    /* for (AZoFTPFile a : afs) {
     retval.add(a);
     }*/
    Collections.sort(afs, new Comparator<AZoFTPFile>() {

        @Override
        public int compare(AZoFTPFile o1, AZoFTPFile o2) {
            return o1.ftpFile.getTimestamp().compareTo(o2.ftpFile.getTimestamp());
        }
    });
    simplyConnect(FTP.BINARY_FILE_TYPE);
    notify(FTPConnectionStatus.CONNECTED, getLastWorkingConnection(), -1);
    if (afs.size() > 0) {
        AZoFTPFile af = afs.getFirst();//) {
        File localFile = null;

        try {
            localFile = localStorage.getLocalFile(af);

        } catch (IOException ex) {
            notify(FTPConnectionStatus.LOCALSTORAGEERROR, af.dir + af.ftpFile.getName(), -1);
            close();
            return retval;
        }
        if (!localStorage.prepareLocalFile(localFile)) {
            notify(FTPConnectionStatus.LOCALSTORAGEERROR, af.dir + af.ftpFile.getName(), -1);
            close();
            return retval;
        }
        FileOutputStream fos = null;
        InputStream is = null;
        try {

            fos = new FileOutputStream(localFile);
            ftpclient.setSoTimeout(LONG_TIMEOUT);
            is = ftpclient.retrieveFileStream(af.dir + af.ftpFile.getName());
            cis = new CountingInputStream(is);
            downloadsize = af.ftpFile.getSize();
            notify(FTPConnectionStatus.DOWNLOADING, af.dir + af.ftpFile.getName(),
                    ((int) (100.0 * ((afs.indexOf(af) + 1.0) / (double) afs.size()))));
            //     ftpclient.setDataTimeout(TIMEOUT);
            //   ftpclient.setSoTimeout(TIMEOUT);

            //  Files.copy(cis, localFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
            try {
                IOUtils.copyLarge(cis, fos);
            } catch (Exception ie) {
                fos.close();
                is.close();
            }
            while (!ftpclient.completePendingCommand()) {
                try {
                    Thread.currentThread().wait(500);
                } catch (InterruptedException ex) {
                    Logger.getLogger(FTPConnection.class.getName()).log(Level.SEVERE, null, ex);
                }
            }
            ;

            is.close();
            fos.close();
            localStorage.setLatestIncoming(localFile);
            localStorage.addSyncedFile(af);
            notify(FTPConnectionStatus.NEW_LOCAL_FILE, localFile.getAbsolutePath(), -1);
            retval.add(af);
            notify(FTPConnectionStatus.SUCCESS, af.dir + af.ftpFile.getName(),
                    ((int) (100.0 * ((afs.indexOf(af) + 2.0) / (double) afs.size()))));

        } catch (Exception ex) {
            try {
                is.close();
                fos.close();
                close();
                localFile.delete();
                simplyConnect(FTP.BINARY_FILE_TYPE);
            } catch (Exception ex2) {
                close();
            }

        }
    }
    close();

    return retval;
}