List of usage examples for org.apache.commons.io IOUtils copyLarge
public static long copyLarge(Reader input, Writer output) throws IOException
Reader
to a Writer
. From source file:alluxio.client.file.FileSystemUtils.java
/** * Persists the given file to the under file system. * * @param fs {@link FileSystem} to carry out Alluxio operations * @param uri the uri of the file to persist * @param status the status info of the file * @return the size of the file persisted * @throws IOException if an I/O error occurs * @throws FileDoesNotExistException if the given file does not exist * @throws AlluxioException if an unexpected Alluxio error occurs *///from w w w . ja v a2 s. c om public static long persistFile(FileSystem fs, AlluxioURI uri, URIStatus status) throws IOException, FileDoesNotExistException, AlluxioException { // TODO(manugoyal) move this logic to the worker, as it deals with the under file system Closer closer = Closer.create(); long ret; try { OpenFileOptions options = OpenFileOptions.defaults().setReadType(ReadType.NO_CACHE); FileInStream in = closer.register(fs.openFile(uri, options)); AlluxioURI dstPath = new AlluxioURI(status.getUfsPath()); UnderFileSystem ufs = UnderFileSystem.get(dstPath.toString()); String parentPath = dstPath.getParent().toString(); if (!ufs.exists(parentPath)) { URIStatus parentStatus = fs.getStatus(uri.getParent()); Permission parentPerm = new Permission(parentStatus.getOwner(), parentStatus.getGroup(), (short) parentStatus.getMode()); MkdirsOptions parentMkdirsOptions = new MkdirsOptions().setCreateParent(true) .setPermission(parentPerm); if (!ufs.mkdirs(parentPath, parentMkdirsOptions)) { throw new IOException("Failed to create " + parentPath); } } URIStatus uriStatus = fs.getStatus(uri); Permission perm = new Permission(uriStatus.getOwner(), uriStatus.getGroup(), (short) uriStatus.getMode()); OutputStream out = closer .register(ufs.create(dstPath.toString(), new CreateOptions().setPermission(perm))); ret = IOUtils.copyLarge(in, out); } catch (Exception e) { throw closer.rethrow(e); } finally { closer.close(); } // Tell the master to mark the file as persisted fs.setAttribute(uri, SetAttributeOptions.defaults().setPersisted(true)); return ret; }
From source file:gobblin.tunnel.TunnelTest.java
@Test(enabled = false) public void mustDownloadLargeFiles() throws Exception { mockServer.when(HttpRequest.request().withMethod("CONNECT").withPath("www.us.apache.org:80")) .respond(HttpResponse.response().withStatusCode(200)); mockServer/*from ww w . ja va 2s .co m*/ .when(HttpRequest.request().withMethod("GET") .withPath("/dist//httpcomponents/httpclient/binary/httpcomponents-client-4.5.1-bin.tar.gz")) .forward(HttpForward.forward().withHost("www.us.apache.org").withPort(80)); Tunnel tunnel = Tunnel.build("www.us.apache.org", 80, "localhost", PORT); try { IOUtils.copyLarge( (InputStream) new URL("http://localhost:" + tunnel.getPort() + "/dist//httpcomponents/httpclient/binary/httpcomponents-client-4.5.1-bin.tar.gz") .getContent(new Class[] { InputStream.class }), new FileOutputStream(File.createTempFile("httpcomponents-client-4.5.1-bin", "tar.gz"))); } finally { tunnel.close(); } }
From source file:cn.lhfei.fu.service.impl.HomeworkBaseServiceImpl.java
/** * /*from w ww.j av a 2s .c o m*/ * * @see cn.lhfei.fu.service.HomeworkBaseService#update(cn.lhfei.fu.web.model.HomeworkBaseModel) */ @Override public boolean update(HomeworkBaseModel model, String userType) throws NullPointerException { OutputStream out = null; BufferedOutputStream bf = null; Date currentTime = new Date(); List<MultipartFile> files = model.getFiles(); try { HomeworkBase base = homeworkBaseDAO.find(model.getBaseId()); base.setModifyTime(currentTime); base.setActionType("" + OperationTypeEnum.SC.getCode()); base.setOperationTime(currentTime); homeworkBaseDAO.save(base); // update homework_base info int num = 1; for (MultipartFile file : files) {// save archive file if (file.getSize() > 0) { String filePath = filePathBuilder.buildFullPath(model, model.getStudentName()); String fileName = filePathBuilder.buildFileName(model, model.getStudentName(), num); String[] names = file.getOriginalFilename().split("[.]"); String fileType = names[names.length - 1]; String fullPath = filePath + File.separator + fileName + "." + fileType; out = new FileOutputStream(new File(fullPath)); bf = new BufferedOutputStream(out); IOUtils.copyLarge(file.getInputStream(), bf); HomeworkArchive archive = new HomeworkArchive(); archive.setArchiveName(fileName); archive.setArchivePath(fullPath); archive.setCreateTime(currentTime); archive.setModifyTime(currentTime); archive.setName(model.getName()); archive.setStudentBaseId(model.getStudentBaseId()); archive.setHomeworkBaseId(model.getBaseId()); /*archive.setHomeworkBase(base);*/ archive.setStudentName(model.getStudentName()); archive.setStudentId(model.getStudentId()); // ? if (userType != null && userType.equals(UserTypeEnum.STUDENT.getCode())) { archive.setStatus("" + ApproveStatusEnum.DSH.getCode()); } else if (userType != null && userType.equals(UserTypeEnum.TEACHER.getCode())) { archive.setStatus("" + ApproveStatusEnum.DSH.getCode()); } else if (userType != null && userType.equals(UserTypeEnum.ADMIN.getCode())) { archive.setStatus("" + ApproveStatusEnum.YSH.getCode()); } homeworkArchiveDAO.save(archive); // auto increment archives number. num++; } } } catch (IOException e) { log.error(e.getMessage(), e); } catch (NullPointerException e) { log.error("File name arguments missed.", e); throw new NullPointerException(e.getMessage()); } finally { if (out != null) { try { out.flush(); out.close(); } catch (IOException e) { log.error(e.getMessage(), e); } } if (bf != null) { try { bf.flush(); bf.close(); } catch (IOException e) { log.error(e.getMessage(), e); } } } return false; }
From source file:edu.uci.ics.hyracks.control.common.deployment.DeploymentUtils.java
/** * Download remote Http URLs and return the stored local file URLs * //w w w.ja v a 2s .c o m * @param urls * the remote Http URLs * @param deploymentDir * the deployment jar storage directory * @param isNC * true is NC/false is CC * @return a list of local file URLs * @throws HyracksException */ private static List<URL> downloadURLs(List<URL> urls, String deploymentDir, boolean isNC) throws HyracksException { //retry 10 times at maximum for downloading binaries int retryCount = 10; int tried = 0; Exception trace = null; while (tried < retryCount) { try { tried++; List<URL> downloadedFileURLs = new ArrayList<URL>(); File dir = new File(deploymentDir); if (!dir.exists()) { FileUtils.forceMkdir(dir); } for (URL url : urls) { String urlString = url.toString(); int slashIndex = urlString.lastIndexOf('/'); String fileName = urlString.substring(slashIndex + 1).split("&")[1]; String filePath = deploymentDir + File.separator + fileName; File targetFile = new File(filePath); if (isNC) { HttpClient hc = HttpClientBuilder.create().build(); HttpGet get = new HttpGet(url.toString()); HttpResponse response = hc.execute(get); InputStream is = response.getEntity().getContent(); OutputStream os = new FileOutputStream(targetFile); try { IOUtils.copyLarge(is, os); } finally { os.close(); is.close(); } } downloadedFileURLs.add(targetFile.toURI().toURL()); } return downloadedFileURLs; } catch (Exception e) { e.printStackTrace(); trace = e; } } throw new HyracksException(trace); }
From source file:de.tudarmstadt.ukp.uby.resource.UbyResource.java
@SuppressWarnings("rawtypes") @Override/* w w w. j a v a 2 s. c o m*/ public boolean initialize(ResourceSpecifier aSpecifier, Map aAdditionalParams) throws ResourceInitializationException { if (!super.initialize(aSpecifier, aAdditionalParams)) { return false; } modelProvider = new ModelProviderBase<Uby>() { { setContextObject(UbyResource.this); setDefault(GROUP_ID, "de.tudarmstadt.ukp.uby"); setDefault(ARTIFACT_ID, "${groupId}-model-data-${language}-${variant}"); setDefault(LOCATION, "classpath:/de/tudarmstadt/ukp/uby/data/lib/data-${language}-${variant}.properties"); setDefault(VARIANT, "light"); setOverride(LOCATION, modelLocation); setOverride(LANGUAGE, language); setOverride(VARIANT, variant); } @Override protected Uby produceResource(URL aUrl) throws IOException { Properties props = getAggregatedProperties(); Properties meta = new Properties(getResourceMetaData()); addOverride(meta, UBY_URL, url); addOverride(meta, UBY_DRIVER, driver); addOverride(meta, UBY_DIALECT, dialect); addOverride(meta, UBY_USERNAME, username); addOverride(meta, UBY_PASSWORD, password); // If an embedded database is to be used, extract database to disk if (aUrl != null) { UbyResource.this.getLogger().info("Using embedded database"); File tmpFolder = File.createTempFile("uby", ".db"); FileUtils.forceDelete(tmpFolder); FileUtils.forceMkdir(tmpFolder); tmpFolder.deleteOnExit(); File tmpDbFile = new File(tmpFolder, DATABASE_FILE); tmpDbFile.deleteOnExit(); UbyResource.this.getLogger().info("Extracting embedded database to [" + tmpDbFile + "]"); InputStream is = null; OutputStream os = null; try { // FIXME should probably just do nothing if database file is not compressed // and if the URL already points to the file system. is = CompressionUtils.getInputStream(aUrl.toString(), aUrl.openStream()); os = new FileOutputStream(tmpDbFile); IOUtils.copyLarge(is, os); } finally { IOUtils.closeQuietly(os); IOUtils.closeQuietly(is); } // Well... we currently only support H2 as embedded DB. If somebody wants to // use a different embedded DB, we'll have to implement something more // generic here. meta.setProperty(UBY_URL, "jdbc:h2:" + tmpFolder.toURI().toURL().toString() + "/" + DATABASE + ";TRACE_LEVEL_FILE=0"); } else { getLogger().info("Connecting to server..."); } getLogger().info("uby.url: " + meta.getProperty(UBY_URL)); getLogger().info("uby.driver: " + meta.getProperty(UBY_DRIVER)); getLogger().info("uby.dialect: " + meta.getProperty(UBY_DIALECT)); getLogger().info("uby.username: " + meta.getProperty(UBY_USERNAME)); getLogger().info("uby.password: " + (StringUtils.isNotEmpty(meta.getProperty(UBY_PASSWORD)) ? "<set>" : "<unset>")); DBConfig dbConfig = new DBConfig(meta.getProperty(UBY_URL), meta.getProperty(UBY_DRIVER), meta.getProperty(UBY_DIALECT), meta.getProperty(UBY_USERNAME), meta.getProperty(UBY_PASSWORD), false); try { return new Uby(dbConfig); } catch (IllegalArgumentException e) { throw new IOException(e); } } }; return true; }
From source file:com.altoukhov.svsync.fileviews.SmbFileSpace.java
@Override public boolean writeFile(InputStream fileStream, FileSnapshot fileInfo) { if (fileStream == null) return false; OutputStream out = null;/*from w w w. j a v a2s. c o m*/ long bytesWritten = 0; boolean setTimestamp = false; try { SmbFile file = new SmbFile(toAbsoluteFilePath(fileInfo.getRelativePath()), auth); out = file.getOutputStream(); if (out == null) { System.out.println("Failed to open file for write: " + fileInfo.getRelativePath()); } if (fileInfo.isLargeFile()) { bytesWritten = IOUtils.copyLarge(fileStream, out); } else { bytesWritten = IOUtils.copy(fileStream, out); } } catch (IOException ex) { System.out.println("Failed to copy file: " + ex.getMessage()); } finally { try { if (fileStream != null) fileStream.close(); if (out != null) out.close(); } catch (IOException ex) { System.out.println("Failed to close stream: " + ex.getMessage()); } } try { SmbFile file = new SmbFile(toAbsoluteFilePath(fileInfo.getRelativePath()), auth); file.setLastModified(fileInfo.getModifiedTimestamp().toDate().getTime()); setTimestamp = true; } catch (IOException ex) { System.out.println("Failed to copy file: " + ex.getMessage()); } return setTimestamp && (bytesWritten == fileInfo.getFileSize()); }
From source file:cz.zcu.kiv.eegdatabase.logic.controller.experiment.DownloadMetadataZipController.java
@Override protected ModelAndView onSubmit(HttpServletRequest request, HttpServletResponse response, Object command, BindException errors) throws Exception { // //ModelAndView mav = new ModelAndView("downloadMetadataZipView"); MetadataCommand mc = (MetadataCommand) command; int id = Integer.parseInt(request.getParameter("id")); Experiment fromDB = experimentDao.read(id); String scenarioName = fromDB.getScenario().getTitle(); Set<DataFile> files = fromDB.getDataFiles(); //gets a parameters from request //contents is in the request if user wants download data file String[] contents = request.getParameterValues("content"); //fileParam represents file metadata params String[] fileParam = request.getParameterValues("fileParam"); FileMetadataParamValId[] params = null; if (fileParam != null) { params = new FileMetadataParamValId[fileParam.length]; //go throws parameters and creates FileMetadataParamsValId instances //in the request data_file_id and file_metadata_param_def_id is separated //by # in the form file_id#file_metadata_param_def_id ///*from ww w .j ava2 s . c o m*/ for (int i = 0; i < fileParam.length; i++) { String[] tmp = fileParam[i].split("#"); params[i] = new FileMetadataParamValId(Integer.parseInt(tmp[1]), Integer.parseInt(tmp[0])); } } Set<DataFile> newFiles = new HashSet<DataFile>(); if (fileParam != null || contents != null) { //go over data files selected from db for (DataFile item : files) { DataFile newItem = null; if (contents != null) { if (Arrays.asList(contents).contains(String.valueOf(item.getDataFileId()))) { newItem = new DataFile(); newItem.setDataFileId(item.getDataFileId()); newItem.setExperiment(item.getExperiment()); newItem.setFileContent(item.getFileContent()); newItem.setFilename(item.getFilename()); newItem.setMimetype(item.getMimetype()); newItem.setDescription(item.getDescription()); } } Set<FileMetadataParamVal> newVals = new HashSet<FileMetadataParamVal>(); if (params != null) { for (FileMetadataParamVal paramVal : item.getFileMetadataParamVals()) { for (FileMetadataParamValId paramId : params) { if (paramVal.getId().getDataFileId() == paramId.getDataFileId() && paramVal.getId() .getFileMetadataParamDefId() == paramId.getFileMetadataParamDefId()) { newVals.add(paramVal); } } } } if (!newVals.isEmpty()) { if (newItem == null) { newItem = new DataFile(); } newItem.setFileMetadataParamVals(newVals); } if (newItem != null) { newFiles.add(newItem); } } } Person user = personDao.getPerson(ControllerUtils.getLoggedUserName()); Timestamp currentTimestamp = new java.sql.Timestamp(Calendar.getInstance().getTime().getTime()); History history = new History(); log.debug("Setting downloading metadata"); history.setExperiment(fromDB); log.debug("Setting user"); history.setPerson(user); log.debug("Setting time of download"); history.setDateOfDownload(currentTimestamp); log.debug("Saving download history"); historyDao.create(history); File file = getZipGenerator().generate(fromDB, mc, newFiles, new byte[0], null); InputStream dataStream = new FileInputStream(file); response.setHeader("Content-Type", "application/zip"); if (scenarioName == null) { response.setHeader("Content-Disposition", "attachment;filename=Experiment_data.zip"); } else { String[] names = scenarioName.split(" "); scenarioName = names[0]; for (int i = 1; i < names.length; i++) { scenarioName += "_" + names[i]; } response.setHeader("Content-Disposition", "attachment;filename=" + scenarioName + ".zip"); } if (dataStream instanceof InputStream) { IOUtils.copyLarge(dataStream, response.getOutputStream()); dataStream.close(); } log.debug(zipGenerator); return null; }
From source file:cz.zcu.kiv.eegdatabase.wui.core.experiments.ExperimentDownloadProvider.java
@Transactional public FileDTO generatePackageFile(ExperimentPackage pckg, MetadataCommand mc, License license, List<Experiment> selectList, Person loggedUser, DownloadPackageManager manager) { ZipOutputStream zipOutputStream = null; FileOutputStream fileOutputStream = null; File tempZipFile = null;// ww w .j a va2 s. co m ZipInputStream in = null; File file = null; try { FileDTO dto = new FileDTO(); dto.setFileName(pckg.getName().replaceAll("\\s", "_") + ".zip"); // create temp zip file tempZipFile = File.createTempFile("experimentDownload_", ".zip"); // open stream to temp zip file fileOutputStream = new FileOutputStream(tempZipFile); // prepare zip stream zipOutputStream = new ZipOutputStream(fileOutputStream); for (Experiment tmp : selectList) { Experiment exp = service.getExperimentForDetail(tmp.getExperimentId()); String experimentDirPrefix = ""; // create directory for each experiment. String scenarioName = exp.getScenario().getTitle(); if (scenarioName != null) { experimentDirPrefix = "Experiment_" + exp.getExperimentId() + "_" + scenarioName.replaceAll("\\s", "_") + "/"; } else experimentDirPrefix = "Experiment_data_" + exp.getExperimentId() + "/"; // generate temp zip file with experiment byte[] licenseFile = licenseService.getLicenseAttachmentContent(license.getLicenseId()); file = zipGenerator.generate(exp, mc, exp.getDataFiles(), licenseFile, license.getAttachmentFileName()); in = new ZipInputStream(new FileInputStream(file)); ZipEntry entryIn = null; // copy unziped experiment in package zip file. // NOTE: its easier solution copy content of one zip in anoter instead create directory structure via java.io.File. while ((entryIn = in.getNextEntry()) != null) { zipOutputStream.putNextEntry(new ZipEntry(experimentDirPrefix + entryIn.getName())); IOUtils.copyLarge(in, zipOutputStream); zipOutputStream.closeEntry(); } // mark all temp files for package for delete on exit FileUtils.deleteOnExitQuietly(file); IOUtils.closeQuietly(in); FileUtils.deleteQuietly(file); createHistoryRecordAboutDownload(exp, loggedUser); synchronized (this) { manager.setNumberOfDownloadedExperiments(manager.getNumberOfDownloadedExperiments() + 1); } } dto.setFile(tempZipFile); // no problem detected - close all streams and mark file for delete on exit. // file is deleted after download action FileUtils.deleteOnExitQuietly(tempZipFile); IOUtils.closeQuietly(zipOutputStream); IOUtils.closeQuietly(fileOutputStream); return dto; } catch (Exception e) { log.error(e.getMessage(), e); // problem detected - close all streams, mark files for delete on exit and try delete them. IOUtils.closeQuietly(zipOutputStream); IOUtils.closeQuietly(fileOutputStream); FileUtils.deleteOnExitQuietly(tempZipFile); FileUtils.deleteOnExitQuietly(file); FileUtils.deleteQuietly(tempZipFile); FileUtils.deleteQuietly(file); return null; } }
From source file:de.extra.client.plugins.responseprocessplugin.filesystem.FileSystemResultDataResponseProcessPlugin.java
/** * @param responseId//w w w . ja v a2 s. c o m * @param responseBody * @return */ private void saveBodyToFilesystem(final String responseId, final DataHandler dataHandler) { try { final String dateiName = buildFilename(responseId); final File responseFile = new File(eingangOrdner, dateiName); final FileOutputStream fileOutputStream = new FileOutputStream(responseFile); IOUtils.copyLarge(dataHandler.getInputStream(), fileOutputStream); transportObserver.responseDataForwarded(responseFile.getAbsolutePath(), responseFile.length()); LOG.info("Response gespeichert in File: '" + dateiName + "'"); } catch (final IOException ioException) { throw new ExtraResponseProcessPluginRuntimeException(ExceptionCode.UNEXPECTED_INTERNAL_EXCEPTION, "Fehler beim schreiben der Antwort", ioException); } }
From source file:alluxio.job.persist.PersistDefinition.java
@Override public SerializableVoid runTask(PersistConfig config, SerializableVoid args, JobWorkerContext context) throws Exception { AlluxioURI uri = new AlluxioURI(config.getFilePath()); String ufsPath = config.getUfsPath(); // check if the file is persisted in UFS and delete it, if we are overwriting it UfsManager.UfsClient ufsClient = context.getUfsManager().get(config.getMountId()); try (CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource()) { UnderFileSystem ufs = ufsResource.get(); if (ufs == null) { throw new IOException("Failed to create UFS instance for " + ufsPath); }/* w ww .ja v a2s . c o m*/ if (ufs.exists(ufsPath)) { if (config.isOverwrite()) { LOG.info("File {} is already persisted in UFS. Removing it.", config.getFilePath()); ufs.deleteFile(ufsPath); } else { throw new IOException("File " + config.getFilePath() + " is already persisted in UFS, to overwrite the file, please set the overwrite flag" + " in the config."); } } FileSystem fs = FileSystem.Factory.get(); long bytesWritten; try (Closer closer = Closer.create()) { OpenFileOptions options = OpenFileOptions.defaults().setReadType(ReadType.NO_CACHE); FileInStream in = closer.register(fs.openFile(uri, options)); AlluxioURI dstPath = new AlluxioURI(ufsPath); // Create ancestor directories from top to the bottom. We cannot use recursive create // parents here because the permission for the ancestors can be different. Stack<Pair<String, MkdirsOptions>> ufsDirsToMakeWithOptions = new Stack<>(); AlluxioURI curAlluxioPath = uri.getParent(); AlluxioURI curUfsPath = dstPath.getParent(); // Stop at the Alluxio root because the mapped directory of Alluxio root in UFS may not // exist. while (!ufs.isDirectory(curUfsPath.toString()) && curAlluxioPath != null) { URIStatus curDirStatus = fs.getStatus(curAlluxioPath); ufsDirsToMakeWithOptions.push(new Pair<>(curUfsPath.toString(), MkdirsOptions.defaults().setCreateParent(false).setOwner(curDirStatus.getOwner()) .setGroup(curDirStatus.getGroup()) .setMode(new Mode((short) curDirStatus.getMode())))); curAlluxioPath = curAlluxioPath.getParent(); curUfsPath = curUfsPath.getParent(); } while (!ufsDirsToMakeWithOptions.empty()) { Pair<String, MkdirsOptions> ufsDirAndPerm = ufsDirsToMakeWithOptions.pop(); // UFS mkdirs might fail if the directory is already created. If so, skip the mkdirs // and assume the directory is already prepared, regardless of permission matching. if (!ufs.mkdirs(ufsDirAndPerm.getFirst(), ufsDirAndPerm.getSecond()) && !ufs.isDirectory(ufsDirAndPerm.getFirst())) { throw new IOException("Failed to create " + ufsDirAndPerm.getFirst() + " with permission " + ufsDirAndPerm.getSecond().toString()); } } URIStatus uriStatus = fs.getStatus(uri); OutputStream out = closer.register( ufs.create(dstPath.toString(), CreateOptions.defaults().setOwner(uriStatus.getOwner()) .setGroup(uriStatus.getGroup()).setMode(new Mode((short) uriStatus.getMode())))); bytesWritten = IOUtils.copyLarge(in, out); incrementPersistedMetric(ufsClient.getUfsMountPointUri(), bytesWritten); } LOG.info("Persisted file {} with size {}", ufsPath, bytesWritten); } return null; }