List of usage examples for java.nio.file.attribute BasicFileAttributes size
long size();
From source file:com.github.zhanhb.ckfinder.download.PathPartial.java
/** * Serve the specified resource, optionally including the data content. * * @param request The servlet request we are processing * @param response The servlet response we are creating * @param content Should the content be included? * @param path the resource to serve// www. j a v a 2s. com * * @exception IOException if an input/output error occurs */ private void serveResource(HttpServletRequest request, HttpServletResponse response, boolean content, Path path) throws IOException, ServletException { ActionContext context = new ActionContext().put(HttpServletRequest.class, request) .put(HttpServletResponse.class, response).put(ServletContext.class, request.getServletContext()) .put(Path.class, path); if (path == null) { notFound.handle(context); return; } BasicFileAttributes attr; try { attr = Files.readAttributes(path, BasicFileAttributes.class); } catch (IOException ex) { notFound.handle(context); return; } context.put(BasicFileAttributes.class, attr); boolean isError = response.getStatus() >= HttpServletResponse.SC_BAD_REQUEST; // Check if the conditions specified in the optional If headers are // satisfied. // Checking If headers boolean included = (request.getAttribute(RequestDispatcher.INCLUDE_CONTEXT_PATH) != null); String etag = this.eTag.getValue(context); if (!included && !isError && !checkIfHeaders(request, response, attr, etag)) { return; } // Find content type. String contentType = contentTypeResolver.getValue(context); // Get content length long contentLength = attr.size(); // Special case for zero length files, which would cause a // (silent) ISE boolean serveContent = content && contentLength != 0; Range[] ranges = null; if (!isError) { if (useAcceptRanges) { // Accept ranges header response.setHeader(HttpHeaders.ACCEPT_RANGES, "bytes"); } // Parse range specifier ranges = serveContent ? parseRange(request, response, attr, etag) : FULL; // ETag header response.setHeader(HttpHeaders.ETAG, etag); // Last-Modified header response.setDateHeader(HttpHeaders.LAST_MODIFIED, attr.lastModifiedTime().toMillis()); } ServletOutputStream ostream = null; if (serveContent) { ostream = response.getOutputStream(); } String disposition = contentDisposition.getValue(context); if (disposition != null) { response.setHeader(HttpHeaders.CONTENT_DISPOSITION, disposition); } // Check to see if a Filter, Valve of wrapper has written some content. // If it has, disable range requests and setting of a content length // since neither can be done reliably. if (isError || ranges == FULL) { // Set the appropriate output headers if (contentType != null) { log.debug("serveFile: contentType='{}'", contentType); response.setContentType(contentType); } if (contentLength >= 0) { setContentLengthLong(response, contentLength); } // Copy the input stream to our output stream (if requested) if (serveContent) { log.trace("Serving bytes"); Files.copy(path, ostream); } } else if (ranges != null && ranges.length != 0) { // Partial content response. response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); if (ranges.length == 1) { Range range = ranges[0]; response.addHeader(HttpHeaders.CONTENT_RANGE, range.toString()); long length = range.end - range.start + 1; setContentLengthLong(response, length); if (contentType != null) { log.debug("serveFile: contentType='{}'", contentType); response.setContentType(contentType); } if (serveContent) { try (InputStream stream = Files.newInputStream(path)) { copyRange(stream, ostream, range, new byte[Math.min((int) length, 8192)]); } } } else { response.setContentType("multipart/byteranges; boundary=" + MIME_SEPARATION); if (serveContent) { copy(path, ostream, ranges, contentType, new byte[Math.min((int) contentLength, 8192)]); } } } }
From source file:com.github.zhanhb.ckfinder.download.PathPartial.java
/** * Parse the range header./*ww w. j a va 2 s. co m*/ * * @param request The servlet request we are processing * @param response The servlet response we are creating * @param attr File attributes * @param etag ETag of the entity * @return array of ranges */ @Nullable @SuppressWarnings("ReturnOfCollectionOrArrayField") private Range[] parseRange(HttpServletRequest request, HttpServletResponse response, BasicFileAttributes attr, String etag) throws IOException { // Checking If-Range String headerValue = request.getHeader(HttpHeaders.IF_RANGE); if (headerValue != null) { long headerValueTime = -1; try { headerValueTime = request.getDateHeader(HttpHeaders.IF_RANGE); } catch (IllegalArgumentException e) { // Ignore } // If the ETag the client gave does not match the entity // eTag, then the entire entity is returned. if (headerValueTime == -1 && !headerValue.trim().equals(etag) || attr.lastModifiedTime().toMillis() > headerValueTime + 1000) { // If the timestamp of the entity the client got is older than // the last modification date of the entity, the entire entity // is returned. return FULL; } } long fileLength = attr.size(); if (fileLength == 0) { return FULL; } // Retrieving the range header (if any is specified String rangeHeader = request.getHeader(HttpHeaders.RANGE); if (rangeHeader == null) { return FULL; } // bytes is the only range unit supported (and I don't see the point // of adding new ones). if (!rangeHeader.startsWith("bytes=")) { return FULL; } // List which will contain all the ranges which are successfully // parsed. List<Range> result = new ArrayList<>(4); // Parsing the range list // "bytes=".length() = 6 for (int index, last = 6;; last = index + 1) { index = rangeHeader.indexOf(',', last); boolean isLast = index == -1; final String rangeDefinition = (isLast ? rangeHeader.substring(last) : rangeHeader.substring(last, index)).trim(); final int dashPos = rangeDefinition.indexOf('-'); if (dashPos == -1) { break; } final Range currentRange = new Range(fileLength); try { if (dashPos == 0) { final long offset = Long.parseLong(rangeDefinition); if (offset == 0) { // -0, --0 break; } currentRange.start = Math.max(fileLength + offset, 0); } else { currentRange.start = Long.parseLong(rangeDefinition.substring(0, dashPos)); if (dashPos < rangeDefinition.length() - 1) { currentRange.end = Long .parseLong(rangeDefinition.substring(dashPos + 1, rangeDefinition.length())); } } } catch (NumberFormatException e) { break; } if (!currentRange.validate()) { break; } result.add(currentRange); if (isLast) { int size = result.size(); if (size == 0) { break; } return result.toArray(new Range[size]); } } response.addHeader(HttpHeaders.CONTENT_RANGE, "bytes */" + fileLength); response.sendError(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE); return null; }
From source file:com.nridge.connector.fs.con_fs.core.FileCrawler.java
private void processFile(Path aPath, BasicFileAttributes aFileAttributes) throws IOException { Logger appLogger = mAppMgr.getLogger(this, "processFile"); appLogger.trace(mAppMgr.LOGMSG_TRACE_ENTER); StopWatch stopWatch = new StopWatch(); stopWatch.start();//from www. jav a 2 s .c o m File fsFile = aPath.toFile(); String docId = generateDocumentId(aPath); String pathFileName = aPath.toAbsolutePath().toString(); appLogger.debug(String.format("Processing File (%s): %s", docId, pathFileName)); boolean isFileFlat = true; Document fsDocument = new Document(Constants.FS_DOCUMENT_TYPE, mBag); DataBag fileBag = fsDocument.getBag(); fileBag.resetValuesWithDefaults(); fileBag.setValueByName("nsd_id", docId); String fileName = fsFile.getName(); fileBag.setValueByName("nsd_url", fsFile.toURI().toURL().toString()); String viewURL = createViewURL(docId); fileBag.setValueByName("nsd_url_view", viewURL); fileBag.setValueByName("nsd_url_display", viewURL); fileBag.setValueByName("nsd_name", fileName); fileBag.setValueByName("nsd_file_name", fileName); fileBag.setValueByName("nsd_file_size", aFileAttributes.size()); FileTime creationTime = aFileAttributes.creationTime(); Date cDate = new Date(creationTime.toMillis()); fileBag.setValueByName("nsd_doc_created_ts", cDate); FileTime lastModifiedTime = aFileAttributes.lastModifiedTime(); Date lmDate = new Date(lastModifiedTime.toMillis()); fileBag.setValueByName("nsd_doc_modified_ts", lmDate); fileBag.setValueByName("nsd_crawl_type", mCrawlQueue.getCrawlType()); DataField dataField = fileBag.getFirstFieldByFeatureName(Field.FEATURE_IS_CONTENT); if (dataField != null) { ContentExtractor contentExtractor = new ContentExtractor(mAppMgr); contentExtractor.setCfgPropertyPrefix(Constants.CFG_PROPERTY_PREFIX + ".extract"); try { String mimeType = contentExtractor.detectType(fsFile); if (StringUtils.isNotEmpty(mimeType)) fileBag.setValueByName("nsd_mime_type", mimeType); if (isExpandableCSVFile(mimeType)) { isFileFlat = false; processCSVFile(aPath, aFileAttributes, viewURL); } else contentExtractor.process(pathFileName, dataField); } catch (NSException e) { String msgStr = String.format("%s: %s", pathFileName, e.getMessage()); appLogger.error(msgStr); } } if (isFileFlat) { fileBag.setValueByName("nsd_doc_hash", fsDocument.generateUniqueHash(false)); saveAddQueueDocument(fsDocument, stopWatch); } appLogger.trace(mAppMgr.LOGMSG_TRACE_DEPART); }
From source file:com.upplication.s3fs.util.AmazonS3ClientMock.java
private S3Element parse(Path elem, Path bucket) throws IOException { S3Object object = new S3Object(); String bucketName = bucket.getFileName().toString(); object.setBucketName(bucketName);//from w w w .ja va 2 s. c o m String key = bucket.relativize(elem).toString().replaceAll("%2F", "/"); boolean dir = key.endsWith("/") || key.isEmpty(); object.setKey(key); ObjectMetadata metadata = new ObjectMetadata(); BasicFileAttributes attr = Files.readAttributes(elem, BasicFileAttributes.class); metadata.setLastModified(new Date(attr.lastAccessTime().toMillis())); if (dir) { metadata.setContentLength(0); object.setObjectContent(null); } else { metadata.setContentLength(attr.size()); object.setObjectContent(new ByteArrayInputStream(Files.readAllBytes(elem))); } object.setObjectMetadata(metadata); AccessControlList permission = createAclPermission(elem, bucketName); return new S3Element(object, permission, dir); }
From source file:com.commander4j.thread.InboundMessageThread.java
public void run() { logger.debug("InboundMessageThread running"); Boolean dbconnected = false;/*from w w w. ja v a 2s .c om*/ if (Common.hostList.getHost(hostID).isConnected(sessionID) == false) { dbconnected = Common.hostList.getHost(hostID).connect(sessionID, hostID); } else { dbconnected = true; } if (dbconnected) { JDBInterfaceLog il = new JDBInterfaceLog(getHostID(), getSessionID()); JeMail mail = new JeMail(getHostID(), getSessionID()); JDBInterface inter = new JDBInterface(getHostID(), getSessionID()); IncommingMaterialDefinition imd = new IncommingMaterialDefinition(getHostID(), getSessionID()); IncommingProcessOrderStatusChange iposc = new IncommingProcessOrderStatusChange(getHostID(), getSessionID()); IncommingProductionDeclarationConfirmation ipd = new IncommingProductionDeclarationConfirmation( getHostID(), getSessionID()); IncommingProcessOrder ipo = new IncommingProcessOrder(getHostID(), getSessionID()); IncommingLocation ilocn = new IncommingLocation(getHostID(), getSessionID()); IncommingPalletStatusChange ipsc = new IncommingPalletStatusChange(getHostID(), getSessionID()); IncommingPalletMove ipmv = new IncommingPalletMove(getHostID(), getSessionID()); IncommingBatchStatusChange bsc = new IncommingBatchStatusChange(getHostID(), getSessionID()); IncommingJourney ij = new IncommingJourney(getHostID(), getSessionID()); IncommingInspectionResult iirslt = new IncommingInspectionResult(getHostID(), getSessionID()); IncommingDespatchConfirmation idc = new IncommingDespatchConfirmation(getHostID(), getSessionID()); IncommingQMInspectionRequest iireq = new IncommingQMInspectionRequest(getHostID(), getSessionID()); IncommingMaterialAutoMove imam = new IncommingMaterialAutoMove(getHostID(), getSessionID()); GenericMessageHeader gmh = new GenericMessageHeader(); LinkedList<String> filenames = new LinkedList<String>(); BasicFileAttributes attrs; while (true) { com.commander4j.util.JWait.milliSec(100); if (allDone) { if (dbconnected) { Common.hostList.getHost(hostID).disconnect(getSessionID()); } return; } if (InboundMessageCollectionThread.recoveringFiles == false) { dir = new File(inputPath); chld = dir.listFiles((FileFilter) FileFileFilter.FILE); if (chld == null) { allDone = true; } else { Arrays.sort(chld, LastModifiedFileComparator.LASTMODIFIED_COMPARATOR); filenames.clear(); for (int i = 0; (i < chld.length) & (i < maxfiles); i++) { fileName = chld[i].getName(); try { attrs = Files.readAttributes(chld[i].getAbsoluteFile().toPath(), BasicFileAttributes.class); if (attrs.size() > 0) { if (fileName.endsWith(".xml")) { filenames.addFirst(fileName); com.commander4j.util.JWait.milliSec(50); } } else { try { chld[i].delete(); } catch (Exception ex) { } } } catch (IOException e) { } } if (filenames.size() > 0) { logger.debug("Begin processing " + String.valueOf(filenames.size()) + " files."); for (int i = filenames.size() - 1; i >= 0; i--) { if (allDone) { if (dbconnected) { Common.hostList.getHost(hostID).disconnect(getSessionID()); } return; } fromFile = filenames.get(i); try { logger.debug("<--- START OF PROCESSING " + fromFile + " ---->"); logger.debug("Reading message header : " + inputPath + fromFile); if (gmh.readAddressInfo(inputPath + fromFile, getSessionID()) == true) { messageProcessedOK = true; errorMessage = ""; if (gmh.getInterfaceType().length() == 0) { messageProcessedOK = false; errorMessage = "Unrecognised Commander4j XML message format in file " + fromFile; logger.debug(errorMessage); String datetime = ""; datetime = JUtility .getISOTimeStampStringFormat(JUtility.getSQLDateTime()); gmh.setMessageDate(datetime); gmh.setInterfaceDirection("Unknown"); gmh.setMessageInformation(fromFile); gmh.setInterfaceType("Unknown"); gmh.setMessageRef("Unknown"); } else { if (gmh.getInterfaceDirection().equals("Input") == false) { messageProcessedOK = false; errorMessage = "Inbound message ignored - Interface Direction = " + gmh.getInterfaceDirection(); } else { String interfaceType = gmh.getInterfaceType(); logger.debug("Processing " + interfaceType + " started."); if (interfaceType.equals("Despatch Confirmation") == true) { messageProcessedOK = idc.processMessage(gmh); errorMessage = idc.getErrorMessage(); } if (interfaceType.equals("Material Definition") == true) { messageProcessedOK = imd.processMessage(gmh); errorMessage = imd.getErrorMessage(); } if (interfaceType.equals("Process Order") == true) { messageProcessedOK = ipo.processMessage(gmh); errorMessage = ipo.getErrorMessage(); } if (interfaceType.equals("Location") == true) { messageProcessedOK = ilocn.processMessage(gmh); errorMessage = ilocn.getErrorMessage(); } if (interfaceType.equals("Pallet Status Change") == true) { messageProcessedOK = ipsc.processMessage(gmh); errorMessage = ipsc.getErrorMessage(); } if (interfaceType.equals("Pallet Move") == true) { messageProcessedOK = ipmv.processMessage(gmh); errorMessage = ipmv.getErrorMessage(); } if (interfaceType.equals("Batch Status Change") == true) { messageProcessedOK = bsc.processMessage(gmh); errorMessage = bsc.getErrorMessage(); } if (interfaceType.equals("Journey Definition") == true) { messageProcessedOK = ij.processMessage(gmh); errorMessage = ij.getErrorMessage(); } if (interfaceType.equals("Process Order Status Change") == true) { messageProcessedOK = iposc.processMessage(gmh); errorMessage = iposc.getErrorMessage(); } if (interfaceType.equals("Production Declaration") == true) { messageProcessedOK = ipd.processMessage(gmh); errorMessage = ipd.getErrorMessage(); } if (interfaceType.equals("QM Inspection Request") == true) { messageProcessedOK = iireq.processMessage(gmh); errorMessage = iireq.getErrorMessage(); } if (interfaceType.equals("QM Inspection Result") == true) { messageProcessedOK = iirslt.processMessage(gmh); errorMessage = iirslt.getErrorMessage(); } if (interfaceType.equals("Material Auto Move") == true) { messageProcessedOK = imam.processMessage(gmh); errorMessage = imam.getErrorMessage(); } GenericMessageHeader.updateStats("Input", interfaceType, messageProcessedOK.toString()); logger.debug("Processing " + interfaceType + " finished."); } } logger.debug( " === RESULT " + messageProcessedOK.toString() + " ==="); if (messageProcessedOK) { il.write(gmh, GenericMessageHeader.msgStatusSuccess, "Processed OK", "DB Update", fromFile); reader.deleteFile(backupPath + gmh.getInterfaceType() + File.separator + fromFile); reader.move_FileToDirectory(inputPath + fromFile, backupPath + gmh.getInterfaceType(), true); } else { il.write(gmh, GenericMessageHeader.msgStatusError, errorMessage, "DB Update", fromFile); if (inter.getInterfaceProperties(gmh.getInterfaceType(), "Input") == true) { if (inter.getEmailError() == true) { String emailaddresses = inter.getEmailAddresses(); StringConverter stringConverter = new StringConverter(); ArrayConverter arrayConverter = new ArrayConverter( String[].class, stringConverter); arrayConverter.setDelimiter(';'); arrayConverter.setAllowedChars(new char[] { '@', '_' }); String[] emailList = (String[]) arrayConverter .convert(String[].class, emailaddresses); if (emailList.length > 0) { String siteName = Common.hostList.getHost(getHostID()) .getSiteDescription(); String attachedFilename = Common.base_dir + java.io.File.separator + inputPath + fromFile; logger.debug("Attaching file " + Common.base_dir + java.io.File.separator + inputPath + fromFile); mail.postMail(emailList, "Error Processing Incoming " + gmh.getInterfaceType() + " for [" + siteName + "] on " + JUtility.getClientName(), errorMessage, fromFile, attachedFilename); com.commander4j.util.JWait.milliSec(2000); } } } reader.deleteFile( errorPath + gmh.getInterfaceType() + File.separator + fromFile); reader.move_FileToDirectory(inputPath + fromFile, errorPath + gmh.getInterfaceType(), true); } } logger.debug("<--- END OF PROCESSING " + fromFile + " ---->"); } catch (Exception e) { e.printStackTrace(); } } } } } } } }
From source file:ch.mattrero.foldersync.FoldersSynchronizer.java
void syncTree(final Path sourceSubDir) { SyncStatus status = null;// w w w. jav a2 s . c o m BasicFileAttributes fromAttributes = null; BasicFileAttributes toAttributes = null; try (final DirectoryStream<Path> sourceStream = Files.newDirectoryStream(sourceSubDir); DirectoryStream<Path> backupStream = Files .newDirectoryStream(resolveBackupItemPath(sourceSubDir))) { final Iterator<Path> sourceIterator = sourceStream.iterator(); final Iterator<Path> backupIterator = backupStream.iterator(); Path sourceItem = (sourceIterator.hasNext() ? sourceIterator.next() : null); Path backupItem = (backupIterator.hasNext() ? backupIterator.next() : null); while (sourceItem != null || backupItem != null) { if (sourceItem == null) { status = DELETED; } else if (backupItem == null) { status = ADDED; } else if (sourceDir.relativize(sourceItem).compareTo(backupDir.relativize(backupItem)) < 0) { status = ADDED; } else if (sourceDir.relativize(sourceItem).compareTo(backupDir.relativize(backupItem)) > 0) { status = DELETED; } else if (Files.isDirectory(sourceItem) != Files.isDirectory(backupItem)) { status = MODIFIED; } else if (Files.isDirectory(sourceItem)) { status = SYNCHRONIZED; } else { fromAttributes = Files.readAttributes(sourceItem, BasicFileAttributes.class); toAttributes = Files.readAttributes(backupItem, BasicFileAttributes.class); if (Math.abs(fromAttributes.lastModifiedTime().toMillis() - toAttributes.lastModifiedTime().toMillis()) > 0) { status = MODIFIED; } else if (fromAttributes.size() != toAttributes.size()) { status = MODIFIED; } else { status = SYNCHRONIZED; } } switch (status) { case ADDED: syncAdded(sourceItem); sourceItem = (sourceIterator.hasNext() ? sourceIterator.next() : null); break; case DELETED: syncDeleted(sourceDir.resolve(backupDir.relativize(backupItem))); backupItem = (backupIterator.hasNext() ? backupIterator.next() : null); break; case MODIFIED: syncModified(sourceItem); case SYNCHRONIZED: default: if (Files.isDirectory(sourceItem)) { syncTree(sourceItem); } sourceItem = (sourceIterator.hasNext() ? sourceIterator.next() : null); backupItem = (backupIterator.hasNext() ? backupIterator.next() : null); break; } } } catch (final IOException | SecurityException e) { logger.debug("Failed to sync tree " + sourceSubDir, e); } }
From source file:gobblin.example.simplejson.SimpleJsonSource.java
@Override public List<WorkUnit> getWorkunits(SourceState state) { List<WorkUnit> workUnits = Lists.newArrayList(); if (!state.contains(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL)) { return workUnits; }//from w w w. j av a 2 s. c om // Create a single snapshot-type extract for all files Extract extract = new Extract(state, Extract.TableType.SNAPSHOT_ONLY, state.getProp(ConfigurationKeys.EXTRACT_NAMESPACE_NAME_KEY, "ExampleNamespace"), "ExampleTable"); String filesToPull = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL); File tempFileDir = new File("test_temp/"); // TODO: Delete the dir after completion. tempFileDir.mkdir(); String tempFileDirAbsolute = ""; try { tempFileDirAbsolute = tempFileDir.getCanonicalPath(); // Retrieve absolute path of temp folder } catch (IOException e) { e.printStackTrace(); } int nameCount = 0; int csvCount = 0; for (String file : Splitter.on(',').omitEmptyStrings().split(filesToPull)) { Iterator it = FileUtils.iterateFiles(new File(file), null, true); while (it.hasNext()) { try { File newFile = (File) it.next(); String basePath = newFile.getCanonicalPath(); // Retrieve absolute path of source Path path = newFile.toPath(); //call to rest api:, provide with file basePath String extension = ""; System.out.println("basePath is" + basePath); int i = basePath.lastIndexOf('.'); System.out.println("i"); if (i > 0) { extension = basePath.substring(i + 1); } String url_file_name = ""; int j = basePath.lastIndexOf('/'); if (j > 0) { url_file_name = basePath.substring(j + 1); } //hand off to rest api if (extension.equals("csv")) { System.out.println("CSVCSVCSV"); csvCount += 1; System.out.println("CURL________________________________________"); //Include basePath, filename, location you want to store file System.out.println( "curl http://localhost:8080" + basePath + "/" + Integer.toString(nameCount)); //10.0.2.2 is localhost from vagrant // Insert the nameCount so that it can be joined back later. Process p = Runtime.getRuntime() .exec("curl http://localhost:8080" + basePath + "/" + Integer.toString(nameCount)); // String myUrl = "http://localhost:8080/parse" + basePath + "&" + url_file_name + "&" + tempFileDirAbsolute; // System.out.println("------------------------------"); // System.out.println(myUrl); // try { // URL url = new URL(myUrl); // HttpURLConnection connection = (HttpURLConnection) url.openConnection(); // connection.setRequestMethod("GET"); // connection.connect(); // } catch (Exception e) { // e.printStackTrace(); // } } // Print filename and associated metadata System.out.println(basePath); BasicFileAttributes attr = Files.readAttributes(path, BasicFileAttributes.class); // System.out.println(" creationTime: " + attr.creationTime()); // System.out.println(" lastAccessTime: " + attr.lastAccessTime()); // System.out.println(" lastModifiedTime: " + attr.lastModifiedTime()); // System.out.println(" isDirectory: " + attr.isDirectory()); // System.out.println(" isOther: " + attr.isOther()); // System.out.println(" isRegularFile: " + attr.isRegularFile()); // System.out.println(" isSymbolicLink: " + attr.isSymbolicLink()); // System.out.println(" size: " + attr.size()); // System.out.println(" "); //creating intermediate JSON JSONObject intermediate = new JSONObject(); intermediate.put("filename", basePath); intermediate.put("timestamp", String.valueOf((new Date()).getTime())); intermediate.put("namespace", getMacAddress()); intermediate.put("creationTime", String.valueOf(attr.creationTime())); intermediate.put("lastAccessTime", String.valueOf(attr.lastAccessTime())); intermediate.put("lastModifiedTime", String.valueOf(attr.lastModifiedTime())); intermediate.put("isDirectory", String.valueOf(attr.isDirectory())); intermediate.put("isOther", String.valueOf(attr.isOther())); intermediate.put("isRegularFile", String.valueOf(attr.isRegularFile())); intermediate.put("isSymbolicLink", String.valueOf(attr.isSymbolicLink())); intermediate.put("size", attr.size()); // Create intermediate temp file nameCount += 1; String intermediateName = "/generated" + String.valueOf(nameCount) + ".json"; String finalName = tempFileDirAbsolute + intermediateName; FileWriter generated = new FileWriter(finalName); generated.write(intermediate.toJSONString()); generated.flush(); generated.close(); // Create one work unit for each file to pull WorkUnit workUnit = new WorkUnit(state, extract); workUnit.setProp(SOURCE_FILE_KEY, finalName); workUnits.add(workUnit); } catch (IOException e) { e.printStackTrace(); } } // write out number of files found to temp file try { FileWriter numCsvFiles = new FileWriter(tempFileDirAbsolute + "/numCsvFiles.txt"); numCsvFiles.write("" + csvCount); numCsvFiles.flush(); numCsvFiles.close(); FileWriter numFiles = new FileWriter(tempFileDirAbsolute + "/numFiles.txt"); numFiles.write("" + nameCount); numFiles.flush(); numFiles.close(); } catch (IOException e) { e.printStackTrace(); } } return workUnits; }
From source file:org.tinymediamanager.core.entities.MediaFile.java
License:asdf
/** * Gathers the media information via the native mediainfo lib. * /*from ww w .j a v a 2s.c om*/ * @param force * forces the execution, will not stop on already imported files */ public void gatherMediaInformation(boolean force) { // check for supported filetype if (!isValidMediainfoFormat()) { // okay, we have no valid MI file, be sure it will not be triggered any more if (StringUtils.isBlank(getContainerFormat())) { setContainerFormat(getExtension()); } return; } // mediainfo already gathered if (!force && !getContainerFormat().isEmpty()) { return; } // gather subtitle infos independent of MI if (getType() == MediaFileType.SUBTITLE) { gatherSubtitleInformation(); } // file size and last modified try { BasicFileAttributes attrs = Files.readAttributes(getFileAsPath(), BasicFileAttributes.class); filedate = attrs.lastModifiedTime().toMillis(); setFilesize(attrs.size()); } catch (IOException e) { if (miSnapshot == null) { // maybe we set it already (from ISO) so only display message when empty LOGGER.warn("could not get file information (size/date): " + e.getMessage()); } // do not set/return here - we might have set it already... and the next check does check for a 0-byte file // setContainerFormat(getExtension()); // return; } // do not work further on 0 byte files if (getFilesize() == 0) { LOGGER.warn("0 Byte file detected: " + this.filename); // set container format to do not trigger it again setContainerFormat(getExtension()); return; } // do not work further on subtitles/NFO files if (type == MediaFileType.SUBTITLE || type == MediaFileType.NFO) { // set container format to do not trigger it again setContainerFormat(getExtension()); return; } // get media info LOGGER.debug("start MediaInfo for " + this.getFileAsPath()); long discFilesSizes = 0L; if (isISO) { discFilesSizes = getMediaInfoSnapshotFromISO(); } else { getMediaInfoSnapshot(); } if (miSnapshot == null) { // MI could not be opened LOGGER.error("error getting MediaInfo for " + this.filename); // set container format to do not trigger it again setContainerFormat(getExtension()); closeMediaInfo(); return; } LOGGER.trace("got MI"); String height = ""; String scanType = ""; String width = ""; String videoCodec = ""; switch (type) { case VIDEO: case VIDEO_EXTRA: case SAMPLE: case TRAILER: height = getMediaInfo(StreamKind.Video, 0, "Height"); scanType = getMediaInfo(StreamKind.Video, 0, "ScanType"); width = getMediaInfo(StreamKind.Video, 0, "Width"); videoCodec = getMediaInfo(StreamKind.Video, 0, "CodecID/Hint", "Format"); // fix for Microsoft VC-1 if (StringUtils.containsIgnoreCase(videoCodec, "Microsoft")) { videoCodec = getMediaInfo(StreamKind.Video, 0, "Format"); } // ***************** // get audio streams // ***************** // int streams = getMediaInfo().streamCount(StreamKind.Audio); int streams = 0; if (streams == 0) { // fallback 1 String cnt = getMediaInfo(StreamKind.General, 0, "AudioCount"); try { streams = Integer.parseInt(cnt); } catch (Exception e) { streams = 0; } } if (streams == 0) { // fallback 2 String cnt = getMediaInfo(StreamKind.Audio, 0, "StreamCount"); try { streams = Integer.parseInt(cnt); } catch (Exception e) { streams = 0; } } audioStreams.clear(); for (int i = 0; i < streams; i++) { MediaFileAudioStream stream = new MediaFileAudioStream(); String audioCodec = getMediaInfo(StreamKind.Audio, i, "CodecID/Hint", "Format"); audioCodec = audioCodec.replaceAll("\\p{Punct}", ""); String audioAddition = getMediaInfo(StreamKind.Audio, i, "Format_Profile"); if ("dts".equalsIgnoreCase(audioCodec) && StringUtils.isNotBlank(audioAddition)) { if (audioAddition.contains("ES")) { audioCodec = "DTSHD-ES"; } if (audioAddition.contains("HRA")) { audioCodec = "DTSHD-HRA"; } if (audioAddition.contains("MA")) { audioCodec = "DTSHD-MA"; } } stream.setCodec(audioCodec); // AAC sometimes codes channels into Channel(s)_Original String channels = getMediaInfo(StreamKind.Audio, i, "Channel(s)_Original", "Channel(s)"); stream.setChannels(StringUtils.isEmpty(channels) ? "" : channels + "ch"); try { String br = getMediaInfo(StreamKind.Audio, i, "BitRate"); stream.setBitrate(Integer.parseInt(br) / 1024); } catch (Exception ignored) { } String language = getMediaInfo(StreamKind.Audio, i, "Language/String", "Language"); if (language.isEmpty()) { if (!isDiscFile()) { // video_ts parsed 'ts' as Tsonga // try to parse from filename String shortname = getBasename().toLowerCase(Locale.ROOT); stream.setLanguage(parseLanguageFromString(shortname)); } } else { stream.setLanguage(parseLanguageFromString(language)); } audioStreams.add(stream); } // ******************** // get subtitle streams // ******************** // streams = getMediaInfo().streamCount(StreamKind.Text); streams = 0; if (streams == 0) { // fallback 1 String cnt = getMediaInfo(StreamKind.General, 0, "TextCount"); try { streams = Integer.parseInt(cnt); } catch (Exception e) { streams = 0; } } if (streams == 0) { // fallback 2 String cnt = getMediaInfo(StreamKind.Text, 0, "StreamCount"); try { streams = Integer.parseInt(cnt); } catch (Exception e) { streams = 0; } } subtitles.clear(); for (int i = 0; i < streams; i++) { MediaFileSubtitle stream = new MediaFileSubtitle(); String codec = getMediaInfo(StreamKind.Text, i, "CodecID/Hint", "Format"); stream.setCodec(codec.replaceAll("\\p{Punct}", "")); String lang = getMediaInfo(StreamKind.Text, i, "Language/String", "Language"); stream.setLanguage(parseLanguageFromString(lang)); String forced = getMediaInfo(StreamKind.Text, i, "Forced"); boolean b = forced.equalsIgnoreCase("true") || forced.equalsIgnoreCase("yes"); stream.setForced(b); subtitles.add(stream); } // detect 3D video (mainly from MKV files) // sample Mediainfo output: // MultiView_Count : 2 // MultiView_Layout : Top-Bottom (left eye first) // MultiView_Layout : Side by Side (left eye first) String mvc = getMediaInfo(StreamKind.Video, 0, "MultiView_Count"); if (!StringUtils.isEmpty(mvc) && mvc.equals("2")) { video3DFormat = VIDEO_3D; String mvl = getMediaInfo(StreamKind.Video, 0, "MultiView_Layout").toLowerCase(Locale.ROOT); LOGGER.debug("3D detected :) " + mvl); if (!StringUtils.isEmpty(mvl) && mvl.contains("top") && mvl.contains("bottom")) { video3DFormat = VIDEO_3D_TAB; } if (!StringUtils.isEmpty(mvl) && mvl.contains("side")) { video3DFormat = VIDEO_3D_SBS; } } break; case AUDIO: MediaFileAudioStream stream = new MediaFileAudioStream(); String audioCodec = getMediaInfo(StreamKind.Audio, 0, "CodecID/Hint", "Format"); stream.setCodec(audioCodec.replaceAll("\\p{Punct}", "")); String channels = getMediaInfo(StreamKind.Audio, 0, "Channel(s)"); stream.setChannels(StringUtils.isEmpty(channels) ? "" : channels + "ch"); try { String br = getMediaInfo(StreamKind.Audio, 0, "BitRate"); stream.setBitrate(Integer.parseInt(br) / 1024); } catch (Exception e) { } String language = getMediaInfo(StreamKind.Audio, 0, "Language/String", "Language"); if (language.isEmpty()) { // try to parse from filename String shortname = getBasename().toLowerCase(Locale.ROOT); stream.setLanguage(parseLanguageFromString(shortname)); } else { stream.setLanguage(parseLanguageFromString(language)); } audioStreams.clear(); audioStreams.add(stream); break; case POSTER: case BANNER: case FANART: case THUMB: case EXTRAFANART: case GRAPHIC: case SEASON_POSTER: case LOGO: case CLEARLOGO: case CLEARART: case DISCART: case EXTRATHUMB: height = getMediaInfo(StreamKind.Image, 0, "Height"); // scanType = getMediaInfo(StreamKind.Image, 0, "ScanType"); // no scantype on graphics width = getMediaInfo(StreamKind.Image, 0, "Width"); videoCodec = getMediaInfo(StreamKind.Image, 0, "CodecID/Hint", "Format"); // System.out.println(height + "-" + width + "-" + videoCodec); break; case NFO: // do nothing here, but do not display default warning (since we got the filedate) break; default: LOGGER.warn("no mediainformation handling for MediaFile type " + getType() + " yet."); break; } // video codec // e.g. XviD, x264, DivX 5, MPEG-4 Visual, AVC, etc. // get first token (e.g. DivX 5 => DivX) setVideoCodec(StringUtils.isEmpty(videoCodec) ? "" : new Scanner(videoCodec).next()); // container format for all except subtitles (subtitle container format is handled another way) if (type == MediaFileType.SUBTITLE) { setContainerFormat(getExtension()); } else { String extensions = getMediaInfo(StreamKind.General, 0, "Codec/Extensions", "Format"); // get first extension setContainerFormat( StringUtils.isBlank(extensions) ? "" : new Scanner(extensions).next().toLowerCase(Locale.ROOT)); // if container format is still empty -> insert the extension if (StringUtils.isBlank(containerFormat)) { setContainerFormat(getExtension()); } } if (height.isEmpty() || scanType.isEmpty()) { setExactVideoFormat(""); } else { setExactVideoFormat(height + Character.toLowerCase(scanType.charAt(0))); } // video dimension if (!width.isEmpty()) { try { setVideoWidth(Integer.parseInt(width)); } catch (NumberFormatException e) { setVideoWidth(0); } } if (!height.isEmpty()) { try { setVideoHeight(Integer.parseInt(height)); } catch (NumberFormatException e) { setVideoHeight(0); } } switch (type) { case VIDEO: case VIDEO_EXTRA: case SAMPLE: case TRAILER: case AUDIO: // overall bitrate (OverallBitRate/String) String br = getMediaInfo(StreamKind.General, 0, "OverallBitRate"); if (!br.isEmpty()) { try { setOverallBitRate(Integer.parseInt(br) / 1024); // in kbps } catch (NumberFormatException e) { setOverallBitRate(0); } } // Duration;Play time of the stream in ms // Duration/String;Play time in format : XXx YYy only, YYy omited if zero // Duration/String1;Play time in format : HHh MMmn SSs MMMms, XX om.if.z. // Duration/String2;Play time in format : XXx YYy only, YYy omited if zero // Duration/String3;Play time in format : HH:MM:SS.MMM if (!isISO) { // ISO files get duration accumulated with snapshot String dur = getMediaInfo(StreamKind.General, 0, "Duration"); if (!dur.isEmpty()) { try { Double d = Double.parseDouble(dur); setDuration(d.intValue() / 1000); } catch (NumberFormatException e) { setDuration(0); } } } else { // do some sanity check, to see, if we have an invalid DVD structure // eg when the sum(filesize) way higher than ISO size LOGGER.trace("ISO size:" + filesize + " dataSize:" + discFilesSizes + " = diff:" + Math.abs(discFilesSizes - filesize)); if (discFilesSizes > 0 && filesize > 0) { long gig = 1024 * 1024 * 1024; if (Math.abs(discFilesSizes - filesize) > gig) { LOGGER.error("ISO file seems to have an invalid structure - ignore duration"); // we set the ISO duration to zero, // so the standard getDuration() will always get the scraped duration setDuration(0); } } } default: break; } LOGGER.trace("extracted MI"); // close mediainfo lib closeMediaInfo(); LOGGER.trace("closed MI"); }
From source file:gov.noaa.pfel.coastwatch.util.FileVisitorDNLS.java
/** Invoked for a file in a directory. */ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { int oSize = directoryPA.size(); try {/*from w w w. j a va 2 s . c o m*/ String name = file.getFileName().toString(); if (!fileNamePattern.matcher(name).matches()) { if (debugMode) String2.log(">> fileName doesn't match: name=" + name + " regex=" + fileNameRegex); return FileVisitResult.CONTINUE; } //getParent returns \\ or /, without trailing / String ttDir = String2.replaceAll(file.getParent().toString(), fromSlash, toSlash) + toSlash; if (debugMode) String2.log(">> add fileName: " + ttDir + name); directoryPA.add(ttDir); namePA.add(name); lastModifiedPA.add(attrs.lastModifiedTime().toMillis()); sizePA.add(attrs.size()); //for debugging only: //String2.log(ttDir + name + // " mod=" + attrs.lastModifiedTime().toMillis() + // " size=" + attrs.size()); } catch (Throwable t) { if (directoryPA.size() > oSize) directoryPA.remove(oSize); if (namePA.size() > oSize) namePA.remove(oSize); if (lastModifiedPA.size() > oSize) lastModifiedPA.remove(oSize); if (sizePA.size() > oSize) sizePA.remove(oSize); String2.log(MustBe.throwableToString(t)); } return FileVisitResult.CONTINUE; }
From source file:org.apache.nifi.controller.repository.FileSystemRepository.java
private long destroyExpiredArchives(final String containerName, final Path container) throws IOException { archiveExpirationLog.debug("Destroying Expired Archives for Container {}", containerName); final List<ArchiveInfo> notYetExceedingThreshold = new ArrayList<>(); long removalTimeThreshold = System.currentTimeMillis() - maxArchiveMillis; long oldestArchiveDateFound = System.currentTimeMillis(); // determine how much space we must have in order to stop deleting old data final Long minRequiredSpace = minUsableContainerBytesForArchive.get(containerName); if (minRequiredSpace == null) { archiveExpirationLog/*w w w . j av a 2s. c om*/ .debug("Could not determine minimum required space so will not destroy any archived data"); return -1L; } final long usableSpace = getContainerUsableSpace(containerName); final ContainerState containerState = containerStateMap.get(containerName); // First, delete files from our queue final long startNanos = System.nanoTime(); final long toFree = minRequiredSpace - usableSpace; final BlockingQueue<ArchiveInfo> fileQueue = archivedFiles.get(containerName); if (archiveExpirationLog.isDebugEnabled()) { if (toFree < 0) { archiveExpirationLog.debug( "Currently {} bytes free for Container {}; requirement is {} byte free, so no need to free space until an additional {} bytes are used", usableSpace, containerName, minRequiredSpace, Math.abs(toFree)); } else { archiveExpirationLog.debug( "Currently {} bytes free for Container {}; requirement is {} byte free, so need to free {} bytes", usableSpace, containerName, minRequiredSpace, toFree); } } ArchiveInfo toDelete; int deleteCount = 0; long freed = 0L; while ((toDelete = fileQueue.peek()) != null) { try { final long fileSize = toDelete.getSize(); removalTimeThreshold = System.currentTimeMillis() - maxArchiveMillis; // we use fileQueue.peek above instead of fileQueue.poll() because we don't always want to // remove the head of the queue. Instead, we want to remove it only if we plan to delete it. // In order to accomplish this, we just peek at the head and check if it should be deleted. // If so, then we call poll() to remove it if (freed < toFree || getLastModTime(toDelete.toPath()) < removalTimeThreshold) { toDelete = fileQueue.poll(); // remove the head of the queue, which is already stored in 'toDelete' Files.deleteIfExists(toDelete.toPath()); containerState.decrementArchiveCount(); LOG.debug( "Deleted archived ContentClaim with ID {} from Container {} because the archival size was exceeding the max configured size", toDelete.getName(), containerName); freed += fileSize; deleteCount++; } // If we'd freed up enough space, we're done... unless the next file needs to be destroyed based on time. if (freed >= toFree) { // If the last mod time indicates that it should be removed, just continue loop. if (deleteBasedOnTimestamp(fileQueue, removalTimeThreshold)) { archiveExpirationLog.debug( "Freed enough space ({} bytes freed, needed to free {} bytes) but will continue to expire data based on timestamp", freed, toFree); continue; } archiveExpirationLog.debug( "Freed enough space ({} bytes freed, needed to free {} bytes). Finished expiring data", freed, toFree); final ArchiveInfo archiveInfo = fileQueue.peek(); final long oldestArchiveDate = archiveInfo == null ? System.currentTimeMillis() : getLastModTime(archiveInfo.toPath()); // Otherwise, we're done. Return the last mod time of the oldest file in the container's archive. final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); if (deleteCount > 0) { LOG.info( "Deleted {} files from archive for Container {}; oldest Archive Date is now {}; container cleanup took {} millis", deleteCount, containerName, new Date(oldestArchiveDate), millis); } else { LOG.debug( "Deleted {} files from archive for Container {}; oldest Archive Date is now {}; container cleanup took {} millis", deleteCount, containerName, new Date(oldestArchiveDate), millis); } return oldestArchiveDate; } } catch (final IOException ioe) { LOG.warn("Failed to delete {} from archive due to {}", toDelete, ioe.toString()); if (LOG.isDebugEnabled()) { LOG.warn("", ioe); } } } // Go through each container and grab the archived data into a List archiveExpirationLog.debug("Searching for more archived data to expire"); final StopWatch stopWatch = new StopWatch(true); for (int i = 0; i < SECTIONS_PER_CONTAINER; i++) { final Path sectionContainer = container.resolve(String.valueOf(i)); final Path archive = sectionContainer.resolve("archive"); if (!Files.exists(archive)) { continue; } try { final long timestampThreshold = removalTimeThreshold; Files.walkFileTree(archive, new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { if (attrs.isDirectory()) { return FileVisitResult.CONTINUE; } final long lastModTime = getLastModTime(file); if (lastModTime < timestampThreshold) { try { Files.deleteIfExists(file); containerState.decrementArchiveCount(); LOG.debug( "Deleted archived ContentClaim with ID {} from Container {} because it was older than the configured max archival duration", file.toFile().getName(), containerName); } catch (final IOException ioe) { LOG.warn( "Failed to remove archived ContentClaim with ID {} from Container {} due to {}", file.toFile().getName(), containerName, ioe.toString()); if (LOG.isDebugEnabled()) { LOG.warn("", ioe); } } } else if (usableSpace < minRequiredSpace) { notYetExceedingThreshold .add(new ArchiveInfo(container, file, attrs.size(), lastModTime)); } return FileVisitResult.CONTINUE; } }); } catch (final IOException ioe) { LOG.warn("Failed to cleanup archived files in {} due to {}", archive, ioe.toString()); if (LOG.isDebugEnabled()) { LOG.warn("", ioe); } } } final long deleteExpiredMillis = stopWatch.getElapsed(TimeUnit.MILLISECONDS); // Sort the list according to last modified time Collections.sort(notYetExceedingThreshold, new Comparator<ArchiveInfo>() { @Override public int compare(final ArchiveInfo o1, final ArchiveInfo o2) { return Long.compare(o1.getLastModTime(), o2.getLastModTime()); } }); final long sortRemainingMillis = stopWatch.getElapsed(TimeUnit.MILLISECONDS) - deleteExpiredMillis; // Delete the oldest data archiveExpirationLog.debug("Deleting data based on timestamp"); final Iterator<ArchiveInfo> itr = notYetExceedingThreshold.iterator(); int counter = 0; while (itr.hasNext()) { final ArchiveInfo archiveInfo = itr.next(); try { final Path path = archiveInfo.toPath(); Files.deleteIfExists(path); containerState.decrementArchiveCount(); LOG.debug( "Deleted archived ContentClaim with ID {} from Container {} because the archival size was exceeding the max configured size", archiveInfo.getName(), containerName); // Check if we've freed enough space every 25 files that we destroy if (++counter % 25 == 0) { if (getContainerUsableSpace(containerName) > minRequiredSpace) { // check if we can stop now LOG.debug("Finished cleaning up archive for Container {}", containerName); break; } } } catch (final IOException ioe) { LOG.warn("Failed to delete {} from archive due to {}", archiveInfo, ioe.toString()); if (LOG.isDebugEnabled()) { LOG.warn("", ioe); } } itr.remove(); } final long deleteOldestMillis = stopWatch.getElapsed(TimeUnit.MILLISECONDS) - sortRemainingMillis - deleteExpiredMillis; long oldestContainerArchive; if (notYetExceedingThreshold.isEmpty()) { oldestContainerArchive = System.currentTimeMillis(); } else { oldestContainerArchive = notYetExceedingThreshold.get(0).getLastModTime(); } if (oldestContainerArchive < oldestArchiveDateFound) { oldestArchiveDateFound = oldestContainerArchive; } // Queue up the files in the order that they should be destroyed so that we don't have to scan the directories for a while. for (final ArchiveInfo toEnqueue : notYetExceedingThreshold.subList(0, Math.min(100000, notYetExceedingThreshold.size()))) { fileQueue.offer(toEnqueue); } final long cleanupMillis = stopWatch.getElapsed(TimeUnit.MILLISECONDS) - deleteOldestMillis - sortRemainingMillis - deleteExpiredMillis; LOG.debug( "Oldest Archive Date for Container {} is {}; delete expired = {} ms, sort remaining = {} ms, delete oldest = {} ms, cleanup = {} ms", containerName, new Date(oldestContainerArchive), deleteExpiredMillis, sortRemainingMillis, deleteOldestMillis, cleanupMillis); return oldestContainerArchive; }