List of usage examples for java.nio.file.attribute BasicFileAttributes isDirectory
boolean isDirectory();
From source file:Main.java
public static void main(String[] args) throws Exception { BasicFileAttributes attr = null; Path path = Paths.get("C:/tutorial/Java/JavaFX", "Topic.txt"); attr = Files.readAttributes(path, BasicFileAttributes.class); System.out.println("Is directory ? " + attr.isDirectory()); }
From source file:com.datazuul.iiif.presentation.api.ManifestGenerator.java
public static void main(String[] args) throws ParseException, JsonProcessingException, IOException, URISyntaxException { Options options = new Options(); options.addOption("d", true, "Absolute file path to the directory containing the image files."); CommandLineParser parser = new DefaultParser(); CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("d")) { String imageDirectoryPath = cmd.getOptionValue("d"); Path imageDirectory = Paths.get(imageDirectoryPath); final List<Path> files = new ArrayList<>(); try {/*from w ww . j av a2s .c o m*/ Files.walkFileTree(imageDirectory, new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (!attrs.isDirectory()) { // TODO there must be a more elegant solution for filtering jpeg files... if (file.getFileName().toString().endsWith("jpg")) { files.add(file); } } return FileVisitResult.CONTINUE; } }); } catch (IOException e) { e.printStackTrace(); } Collections.sort(files, new Comparator() { @Override public int compare(Object fileOne, Object fileTwo) { String filename1 = ((Path) fileOne).getFileName().toString(); String filename2 = ((Path) fileTwo).getFileName().toString(); try { // numerical sorting Integer number1 = Integer.parseInt(filename1.substring(0, filename1.lastIndexOf("."))); Integer number2 = Integer.parseInt(filename2.substring(0, filename2.lastIndexOf("."))); return number1.compareTo(number2); } catch (NumberFormatException nfe) { // alpha-numerical sorting return filename1.compareToIgnoreCase(filename2); } } }); generateManifest(imageDirectory.getFileName().toString(), files); } else { // automatically generate the help statement HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("ManifestGenerator", options); } }
From source file:Test.java
public static void main(String[] args) throws Exception { Path path = FileSystems.getDefault().getPath("/home/docs/users.txt"); // BasicFileAttributes attributes = Files.readAttributes(path, // BasicFileAttributes.class); BasicFileAttributeView view = Files.getFileAttributeView(path, BasicFileAttributeView.class); BasicFileAttributes attributes = view.readAttributes(); System.out.println("Creation Time: " + attributes.creationTime()); System.out.println("Last Accessed Time: " + attributes.lastAccessTime()); System.out.println("Last Modified Time: " + attributes.lastModifiedTime()); System.out.println("File Key: " + attributes.fileKey()); System.out.println("Directory: " + attributes.isDirectory()); System.out.println("Other Type of File: " + attributes.isOther()); System.out.println("Regular File: " + attributes.isRegularFile()); System.out.println("Symbolic File: " + attributes.isSymbolicLink()); System.out.println("Size: " + attributes.size()); }
From source file:com.google.jenkins.plugins.persistentmaster.volume.zip.ZipCreator.java
@Override public void addFile(Path file, String pathInVolume, @Nullable BasicFileAttributes attrs) throws IOException { Preconditions.checkState(!closed, "Volume closed"); if (attrs == null) { // make sure attrs are available attrs = Files.readAttributes(file, BasicFileAttributes.class); }//from w w w.j a v a2 s. c o m if (attrs.isSymbolicLink()) { copySymlink(file, pathInVolume); } else if (attrs.isDirectory()) { copyDirectory(pathInVolume); } else { copyRegularFile(file, pathInVolume); } fileCount++; }
From source file:net.mozq.picto.core.ProcessCore.java
public static void findFiles(ProcessCondition processCondition, Consumer<ProcessData> processDataSetter, BooleanSupplier processStopper) throws IOException { Set<FileVisitOption> fileVisitOptionSet; if (processCondition.isFollowLinks()) { fileVisitOptionSet = EnumSet.of(FileVisitOption.FOLLOW_LINKS); } else {//from ww w .j a va2 s . c om fileVisitOptionSet = Collections.emptySet(); } Files.walkFileTree(processCondition.getSrcRootPath(), fileVisitOptionSet, processCondition.getDept(), new SimpleFileVisitor<Path>() { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { if (processStopper.getAsBoolean()) { return FileVisitResult.TERMINATE; } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (attrs.isDirectory()) { return FileVisitResult.SKIP_SUBTREE; } if (processStopper.getAsBoolean()) { return FileVisitResult.TERMINATE; } if (!processCondition.getPathFilter().accept(file, attrs)) { return FileVisitResult.SKIP_SUBTREE; } Path rootRelativeSubPath = processCondition.getSrcRootPath().relativize(file.getParent()); ImageMetadata imageMetadata = getImageMetadata(file); Date baseDate; if (processCondition.isChangeFileCreationDate() || processCondition.isChangeFileModifiedDate() || processCondition.isChangeFileAccessDate() || processCondition.isChangeExifDate()) { baseDate = getBaseDate(processCondition, file, attrs, imageMetadata); } else { baseDate = null; } String destSubPathname = processCondition.getDestSubPathFormat().format(varName -> { try { switch (varName) { case "Now": return new Date(); case "ParentSubPath": return rootRelativeSubPath.toString(); case "FileName": return file.getFileName().toString(); case "BaseName": return FileUtilz.getBaseName(file.getFileName().toString()); case "Extension": return FileUtilz.getExt(file.getFileName().toString()); case "Size": return Long.valueOf(Files.size(file)); case "CreationDate": return (processCondition.isChangeFileCreationDate()) ? baseDate : new Date(attrs.creationTime().toMillis()); case "ModifiedDate": return (processCondition.isChangeFileModifiedDate()) ? baseDate : new Date(attrs.lastModifiedTime().toMillis()); case "AccessDate": return (processCondition.isChangeFileAccessDate()) ? baseDate : new Date(attrs.lastAccessTime().toMillis()); case "PhotoTakenDate": return (processCondition.isChangeExifDate()) ? baseDate : getPhotoTakenDate(file, imageMetadata); case "Width": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_EXIF_IMAGE_WIDTH); case "Height": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_EXIF_IMAGE_LENGTH); case "FNumber": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_FNUMBER); case "Aperture": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_APERTURE_VALUE); case "MaxAperture": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_MAX_APERTURE_VALUE); case "ISO": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_ISO); case "FocalLength": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_FOCAL_LENGTH); // ? case "FocalLength35mm": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_FOCAL_LENGTH_IN_35MM_FORMAT); case "ShutterSpeed": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_SHUTTER_SPEED_VALUE); case "Exposure": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_EXPOSURE); // case "ExposureTime": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_EXPOSURE_TIME); // case "ExposureMode": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_EXPOSURE_MODE); case "ExposureProgram": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_EXPOSURE_PROGRAM); case "Brightness": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_BRIGHTNESS_VALUE); case "WhiteBalance": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_WHITE_BALANCE_1); case "LightSource": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_LIGHT_SOURCE); case "Lens": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_LENS); case "LensMake": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_LENS_MAKE); case "LensModel": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_LENS_MODEL); case "LensSerialNumber": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_LENS_SERIAL_NUMBER); case "Make": return getEXIFStringValue(imageMetadata, TiffTagConstants.TIFF_TAG_MAKE); case "Model": return getEXIFStringValue(imageMetadata, TiffTagConstants.TIFF_TAG_MODEL); case "SerialNumber": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_SERIAL_NUMBER); case "Software": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_SOFTWARE); case "ProcessingSoftware": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_PROCESSING_SOFTWARE); case "OwnerName": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_OWNER_NAME); case "CameraOwnerName": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_CAMERA_OWNER_NAME); case "GPSLat": return getEXIFGpsLat(imageMetadata); case "GPSLatDeg": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LATITUDE, 0); case "GPSLatMin": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LATITUDE, 1); case "GPSLatSec": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LATITUDE, 2); case "GPSLatRef": return getEXIFStringValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LATITUDE_REF); case "GPSLon": return getEXIFGpsLon(imageMetadata); case "GPSLonDeg": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LONGITUDE, 0); case "GPSLonMin": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LONGITUDE, 1); case "GPSLonSec": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LONGITUDE, 2); case "GPSLonRef": return getEXIFStringValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LONGITUDE_REF); case "GPSAlt": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_ALTITUDE); case "GPSAltRef": return getEXIFIntValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_ALTITUDE_REF); default: throw new PictoInvalidDestinationPathException(Messages .getString("message.warn.invalid.destSubPath.varName", varName)); } } catch (PictoException e) { throw e; } catch (Exception e) { throw new PictoInvalidDestinationPathException( Messages.getString("message.warn.invalid.destSubPath.pattern"), e); } }); Path destSubPath = processCondition.getDestRootPath().resolve(destSubPathname).normalize(); if (!destSubPath.startsWith(processCondition.getDestRootPath())) { throw new PictoInvalidDestinationPathException( Messages.getString("message.warn.invalid.destination.path", destSubPath)); } ProcessData processData = new ProcessData(); processData.setSrcPath(file); processData.setSrcFileAttributes(attrs); processData.setDestPath(destSubPath); processData.setBaseDate(baseDate); processDataSetter.accept(processData); return FileVisitResult.CONTINUE; } }); }
From source file:de.tiqsolutions.hdfs.BasicFileAttributeViewImpl.java
Map<String, Object> readAttributes(String attributes) throws IOException { BasicFileAttributes attr = readAttributes(); List<String> attrlist = Arrays.asList(attributes.split(",")); boolean readall = attrlist.contains("*"); Map<String, Object> ret = new HashMap<>(); if (readall || attrlist.contains("fileKey")) ret.put("fileKey", attr.fileKey()); if (readall || attrlist.contains("creationTime")) ret.put("creationTime", attr.creationTime()); if (readall || attrlist.contains("isDirectory")) ret.put("isDirectory", attr.isDirectory()); if (readall || attrlist.contains("isOther")) ret.put("isOther", attr.isOther()); if (readall || attrlist.contains("isRegularFile")) ret.put("isRegularFile", attr.isRegularFile()); if (readall || attrlist.contains("isSymbolicLink")) ret.put("isSymbolicLink", attr.isSymbolicLink()); if (readall || attrlist.contains("lastAccessTime")) ret.put("lastAccessTime", attr.lastAccessTime()); if (readall || attrlist.contains("lastModifiedTime")) ret.put("lastModifiedTime", attr.lastModifiedTime()); if (readall || attrlist.contains("size")) ret.put("size", attr.size()); return ret;/*from w w w . ja v a2 s.co m*/ }
From source file:gobblin.example.simplejson.SimpleJsonSource.java
@Override public List<WorkUnit> getWorkunits(SourceState state) { List<WorkUnit> workUnits = Lists.newArrayList(); if (!state.contains(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL)) { return workUnits; }/*from w w w . jav a 2 s . c om*/ // Create a single snapshot-type extract for all files Extract extract = new Extract(state, Extract.TableType.SNAPSHOT_ONLY, state.getProp(ConfigurationKeys.EXTRACT_NAMESPACE_NAME_KEY, "ExampleNamespace"), "ExampleTable"); String filesToPull = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL); File tempFileDir = new File("test_temp/"); // TODO: Delete the dir after completion. tempFileDir.mkdir(); String tempFileDirAbsolute = ""; try { tempFileDirAbsolute = tempFileDir.getCanonicalPath(); // Retrieve absolute path of temp folder } catch (IOException e) { e.printStackTrace(); } int nameCount = 0; int csvCount = 0; for (String file : Splitter.on(',').omitEmptyStrings().split(filesToPull)) { Iterator it = FileUtils.iterateFiles(new File(file), null, true); while (it.hasNext()) { try { File newFile = (File) it.next(); String basePath = newFile.getCanonicalPath(); // Retrieve absolute path of source Path path = newFile.toPath(); //call to rest api:, provide with file basePath String extension = ""; System.out.println("basePath is" + basePath); int i = basePath.lastIndexOf('.'); System.out.println("i"); if (i > 0) { extension = basePath.substring(i + 1); } String url_file_name = ""; int j = basePath.lastIndexOf('/'); if (j > 0) { url_file_name = basePath.substring(j + 1); } //hand off to rest api if (extension.equals("csv")) { System.out.println("CSVCSVCSV"); csvCount += 1; System.out.println("CURL________________________________________"); //Include basePath, filename, location you want to store file System.out.println( "curl http://localhost:8080" + basePath + "/" + Integer.toString(nameCount)); //10.0.2.2 is localhost from vagrant // Insert the nameCount so that it can be joined back later. Process p = Runtime.getRuntime() .exec("curl http://localhost:8080" + basePath + "/" + Integer.toString(nameCount)); // String myUrl = "http://localhost:8080/parse" + basePath + "&" + url_file_name + "&" + tempFileDirAbsolute; // System.out.println("------------------------------"); // System.out.println(myUrl); // try { // URL url = new URL(myUrl); // HttpURLConnection connection = (HttpURLConnection) url.openConnection(); // connection.setRequestMethod("GET"); // connection.connect(); // } catch (Exception e) { // e.printStackTrace(); // } } // Print filename and associated metadata System.out.println(basePath); BasicFileAttributes attr = Files.readAttributes(path, BasicFileAttributes.class); // System.out.println(" creationTime: " + attr.creationTime()); // System.out.println(" lastAccessTime: " + attr.lastAccessTime()); // System.out.println(" lastModifiedTime: " + attr.lastModifiedTime()); // System.out.println(" isDirectory: " + attr.isDirectory()); // System.out.println(" isOther: " + attr.isOther()); // System.out.println(" isRegularFile: " + attr.isRegularFile()); // System.out.println(" isSymbolicLink: " + attr.isSymbolicLink()); // System.out.println(" size: " + attr.size()); // System.out.println(" "); //creating intermediate JSON JSONObject intermediate = new JSONObject(); intermediate.put("filename", basePath); intermediate.put("timestamp", String.valueOf((new Date()).getTime())); intermediate.put("namespace", getMacAddress()); intermediate.put("creationTime", String.valueOf(attr.creationTime())); intermediate.put("lastAccessTime", String.valueOf(attr.lastAccessTime())); intermediate.put("lastModifiedTime", String.valueOf(attr.lastModifiedTime())); intermediate.put("isDirectory", String.valueOf(attr.isDirectory())); intermediate.put("isOther", String.valueOf(attr.isOther())); intermediate.put("isRegularFile", String.valueOf(attr.isRegularFile())); intermediate.put("isSymbolicLink", String.valueOf(attr.isSymbolicLink())); intermediate.put("size", attr.size()); // Create intermediate temp file nameCount += 1; String intermediateName = "/generated" + String.valueOf(nameCount) + ".json"; String finalName = tempFileDirAbsolute + intermediateName; FileWriter generated = new FileWriter(finalName); generated.write(intermediate.toJSONString()); generated.flush(); generated.close(); // Create one work unit for each file to pull WorkUnit workUnit = new WorkUnit(state, extract); workUnit.setProp(SOURCE_FILE_KEY, finalName); workUnits.add(workUnit); } catch (IOException e) { e.printStackTrace(); } } // write out number of files found to temp file try { FileWriter numCsvFiles = new FileWriter(tempFileDirAbsolute + "/numCsvFiles.txt"); numCsvFiles.write("" + csvCount); numCsvFiles.flush(); numCsvFiles.close(); FileWriter numFiles = new FileWriter(tempFileDirAbsolute + "/numFiles.txt"); numFiles.write("" + nameCount); numFiles.flush(); numFiles.close(); } catch (IOException e) { e.printStackTrace(); } } return workUnits; }
From source file:org.apache.nifi.controller.repository.FileSystemRepository.java
private synchronized void initializeRepository() throws IOException { final Map<String, Path> realPathMap = new HashMap<>(); final ExecutorService executor = Executors.newFixedThreadPool(containers.size()); final List<Future<Long>> futures = new ArrayList<>(); // Run through each of the containers. For each container, create the sections if necessary. // Then, we need to scan through the archived data so that we can determine what the oldest // archived data is, so that we know when we have to start aging data off. for (final Map.Entry<String, Path> container : containers.entrySet()) { final String containerName = container.getKey(); final ContainerState containerState = containerStateMap.get(containerName); final Path containerPath = container.getValue(); final boolean pathExists = Files.exists(containerPath); final Path realPath; if (pathExists) { realPath = containerPath.toRealPath(); } else {/*ww w. j av a 2 s.co m*/ realPath = Files.createDirectories(containerPath).toRealPath(); } for (int i = 0; i < SECTIONS_PER_CONTAINER; i++) { Files.createDirectories(realPath.resolve(String.valueOf(i))); } realPathMap.put(containerName, realPath); // We need to scan the archive directories to find out the oldest timestamp so that know whether or not we // will have to delete archived data based on time threshold. Scanning all of the directories can be very // expensive because of all of the disk accesses. So we do this in multiple threads. Since containers are // often unique to a disk, we just map 1 thread to each container. final Callable<Long> scanContainer = new Callable<Long>() { @Override public Long call() throws IOException { final AtomicLong oldestDateHolder = new AtomicLong(0L); // the path already exists, so scan the path to find any files and update maxIndex to the max of // all filenames seen. Files.walkFileTree(realPath, new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { LOG.warn("Content repository contains un-readable file or directory '" + file.getFileName() + "'. Skipping. ", exc); return FileVisitResult.SKIP_SUBTREE; } @Override public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { if (attrs.isDirectory()) { return FileVisitResult.CONTINUE; } // Check if this is an 'archive' directory final Path relativePath = realPath.relativize(file); if (relativePath.getNameCount() > 3 && ARCHIVE_DIR_NAME.equals(relativePath.subpath(1, 2).toString())) { final long lastModifiedTime = getLastModTime(file); if (lastModifiedTime < oldestDateHolder.get()) { oldestDateHolder.set(lastModifiedTime); } containerState.incrementArchiveCount(); } return FileVisitResult.CONTINUE; } }); return oldestDateHolder.get(); } }; // If the path didn't exist to begin with, there's no archive directory, so don't bother scanning. if (pathExists) { futures.add(executor.submit(scanContainer)); } } executor.shutdown(); for (final Future<Long> future : futures) { try { final Long oldestDate = future.get(); if (oldestDate < oldestArchiveDate.get()) { oldestArchiveDate.set(oldestDate); } } catch (final ExecutionException | InterruptedException e) { if (e.getCause() instanceof IOException) { throw (IOException) e.getCause(); } else { throw new RuntimeException(e); } } } containers.clear(); containers.putAll(realPathMap); }
From source file:org.apache.nifi.controller.repository.FileSystemRepository.java
private long destroyExpiredArchives(final String containerName, final Path container) throws IOException { archiveExpirationLog.debug("Destroying Expired Archives for Container {}", containerName); final List<ArchiveInfo> notYetExceedingThreshold = new ArrayList<>(); long removalTimeThreshold = System.currentTimeMillis() - maxArchiveMillis; long oldestArchiveDateFound = System.currentTimeMillis(); // determine how much space we must have in order to stop deleting old data final Long minRequiredSpace = minUsableContainerBytesForArchive.get(containerName); if (minRequiredSpace == null) { archiveExpirationLog/*from w w w . ja v a 2s . c o m*/ .debug("Could not determine minimum required space so will not destroy any archived data"); return -1L; } final long usableSpace = getContainerUsableSpace(containerName); final ContainerState containerState = containerStateMap.get(containerName); // First, delete files from our queue final long startNanos = System.nanoTime(); final long toFree = minRequiredSpace - usableSpace; final BlockingQueue<ArchiveInfo> fileQueue = archivedFiles.get(containerName); if (archiveExpirationLog.isDebugEnabled()) { if (toFree < 0) { archiveExpirationLog.debug( "Currently {} bytes free for Container {}; requirement is {} byte free, so no need to free space until an additional {} bytes are used", usableSpace, containerName, minRequiredSpace, Math.abs(toFree)); } else { archiveExpirationLog.debug( "Currently {} bytes free for Container {}; requirement is {} byte free, so need to free {} bytes", usableSpace, containerName, minRequiredSpace, toFree); } } ArchiveInfo toDelete; int deleteCount = 0; long freed = 0L; while ((toDelete = fileQueue.peek()) != null) { try { final long fileSize = toDelete.getSize(); removalTimeThreshold = System.currentTimeMillis() - maxArchiveMillis; // we use fileQueue.peek above instead of fileQueue.poll() because we don't always want to // remove the head of the queue. Instead, we want to remove it only if we plan to delete it. // In order to accomplish this, we just peek at the head and check if it should be deleted. // If so, then we call poll() to remove it if (freed < toFree || getLastModTime(toDelete.toPath()) < removalTimeThreshold) { toDelete = fileQueue.poll(); // remove the head of the queue, which is already stored in 'toDelete' Files.deleteIfExists(toDelete.toPath()); containerState.decrementArchiveCount(); LOG.debug( "Deleted archived ContentClaim with ID {} from Container {} because the archival size was exceeding the max configured size", toDelete.getName(), containerName); freed += fileSize; deleteCount++; } // If we'd freed up enough space, we're done... unless the next file needs to be destroyed based on time. if (freed >= toFree) { // If the last mod time indicates that it should be removed, just continue loop. if (deleteBasedOnTimestamp(fileQueue, removalTimeThreshold)) { archiveExpirationLog.debug( "Freed enough space ({} bytes freed, needed to free {} bytes) but will continue to expire data based on timestamp", freed, toFree); continue; } archiveExpirationLog.debug( "Freed enough space ({} bytes freed, needed to free {} bytes). Finished expiring data", freed, toFree); final ArchiveInfo archiveInfo = fileQueue.peek(); final long oldestArchiveDate = archiveInfo == null ? System.currentTimeMillis() : getLastModTime(archiveInfo.toPath()); // Otherwise, we're done. Return the last mod time of the oldest file in the container's archive. final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); if (deleteCount > 0) { LOG.info( "Deleted {} files from archive for Container {}; oldest Archive Date is now {}; container cleanup took {} millis", deleteCount, containerName, new Date(oldestArchiveDate), millis); } else { LOG.debug( "Deleted {} files from archive for Container {}; oldest Archive Date is now {}; container cleanup took {} millis", deleteCount, containerName, new Date(oldestArchiveDate), millis); } return oldestArchiveDate; } } catch (final IOException ioe) { LOG.warn("Failed to delete {} from archive due to {}", toDelete, ioe.toString()); if (LOG.isDebugEnabled()) { LOG.warn("", ioe); } } } // Go through each container and grab the archived data into a List archiveExpirationLog.debug("Searching for more archived data to expire"); final StopWatch stopWatch = new StopWatch(true); for (int i = 0; i < SECTIONS_PER_CONTAINER; i++) { final Path sectionContainer = container.resolve(String.valueOf(i)); final Path archive = sectionContainer.resolve("archive"); if (!Files.exists(archive)) { continue; } try { final long timestampThreshold = removalTimeThreshold; Files.walkFileTree(archive, new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { if (attrs.isDirectory()) { return FileVisitResult.CONTINUE; } final long lastModTime = getLastModTime(file); if (lastModTime < timestampThreshold) { try { Files.deleteIfExists(file); containerState.decrementArchiveCount(); LOG.debug( "Deleted archived ContentClaim with ID {} from Container {} because it was older than the configured max archival duration", file.toFile().getName(), containerName); } catch (final IOException ioe) { LOG.warn( "Failed to remove archived ContentClaim with ID {} from Container {} due to {}", file.toFile().getName(), containerName, ioe.toString()); if (LOG.isDebugEnabled()) { LOG.warn("", ioe); } } } else if (usableSpace < minRequiredSpace) { notYetExceedingThreshold .add(new ArchiveInfo(container, file, attrs.size(), lastModTime)); } return FileVisitResult.CONTINUE; } }); } catch (final IOException ioe) { LOG.warn("Failed to cleanup archived files in {} due to {}", archive, ioe.toString()); if (LOG.isDebugEnabled()) { LOG.warn("", ioe); } } } final long deleteExpiredMillis = stopWatch.getElapsed(TimeUnit.MILLISECONDS); // Sort the list according to last modified time Collections.sort(notYetExceedingThreshold, new Comparator<ArchiveInfo>() { @Override public int compare(final ArchiveInfo o1, final ArchiveInfo o2) { return Long.compare(o1.getLastModTime(), o2.getLastModTime()); } }); final long sortRemainingMillis = stopWatch.getElapsed(TimeUnit.MILLISECONDS) - deleteExpiredMillis; // Delete the oldest data archiveExpirationLog.debug("Deleting data based on timestamp"); final Iterator<ArchiveInfo> itr = notYetExceedingThreshold.iterator(); int counter = 0; while (itr.hasNext()) { final ArchiveInfo archiveInfo = itr.next(); try { final Path path = archiveInfo.toPath(); Files.deleteIfExists(path); containerState.decrementArchiveCount(); LOG.debug( "Deleted archived ContentClaim with ID {} from Container {} because the archival size was exceeding the max configured size", archiveInfo.getName(), containerName); // Check if we've freed enough space every 25 files that we destroy if (++counter % 25 == 0) { if (getContainerUsableSpace(containerName) > minRequiredSpace) { // check if we can stop now LOG.debug("Finished cleaning up archive for Container {}", containerName); break; } } } catch (final IOException ioe) { LOG.warn("Failed to delete {} from archive due to {}", archiveInfo, ioe.toString()); if (LOG.isDebugEnabled()) { LOG.warn("", ioe); } } itr.remove(); } final long deleteOldestMillis = stopWatch.getElapsed(TimeUnit.MILLISECONDS) - sortRemainingMillis - deleteExpiredMillis; long oldestContainerArchive; if (notYetExceedingThreshold.isEmpty()) { oldestContainerArchive = System.currentTimeMillis(); } else { oldestContainerArchive = notYetExceedingThreshold.get(0).getLastModTime(); } if (oldestContainerArchive < oldestArchiveDateFound) { oldestArchiveDateFound = oldestContainerArchive; } // Queue up the files in the order that they should be destroyed so that we don't have to scan the directories for a while. for (final ArchiveInfo toEnqueue : notYetExceedingThreshold.subList(0, Math.min(100000, notYetExceedingThreshold.size()))) { fileQueue.offer(toEnqueue); } final long cleanupMillis = stopWatch.getElapsed(TimeUnit.MILLISECONDS) - deleteOldestMillis - sortRemainingMillis - deleteExpiredMillis; LOG.debug( "Oldest Archive Date for Container {} is {}; delete expired = {} ms, sort remaining = {} ms, delete oldest = {} ms, cleanup = {} ms", containerName, new Date(oldestContainerArchive), deleteExpiredMillis, sortRemainingMillis, deleteOldestMillis, cleanupMillis); return oldestContainerArchive; }
From source file:org.cryptomator.cryptofs.CryptoDirectoryStreamTest.java
License:asdf
@Before @SuppressWarnings("unchecked") public void setup() throws IOException { filenameCryptor = cryptorProvider.createNew().fileNameCryptor(); ciphertextDirPath = Mockito.mock(Path.class); FileSystem fs = Mockito.mock(FileSystem.class); Mockito.when(ciphertextDirPath.getFileSystem()).thenReturn(fs); FileSystemProvider provider = Mockito.mock(FileSystemProvider.class); Mockito.when(fs.provider()).thenReturn(provider); dirStream = Mockito.mock(DirectoryStream.class); Mockito.when(provider.newDirectoryStream(Mockito.same(ciphertextDirPath), Mockito.any())) .thenReturn(dirStream);//from ww w . jav a2 s . com longFileNameProvider = Mockito.mock(LongFileNameProvider.class); conflictResolver = Mockito.mock(ConflictResolver.class); finallyUtil = mock(FinallyUtil.class); Mockito.when(longFileNameProvider.inflate(Mockito.anyString())).then(invocation -> { String shortName = invocation.getArgument(0); if (shortName.contains("invalid")) { throw new IOException("invalid shortened name"); } else { return StringUtils.removeEnd(shortName, ".lng"); } }); cryptoPathMapper = Mockito.mock(CryptoPathMapper.class); Mockito.when(cryptoPathMapper.resolveDirectory(Mockito.any())).then(invocation -> { Path dirFilePath = invocation.getArgument(0); if (dirFilePath.toString().contains("invalid")) { throw new IOException("Invalid directory."); } Path dirPath = Mockito.mock(Path.class); BasicFileAttributes attrs = Mockito.mock(BasicFileAttributes.class); Mockito.when(dirPath.getFileSystem()).thenReturn(fs); Mockito.when(provider.readAttributes(dirPath, BasicFileAttributes.class)).thenReturn(attrs); Mockito.when(attrs.isDirectory()).thenReturn(!dirFilePath.toString().contains("noDirectory")); return new Directory("asdf", dirPath); }); Mockito.when(conflictResolver.resolveConflictsIfNecessary(Mockito.any(), Mockito.any())) .then(returnsFirstArg()); doAnswer(invocation -> { for (Object runnable : invocation.getArguments()) { ((RunnableThrowingException<?>) runnable).run(); } return null; }).when(finallyUtil).guaranteeInvocationOf(any(RunnableThrowingException.class), any(RunnableThrowingException.class), any(RunnableThrowingException.class)); }