List of usage examples for org.apache.commons.io.filefilter HiddenFileFilter VISIBLE
IOFileFilter VISIBLE
To view the source code for org.apache.commons.io.filefilter HiddenFileFilter VISIBLE.
Click Source Link
From source file:com.ut.healthelink.service.impl.transactionInManagerImpl.java
@Override public Integer moveFilesByPath(String inPath, Integer transportMethodId, Integer orgId, Integer transportId) { Integer sysErrors = 0;/*w w w. j av a2 s . c o m*/ try { fileSystem fileSystem = new fileSystem(); String fileInPath = fileSystem.setPathFromRoot(inPath); File folder = new File(fileInPath); //list files //we only list visible files File[] listOfFiles = folder.listFiles((FileFilter) HiddenFileFilter.VISIBLE); Organization orgDetails = organizationmanager.getOrganizationById(orgId); String defPath = "/bowlink/" + orgDetails.getcleanURL() + "/input files/"; String outPath = fileSystem.setPath(defPath); //too many variables that could come into play regarding file types, will check files with one method //loop files for (File file : listOfFiles) { String fileName = file.getName(); DateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmssS"); Date date = new Date(); /* Create the batch name (TransportMethodId+OrgId+Date/Time/Seconds) */ String batchName = new StringBuilder().append(transportMethodId).append(orgId) .append(dateFormat.format(date)).toString(); if (!fileName.endsWith("_error")) { try { String fileExt = fileName.substring(fileName.lastIndexOf(".") + 1); //figure out how many active transports are using fileExt method for this particular path List<configurationTransport> transportList = configurationtransportmanager .getTransportListForFileExtAndPath(fileExt, transportMethodId, 1, inPath); //figure out if files has distinct delimiters List<configurationTransport> transports = configurationtransportmanager .getConfigTransportForFileExtAndPath(fileExt, transportMethodId, 1, inPath); batchUploads batchInfo = new batchUploads(); batchInfo.setOrgId(orgId); batchInfo.settransportMethodId(transportMethodId); batchInfo.setstatusId(4); batchInfo.setstartDateTime(date); batchInfo.setutBatchName(batchName); batchInfo.setOriginalFolder(inPath); Integer batchId = 0; String newFileName = ""; Integer statusId = 4; Integer configId = 0; Integer fileSize = 0; Integer encodingId = 1; Integer errorId = 0; if (transportList.size() == 0 || transports.size() == 0) { //neither of them should be 0 //no source transport is associated with this method / file batchInfo.setuserId(usermanager.getUserByTypeByOrganization(orgId).get(0).getId()); batchInfo.setConfigId(0); newFileName = newFileName(outPath, fileName); batchInfo.setoriginalFileName(newFileName); batchInfo.setFileLocation(defPath); batchInfo.setEncodingId(encodingId); batchId = (Integer) submitBatchUpload(batchInfo); //insert error errorId = 13; statusId = 7; } else if (transports.size() == 1) { encodingId = transports.get(0).getEncodingId(); configurationTransport ct = configurationtransportmanager .getTransportDetailsByTransportId(transportId); fileSize = ct.getmaxFileSize(); if (transportList.size() > 1) { configId = 0; fileSize = configurationtransportmanager.getMinMaxFileSize(fileExt, transportMethodId); } else { configId = ct.getconfigId(); } batchInfo.setConfigId(configId); batchInfo.setContainsHeaderRow(transports.get(0).getContainsHeaderRow()); batchInfo.setDelimChar(transports.get(0).getDelimChar()); batchInfo.setFileLocation(ct.getfileLocation()); outPath = fileSystem.setPath(ct.getfileLocation()); batchInfo.setOrgId(orgId); newFileName = newFileName(outPath, fileName); batchInfo.setoriginalFileName(newFileName); batchInfo.setEncodingId(encodingId); //find user List<User> users = usermanager.getSendersForConfig(Arrays.asList(ct.getconfigId())); if (users.size() == 0) { users = usermanager.getOrgUsersForConfig(Arrays.asList(ct.getconfigId())); } batchInfo.setuserId(users.get(0).getId()); batchId = (Integer) submitBatchUpload(batchInfo); statusId = 2; } else if (transportList.size() > 1 && transports.size() > 1) { //we loop though our delimiters for this type of fileExt String delimiter = ""; Integer fileDelimiter = 0; String fileLocation = ""; Integer userId = 0; //get distinct delimiters List<configurationTransport> delimList = configurationtransportmanager .getDistinctDelimCharForFileExt(fileExt, transportMethodId); List<configurationTransport> encodings = configurationtransportmanager .getTransportEncoding(fileExt, transportMethodId); //we reject file is multiple encodings/delimiters are found for extension type as we won't know how to decode it and read delimiter if (encodings.size() != 1) { batchInfo.setuserId(usermanager.getUserByTypeByOrganization(orgId).get(0).getId()); statusId = 7; errorId = 16; } else { encodingId = encodings.get(0).getEncodingId(); for (configurationTransport ctdelim : delimList) { fileSystem dir = new fileSystem(); int delimCount = (Integer) dir.checkFileDelimiter(file, ctdelim.getDelimChar()); if (delimCount > 3) { delimiter = ctdelim.getDelimChar(); fileDelimiter = ctdelim.getfileDelimiter(); statusId = 2; fileLocation = ctdelim.getfileLocation(); break; } } } // we don't have an error yet if (errorId > 0) { // some error detected from previous checks userId = usermanager.getUserByTypeByOrganization(orgId).get(0).getId(); batchInfo.setConfigId(configId); batchInfo.setFileLocation(defPath); batchInfo.setOrgId(orgId); newFileName = newFileName(outPath, fileName); batchInfo.setoriginalFileName(newFileName); batchInfo.setuserId(userId); batchId = (Integer) submitBatchUpload(batchInfo); batchInfo.setEncodingId(encodingId); } else if (statusId != 2) { //no vaild delimiter detected statusId = 7; userId = usermanager.getUserByTypeByOrganization(orgId).get(0).getId(); batchInfo.setConfigId(configId); batchInfo.setFileLocation(defPath); batchInfo.setOrgId(orgId); newFileName = newFileName(outPath, fileName); batchInfo.setoriginalFileName(newFileName); batchInfo.setuserId(userId); batchId = (Integer) submitBatchUpload(batchInfo); batchInfo.setEncodingId(encodingId); errorId = 15; } else if (statusId == 2) { encodingId = encodings.get(0).getEncodingId(); //we check to see if there is multi header row, if so, we reject because we don't know what header rows value to look for List<configurationTransport> containsHeaderRowCount = configurationtransportmanager .getCountContainsHeaderRow(fileExt, transportMethodId); if (containsHeaderRowCount.size() != 1) { batchInfo.setuserId( usermanager.getUserByTypeByOrganization(orgId).get(0).getId()); statusId = 7; errorId = 14; } else { List<Integer> totalConfigs = configurationtransportmanager .getConfigCount(fileExt, transportMethodId, fileDelimiter); //set how many configs we have if (totalConfigs.size() > 1) { configId = 0; } else { configId = totalConfigs.get(0); } //get path fileLocation = configurationtransportmanager .getTransportDetails(totalConfigs.get(0)).getfileLocation(); fileSize = configurationtransportmanager .getTransportDetails(totalConfigs.get(0)).getmaxFileSize(); List<User> users = usermanager.getSendersForConfig(totalConfigs); if (users.size() == 0) { users = usermanager.getOrgUsersForConfig(totalConfigs); } userId = users.get(0).getId(); batchInfo.setContainsHeaderRow( containsHeaderRowCount.get(0).getContainsHeaderRow()); batchInfo.setDelimChar(delimiter); batchInfo.setConfigId(configId); batchInfo.setFileLocation(fileLocation); outPath = fileSystem.setPath(fileLocation); batchInfo.setOrgId(orgId); newFileName = newFileName(outPath, fileName); batchInfo.setoriginalFileName(newFileName); batchInfo.setuserId(userId); batchInfo.setEncodingId(encodingId); batchId = (Integer) submitBatchUpload(batchInfo); } } } /** insert log**/ try { //log user activity UserActivity ua = new UserActivity(); ua.setUserId(0); ua.setFeatureId(0); ua.setAccessMethod("System"); ua.setActivity("System Uploaded File"); ua.setBatchUploadId(batchInfo.getId()); usermanager.insertUserLog(ua); } catch (Exception ex) { ex.printStackTrace(); System.err.println("moveFilesByPath - insert user log" + ex.toString()); } //we encoded user's file if it is not File newFile = new File(outPath + newFileName); // now we move file Path source = file.toPath(); Path target = newFile.toPath(); File archiveFile = new File(fileSystem.setPath(archivePath) + batchName + newFileName.substring(newFileName.lastIndexOf("."))); Path archive = archiveFile.toPath(); //we keep original file in archive folder Files.copy(source, archive); /** * we check encoding here * */ if (encodingId < 2) { //file is not encoded String encodedOldFile = filemanager.encodeFileToBase64Binary(file); filemanager.writeFile(newFile.getAbsolutePath(), encodedOldFile); Files.delete(source); } else { Files.move(source, target); } if (statusId == 2) { /** * check file size if configId is 0 we go with the smallest file size * */ long maxFileSize = fileSize * 1000000; if (Files.size(target) > maxFileSize) { statusId = 7; errorId = 12; } } if (statusId != 2) { insertProcessingError(errorId, 0, batchId, null, null, null, null, false, false, ""); } updateBatchStatus(batchId, statusId, "endDateTime"); } catch (Exception exAtFile) { exAtFile.printStackTrace(); System.err.println("moveFilesByPath " + exAtFile.toString()); try { sendEmailToAdmin( (exAtFile.toString() + "<br/>" + Arrays.toString(exAtFile.getStackTrace())), "moveFilesByPath - at rename file to error "); //we need to move that file out of the way file.renameTo((new File(file.getAbsolutePath() + batchName + "_error"))); } catch (Exception ex1) { ex1.printStackTrace(); System.err.println("moveFilesByPath " + ex1.getMessage()); } } } } } catch (Exception ex) { ex.printStackTrace(); try { sendEmailToAdmin((ex.toString() + "<br/>" + Arrays.toString(ex.getStackTrace())), "moveFilesByPath - issue with looping folder files"); } catch (Exception ex1) { ex1.printStackTrace(); System.err.println("moveFilesByPath " + ex1.getMessage()); } return 1; } return sysErrors; }
From source file:opendap.aws.glacier.GlacierManager.java
public void loadVaults() throws IOException, JDOMException { GlacierVaultManager gvm;//from w w w. j a v a2 s . c om File gRootDir = getGlacierRootDir(); log.debug("loadVaults(): getGlacierRootDir", getGlacierRootDir()); File[] vaults = gRootDir.listFiles((FileFilter) HiddenFileFilter.VISIBLE); if (vaults != null) { log.debug("loadVaults(): Got {} vaults", vaults.length); for (File vault : vaults) { if (!vault.getName().equals(DefaultResourceCacheDirectoryName)) { String vaultName = vault.getName(); log.debug("loadVaults(): Loading vault: {} vaultName: {}", vault, vaultName); if (vault.isDirectory()) { gvm = new GlacierVaultManager(vaultName, gRootDir); gvm.setParentContext(getGlacierServiceContext()); // gvm.loadArchiveRecords(); gvm.loadIndexObjects(); _vaults.put(gvm.name(), gvm); } } } } }
From source file:opendap.aws.glacier.GlacierVaultManager.java
public void loadIndexObjects() throws IOException, JDOMException { Index index;/* w w w.j a va 2 s .com*/ File indexDir = getIndexDir(); File[] indexFiles = indexDir.listFiles((FileFilter) HiddenFileFilter.VISIBLE); if (indexFiles != null) { for (File indexFile : indexFiles) { if (indexFile.isFile()) { index = new Index(indexFile); StringBuilder resourceId = new StringBuilder(); String pathDelimiter = index.getDelimiter(); resourceId //.append(getGlacierServiceContext()) .append(index.getPath()).append(pathDelimiter).append(index.getIndexFileConvention()); index.setResourceId(resourceId.toString()); _indexObjects.put(index.getResourceId(), index); _log.debug("Loaded Index. Vault: {} resourceId: {}", name(), index.getResourceId()); } else { _log.debug("Skipping directory/link {}", indexFile); } } } else { _log.debug("No index files found for vault {}", name()); } }
From source file:org.apache.nifi.processors.flume.ExecuteFlumeSinkTest.java
@Test @Ignore("Does not work on Windows") public void testHdfsSink() throws IOException { File destDir = temp.newFolder("hdfs"); TestRunner runner = TestRunners.newTestRunner(ExecuteFlumeSink.class); runner.setProperty(ExecuteFlumeSink.SINK_TYPE, "hdfs"); runner.setProperty(ExecuteFlumeSink.FLUME_CONFIG, "tier1.sinks.sink-1.hdfs.path = " + destDir.toURI().toString() + "\n" + "tier1.sinks.sink-1.hdfs.fileType = DataStream\n" + "tier1.sinks.sink-1.hdfs.serializer = TEXT\n" + "tier1.sinks.sink-1.serializer.appendNewline = false"); try (InputStream inputStream = getClass().getResourceAsStream("/testdata/records.txt")) { Map<String, String> attributes = new HashMap<>(); attributes.put(CoreAttributes.FILENAME.key(), "records.txt"); runner.enqueue(inputStream, attributes); runner.run();/*from w w w .ja v a 2 s. co m*/ } File[] files = destDir.listFiles((FilenameFilter) HiddenFileFilter.VISIBLE); assertEquals("Unexpected number of destination files.", 1, files.length); File dst = files[0]; byte[] expectedMd5; try (InputStream md5Stream = getClass().getResourceAsStream("/testdata/records.txt")) { expectedMd5 = FileUtils.computeMd5Digest(md5Stream); } byte[] actualMd5 = FileUtils.computeMd5Digest(dst); Assert.assertArrayEquals("Destination file doesn't match source data", expectedMd5, actualMd5); }
From source file:org.callistasoftware.maven.plugins.propertyscanner.MyDirectoryWalker.java
public MyDirectoryWalker(IOFileFilter fileFilter) { super(HiddenFileFilter.VISIBLE, fileFilter, -1); }
From source file:org.codehaus.mojo.webtest.components.ReportCollector.java
/** * Constructor/* w w w .j a v a 2 s. c o m*/ * * @param name the name of the files to be collected */ public ReportCollector(String name) { super(HiddenFileFilter.VISIBLE, -1); this.name = name; }
From source file:org.craftercms.deployer.impl.processors.GitPullProcessor.java
protected void addClonedFilesToChangeSet(File parent, String parentPath, ChangeSet changeSet) { String[] filenames = parent.list(HiddenFileFilter.VISIBLE); if (filenames != null) { for (String filename : filenames) { File file = new File(parent, filename); String path = FilenameUtils.concat(parentPath, filename); if (file.isDirectory()) { addClonedFilesToChangeSet(file, path, changeSet); } else { logger.debug("New file: {}", path); changeSet.getCreatedFiles().add(path); }//w ww . j a va2 s .co m } } }
From source file:org.forgerock.doc.maven.HTMLUtils.java
/** * Replace HTML tags with additional content. * * @param baseDir/*w ww.j a va2s . c o m*/ * Base directory under which to find HTML files recursively * @param replacements * Keys are tags to replace. Values are replacements, including * the original tag. * @return List of files updated * @throws IOException * Something went wrong reading or writing files. */ static List<File> updateHTML(final String baseDir, final Map<String, String> replacements) throws IOException { // Match normal directories, and HTML files. IOFileFilter dirFilter = FileFilterUtils.and(FileFilterUtils.directoryFileFilter(), HiddenFileFilter.VISIBLE); IOFileFilter fileFilter = FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter(".html")); FileFilter filter = FileFilterUtils.or(dirFilter, fileFilter); FilteredFileUpdater ffu = new FilteredFileUpdater(replacements, filter); return ffu.update(new File(baseDir)); }
From source file:org.gbif.dwca.action.ValidateAction.java
/** * @param dwcaFolder//from www .j a v a 2s .co m */ private void validateArchive(File dwcaFolder) { if (dwcaFolder == null) { return; } log.info("Inspecting uploaded dwc archive"); try { archive = ArchiveFactory.openArchive(dwcaFolder); // inspect dwca folder files if (dwcaFolder.isDirectory()) { if (archive != null && archive.getCore() != null) { coreFile = archive.getCore().getLocation(); } dwcaFiles = new HashSet<String>(Arrays.asList(dwcaFolder.list(HiddenFileFilter.VISIBLE))); dwcaFiles.remove("meta.xml"); if (archive.getMetadataLocation() != null) { dwcaFiles.remove(new File(archive.getMetadataLocation()).getName()); } } else { coreFile = dwcaFolder.getName(); dwcaFiles.add(coreFile); } // inspect archive files ArchiveFile af = archive.getCore(); inspectArchiveFile(af, true); for (ArchiveFile ext : archive.getExtensions()) { inspectArchiveFile(ext, false); } // read records if (!metaOnly) { setRecords(); } } catch (Exception e) { setDwcaException(e); } // read metadata try { metadata = archive.getMetadata(); } catch (Exception e) { setMetadataException(e); } }
From source file:org.gbif.dwca.io.ArchiveFactory.java
/** * Opens an archive from a local file and decompresses or copies it into the given archive directory. * Make sure the archive directory does not contain files already, any existing files will be removed! * * If the source archive is an uncompressed, single data file and a valid archive, it will be copied as is * to the archiveDir./*from www . jav a 2 s . com*/ * * @param archiveFile the location of a compressed archive or single data file * @param archiveDir empty, writable directory used to keep decompress archive in */ public static Archive openArchive(File archiveFile, File archiveDir) throws IOException, UnsupportedArchiveException { if (archiveDir.exists()) { // clean up any existing folder LOG.debug("Deleting existing archive folder [{}]", archiveDir.getAbsolutePath()); org.gbif.utils.file.FileUtils.deleteDirectoryRecursively(archiveDir); } FileUtils.forceMkdir(archiveDir); // try to decompress archive try { CompressionUtil.decompressFile(archiveDir, archiveFile, true); // we keep subfolder, but often the entire archive is within one subfolder. Remove that root folder if present File[] rootFiles = archiveDir.listFiles((FileFilter) HiddenFileFilter.VISIBLE); if (rootFiles.length == 1) { File root = rootFiles[0]; if (root.isDirectory()) { // single root dir, flatten structure LOG.debug("Removing single root folder {} found in decompressed archive", root.getAbsoluteFile()); for (File f : FileUtils.listFiles(root, TrueFileFilter.TRUE, null)) { File f2 = new File(archiveDir, f.getName()); f.renameTo(f2); } } } // continue to read archive from the tmp dir return openArchive(archiveDir); } catch (CompressionUtil.UnsupportedCompressionType e) { LOG.debug("Could not uncompress archive [{}], try to read as single text file", archiveFile, e); // If its a text file only we will get this exception - but also for corrupt compressions // try to open as text file only and if successful copy file to archive dir Archive arch = openArchiveDataFile(archiveFile); Files.copy(archiveFile, new File(archiveDir, archiveFile.getName())); return arch; } }