List of usage examples for org.apache.commons.vfs2 FileObject getType
FileType getType() throws FileSystemException;
From source file:org.pentaho.di.plugins.fileopensave.providers.vfs.VFSFileProvider.java
/** * @param file/*w ww . j a v a 2 s . c o m*/ * @param filters * @return */ @Override public List<VFSFile> getFiles(VFSFile file, String filters) { if (file.getPath() == null) { return getRoot(file); } List<VFSFile> files = new ArrayList<>(); try { FileObject fileObject = KettleVFS.getFileObject(file.getPath(), new Variables(), VFSHelper.getOpts(file.getPath(), file.getConnection())); FileType fileType = fileObject.getType(); if (fileType.hasChildren()) { FileObject[] children = fileObject.getChildren(); for (FileObject child : children) { FileType fileType1 = child.getType(); if (fileType1.hasChildren()) { files.add(VFSDirectory.create(file.getPath(), child, file.getConnection())); } else { if (Utils.matches(child.getName().getBaseName(), filters)) { files.add(VFSFile.create(file.getPath(), child, file.getConnection())); } } } } } catch (KettleFileException | FileSystemException ignored) { // File does not exist } return files; }
From source file:org.pentaho.di.trans.steps.file.BaseFileInputStep.java
/** * Prepare file-dependent data for fill additional fields. *//* www .j av a 2 s. com*/ protected void fillFileAdditionalFields(D data, FileObject file) throws FileSystemException { data.shortFilename = file.getName().getBaseName(); data.path = KettleVFS.getFilename(file.getParent()); data.hidden = file.isHidden(); data.extension = file.getName().getExtension(); data.uriName = file.getName().getURI(); data.rootUriName = file.getName().getRootURI(); if (file.getType().hasContent()) { data.lastModificationDateTime = new Date(file.getContent().getLastModifiedTime()); data.size = file.getContent().getSize(); } else { data.lastModificationDateTime = null; data.size = null; } }
From source file:org.pentaho.hadoop.shim.HadoopConfigurationLocator.java
/** * Create a ClassLoader to load resources for a {@code HadoopConfiguration}. * * @param root Configuration root directory * @param parent Parent class loader to delegate to if resources cannot be found in the configuration's * directory or provided classpath * @param classpathUrls Additional URLs to add to the class loader. These will be added before any internal * resources./*from w w w . j a va 2s . c o m*/ * @param ignoredClasses Classes (or packages) that should not be loaded by the class loader * @return A class loader capable of loading a Hadoop configuration located at {@code root}. * @throws ConfigurationException Error creating a class loader for the Hadoop configuration located at {@code root} */ protected ClassLoader createConfigurationLoader(FileObject root, ClassLoader parent, List<URL> classpathUrls, ShimProperties configurationProperties, String... ignoredClasses) throws ConfigurationException { try { if (root == null || !FileType.FOLDER.equals(root.getType())) { throw new IllegalArgumentException("root must be a folder: " + root); } // Find all jar files in the configuration, at most 2 folders deep List<URL> jars = findJarsIn(root, 3, configurationProperties.getConfigSet(SHIM_CLASSPATH_IGNORE)); // Add the root of the configuration jars.add(0, new URL(root.getURL().toExternalForm() + "/")); // Inject any overriding URLs before all other paths if (classpathUrls != null) { jars.addAll(0, classpathUrls); } //Exclude jars contained in exclude.jars property in config.properties file from the list of jars jars = filterJars(jars, configurationProperties.getProperty(CONFIG_PROPERTY_EXCLUDE_JARS)); return new HadoopConfigurationClassLoader(jars.toArray(EMPTY_URL_ARRAY), parent, ignoredClasses); } catch (Exception ex) { throw new ConfigurationException(BaseMessages.getString(PKG, "Error.CreatingClassLoader"), ex); } }
From source file:org.pentaho.hadoop.shim.HadoopConfigurationLocator.java
/** * Parse a set of URLs from a comma-separated list of URLs. If the URL points to a directory all jar files within that * directory will be returned as well.//from w w w .j a v a 2s .c o m * * @param urlString Comma-separated list of URLs (relative or absolute) * @return List of URLs resolved from {@code urlString} */ protected List<URL> parseURLs(FileObject root, String urlString) { if (urlString == null || urlString.trim().isEmpty()) { return Collections.emptyList(); } String[] paths = urlString.split(","); List<URL> urls = new ArrayList<URL>(); for (String path : paths) { try { FileObject file = root.resolveFile(path.trim()); if (!file.exists()) { file = defaultFsm.resolveFile(path.trim()); } if (FileType.FOLDER.equals(file.getType())) { // Add directories with a trailing / so the URL ClassLoader interprets // them as directories urls.add(new URL(file.getURL().toExternalForm() + "/")); // Also add all jars within this directory urls.addAll(findJarsIn(file, 1, new HashSet<String>())); } else { urls.add(file.getURL()); } } catch (Exception e) { // Log invalid path logger.error(BaseMessages.getString(PKG, "Error.InvalidClasspathEntry", path)); } } return urls; }
From source file:org.pentaho.hadoop.shim.HadoopConfigurationLocatorTest.java
@BeforeClass public static void setup() throws Exception { // Create a test hadoop configuration "a" FileObject ramRoot = VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH); FileObject aConfigFolder = ramRoot.resolveFile("a"); if (aConfigFolder.exists()) { aConfigFolder.delete(new AllFileSelector()); }/* w w w. j av a 2 s . c om*/ aConfigFolder.createFolder(); assertEquals(FileType.FOLDER, aConfigFolder.getType()); // Create the properties file for the configuration as hadoop-configurations/a/config.properties configFile = aConfigFolder.resolveFile("config.properties"); Properties p = new Properties(); p.setProperty("name", "Test Configuration A"); p.setProperty("classpath", ""); p.setProperty("ignore.classes", ""); p.setProperty("library.path", ""); p.setProperty("required.classes", HadoopConfigurationLocatorTest.class.getName()); p.store(configFile.getContent().getOutputStream(), "Test Configuration A"); configFile.close(); // Create the implementation jar FileObject implJar = aConfigFolder.resolveFile("a-config.jar"); implJar.createFile(); // Use ShrinkWrap to create the jar and write it out to VFS JavaArchive archive = ShrinkWrap.create(JavaArchive.class, "a-configuration.jar") .addAsServiceProvider(HadoopShim.class, MockHadoopShim.class).addClass(MockHadoopShim.class); archive.as(ZipExporter.class).exportTo(implJar.getContent().getOutputStream()); }
From source file:org.pentaho.hadoop.shim.HadoopConfigurationLocatorTest.java
@Test(expected = ConfigurationException.class) public void createConfigurationLoader_root_not_a_folder() throws Exception { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); // Try to create a configuration based on a file, not a folder FileObject buildProperties = VFS.getManager().resolveFile("ram:///test.file"); buildProperties.createFile();// w ww .j a v a2s. co m assertEquals(FileType.FILE, buildProperties.getType()); locator.createConfigurationLoader(buildProperties, null, null, new ShimProperties()); }
From source file:org.pentaho.hadoop.shim.hsp101.HadoopShim.java
@Override public void onLoad(HadoopConfiguration config, HadoopConfigurationFileSystemManager fsm) throws Exception { fsm.addProvider(config, "hdfs", config.getIdentifier(), new HDFSFileProvider()); setDistributedCacheUtil(new DistributedCacheUtilImpl(config) { /**//from www. j av a 2s . c o m * Default permission for cached files * <p/> * Not using FsPermission.createImmutable due to EOFExceptions when using it with Hadoop 0.20.2 */ private final FsPermission CACHED_FILE_PERMISSION = new FsPermission((short) 0755); public void addFileToClassPath(Path file, Configuration conf) throws IOException { String classpath = conf.get("mapred.job.classpath.files"); conf.set("mapred.job.classpath.files", classpath == null ? file.toString() : classpath + getClusterPathSeparator() + file.toString()); FileSystem fs = FileSystem.get(conf); URI uri = fs.makeQualified(file).toUri(); DistributedCache.addCacheFile(uri, conf); } /** * Stages the source file or folder to a Hadoop file system and sets their permission and replication * value appropriately to be used with the Distributed Cache. WARNING: This will delete the contents of * dest before staging the archive. * * @param source File or folder to copy to the file system. If it is a folder all contents will be * copied into dest. * @param fs Hadoop file system to store the contents of the archive in * @param dest Destination to copy source into. If source is a file, the new file name will be * exactly dest. If source is a folder its contents will be copied into dest. For more * info see {@link FileSystem#copyFromLocalFile(org.apache.hadoop.fs.Path, * org.apache.hadoop.fs.Path)}. * @param overwrite Should an existing file or folder be overwritten? If not an exception will be * thrown. * @throws IOException Destination exists is not a directory * @throws KettleFileException Source does not exist or destination exists and overwrite is false. */ public void stageForCache(FileObject source, FileSystem fs, Path dest, boolean overwrite) throws IOException, KettleFileException { if (!source.exists()) { throw new KettleFileException(BaseMessages.getString(DistributedCacheUtilImpl.class, "DistributedCacheUtil.SourceDoesNotExist", source)); } if (fs.exists(dest)) { if (overwrite) { // It is a directory, clear it out fs.delete(dest, true); } else { throw new KettleFileException(BaseMessages.getString(DistributedCacheUtilImpl.class, "DistributedCacheUtil.DestinationExists", dest.toUri().getPath())); } } // Use the same replication we'd use for submitting jobs short replication = (short) fs.getConf().getInt("mapred.submit.replication", 10); copyFile(source, fs, dest, overwrite); fs.setReplication(dest, replication); } private void copyFile(FileObject source, FileSystem fs, Path dest, boolean overwrite) throws IOException { if (source.getType() == FileType.FOLDER) { fs.mkdirs(dest); fs.setPermission(dest, CACHED_FILE_PERMISSION); for (FileObject fileObject : source.getChildren()) { copyFile(fileObject, fs, new Path(dest, fileObject.getName().getBaseName()), overwrite); } } else { try (FSDataOutputStream fsDataOutputStream = fs.create(dest, overwrite)) { IOUtils.copy(source.getContent().getInputStream(), fsDataOutputStream); fs.setPermission(dest, CACHED_FILE_PERMISSION); } } } public String getClusterPathSeparator() { return System.getProperty("hadoop.cluster.path.separator", ","); } }); }
From source file:org.pentaho.metaverse.impl.VfsLineageCollector.java
@Override public List<String> listArtifacts(final String startingDate, final String endingDate) throws IllegalArgumentException { List<String> paths = new ArrayList<>(); try {//from w w w. j a v a 2 s . co m FileSystemOptions opts = new FileSystemOptions(); FileObject lineageRootFolder = KettleVFS.getFileObject(getOutputFolder(), opts); FileSelector dateRangeFilter = new VfsDateRangeFilter(format, startingDate, endingDate); FileSelector depthFilter = new FileDepthSelector(1, 256); if (lineageRootFolder.exists() && lineageRootFolder.getType() == FileType.FOLDER) { // get the folders that come on or after the startingDate FileObject[] dayFolders = lineageRootFolder.findFiles(dateRangeFilter); for (FileObject dayFolder : dayFolders) { FileObject[] listThisFolder = dayFolder.findFiles(depthFilter); for (FileObject currentFile : listThisFolder) { if (currentFile.getType() == FileType.FILE) { paths.add(currentFile.getName().getPath()); } } } } return paths; } catch (Exception e) { throw new IllegalArgumentException(e); } }
From source file:org.pentaho.metaverse.impl.VfsLineageCollector.java
@Override public List<String> listArtifactsForFile(String pathToArtifact, String startingDate, String endingDate) throws IllegalArgumentException { List<String> paths = new ArrayList<>(); try {/*from w ww. j a v a 2 s. c o m*/ FileSystemOptions opts = new FileSystemOptions(); FileObject lineageRootFolder = KettleVFS.getFileObject(getOutputFolder(), opts); FileSelector dateRangeFilter = new VfsDateRangeFilter(format, startingDate, endingDate); FileSelector depthFilter = new FileDepthSelector(1, 256); if (lineageRootFolder.exists() && lineageRootFolder.getType() == FileType.FOLDER) { // get all of the date folders of lineage we have FileObject[] dayFolders = lineageRootFolder.findFiles(dateRangeFilter); for (FileObject dayFolder : dayFolders) { FileObject[] listThisFolder = dayFolder.findFiles(depthFilter); for (FileObject currentFile : listThisFolder) { FileObject requested = currentFile.resolveFile(pathToArtifact); if (requested.exists() && requested.getType() == FileType.FOLDER) { FileObject[] requestedChildren = requested.getChildren(); for (FileObject requestedChild : requestedChildren) { if (requestedChild.getType() == FileType.FILE) { paths.add(requestedChild.getName().getPath()); } } } } } } return paths; } catch (Exception e) { throw new IllegalArgumentException(e); } }
From source file:org.pentaho.metaverse.util.VfsDateRangeFilterTest.java
@Test public void testAccept_notFolder() throws Exception { filter = new VfsDateRangeFilter(format, start); FileSelectInfo fsi = mock(FileSelectInfo.class); FileObject fo = mock(FileObject.class); when(fo.getType()).thenReturn(FileType.FILE); when(fsi.getFile()).thenReturn(fo);/* w ww. j a v a 2s. c o m*/ when(fsi.getDepth()).thenReturn(1); assertFalse(filter.includeFile(fsi)); }