List of usage examples for org.apache.hadoop.fs FileSystem getConf
@Override
public Configuration getConf()
From source file:gaffer.accumulo.TestAccumuloBackedGraphUpdatingConf.java
License:Apache License
public static int readResults(FileSystem fs, Path path, Set<GraphElementWithStatistics> results) throws IOException { FileStatus[] fileStatus = fs.listStatus(path, new PathFilter() { @Override//ww w .j ava 2 s .c om public boolean accept(Path path) { return path.getName().contains("part-m-"); } }); int count = 0; for (int i = 0; i < fileStatus.length; i++) { SequenceFile.Reader reader = new SequenceFile.Reader(fs, fileStatus[i].getPath(), fs.getConf()); GraphElement element = new GraphElement(); SetOfStatistics statistics = new SetOfStatistics(); while (reader.next(element, statistics)) { count++; results.add(new GraphElementWithStatistics(element.clone(), statistics.clone())); } reader.close(); } return count; }
From source file:gobblin.util.AvroUtils.java
License:Apache License
/** * Get Avro schema from an Avro data file. *//* w w w .j av a 2 s . c o m*/ public static Schema getSchemaFromDataFile(Path dataFile, FileSystem fs) throws IOException { try (SeekableInput sin = new FsInput(dataFile, fs.getConf()); DataFileReader<GenericRecord> reader = new DataFileReader<>(sin, new GenericDatumReader<GenericRecord>())) { return reader.getSchema(); } }
From source file:gobblin.util.AvroUtils.java
License:Apache License
/** * Get the latest avro schema for a directory * @param directory the input dir that contains avro files * @param fs the {@link FileSystem} for the given directory. * @param latest true to return latest schema, false to return oldest schema * @return the latest/oldest schema in the directory * @throws IOException//from w ww .ja va2 s .c o m */ public static Schema getDirectorySchema(Path directory, FileSystem fs, boolean latest) throws IOException { Schema schema = null; try (Closer closer = Closer.create()) { List<FileStatus> files = getDirectorySchemaHelper(directory, fs); if (files == null || files.size() == 0) { LOG.warn("There is no previous avro file in the directory: " + directory); } else { FileStatus file = latest ? files.get(0) : files.get(files.size() - 1); LOG.debug("Path to get the avro schema: " + file); FsInput fi = new FsInput(file.getPath(), fs.getConf()); GenericDatumReader<GenericRecord> genReader = new GenericDatumReader<>(); schema = closer.register(new DataFileReader<>(fi, genReader)).getSchema(); } } catch (IOException ioe) { throw new IOException("Cannot get the schema for directory " + directory, ioe); } return schema; }
From source file:gobblin.util.ParallelRunner.java
License:Apache License
/** * Move a {@link Path}./* www . j a va2s.c o m*/ * * <p> * This method submits a task to move a {@link Path} and returns immediately * after the task is submitted. * </p> * * @param src path to be moved * @param dstFs the destination {@link FileSystem} * @param dst the destination path * @param overwrite true to overwrite the destination * @param group an optional group name for the destination path */ public void movePath(final Path src, final FileSystem dstFs, final Path dst, final boolean overwrite, final Optional<String> group) { this.futures.add(new NamedFuture(this.executor.submit(new Callable<Void>() { @Override public Void call() throws Exception { Lock lock = ParallelRunner.this.locks.get(src.toString()); lock.lock(); try { if (ParallelRunner.this.fs.exists(src)) { HadoopUtils.movePath(ParallelRunner.this.fs, src, dstFs, dst, overwrite, dstFs.getConf()); if (group.isPresent()) { HadoopUtils.setGroup(dstFs, dst, group.get()); } } return null; } catch (FileAlreadyExistsException e) { LOGGER.warn(String.format("Failed to move %s to %s: dst already exists", src, dst), e); return null; } finally { lock.unlock(); } } }), "Move " + src + " to " + dst)); }
From source file:gobblin.util.ParallelRunnerTest.java
License:Apache License
@Test public void testMovePath() throws IOException, URISyntaxException { String expected = "test"; ByteArrayOutputStream actual = new ByteArrayOutputStream(); Path src = new Path("/src/file.txt"); Path dst = new Path("/dst/file.txt"); FileSystem fs1 = Mockito.mock(FileSystem.class); Mockito.when(fs1.exists(src)).thenReturn(true); Mockito.when(fs1.isFile(src)).thenReturn(true); Mockito.when(fs1.getUri()).thenReturn(new URI("fs1:////")); Mockito.when(fs1.getFileStatus(src)).thenReturn(new FileStatus(1, false, 1, 1, 1, src)); Mockito.when(fs1.open(src)).thenReturn( new FSDataInputStream(new SeekableFSInputStream(new ByteArrayInputStream(expected.getBytes())))); Mockito.when(fs1.delete(src, true)).thenReturn(true); FileSystem fs2 = Mockito.mock(FileSystem.class); Mockito.when(fs2.exists(dst)).thenReturn(false); Mockito.when(fs2.getUri()).thenReturn(new URI("fs2:////")); Mockito.when(fs2.getConf()).thenReturn(new Configuration()); Mockito.when(fs2.create(dst, false)).thenReturn(new FSDataOutputStream(actual, null)); try (ParallelRunner parallelRunner = new ParallelRunner(1, fs1)) { parallelRunner.movePath(src, fs2, dst, Optional.<String>absent()); }// ww w.jav a2 s . co m Assert.assertEquals(actual.toString(), expected); }
From source file:gobblin.util.ProxiedFileSystemCache.java
License:Apache License
private static Configuration resolveConfiguration(Configuration configuration, FileSystem fileSystem) throws IOException { if (configuration != null) { return configuration; }/* w w w . j a v a2 s . c o m*/ if (fileSystem != null) { return fileSystem.getConf(); } throw new IOException("FileSystem configuration could not be determined from available inputs."); }
From source file:gobblin.util.ProxiedFileSystemUtils.java
License:Apache License
/** * Retrives a {@link Token} from a given sequence file for a specified user. The sequence file should contain a list * of key, value pairs where each key corresponds to a user and each value corresponds to a {@link Token} for that * user./* w w w. j a va 2s .c o m*/ * * @param userNameKey The name of the user to retrieve a {@link Token} for * @param tokenFilePath The path to the sequence file containing the {@link Token}s * * @return A {@link Token} for the given user name */ public static Optional<Token<?>> getTokenFromSeqFile(String userNameKey, Path tokenFilePath) throws IOException { log.info("Reading tokens from sequence file " + tokenFilePath); try (Closer closer = Closer.create()) { FileSystem localFs = FileSystem.getLocal(new Configuration()); @SuppressWarnings("deprecation") SequenceFile.Reader tokenReader = closer .register(new SequenceFile.Reader(localFs, tokenFilePath, localFs.getConf())); Text key = new Text(); Token<?> value = new Token<>(); while (tokenReader.next(key, value)) { log.debug("Found token for user: " + key); if (key.toString().equals(userNameKey)) { return Optional.<Token<?>>of(value); } } } log.warn("Did not find any tokens for user " + userNameKey); return Optional.absent(); }
From source file:gobblin.util.ProxiedFileSystemWrapper.java
License:Apache License
/** * Get token from the token sequence file. * @param authPath//w w w. j a v a2s . c om * @param proxyUserName * @return Token for proxyUserName if it exists. * @throws IOException */ private static Optional<Token<?>> getTokenFromSeqFile(String authPath, String proxyUserName) throws IOException { try (Closer closer = Closer.create()) { FileSystem localFs = FileSystem.getLocal(new Configuration()); SequenceFile.Reader tokenReader = closer .register(new SequenceFile.Reader(localFs, new Path(authPath), localFs.getConf())); Text key = new Text(); Token<?> value = new Token<>(); while (tokenReader.next(key, value)) { LOG.info("Found token for " + key); if (key.toString().equals(proxyUserName)) { return Optional.<Token<?>>of(value); } } } return Optional.absent(); }
From source file:hdfs.HdfsFileWriter.java
License:Apache License
public HdfsFileWriter(FileSystem fileSystem, Path path) throws IOException { LOG.debug("Creating writer on {}", path); this.path = path; Configuration conf = fileSystem.getConf(); // FsServerDefaults fsDefaults = fileSystem.getServerDefaults(path); // /* ww w. j av a 2 s .c o m*/ // EnumSet<CreateFlag> flags = EnumSet.of(CreateFlag.CREATE, // CreateFlag.OVERWRITE); // if (Boolean.getBoolean(HDFS_SYNC_BLOCK)) { // flags.add(CreateFlag.SYNC_BLOCK); // } // fileSystem.getoutputStream = fileSystem.create( // path, // FsPermission.getDefault().applyUMask(FsPermission.getUMask(conf)), // flags, // fsDefaults.getFileBufferSize(), // fsDefaults.getReplication(), // fsDefaults.getBlockSize(), null); // fileSystem. this.outputStream = fileSystem.create(path, FsPermission.getDefault(), true, 50000, fileSystem.getDefaultReplication(), fileSystem.getDefaultBlockSize(), null); }
From source file:io.dstream.tez.utils.HadoopUtils.java
License:Apache License
/** * Will provision current classpath to YARN and return an array of * {@link Path}s representing provisioned resources * If 'generate-jar' system property is set it will also generate the JAR for the current * working directory (mainly used when executing from IDE) *//*w w w . java 2 s . c om*/ private static Path[] provisionClassPath(FileSystem fs, String applicationName, String[] classPathExclusions) { String genJarProperty = System.getProperty(TezConstants.GENERATE_JAR); boolean generateJar = genJarProperty != null && Boolean.parseBoolean(genJarProperty); List<Path> provisionedPaths = new ArrayList<Path>(); List<File> generatedJars = new ArrayList<File>(); boolean confFromHadoopConfDir = generateConfigJarFromHadoopConfDir(fs, applicationName, provisionedPaths, generatedJars); TezConfiguration tezConf = new TezConfiguration(fs.getConf()); boolean provisionTez = true; if (tezConf.get("tez.lib.uris") != null) { provisionTez = false; } URL[] classpath = ((URLClassLoader) ClassLoader.getSystemClassLoader()).getURLs(); for (URL classpathUrl : classpath) { File f = new File(classpathUrl.getFile()); if (f.isDirectory()) { if (generateJar) { String jarFileName = ClassPathUtils.generateJarFileName("application"); f = doGenerateJar(f, jarFileName, generatedJars, "application"); } else if (f.getName().equals("conf") && !confFromHadoopConfDir) { String jarFileName = ClassPathUtils.generateJarFileName("conf_application"); f = doGenerateJar(f, jarFileName, generatedJars, "configuration"); } else { f = null; } } if (f != null) { if (f.getName().startsWith("tez-") && !provisionTez) { logger.info("Skipping provisioning of " + f.getName() + " since Tez libraries are already provisioned"); continue; } String destinationFilePath = applicationName + "/" + f.getName(); Path provisionedPath = new Path(fs.getHomeDirectory(), destinationFilePath); if (shouldProvision(provisionedPath.getName(), classPathExclusions)) { try { provisioinResourceToFs(fs, new Path(f.getAbsolutePath()), provisionedPath); provisionedPaths.add(provisionedPath); } catch (Exception e) { logger.warn("Failed to provision " + provisionedPath + "; " + e.getMessage()); if (logger.isDebugEnabled()) { logger.trace("Failed to provision " + provisionedPath, e); } } } } } for (File generatedJar : generatedJars) { try { generatedJar.delete(); } catch (Exception e) { logger.warn("Failed to delete generated jars", e); } } return provisionedPaths.toArray(new Path[] {}); }