List of usage examples for org.apache.hadoop.fs FileSystem getUri
public abstract URI getUri();
From source file:org.apache.falcon.regression.core.util.HadoopUtil.java
License:Apache License
/** * Removes given directory from a filesystem. * @param hdfsPath path to a given directory * @param fs filesystem// w w w . j av a2 s. c om * @throws IOException */ public static void deleteDirIfExists(String hdfsPath, FileSystem fs) throws IOException { Path path = new Path(hdfsPath); if (fs.exists(path)) { LOGGER.info(String.format("Deleting HDFS path: %s on %s", path, fs.getUri())); fs.delete(path, true); } else { LOGGER.info(String.format("Not deleting non-existing HDFS path: %s on %s", path, fs.getUri())); } }
From source file:org.apache.falcon.regression.core.util.HadoopUtil.java
License:Apache License
/** * Copies files from a source directory to target directories on hdfs. * @param fs target filesystem// w ww . j av a 2 s . co m * @param inputPath source location * @param remotePathPrefix prefix for target directories * @param remoteLocations target directories * @return list of exact locations where data was copied * @throws IOException */ public static List<String> flattenAndPutDataInFolder(FileSystem fs, String inputPath, String remotePathPrefix, List<String> remoteLocations) throws IOException { if (StringUtils.isNotEmpty(remotePathPrefix)) { deleteDirIfExists(remotePathPrefix, fs); } LOGGER.info("Creating data in folders: \n" + remoteLocations); File input = new File(inputPath); File[] files = input.isDirectory() ? input.listFiles() : new File[] { input }; List<Path> filePaths = new ArrayList<>(); assert files != null; for (final File file : files) { if (!file.isDirectory()) { final Path filePath = new Path(file.getAbsolutePath()); filePaths.add(filePath); } } if (!remotePathPrefix.endsWith("/") && !remoteLocations.get(0).startsWith("/")) { remotePathPrefix += "/"; } List<String> locations = new ArrayList<>(); for (String remoteDir : remoteLocations) { String remoteLocation = remotePathPrefix + remoteDir; remoteLocation = cutProtocol(remoteLocation); locations.add(remoteLocation); LOGGER.info(String.format("copying to: %s files: %s", fs.getUri() + remoteLocation, Arrays.toString(files))); if (!fs.exists(new Path(remoteLocation))) { fs.mkdirs(new Path(remoteLocation)); } fs.copyFromLocalFile(false, true, filePaths.toArray(new Path[filePaths.size()]), new Path(remoteLocation)); } return locations; }
From source file:org.apache.falcon.regression.core.util.HadoopUtil.java
License:Apache License
/** * Copies data from local sources to remote directories. * @param fs target filesystem/*from ww w . j a va2 s .c om*/ * @param folderPrefix prefix for remote directories * @param folderList remote directories * @param fileLocations sources * @throws IOException */ public static void copyDataToFolders(FileSystem fs, final String folderPrefix, List<String> folderList, String... fileLocations) throws IOException { for (final String folder : folderList) { String folderSpace = folder.replaceAll("/", "_"); File file = new File(OSUtil.NORMAL_INPUT + folderSpace + ".txt"); FileUtils.writeStringToFile(file, "folder", true); fs.copyFromLocalFile(new Path(file.getAbsolutePath()), new Path(folderPrefix + folder)); if (!file.delete()) { LOGGER.info("delete was not successful for file: " + file); } Path[] srcPaths = new Path[fileLocations.length]; for (int i = 0; i < srcPaths.length; ++i) { srcPaths[i] = new Path(fileLocations[i]); } LOGGER.info(String.format("copying %s to %s%s on %s", Arrays.toString(srcPaths), folderPrefix, folder, fs.getUri())); fs.copyFromLocalFile(false, true, srcPaths, new Path(folderPrefix + folder)); } }
From source file:org.apache.falcon.regression.core.util.HadoopUtil.java
License:Apache License
/** * Creates list of folders on remote filesystem. * @param fs remote filesystem//from www .j a va2s. co m * @param folderPrefix prefix for remote directories * @param folderList list of folders * @throws IOException */ public static void createFolders(FileSystem fs, final String folderPrefix, List<String> folderList) throws IOException { for (final String folder : folderList) { final String pathString = cutProtocol(folderPrefix + folder); LOGGER.info("Creating " + fs.getUri() + "/" + pathString); fs.mkdirs(new Path(pathString)); } }
From source file:org.apache.falcon.regression.ExternalFSTest.java
License:Apache License
@Test(dataProvider = "getData") public void replicateToExternalFS(final FileSystem externalFS, final String separator, final boolean withData) throws Exception { final String endpoint = externalFS.getUri().toString(); Bundle.submitCluster(bundles[0], externalBundle); String startTime = TimeUtil.getTimeWrtSystemTime(0); String endTime = TimeUtil.addMinsToTime(startTime, 5); LOGGER.info("Time range between : " + startTime + " and " + endTime); String datePattern = StringUtils .join(new String[] { "${YEAR}", "${MONTH}", "${DAY}", "${HOUR}", "${MINUTE}" }, separator); //configure feed FeedMerlin feed = new FeedMerlin(bundles[0].getDataSets().get(0)); String targetDataLocation = endpoint + testWasbTargetDir + datePattern; feed.setFilePath(sourcePath + '/' + datePattern); //erase all clusters from feed definition feed.clearFeedClusters();// ww w. j a va2 s. co m //set local cluster as source feed.addFeedCluster(new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) .withRetention("days(1000000)", ActionType.DELETE).withValidity(startTime, endTime) .withClusterType(ClusterType.SOURCE).build()); //set externalFS cluster as target feed.addFeedCluster( new FeedMerlin.FeedClusterBuilder(Util.readEntityName(externalBundle.getClusters().get(0))) .withRetention("days(1000000)", ActionType.DELETE).withValidity(startTime, endTime) .withClusterType(ClusterType.TARGET).withDataLocation(targetDataLocation).build()); //submit and schedule feed LOGGER.info("Feed : " + Util.prettyPrintXml(feed.toString())); AssertUtil.assertSucceeded(prism.getFeedHelper().submitAndSchedule(feed.toString())); datePattern = StringUtils.join(new String[] { "yyyy", "MM", "dd", "HH", "mm" }, separator); //upload necessary data DateTime date = new DateTime(startTime, DateTimeZone.UTC); DateTimeFormatter fmt = DateTimeFormat.forPattern(datePattern); String timePattern = fmt.print(date); HadoopUtil.recreateDir(clusterFS, sourcePath + '/' + timePattern); if (withData) { HadoopUtil.copyDataToFolder(clusterFS, sourcePath + '/' + timePattern, OSUtil.SINGLE_FILE); } Path srcPath = new Path(sourcePath + '/' + timePattern); Path dstPath = new Path(endpoint + testWasbTargetDir + '/' + timePattern); //check if coordinator exists TimeUtil.sleepSeconds(10); InstanceUtil.waitTillInstancesAreCreated(clusterOC, feed.toString(), 0); Assert.assertEquals(OozieUtil.checkIfFeedCoordExist(clusterOC, feed.getName(), "REPLICATION"), 1); //replication should start, wait while it ends InstanceUtil.waitTillInstanceReachState(clusterOC, Util.readEntityName(feed.toString()), 1, CoordinatorAction.Status.SUCCEEDED, EntityType.FEED); //check if data has been replicated correctly List<Path> cluster1ReplicatedData = HadoopUtil.getAllFilesRecursivelyHDFS(clusterFS, srcPath); List<Path> cluster2ReplicatedData = HadoopUtil.getAllFilesRecursivelyHDFS(externalFS, dstPath); AssertUtil.checkForListSizes(cluster1ReplicatedData, cluster2ReplicatedData); final ContentSummary srcSummary = clusterFS.getContentSummary(srcPath); final ContentSummary dstSummary = externalFS.getContentSummary(dstPath); Assert.assertEquals(dstSummary.getLength(), srcSummary.getLength()); }
From source file:org.apache.falcon.replication.FilteredCopyListingTest.java
License:Apache License
private static void recordInExpectedValues(String path) throws Exception { FileSystem fileSystem = FileSystem.getLocal(new Configuration()); Path sourcePath = new Path(fileSystem.getUri().toString() + path); EXPECTED_VALUES.put(sourcePath.toString(), DistCpUtils.getRelativePath(new Path("/tmp/source"), sourcePath)); }
From source file:org.apache.falcon.service.SharedLibraryHostingService.java
License:Apache License
@SuppressWarnings("ConstantConditions") public static void pushLibsToHDFS(FileSystem fs, String src, Path target, FalconPathFilter pathFilter) throws IOException, FalconException { if (StringUtils.isEmpty(src)) { return;// w ww . j a v a 2 s.com } LOG.debug("Copying libs from {}", src); createTargetPath(fs, target); for (String srcPaths : src.split(",")) { File srcFile = new File(srcPaths); File[] srcFiles = new File[] { srcFile }; if (srcFiles != null) { if (srcFile.isDirectory()) { srcFiles = srcFile.listFiles(); } } if (srcFiles != null) { for (File file : srcFiles) { Path path = new Path(file.getAbsolutePath()); String jarName = StringUtils.removeEnd(path.getName(), ".jar"); if (pathFilter != null) { if (!pathFilter.accept(path)) { continue; } jarName = pathFilter.getJarName(path); } Path targetFile = new Path(target, jarName + ".jar"); if (fs.exists(targetFile)) { FileStatus fstat = fs.getFileStatus(targetFile); if (fstat.getLen() == file.length()) { continue; } } fs.copyFromLocalFile(false, true, new Path(file.getAbsolutePath()), targetFile); fs.setPermission(targetFile, HadoopClientFactory.READ_EXECUTE_PERMISSION); LOG.info("Copied {} to {} in {}", file.getAbsolutePath(), targetFile.toString(), fs.getUri()); } } } }
From source file:org.apache.flink.streaming.connectors.fs.bucketing.BucketingSinkFsInitTest.java
License:Apache License
@Test public void testInitForLocalFileSystem() throws Exception { final Path path = new Path(tempFolder.newFolder().toURI()); FileSystem fs = BucketingSink.createHadoopFileSystem(path, null); assertEquals("file", fs.getUri().getScheme()); }
From source file:org.apache.flink.streaming.connectors.fs.bucketing.BucketingSinkFsInitTest.java
License:Apache License
@Test public void testInitForHadoopFileSystem() throws Exception { final Path path = new Path("hdfs://localhost:51234/some/path/"); FileSystem fs = BucketingSink.createHadoopFileSystem(path, null); assertEquals("hdfs", fs.getUri().getScheme()); }
From source file:org.apache.giraph.debugger.gui.ServerUtils.java
License:Apache License
/** * @param jobId id of the job, whose jar path will be returned. * @return a url wrapped inside an array for convenience. *///ww w . j a v a2s.c o m public static URL[] getCachedJobJarPath(String jobId) { // read the jar signature file under the TRACE_ROOT/jobId/ Path jarSignaturePath = new Path(DebuggerUtils.getTraceFileRoot(jobId) + "/" + "jar.signature"); try { FileSystem fs = getFileSystem(); try (FSDataInputStream jarSignatureInput = fs.open(jarSignaturePath)) { List<String> lines = IOUtils.readLines(jarSignatureInput); if (lines.size() > 0) { String jarSignature = lines.get(0); // check if jar is already in JARCACHE_LOCAL File localFile = new File(DebuggerUtils.JARCACHE_LOCAL + "/" + jarSignature + ".jar"); if (!localFile.exists()) { // otherwise, download from HDFS Path hdfsPath = new Path( fs.getUri().resolve(DebuggerUtils.JARCACHE_HDFS + "/" + jarSignature + ".jar")); Logger.getLogger(ServerUtils.class) .info("Copying from HDFS: " + hdfsPath + " to " + localFile); localFile.getParentFile().mkdirs(); fs.copyToLocalFile(hdfsPath, new Path(localFile.toURI())); } return new URL[] { localFile.toURI().toURL() }; } } } catch (IOException e) { // gracefully ignore if we failed to read the jar.signature LOG.warn("An IOException is thrown but will be ignored: " + e.toString()); } return new URL[0]; }