List of usage examples for org.apache.hadoop.fs Path SEPARATOR_CHAR
char SEPARATOR_CHAR
To view the source code for org.apache.hadoop.fs Path SEPARATOR_CHAR.
Click Source Link
From source file:com.quest.orahive.Utilities.java
License:Apache License
public static String getOraHiveJarFile() { Path jarFilePath = new Path(Utilities.class.getProtectionDomain().getCodeSource().getLocation().getPath()); String result = jarFilePath.toUri().getPath(); if (!result.endsWith(".jar")) result = result + Path.SEPARATOR_CHAR + Constants.ORAHIVE_JAR_FILENAME; return result; }
From source file:com.talis.hadoop.rdf.ZipUtils.java
License:Apache License
/** * Write a file to a zip output stream, removing leading path name components * from the actual file name when creating the zip file entry. * //from w w w .j av a 2 s. c o m * The entry placed in the zip file is <code>baseName</code>/ * <code>relativePath</code>, where <code>relativePath</code> is constructed * by removing a leading <code>root</code> from the path for * <code>itemToZip</code>. * * If <code>itemToZip</code> is an empty directory, it is ignored. If * <code>itemToZip</code> is a directory, the contents of the directory are * added recursively. * * @param zos The zip output stream * @param baseName The base name to use for the file name entry in the zip * file * @param root The path to remove from <code>itemToZip</code> to make a * relative path name * @param itemToZip The path to the file to be added to the zip file * @return the number of entries added * @throws IOException */ static public int zipDirectory(final Configuration conf, final ZipOutputStream zos, final String baseName, final String root, final Path itemToZip) throws IOException { LOG.info("zipDirectory: {} {} {}", new Object[] { baseName, root, itemToZip }); LocalFileSystem localFs = FileSystem.getLocal(conf); int count = 0; final FileStatus itemStatus = localFs.getFileStatus(itemToZip); if (itemStatus.isDir()) { final FileStatus[] statai = localFs.listStatus(itemToZip); // Add a directory entry to the zip file final String zipDirName = relativePathForZipEntry(itemToZip.toUri().getPath(), baseName, root); final ZipEntry dirZipEntry = new ZipEntry(zipDirName + Path.SEPARATOR_CHAR); LOG.info(String.format("Adding directory %s to zip", zipDirName)); zos.putNextEntry(dirZipEntry); zos.closeEntry(); count++; if (statai == null || statai.length == 0) { LOG.info(String.format("Skipping empty directory %s", itemToZip)); return count; } for (FileStatus status : statai) { count += zipDirectory(conf, zos, baseName, root, status.getPath()); } LOG.info(String.format("Wrote %d entries for directory %s", count, itemToZip)); return count; } final String inZipPath = relativePathForZipEntry(itemToZip.toUri().getPath(), baseName, root); if (inZipPath.length() == 0) { LOG.warn(String.format("Skipping empty zip file path for %s (%s %s)", itemToZip, root, baseName)); return 0; } // Take empty files in case the place holder is needed FSDataInputStream in = null; try { in = localFs.open(itemToZip); final ZipEntry ze = new ZipEntry(inZipPath); ze.setTime(itemStatus.getModificationTime()); // Comments confuse looking at the zip file // ze.setComment(itemToZip.toString()); zos.putNextEntry(ze); IOUtils.copyBytes(in, zos, conf, false); zos.closeEntry(); LOG.info(String.format("Wrote %d entries for file %s", count, itemToZip)); return 1; } finally { in.close(); } }
From source file:fr.ens.biologie.genomique.eoulsan.modules.mgmt.hadoop.DistCp.java
License:LGPL
static private boolean isAncestorPath(final String x, final String y) { if (!y.startsWith(x)) { return false; }//from www. j a v a2 s . c om final int len = x.length(); return y.length() == len || y.charAt(len) == Path.SEPARATOR_CHAR; }
From source file:ml.shifu.dtrain.NNUtils.java
License:Apache License
public static String getTmpNNModelName(String tmpModelsFolder, String trainerId, int iteration) { return new StringBuilder(200).append(tmpModelsFolder).append(Path.SEPARATOR_CHAR).append("model") .append(trainerId).append('-').append(iteration).append(".nn").toString(); }
From source file:ml.shifu.shifu.core.dtrain.DTrainUtils.java
License:Apache License
public static String getTmpModelName(String tmpModelsFolder, String trainerId, int iteration, String modelPost) {//from ww w.j a v a 2s.c om return new StringBuilder(200).append(tmpModelsFolder).append(Path.SEPARATOR_CHAR).append("model") .append(trainerId).append('-').append(iteration).append(".").append(modelPost).toString(); }
From source file:org.apache.accumulo.gc.SimpleGarbageCollector.java
License:Apache License
/** * Move a file, that would otherwise be deleted, to the archive directory for files * * @param fileToArchive//from ww w. j av a 2s .c o m * Path to file that is to be archived * @return True if the file was successfully moved to the file archive directory, false otherwise */ boolean archiveFile(Path fileToArchive) throws IOException { // Figure out what the base path this volume uses on this FileSystem Volume sourceVolume = fs.getVolumeByPath(fileToArchive); String sourceVolumeBasePath = sourceVolume.getBasePath(); log.debug("Base path for volume: " + sourceVolumeBasePath); // Get the path for the file we want to archive String sourcePathBasePath = fileToArchive.toUri().getPath(); // Strip off the common base path for the file to archive String relativeVolumePath = sourcePathBasePath.substring(sourceVolumeBasePath.length()); if (Path.SEPARATOR_CHAR == relativeVolumePath.charAt(0)) { if (relativeVolumePath.length() > 1) { relativeVolumePath = relativeVolumePath.substring(1); } else { relativeVolumePath = ""; } } log.debug("Computed relative path for file to archive: " + relativeVolumePath); // The file archive path on this volume (we can't archive this file to a different volume) Path archivePath = new Path(sourceVolumeBasePath, ServerConstants.FILE_ARCHIVE_DIR); log.debug("File archive path: " + archivePath); fs.mkdirs(archivePath); // Preserve the path beneath the Volume's base directory (e.g. tables/1/A_0000001.rf) Path fileArchivePath = new Path(archivePath, relativeVolumePath); log.debug("Create full path of " + fileArchivePath + " from " + archivePath + " and " + relativeVolumePath); // Make sure that it doesn't already exist, something is wrong. if (fs.exists(fileArchivePath)) { log.warn("Tried to archive file, but it already exists: " + fileArchivePath); return false; } log.debug("Moving " + fileToArchive + " to " + fileArchivePath); return fs.rename(fileToArchive, fileArchivePath); }
From source file:org.apache.accumulo.master.tableOps.CleanUp.java
License:Apache License
protected void archiveFile(VolumeManager fs, String dir, String tableId) throws IOException { Path tableDirectory = new Path(dir, tableId); Volume v = fs.getVolumeByPath(tableDirectory); String basePath = v.getBasePath(); // Path component of URI String tableDirPath = tableDirectory.toUri().getPath(); // Just the suffix of the path (after the Volume's base path) String tableDirSuffix = tableDirPath.substring(basePath.length()); // Remove a leading path separator char because Path will treat the "child" as an absolute path with it if (Path.SEPARATOR_CHAR == tableDirSuffix.charAt(0)) { if (tableDirSuffix.length() > 1) { tableDirSuffix = tableDirSuffix.substring(1); } else {//from w ww . ja va2 s . c om tableDirSuffix = ""; } } // Get the file archive directory on this volume final Path fileArchiveDir = new Path(basePath, ServerConstants.FILE_ARCHIVE_DIR); // Make sure it exists just to be safe fs.mkdirs(fileArchiveDir); // The destination to archive this table to final Path destTableDir = new Path(fileArchiveDir, tableDirSuffix); log.debug("Archiving " + tableDirectory + " to " + tableDirectory); if (fs.exists(destTableDir)) { merge(fs, tableDirectory, destTableDir); } else { fs.rename(tableDirectory, destTableDir); } }
From source file:org.apache.accumulo.server.fs.VolumeManagerImpl.java
License:Apache License
@Override public Path getFullPath(FileType fileType, String path) { int colon = path.indexOf(':'); if (colon > -1) { // Check if this is really an absolute path or if this is a 1.4 style relative path for a WAL if (fileType == FileType.WAL && path.charAt(colon + 1) != '/') { path = path.substring(path.indexOf('/')); } else {//from www. j av a 2s . co m return new Path(path); } } if (path.startsWith("/")) path = path.substring(1); // ACCUMULO-2974 To ensure that a proper absolute path is created, the caller needs to include the table ID // in the relative path. Fail when this doesn't appear to happen. if (FileType.TABLE == fileType) { // Trailing slash doesn't create an additional element String[] pathComponents = StringUtils.split(path, Path.SEPARATOR_CHAR); // Is an rfile if (path.endsWith(RFILE_SUFFIX)) { if (pathComponents.length < 3) { throw new IllegalArgumentException("Fewer components in file path than expected"); } } else { // is a directory if (pathComponents.length < 2) { throw new IllegalArgumentException("Fewer components in directory path than expected"); } } } // normalize the path Path fullPath = new Path(defaultVolume.getBasePath(), fileType.getDirectory()); fullPath = new Path(fullPath, path); FileSystem fs = getVolumeByPath(fullPath).getFileSystem(); return fs.makeQualified(fullPath); }
From source file:org.apache.giraph.master.BspServiceMaster.java
License:Apache License
/** * Write superstep metrics to own file in HDFS * /* w ww. j ava2 s .c o m*/ * @param superstep * the current superstep * @param aggregatedMetrics * the aggregated metrics to write */ private void printAggregatedMetricsToHDFS(long superstep, AggregatedMetrics aggregatedMetrics) { ImmutableClassesGiraphConfiguration conf = getConfiguration(); PrintStream out = null; Path dir = new Path(GiraphConstants.METRICS_DIRECTORY.get(conf)); Path outFile = new Path(GiraphConstants.METRICS_DIRECTORY.get(conf) + Path.SEPARATOR_CHAR + "superstep_" + superstep + ".metrics"); try { FileSystem fs; fs = FileSystem.get(conf); if (!fs.exists(dir)) { fs.mkdirs(dir); } if (fs.exists(outFile)) { throw new RuntimeException("printAggregatedMetricsToHDFS: metrics file exists"); } out = new PrintStream(fs.create(outFile), false, Charset.defaultCharset().name()); aggregatedMetrics.print(superstep, out); } catch (IOException e) { throw new RuntimeException("printAggregatedMetricsToHDFS: error creating metrics file", e); } finally { if (out != null) { out.close(); } } }
From source file:org.apache.hcatalog.har.HarOutputCommitterPostProcessor.java
License:Apache License
/** * Creates a har file from the contents of a given directory, using that as root. * @param dir Directory to archive/*from w w w . j a v a 2s.c o m*/ * @param harFile The HAR file to create */ public static void makeHar(JobContext context, String dir, String harFile) throws IOException { // Configuration conf = context.getConfiguration(); // Credentials creds = context.getCredentials(); // HCatUtil.logAllTokens(LOG,context); int lastSep = harFile.lastIndexOf(Path.SEPARATOR_CHAR); Path archivePath = new Path(harFile.substring(0, lastSep)); final String[] args = { "-archiveName", harFile.substring(lastSep + 1, harFile.length()), "-p", dir, "*", archivePath.toString() }; // for (String arg : args){ // LOG.info("Args to har : "+ arg); // } try { Configuration newConf = new Configuration(); FileSystem fs = archivePath.getFileSystem(newConf); String hadoopTokenFileLocationEnvSetting = System .getenv(HCatConstants.SYSENV_HADOOP_TOKEN_FILE_LOCATION); if ((hadoopTokenFileLocationEnvSetting != null) && (!hadoopTokenFileLocationEnvSetting.isEmpty())) { newConf.set(HCatConstants.CONF_MAPREDUCE_JOB_CREDENTIALS_BINARY, hadoopTokenFileLocationEnvSetting); // LOG.info("System.getenv(\"HADOOP_TOKEN_FILE_LOCATION\") =["+ System.getenv("HADOOP_TOKEN_FILE_LOCATION")+"]"); } // for (FileStatus ds : fs.globStatus(new Path(dir, "*"))){ // LOG.info("src : "+ds.getPath().toUri().toString()); // } final HadoopArchives har = new HadoopArchives(newConf); int rc = ToolRunner.run(har, args); if (rc != 0) { throw new Exception("Har returned error code " + rc); } // for (FileStatus hs : fs.globStatus(new Path(harFile, "*"))){ // LOG.info("dest : "+hs.getPath().toUri().toString()); // } // doHarCheck(fs,harFile); // LOG.info("Nuking " + dir); fs.delete(new Path(dir), true); } catch (Exception e) { throw new HCatException("Error creating Har [" + harFile + "] from [" + dir + "]", e); } }