List of usage examples for org.apache.hadoop.fs FileStatus getReplication
public short getReplication()
From source file:com.ning.metrics.action.hdfs.reader.HdfsEntry.java
License:Apache License
public HdfsEntry(FileSystem fs, FileStatus status, boolean raw, RowFileContentsIteratorFactory rowFileContentsIteratorFactory) throws IOException { this.fs = fs; this.path = status.getPath(); this.modificationDate = new DateTime(status.getModificationTime()); this.blockSize = status.getBlockSize(); this.size = status.getLen(); this.replication = status.getReplication(); this.replicatedSize = status.getReplication() * status.getLen(); this.directory = status.isDir(); this.raw = raw; this.rowFileContentsIteratorFactory = rowFileContentsIteratorFactory; }
From source file:com.ning.metrics.action.hdfs.writer.TestHdfsWriter.java
License:Apache License
@Test(groups = "fast") public void testWrite() throws Exception { final HdfsWriter writer = new HdfsWriter(fileSystemAccess); final String outputFile = outputPath + "/lorem.txt"; final URI outputURI = writer.write(new FileInputStream(file), outputFile, false, (short) 1, 1024, "ugo=r"); final FileStatus status = fileSystemAccess.get().getFileStatus(new Path(outputURI)); Assert.assertEquals(status.getPath().toString(), "file:" + outputFile); Assert.assertEquals(status.getReplication(), (short) 1); //Broken for some reason? //Assert.assertEquals(status.getBlockSize(), 1024); //Assert.assertEquals(status.getPermission().toString(), "-r--r--r--"); // Test that we don't overwrite the file try {// w ww.j a v a 2 s . c o m writer.write(new FileInputStream(file), outputFile, false, (short) 1, 1024, "ugo=r"); Assert.fail(); } catch (IOException e) { Assert.assertEquals(e.getLocalizedMessage(), String.format("File already exists:%s", outputFile)); } }
From source file:com.pivotal.hawq.mapreduce.parquet.HAWQParquetInputFormat.java
License:Apache License
@Override protected List<FileStatus> listStatus(JobContext jobContext) throws IOException { List<FileStatus> result = Lists.newArrayList(); for (HAWQFileStatus hawqFileStatus : hawqFileStatuses) { if (hawqFileStatus.getFileLength() == 0) continue; // skip empty file Path path = new Path(hawqFileStatus.getFilePath()); FileSystem fs = path.getFileSystem(jobContext.getConfiguration()); FileStatus dfsStat = fs.getFileStatus(path); // rewrite file length because HAWQ records the logicalEOF of file, which may // be smaller than the file's actual EOF FileStatus hawqStat = new FileStatus(hawqFileStatus.getFileLength(), // rewrite to logicalEOF dfsStat.isDirectory(), dfsStat.getReplication(), dfsStat.getBlockSize(), dfsStat.getModificationTime(), dfsStat.getAccessTime(), dfsStat.getPermission(), dfsStat.getOwner(), dfsStat.getGroup(), dfsStat.getPath()); result.add(hawqStat);/*w ww . j a v a2 s.c o m*/ } return result; }
From source file:com.quantcast.qfs.hadoop.QFSEmulationImpl.java
License:Apache License
public KfsFileAttr fullStat(Path path) throws IOException { FileStatus fs = localFS.getFileStatus(new Path(path.toUri().getPath())); KfsFileAttr fa = new KfsFileAttr(); fa.filename = fs.getPath().toUri().getPath(); fa.isDirectory = fs.isDir();/* w ww . j av a 2 s .c om*/ fa.filesize = fs.getLen(); fa.replication = fs.getReplication(); fa.modificationTime = fs.getModificationTime(); return fa; }
From source file:com.ruizhan.hadoop.hdfs.Ls.java
License:Apache License
@Override protected void processPath(PathData item) throws IOException { FileStatus stat = item.stat; String line = String.format(lineFormat, (stat.isDirectory() ? "d" : "-"), stat.getPermission(), (stat.isFile() ? stat.getReplication() : "-"), stat.getOwner(), stat.getGroup(), formatSize(stat.getLen()), dateFormat.format(new Date(stat.getModificationTime())), item); out.println(line);/*from www . ja va 2 s. com*/ }
From source file:com.ruizhan.hadoop.hdfs.Ls.java
License:Apache License
/** * Compute column widths and rebuild the format string * @param items to find the max field width for each column *//*w ww. jav a2 s. c o m*/ private void adjustColumnWidths(PathData items[]) { for (PathData item : items) { FileStatus stat = item.stat; maxRepl = maxLength(maxRepl, stat.getReplication()); maxLen = maxLength(maxLen, stat.getLen()); maxOwner = maxLength(maxOwner, stat.getOwner()); maxGroup = maxLength(maxGroup, stat.getGroup()); } StringBuilder fmt = new StringBuilder(); fmt.append("%s%s "); // permission string fmt.append("%" + maxRepl + "s "); // Do not use '%-0s' as a formatting conversion, since it will throw a // a MissingFormatWidthException if it is used in String.format(). // http://docs.oracle.com/javase/1.5.0/docs/api/java/util/Formatter.html#intFlags fmt.append((maxOwner > 0) ? "%-" + maxOwner + "s " : "%s"); fmt.append((maxGroup > 0) ? "%-" + maxGroup + "s " : "%s"); fmt.append("%" + maxLen + "s "); fmt.append("%s %s"); // mod time & path lineFormat = fmt.toString(); }
From source file:com.streamsets.pipeline.stage.origin.hdfs.spooler.HdfsFile.java
License:Apache License
@SuppressWarnings("unchecked") public Map<String, Object> getFileMetadata() throws IOException { FileStatus file = fs.getFileStatus(filePath); Map<String, Object> metadata = new HashMap<>(); metadata.put(HeaderAttributeConstants.FILE_NAME, file.getPath().getName()); metadata.put(HeaderAttributeConstants.FILE, file.getPath().toUri().getPath()); metadata.put(HeaderAttributeConstants.LAST_MODIFIED_TIME, file.getModificationTime()); metadata.put(HeaderAttributeConstants.LAST_ACCESS_TIME, file.getAccessTime()); metadata.put(HeaderAttributeConstants.IS_DIRECTORY, file.isDirectory()); metadata.put(HeaderAttributeConstants.IS_SYMBOLIC_LINK, file.isSymlink()); metadata.put(HeaderAttributeConstants.SIZE, file.getLen()); metadata.put(HeaderAttributeConstants.OWNER, file.getOwner()); metadata.put(HeaderAttributeConstants.GROUP, file.getGroup()); metadata.put(HeaderAttributeConstants.BLOCK_SIZE, file.getBlockSize()); metadata.put(HeaderAttributeConstants.REPLICATION, file.getReplication()); metadata.put(HeaderAttributeConstants.IS_ENCRYPTED, file.isEncrypted()); FsPermission permission = file.getPermission(); if (permission != null) { metadata.put(PERMISSIONS, permission.toString()); }/*from w w w . j ava2s . c om*/ return metadata; }
From source file:com.uber.hoodie.common.table.timeline.dto.FileStatusDTO.java
License:Apache License
public static FileStatusDTO fromFileStatus(FileStatus fileStatus) { if (null == fileStatus) { return null; }/*from ww w . ja v a 2s.c o m*/ FileStatusDTO dto = new FileStatusDTO(); try { dto.path = FilePathDTO.fromPath(fileStatus.getPath()); dto.length = fileStatus.getLen(); dto.isdir = fileStatus.isDirectory(); dto.blockReplication = fileStatus.getReplication(); dto.blocksize = fileStatus.getBlockSize(); dto.modificationTime = fileStatus.getModificationTime(); dto.accessTime = fileStatus.getModificationTime(); dto.symlink = fileStatus.isSymlink() ? FilePathDTO.fromPath(fileStatus.getSymlink()) : null; safeReadAndSetMetadata(dto, fileStatus); } catch (IOException ioe) { throw new HoodieException(ioe); } return dto; }
From source file:crunch.MaxTemperature.java
License:Apache License
@Test
public void fileStatusForFile() throws IOException {
Path file = new Path("/dir/file"); // XXX new Path creates the file
FileStatus stat = fs.getFileStatus(file);
assertThat(stat.getPath().toUri().getPath(), is("/dir/file")); // XXX FileStatus.getPath().toUri() -> URI .getPath()
assertThat(stat.isDir(), is(false)); // XXX assertThat(actual, Matcher) Matcher provides matches method
assertThat(stat.getLen(), is(7L));
assertThat(stat.getModificationTime(), is(lessThanOrEqualTo(System.currentTimeMillis())));
assertThat(stat.getReplication(), is((short) 1)); // XXX Matcher<Short> is(Short value) -> o.h.core.Is.<Short>is(Short) static factory method from oh.core.Is
// XXX which calls the constructor is(Matcher<Short> matcher) with the matcher equalTo(Short) hamcrest-all-1.3-source.java:Is.java:65
assertThat(stat.getBlockSize(), is(64 * 1024 * 1024L));
assertThat(stat.getOwner(), is(System.getProperty("user.name")));
assertThat(stat.getGroup(), is("supergroup"));
assertThat(stat.getPermission().toString(), is("rw-r--r--"));
}//from w w w . j a va 2 s . c om
From source file:crunch.MaxTemperature.java
License:Apache License
@Test
public void fileStatusForDirectory() throws IOException {
Path dir = new Path("/dir"); // XXX new Path creates the directory
FileStatus stat = fs.getFileStatus(dir);
assertThat(stat.getPath().toUri().getPath(), is("/dir"));
assertThat(stat.isDir(), is(true));
assertThat(stat.getLen(), is(0L));
assertThat(stat.getModificationTime(), is(lessThanOrEqualTo(System.currentTimeMillis())));
assertThat(stat.getReplication(), is((short) 0));
assertThat(stat.getBlockSize(), is(0L));
assertThat(stat.getOwner(), is(System.getProperty("user.name")));
assertThat(stat.getGroup(), is("supergroup"));
assertThat(stat.getPermission().toString(), is("rwxr-xr-x"));
}/* ww w .j a v a 2 s . c om*/