List of usage examples for org.apache.hadoop.fs FileStatus getReplication
public short getReplication()
From source file:a.TestConcatExample.java
License:Apache License
@Test public void concatIsPermissive() throws IOException, URISyntaxException { MiniDFSCluster cluster = null;/*from ww w .j a v a2 s .c o m*/ final Configuration conf = WebHdfsTestUtil.createConf(); conf.set("dfs.namenode.fs-limits.min-block-size", "1000"); // Allow tiny blocks for the test try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); cluster.waitActive(); final FileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsFileSystem.SCHEME); final FileSystem dfs = cluster.getFileSystem(); final FileSystem fs = dfs; // WebHDFS has a bug in getLocatedBlocks Path root = new Path("/dir"); fs.mkdirs(root); short origRep = 3; short secondRep = (short) (origRep - 1); Path f1 = new Path("/dir/f1"); long size1 = writeFile(fs, f1, /* blocksize */ 4096, origRep, 5); long f1NumBlocks = fs.getFileBlockLocations(f1, 0, size1).length; assertEquals(5, f1NumBlocks); Path f2 = new Path("/dir/f2"); long size2 = writeFile(fs, f2, /* blocksize (must divide 512 for checksum) */ 4096 - 512, secondRep, 4); long f2NumBlocks = fs.getFileBlockLocations(f2, 0, size2).length; assertEquals(5, f2NumBlocks); fs.concat(f1, new Path[] { f2 }); FileStatus[] fileStatuses = fs.listStatus(root); // Only one file should remain assertEquals(1, fileStatuses.length); FileStatus fileStatus = fileStatuses[0]; // And it should be named after the first file assertEquals("f1", fileStatus.getPath().getName()); // The entire file takes the replication of the first argument assertEquals(origRep, fileStatus.getReplication()); // As expected, the new concated file is the length of both the previous files assertEquals(size1 + size2, fileStatus.getLen()); // And we should have the same number of blocks assertEquals(f1NumBlocks + f2NumBlocks, fs.getFileBlockLocations(fileStatus.getPath(), 0, size1 + size2).length); } finally { if (cluster != null) { cluster.shutdown(); } } }
From source file:alluxio.hadoop.HadoopUtils.java
License:Apache License
/** * Returns a string representation of a Hadoop {@link FileStatus}. * * @param fs Hadoop {@link FileStatus}/* w w w .j a v a2s . c o m*/ * @return its string representation */ public static String toStringHadoopFileStatus(FileStatus fs) { StringBuilder sb = new StringBuilder(); sb.append("HadoopFileStatus: Path: ").append(fs.getPath()); sb.append(" , Length: ").append(fs.getLen()); // Use isDir instead of isDirectory for compatibility with hadoop 1. sb.append(" , IsDir: ").append(fs.isDir()); sb.append(" , BlockReplication: ").append(fs.getReplication()); sb.append(" , BlockSize: ").append(fs.getBlockSize()); sb.append(" , ModificationTime: ").append(fs.getModificationTime()); sb.append(" , AccessTime: ").append(fs.getAccessTime()); sb.append(" , Permission: ").append(fs.getPermission()); sb.append(" , Owner: ").append(fs.getOwner()); sb.append(" , Group: ").append(fs.getGroup()); return sb.toString(); }
From source file:cascading.tap.hadoop.Hfs.java
License:Open Source License
/** * Method getReplication returns the {@code replication} specified by the underlying file system for * this resource.//from w w w . ja v a2 s .c o m * * @param conf of JobConf * @return int * @throws IOException when */ public int getReplication(Configuration conf) throws IOException { if (!resourceExists(conf)) return 0; FileStatus fileStatus = getFileSystem(conf).getFileStatus(getPath()); if (fileStatus.isDir()) return 0; return fileStatus.getReplication(); }
From source file:ch.cern.db.hdfs.Main.java
License:GNU General Public License
private void printFileStatus(FileStatus status) { System.out.println();//from ww w .ja va 2 s . c o m System.out.println("Showing metadata for: " + status.getPath()); System.out.println(" isDirectory: " + status.isDirectory()); System.out.println(" isFile: " + status.isFile()); System.out.println(" isSymlink: " + status.isSymlink()); System.out.println(" encrypted: " + status.isEncrypted()); System.out.println(" length: " + status.getLen()); System.out.println(" replication: " + status.getReplication()); System.out.println(" blocksize: " + status.getBlockSize()); System.out.println(" modification_time: " + new Date(status.getModificationTime())); System.out.println(" access_time: " + new Date(status.getAccessTime())); System.out.println(" owner: " + status.getOwner()); System.out.println(" group: " + status.getGroup()); System.out.println(" permission: " + status.getPermission()); System.out.println(); }
From source file:cn.ict.magicube.fs.shell.FixedLs.java
License:Apache License
@Override protected void processPath(PathData item) throws IOException { FileStatus stat = item.stat; String line = String.format(lineFormat, (stat.isDirectory() ? "d" : "-"), stat.getPermission(), (stat.isFile() ? stat.getReplication() : "-"), formatSize(stat.getLen()), dateFormat.format(new Date(stat.getModificationTime())), item); out.println(line);//from w w w .j av a 2s .c o m }
From source file:cn.ict.magicube.fs.shell.FixedLs.java
License:Apache License
/** * Compute column widths and rebuild the format string * @param items to find the max field width for each column */// w ww . j a v a2s.c o m private void adjustColumnWidths(PathData items[]) { for (PathData item : items) { FileStatus stat = item.stat; maxRepl = maxLength(maxRepl, stat.getReplication()); maxLen = maxLength(maxLen, stat.getLen()); } StringBuilder fmt = new StringBuilder(); fmt.append("%s%s "); // permission string fmt.append("%" + maxRepl + "s "); fmt.append("%" + maxLen + "s "); fmt.append("%s %s"); // mod time & path lineFormat = fmt.toString(); }
From source file:com.amintor.hdfs.client.kerberizedhdfsclient.KerberizedHDFSClient.java
/** * @param args the command line arguments *///from w ww .j a v a 2 s .c o m public static void main(String[] args) { try { Configuration conf = new Configuration(); conf.addResource(new FileInputStream(HDFS_SITE_LOCATION)); conf.addResource(new FileInputStream(CORE_SITE_LOCATION)); String authType = conf.get("hadoop.security.authentication"); System.out.println("Authentication Type:" + authType); if (authType.trim().equalsIgnoreCase("kerberos")) { // Login through UGI keytab UserGroupInformation.setConfiguration(conf); UserGroupInformation.loginUserFromKeytab("vijay", "/Users/vsingh/Software/vijay.keytab"); FileSystem hdFS = FileSystem.get(conf); FileStatus[] listStatus = hdFS.listStatus(new Path(args[0])); for (FileStatus statusFile : listStatus) { System.out.print("Replication:" + statusFile.getReplication() + "\t"); System.out.print("Owner:" + statusFile.getOwner() + "\t"); System.out.print("Group:" + statusFile.getGroup() + "\t"); System.out.println("Path:" + statusFile.getPath() + "\t"); } } } catch (IOException ex) { Logger.getLogger(KerberizedHDFSClient.class.getName()).log(Level.SEVERE, null, ex); } }
From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java
License:Open Source License
private void testCreate(Path path, boolean override) throws Exception { Configuration conf = new Configuration(); conf.set("fs.http.impl", HoopFileSystem.class.getName()); FileSystem fs = FileSystem.get(getJettyURL().toURI(), conf); FsPermission permission = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE); OutputStream os = fs.create(new Path(path.toUri().getPath()), permission, override, 1024, (short) 2, 100 * 1024 * 1024, null);/*from ww w . j a va 2s .co m*/ os.write(1); os.close(); fs.close(); fs = FileSystem.get(getHadoopConf()); FileStatus status = fs.getFileStatus(path); Assert.assertEquals(status.getReplication(), 2); Assert.assertEquals(status.getBlockSize(), 100 * 1024 * 1024); Assert.assertEquals(status.getPermission(), permission); InputStream is = fs.open(path); Assert.assertEquals(is.read(), 1); is.close(); fs.close(); }
From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java
License:Open Source License
private void testListStatus() throws Exception { FileSystem fs = FileSystem.get(getHadoopConf()); Path path = new Path(getHadoopTestDir(), "foo.txt"); OutputStream os = fs.create(path); os.write(1);//from w w w. j ava 2 s . c o m os.close(); FileStatus status1 = fs.getFileStatus(path); fs.close(); Configuration conf = new Configuration(); conf.set("fs.http.impl", HoopFileSystem.class.getName()); fs = FileSystem.get(getJettyURL().toURI(), conf); FileStatus status2 = fs.getFileStatus(new Path(path.toUri().getPath())); fs.close(); Assert.assertEquals(status2.getPermission(), status1.getPermission()); Assert.assertEquals(status2.getPath().toUri().getPath(), status1.getPath().toUri().getPath()); Assert.assertEquals(status2.getReplication(), status1.getReplication()); Assert.assertEquals(status2.getBlockSize(), status1.getBlockSize()); Assert.assertEquals(status2.getAccessTime(), status1.getAccessTime()); Assert.assertEquals(status2.getModificationTime(), status1.getModificationTime()); Assert.assertEquals(status2.getOwner(), status1.getOwner()); Assert.assertEquals(status2.getGroup(), status1.getGroup()); Assert.assertEquals(status2.getLen(), status1.getLen()); FileStatus[] stati = fs.listStatus(path.getParent()); Assert.assertEquals(stati.length, 1); Assert.assertEquals(stati[0].getPath().getName(), path.getName()); }
From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java
License:Open Source License
private void testSetReplication() throws Exception { FileSystem fs = FileSystem.get(getHadoopConf()); Path path = new Path(getHadoopTestDir(), "foo.txt"); OutputStream os = fs.create(path); os.write(1);/*from w w w .j a va 2s. co m*/ os.close(); fs.close(); fs.setReplication(path, (short) 2); Configuration conf = new Configuration(); conf.set("fs.http.impl", HoopFileSystem.class.getName()); fs = FileSystem.get(getJettyURL().toURI(), conf); fs.setReplication(path, (short) 1); fs.close(); fs = FileSystem.get(getHadoopConf()); FileStatus status1 = fs.getFileStatus(path); fs.close(); Assert.assertEquals(status1.getReplication(), (short) 1); }