List of usage examples for org.apache.hadoop.fs FileStatus getOwner
public String getOwner()
From source file:com.trendmicro.hdfs.webdav.HDFSResource.java
License:Apache License
@SuppressWarnings({ "rawtypes", "unchecked" })
private void populateProperties() {
if (properties != null) {
return;/*ww w . j a va2 s . c o m*/
}
properties = new DavPropertySet();
FileStatus stat = null;
try {
stat = user.doAs(new PrivilegedExceptionAction<FileStatus>() {
public FileStatus run() throws Exception {
return FileSystem.get(conf).getFileStatus(getPath());
}
});
} catch (IOException ex) {
LOG.warn(StringUtils.stringifyException(ex));
} catch (InterruptedException e) {
LOG.warn(StringUtils.stringifyException(e));
}
if (stat != null) {
properties.add(new DefaultDavProperty(DavPropertyName.GETCONTENTLENGTH, stat.getLen()));
SimpleDateFormat simpleFormat = (SimpleDateFormat) DavConstants.modificationDateFormat.clone();
simpleFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
Date date = new Date(stat.getModificationTime());
properties.add(new DefaultDavProperty(DavPropertyName.GETLASTMODIFIED, simpleFormat.format(date)));
properties.add(new DefaultDavProperty(SecurityConstants.OWNER, stat.getOwner()));
properties.add(new DefaultDavProperty(SecurityConstants.GROUP, stat.getGroup()));
// TODO: Populate DAV property SecurityConstants.CURRENT_USER_PRIVILEGE_SET
}
if (getDisplayName() != null) {
properties.add(new DefaultDavProperty(DavPropertyName.DISPLAYNAME, getDisplayName()));
}
if (isCollection()) {
properties.add(new ResourceType(ResourceType.COLLECTION));
// Windows XP support
properties.add(new DefaultDavProperty(DavPropertyName.ISCOLLECTION, "1"));
} else {
properties.add(new ResourceType(ResourceType.DEFAULT_RESOURCE));
// Windows XP support
properties.add(new DefaultDavProperty(DavPropertyName.ISCOLLECTION, "0"));
}
}
From source file:com.tripadvisor.hadoop.BackupHdfs.java
License:Apache License
/** * Method to move files from HDFS to local filesystem * * localPath: Path on the machines filesystem * fs:FileSystem object from HDFS/* w ww . java 2s . co m*/ * pathList:List of paths for files that might need to be backed * up * size:max size in bytes to be backed up * * ReturnsDate of the last files backed up if reached size limit, * else, zero **/ public long backupFiles(String localPath, String preservePath, FileSystem fs, ArrayList<Path> pathList, long size) { Path fsPath; long tmpSize = 0; long tmpDate = 0; // Start iterating over all paths for (Path hdfsPath : pathList) { try { long nFileSize = fs.getContentSummary(hdfsPath).getLength(); tmpSize = tmpSize + nFileSize; if ((tmpSize <= size) || (size == 0)) { FileStatus stat = fs.getFileStatus(hdfsPath); System.err.println("File " + hdfsPath.toUri().getPath() + " " + nFileSize + " bytes, " + "perms: " + stat.getOwner() + "/" + stat.getGroup() + ", " + stat.getPermission().toString()); tmpDate = stat.getModificationTime() / 1000; String sFsPath = localPath + hdfsPath.toUri().getPath(); fsPath = new Path(sFsPath); File f = new File(sFsPath); // COMMENTED OUT: until a few backup cycles run // and the mtime gets in fact set on all copied // files. // // ignore it if the file exists and has the same mtime // if (f.exists() && f.isFile() && f.lastModified() == stat.getModificationTime()) // { // System.out.println("no need to backup " + f.toString() + ", mtime matches hdfs"); // continue; // } if (false == m_bDryRun) { // check if we need to back up the local file // (not directory), if it already exists. if (f.exists() && f.isFile()) { // ignore files with substrings in the // no-preserve file if (true == doPreserveFile(sFsPath)) { // move it to the backup path String sNewPath = preservePath + hdfsPath.toUri().getPath(); File newFile = new File(sNewPath); // create directory structure for new file? if (false == newFile.getParentFile().exists()) { if (false == newFile.getParentFile().mkdirs()) { System.err .println("Failed to mkdirs " + newFile.getParentFile().toString()); System.exit(1); } } // rename existing file to new location if (false == f.renameTo(newFile)) { System.err.println( "Failed to renameTo " + f.toString() + " to " + newFile.toString()); System.exit(1); } System.out.println("preserved " + f.toString() + " into " + newFile.toString()); } else { System.out.println("skipped preservation of " + f.toString()); } } // copy from hdfs to local filesystem fs.copyToLocalFile(hdfsPath, fsPath); // set the mtime to match hdfs file f.setLastModified(stat.getModificationTime()); // compare checksums on both files compareChecksums(fs, hdfsPath, sFsPath); } // don't print the progress after every file -- go // by at least 1% increments long nPercentDone = (long) (100 * tmpSize / m_nTotalBytes); if (nPercentDone > m_nLastPercentBytesDone) { System.out.println("progress: copied " + prettyPrintBytes(tmpSize) + ", " + nPercentDone + "% done" + ", tstamp=" + tmpDate); m_nLastPercentBytesDone = nPercentDone; } if (m_nSleepSeconds > 0) { try { Thread.sleep(1000 * m_nSleepSeconds); } catch (Exception e2) { // ignore } } } else { return tmpDate; } } catch (IOException e) { System.err.println("FATAL ERROR: Something wrong with the file"); System.err.println(e); System.out.println(tmpDate); System.exit(1); return 0; } } return 0; }
From source file:com.tripadvisor.hadoop.BackupHdfs.java
License:Apache License
/** * Method to go though the HDFS filesystem in a DFS to find all * files/*from w w w . j av a2 s . co m*/ * * fs:FileSystem object from HDFS * minDate: Oldest date for files to be backed up * maxDate:Newest date for files to be backed up * p:Path in HDFS to look for files * pathList:Will be filled with all files in p * hmTimestamps: hashmap of timestamps for later sorting **/ public void checkDir(FileSystem fs, long minDate, long maxDate, Path p, ArrayList<Path> pathList, HashMap<Path, Long> hmTimestamps) { long tmpDate; FileStatus[] fStat; try { String sPath = p.toUri().getPath(); // If this is a directory if (fs.getFileStatus(p).isDir()) { // ignore certain directories if ("dfstmp".equals(p.getName()) || "tmp".equals(p.getName()) || "jobtracker".equals(p.getName()) || sPath.startsWith("/mapred") || "ops".equals(p.getName()) || p.getName().startsWith("_distcp_logs")) { return; } // dump the mkdir and chmod commands for this // directory -- skip root directory only { FileStatus stat = fs.getFileStatus(p); if (!sPath.equals("/")) { m_wrMkdirs.println("hadoop fs -mkdir " + sPath); } m_wrChmods.println("hadoop fs -chown " + stat.getOwner() + ":" + stat.getGroup() + " " + sPath); Short sh = new Short(stat.getPermission().toShort()); m_wrChmods.println("hadoop fs -chmod " + Long.toOctalString(sh.longValue()) + " " + sPath); } fStat = fs.listStatus(p); // Do a recursive call to all elements for (int i = 0; i < fStat.length; i++) { checkDir(fs, minDate, maxDate, fStat[i].getPath(), pathList, hmTimestamps); } } else { // If not a directory then we've found a file // ignore crc files if (p.getName().endsWith(".crc")) { return; } // ignore other files if (sPath.startsWith("/user/oozie/etl/workflows/")) { return; } // try to get the table name from the path. There are // various types of tables, from those replicated from // another database to regular hive tables to // partitioned hive tables. We use table names to // both exclude some from the backup, and for the rest // to dump out the schema and partition name. if (m_ignoreTables != null && m_ignoreTables.doIgnoreFile(sPath)) { m_nIgnoredTables++; if (m_nIgnoredTables < 5) { System.out.println("Skipping ignore-table file: " + sPath); } else if (m_nIgnoredTables == 5) { System.out.println("(...not showing other skipped tables...)"); } return; } FileStatus stat = fs.getFileStatus(p); tmpDate = stat.getModificationTime() / 1000; // store the chmods/chowns for all files m_wrChmods.println("hadoop fs -chown " + stat.getOwner() + ":" + stat.getGroup() + " " + sPath); m_wrChmods.println("hadoop fs -chmod " + stat.getPermission().toShort() + " " + sPath); // check dates. is it too young? if (tmpDate < minDate) { return; } // is the file too recent? if (tmpDate > maxDate) { //System.out.println("file too recent: " + sPath); return; } // file timestamp is ok pathList.add(p); hmTimestamps.put(p, new Long(tmpDate)); // store info about total bytes neeed to backup m_nTotalBytes += fs.getContentSummary(p).getLength(); } } catch (IOException e) { System.err.println("ERROR: could not open " + p + ": " + e); // System.exit(1) ; } }
From source file:com.tripadvisor.hadoop.VerifyHdfsBackup.java
License:Apache License
/** * Method to go though the HDFS filesystem in a DFS to find all * files/*from www . j a v a2 s. c o m*/ * * fs:FileSystem object from HDFS * maxDate:Newest date for files to be backed up * p:Path in HDFS to look for files **/ public void checkDir(FileSystem fs, Path p, String sLocalPathRoot, long maxDate) { FileStatus[] fStat; try { String sPath = p.toUri().getPath(); // If this is a directory if (fs.getFileStatus(p).isDir()) { // ignore certain directories if ("dfstmp".equals(p.getName()) || "tmp".equals(p.getName()) || "jobtracker".equals(p.getName()) || sPath.startsWith("/mapred") || "ops".equals(p.getName()) || p.getName().startsWith("_distcp_logs")) { return; } fStat = fs.listStatus(p); // Do a recursive call to all elements for (int i = 0; i < fStat.length; i++) { checkDir(fs, fStat[i].getPath(), sLocalPathRoot, maxDate); } } else { // If not a directory then we've found a file // ignore crc files if (p.getName().endsWith(".crc")) { return; } // ignore other files if (sPath.startsWith("/user/oozie/etl/workflows/")) { return; } // try to get the table name from the path. There are // various types of tables, from those replicated from // tripmonster to regular hive tables to partitioned // hive tables. We use table names to both exclude // some from the backup, and for the rest to dump out // the schema and partition name. if (m_ignoreTables != null && m_ignoreTables.doIgnoreFile(sPath)) { return; } // check the file FileStatus stat = fs.getFileStatus(p); // ignore files that are too new if ((stat.getModificationTime() / 1000) > maxDate) { System.out.println("IGNORING: " + sPath + " too new"); return; } // warn about files that have a mis-matching block // size. The checksum check will fail for them // anyways, so just catch it here. if (stat.getBlockSize() != N_BLOCK_SIZE) { System.out.println("ERROR: non-default block size (" + (stat.getBlockSize() / (1024 * 1024)) + "M) would fail checksum: " + sPath); return; } // get HDFS checksum FileChecksum ck = fs.getFileChecksum(p); String sCk, sCkShort; if (ck == null) { sCk = sCkShort = "<null>"; } else { sCk = ck.toString(); sCkShort = sCk.replaceAll("^.*:", ""); } System.out.println(sPath + " len=" + stat.getLen() + " " + stat.getOwner() + "/" + stat.getGroup() + " checksum=" + sCk); // find the local file String sFsPath = sLocalPathRoot + p.toUri().getPath(); File fLocal = new File(sFsPath); if (!fLocal.exists()) { Calendar cal = Calendar.getInstance(); cal.setTimeInMillis(stat.getModificationTime()); System.out.println("ERROR: file does not exist: " + sFsPath + " hdfs-last-mtime=" + cal.getTime().toString()); return; } if (!fLocal.isFile()) { System.out.println("ERROR: path is not a file: " + sFsPath); return; } if (stat.getLen() != fLocal.length()) { System.out.println("ERROR: length mismatch: " + sFsPath + " hdfslen=" + stat.getLen() + " fslen=" + fLocal.length()); return; } // get local fs checksum FileChecksum ckLocal = getLocalFileChecksum(sFsPath); if (ckLocal == null) { System.out.println("ERROR Failed to get checksum for local file " + sFsPath); return; } // compare checksums as a string, to strip the // algorithm name from the beginning String sCkLocal = ckLocal.toString(); String sCkLocalShort = sCkLocal.replaceAll("^.*:", ""); if (false == sCkShort.equals(sCkLocalShort)) { System.out.println( "ERROR: checksum mismatch: " + sFsPath + "\nhdfs = " + sCk + "\nlocal= " + sCkLocal); return; } } } catch (IOException e) { System.out.println("ERROR: could not open " + p + ": " + e); // System.exit(1) ; } }
From source file:com.uber.hoodie.common.table.timeline.dto.FileStatusDTO.java
License:Apache License
/** * Used to safely handle FileStatus calls which might fail on some FileSystem implementation. * (DeprecatedLocalFileSystem)//w w w .j a v a 2s . co m */ private static void safeReadAndSetMetadata(FileStatusDTO dto, FileStatus fileStatus) { try { dto.owner = fileStatus.getOwner(); dto.group = fileStatus.getGroup(); dto.permission = FSPermissionDTO.fromFsPermission(fileStatus.getPermission()); } catch (IllegalArgumentException ie) { // Deprecated File System (testing) does not work well with this call // skipping } }
From source file:com.wandisco.s3hdfs.rewrite.filter.TestCurlCommands.java
License:Apache License
@Test public void testCurlCreateBucket1() throws IOException, URISyntaxException, S3ServiceException, InterruptedException { S3HdfsPath s3HdfsPath = testUtil.setUpS3HdfsPath("rewrite", null, "flavaflav"); ProcessBuilder pb = new ProcessBuilder("curl", "-v", "-L", "-X", "PUT", "http://" + hostName + ":" + PROXY_PORT + "/rewrite?user.name=flavaflav"); Process proc = pb.start();//from w ww. j a v a 2s .c om proc.waitFor(); String out = testUtil.readInputStream(proc.getInputStream()); String out2 = testUtil.readInputStream(proc.getErrorStream()); System.err.println(out); System.err.println(out2); FileStatus retVal = hdfs.listStatus(new Path(s3HdfsPath.getHdfsRootUserPath()))[0]; System.out.println(hdfs.listStatus(new Path(s3HdfsPath.getHdfsRootUserPath()))[0].getPath()); assertEquals("rewrite", retVal.getPath().getName()); assertEquals("flavaflav", retVal.getOwner()); assertTrue(retVal.isDirectory()); FileStatus[] inside = hdfs.listStatus(new Path(retVal.getPath().toString())); assertEquals(0, inside.length); }
From source file:com.wandisco.s3hdfs.rewrite.filter.TestCurlCommands.java
License:Apache License
@Test public void testCurlCreateBucket2() throws IOException, URISyntaxException, S3ServiceException, InterruptedException { S3HdfsPath s3HdfsPath = testUtil.setUpS3HdfsPath("rewrite", null, "flavaflav"); ProcessBuilder pb = new ProcessBuilder("curl", "-v", "-L", "-X", "PUT", "-H", "Host: rewrite." + hostName, "http://" + hostName + ":" + PROXY_PORT + "/?user.name=flavaflav"); Process proc = pb.start();/*from w w w . j a v a2s . com*/ proc.waitFor(); String out = testUtil.readInputStream(proc.getInputStream()); String out2 = testUtil.readInputStream(proc.getErrorStream()); System.out.println(out); System.out.println(out2); FileStatus retVal = hdfs.listStatus(new Path(s3HdfsPath.getHdfsRootUserPath()))[0]; System.out.println(hdfs.listStatus(new Path(s3HdfsPath.getHdfsRootUserPath()))[0].getPath()); assertEquals("rewrite", retVal.getPath().getName()); assertEquals("flavaflav", retVal.getOwner()); assertTrue(retVal.isDirectory()); FileStatus[] inside = hdfs.listStatus(new Path(retVal.getPath().toString())); assertEquals(0, inside.length); }
From source file:com.wandisco.s3hdfs.rewrite.filter.TestCurlCommands.java
License:Apache License
@Test public void testCurlGetNonExistantObject() throws IOException, URISyntaxException, S3ServiceException, InterruptedException { // WITHOUT BUCKET S3HdfsPath s3HdfsPath = testUtil.setUpS3HdfsPath("myBucket", "S3HDFS%2Fslot%2D01special%2Dtapestart", "flavaflav"); ProcessBuilder pb = new ProcessBuilder("curl", "-v", "-L", "-X", "GET", "http://" + hostName + ":" + PROXY_PORT + "/myBucket/" + s3HdfsPath.getObjectName() + "?user.name=flavaflav"); Process proc = pb.start();/* w w w .j av a 2 s . co m*/ proc.waitFor(); String out = testUtil.readInputStream(proc.getInputStream()); String out2 = testUtil.readInputStream(proc.getErrorStream()); System.out.println(out); System.out.println(out2); assert out2.contains("HTTP/1.1 404 Not Found"); // MAKE BUCKET ProcessBuilder pb2 = new ProcessBuilder("curl", "-v", "-L", "-X", "PUT", "-H", "Host: myBucket." + hostName, "http://" + hostName + ":" + PROXY_PORT + "/?user.name=flavaflav"); Process proc2 = pb2.start(); proc2.waitFor(); out = testUtil.readInputStream(proc2.getInputStream()); out2 = testUtil.readInputStream(proc2.getErrorStream()); System.out.println(out); System.out.println(out2); FileStatus retVal = hdfs.listStatus(new Path(s3HdfsPath.getHdfsRootUserPath()))[0]; System.out.println(hdfs.listStatus(new Path(s3HdfsPath.getHdfsRootUserPath()))[0].getPath()); assertEquals("myBucket", retVal.getPath().getName()); assertEquals("flavaflav", retVal.getOwner()); assertTrue(retVal.isDirectory()); FileStatus[] inside = hdfs.listStatus(new Path(retVal.getPath().toString())); assertEquals(0, inside.length); //WITH BUCKET ProcessBuilder pb3 = new ProcessBuilder("curl", "-v", "-L", "-X", "GET", "http://" + hostName + ":" + PROXY_PORT + "/myBucket/" + s3HdfsPath.getObjectName() + "?user.name=flavaflav"); Process proc3 = pb3.start(); proc3.waitFor(); out = testUtil.readInputStream(proc3.getInputStream()); out2 = testUtil.readInputStream(proc3.getErrorStream()); System.out.println("LAST: " + out); System.out.println("LAST: " + out2); assert out2.contains("HTTP/1.1 404 Not Found"); }
From source file:crunch.MaxTemperature.java
License:Apache License
@Test
public void fileStatusForFile() throws IOException {
Path file = new Path("/dir/file"); // XXX new Path creates the file
FileStatus stat = fs.getFileStatus(file);
assertThat(stat.getPath().toUri().getPath(), is("/dir/file")); // XXX FileStatus.getPath().toUri() -> URI .getPath()
assertThat(stat.isDir(), is(false)); // XXX assertThat(actual, Matcher) Matcher provides matches method
assertThat(stat.getLen(), is(7L));
assertThat(stat.getModificationTime(), is(lessThanOrEqualTo(System.currentTimeMillis())));
assertThat(stat.getReplication(), is((short) 1)); // XXX Matcher<Short> is(Short value) -> o.h.core.Is.<Short>is(Short) static factory method from oh.core.Is
// XXX which calls the constructor is(Matcher<Short> matcher) with the matcher equalTo(Short) hamcrest-all-1.3-source.java:Is.java:65
assertThat(stat.getBlockSize(), is(64 * 1024 * 1024L));
assertThat(stat.getOwner(), is(System.getProperty("user.name")));
assertThat(stat.getGroup(), is("supergroup"));
assertThat(stat.getPermission().toString(), is("rw-r--r--"));
}/*w w w . j av a 2 s . co m*/
From source file:crunch.MaxTemperature.java
License:Apache License
@Test
public void fileStatusForDirectory() throws IOException {
Path dir = new Path("/dir"); // XXX new Path creates the directory
FileStatus stat = fs.getFileStatus(dir);
assertThat(stat.getPath().toUri().getPath(), is("/dir"));
assertThat(stat.isDir(), is(true));
assertThat(stat.getLen(), is(0L));
assertThat(stat.getModificationTime(), is(lessThanOrEqualTo(System.currentTimeMillis())));
assertThat(stat.getReplication(), is((short) 0));
assertThat(stat.getBlockSize(), is(0L));
assertThat(stat.getOwner(), is(System.getProperty("user.name")));
assertThat(stat.getGroup(), is("supergroup"));
assertThat(stat.getPermission().toString(), is("rwxr-xr-x"));
}//from w ww. ja v a 2 s. c o m