List of usage examples for org.apache.hadoop.fs FileStatus getReplication
public short getReplication()
From source file:io.apigee.lembos.node.types.DistributedCacheWrap.java
License:Apache License
/** * Java wrapper for {@link DistributedCache#getFileStatus(Configuration, URI)}. * * @param ctx the JavaScript context//from w w w . j a va 2s .c o m * @param thisObj the 'this' object * @param args the function arguments * @param func the function being called * * @return array of archive class paths */ @JSStaticFunction public static Object getFileStatus(final Context ctx, final Scriptable thisObj, final Object[] args, final Function func) { final Object arg0 = args.length >= 1 ? args[0] : Undefined.instance; final Object arg1 = args.length >= 2 ? args[1] : Undefined.instance; if (args.length < 2) { throw Utils.makeError(ctx, thisObj, LembosMessages.TWO_ARGS_EXPECTED); } else if (!JavaScriptUtils.isDefined(arg0)) { throw Utils.makeError(ctx, thisObj, LembosMessages.FIRST_ARG_REQUIRED); } else if (!JavaScriptUtils.isDefined(arg1)) { throw Utils.makeError(ctx, thisObj, LembosMessages.SECOND_ARG_REQUIRED); } else if (!(arg0 instanceof ConfigurationWrap)) { throw Utils.makeError(ctx, thisObj, LembosMessages.FIRST_ARG_MUST_BE_CONF); } final URI hdfsUri = URI.create(arg1.toString()); FileStatus status; try { status = DistributedCache.getFileStatus(((ConfigurationWrap) arg0).getConf(), hdfsUri); } catch (IOException e) { throw Utils.makeError(ctx, thisObj, e.getMessage()); } if (status == null) { throw Utils.makeError(ctx, thisObj, "Unable to get file status for HDFS uri: " + hdfsUri.toString()); } final Scriptable jsStatus = ctx.newObject(thisObj); ScriptableObject.defineProperty(jsStatus, "accessTime", status.getAccessTime(), ScriptableObject.READONLY); ScriptableObject.defineProperty(jsStatus, "blockSize", status.getBlockSize(), ScriptableObject.READONLY); ScriptableObject.defineProperty(jsStatus, "group", status.getGroup(), ScriptableObject.READONLY); ScriptableObject.defineProperty(jsStatus, "len", status.getLen(), ScriptableObject.READONLY); ScriptableObject.defineProperty(jsStatus, "modificationTime", status.getModificationTime(), ScriptableObject.READONLY); ScriptableObject.defineProperty(jsStatus, "owner", status.getOwner(), ScriptableObject.READONLY); ScriptableObject.defineProperty(jsStatus, "path", status.getPath().toString(), ScriptableObject.READONLY); ScriptableObject.defineProperty(jsStatus, "permission", status.getPermission().toString(), ScriptableObject.READONLY); ScriptableObject.defineProperty(jsStatus, "replication", status.getReplication(), ScriptableObject.READONLY); return jsStatus; }
From source file:io.hops.erasure_coding.BaseEncodingManager.java
License:Apache License
/** * RAID an individual file//from w w w. ja v a 2s.c o m */ public static boolean doFileRaid(Configuration conf, Path sourceFile, Path parityPath, Codec codec, Statistics statistics, Progressable reporter, int targetRepl, int metaRepl) throws IOException { FileSystem srcFs = sourceFile.getFileSystem(conf); FileStatus sourceStatus = srcFs.getFileStatus(sourceFile); // extract block locations from File system BlockLocation[] locations = srcFs.getFileBlockLocations(sourceFile, 0, sourceStatus.getLen()); // if the file has fewer than 2 blocks, then nothing to do if (locations.length <= 2) { return false; } // add up the raw disk space occupied by this file long diskSpace = 0; for (BlockLocation l : locations) { diskSpace += (l.getLength() * sourceStatus.getReplication()); } statistics.numProcessedBlocks += locations.length; statistics.processedSize += diskSpace; // generate parity file generateParityFile(conf, sourceStatus, targetRepl, reporter, srcFs, parityPath, codec, locations.length, sourceStatus.getReplication(), metaRepl, sourceStatus.getBlockSize()); if (srcFs.setReplication(sourceFile, (short) targetRepl) == false) { LOG.info("Error in reducing replication of " + sourceFile + " to " + targetRepl); statistics.remainingSize += diskSpace; return false; } ; diskSpace = 0; for (BlockLocation l : locations) { diskSpace += (l.getLength() * targetRepl); } statistics.remainingSize += diskSpace; // the metafile will have this many number of blocks int numMeta = locations.length / codec.stripeLength; if (locations.length % codec.stripeLength != 0) { numMeta++; } // we create numMeta for every file. This metablock has metaRepl # replicas. // the last block of the metafile might not be completely filled up, but we // ignore that for now. statistics.numMetaBlocks += (numMeta * metaRepl); statistics.metaSize += (numMeta * metaRepl * sourceStatus.getBlockSize()); return true; }
From source file:org.apache.ambari.view.filebrowser.HdfsApi.java
License:Apache License
/** * Converts a Hadoop <code>FileStatus</code> object into a JSON array object. * It replaces the <code>SCHEME://HOST:PORT</code> of the path with the * specified URL.// www . j av a 2 s .com * <p/> * * @param status * Hadoop file status. * @return The JSON representation of the file status. */ public Map<String, Object> fileStatusToJSON(FileStatus status) { Map<String, Object> json = new LinkedHashMap<String, Object>(); json.put("path", Path.getPathWithoutSchemeAndAuthority(status.getPath()).toString()); json.put("replication", status.getReplication()); json.put("isDirectory", status.isDirectory()); json.put("len", status.getLen()); json.put("owner", status.getOwner()); json.put("group", status.getGroup()); json.put("permission", permissionToString(status.getPermission())); json.put("accessTime", status.getAccessTime()); json.put("modificationTime", status.getModificationTime()); json.put("blockSize", status.getBlockSize()); json.put("replication", status.getReplication()); json.put("readAccess", checkAccessPermissions(status, FsAction.READ, ugi)); json.put("writeAccess", checkAccessPermissions(status, FsAction.WRITE, ugi)); json.put("executeAccess", checkAccessPermissions(status, FsAction.EXECUTE, ugi)); return json; }
From source file:org.apache.ambari.view.hive.utils.HdfsApi.java
License:Apache License
/** * Converts a Hadoop <code>FileStatus</code> object into a JSON array object. * It replaces the <code>SCHEME://HOST:PORT</code> of the path with the * specified URL./*from w w w . ja v a 2 s. c om*/ * <p/> * * @param status * Hadoop file status. * @return The JSON representation of the file status. */ public static Map<String, Object> fileStatusToJSON(FileStatus status) { Map<String, Object> json = new LinkedHashMap<String, Object>(); json.put("path", status.getPath().toString()); json.put("isDirectory", status.isDirectory()); json.put("len", status.getLen()); json.put("owner", status.getOwner()); json.put("group", status.getGroup()); json.put("permission", permissionToString(status.getPermission())); json.put("accessTime", status.getAccessTime()); json.put("modificationTime", status.getModificationTime()); json.put("blockSize", status.getBlockSize()); json.put("replication", status.getReplication()); return json; }
From source file:org.apache.ambari.view.utils.hdfs.HdfsApi.java
License:Apache License
/** * Converts a Hadoop <code>FileStatus</code> object into a JSON array object. * It replaces the <code>SCHEME://HOST:PORT</code> of the path with the * specified URL.//from w w w.j a v a2s .co m * <p/> * * @param status * Hadoop file status. * @return The JSON representation of the file status. */ public Map<String, Object> fileStatusToJSON(FileStatus status) { Map<String, Object> json = new LinkedHashMap<String, Object>(); json.put("path", Path.getPathWithoutSchemeAndAuthority(status.getPath()).toString()); json.put("replication", status.getReplication()); json.put("isDirectory", status.isDirectory()); json.put("len", status.getLen()); json.put("owner", status.getOwner()); json.put("group", status.getGroup()); json.put("permission", permissionToString(status.getPermission())); json.put("accessTime", status.getAccessTime()); json.put("modificationTime", status.getModificationTime()); json.put("blockSize", status.getBlockSize()); json.put("replication", status.getReplication()); json.put("readAccess", checkAccessPermissions(status, FsAction.READ, ugi)); json.put("writeAccess", checkAccessPermissions(status, FsAction.WRITE, ugi)); json.put("executeAccess", checkAccessPermissions(status, FsAction.EXECUTE, ugi)); return json; }
From source file:org.apache.distributedlog.fs.TestDLFileSystem.java
License:Apache License
@Test public void testListStatuses() throws Exception { Path parentPath = new Path("/path/to/" + runtime.getMethodName()); assertFalse(fs.exists(parentPath));/*w ww . j av a 2 s . c om*/ try (FSDataOutputStream parentOut = fs.create(parentPath)) { parentOut.writeBytes("parent"); parentOut.flush(); } assertTrue(fs.exists(parentPath)); int numLogs = 3; for (int i = 0; i < numLogs; i++) { Path path = new Path("/path/to/" + runtime.getMethodName() + "/" + runtime.getMethodName() + "-" + i); assertFalse(fs.exists(path)); try (FSDataOutputStream out = fs.create(path)) { out.writeBytes("line"); out.flush(); } assertTrue(fs.exists(path)); } FileStatus[] files = fs.listStatus(new Path("/path/to/" + runtime.getMethodName())); assertEquals(3, files.length); for (int i = 0; i < numLogs; i++) { FileStatus file = files[i]; assertEquals(4, file.getLen()); assertFalse(file.isDirectory()); assertEquals(3, file.getReplication()); assertEquals(0L, file.getModificationTime()); assertEquals(new Path("/path/to/" + runtime.getMethodName() + "/" + runtime.getMethodName() + "-" + i), file.getPath()); } }
From source file:org.apache.falcon.hadoop.JailedFileSystem.java
License:Apache License
@Override public FileStatus[] listStatus(Path f) throws IOException { FileStatus[] fileStatuses = localFS.listStatus(toLocalPath(f)); if (fileStatuses == null || fileStatuses.length == 0) { return fileStatuses; } else {//ww w . j av a 2s . c o m FileStatus[] jailFileStatuses = new FileStatus[fileStatuses.length]; for (int index = 0; index < fileStatuses.length; index++) { FileStatus status = fileStatuses[index]; jailFileStatuses[index] = new FileStatus(status.getLen(), status.isDirectory(), status.getReplication(), status.getBlockSize(), status.getModificationTime(), status.getAccessTime(), status.getPermission(), status.getOwner(), status.getGroup(), fromLocalPath(status.getPath()).makeQualified(this.getUri(), this.getWorkingDirectory())); } return jailFileStatuses; } }
From source file:org.apache.falcon.hadoop.JailedFileSystem.java
License:Apache License
@Override public FileStatus getFileStatus(Path f) throws IOException { FileStatus status = localFS.getFileStatus(toLocalPath(f)); if (status == null) { return null; }/*from www.j a va 2s.co m*/ return new FileStatus(status.getLen(), status.isDirectory(), status.getReplication(), status.getBlockSize(), status.getModificationTime(), status.getAccessTime(), status.getPermission(), status.getOwner(), status.getGroup(), fromLocalPath(status.getPath()).makeQualified(this.getUri(), this.getWorkingDirectory())); }
From source file:org.apache.gobblin.data.management.copy.writer.FileAwareInputStreamDataWriter.java
License:Apache License
/** * Write the contents of input stream into staging path. * * <p>// w ww. j a va 2s . co m * WriteAt indicates the path where the contents of the input stream should be written. When this method is called, * the path writeAt.getParent() will exist already, but the path writeAt will not exist. When this method is returned, * the path writeAt must exist. Any data written to any location other than writeAt or a descendant of writeAt * will be ignored. * </p> * * @param inputStream {@link FSDataInputStream} whose contents should be written to staging path. * @param writeAt {@link Path} at which contents should be written. * @param copyableFile {@link org.apache.gobblin.data.management.copy.CopyEntity} that generated this copy operation. * @param record The actual {@link FileAwareInputStream} passed to the write method. * @throws IOException */ protected void writeImpl(InputStream inputStream, Path writeAt, CopyableFile copyableFile, FileAwareInputStream record) throws IOException { final short replication = this.state.getPropAsShort(ConfigurationKeys.WRITER_FILE_REPLICATION_FACTOR, copyableFile.getReplication(this.fs)); final long blockSize = copyableFile.getBlockSize(this.fs); final long fileSize = copyableFile.getFileStatus().getLen(); long expectedBytes = fileSize; Long maxBytes = null; // Whether writer must write EXACTLY maxBytes. boolean mustMatchMaxBytes = false; if (record.getSplit().isPresent()) { maxBytes = record.getSplit().get().getHighPosition() - record.getSplit().get().getLowPosition(); if (record.getSplit().get().isLastSplit()) { expectedBytes = fileSize % blockSize; mustMatchMaxBytes = false; } else { expectedBytes = maxBytes; mustMatchMaxBytes = true; } } Predicate<FileStatus> fileStatusAttributesFilter = new Predicate<FileStatus>() { @Override public boolean apply(FileStatus input) { return input.getReplication() == replication && input.getBlockSize() == blockSize; } }; Optional<FileStatus> persistedFile = this.recoveryHelper.findPersistedFile(this.state, copyableFile, fileStatusAttributesFilter); if (persistedFile.isPresent()) { log.info(String.format("Recovering persisted file %s to %s.", persistedFile.get().getPath(), writeAt)); this.fs.rename(persistedFile.get().getPath(), writeAt); } else { // Copy empty directories if (copyableFile.getFileStatus().isDirectory()) { this.fs.mkdirs(writeAt); return; } OutputStream os = this.fs.create(writeAt, true, this.fs.getConf().getInt("io.file.buffer.size", 4096), replication, blockSize); if (encryptionConfig != null) { os = EncryptionFactory.buildStreamCryptoProvider(encryptionConfig).encodeOutputStream(os); } try { FileSystem defaultFS = FileSystem.get(new Configuration()); StreamThrottler<GobblinScopeTypes> throttler = this.taskBroker .getSharedResource(new StreamThrottler.Factory<GobblinScopeTypes>(), new EmptyKey()); ThrottledInputStream throttledInputStream = throttler.throttleInputStream().inputStream(inputStream) .sourceURI(copyableFile.getOrigin().getPath() .makeQualified(defaultFS.getUri(), defaultFS.getWorkingDirectory()).toUri()) .targetURI(this.fs.makeQualified(writeAt).toUri()).build(); StreamCopier copier = new StreamCopier(throttledInputStream, os, maxBytes) .withBufferSize(this.bufferSize); log.info("File {}: Starting copy", copyableFile.getOrigin().getPath()); if (isInstrumentationEnabled()) { copier.withCopySpeedMeter(this.copySpeedMeter); } long numBytes = copier.copy(); if ((this.checkFileSize || mustMatchMaxBytes) && numBytes != expectedBytes) { throw new IOException(String.format("Incomplete write: expected %d, wrote %d bytes.", expectedBytes, numBytes)); } this.bytesWritten.addAndGet(numBytes); if (isInstrumentationEnabled()) { log.info("File {}: copied {} bytes, average rate: {} B/s", copyableFile.getOrigin().getPath(), this.copySpeedMeter.getCount(), this.copySpeedMeter.getMeanRate()); } else { log.info("File {} copied.", copyableFile.getOrigin().getPath()); } } catch (NotConfiguredException nce) { log.warn("Broker error. Some features of stream copier may not be available.", nce); } finally { os.close(); inputStream.close(); } } }
From source file:org.apache.hdt.dfs.core.DFSFile.java
License:Apache License
public DFSFile(DFSPath parent, Path path) { super(parent, path); try {// ww w . ja v a 2 s . c o m FileStatus fs = getDFS().getFileStatus(path); this.length = fs.getLen(); this.replication = fs.getReplication(); } catch (IOException e) { e.printStackTrace(); } }