List of usage examples for org.apache.hadoop.hdfs.protocol HdfsFileStatus getLen
long getLen();
From source file:com.bigstep.datalake.DLFileSystem.java
License:Apache License
private FileStatus makeQualified(HdfsFileStatus f, Path parent) { return new FileStatus(f.getLen(), f.isDir(), f.getReplication(), f.getBlockSize(), f.getModificationTime(), f.getAccessTime(), f.getPermission(), f.getOwner(), f.getGroup(), f.isSymlink() ? new Path(f.getSymlink()) : null, makeQualified(f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()))); }
From source file:com.bigstep.datalake.JsonUtil.java
License:Apache License
/** * Convert a HdfsFileStatus object to a Json string. * @param status input status//from w w w .j a v a 2 s .com * @param includeType type to use * @return the json */ public static String toJsonString(final HdfsFileStatus status, boolean includeType) { if (status == null) { return null; } final Map<String, Object> m = new TreeMap<String, Object>(); m.put("pathSuffix", status.getLocalName()); m.put("type", PathType.valueOf(status)); if (status.isSymlink()) { m.put("symlink", status.getSymlink()); } m.put("length", status.getLen()); m.put("owner", status.getOwner()); m.put("group", status.getGroup()); FsPermission perm = status.getPermission(); m.put("permission", toString(perm)); if (perm.getAclBit()) { m.put("aclBit", true); } if (perm.getEncryptedBit()) { m.put("encBit", true); } m.put("accessTime", status.getAccessTime()); m.put("modificationTime", status.getModificationTime()); m.put("blockSize", status.getBlockSize()); m.put("replication", status.getReplication()); m.put("fileId", status.getFileId()); m.put("childrenNum", status.getChildrenNum()); m.put("storagePolicy", status.getStoragePolicy()); Gson gson = new Gson(); return includeType ? toJsonString(FileStatus.class, m) : gson.toJson(m); }
From source file:com.mellanox.r4h.DFSOutputStream.java
License:Apache License
/** Construct a new output stream for append. */ private DFSOutputStream(DFSClient dfsClient, String src, EnumSet<CreateFlag> flags, Progressable progress, LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum) throws IOException { this(dfsClient, src, progress, stat, checksum); initialFileSize = stat.getLen(); // length of file when opened this.shouldSyncBlock = flags.contains(CreateFlag.SYNC_BLOCK); boolean toNewBlock = flags.contains(CreateFlag.NEW_BLOCK); // The last partial block of the file has to be filled. if (!toNewBlock && lastBlock != null) { // indicate that we are appending to an existing block bytesCurBlock = lastBlock.getBlockSize(); streamer = new DataStreamer(lastBlock, stat, bytesPerChecksum); } else {/* w w w . j a v a2 s . c o m*/ computePacketChunkSize(dfsClient.getConf().getWritePacketSize(), bytesPerChecksum); streamer = new DataStreamer(stat, lastBlock != null ? lastBlock.getBlock() : null); } this.fileEncryptionInfo = stat.getFileEncryptionInfo(); }
From source file:com.pinterest.terrapin.controller.ControllerUtil.java
License:Apache License
/** * Builds the helix ideal state for HDFS directory by finding the locations of HDFS blocks and * creating an ideal state assignment based on those. * * @param hdfsClient The HDFS client object. * @param hdfsDir The HDFS directory containing the various files. * @param resourceName The name of the Helix resource for which the ideal state is being created. * @param partitioner The partitioner type, used for extracting helix partition names from * HDFS files./*from ww w . jav a 2 s . co m*/ * @param numReplicas The number of replicas for each partition. * @param enableZkCompression Whether data in zk is kept compressed. * @return The ideal state as computed based on HDFS block placement. * @throws ControllerException */ public static IdealState buildIdealStateForHdfsDir(DFSClient hdfsClient, String hdfsDir, String resourceName, PartitionerType partitioner, int numReplicas, boolean enableZkCompression) throws ControllerException { List<HdfsFileStatus> fileList; try { fileList = TerrapinUtil.getHdfsFileList(hdfsClient, hdfsDir); } catch (IOException e) { throw new ControllerException("Exception while listing files in " + hdfsDir, ControllerErrorCode.HDFS_ERROR); } // Mapping from file to HDFS block locations. Map<Integer, Set<String>> hdfsBlockMapping = Maps.newHashMapWithExpectedSize(fileList.size()); for (HdfsFileStatus fileStatus : fileList) { Integer partitionName = TerrapinUtil.extractPartitionName(fileStatus.getLocalName(), partitioner); if (partitionName == null) { LOG.info("Skipping " + fileStatus.getLocalName() + " for " + hdfsDir); continue; } String fullName = fileStatus.getFullName(hdfsDir); BlockLocation[] locations = null; try { locations = hdfsClient.getBlockLocations(fullName, 0, fileStatus.getLen()); } catch (Exception e) { throw new ControllerException("Exception while getting block locations " + e.getMessage(), ControllerErrorCode.HDFS_ERROR); } Set<String> instanceSet = Sets.newHashSetWithExpectedSize(3); BlockLocation firstLocation = locations[0]; String[] hosts = null; try { hosts = firstLocation.getHosts(); } catch (IOException e) { throw new ControllerException("Exception while getting hosts " + e.getMessage(), ControllerErrorCode.HDFS_ERROR); } for (String host : hosts) { instanceSet.add(host); } hdfsBlockMapping.put(partitionName, instanceSet); } // Assign helix partitions for the resource - which is the HDFS directory. int bucketSize = TerrapinUtil.getBucketSize(hdfsBlockMapping.size(), enableZkCompression); CustomModeISBuilder idealStateBuilder = new CustomModeISBuilder(resourceName); for (Map.Entry<Integer, Set<String>> mapping : hdfsBlockMapping.entrySet()) { // Make partitions globally unique String partitionName = null; // This is needed because of the way helix parses partition numbers for buckets. if (bucketSize > 0) { partitionName = resourceName + "_" + mapping.getKey(); } else { partitionName = resourceName + "$" + mapping.getKey(); } Set<String> instanceSet = mapping.getValue(); for (String instance : instanceSet) { idealStateBuilder.assignInstanceAndState(partitionName, TerrapinUtil.getHelixInstanceFromHDFSHost(instance), "ONLINE"); } } idealStateBuilder.setStateModel("OnlineOffline"); idealStateBuilder.setNumReplica(numReplicas); idealStateBuilder.setNumPartitions(hdfsBlockMapping.size()); IdealState is = idealStateBuilder.build(); if (bucketSize > 0) { is.setBucketSize(bucketSize); } is.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED); if (enableZkCompression) { TerrapinUtil.compressIdealState(is); } return is; }
From source file:com.pinterest.terrapin.controller.HdfsManagerTest.java
License:Apache License
private HdfsFileStatus buildHdfsStatus(String path) { HdfsFileStatus status = PowerMockito.mock(HdfsFileStatus.class); when(status.getLocalName()).thenReturn(new Path(path).getName()); when(status.getFullName(any(String.class))).thenReturn(path); when(status.getLen()).thenReturn(1000L); return status; }
From source file:com.pinterest.terrapin.hadoop.HdfsUploader.java
License:Apache License
@Override
List<Pair<Path, Long>> getFileList() {
List<Pair<Path, Long>> fileSizePairList = Lists.newArrayList();
try {//from w ww . j av a 2 s . co m
List<HdfsFileStatus> fileStatusList = TerrapinUtil.getHdfsFileList(dfsClient, hdfsDir.toString());
for (HdfsFileStatus fileStatus : fileStatusList) {
fileSizePairList.add(new ImmutablePair(fileStatus.getFullPath(hdfsDir), fileStatus.getLen()));
}
} catch (IOException e) {
throw new RuntimeException(e);
}
return fileSizePairList;
}