List of usage examples for org.apache.hadoop.fs BlockLocation BlockLocation
public BlockLocation()
From source file:com.facebook.presto.hive.TestBackgroundHiveSplitLoader.java
License:Apache License
private static LocatedFileStatus locatedFileStatus(Path path) { return new LocatedFileStatus(0L, false, 0, 0L, 0L, 0L, null, null, null, null, path, new BlockLocation[] { new BlockLocation() }); }
From source file:com.facebook.presto.hive.util.InternalHiveSplitFactory.java
License:Apache License
private Optional<InternalHiveSplit> createInternalHiveSplit(Path path, BlockLocation[] blockLocations, long start, long length, long fileSize, OptionalInt bucketNumber, boolean splittable) { String pathString = path.toString(); if (!pathMatchesPredicate(pathDomain, pathString)) { return Optional.empty(); }/* w w w .j a v a2s . c om*/ boolean forceLocalScheduling = this.forceLocalScheduling; // For empty files, some filesystem (e.g. LocalFileSystem) produce one empty block // while others (e.g. hdfs.DistributedFileSystem) produces no block. // Synthesize an empty block if one does not already exist. if (fileSize == 0 && blockLocations.length == 0) { blockLocations = new BlockLocation[] { new BlockLocation() }; // Turn off force local scheduling because hosts list doesn't exist. forceLocalScheduling = false; } ImmutableList.Builder<InternalHiveBlock> blockBuilder = ImmutableList.builder(); for (BlockLocation blockLocation : blockLocations) { // clamp the block range long blockStart = Math.max(start, blockLocation.getOffset()); long blockEnd = Math.min(start + length, blockLocation.getOffset() + blockLocation.getLength()); if (blockStart > blockEnd) { // block is outside split range continue; } if (blockStart == blockEnd && !(blockStart == start && blockEnd == start + length)) { // skip zero-width block, except in the special circumstance: slice is empty, and the block covers the empty slice interval. continue; } blockBuilder.add(new InternalHiveBlock(blockStart, blockEnd, getHostAddresses(blockLocation))); } List<InternalHiveBlock> blocks = blockBuilder.build(); checkBlocks(blocks, start, length); if (!splittable) { // not splittable, use the hosts from the first block if it exists blocks = ImmutableList.of(new InternalHiveBlock(start, start + length, blocks.get(0).getAddresses())); } return Optional.of(new InternalHiveSplit(partitionName, pathString, start, start + length, fileSize, schema, partitionKeys, blocks, bucketNumber, splittable, forceLocalScheduling && allBlocksHaveRealAddress(blocks), columnCoercions, bucketConversion, s3SelectPushdownEnabled && S3SelectPushdown.isCompressionCodecSupported(inputFormat, path))); }
From source file:com.ibm.crail.hdfs.CrailHadoopFileSystem.java
License:Apache License
@Override public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { try {// ww w . ja va 2s . co m CrailBlockLocation[] _locations = dfs.lookup(file.getPath().toUri().getRawPath()).get().asFile() .getBlockLocations(start, len); BlockLocation[] locations = new BlockLocation[_locations.length]; for (int i = 0; i < locations.length; i++) { locations[i] = new BlockLocation(); locations[i].setOffset(_locations[i].getOffset()); locations[i].setLength(_locations[i].getLength()); locations[i].setNames(_locations[i].getNames()); locations[i].setHosts(_locations[i].getHosts()); locations[i].setTopologyPaths(_locations[i].getTopology()); } return locations; } catch (Exception e) { throw new IOException(e); } }
From source file:com.ibm.crail.hdfs.CrailHadoopFileSystem.java
License:Apache License
@Override public BlockLocation[] getFileBlockLocations(Path path, long start, long len) throws IOException { try {//w w w.j av a 2 s . c om CrailBlockLocation[] _locations = dfs.lookup(path.toUri().getRawPath()).get().asFile() .getBlockLocations(start, len); BlockLocation[] locations = new BlockLocation[_locations.length]; for (int i = 0; i < locations.length; i++) { locations[i] = new BlockLocation(); locations[i].setOffset(_locations[i].getOffset()); locations[i].setLength(_locations[i].getLength()); locations[i].setNames(_locations[i].getNames()); locations[i].setHosts(_locations[i].getHosts()); locations[i].setTopologyPaths(_locations[i].getTopology()); } return locations; } catch (Exception e) { throw new IOException(e); } }
From source file:com.ibm.crail.hdfs.CrailHDFS.java
License:Apache License
@Override public BlockLocation[] getFileBlockLocations(Path path, long start, long len) throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException { try {/* ww w . j a va 2s . c o m*/ CrailBlockLocation[] _locations = dfs.lookup(path.toUri().getRawPath()).get().asFile() .getBlockLocations(start, len); BlockLocation[] locations = new BlockLocation[_locations.length]; for (int i = 0; i < locations.length; i++) { locations[i] = new BlockLocation(); locations[i].setOffset(_locations[i].getOffset()); locations[i].setLength(_locations[i].getLength()); locations[i].setNames(_locations[i].getNames()); locations[i].setHosts(_locations[i].getHosts()); locations[i].setTopologyPaths(_locations[i].getTopology()); } return locations; } catch (Exception e) { throw new IOException(e); } }
From source file:hsyndicate.hadoop.dfs.HSyndicateDFS.java
License:Apache License
@Override public synchronized BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) { HSyndicateUGMonitor monitor = null;//from w ww . j a v a 2s .c o m try { monitor = new HSyndicateUGMonitor(this.getConf()); SyndicateFSPath hpath = makeSyndicateFSPath(file.getPath()); long filesize = file.getLen(); long pblocksize = this.syndicateFS.getBlockSize(hpath); long lblocksize = getDefaultBlockSize(); int groupof = 64; int pstartblockID = BlockUtils.getBlockID(start, pblocksize); int pendblockID = BlockUtils.getBlockID(start + len, pblocksize); int peffectiveblocklen = pendblockID - pstartblockID + 1; int lstartblockID = BlockUtils.getBlockID(start, lblocksize); int lendblockID = BlockUtils.getBlockID(start + len, lblocksize); int leffectiveblocklen = lendblockID - lstartblockID + 1; BlockLocation[] locations = new BlockLocation[leffectiveblocklen]; List<HSyndicateUGMonitorResults<byte[]>> localCachedBlockInfo = monitor.getLocalCachedBlockInfo(hpath); for (int i = 0; i < leffectiveblocklen; i++) { locations[i] = new BlockLocation(); locations[i].setOffset(BlockUtils.getBlockStartOffset(lstartblockID + i, lblocksize)); locations[i].setLength(BlockUtils.getBlockLength(filesize, lblocksize, lstartblockID + i)); List<String> gateway_hosts = new ArrayList<String>(); List<String> gateway_names = new ArrayList<String>(); List<String> gateway_topology = new ArrayList<String>(); int[] lcachedcnt = new int[localCachedBlockInfo.size()]; for (int k = 0; k < lcachedcnt.length; k++) { lcachedcnt[k] = 0; } for (int k = 0; k < lcachedcnt.length; k++) { HSyndicateUGMonitorResults<byte[]> info = localCachedBlockInfo.get(k); if (info.getResult() != null) { for (int j = 0; j < groupof; j++) { int pcurblockID = pstartblockID + (i * groupof) + j; if (pcurblockID >= peffectiveblocklen) { break; } boolean hasCache = BlockUtils.checkBlockPresence(pcurblockID, info.getResult()); if (hasCache) { lcachedcnt[k]++; } } } } int maxCnt = 0; for (int k = 0; k < lcachedcnt.length; k++) { if (maxCnt < lcachedcnt[k]) { maxCnt = lcachedcnt[k]; } } if (maxCnt > 0) { for (int k = 0; k < lcachedcnt.length; k++) { if (lcachedcnt[k] == maxCnt) { HSyndicateUGMonitorResults<byte[]> info = localCachedBlockInfo.get(k); gateway_names.add(info.getHostname()); } } } if (gateway_names.isEmpty()) { gateway_names.addAll(monitor.getUserGatewayHosts()); } for (String name : gateway_names) { gateway_hosts.add(IPUtils.parseHost(name)); gateway_topology.add("/default-rack/" + name); } locations[i].setHosts(gateway_hosts.toArray(new String[0])); locations[i].setNames(gateway_names.toArray(new String[0])); locations[i].setTopologyPaths(gateway_topology.toArray(new String[0])); } monitor.close(); return locations; } catch (Exception ex) { LOG.info(ex); if (monitor != null) { try { monitor.close(); } catch (IOException ex1) { } } } return null; }