List of usage examples for org.apache.hadoop.fs BlockLocation BlockLocation
public BlockLocation(String[] names, String[] hosts, long offset, long length)
From source file:RawParascaleFileSystem.java
License:Apache License
/** * {@inheritDoc}/*from ww w . ja v a 2 s. co m*/ */ @Override public BlockLocation[] getFileBlockLocations(final FileStatus file, final long start, final long len) throws IOException { ChunkLocator newChunkLocator = null; if (file.getLen() < start + len) { throw new IOException("start+len must be less or equal than file length"); } final ArrayList<BlockLocation> locations = new ArrayList<BlockLocation>(); try { newChunkLocator = newChunkLocator(); final Path makeQualified = file.getPath().makeQualified(this.getUri(), this.getWorkingDirectory()); // sorted by offset final ChunkLocation[] chunkLocations = newChunkLocator.getChunkLocations(pathToFile(makeQualified), getVirtualFSFromPath(makeQualified, true)); long begin = start; long length = len; for (final ChunkLocation chunkLocation : chunkLocations) { final ChunkInfo chunkInfo = chunkLocation.getChunkInfo(); final StorageNodeInfo[] storageNodeInfo = chunkLocation.getStorageNodeInfo(); if (length <= 0) { // stop when length exceeded break; } if (begin < chunkInfo.getChunkOffset()) { // skip if location not reached yet continue; } final List<String> hosts = new ArrayList<String>(0); for (int j = 0; j < storageNodeInfo.length; j++) { // select all enabled and running nodes if (storageNodeInfo[j].isUp() && storageNodeInfo[j].isEnabled()) { hosts.add(storageNodeInfo[j].getNodeName()); } } final long lengthInChunk = chunkInfo.getChunkLength() - (begin - chunkInfo.getChunkOffset()); final BlockLocation blockLocation = new BlockLocation(null, hosts.toArray(new String[0]), begin, lengthInChunk < length ? lengthInChunk : length); begin += blockLocation.getLength(); length -= blockLocation.getLength(); locations.add(blockLocation); } if (pLog.isDebugEnabled()) { pLog.debug("Fetched " + locations.size() + " chunk locations for " + makeQualified); } return locations.toArray(new BlockLocation[0]); } catch (final ChunkStorageException e) { throw new IOException( "can not fetch chunk locations " + newChunkLocator == null ? "" : newChunkLocator.toString(), e); } finally { if (newChunkLocator != null) { newChunkLocator.close(); } } }
From source file:alluxio.hadoop.AbstractFileSystem.java
License:Apache License
@Override public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { if (file == null) { return null; }// ww w . j a va 2 s. com if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI path = new AlluxioURI(HadoopUtils.getPathWithoutScheme(file.getPath())); List<FileBlockInfo> blocks = getFileBlocks(path); List<BlockLocation> blockLocations = new ArrayList<>(); for (FileBlockInfo fileBlockInfo : blocks) { long offset = fileBlockInfo.getOffset(); long end = offset + fileBlockInfo.getBlockInfo().getLength(); // Check if there is any overlapping between [start, start+len] and [offset, end] if (end >= start && offset <= start + len) { ArrayList<String> names = new ArrayList<>(); ArrayList<String> hosts = new ArrayList<>(); // add the existing in-memory block locations for (alluxio.wire.BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { HostAndPort address = HostAndPort.fromParts(location.getWorkerAddress().getHost(), location.getWorkerAddress().getDataPort()); names.add(address.toString()); hosts.add(address.getHostText()); } // add under file system locations for (String location : fileBlockInfo.getUfsLocations()) { names.add(location); hosts.add(HostAndPort.fromString(location).getHostText()); } blockLocations.add(new BlockLocation(CommonUtils.toStringArray(names), CommonUtils.toStringArray(hosts), offset, fileBlockInfo.getBlockInfo().getLength())); } } BlockLocation[] ret = new BlockLocation[blockLocations.size()]; blockLocations.toArray(ret); return ret; }
From source file:ch.cern.db.hdfs.DistributedFileSystemMetadataTest.java
License:GNU General Public License
@Test public void computeHostsDiskIdsCount() throws IOException { List<BlockLocation> blockStorageLocations = new LinkedList<>(); blockStorageLocations//from w w w. j a va2 s . c o m .add(new BlockStorageLocation(new BlockLocation(null, new String[] { "host1", "host2" }, 0, 0), new VolumeId[] { new TVolumeId("3"), new TVolumeId("4") })); blockStorageLocations .add(new BlockStorageLocation(new BlockLocation(null, new String[] { "host2", "host3" }, 0, 0), new VolumeId[] { new TVolumeId("4"), new TVolumeId("5") })); blockStorageLocations .add(new BlockStorageLocation(new BlockLocation(null, new String[] { "host10", "host2" }, 0, 0), new VolumeId[] { new TVolumeId("3"), new TVolumeId("4") })); blockStorageLocations .add(new BlockStorageLocation(new BlockLocation(null, new String[] { "host10", "host3" }, 0, 0), new VolumeId[] { new TVolumeId("8"), new TVolumeId("5") })); blockStorageLocations.add(new BlockLocation(null, new String[] { "host10", "host3", "host3" }, 0, 0)); HashMap<String, HashMap<Integer, Integer>> hosts_diskids = DistributedFileSystemMetadata .computeHostsDiskIdsCount(blockStorageLocations); Assert.assertEquals(1, hosts_diskids.get("host1").get(3).intValue()); Assert.assertEquals(3, hosts_diskids.get("host2").get(4).intValue()); Assert.assertEquals(2, hosts_diskids.get("host3").get(5).intValue()); Assert.assertEquals(2, hosts_diskids.get("host3").get(-1).intValue()); Assert.assertEquals(1, hosts_diskids.get("host10").get(3).intValue()); Assert.assertEquals(1, hosts_diskids.get("host10").get(8).intValue()); Assert.assertEquals(1, hosts_diskids.get("host10").get(-1).intValue()); }
From source file:com.pinterest.terrapin.controller.ControllerUtilTest.java
License:Apache License
public void testBuildIdealStateForHdfsDirHelper(boolean zkCompression, int numPartitions) throws Exception { String hdfsDir = Constants.HDFS_DATA_DIR + "/fileset"; DFSClient dfsClient = mock(DFSClient.class); // Create three hosts in the clusters. List<BlockLocation> locations = ImmutableList .of(new BlockLocation(new String[] { "host1", "host2" }, new String[] { "host1", "host2" }, 0, 0)); HdfsFileStatus[] fileStatuses = new HdfsFileStatus[numPartitions]; for (int i = 0; i < numPartitions; ++i) { fileStatuses[i] = PowerMockito.mock(HdfsFileStatus.class); String localName = TerrapinUtil.formatPartitionName(i); when(fileStatuses[i].getLocalName()).thenReturn(localName); when(fileStatuses[i].getFullName(eq(hdfsDir))).thenReturn(hdfsDir + "/" + localName); when(fileStatuses[i].getLen()).thenReturn(1000L); BlockLocation[] locationArray = new BlockLocation[1]; locations.subList(0, 1).toArray(locationArray); when(dfsClient.getBlockLocations(eq(fileStatuses[i].getFullName(hdfsDir)), anyLong(), anyLong())) .thenReturn(locationArray); }// w w w . j a v a2 s. c om when(dfsClient.listPaths(eq(hdfsDir), any(byte[].class))).thenReturn(new DirectoryListing(fileStatuses, 0)); IdealState is = ControllerUtil.buildIdealStateForHdfsDir(dfsClient, hdfsDir, "resource", PartitionerType.CASCADING, 2, zkCompression); assertEquals(numPartitions, is.getNumPartitions()); assertEquals("resource", is.getResourceName()); for (int i = 0; i < numPartitions; ++i) { String partition; if (numPartitions > 1000 && !zkCompression) { partition = "resource_" + i; } else { partition = "resource$" + i; } assertEquals(Sets.newHashSet("host1", "host2"), is.getInstanceSet(partition)); } assertEquals("OnlineOffline", is.getStateModelDefRef()); if (zkCompression) { assertTrue(is.getRecord().getBooleanField("enableCompression", false)); } assertEquals(IdealState.RebalanceMode.CUSTOMIZED, is.getRebalanceMode()); }
From source file:com.pinterest.terrapin.controller.HdfsManagerTest.java
License:Apache License
private void setupBlockLocations(String resource, Map<Integer, List<String>> locationMap) throws IOException { for (Map.Entry<Integer, List<String>> entry : locationMap.entrySet()) { String path = TerrapinUtil.helixResourceToHdfsDir(resource) + String.format("/%s", TerrapinUtil.formatPartitionName(entry.getKey())); BlockLocation[] locations = new BlockLocation[1]; String[] hostsArray = new String[entry.getValue().size()]; entry.getValue().toArray(hostsArray); locations[0] = new BlockLocation(hostsArray, hostsArray, 0, 1000); when(mockDfsClient.getBlockLocations(eq(path), eq(0L), eq(1000L))).thenReturn(locations); }/* w w w . ja v a 2s .co m*/ }
From source file:com.quantcast.qfs.hadoop.QuantcastFileSystem.java
License:Apache License
/** * Return null if the file doesn't exist; otherwise, get the * locations of the various chunks of the file file from QFS. *///from ww w . j av a2s . c o m @Override public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { if (file == null || start < 0 || len < 0) { throw new IllegalArgumentException("file: " + file + " start: " + start + " length: " + len); } final String srep = makeAbsolute(file.getPath()).toUri().getPath(); if (file.isDir()) { throw new IOException(srep + ": is a directory"); } final String[][] hints = qfsImpl.getBlocksLocation(srep, start, len); if (hints == null || hints.length < 1 || hints[0].length != 1) { throw new Error(srep + ": getBlocksLocation internal error"); } final long blockSize = Long.parseLong(hints[0][0], 16); if (blockSize < 0) { try { qfsImpl.retToIoException((int) blockSize); } catch (FileNotFoundException ex) { } return null; } if (blockSize == 0) { throw new Error(srep + ": getBlocksLocation internal error: 0 block size"); } final long end = Math.min(file.getLen(), start + len); if (hints.length <= 1 || end <= start) { // Return an emtpy host list, as hadoop expects at least one location. final BlockLocation[] result = new BlockLocation[1]; result[0] = new BlockLocation(null, null, start, Math.max(0L, end - start)); return result; } final int blkcnt = (int) ((end - 1) / blockSize - start / blockSize + 1); final BlockLocation[] result = new BlockLocation[blkcnt]; final ArrayList<String> hlist = new ArrayList<String>(); long pos = start - start % blockSize; for (int i = 0, m = 1; i < blkcnt; ++i) { hlist.clear(); if (m < hints.length) { final String[] locs = hints[m++]; hlist.ensureCapacity(locs.length); for (int k = 0; k < locs.length; ++k) { final int idx = locs[k].lastIndexOf(':'); final String host = 0 < idx ? locs[k].substring(0, idx) : locs[k]; if (!hlist.contains(host)) { hlist.add(host); } } } final long lpos = pos < start ? start : pos; final long bend = pos + blockSize; final int hsz = hlist.size(); result[i] = new BlockLocation(null, hsz <= 0 ? null : hlist.toArray(new String[hsz]), lpos, (bend < end ? bend : end) - lpos); pos = bend; } return result; }
From source file:com.qubole.rubix.core.CachingFileSystem.java
License:Apache License
@Override public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { if (!clusterManager.isMaster() || cacheSkipped) { // If in worker node, blockLocation does not matter return fs.getFileBlockLocations(file, start, len); }/*from w ww . j a va 2 s . c om*/ List<String> nodes = clusterManager.getNodes(); if (file == null) { return null; } else if (start >= 0L && len >= 0L) { if (file.getLen() < start) { return new BlockLocation[0]; } else { // Using similar logic of returning all Blocks as FileSystem.getFileBlockLocations does instead of only returning blocks from start till len BlockLocation[] blockLocations = new BlockLocation[(int) Math .ceil((double) file.getLen() / clusterManager.getSplitSize())]; int blockNumber = 0; for (long i = 0; i < file.getLen(); i = i + clusterManager.getSplitSize()) { long end = i + clusterManager.getSplitSize(); if (end > file.getLen()) { end = file.getLen(); } String key = file.getPath().toString() + i + end; HashFunction hf = Hashing.md5(); HashCode hc = hf.hashString(key, Charsets.UTF_8); int nodeIndex = Hashing.consistentHash(hc, nodes.size()); String[] name = new String[] { nodes.get(nodeIndex) }; String[] host = new String[] { nodes.get(nodeIndex) }; blockLocations[blockNumber++] = new BlockLocation(name, host, i, end - i); log.info(String.format("BlockLocation %s %d %d %s totalHosts: %s", file.getPath().toString(), i, end - i, host[0], nodes.size())); } return blockLocations; } } else { throw new IllegalArgumentException("Invalid start or len parameter"); } }
From source file:eu.scape_project.pt.mapred.input.ControlFileInputFormatTest.java
License:Apache License
@Test public void testGetSortedHosts() throws IOException { MockupFileSystem fs = new MockupFileSystem(); Path pInputFile1 = new Path("inputFile1"); Path pInputFile2 = new Path("inputFile2"); Path pInputFile3 = new Path("inputFile3"); fs.addFile(pInputFile1.toString(), true, new BlockLocation[] { new BlockLocation(null, new String[] { "hostA", "hostB" }, 0, 0), new BlockLocation(null, new String[] { "hostB", "hostC" }, 0, 0) }); fs.addFile(pInputFile2.toString(), true, new BlockLocation[] { new BlockLocation(null, new String[] { "hostA" }, 0, 0) }); fs.addFile(pInputFile3.toString(), true, new BlockLocation[] { new BlockLocation(null, new String[] { "hostA" }, 0, 0) }); String[] hosts = ControlFileInputFormat.getSortedHosts(fs, new Path[] { pInputFile1, pInputFile2, pInputFile3 }); int i = 1;/*from w w w. jav a 2s .com*/ for (String host : hosts) { LOG.debug(i++ + ". Host = " + host); } assertArrayEquals(new String[] { "hostA", "hostB", "hostC" }, hosts); }
From source file:org.apache.cassandra.hadoop.fs.CassandraFileSystemThriftStore.java
License:Apache License
public BlockLocation[] getBlockLocation(List<Block> blocks, long start, long len) throws IOException { if (blocks.isEmpty()) return null; List<ByteBuffer> blockKeys = new ArrayList<ByteBuffer>(blocks.size()); for (Block b : blocks) blockKeys.add(uuidToByteBuffer(b.id)); BlockLocation[] locations = new BlockLocation[blocks.size()]; try {// w w w .jav a 2 s .c om List<List<String>> blockEndpoints = ((Brisk.Iface) client).describe_keys(keySpace, blockKeys); for (int i = 0; i < blockEndpoints.size(); i++) { List<String> endpoints = blockEndpoints.get(i); Block b = blocks.get(i); long offset = (i == 0 && b.offset > start) ? start : b.offset; // TODO: Add topology info if at all possible? locations[i] = new BlockLocation(null, endpoints.toArray(new String[0]), offset, b.length); } return locations; } catch (Exception e) { throw new IOException(e); } }
From source file:org.apache.drill.exec.store.TestAffinityCalculator.java
License:Apache License
public BlockLocation[] buildBlockLocations(String[] hosts, long blockSize) { String[] names = new String[hosts.length]; for (int i = 0; i < hosts.length; i++) { hosts[i] = "host" + i; names[i] = "host:" + port; }/*from w w w . ja v a 2s . c o m*/ BlockLocation[] blockLocations = new BlockLocation[3]; blockLocations[0] = new BlockLocation(new String[] { names[0], names[1], names[2] }, new String[] { hosts[0], hosts[1], hosts[2] }, 0, blockSize); blockLocations[1] = new BlockLocation(new String[] { names[0], names[2], names[3] }, new String[] { hosts[0], hosts[2], hosts[3] }, blockSize, blockSize); blockLocations[2] = new BlockLocation(new String[] { names[0], names[1], names[3] }, new String[] { hosts[0], hosts[1], hosts[3] }, blockSize * 2, blockSize); return blockLocations; }