Example usage for org.apache.hadoop.fs BlockLocation getTopologyPaths

Introduction

In this page you can find the example usage for org.apache.hadoop.fs BlockLocation getTopologyPaths.

Prototype

public String[] getTopologyPaths() throws IOException 

Source Link

Document

Get the list of network topology paths for each of the hosts.

Usage

From source file:ch.cern.db.hdfs.Main.java

License:GNU General Public License

private void printBlockMetadata(BlockLocation blockLocation, String[] dataDirs) throws IOException {

    System.out.println("   Offset: " + blockLocation.getOffset());
    System.out.println("   Length: " + blockLocation.getLength());

    String[] cachedHosts = blockLocation.getCachedHosts();
    if (cachedHosts.length == 0) {
        System.out.println("   No cached hosts");
    }/*from  www.  ja  va 2 s  . co  m*/

    System.out.println("   Replicas:");
    VolumeId[] volumeIds = blockLocation instanceof BlockStorageLocation
            ? (((BlockStorageLocation) blockLocation).getVolumeIds())
            : null;
    String[] hosts = blockLocation.getHosts();
    String[] names = blockLocation.getNames();
    String[] topologyPaths = blockLocation.getTopologyPaths();
    for (int i = 0; i < topologyPaths.length; i++) {
        int diskId = volumeIds != null ? DistributedFileSystemMetadata.getDiskId(volumeIds[i]) : -1;

        System.out.println("      Replica (" + i + "):");
        System.out.println("         Host: " + hosts[i]);

        if (diskId == -1)
            System.out.println("         DiskId: unknown");
        else if (dataDirs != null && diskId < dataDirs.length)
            System.out.println("         Location: " + dataDirs[diskId] + " (DiskId: " + diskId + ")");
        else
            System.out.println("         DiskId: " + diskId);

        System.out.println("         Name: " + names[i]);
        System.out.println("         TopologyPaths: " + topologyPaths[i]);
    }

    if (cachedHosts.length > 0) {
        System.out.println("   Cached hosts:");
        for (String cachedHost : cachedHosts) {
            System.out.println("      Host: " + cachedHost);
        }
    }
}

---

From source file:com.cloudera.GetBlockLocations.java

License:Apache License

public static void main(String[] args) throws Exception {
    final Configuration conf = new Configuration();
    String url = getStringOrDie("get.block.locations.path");
    final FileSystem fs = FileSystem.get(new URI(url), conf);

    if (!fs.exists(new Path(url))) {
        System.out.println("no file at " + url);
        System.exit(1);/*from   ww w  . j  av  a  2s. co m*/
    }
    BlockLocation locs[] = null;
    try {
        locs = fs.getFileBlockLocations(new Path(url), 0, Long.MAX_VALUE);
    } catch (IOException e) {
        System.out.println("Error calling getFileBlockLocations(" + url + ")\n");
        e.printStackTrace(System.err);
        System.exit(1);
    }

    String prefix = "";
    for (BlockLocation loc : locs) {
        System.out.println(prefix);
        System.out.println("{");
        System.out.println("  hosts =         " + Arrays.toString(loc.getHosts()));
        System.out.println("  cachedHosts =   " + Arrays.toString(loc.getCachedHosts()));
        System.out.println("  names    =      " + Arrays.toString(loc.getNames()));
        System.out.println("  topologyPaths = " + Arrays.toString(loc.getTopologyPaths()));
        System.out.println("  offset =        " + loc.getOffset());
        System.out.println("  length =        " + loc.getLength());
        System.out.println("  corrupt =       " + loc.isCorrupt());
        System.out.println("}");
        prefix = ",";
    }
}

---

From source file:hsyndicate.tools.BlockLocations.java

License:Apache License

public static void main(String[] args) throws Exception {
    Path p = new Path(args[0]);
    Configuration conf = new Configuration();
    FileSystem fs = p.getFileSystem(conf);
    FileStatus f = fs.getFileStatus(p);/*from w w w. j a v a2 s. c  om*/
    BlockLocation[] bla = fs.getFileBlockLocations(f, 0, f.getLen());

    System.out.println("File : " + f.getPath().toString());
    for (BlockLocation bl : bla) {
        System.out.println("Offset : " + bl.getOffset());
        System.out.println("Len : " + bl.getLength());
        System.out.println("Hosts : " + makeCommaSeparated(bl.getHosts()));
        System.out.println("Names : " + makeCommaSeparated(bl.getNames()));
        System.out.println("TopologyPaths : " + makeCommaSeparated(bl.getTopologyPaths()));
    }
}

---

From source file:org.springframework.yarn.batch.partition.HdfsSplitBatchPartitionHandler.java

License:Apache License

@Override
protected Map<StepExecution, ContainerRequestHint> createResourceRequestData(Set<StepExecution> stepExecutions)
        throws Exception {
    Map<StepExecution, ContainerRequestHint> requests = new HashMap<StepExecution, ContainerRequestHint>();

    for (StepExecution execution : stepExecutions) {
        String fileName = execution.getExecutionContext().getString("fileName");
        long splitStart = execution.getExecutionContext().getLong("splitStart");
        long splitLength = execution.getExecutionContext().getLong("splitLength");

        log.debug("Creating request data for stepExecution=" + execution + " with fileName=" + fileName
                + " splitStart=" + splitStart + " splitLength=" + splitLength);

        FileSystem fs = FileSystem.get(configuration);
        Path path = new Path(execution.getExecutionContext().getString("fileName"));

        HashSet<String> hostsSet = new HashSet<String>();

        BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(path, splitStart, splitLength);
        for (BlockLocation blockLocation : fileBlockLocations) {
            for (String host : blockLocation.getHosts()) {
                hostsSet.add(host);// w ww  .ja  v  a2 s . c  o  m
            }
            log.debug("block: " + blockLocation + " topologypaths="
                    + StringUtils.arrayToCommaDelimitedString(blockLocation.getTopologyPaths()));
        }

        String[] hosts = hostsSet.toArray(new String[0]);
        String[] racks = new String[0];
        // hints only for hosts
        requests.put(execution, new ContainerRequestHint(execution, null, hosts, racks, null));
    }

    return requests;
}

---