Example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_LIST_LIMIT

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_LIST_LIMIT.

Prototype

String DFS_LIST_LIMIT

To view the source code for org.apache.hadoop.hdfs DFSConfigKeys DFS_LIST_LIMIT.

Click Source Link

Usage

From source file:org.apache.tajo.storage.AbstractStorageManager.java

License:Apache License

private void setVolumeMeta(List<FileFragment> splits, final List<BlockLocation> blockLocations)
        throws IOException {

    int locationSize = blockLocations.size();
    int splitSize = splits.size();
    if (locationSize == 0 || splitSize == 0)
        return;//from  w  w  w  .j av a  2  s. c o  m

    if (locationSize != splitSize) {
        // splits and locations don't match up
        LOG.warn("Number of block locations not equal to number of splits: " + "#locations=" + locationSize
                + " #splits=" + splitSize);
        return;
    }

    DistributedFileSystem fs = (DistributedFileSystem) DistributedFileSystem.get(conf);
    int lsLimit = conf.getInt(DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
    int blockLocationIdx = 0;

    Iterator<FileFragment> iter = splits.iterator();
    while (locationSize > blockLocationIdx) {

        int subSize = Math.min(locationSize - blockLocationIdx, lsLimit);
        List<BlockLocation> locations = blockLocations.subList(blockLocationIdx, blockLocationIdx + subSize);
        //BlockStorageLocation containing additional volume location information for each replica of each block.
        BlockStorageLocation[] blockStorageLocations = fs.getFileBlockStorageLocations(locations);

        for (BlockStorageLocation blockStorageLocation : blockStorageLocations) {
            iter.next().setDiskIds(getDiskIds(blockStorageLocation.getVolumeIds()));
            blockLocationIdx++;
        }
    }
    LOG.info("# of splits with volumeId " + splitSize);
}

---

From source file:org.apache.tajo.storage.FileStorageManager.java

License:Apache License

private void setVolumeMeta(List<Fragment> splits, final List<BlockLocation> blockLocations) throws IOException {

    int locationSize = blockLocations.size();
    int splitSize = splits.size();
    if (locationSize == 0 || splitSize == 0)
        return;/* ww w.j  av a  2s .  com*/

    if (locationSize != splitSize) {
        // splits and locations don't match up
        LOG.warn("Number of block locations not equal to number of splits: " + "#locations=" + locationSize
                + " #splits=" + splitSize);
        return;
    }

    DistributedFileSystem fs = (DistributedFileSystem) DistributedFileSystem.get(conf);
    int lsLimit = conf.getInt(DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
    int blockLocationIdx = 0;

    Iterator<Fragment> iter = splits.iterator();
    while (locationSize > blockLocationIdx) {

        int subSize = Math.min(locationSize - blockLocationIdx, lsLimit);
        List<BlockLocation> locations = blockLocations.subList(blockLocationIdx, blockLocationIdx + subSize);
        //BlockStorageLocation containing additional volume location information for each replica of each block.
        BlockStorageLocation[] blockStorageLocations = fs.getFileBlockStorageLocations(locations);

        for (BlockStorageLocation blockStorageLocation : blockStorageLocations) {
            ((FileFragment) iter.next()).setDiskIds(getDiskIds(blockStorageLocation.getVolumeIds()));
            blockLocationIdx++;
        }
    }
    LOG.info("# of splits with volumeId " + splitSize);
}

---

From source file:org.apache.tajo.storage.FileTablespace.java

License:Apache License

private void setVolumeMeta(List<Fragment> splits, final List<BlockLocation> blockLocations) throws IOException {

    int locationSize = blockLocations.size();
    int splitSize = splits.size();
    if (locationSize == 0 || splitSize == 0)
        return;//from   ww  w .j  a  va2s .co m

    if (locationSize != splitSize) {
        // splits and locations don't match up
        LOG.warn("Number of block locations not equal to number of splits: " + "#locations=" + locationSize
                + " #splits=" + splitSize);
        return;
    }

    DistributedFileSystem fs = (DistributedFileSystem) this.fs;
    int lsLimit = conf.getInt(DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
    int blockLocationIdx = 0;

    Iterator<Fragment> iter = splits.iterator();
    while (locationSize > blockLocationIdx) {

        int subSize = Math.min(locationSize - blockLocationIdx, lsLimit);
        List<BlockLocation> locations = blockLocations.subList(blockLocationIdx, blockLocationIdx + subSize);
        //BlockStorageLocation containing additional volume location information for each replica of each block.
        BlockStorageLocation[] blockStorageLocations = fs.getFileBlockStorageLocations(locations);

        for (BlockStorageLocation blockStorageLocation : blockStorageLocations) {
            ((FileFragment) iter.next()).setDiskIds(getDiskIds(blockStorageLocation.getVolumeIds()));
            blockLocationIdx++;
        }
    }
    LOG.info("# of splits with volumeId " + splitSize);
}

---