Example usage for org.apache.hadoop.fs FileStatus getBlockSize

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getBlockSize.

Prototype

public long getBlockSize() 

Source Link

Document

Get the block size of the file.

Usage

From source file:hadoop.example.hdfs.ShowFileStatusTestCase.java

License:Open Source License

@Test
public void fileStatusForDirectory() throws IOException {
    Path dir = new Path("/dir");
    FileStatus stat = fs.getFileStatus(dir);
    Assert.assertTrue(stat.isDir());/*w  ww  .j a va  2s  . com*/
    Assert.assertEquals(stat.getLen(), 0L);
    Assert.assertEquals(stat.getReplication(), 0L);
    Assert.assertEquals(stat.getBlockSize(), 0L);
    Assert.assertEquals(stat.getOwner(), "haint");
    Assert.assertEquals(stat.getGroup(), "supergroup");
    Assert.assertEquals(stat.getPermission().toString(), "rwxr-xr-x");
}

---

From source file:hdfs.jsr203.attribute.HadoopFileAttributeView.java

License:Apache License

Object attribute(AttrID id, FileStatus hfas) {
    switch (id) {
    case accessTime:
        return hfas.getAccessTime();
    case blockSize:
        return hfas.getBlockSize();
    case group:/*from   w w w. j  av  a 2s. com*/
        return hfas.getGroup();
    case len:
        return hfas.getLen();
    case modificationTime:
        return hfas.getModificationTime();
    case owner:
        return hfas.getOwner();
    case replication:
        return hfas.getReplication();
    case isDirectory:
        return hfas.isDirectory();
    // TODO enable encryption
    //case isEncrypted:
    //    return hfas.isEncrypted();
    case isFile:
        return hfas.isFile();
    case isSymLink:
        return hfas.isSymlink();
    }
    return null;
}

---

From source file:input_format.MyFileInputFormat.java

License:Open Source License

/** 
 * Generate the list of files and make them into FileSplits.
 *//*  w  ww .  java 2s.c o m*/
public List<InputSplit> getSplits(JobContext job) throws IOException {
    long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
    long maxSize = getMaxSplitSize(job);

    // generate splits
    List<InputSplit> splits = new ArrayList<InputSplit>();
    for (FileStatus file : listStatus(job)) {
        Path path = file.getPath();
        FileSystem fs = path.getFileSystem(job.getConfiguration());
        long length = file.getLen();
        BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length);
        if ((length != 0) && isSplitable(job, path)) {
            long blockSize = file.getBlockSize();
            long splitSize = computeSplitSize(blockSize, minSize, maxSize);

            long bytesRemaining = length;
            while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) {
                int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
                splits.add(new MyFileSplit(path, length - bytesRemaining, splitSize,
                        blkLocations[blkIndex].getHosts()));
                bytesRemaining -= splitSize;
            }

            if (bytesRemaining != 0) {
                splits.add(new MyFileSplit(path, length - bytesRemaining, bytesRemaining,
                        blkLocations[blkLocations.length - 1].getHosts()));
            }
        } else if (length != 0) {
            splits.add(new MyFileSplit(path, 0, length, blkLocations[0].getHosts()));
        } else {
            //Create empty hosts array for zero length files
            splits.add(new MyFileSplit(path, 0, length, new String[0]));
        }
    }
    LOG.debug("Total # of splits: " + splits.size());

    //String p=job.getConfiguration().get("mapred.fairscheduler.pool");
    //int max = Integer.parseInt(p.substring(p.indexOf("l")+1));
    int max = 1000;

    if (splits.size() <= max)
        job.getConfiguration().setInt("mapred.reduce.tasks", splits.size());
    else
        job.getConfiguration().setInt("mapred.reduce.tasks", max);
    return splits;
}

---

From source file:io.aos.hdfs.ShowFileStatusTest.java

License:Apache License

@Test
public void fileStatusForFile() throws IOException {
    Path file = new Path("/dir/file");
    FileStatus stat = fs.getFileStatus(file);
    assertThat(stat.getPath().toUri().getPath(), is("/dir/file"));
    assertThat(stat.isDir(), is(false));
    assertThat(stat.getLen(), is(7L));/* w w  w  . j  a v a  2  s. c om*/
    assertThat(stat.getModificationTime(), is(lessThanOrEqualTo(System.currentTimeMillis())));
    assertThat(stat.getReplication(), is((short) 1));
    assertThat(stat.getBlockSize(), is(64 * 1024 * 1024L));
    assertThat(stat.getOwner(), is("tom"));
    assertThat(stat.getGroup(), is("supergroup"));
    assertThat(stat.getPermission().toString(), is("rw-r--r--"));
}

---

From source file:io.aos.hdfs.ShowFileStatusTest.java

License:Apache License

@Test
public void fileStatusForDirectory() throws IOException {
    Path dir = new Path("/dir");
    FileStatus stat = fs.getFileStatus(dir);
    assertThat(stat.getPath().toUri().getPath(), is("/dir"));
    assertThat(stat.isDir(), is(true));//from w ww.  jav a2  s.  c  o  m
    assertThat(stat.getLen(), is(0L));
    assertThat(stat.getModificationTime(), is(lessThanOrEqualTo(System.currentTimeMillis())));
    assertThat(stat.getReplication(), is((short) 0));
    assertThat(stat.getBlockSize(), is(0L));
    assertThat(stat.getOwner(), is("tom"));
    assertThat(stat.getGroup(), is("supergroup"));
    assertThat(stat.getPermission().toString(), is("rwxr-xr-x"));
}

---

From source file:io.apigee.lembos.node.types.DistributedCacheWrap.java

License:Apache License

/**
 * Java wrapper for {@link DistributedCache#getFileStatus(Configuration, URI)}.
 *
 * @param ctx the JavaScript context//from w  ww.  j  a va  2s . c  o  m
 * @param thisObj the 'this' object
 * @param args the function arguments
 * @param func the function being called
 *
 * @return array of archive class paths
 */
@JSStaticFunction
public static Object getFileStatus(final Context ctx, final Scriptable thisObj, final Object[] args,
        final Function func) {
    final Object arg0 = args.length >= 1 ? args[0] : Undefined.instance;
    final Object arg1 = args.length >= 2 ? args[1] : Undefined.instance;

    if (args.length < 2) {
        throw Utils.makeError(ctx, thisObj, LembosMessages.TWO_ARGS_EXPECTED);
    } else if (!JavaScriptUtils.isDefined(arg0)) {
        throw Utils.makeError(ctx, thisObj, LembosMessages.FIRST_ARG_REQUIRED);
    } else if (!JavaScriptUtils.isDefined(arg1)) {
        throw Utils.makeError(ctx, thisObj, LembosMessages.SECOND_ARG_REQUIRED);
    } else if (!(arg0 instanceof ConfigurationWrap)) {
        throw Utils.makeError(ctx, thisObj, LembosMessages.FIRST_ARG_MUST_BE_CONF);
    }

    final URI hdfsUri = URI.create(arg1.toString());
    FileStatus status;

    try {
        status = DistributedCache.getFileStatus(((ConfigurationWrap) arg0).getConf(), hdfsUri);
    } catch (IOException e) {
        throw Utils.makeError(ctx, thisObj, e.getMessage());
    }

    if (status == null) {
        throw Utils.makeError(ctx, thisObj, "Unable to get file status for HDFS uri: " + hdfsUri.toString());
    }

    final Scriptable jsStatus = ctx.newObject(thisObj);

    ScriptableObject.defineProperty(jsStatus, "accessTime", status.getAccessTime(), ScriptableObject.READONLY);
    ScriptableObject.defineProperty(jsStatus, "blockSize", status.getBlockSize(), ScriptableObject.READONLY);
    ScriptableObject.defineProperty(jsStatus, "group", status.getGroup(), ScriptableObject.READONLY);
    ScriptableObject.defineProperty(jsStatus, "len", status.getLen(), ScriptableObject.READONLY);
    ScriptableObject.defineProperty(jsStatus, "modificationTime", status.getModificationTime(),
            ScriptableObject.READONLY);
    ScriptableObject.defineProperty(jsStatus, "owner", status.getOwner(), ScriptableObject.READONLY);
    ScriptableObject.defineProperty(jsStatus, "path", status.getPath().toString(), ScriptableObject.READONLY);
    ScriptableObject.defineProperty(jsStatus, "permission", status.getPermission().toString(),
            ScriptableObject.READONLY);
    ScriptableObject.defineProperty(jsStatus, "replication", status.getReplication(),
            ScriptableObject.READONLY);

    return jsStatus;
}

---

From source file:io.hops.erasure_coding.BaseEncodingManager.java

License:Apache License

static long numBlocks(FileStatus stat) {
    return (long) Math.ceil(stat.getLen() * 1.0 / stat.getBlockSize());
}

---

From source file:io.hops.erasure_coding.BaseEncodingManager.java

License:Apache License

public static long getNumBlocks(FileStatus stat) {
    long numBlocks = stat.getLen() / stat.getBlockSize();
    if (stat.getLen() % stat.getBlockSize() == 0) {
        return numBlocks;
    } else {//from   www. ja v a2 s .  c om
        return numBlocks + 1;
    }
}

---

From source file:io.hops.erasure_coding.BaseEncodingManager.java

License:Apache License

/**
 * RAID an individual file//from   ww w .  java 2s  . c  om
 */
public static boolean doFileRaid(Configuration conf, Path sourceFile, Path parityPath, Codec codec,
        Statistics statistics, Progressable reporter, int targetRepl, int metaRepl) throws IOException {
    FileSystem srcFs = sourceFile.getFileSystem(conf);
    FileStatus sourceStatus = srcFs.getFileStatus(sourceFile);

    // extract block locations from File system
    BlockLocation[] locations = srcFs.getFileBlockLocations(sourceFile, 0, sourceStatus.getLen());
    // if the file has fewer than 2 blocks, then nothing to do
    if (locations.length <= 2) {
        return false;
    }

    // add up the raw disk space occupied by this file
    long diskSpace = 0;
    for (BlockLocation l : locations) {
        diskSpace += (l.getLength() * sourceStatus.getReplication());
    }
    statistics.numProcessedBlocks += locations.length;
    statistics.processedSize += diskSpace;

    // generate parity file
    generateParityFile(conf, sourceStatus, targetRepl, reporter, srcFs, parityPath, codec, locations.length,
            sourceStatus.getReplication(), metaRepl, sourceStatus.getBlockSize());
    if (srcFs.setReplication(sourceFile, (short) targetRepl) == false) {
        LOG.info("Error in reducing replication of " + sourceFile + " to " + targetRepl);
        statistics.remainingSize += diskSpace;
        return false;
    }
    ;

    diskSpace = 0;
    for (BlockLocation l : locations) {
        diskSpace += (l.getLength() * targetRepl);
    }
    statistics.remainingSize += diskSpace;

    // the metafile will have this many number of blocks
    int numMeta = locations.length / codec.stripeLength;
    if (locations.length % codec.stripeLength != 0) {
        numMeta++;
    }

    // we create numMeta for every file. This metablock has metaRepl # replicas.
    // the last block of the metafile might not be completely filled up, but we
    // ignore that for now.
    statistics.numMetaBlocks += (numMeta * metaRepl);
    statistics.metaSize += (numMeta * metaRepl * sourceStatus.getBlockSize());
    return true;
}

---

From source file:io.hops.erasure_coding.FileStripeReader.java

License:Apache License

@Override
public InputStream buildOneInput(int locationIndex, long offsetInBlock, FileSystem srcFs, Path srcFile,
        FileStatus srcStat, FileSystem parityFs, Path parityFile, FileStatus parityStat) throws IOException {
    final long blockSize = srcStat.getBlockSize();

    LOG.info("buildOneInput srcfile " + srcFile + " srclen " + srcStat.getLen() + " parityfile " + parityFile
            + " paritylen " + parityStat.getLen() + " stripeindex " + stripeStartIdx + " locationindex "
            + locationIndex + " offsetinblock " + offsetInBlock);
    if (locationIndex < codec.parityLength) {
        return this.getParityFileInput(locationIndex, parityFile, parityFs, parityStat, offsetInBlock);
    } else {// w  w  w.  j  a va 2 s .c o  m
        // Dealing with a src file here.
        int blockIdxInStripe = locationIndex - codec.parityLength;
        int blockIdx = (int) (codec.stripeLength * stripeStartIdx + blockIdxInStripe);
        long offset = blockSize * blockIdx + offsetInBlock;
        if (offset >= srcStat.getLen()) {
            LOG.info("Using zeros for " + srcFile + ":" + offset + " for location " + locationIndex);
            return new RaidUtils.ZeroInputStream(blockSize * (blockIdx + 1));
        } else {
            LOG.info("Opening " + srcFile + ":" + offset + " for location " + locationIndex);
            FSDataInputStream s = fs.open(srcFile, conf.getInt("io.file.buffer.size", 64 * 1024));
            s.seek(offset);
            return s;
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• »