Example usage for org.apache.hadoop.fs FileStatus getReplication

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getReplication.

Prototype

public short getReplication() 

Source Link

Document

Get the replication factor of a file.

Usage

From source file:eagle.security.hdfs.entity.FileStatusEntity.java

License:Apache License

public FileStatusEntity(FileStatus status) throws IOException {
    //this.path = status.getPath();
    this.length = status.getLen();
    this.isdir = status.isDirectory();
    this.block_replication = status.getReplication();
    this.blocksize = status.getBlockSize();
    this.modification_time = status.getModificationTime();
    this.access_time = status.getAccessTime();
    this.permission = status.getPermission();
    this.owner = status.getOwner();
    this.group = status.getGroup();
    if (status.isSymlink()) {
        this.symlink = status.getSymlink();
    }//ww  w  .ja  v a  2 s .  c om
}

---

From source file:edu.umn.cs.spatialHadoop.visualization.HadoopvizServer.java

License:Open Source License

/**
 * Lists the contents of a directory/* w  ww  . j a v a2  s  .c  o  m*/
 * @param request
 * @param response
 */
private void handleListFiles(HttpServletRequest request, HttpServletResponse response) {
    try {
        String pathStr = request.getParameter("path");
        Path path = new Path(pathStr == null || pathStr.isEmpty() ? "/" : pathStr);
        FileSystem fs = path.getFileSystem(commonParams);
        FileStatus[] fileStatuses = fs.listStatus(path, SpatialSite.NonHiddenFileFilter);
        Arrays.sort(fileStatuses, new Comparator<FileStatus>() {
            @Override
            public int compare(FileStatus o1, FileStatus o2) {
                if (o1.isDirectory() && o2.isFile())
                    return -1;
                if (o1.isFile() && o2.isDirectory())
                    return 1;
                return o1.getPath().getName().toLowerCase().compareTo(o2.getPath().getName().toLowerCase());
            }
        });
        response.setContentType("application/json;charset=utf-8");
        response.setStatus(HttpServletResponse.SC_OK);
        PrintWriter out = response.getWriter();
        out.print("{\"FileStatuses\":{");
        if (pathStr.endsWith("/")) {
            pathStr = pathStr.substring(0, pathStr.length() - 1);
        }
        out.printf("\"BaseDir\":\"%s\",", pathStr);
        if (path.getParent() != null)
            out.printf("\"ParentDir\":\"%s\",", path.getParent());
        out.print("\"FileStatus\":[");
        for (int i = 0; i < fileStatuses.length; i++) {
            FileStatus fileStatus = fileStatuses[i];
            if (i != 0)
                out.print(',');
            String filename = fileStatus.getPath().getName();
            int idot = filename.lastIndexOf('.');
            String extension = idot == -1 ? "" : filename.substring(idot + 1);
            out.printf(
                    "{\"accessTime\":%d,\"blockSize\":%d,\"childrenNum\":%d,\"fileId\":%d,"
                            + "\"group\":\"%s\",\"length\":%d,\"modificationTime\":%d,"
                            + "\"owner\":\"%s\",\"pathSuffix\":\"%s\",\"permission\":\"%s\","
                            + "\"replication\":%d,\"storagePolicy\":%d,\"type\":\"%s\",\"extension\":\"%s\"}",
                    fileStatus.getAccessTime(), fileStatus.getBlockSize(), 0, 0, fileStatus.getGroup(),
                    fileStatus.getLen(), fileStatus.getModificationTime(), fileStatus.getOwner(),
                    fileStatus.getPath().getName(), fileStatus.getPermission(), fileStatus.getReplication(), 0,
                    fileStatus.isDirectory() ? "DIRECTORY" : "FILE", extension.toLowerCase());
        }
        out.print("]}");
        // Check if there is an image or master file
        FileStatus[] metaFiles = fs.listStatus(path, new PathFilter() {
            @Override
            public boolean accept(Path path) {
                return path.getName().startsWith("_master") || path.getName().equals("_data.png");
            }
        });
        for (FileStatus metaFile : metaFiles) {
            String metaFileName = metaFile.getPath().getName();
            if (metaFileName.startsWith("_master")) {
                out.printf(",\"MasterPath\":\"%s\"", metaFileName);
                String shape = OperationsParams.detectShape(fileStatuses[0].getPath(), commonParams);
                if (shape != null)
                    out.printf(",\"Shape\":\"%s\"", shape);
            } else if (metaFileName.equals("_data.png"))
                out.printf(",\"ImagePath\":\"%s\"", metaFileName);
        }
        out.print("}");

        out.close();
    } catch (Exception e) {
        System.out.println("error happened");
        e.printStackTrace();
        try {
            e.printStackTrace(response.getWriter());
        } catch (IOException ioe) {
            ioe.printStackTrace();
            e.printStackTrace();
        }
        response.setContentType("text/plain;charset=utf-8");
        response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
    }
}

---

From source file:fm.last.hadoop.tools.ReplicationPolicyFixer.java

License:Apache License

private void findMissReplicatedFiles(FileStatus file, Set<Path> missReplicatedFiles) throws IOException {
    Path path = file.getPath();/*from  w  ww  .  ja va2 s.  c  om*/

    if (file.isDir()) {
        FileStatus[] files = fs.listStatus(path);
        if (files == null) {
            return;
        }
        for (FileStatus subFile : files) {
            findMissReplicatedFiles(subFile, missReplicatedFiles);
        }
        return;
    }

    int pathNameLength = path.toUri().getPath().length();
    String padding = StringUtils.repeat(" ", Math.max(0, lastPathNameLength - pathNameLength));
    lastPathNameLength = pathNameLength;
    out.print(path.toUri().getPath() + padding + "\r");
    out.flush();

    LocatedBlocks blocks = nameNode.getBlockLocations(path.toUri().getPath(), 0, file.getLen());
    if (blocks == null) { // the file is deleted
        return;
    }
    if (blocks.isUnderConstruction()) {
        out.println("\nNot checking open file : " + path.toString());
        return;
    }

    for (LocatedBlock lBlk : blocks.getLocatedBlocks()) {
        if (lBlk.isCorrupt()) {
            out.println("\n" + lBlk.toString() + " is corrupt so skipping file : " + path.toString());
            return;
        }

        Block block = lBlk.getBlock();
        DatanodeInfo[] locs = lBlk.getLocations();
        short targetFileReplication = file.getReplication();
        // verify block placement policy
        int missingRacks = verifyBlockPlacement(lBlk, targetFileReplication, cluster);
        if (missingRacks > 0 && locs.length > 0) {
            out.println("\nReplica placement policy is violated for " + block.toString() + " of file "
                    + path.toString() + ". Block should be additionally replicated on " + missingRacks
                    + " more rack(s).");
            missReplicatedFiles.add(path);
        }
    }
}

---

From source file:gobblin.data.management.copy.writer.FileAwareInputStreamDataWriter.java

License:Apache License

/**
 * Write the contents of input stream into staging path.
 *
 * <p>/*from w w  w  .j  a v  a  2s  .  c  o m*/
 *   WriteAt indicates the path where the contents of the input stream should be written. When this method is called,
 *   the path writeAt.getParent() will exist already, but the path writeAt will not exist. When this method is returned,
 *   the path writeAt must exist. Any data written to any location other than writeAt or a descendant of writeAt
 *   will be ignored.
 * </p>
 *
 * @param inputStream {@link FSDataInputStream} whose contents should be written to staging path.
 * @param writeAt {@link Path} at which contents should be written.
 * @param copyableFile {@link gobblin.data.management.copy.CopyEntity} that generated this copy operation.
 * @throws IOException
 */
protected void writeImpl(InputStream inputStream, Path writeAt, CopyableFile copyableFile) throws IOException {

    final short replication = copyableFile.getPreserve().preserve(PreserveAttributes.Option.REPLICATION)
            ? copyableFile.getOrigin().getReplication()
            : this.fs.getDefaultReplication(writeAt);
    final long blockSize = copyableFile.getPreserve().preserve(PreserveAttributes.Option.BLOCK_SIZE)
            ? copyableFile.getOrigin().getBlockSize()
            : this.fs.getDefaultBlockSize(writeAt);

    Predicate<FileStatus> fileStatusAttributesFilter = new Predicate<FileStatus>() {
        @Override
        public boolean apply(FileStatus input) {
            return input.getReplication() == replication && input.getBlockSize() == blockSize;
        }
    };
    Optional<FileStatus> persistedFile = this.recoveryHelper.findPersistedFile(this.state, copyableFile,
            fileStatusAttributesFilter);

    if (persistedFile.isPresent()) {
        log.info(String.format("Recovering persisted file %s to %s.", persistedFile.get().getPath(), writeAt));
        this.fs.rename(persistedFile.get().getPath(), writeAt);
    } else {

        OutputStream os = this.fs.create(writeAt, true, this.fs.getConf().getInt("io.file.buffer.size", 4096),
                replication, blockSize);
        if (encryptionConfig != null) {
            os = EncryptionFactory.buildStreamCryptoProvider(encryptionConfig).encodeOutputStream(os);
        }
        try {

            StreamThrottler<GobblinScopeTypes> throttler = this.taskBroker
                    .getSharedResource(new StreamThrottler.Factory<GobblinScopeTypes>(), new EmptyKey());
            ThrottledInputStream throttledInputStream = throttler.throttleInputStream().inputStream(inputStream)
                    .sourceURI(copyableFile.getOrigin().getPath().toUri())
                    .targetURI(this.fs.makeQualified(writeAt).toUri()).build();
            StreamCopier copier = new StreamCopier(throttledInputStream, os).withBufferSize(this.bufferSize);

            if (isInstrumentationEnabled()) {
                copier.withCopySpeedMeter(this.copySpeedMeter);
            }
            this.bytesWritten.addAndGet(copier.copy());
            if (isInstrumentationEnabled()) {
                log.info("File {}: copied {} bytes, average rate: {} B/s", copyableFile.getOrigin().getPath(),
                        this.copySpeedMeter.getCount(), this.copySpeedMeter.getMeanRate());
            } else {
                log.info("File {} copied.", copyableFile.getOrigin().getPath());
            }
        } catch (NotConfiguredException nce) {
            log.warn("Broker error. Some features of stream copier may not be available.", nce);
        } finally {
            os.close();
            inputStream.close();
        }
    }
}

---

From source file:gobblin.util.filesystem.InstrumentedFileSystemUtils.java

License:Apache License

/**
 * Replace the scheme of the input {@link FileStatus} if it matches the string to replace.
 *//*from   ww  w .j  a v  a  2 s.  c  o  m*/
public static FileStatus replaceScheme(FileStatus st, String replace, String replacement) {
    if (replace != null && replace.equals(replacement)) {
        return st;
    }
    try {
        return new FileStatus(st.getLen(), st.isDir(), st.getReplication(), st.getBlockSize(),
                st.getModificationTime(), st.getAccessTime(), st.getPermission(), st.getOwner(), st.getGroup(),
                st.isSymlink() ? st.getSymlink() : null, replaceScheme(st.getPath(), replace, replacement));
    } catch (IOException ioe) {
        throw new RuntimeException(ioe);
    }
}

---

From source file:hadoop.example.hdfs.ShowFileStatusTestCase.java

License:Open Source License

@Test
public void fileStatusForFile() throws IOException {
    Path file = new Path("/dir/file");
    FileStatus stat = fs.getFileStatus(file);
    Assert.assertEquals("/dir/file", stat.getPath().toUri().getPath());
    Assert.assertFalse(stat.isDir());/*from ww w  .j  a v  a 2 s  . c  o  m*/
    Assert.assertEquals(stat.getLen(), 7L);
    Assert.assertEquals(stat.getReplication(), 1);
    Assert.assertEquals(stat.getBlockSize(), 64 * 1024 * 1024L);
    Assert.assertEquals(stat.getOwner(), "haint");
    Assert.assertEquals(stat.getGroup(), "supergroup");
    Assert.assertEquals(stat.getPermission().toString(), "rw-r--r--");
}

---

From source file:hadoop.example.hdfs.ShowFileStatusTestCase.java

License:Open Source License

@Test
public void fileStatusForDirectory() throws IOException {
    Path dir = new Path("/dir");
    FileStatus stat = fs.getFileStatus(dir);
    Assert.assertTrue(stat.isDir());//from  w  ww.j  av  a 2 s.c o  m
    Assert.assertEquals(stat.getLen(), 0L);
    Assert.assertEquals(stat.getReplication(), 0L);
    Assert.assertEquals(stat.getBlockSize(), 0L);
    Assert.assertEquals(stat.getOwner(), "haint");
    Assert.assertEquals(stat.getGroup(), "supergroup");
    Assert.assertEquals(stat.getPermission().toString(), "rwxr-xr-x");
}

---

From source file:hdfs.jsr203.attribute.HadoopFileAttributeView.java

License:Apache License

Object attribute(AttrID id, FileStatus hfas) {
    switch (id) {
    case accessTime:
        return hfas.getAccessTime();
    case blockSize:
        return hfas.getBlockSize();
    case group://ww  w . j ava2 s  . c o m
        return hfas.getGroup();
    case len:
        return hfas.getLen();
    case modificationTime:
        return hfas.getModificationTime();
    case owner:
        return hfas.getOwner();
    case replication:
        return hfas.getReplication();
    case isDirectory:
        return hfas.isDirectory();
    // TODO enable encryption
    //case isEncrypted:
    //    return hfas.isEncrypted();
    case isFile:
        return hfas.isFile();
    case isSymLink:
        return hfas.isSymlink();
    }
    return null;
}

---

From source file:io.aos.hdfs.ShowFileStatusTest.java

License:Apache License

@Test
public void fileStatusForFile() throws IOException {
    Path file = new Path("/dir/file");
    FileStatus stat = fs.getFileStatus(file);
    assertThat(stat.getPath().toUri().getPath(), is("/dir/file"));
    assertThat(stat.isDir(), is(false));
    assertThat(stat.getLen(), is(7L));/*from w ww  .  jav a  2 s  . co  m*/
    assertThat(stat.getModificationTime(), is(lessThanOrEqualTo(System.currentTimeMillis())));
    assertThat(stat.getReplication(), is((short) 1));
    assertThat(stat.getBlockSize(), is(64 * 1024 * 1024L));
    assertThat(stat.getOwner(), is("tom"));
    assertThat(stat.getGroup(), is("supergroup"));
    assertThat(stat.getPermission().toString(), is("rw-r--r--"));
}

---

From source file:io.aos.hdfs.ShowFileStatusTest.java

License:Apache License

@Test
public void fileStatusForDirectory() throws IOException {
    Path dir = new Path("/dir");
    FileStatus stat = fs.getFileStatus(dir);
    assertThat(stat.getPath().toUri().getPath(), is("/dir"));
    assertThat(stat.isDir(), is(true));//from w  w  w  . jav a  2s. com
    assertThat(stat.getLen(), is(0L));
    assertThat(stat.getModificationTime(), is(lessThanOrEqualTo(System.currentTimeMillis())));
    assertThat(stat.getReplication(), is((short) 0));
    assertThat(stat.getBlockSize(), is(0L));
    assertThat(stat.getOwner(), is("tom"));
    assertThat(stat.getGroup(), is("supergroup"));
    assertThat(stat.getPermission().toString(), is("rwxr-xr-x"));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• »