Example usage for org.apache.hadoop.fs FileStatus isFile

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus isFile.

Prototype

public boolean isFile()

Source Link

Document

Is this a file?

Usage

From source file:org.springframework.yarn.fs.DefaultResourceLocalizerTests.java

License:Apache License

@Test
public void testRawEntries() throws Exception {
    String dir = "/DefaultResourceLocalizerTests-testRawEntries";
    LocalResourcesFactoryBean factory = new LocalResourcesFactoryBean();
    factory.setConfiguration(configuration);

    List<CopyEntry> copyEntries = new ArrayList<CopyEntry>();
    factory.setCopyEntries(copyEntries);
    factory.setHdfsEntries(new ArrayList<LocalResourcesFactoryBean.TransferEntry>());

    List<RawCopyEntry> rawEntries = new ArrayList<RawCopyEntry>();
    rawEntries.add(new RawCopyEntry(new byte[10], dir + "/rawContent1", false));
    factory.setRawCopyEntries(rawEntries);
    factory.afterPropertiesSet();//  w  ww  .jav  a  2 s.co m

    SmartResourceLocalizer localizer = (SmartResourceLocalizer) factory.getObject();
    localizer.copy();

    FileSystem fs = FileSystem.get(configuration);
    FileStatus fileStatus = fs.getFileStatus(new Path(dir + "/rawContent1"));
    assertThat(fileStatus.isFile(), is(true));
    assertThat(fileStatus.getLen(), is(10l));
}

From source file:org.springframework.yarn.fs.DefaultResourceLocalizerTests.java

License:Apache License

@Test
public void testRawEntriesStaging() throws Exception {
    String dir = "DefaultResourceLocalizerTests-testRawEntriesStaging";
    LocalResourcesFactoryBean factory = new LocalResourcesFactoryBean();
    factory.setConfiguration(configuration);

    List<CopyEntry> copyEntries = new ArrayList<CopyEntry>();
    factory.setCopyEntries(copyEntries);
    factory.setHdfsEntries(new ArrayList<LocalResourcesFactoryBean.TransferEntry>());

    List<RawCopyEntry> rawEntries = new ArrayList<RawCopyEntry>();
    rawEntries.add(new RawCopyEntry(new byte[10], "rawContent1", true));
    factory.setRawCopyEntries(rawEntries);
    factory.afterPropertiesSet();//from   w ww. j av  a2  s. c om

    SmartResourceLocalizer localizer = (SmartResourceLocalizer) factory.getObject();
    localizer.setStagingDirectory(new Path(dir));
    localizer.setStagingId("foo-id");
    localizer.copy();

    FileSystem fs = FileSystem.get(configuration);
    FileStatus fileStatus = fs.getFileStatus(new Path(dir + "/foo-id/rawContent1"));
    assertThat(fileStatus.isFile(), is(true));
    assertThat(fileStatus.getLen(), is(10l));
}

From source file:streaming.core.DownloadRunner.java

License:Apache License

public static int getTarFileByPath(HttpServletResponse res, String pathStr) {
    String[] paths = pathStr.split(",");
    try {//from w w  w .  jav  a2 s .co  m
        OutputStream outputStream = res.getOutputStream();

        TarOutputStream tarOutputStream = new TarOutputStream(new BufferedOutputStream(outputStream));

        FileSystem fs = FileSystem.get(new Configuration());
        List<FileStatus> files = new ArrayList<FileStatus>();

        for (String path : paths) {
            Path p = new Path(path);
            if (fs.exists(p)) {
                if (fs.isFile(p)) {
                    files.add(fs.getFileStatus(p));
                } else if (fs.isDirectory(p)) {
                    FileStatus[] fileStatusArr = fs.listStatus(p);
                    if (fileStatusArr != null && fileStatusArr.length > 0) {

                        for (FileStatus cur : fileStatusArr) {
                            if (cur.isFile()) {
                                files.add(cur);
                            }
                        }
                    }
                }
            }

        }

        if (files.size() > 0) {
            FSDataInputStream inputStream = null;
            int len = files.size();
            int i = 1;
            for (FileStatus cur : files) {
                logger.info("[" + i++ + "/" + len + "]" + ",?" + cur);
                inputStream = fs.open(cur.getPath());

                tarOutputStream.putNextEntry(new HDFSTarEntry(cur, cur.getPath().getName()));
                org.apache.commons.io.IOUtils.copyLarge(inputStream, tarOutputStream);
                inputStream.close();

            }
            tarOutputStream.flush();
            tarOutputStream.close();
            return 200;
        } else
            return 400;

    } catch (Exception e) {
        e.printStackTrace();
        return 500;

    }
}

From source file:yarnkit.utils.YarnUtils.java

License:Apache License

@Nonnull
public static LocalResource createLocalResource(@Nonnull FileSystem fs, @Nonnull Path hdfsPath)
        throws IOException {
    LocalResource resource = Records.newRecord(LocalResource.class);
    FileStatus fileStat = fs.getFileStatus(hdfsPath);
    resource.setResource(ConverterUtils.getYarnUrlFromPath(hdfsPath));
    resource.setSize(fileStat.getLen());
    resource.setTimestamp(fileStat.getModificationTime());
    resource.setType(fileStat.isFile() ? LocalResourceType.FILE : LocalResourceType.ARCHIVE);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    return resource;
}