Example usage for org.apache.hadoop.fs FileStatus isFile

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus isFile.

Prototype

public boolean isFile() 

Source Link

Document

Is this a file?

Usage

From source file:org.springframework.data.hadoop.store.output.PartitionTextFileWriterSmokeTests.java

License:Apache License

private long getTotalWritten(String path) {
    @SuppressWarnings("resource")
    FsShell shell = new FsShell(hadoopConfiguration);
    long total = 0;
    for (FileStatus s : shell.ls(true, path)) {
        if (s.isFile()) {
            total += s.getLen();//  www  .  jav  a2 s .c o  m
        }
    }
    return total;
}

---

From source file:org.springframework.data.hadoop.store.output.TextFileWriterSmokeTests.java

License:Apache License

private long getTotalWritten() {
    @SuppressWarnings("resource")
    FsShell shell = new FsShell(hadoopConfiguration);
    long total = 0;
    for (FileStatus s : shell.ls(true, PATH)) {
        if (s.isFile()) {
            total += s.getLen();/*w  ww.  ja  va2s .c om*/
        }
    }
    return total;
}

---

From source file:org.springframework.yarn.batch.item.HdfsFileSplitItemReaderIntegrationTests.java

License:Apache License

@Test
public void testWithOneReader() throws Exception {
    createDataFile(PDATA1, "data1-line-", 100);
    Resource resource = resourceLoader.getResource(DATA1);

    FileStatus fileStatus = fs.getFileStatus(PDATA1);
    assertThat(fileStatus, notNullValue());
    assertThat(fileStatus.isFile(), is(true));

    HdfsFileSplitItemReader<String> itemReader = createItemReader(resource, 0, fileStatus.getLen());

    readAndTestRange(itemReader, "data1-line-", 1, 100);
    itemReader.close();/* w w w.j  av a2  s. co m*/
}

---

From source file:org.springframework.yarn.client.ClientLocalizerIntegrationTests.java

License:Apache License

@Test
public void testDistributeWithCopy() throws IOException {
    assertThat(client, notNullValue());/*from w w  w  .j  a v  a2 s.c  o m*/

    localizer.setStagingDirectory(new Path("/syarn-tmp/ClientLocalizerIntegrationTests/1"));
    localizer.distribute();

    FileStatus fileStatus = fs
            .getFileStatus(new Path("/syarn-tmp/ClientLocalizerIntegrationTests/1/hadoop.properties"));
    assertThat(fileStatus, notNullValue());
    assertThat(fileStatus.isFile(), is(true));

    Map<String, LocalResource> resources = localizer.getResources();
    assertThat(resources, notNullValue());
    LocalResource localResource = resources.get("hadoop.properties");
    assertThat(localResource, notNullValue());
    URL resource = localResource.getResource();
    assertThat(resource, notNullValue());
    assertThat(resource.getFile(), is("/syarn-tmp/ClientLocalizerIntegrationTests/1/hadoop.properties"));
}

---

From source file:org.springframework.yarn.fs.DefaultResourceLocalizer.java

License:Apache License

/**
 * Gets a map of localized resources.//  w  w  w  .j  av  a  2s . c  o m
 *
 * @param fs the file system
 * @return a map of localized resources
 * @throws IOException if problem occurred getting file status
 * @throws URISyntaxException if file path is wrong
 */
protected Map<String, LocalResource> doFileTransfer(FileSystem fs) throws IOException, URISyntaxException {
    Map<String, LocalResource> returned = new HashMap<String, LocalResource>();
    Path resolvedStagingDirectory = resolveStagingDirectory();
    for (TransferEntry e : transferEntries) {
        Path remotePath = (!e.staging) ? new Path(e.remote + e.path)
                : new Path(e.remote + resolvedStagingDirectory.toUri().getPath() + e.path);
        URI localUri = new URI(e.local);
        FileStatus[] fileStatuses = fs.globStatus(remotePath);
        if (log.isDebugEnabled()) {
            log.debug("Trying path " + remotePath + " glob fileStatus length="
                    + (fileStatuses != null ? fileStatuses.length : "null"));
        }
        if (!ObjectUtils.isEmpty(fileStatuses)) {
            for (FileStatus status : fileStatuses) {
                if (log.isDebugEnabled()) {
                    log.debug("FileStatus=" + status);
                }
                if (status.isFile()) {
                    URI remoteUri = status.getPath().toUri();
                    Path path = new Path(new Path(localUri), remoteUri.getPath());
                    LocalResource res = Records.newRecord(LocalResource.class);
                    res.setType(e.type);
                    res.setVisibility(e.visibility);
                    res.setResource(ConverterUtils.getYarnUrlFromPath(path));
                    res.setTimestamp(status.getModificationTime());
                    res.setSize(status.getLen());
                    if (log.isDebugEnabled()) {
                        log.debug("Using remote uri [" + remoteUri + "] and local uri [" + localUri
                                + "] converted to path [" + path + "]");
                    }
                    returned.put(status.getPath().getName(), res);
                }
            }
        }
    }
    return returned;
}

---

From source file:org.springframework.yarn.fs.DefaultResourceLocalizerTests.java

License:Apache License

@Test
public void testSimpleCopy() throws Exception {
    String dir = "/DefaultResourceLocalizerTests-testSimpleCopy";
    LocalResourcesFactoryBean factory = new LocalResourcesFactoryBean();
    factory.setConfiguration(configuration);

    List<CopyEntry> copyEntries = new ArrayList<CopyEntry>();
    CopyEntry entry = new CopyEntry("classpath:/test-site-1.xml", dir, false);
    copyEntries.add(entry);/*from   w  w w .  java2s  . c  o m*/
    factory.setCopyEntries(copyEntries);
    factory.setHdfsEntries(new ArrayList<LocalResourcesFactoryBean.TransferEntry>());
    factory.afterPropertiesSet();

    SmartResourceLocalizer localizer = (SmartResourceLocalizer) factory.getObject();
    localizer.copy();

    FileSystem fs = FileSystem.get(configuration);
    FileStatus fileStatus = fs.getFileStatus(new Path(dir + "/test-site-1.xml"));
    assertThat(fileStatus.isFile(), is(true));
    assertThat(fileStatus.getLen(), greaterThan(0l));
}

---

From source file:org.springframework.yarn.fs.DefaultResourceLocalizerTests.java

License:Apache License

@Test
public void testSimpleCopyHomeStaging() throws Exception {
    String dir = "DefaultResourceLocalizerTests-testSimpleCopyHomeStaging";
    LocalResourcesFactoryBean factory = new LocalResourcesFactoryBean();
    factory.setConfiguration(configuration);

    List<CopyEntry> copyEntries = new ArrayList<CopyEntry>();
    CopyEntry entry = new CopyEntry("classpath:/test-site-1.xml", null, true);
    copyEntries.add(entry);//  w w w.ja v  a2s  .  co  m
    factory.setCopyEntries(copyEntries);
    factory.setHdfsEntries(new ArrayList<LocalResourcesFactoryBean.TransferEntry>());
    factory.afterPropertiesSet();

    SmartResourceLocalizer localizer = (SmartResourceLocalizer) factory.getObject();
    localizer.setStagingDirectory(new Path(dir));
    localizer.setStagingId("foo-id");
    localizer.copy();

    FileSystem fs = FileSystem.get(configuration);
    FileStatus fileStatus = fs.getFileStatus(new Path(dir + "/foo-id/test-site-1.xml"));
    assertThat(fileStatus.isFile(), is(true));
    assertThat(fileStatus.getLen(), greaterThan(0l));
}

---

From source file:org.springframework.yarn.fs.DefaultResourceLocalizerTests.java

License:Apache License

@Test
public void testDistributeGlobalCustomStaging() throws Exception {
    LocalResourcesFactoryBean factory = new LocalResourcesFactoryBean();
    factory.setConfiguration(configuration);

    List<CopyEntry> copyEntries = new ArrayList<CopyEntry>();
    CopyEntry entry = new CopyEntry("classpath:/test-site-1.xml", null, true);
    copyEntries.add(entry);//  ww  w.j  a  va2  s.c o m
    factory.setCopyEntries(copyEntries);

    List<TransferEntry> transferEntries = new ArrayList<TransferEntry>();
    TransferEntry tEntry = new TransferEntry(null, null, "/test-site-1.xml", true);
    transferEntries.add(tEntry);
    factory.setHdfsEntries(transferEntries);

    factory.afterPropertiesSet();
    SmartResourceLocalizer localizer = (SmartResourceLocalizer) factory.getObject();

    localizer.setStagingDirectory(new Path("/tmp/foo"));
    localizer.setStagingId("foo-id");
    localizer.distribute();

    Map<String, LocalResource> resources = localizer.getResources();
    assertThat(resources, notNullValue());
    assertThat(resources.size(), is(1));

    FileSystem fs = FileSystem.get(configuration);
    FileStatus fileStatus = fs.getFileStatus(new Path("/tmp/foo/foo-id/test-site-1.xml"));
    assertThat(fileStatus.isFile(), is(true));
    assertThat(fileStatus.getLen(), greaterThan(0l));

}

---

From source file:org.springframework.yarn.fs.DefaultResourceLocalizerTests.java

License:Apache License

@Test
public void testDistributeGlobalDefaultStaging() throws Exception {
    LocalResourcesFactoryBean factory = new LocalResourcesFactoryBean();
    factory.setConfiguration(configuration);

    List<CopyEntry> copyEntries = new ArrayList<CopyEntry>();
    CopyEntry entry = new CopyEntry("classpath:/test-site-1.xml", null, true);
    copyEntries.add(entry);/*  w  ww.  j a  v  a  2 s. c o m*/
    factory.setCopyEntries(copyEntries);

    List<TransferEntry> transferEntries = new ArrayList<TransferEntry>();
    TransferEntry tEntry = new TransferEntry(null, null, "/test-site-1.xml", true);
    transferEntries.add(tEntry);
    factory.setHdfsEntries(transferEntries);
    factory.afterPropertiesSet();

    SmartResourceLocalizer localizer = (SmartResourceLocalizer) factory.getObject();

    localizer.setStagingId("foo-id");

    localizer.distribute();

    Map<String, LocalResource> resources = localizer.getResources();
    assertThat(resources, notNullValue());
    assertThat(resources.size(), is(1));

    FileSystem fs = FileSystem.get(configuration);
    FileStatus fileStatus = fs.getFileStatus(new Path("/syarn/staging/foo-id/test-site-1.xml"));
    assertThat(fileStatus.isFile(), is(true));
    assertThat(fileStatus.getLen(), greaterThan(0l));
}

---

From source file:org.springframework.yarn.fs.DefaultResourceLocalizerTests.java

License:Apache License

@Test
public void testDistributeHomeStaging() throws Exception {
    String dir = "DefaultResourceLocalizerTests-testDistributeHomeStaging";
    LocalResourcesFactoryBean factory = new LocalResourcesFactoryBean();
    factory.setConfiguration(configuration);

    List<CopyEntry> copyEntries = new ArrayList<CopyEntry>();
    CopyEntry entry = new CopyEntry("classpath:/test-site-1.xml", null, true);
    copyEntries.add(entry);//w ww . j av a2 s  .  c  o m
    factory.setCopyEntries(copyEntries);

    List<TransferEntry> transferEntries = new ArrayList<TransferEntry>();
    TransferEntry tEntry = new TransferEntry(null, null, "/test-site-1.xml", true);
    transferEntries.add(tEntry);
    factory.setHdfsEntries(transferEntries);
    factory.afterPropertiesSet();

    SmartResourceLocalizer localizer = (SmartResourceLocalizer) factory.getObject();

    localizer.setStagingDirectory(new Path(dir));
    localizer.setStagingId("foo-id");
    localizer.distribute();

    Map<String, LocalResource> resources = localizer.getResources();
    assertThat(resources, notNullValue());
    assertThat(resources.size(), is(1));

    FileSystem fs = FileSystem.get(configuration);
    FileStatus fileStatus = fs.getFileStatus(new Path(dir + "/foo-id/test-site-1.xml"));
    assertThat(fileStatus.isFile(), is(true));
    assertThat(fileStatus.getLen(), greaterThan(0l));

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• »