Example usage for org.apache.hadoop.fs Path SEPARATOR

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path SEPARATOR.

Prototype

String SEPARATOR

To view the source code for org.apache.hadoop.fs Path SEPARATOR.

Click Source Link

Document

The directory separator, a slash.

Usage

From source file:com.datatorrent.lib.io.fs.FileMergerTest.java

License:Apache License

@Test
public void testBlocksPath() {
    Assert.assertEquals("Blocks path not initialized in application context",
            context.getValue(DAGContext.APPLICATION_PATH) + Path.SEPARATOR + BlockWriter.DEFAULT_BLOCKS_DIR
                    + Path.SEPARATOR,
            testFM.blocksDir);//www . ja  v a2 s .  com
}

From source file:com.datatorrent.lib.io.fs.FileSplitterInputTest.java

License:Apache License

@Test
public void testScannerFilterForDuplicates() throws InterruptedException {
    String filePath = testMeta.dataDirectory + Path.SEPARATOR + "file0.txt";
    testMeta.scanner = new MockScanner();
    testMeta.fileSplitterInput.setScanner(testMeta.scanner);
    testMeta.fileSplitterInput.getScanner().setScanIntervalMillis(500);
    testMeta.fileSplitterInput.getScanner().setFilePatternRegularExp(".*[.]txt");
    testMeta.fileSplitterInput.getScanner().setFiles(filePath);

    testMeta.fileSplitterInput.setup(testMeta.context);
    testMeta.fileSplitterInput.beginWindow(1);
    testMeta.scanner.semaphore.acquire();

    testMeta.fileSplitterInput.emitTuples();
    testMeta.fileSplitterInput.endWindow();

    testMeta.fileSplitterInput.beginWindow(2);
    testMeta.scanner.semaphore.release();
    testMeta.scanner.semaphore.acquire();
    testMeta.fileSplitterInput.emitTuples();
    testMeta.fileSplitterInput.endWindow();

    Assert.assertEquals("File metadata", 1, testMeta.fileMetadataSink.collectedTuples.size());
    for (Object fileMetadata : testMeta.fileMetadataSink.collectedTuples) {
        FileSplitterInput.FileMetadata metadata = (FileSplitterInput.FileMetadata) fileMetadata;
        Assert.assertTrue("path: " + metadata.getFilePath(),
                testMeta.filePaths.contains(metadata.getFilePath()));
        Assert.assertNotNull("name: ", metadata.getFileName());
    }/*from   ww  w  . j  a  v  a2  s .co  m*/

    testMeta.fileMetadataSink.collectedTuples.clear();
    testMeta.fileSplitterInput.teardown();
}

From source file:com.datatorrent.lib.io.fs.FileStitcher.java

License:Apache License

@Override
public void setup(Context.OperatorContext context) {

    blocksDirectoryPath = context.getValue(DAGContext.APPLICATION_PATH) + Path.SEPARATOR + blocksDirectory;

    try {/*  w w w. j av a 2 s.c  o m*/
        outputFS = getOutputFSInstance();
        outputFS.setWriteChecksum(writeChecksum);
    } catch (IOException ex) {
        throw new RuntimeException("Exception in getting output file system.", ex);
    }
    try {
        appFS = getAppFSInstance();
    } catch (IOException ex) {
        try {
            outputFS.close();
        } catch (IOException e) {
            throw new RuntimeException("Exception in closing output file system.", e);
        }
        throw new RuntimeException("Exception in getting application file system.", ex);
    }

    super.setup(context); // Calling it at the end as the reconciler thread uses resources allocated above.
}

From source file:com.datatorrent.lib.io.fs.HDFSFileMerger.java

License:Apache License

/**
 * Attempt for recovery if block concat is successful but temp file is not
 * moved to final file//from   w ww  .j  a va  2 s .c o  m
 * 
 * @param outputFileMetadata
 * @throws IOException
 */
@VisibleForTesting
protected boolean recover(OutputFileMetadata outputFileMetadata) throws IOException {
    Path firstBlockPath = new Path(blocksDirectoryPath + Path.SEPARATOR + outputFileMetadata.getBlockIds()[0]);
    Path outputFilePath = new Path(filePath, outputFileMetadata.getRelativePath());
    if (appFS.exists(firstBlockPath)) {
        FileStatus status = appFS.getFileStatus(firstBlockPath);
        if (status.getLen() == outputFileMetadata.getFileLength()) {
            moveToFinalFile(firstBlockPath, outputFilePath);
            return true;
        }
        LOG.error("Unable to recover in FileMerger for file: {}", outputFilePath);
        return false;
    }

    if (outputFS.exists(outputFilePath)) {
        LOG.debug("Output file already present at the destination, nothing to recover.");
        return true;
    }
    LOG.error("Unable to recover in FileMerger for file: {}", outputFilePath);
    return false;
}

From source file:com.datatorrent.stram.client.EventsAgent.java

License:Apache License

private String getEventsDirectory(String appId) {
    String appPath = stramAgent.getAppPath(appId);
    if (appPath == null) {
        return null;
    }/*from  www  .j av a2 s .com*/
    return appPath + Path.SEPARATOR + "events";
}

From source file:com.datatorrent.stram.client.RecordingsAgent.java

License:Apache License

public String getRecordingsDirectory(String appId, String opId) {
    return getRecordingsDirectory(appId) + Path.SEPARATOR + opId;
}

From source file:com.datatorrent.stram.client.RecordingsAgent.java

License:Apache License

public String getRecordingsDirectory(String appId) {
    String appPath = stramAgent.getAppPath(appId);
    if (appPath == null) {
        return null;
    }//from w  w w . j ava2  s. c o  m
    return appPath + Path.SEPARATOR + "recordings";
}

From source file:com.datatorrent.stram.client.RecordingsAgent.java

License:Apache License

public String getRecordingDirectory(String appId, String opId, String id) {
    String dir = getRecordingsDirectory(appId, opId);
    return (dir == null) ? null : dir + Path.SEPARATOR + id;
}

From source file:com.datatorrent.stram.client.StatsAgent.java

License:Apache License

public String getOperatorStatsDirectory(String appId, String opName) {
    return getStatsDirectory(appId) + Path.SEPARATOR + "operators" + Path.SEPARATOR + opName;
}

From source file:com.datatorrent.stram.client.StatsAgent.java

License:Apache License

public String getContainerStatsDirectory(String appId) {
    return getStatsDirectory(appId) + Path.SEPARATOR + "containers";
}