Example usage for org.apache.hadoop.fs Path SEPARATOR

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path SEPARATOR.

Prototype

String SEPARATOR

To view the source code for org.apache.hadoop.fs Path SEPARATOR.

Click Source Link

Document

The directory separator, a slash.

Usage

From source file:org.apache.apex.malhar.lib.join.AbstractManagedStateInnerJoinOperator.java

License:Apache License

@Override
public void setup(Context.OperatorContext context) {
    super.setup(context);
    ((FileAccessFSImpl) stream1Store.getFileAccess())
            .setBasePath(context.getValue(DAG.APPLICATION_PATH) + Path.SEPARATOR + stateDir + Path.SEPARATOR
                    + String.valueOf(context.getId()) + Path.SEPARATOR + stream1State);
    ((FileAccessFSImpl) stream2Store.getFileAccess())
            .setBasePath(context.getValue(DAG.APPLICATION_PATH) + Path.SEPARATOR + stateDir + Path.SEPARATOR
                    + String.valueOf(context.getId()) + Path.SEPARATOR + stream2State);
    stream1Store.getCheckpointManager().setStatePath("managed_state_" + stream1State);
    stream1Store.getCheckpointManager().setStatePath("managed_state_" + stream2State);
    stream1Store.setup(context);/*from w w w. j  a  v a  2 s . com*/
    stream2Store.setup(context);
}

---

From source file:org.apache.apex.malhar.lib.wal.FSWindowDataManager.java

License:Apache License

@Override
public void setup(Context.OperatorContext context) {
    serializationBuffer = new SerializationBuffer(new WindowedBlockStream());
    operatorId = context.getId();//from   w  ww  . j a  v a2s  . c o  m

    if (isStatePathRelativeToAppPath) {
        fullStatePath = context.getValue(DAG.APPLICATION_PATH) + Path.SEPARATOR + statePath;
    } else {
        fullStatePath = statePath;
    }

    try {
        fileContext = FileContextUtils.getFileContext(fullStatePath);
        setupWals(context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID));
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

---

From source file:org.apache.apex.malhar.lib.wal.FSWindowDataManager.java

License:Apache License

private void configureWal(FSWindowReplayWAL wal, int operatorId, boolean updateWalState) throws IOException {
    String operatorDir = fullStatePath + Path.SEPARATOR + operatorId;
    wal.setFilePath(operatorDir + Path.SEPARATOR + WAL_FILE_NAME);
    wal.fileContext = fileContext;//from   w w  w .  j  av a 2  s .  com

    if (updateWalState) {
        if (!wal.fileDescriptors.isEmpty()) {
            SortedSet<Integer> sortedParts = wal.fileDescriptors.keySet();

            wal.walStartPointer = new FileSystemWAL.FileSystemWALPointer(sortedParts.first(), 0);

            FSWindowReplayWAL.FileDescriptor last = wal.fileDescriptors.get(sortedParts.last()).last();
            if (last.isTmp) {
                wal.tempPartFiles.put(last.part, last.filePath.toString());
            }
        }
    }
}

---

From source file:org.apache.apex.malhar.lib.wal.FSWindowDataManager.java

License:Apache License

private void findFiles(FSWindowReplayWAL wal, int operatorId) throws IOException {
    String operatorDir = fullStatePath + Path.SEPARATOR + operatorId;
    Path operatorPath = new Path(operatorDir);
    if (fileContext.util().exists(operatorPath)) {
        RemoteIterator<FileStatus> walFilesIter = fileContext.listStatus(operatorPath);

        while (walFilesIter.hasNext()) {
            FileStatus fileStatus = walFilesIter.next();
            FSWindowReplayWAL.FileDescriptor descriptor = FSWindowReplayWAL.FileDescriptor
                    .create(fileStatus.getPath());
            wal.fileDescriptors.put(descriptor.part, descriptor);
        }//from  w  w  w.j  a va  2 s.  co  m
    }
}

---

From source file:org.apache.apex.malhar.lib.wal.FSWindowDataManager.java

License:Apache License

/**
 * Deletes artifacts for all windows less than equal to committed window id.<p/>
 *
 * {@link FSWindowDataManager} uses {@link FSWindowReplayWAL} to record data which writes to temp part files.
 * The temp part files are finalized only when they are rotated. So when a window is committed, artifacts for
 * windows <= committed window may still be in temporary files. These temporary files are needed for Wal recovery so
 * we do not alter them and we delete a part file completely (opposed to partial deletion) for efficiency.<br/>
 * Therefore, data of a window gets deleted only when it satisfies all the following criteria:
 * <ul>//www.  j a v  a2  s. c om
 *   <li>window <= committed window id</li>
 *   <li>the part file of the artifact is rotated.</li>
 *   <li>the part file doesn't contain artifacts for windows greater than the artifact's window to avoid partial
 *   file deletion.</li>
 * </ul>
 *
 * In addition to this we also delete:
 * <ol>
 *   <li>Some stray temporary files are also deleted which correspond to completely deleted parts.</li>
 *   <li>Once the committed window > largest recovery window, we delete the files of partitions that were removed.</li>
 * </ol>
 *
 * @param committedWindowId   window id
 * @throws IOException
 */
@Override
public void committed(long committedWindowId) throws IOException {
    closeReaders();
    //find the largest window <= committed window id and the part file corresponding to it is finalized.
    Map.Entry<Long, Integer> largestEntryForDeletion = null;

    Iterator<Map.Entry<Long, Integer>> iterator = wal.windowWalParts.entrySet().iterator();

    while (iterator.hasNext()) {
        Map.Entry<Long, Integer> entry = iterator.next();
        //only completely finalized part files are deleted.
        if (entry.getKey() <= committedWindowId && !wal.tempPartFiles.containsKey(entry.getValue())) {
            largestEntryForDeletion = entry;
            iterator.remove();
        }
        if (entry.getKey() > committedWindowId) {
            break;
        }
    }

    if (largestEntryForDeletion != null && !wal.windowWalParts
            .containsValue(largestEntryForDeletion.getValue()) /* no artifacts for higher window present*/) {

        int highestPartToDelete = largestEntryForDeletion.getValue();
        wal.getWriter().delete(new FileSystemWAL.FileSystemWALPointer(highestPartToDelete + 1, 0));

        //also delete any old stray temp files that correspond to parts < deleteTillPointer.partNum
        Iterator<Map.Entry<Integer, FSWindowReplayWAL.FileDescriptor>> fileIterator = wal.fileDescriptors
                .entries().iterator();
        while (fileIterator.hasNext()) {
            Map.Entry<Integer, FSWindowReplayWAL.FileDescriptor> entry = fileIterator.next();
            if (entry.getKey() <= highestPartToDelete && entry.getValue().isTmp) {
                if (fileContext.util().exists(entry.getValue().filePath)) {
                    fileContext.delete(entry.getValue().filePath, true);
                }
            } else if (entry.getKey() > highestPartToDelete) {
                break;
            }
        }
    }

    //delete data of partitions that have been removed
    if (deletedOperators != null) {
        Iterator<Integer> operatorIter = deletedOperators.iterator();

        while (operatorIter.hasNext()) {
            int deletedOperatorId = operatorIter.next();
            FSWindowReplayWAL wal = readOnlyWals.get(deletedOperatorId);
            if (committedWindowId > largestCompletedWindow) {
                Path operatorDir = new Path(fullStatePath + Path.SEPARATOR + deletedOperatorId);

                if (fileContext.util().exists(operatorDir)) {
                    fileContext.delete(operatorDir, true);
                }
                wal.teardown();
                operatorIter.remove();
                readOnlyWals.remove(deletedOperatorId);
            }
        }

        if (deletedOperators.isEmpty()) {
            deletedOperators = null;
        }
    }
}

---

From source file:org.apache.asterix.aoya.AsterixApplicationMaster.java

License:Apache License

/**
 * Here I am just pointing the Containers to the exisiting HDFS resources given by the Client
 * filesystem of the nodes.//from  ww  w.j  a va  2s  .co m
 *
 * @throws IOException
 */
private void localizeDFSResources() throws IOException {
    //if performing an 'offline' task, skip a lot of resource distribution
    if (obliterate || backup || restore) {
        if (appMasterJar == null || ("").equals(appMasterJar)) {
            //this can happen in a jUnit testing environment. we don't need to set it there.
            if (!conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
                throw new IllegalStateException("AM jar not provided in environment.");
            } else {
                return;
            }
        }
        FileSystem fs = FileSystem.get(conf);
        FileStatus appMasterJarStatus = fs.getFileStatus(appMasterJar);
        LocalResource obliteratorJar = Records.newRecord(LocalResource.class);
        obliteratorJar.setType(LocalResourceType.FILE);
        obliteratorJar.setVisibility(LocalResourceVisibility.PRIVATE);
        obliteratorJar.setResource(ConverterUtils.getYarnUrlFromPath(appMasterJar));
        obliteratorJar.setTimestamp(appMasterJarStatus.getModificationTime());
        obliteratorJar.setSize(appMasterJarStatus.getLen());
        localResources.put("asterix-yarn.jar", obliteratorJar);
        LOG.info(localResources.values());
        return;
    }
    //otherwise, distribute evertything to start up asterix

    LocalResource asterixZip = Records.newRecord(LocalResource.class);

    //this un-tar's the asterix distribution
    asterixZip.setType(LocalResourceType.ARCHIVE);

    asterixZip.setVisibility(LocalResourceVisibility.PRIVATE);
    try {
        asterixZip.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixZipPath)));

    } catch (URISyntaxException e) {
        LOG.error("Error locating Asterix zip" + " in env, path=" + asterixZipPath);
        throw new IOException(e);
    }

    asterixZip.setTimestamp(asterixZipTimestamp);
    asterixZip.setSize(asterixZipLen);
    localResources.put(ASTERIX_ZIP_NAME, asterixZip);

    //now let's do the same for the cluster description XML
    LocalResource asterixConf = Records.newRecord(LocalResource.class);
    asterixConf.setType(LocalResourceType.FILE);

    asterixConf.setVisibility(LocalResourceVisibility.PRIVATE);
    try {
        asterixConf.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixConfPath)));
    } catch (URISyntaxException e) {
        LOG.error("Error locating Asterix config" + " in env, path=" + asterixConfPath);
        throw new IOException(e);
    }
    //TODO: I could avoid localizing this everywhere by only calling this block on the metadata node.
    asterixConf.setTimestamp(asterixConfTimestamp);
    asterixConf.setSize(asterixConfLen);
    localResources.put("cluster-config.xml", asterixConf);
    //now add the libraries if there are any
    try {
        FileSystem fs = FileSystem.get(conf);
        Path p = new Path(dfsBasePath, instanceConfPath + File.separator + "library" + Path.SEPARATOR);
        if (fs.exists(p)) {
            FileStatus[] dataverses = fs.listStatus(p);
            for (FileStatus d : dataverses) {
                if (!d.isDirectory())
                    throw new IOException("Library configuration directory structure is incorrect");
                FileStatus[] libraries = fs.listStatus(d.getPath());
                for (FileStatus l : libraries) {
                    if (l.isDirectory())
                        throw new IOException("Library configuration directory structure is incorrect");
                    LocalResource lr = Records.newRecord(LocalResource.class);
                    lr.setResource(ConverterUtils.getYarnUrlFromURI(l.getPath().toUri()));
                    lr.setSize(l.getLen());
                    lr.setTimestamp(l.getModificationTime());
                    lr.setType(LocalResourceType.ARCHIVE);
                    lr.setVisibility(LocalResourceVisibility.PRIVATE);
                    localResources.put("library" + Path.SEPARATOR + d.getPath().getName() + Path.SEPARATOR
                            + l.getPath().getName().split("\\.")[0], lr);
                    LOG.info("Found library: " + l.getPath().toString());
                    LOG.info(l.getPath().getName());
                }
            }
        }
    } catch (FileNotFoundException e) {
        LOG.info("No external libraries present");
        //do nothing, it just means there aren't libraries. that is possible and ok
        // it should be handled by the fs.exists(p) check though.
    }
    LOG.info(localResources.values());

}

---

From source file:org.apache.drill.exec.impersonation.BaseTestImpersonation.java

License:Apache License

/**
 * Start a MiniDFS cluster backed Drillbit cluster
 * @param testClass//from  w ww . ja va2  s. c  o  m
 * @param isImpersonationEnabled Enable impersonation in the cluster?
 * @throws Exception
 */
protected static void startMiniDfsCluster(final String testClass, final boolean isImpersonationEnabled)
        throws Exception {
    Preconditions.checkArgument(!Strings.isNullOrEmpty(testClass),
            "Expected a non-null and non-empty test class name");
    dfsConf = new Configuration();

    // Set the MiniDfs base dir to be the temp directory of the test, so that all files created within the MiniDfs
    // are properly cleanup when test exits.
    miniDfsStoragePath = System.getProperty("java.io.tmpdir") + Path.SEPARATOR + testClass;
    dfsConf.set("hdfs.minidfs.basedir", miniDfsStoragePath);

    if (isImpersonationEnabled) {
        // Set the proxyuser settings so that the user who is running the Drillbits/MiniDfs can impersonate other users.
        dfsConf.set(String.format("hadoop.proxyuser.%s.hosts", processUser), "*");
        dfsConf.set(String.format("hadoop.proxyuser.%s.groups", processUser), "*");
    }

    // Start the MiniDfs cluster
    dfsCluster = new MiniDFSCluster.Builder(dfsConf).numDataNodes(3).format(true).build();

    fs = dfsCluster.getFileSystem();
}

---

From source file:org.apache.drill.exec.impersonation.TestImpersonationMetadata.java

License:Apache License

private static void testCreateTableTestHelper(String user, String tableWS, String tableName) throws Exception {
    try {/*ww  w  .  j  a v a2s  .c  o m*/
        updateClient(user);

        test("USE " + Joiner.on(".").join(MINIDFS_STORAGE_PLUGIN_NAME, tableWS));

        test("CREATE TABLE " + tableName + " AS SELECT "
                + "c_custkey, c_nationkey FROM cp.`tpch/customer.parquet` ORDER BY c_custkey;");

        test("SHOW FILES");

        testBuilder().sqlQuery("SELECT * FROM " + tableName + " LIMIT 1").ordered()
                .baselineColumns("c_custkey", "c_nationkey").baselineValues(1, 15).go();

    } finally {
        // There is no drop table, we need to delete the table directory through FileSystem object
        final Path tablePath = new Path(Path.SEPARATOR + tableWS + Path.SEPARATOR + tableName);
        if (fs.exists(tablePath)) {
            fs.delete(tablePath, true);
        }
    }
}

---

From source file:org.apache.drill.exec.impersonation.TestImpersonationMetadata.java

License:Apache License

@Test
public void testRefreshMetadata() throws Exception {
    final String tableName = "nation1";
    final String tableWS = "drillTestGrp1_700";

    updateClient(user1);/*from   w ww.java  2  s. com*/
    test("USE " + Joiner.on(".").join(MINIDFS_STORAGE_PLUGIN_NAME, tableWS));

    test("CREATE TABLE " + tableName + " partition by (n_regionkey) AS SELECT * "
            + "FROM cp.`tpch/nation.parquet`;");

    test("refresh table metadata " + tableName + ";");

    test("SELECT * FROM " + tableName + ";");

    final Path tablePath = new Path(Path.SEPARATOR + tableWS + Path.SEPARATOR + tableName);
    assertTrue(fs.exists(tablePath) && fs.isDirectory(tablePath));
    fs.mkdirs(new Path(tablePath, "tmp5"));

    test("SELECT * from " + tableName + ";");

}

---

From source file:org.apache.drill.exec.store.parquet.TestParquetMetadataCache.java

License:Apache License

@BeforeClass
public static void copyData() throws Exception {
    // copy the data into the temporary location
    String tmpLocation = getDfsTestTmpSchemaLocation();
    File dataDir1 = new File(tmpLocation + Path.SEPARATOR + tableName1);
    dataDir1.mkdir();//from   w w w  . j  av a2s .  com
    FileUtils.copyDirectory(new File(String.format(String.format("%s/multilevel/parquet", TEST_RES_PATH))),
            dataDir1);

    File dataDir2 = new File(tmpLocation + Path.SEPARATOR + tableName2);
    dataDir2.mkdir();
    FileUtils.copyDirectory(new File(String.format(String.format("%s/multilevel/parquet2", TEST_RES_PATH))),
            dataDir2);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »