Example usage for org.apache.hadoop.fs FileSystem getUri

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getUri.

Prototype

public abstract URI getUri();

Source Link

Document

Returns a URI which identifies this FileSystem.

Usage

From source file:org.apache.blur.mapreduce.lib.update.DriverTest.java

License:Apache License

@Test
public void testDriverAddSingleRecordToExistingRow() throws Exception {
    FileSystem fileSystem = miniCluster.getFileSystem();
    Path root = new Path(fileSystem.getUri() + "/");

    String tableName = "testDriverAddSingleRecordToExistingRow";
    Iface client = getClient();/*from   w w w  .  j  a v a2s  . c  o  m*/
    creatTable(tableName, new Path(root, "tables"), true);
    addRow(client, tableName, "row1", "record1", "value1");

    Driver driver = new Driver();
    driver.setConf(conf);

    String mrIncWorkingPathStr = new Path(root, "working").toString();
    generateData(mrIncWorkingPathStr);
    String outputPathStr = new Path(root, "output").toString();
    String blurZkConnection = miniCluster.getZkConnectionString();

    assertEquals(0,
            driver.run(new String[] { tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1" }));

    client.loadData(tableName, outputPathStr);

    waitUntilAllImportsAreCompleted(client, tableName);

    TableStats tableStats = client.tableStats(tableName);
    assertEquals(1, tableStats.getRowCount());
    assertEquals(2, tableStats.getRecordCount());
}

---

From source file:org.apache.blur.mapreduce.lib.update.DriverTest.java

License:Apache License

@Test
public void testDriverUpdateRecordToExistingRow() throws Exception {
    FileSystem fileSystem = miniCluster.getFileSystem();
    Path root = new Path(fileSystem.getUri() + "/");

    String tableName = "testDriverUpdateRecordToExistingRow";
    Iface client = getClient();/*from   w  w w.  j  a  va2  s .  c  o m*/
    creatTable(tableName, new Path(root, "tables"), true);
    String rowId = "row1";
    String recordId = "record1";
    addRow(client, tableName, rowId, recordId, "value1");

    Driver driver = new Driver();
    driver.setConf(conf);

    String mrIncWorkingPathStr = new Path(root, "working").toString();
    generateData(mrIncWorkingPathStr, rowId, recordId, "value2");
    String outputPathStr = new Path(root, "output").toString();
    String blurZkConnection = miniCluster.getZkConnectionString();

    assertEquals(0,
            driver.run(new String[] { tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1" }));
    {
        Selector selector = new Selector();
        selector.setRowId(rowId);
        FetchResult fetchRow = client.fetchRow(tableName, selector);
        Row row = fetchRow.getRowResult().getRow();
        assertEquals(rowId, row.getId());
        List<Record> records = row.getRecords();
        assertEquals(1, records.size());
        Record record = records.get(0);
        assertEquals(recordId, record.getRecordId());
        List<Column> columns = record.getColumns();
        assertEquals(1, columns.size());
        Column column = columns.get(0);
        assertEquals("col0", column.getName());
        assertEquals("value1", column.getValue());
    }

    client.loadData(tableName, outputPathStr);

    waitUntilAllImportsAreCompleted(client, tableName);

    TableStats tableStats = client.tableStats(tableName);
    assertEquals(1, tableStats.getRowCount());
    assertEquals(1, tableStats.getRecordCount());

    {
        Selector selector = new Selector();
        selector.setRowId(rowId);
        FetchResult fetchRow = client.fetchRow(tableName, selector);
        Row row = fetchRow.getRowResult().getRow();
        assertEquals(rowId, row.getId());
        List<Record> records = row.getRecords();
        assertEquals(1, records.size());
        Record record = records.get(0);
        assertEquals(recordId, record.getRecordId());
        List<Column> columns = record.getColumns();
        assertEquals(1, columns.size());
        Column column = columns.get(0);
        assertEquals("col0", column.getName());
        assertEquals("value2", column.getValue());
    }
}

---

From source file:org.apache.blur.mapreduce.lib.update.DriverTest.java

License:Apache License

@Test
public void testBulkTableUpdateCommandUpdateRecordToExistingRow() throws Exception {
    FileSystem fileSystem = miniCluster.getFileSystem();
    Path root = new Path(fileSystem.getUri() + "/");

    String tableName = "testBulkTableUpdateCommandUpdateRecordToExistingRow";
    Iface client = getClient();/*from w w w .  j a  va2  s.  co m*/
    Path mrIncWorkingPath = new Path(new Path(root, "working"), tableName);
    creatTable(tableName, new Path(root, "tables"), true, mrIncWorkingPath.toString());
    String rowId = "row1";
    String recordId = "record1";
    addRow(client, tableName, rowId, recordId, "value1");

    generateData(mrIncWorkingPath.toString(), rowId, recordId, "value2");

    {
        Selector selector = new Selector();
        selector.setRowId(rowId);
        FetchResult fetchRow = client.fetchRow(tableName, selector);
        Row row = fetchRow.getRowResult().getRow();
        assertEquals(rowId, row.getId());
        List<Record> records = row.getRecords();
        assertEquals(1, records.size());
        Record record = records.get(0);
        assertEquals(recordId, record.getRecordId());
        List<Column> columns = record.getColumns();
        assertEquals(1, columns.size());
        Column column = columns.get(0);
        assertEquals("col0", column.getName());
        assertEquals("value1", column.getValue());
    }

    BulkTableUpdateCommand bulkTableUpdateCommand = new BulkTableUpdateCommand();
    bulkTableUpdateCommand.setAutoLoad(true);
    bulkTableUpdateCommand.setTable(tableName);
    bulkTableUpdateCommand.setWaitForDataBeVisible(true);
    bulkTableUpdateCommand.addExtraConfig(conf);
    assertEquals(0, (int) bulkTableUpdateCommand.run(getClient()));

    TableStats tableStats = client.tableStats(tableName);
    assertEquals(1, tableStats.getRowCount());
    assertEquals(1, tableStats.getRecordCount());

    {
        Selector selector = new Selector();
        selector.setRowId(rowId);
        FetchResult fetchRow = client.fetchRow(tableName, selector);
        Row row = fetchRow.getRowResult().getRow();
        assertEquals(rowId, row.getId());
        List<Record> records = row.getRecords();
        assertEquals(1, records.size());
        Record record = records.get(0);
        assertEquals(recordId, record.getRecordId());
        List<Column> columns = record.getColumns();
        assertEquals(1, columns.size());
        Column column = columns.get(0);
        assertEquals("col0", column.getName());
        assertEquals("value2", column.getValue());
    }
}

---

From source file:org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry.java

License:Apache License

/**
 * Concatenates udf are with root directory.
 * Creates udf area, if area does not exist.
 * Checks if area exists and is directory, if it is writable for current user,
 * throws {@link DrillRuntimeException} otherwise.
 *
 * @param fs file system where area should be created or checked
 * @param root root directory/* w  w w . j  a v  a 2 s  . c om*/
 * @param directory directory path
 * @return path to area
 */
private Path createArea(FileSystem fs, String root, String directory) {
    Path path = new Path(new File(root, directory).toURI().getPath());
    String fullPath = path.toUri().getPath();
    try {
        fs.mkdirs(path);
        Preconditions.checkState(fs.exists(path), "Area [%s] must exist", fullPath);
        FileStatus fileStatus = fs.getFileStatus(path);
        Preconditions.checkState(fileStatus.isDirectory(), "Area [%s] must be a directory", fullPath);
        FsPermission permission = fileStatus.getPermission();
        // It is considered that process user has write rights on directory if:
        // 1. process user is owner of the directory and has write rights
        // 2. process user is in group that has write rights
        // 3. any user has write rights
        Preconditions.checkState(
                (ImpersonationUtil.getProcessUserName().equals(fileStatus.getOwner())
                        && permission.getUserAction().implies(FsAction.WRITE))
                        || (Sets.newHashSet(ImpersonationUtil.getProcessUserGroupNames()).contains(
                                fileStatus.getGroup()) && permission.getGroupAction().implies(FsAction.WRITE))
                        || permission.getOtherAction().implies(FsAction.WRITE),
                "Area [%s] must be writable and executable for application user", fullPath);
    } catch (Exception e) {
        if (e instanceof DrillRuntimeException) {
            throw (DrillRuntimeException) e;
        }
        // throws
        DrillRuntimeException.format(e, "Error during udf area creation [%s] on file system [%s]", fullPath,
                fs.getUri());
    }
    logger.info("Created remote udf area [{}] on file system [{}]", fullPath, fs.getUri());
    return path;
}

---

From source file:org.apache.drill.exec.rpc.user.UserSession.java

License:Apache License

/** On session close deletes all session temporary locations recursively and clears temporary locations list. */
@Override//from w  ww  . java 2 s  . c o  m
public void close() {
    for (Map.Entry<Path, FileSystem> entry : temporaryLocations.entrySet()) {
        Path path = entry.getKey();
        FileSystem fs = entry.getValue();
        try {
            fs.delete(path, true);
            logger.info("Deleted session temporary location [{}] from file system [{}]", path.toUri().getPath(),
                    fs.getUri());
        } catch (Exception e) {
            logger.warn("Error during session temporary location [{}] deletion from file system [{}]: [{}]",
                    path.toUri().getPath(), fs.getUri(), e.getMessage());
        }
    }
    temporaryLocations.clear();
}

---

From source file:org.apache.drill.exec.store.StorageStrategy.java

License:Apache License

/**
 * Creates passed file on appropriate file system.
 * Before creation checks which parent directories do not exists.
 * Applies storage strategy rules to all newly created directories and file.
 * Will return first created parent path or file if no new parent paths created.
 *
 * Case 1: /a/b -> already exists, attempt to create /a/b/c/some_file.txt
 * Will create file and return /a/b/c./*from  w  w  w .j a va  2 s  .  c  om*/
 * Case 2: /a/b/c -> already exists, attempt to create /a/b/c/some_file.txt
 * Will create file and return /a/b/c/some_file.txt.
 * Case 3: /a/b/c/some_file.txt -> already exists, will fail.
 *
 * @param fs file system where file should be located
 * @param file file path
 * @return first created parent path or file
 * @throws IOException is thrown in case of problems while creating path, setting permission
 *         or adding path to delete on exit list
 */
public Path createFileAndApply(FileSystem fs, Path file) throws IOException {
    List<Path> locations = getNonExistentLocations(fs, file.getParent());
    if (!fs.createNewFile(file)) {
        throw new IOException(String.format("File [%s] already exists on file system [%s].",
                file.toUri().getPath(), fs.getUri()));
    }
    applyToFile(fs, file);

    if (locations.isEmpty()) {
        return file;
    }

    for (Path location : locations) {
        applyStrategy(fs, location, folderPermission, deleteOnExit);
    }
    return locations.get(locations.size() - 1);
}

---

From source file:org.apache.druid.storage.hdfs.HdfsDataSegmentFinder.java

License:Apache License

@Override
public Set<DataSegment> findSegments(String workingDirPathStr, boolean updateDescriptor)
        throws SegmentLoadingException {
    final Map<String, Pair<DataSegment, Long>> timestampedSegments = new HashMap<>();
    final Path workingDirPath = new Path(workingDirPathStr);
    FileSystem fs;
    try {//from  w  w w  . ja  v  a2  s  . c o  m
        fs = workingDirPath.getFileSystem(config);

        log.info(fs.getScheme());
        log.info("FileSystem URI:" + fs.getUri().toString());

        if (!fs.exists(workingDirPath)) {
            throw new SegmentLoadingException("Working directory [%s] doesn't exist.", workingDirPath);
        }

        if (!fs.isDirectory(workingDirPath)) {
            throw new SegmentLoadingException("Working directory [%s] is not a directory!?", workingDirPath);
        }

        final RemoteIterator<LocatedFileStatus> it = fs.listFiles(workingDirPath, true);
        while (it.hasNext()) {
            final LocatedFileStatus locatedFileStatus = it.next();
            final Path path = locatedFileStatus.getPath();
            if (path.getName().endsWith("descriptor.json")) {

                // There are 3 supported path formats:
                //    - hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum/descriptor.json
                //    - hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum_descriptor.json
                //    - hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum_UUID_descriptor.json
                final String descriptorParts[] = path.getName().split("_");

                Path indexZip = new Path(path.getParent(), "index.zip");
                if (descriptorParts.length > 1) {
                    Preconditions
                            .checkState(
                                    descriptorParts.length <= 3
                                            && org.apache.commons.lang.StringUtils.isNumeric(descriptorParts[0])
                                            && "descriptor.json"
                                                    .equals(descriptorParts[descriptorParts.length - 1]),
                                    "Unexpected descriptor filename format [%s]", path);

                    indexZip = new Path(path.getParent(), StringUtils.format("%s_%sindex.zip",
                            descriptorParts[0], descriptorParts.length == 2 ? "" : descriptorParts[1] + "_"));
                }

                if (fs.exists(indexZip)) {
                    final DataSegment dataSegment = mapper.readValue(fs.open(path), DataSegment.class);
                    log.info("Found segment [%s] located at [%s]", dataSegment.getIdentifier(), indexZip);

                    final Map<String, Object> loadSpec = dataSegment.getLoadSpec();
                    final String pathWithoutScheme = indexZip.toUri().getPath();

                    if (!loadSpec.get("type").equals(HdfsStorageDruidModule.SCHEME)
                            || !loadSpec.get("path").equals(pathWithoutScheme)) {
                        loadSpec.put("type", HdfsStorageDruidModule.SCHEME);
                        loadSpec.put("path", pathWithoutScheme);
                        if (updateDescriptor) {
                            log.info("Updating loadSpec in descriptor.json at [%s] with new path [%s]", path,
                                    pathWithoutScheme);
                            mapper.writeValue(fs.create(path, true), dataSegment);
                        }
                    }

                    DataSegmentFinder.putInMapRetainingNewest(timestampedSegments, dataSegment,
                            locatedFileStatus.getModificationTime());
                } else {
                    throw new SegmentLoadingException(
                            "index.zip didn't exist at [%s] while descripter.json exists!?", indexZip);
                }
            }
        }
    } catch (IOException e) {
        throw new SegmentLoadingException(e, "Problems interacting with filesystem[%s].", workingDirPath);
    }

    return timestampedSegments.values().stream().map(x -> x.lhs).collect(Collectors.toSet());
}

---

From source file:org.apache.falcon.hive.util.FileUtils.java

License:Apache License

public static void validatePath(final FileSystem fileSystem, final Path basePath) throws IOException {
    if (!fileSystem.exists(basePath)) {
        throw new IOException("Please create base dir " + fileSystem.getUri() + basePath
                + ". Please set group to " + DRStatusStore.getStoreGroup() + " and permissions to "
                + DRStatusStore.DEFAULT_STORE_PERMISSION.toString());
    }/*from ww  w  .  jav a  2  s . co  m*/

    if (!fileSystem.getFileStatus(basePath).getPermission().equals(DRStatusStore.DEFAULT_STORE_PERMISSION)
            || !fileSystem.getFileStatus(basePath).getGroup().equalsIgnoreCase(DRStatusStore.getStoreGroup())) {
        throw new IOException(
                "Base dir " + fileSystem.getUri() + basePath + " does not have correct ownership/permissions."
                        + " Please set group to " + DRStatusStore.getStoreGroup() + " and permissions to "
                        + DRStatusStore.DEFAULT_STORE_PERMISSION.toString());
    }

}

---

From source file:org.apache.falcon.regression.core.util.HadoopUtil.java

License:Apache License

/**
 * Recursively retrieves all data file names from a given location.
 * @param fs filesystem/*from  w w  w .  j  a  v a  2  s.  c  o  m*/
 * @param location given location
 * @return list of all files
 * @throws IOException
 */
public static List<Path> getAllFilesRecursivelyHDFS(FileSystem fs, Path location) throws IOException {
    List<Path> returnList = new ArrayList<>();
    RemoteIterator<LocatedFileStatus> remoteIterator;
    try {
        remoteIterator = fs.listFiles(location, true);
    } catch (FileNotFoundException e) {
        LOGGER.info("Path '" + location + "' is not found on " + fs.getUri());
        return returnList;
    }
    while (remoteIterator.hasNext()) {
        Path path = remoteIterator.next().getPath();
        if (!path.toUri().toString().contains("_SUCCESS")) {
            returnList.add(path);
        }
    }
    return returnList;
}

---

From source file:org.apache.falcon.regression.core.util.HadoopUtil.java

License:Apache License

/**
 * Copies file from local place to hdfs location.
 * @param fs target filesystem/*from www.  j ava  2  s  .c om*/
 * @param dstHdfsDir destination
 * @param srcFileLocation source location
 * @throws IOException
 */
public static void copyDataToFolder(final FileSystem fs, String dstHdfsDir, final String srcFileLocation)
        throws IOException {
    LOGGER.info(String.format("Copying local dir %s to hdfs location %s on %s", srcFileLocation, dstHdfsDir,
            fs.getUri()));
    fs.copyFromLocalFile(new Path(srcFileLocation), new Path(cutProtocol(dstHdfsDir)));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• »