Example usage for org.apache.hadoop.fs FileSystem exists

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem exists.

Prototype

public boolean exists(Path f) throws IOException 

Source Link

Document

Check if a path exists.

Usage

From source file:com.facebook.hiveio.output.HiveApiOutputFormat.java

License:Apache License

/**
 * Check if the given table is empty, that is has no files
 * @param conf Configuration to use//from w w  w .j a  va 2  s .c  o m
 * @param description HiveOutputDescription
 * @param oti OutputInfo
 * @throws IOException Hadoop Filesystem issues
 */
private void checkTableIsEmpty(Configuration conf, HiveOutputDescription description, OutputInfo oti)
        throws IOException {
    Path tablePath = new Path(oti.getTableRoot());
    FileSystem fs = tablePath.getFileSystem(conf);

    if (fs.exists(tablePath) && FileSystems.dirHasNonHiddenFiles(fs, tablePath)) {
        throw new IOException("Table " + description.getTableDesc().getTableName() + " has existing data");
    }
}

---

From source file:com.facebook.LinkBench.LinkBenchDriverMR.java

License:Apache License

/**
 * setup input files for map reduce job//  w  w  w .  j  a  v  a  2 s .  c om
 * @param jobconf configuration of the map reduce job
 * @param nmappers number of mappers (loader or requester)
 */
private static FileSystem setupInputFiles(JobConf jobconf, int nmappers)
        throws IOException, InterruptedException {
    //setup input/output directories
    final Path indir = new Path(TMP_DIR, "in");
    final Path outdir = new Path(TMP_DIR, "out");
    FileInputFormat.setInputPaths(jobconf, indir);
    FileOutputFormat.setOutputPath(jobconf, outdir);

    final FileSystem fs = FileSystem.get(jobconf);
    if (fs.exists(TMP_DIR)) {
        throw new IOException(
                "Tmp directory " + fs.makeQualified(TMP_DIR) + " already exists.  Please remove it first.");
    }
    if (!fs.mkdirs(indir)) {
        throw new IOException("Cannot create input directory " + indir);
    }

    //generate an input file for each map task
    if (USE_INPUT_FILES) {
        for (int i = 0; i < nmappers; ++i) {
            final Path file = new Path(indir, "part" + i);
            final IntWritable mapperid = new IntWritable(i);
            final IntWritable nummappers = new IntWritable(nmappers);
            final SequenceFile.Writer writer = SequenceFile.createWriter(fs, jobconf, file, IntWritable.class,
                    IntWritable.class, CompressionType.NONE);
            try {
                writer.append(mapperid, nummappers);
            } finally {
                writer.close();
            }
            logger.info("Wrote input for Map #" + i);
        }
    }
    return fs;
}

---

From source file:com.facebook.presto.accumulo.examples.TpcHBatchWriter.java

License:Apache License

@Override
public int run(AccumuloConfig config, CommandLine cmd) throws Exception {
    Path orders = new Path(cmd.getOptionValue(ORDERS_OPT));
    final FileSystem fs = FileSystem.get(new Configuration());
    if (!fs.exists(orders)) {
        throw new FileNotFoundException(format("File %s does not exist or is a directory", orders));
    }/*  ww  w .  j  a v a 2  s  .c o m*/

    ZooKeeperInstance inst = new ZooKeeperInstance(config.getInstance(), config.getZooKeepers());
    Connector conn = inst.getConnector(config.getUsername(), new PasswordToken(config.getPassword()));

    validateTable(conn, DATA_TABLE);
    validateTable(conn, INDEX_TABLE);

    BatchWriterConfig bwc = new BatchWriterConfig();
    MultiTableBatchWriter mtbw = conn.createMultiTableBatchWriter(bwc);
    BatchWriter mainWrtr = mtbw.getBatchWriter(DATA_TABLE);
    BatchWriter indexWrtr = mtbw.getBatchWriter(INDEX_TABLE);

    long numTweets = 0;
    long numIndex = 0;

    System.out.println(format("Reading from file: %s", orders));
    BufferedReader rdr = new BufferedReader(new InputStreamReader(fs.open(orders)));

    // For each record in the file
    String line;
    while ((line = rdr.readLine()) != null) {
        // Split the line into fields
        String[] fields = line.split("\\|");
        if (fields.length < 9) {
            System.err.println(format("Record does not contain at least nine fields:\n%s", line));
            continue;
        }

        // Parse out the fields from strings
        Long orderkey = Long.parseLong(fields[0]);
        Long custkey = Long.parseLong(fields[1]);
        String orderstatus = fields[2];
        Double totalprice = Double.parseDouble(fields[3]);
        Date orderdate = sdformat.parse(fields[4]);
        String orderpriority = fields[5];
        String clerk = fields[6];
        Long shippriority = Long.parseLong(fields[7]);
        String comment = fields[8];

        // Create mutation for the row
        Mutation mutation = new Mutation(encode(orderkey));
        mutation.put(CF, CUSTKEY, encode(custkey));
        mutation.put(CF, ORDERSTATUS, encode(orderstatus));
        mutation.put(CF, TOTALPRICE, encode(totalprice));
        mutation.put(CF, ORDERDATE, encode(orderdate));
        mutation.put(CF, ORDERPRIORITY, encode(orderpriority));
        mutation.put(CF, CLERK, encode(clerk));
        mutation.put(CF, SHIPPRIORITY, encode(shippriority));
        mutation.put(CF, COMMENT, encode(comment));
        mainWrtr.addMutation(mutation);
        ++numTweets;

        // Create index mutation for the clerk
        Mutation idxClerk = new Mutation(encode(clerk));
        idxClerk.put(CF, encode(orderkey), EMPTY_BYTES);
        indexWrtr.addMutation(idxClerk);
        ++numIndex;
    }
    rdr.close();

    // Send the mutations to Accumulo and release resources
    mtbw.close();

    // Display how many tweets were inserted into Accumulo
    System.out.println(format("%d tweets Mutations inserted", numTweets));
    System.out.println(format("%d index Mutations inserted", numIndex));
    return 0;
}

---

From source file:com.facebook.presto.example.ExampleClient.java

License:Apache License

private static Map<String, Map<String, ExampleTable>> lookupSchemas(URI metadataUri,
        JsonCodec<Map<String, List<ExampleTable>>> catalogCodec) throws IOException {
    String json = null;//from   www.j  av a2  s  .c om
    if (metadataUri.getScheme().equalsIgnoreCase("hdfs")) {
        // schema file on hdfs
        String hdfsSiteLocation = "/etc/hadoop/conf/hdfs-site.xml";
        String coreSiteLocation = "/etc/hadoop/conf/core-site.xml";

        Configuration conf = new Configuration();
        final Path hdfsConf = new Path(hdfsSiteLocation);
        final Path coreConf = new Path(coreSiteLocation);
        conf.addResource(hdfsConf);
        conf.addResource(coreConf);

        Path schemaPath = new Path(metadataUri);

        FileSystem fs = FileSystem.get(conf);

        if (!fs.exists(schemaPath)) {
            byte[] schemaBytes = ByteStreams.toByteArray(fs.open(schemaPath));
            json = new String(schemaBytes, UTF_8);
        }
    } else {
        URL result = metadataUri.toURL();
        json = Resources.toString(result, UTF_8);
    }
    Map<String, List<ExampleTable>> catalog = catalogCodec.fromJson(json);

    return ImmutableMap.copyOf(transformValues(catalog, resolveAndIndexTables(metadataUri)));
}

---

From source file:com.facebook.presto.hive.AbstractTestHiveClient.java

License:Apache License

protected Set<String> listAllDataFiles(Path path) throws IOException {
    Set<String> result = new HashSet<>();
    FileSystem fileSystem = hdfsEnvironment.getFileSystem("user", path);
    if (fileSystem.exists(path)) {
        for (FileStatus fileStatus : fileSystem.listStatus(path)) {
            if (HadoopFileStatus.isFile(fileStatus)) {
                result.add(fileStatus.getPath().toString());
            } else if (HadoopFileStatus.isDirectory(fileStatus)) {
                result.addAll(listAllDataFiles(fileStatus.getPath()));
            }/*from w  w  w  .  j  a  v a  2 s .c o  m*/
        }
    }
    return result;
}

---

From source file:com.facebook.presto.hive.AbstractTestHiveClient.java

License:Apache License

/**
 * @return query id/*w w  w.j av  a2 s .c  o  m*/
 */
private String insertData(SchemaTableName tableName, MaterializedResult data) throws Exception {
    Path writePath;
    Path targetPath;
    String queryId;
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle);
        queryId = session.getQueryId();
        writePath = getStagingPathRoot(insertTableHandle);
        targetPath = getTargetPathRoot(insertTableHandle);

        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session,
                insertTableHandle);

        // write data
        sink.appendPage(data.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());

        // commit the insert
        metadata.finishInsert(session, insertTableHandle, fragments);
        transaction.commit();
    }

    // check that temporary files are removed
    if (!writePath.equals(targetPath)) {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem("user", writePath);
        assertFalse(fileSystem.exists(writePath));
    }

    return queryId;
}

---

From source file:com.facebook.presto.hive.AbstractTestHiveClient.java

License:Apache License

private void doTestTransactionDeleteInsert(HiveStorageFormat storageFormat, SchemaTableName tableName,
        Domain domainToDrop, MaterializedResult insertData, MaterializedResult expectedData,
        TransactionDeleteInsertTestTag tag, boolean expectQuerySucceed,
        Optional<ConflictTrigger> conflictTrigger) throws Exception {
    Path writePath = null;//from   ww w  . j a  v  a 2 s .co  m
    Path targetPath = null;

    try (Transaction transaction = newTransaction()) {
        try {
            ConnectorMetadata metadata = transaction.getMetadata();
            ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
            ConnectorSession session;
            rollbackIfEquals(tag, ROLLBACK_RIGHT_AWAY);

            // Query 1: delete
            session = newSession();
            HiveColumnHandle dsColumnHandle = (HiveColumnHandle) metadata.getColumnHandles(session, tableHandle)
                    .get("pk2");
            TupleDomain<ColumnHandle> tupleDomain = TupleDomain
                    .withColumnDomains(ImmutableMap.of(dsColumnHandle, domainToDrop));
            Constraint<ColumnHandle> constraint = new Constraint<>(tupleDomain,
                    convertToPredicate(tupleDomain));
            List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle,
                    constraint, Optional.empty());
            ConnectorTableLayoutHandle tableLayoutHandle = Iterables.getOnlyElement(tableLayoutResults)
                    .getTableLayout().getHandle();
            metadata.metadataDelete(session, tableHandle, tableLayoutHandle);
            rollbackIfEquals(tag, ROLLBACK_AFTER_DELETE);

            // Query 2: insert
            session = newSession();
            ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle);
            rollbackIfEquals(tag, ROLLBACK_AFTER_BEGIN_INSERT);
            writePath = getStagingPathRoot(insertTableHandle);
            targetPath = getTargetPathRoot(insertTableHandle);
            ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(),
                    session, insertTableHandle);
            sink.appendPage(insertData.toPage());
            rollbackIfEquals(tag, ROLLBACK_AFTER_APPEND_PAGE);
            Collection<Slice> fragments = getFutureValue(sink.finish());
            rollbackIfEquals(tag, ROLLBACK_AFTER_SINK_FINISH);
            metadata.finishInsert(session, insertTableHandle, fragments);
            rollbackIfEquals(tag, ROLLBACK_AFTER_FINISH_INSERT);

            assertEquals(tag, COMMIT);

            if (conflictTrigger.isPresent()) {
                JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class);
                List<PartitionUpdate> partitionUpdates = fragments.stream().map(Slice::getBytes)
                        .map(partitionUpdateCodec::fromJson).collect(toList());
                conflictTrigger.get().triggerConflict(session, tableName, insertTableHandle, partitionUpdates);
            }
            transaction.commit();
            if (conflictTrigger.isPresent()) {
                assertTrue(expectQuerySucceed);
                conflictTrigger.get().verifyAndCleanup(tableName);
            }
        } catch (TestingRollbackException e) {
            transaction.rollback();
        } catch (PrestoException e) {
            assertFalse(expectQuerySucceed);
            if (conflictTrigger.isPresent()) {
                conflictTrigger.get().verifyAndCleanup(tableName);
            }
        }
    }

    // check that temporary files are removed
    if (writePath != null && !writePath.equals(targetPath)) {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem("user", writePath);
        assertFalse(fileSystem.exists(writePath));
    }

    try (Transaction transaction = newTransaction()) {
        // verify partitions
        List<String> partitionNames = transaction.getMetastore(tableName.getSchemaName())
                .getPartitionNames(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(
                        () -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available"));
        assertEqualsIgnoreOrder(partitionNames,
                expectedData.getMaterializedRows().stream()
                        .map(row -> format("pk1=%s/pk2=%s", row.getField(1), row.getField(2))).distinct()
                        .collect(toList()));

        // load the new table
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(
                metadata.getColumnHandles(session, tableHandle).values());

        // verify the data
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session,
                TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat));
        assertEqualsIgnoreOrder(result.getMaterializedRows(), expectedData.getMaterializedRows());
    }
}

---

From source file:com.facebook.presto.hive.AbstractTestHiveClientS3.java

License:Apache License

@Test
public void testGetFileStatus() throws Exception {
    Path basePath = new Path("s3://presto-test-hive/");
    Path tablePath = new Path(basePath, "presto_test_s3");
    Path filePath = new Path(tablePath, "test1.csv");
    FileSystem fs = basePath.getFileSystem(hdfsEnvironment.getConfiguration(basePath));

    assertTrue(isDirectory(fs.getFileStatus(basePath)));
    assertTrue(isDirectory(fs.getFileStatus(tablePath)));
    assertFalse(isDirectory(fs.getFileStatus(filePath)));
    assertFalse(fs.exists(new Path(basePath, "foo")));
}

---

From source file:com.facebook.presto.hive.AbstractTestHiveFileSystem.java

License:Apache License

@Test
public void testGetFileStatus() throws Exception {
    Path basePath = getBasePath();
    Path tablePath = new Path(basePath, "presto_test_external_fs");
    Path filePath = new Path(tablePath, "test1.csv");
    FileSystem fs = hdfsEnvironment.getFileSystem(TESTING_CONTEXT, basePath);

    assertTrue(fs.getFileStatus(basePath).isDirectory());
    assertTrue(fs.getFileStatus(tablePath).isDirectory());
    assertFalse(fs.getFileStatus(filePath).isDirectory());
    assertFalse(fs.exists(new Path(basePath, "foo")));
}

---

From source file:com.facebook.presto.hive.AbstractTestHiveFileSystem.java

License:Apache License

@Test
public void testRename() throws Exception {
    Path basePath = new Path(getBasePath(), UUID.randomUUID().toString());
    FileSystem fs = hdfsEnvironment.getFileSystem(TESTING_CONTEXT, basePath);
    assertFalse(fs.exists(basePath));

    // create file foo.txt
    Path path = new Path(basePath, "foo.txt");
    assertTrue(fs.createNewFile(path));//from w  w  w  . j a v  a 2s  . co  m
    assertTrue(fs.exists(path));

    // rename foo.txt to bar.txt when bar does not exist
    Path newPath = new Path(basePath, "bar.txt");
    assertFalse(fs.exists(newPath));
    assertTrue(fs.rename(path, newPath));
    assertFalse(fs.exists(path));
    assertTrue(fs.exists(newPath));

    // rename foo.txt to foo.txt when foo.txt does not exist
    assertFalse(fs.rename(path, path));

    // create file foo.txt and rename to existing bar.txt
    assertTrue(fs.createNewFile(path));
    assertFalse(fs.rename(path, newPath));

    // rename foo.txt to foo.txt when foo.txt exists
    assertFalse(fs.rename(path, path));

    // delete foo.txt
    assertTrue(fs.delete(path, false));
    assertFalse(fs.exists(path));

    // create directory source with file
    Path source = new Path(basePath, "source");
    assertTrue(fs.createNewFile(new Path(source, "test.txt")));

    // rename source to non-existing target
    Path target = new Path(basePath, "target");
    assertFalse(fs.exists(target));
    assertTrue(fs.rename(source, target));
    assertFalse(fs.exists(source));
    assertTrue(fs.exists(target));

    // create directory source with file
    assertTrue(fs.createNewFile(new Path(source, "test.txt")));

    // rename source to existing target
    assertTrue(fs.rename(source, target));
    assertFalse(fs.exists(source));
    target = new Path(target, "source");
    assertTrue(fs.exists(target));
    assertTrue(fs.exists(new Path(target, "test.txt")));

    // delete target
    target = new Path(basePath, "target");
    assertTrue(fs.exists(target));
    assertTrue(fs.delete(target, true));
    assertFalse(fs.exists(target));

    // cleanup
    fs.delete(basePath, true);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »