Example usage for org.apache.hadoop.fs FileSystem mkdirs

List of usage examples for org.apache.hadoop.fs FileSystem mkdirs

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. mkdirs

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem mkdirs.

Prototype

public boolean mkdirs(Path f) throws IOException

Source Link

Document

Call #mkdirs(Path,FsPermission) with default permission.

Usage

From source file:org.apache.accumulo.test.functional.BulkFailureIT.java

License:Apache License

private String createTestFile(long txid, SortedMap<Key, Value> testData, FileSystem fs) throws IOException {
    Path base = new Path(getCluster().getTemporaryPath(), "testBulk_ICI_" + txid);

    fs.delete(base, true);//from   www .  j av a 2 s  .  c  o  m
    fs.mkdirs(base);
    Path files = new Path(base, "files");

    try (RFileWriter writer = RFile.newWriter().to(new Path(files, "ici_01.rf").toString()).withFileSystem(fs)
            .build()) {
        writer.append(testData.entrySet());
    }

    String filesStr = fs.makeQualified(files).toString();
    return filesStr;
}

From source file:org.apache.accumulo.test.functional.BulkIT.java

License:Apache License

static void runTest(Connector c, FileSystem fs, Path basePath, String principal, String tableName,
        String filePrefix, String dirSuffix) throws Exception {
    c.tableOperations().create(tableName);

    Path base = new Path(basePath, "testBulkFail_" + dirSuffix);
    fs.delete(base, true);/*  w ww  .j a va 2s  . c  o  m*/
    fs.mkdirs(base);
    Path bulkFailures = new Path(base, "failures");
    Path files = new Path(base, "files");
    fs.mkdirs(bulkFailures);
    fs.mkdirs(files);

    Opts opts = new Opts();
    opts.timestamp = 1;
    opts.random = 56;
    opts.rows = N;
    opts.instance = c.getInstance().getInstanceName();
    opts.cols = 1;
    opts.setTableName(tableName);
    opts.conf = new Configuration(false);
    opts.fs = fs;
    String fileFormat = filePrefix + "rf%02d";
    for (int i = 0; i < COUNT; i++) {
        opts.outputFile = new Path(files, String.format(fileFormat, i)).toString();
        opts.startRow = N * i;
        TestIngest.ingest(c, fs, opts, BWOPTS);
    }
    opts.outputFile = new Path(files, String.format(fileFormat, N)).toString();
    opts.startRow = N;
    opts.rows = 1;
    // create an rfile with one entry, there was a bug with this:
    TestIngest.ingest(c, fs, opts, BWOPTS);

    // Make sure the server can modify the files
    c.tableOperations().importDirectory(tableName, files.toString(), bulkFailures.toString(), false);
    VerifyIngest.Opts vopts = new VerifyIngest.Opts();
    vopts.setTableName(tableName);
    vopts.random = 56;
    vopts.setPrincipal(principal);
    for (int i = 0; i < COUNT; i++) {
        vopts.startRow = i * N;
        vopts.rows = N;
        VerifyIngest.verifyIngest(c, vopts, SOPTS);
    }
    vopts.startRow = N;
    vopts.rows = 1;
    VerifyIngest.verifyIngest(c, vopts, SOPTS);
}

From source file:org.apache.accumulo.test.functional.FunctionalTestUtils.java

License:Apache License

static public void bulkImport(Connector c, FileSystem fs, String table, String dir) throws Exception {
    String failDir = dir + "_failures";
    Path failPath = new Path(failDir);
    fs.delete(failPath, true);/* w w  w.j  a  v a  2  s. co  m*/
    fs.mkdirs(failPath);

    // Ensure server can read/modify files
    c.tableOperations().importDirectory(table, dir, failDir, false);

    if (fs.listStatus(failPath).length > 0) {
        throw new Exception("Some files failed to bulk import");
    }

}

From source file:org.apache.accumulo.test.GetFileInfoBulkIT.java

License:Apache License

@Test
public void test() throws Exception {
    final Connector c = getConnector();
    getCluster().getClusterControl().kill(ServerType.GARBAGE_COLLECTOR, "localhost");
    final String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    // turn off compactions
    c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "2000");
    c.tableOperations().setProperty(tableName, Property.TABLE_FILE_MAX.getKey(), "2000");
    // splits to slow down bulk import
    SortedSet<Text> splits = new TreeSet<>();
    for (int i = 1; i < 0xf; i++) {
        splits.add(new Text(Integer.toHexString(i)));
    }/* www.  ja v  a2 s  .  c om*/
    c.tableOperations().addSplits(tableName, splits);

    MasterMonitorInfo stats = getCluster().getMasterMonitorInfo();
    assertEquals(1, stats.tServerInfo.size());

    log.info("Creating lots of bulk import files");
    final FileSystem fs = getCluster().getFileSystem();
    final Path basePath = getCluster().getTemporaryPath();
    CachedConfiguration.setInstance(fs.getConf());

    final Path base = new Path(basePath, "testBulkLoad" + tableName);
    fs.delete(base, true);
    fs.mkdirs(base);

    ExecutorService es = Executors.newFixedThreadPool(5);
    List<Future<Pair<String, String>>> futures = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        final int which = i;
        futures.add(es.submit(new Callable<Pair<String, String>>() {
            @Override
            public Pair<String, String> call() throws Exception {
                Path bulkFailures = new Path(base, "failures" + which);
                Path files = new Path(base, "files" + which);
                fs.mkdirs(bulkFailures);
                fs.mkdirs(files);
                for (int i = 0; i < 100; i++) {
                    FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder()
                            .forFile(files.toString() + "/bulk_" + i + "." + RFile.EXTENSION, fs, fs.getConf())
                            .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build();
                    writer.startDefaultLocalityGroup();
                    for (int j = 0x100; j < 0xfff; j += 3) {
                        writer.append(new Key(Integer.toHexString(j)), new Value(new byte[0]));
                    }
                    writer.close();
                }
                return new Pair<>(files.toString(), bulkFailures.toString());
            }
        }));
    }
    List<Pair<String, String>> dirs = new ArrayList<>();
    for (Future<Pair<String, String>> f : futures) {
        dirs.add(f.get());
    }
    log.info("Importing");
    long startOps = getOpts();
    long now = System.currentTimeMillis();
    List<Future<Object>> errs = new ArrayList<>();
    for (Pair<String, String> entry : dirs) {
        final String dir = entry.getFirst();
        final String err = entry.getSecond();
        errs.add(es.submit(new Callable<Object>() {
            @Override
            public Object call() throws Exception {
                c.tableOperations().importDirectory(tableName, dir, err, false);
                return null;
            }
        }));
    }
    for (Future<Object> err : errs) {
        err.get();
    }
    es.shutdown();
    es.awaitTermination(2, TimeUnit.MINUTES);
    log.info(String.format("Completed in %.2f seconds", (System.currentTimeMillis() - now) / 1000.));
    Uninterruptibles.sleepUninterruptibly(30, TimeUnit.SECONDS);
    long getFileInfoOpts = getOpts() - startOps;
    log.info("# opts: {}", getFileInfoOpts);
    assertTrue("unexpected number of getFileOps", getFileInfoOpts < 2100 && getFileInfoOpts > 1000);
}

From source file:org.apache.accumulo.test.ImportExportIT.java

License:Apache License

@Test
public void testExportImportThenScan() throws Exception {
    Connector conn = getConnector();//from  ww w  .  java  2  s .co  m

    String[] tableNames = getUniqueNames(2);
    String srcTable = tableNames[0], destTable = tableNames[1];
    conn.tableOperations().create(srcTable);

    BatchWriter bw = conn.createBatchWriter(srcTable, new BatchWriterConfig());
    for (int row = 0; row < 1000; row++) {
        Mutation m = new Mutation(Integer.toString(row));
        for (int col = 0; col < 100; col++) {
            m.put(Integer.toString(col), "", Integer.toString(col * 2));
        }
        bw.addMutation(m);
    }

    bw.close();

    conn.tableOperations().compact(srcTable, null, null, true, true);

    // Make a directory we can use to throw the export and import directories
    // Must exist on the filesystem the cluster is running.
    FileSystem fs = cluster.getFileSystem();
    Path tmp = cluster.getTemporaryPath();
    log.info("Using FileSystem: " + fs);
    Path baseDir = new Path(tmp, getClass().getName());
    if (fs.exists(baseDir)) {
        log.info("{} exists on filesystem, deleting", baseDir);
        assertTrue("Failed to deleted " + baseDir, fs.delete(baseDir, true));
    }
    log.info("Creating {}", baseDir);
    assertTrue("Failed to create " + baseDir, fs.mkdirs(baseDir));
    Path exportDir = new Path(baseDir, "export");
    Path importDir = new Path(baseDir, "import");
    for (Path p : new Path[] { exportDir, importDir }) {
        assertTrue("Failed to create " + baseDir, fs.mkdirs(p));
    }

    log.info("Exporting table to {}", exportDir);
    log.info("Importing table from {}", importDir);

    // Offline the table
    conn.tableOperations().offline(srcTable, true);
    // Then export it
    conn.tableOperations().exportTable(srcTable, exportDir.toString());

    // Make sure the distcp.txt file that exporttable creates is available
    Path distcp = new Path(exportDir, "distcp.txt");
    Assert.assertTrue("Distcp file doesn't exist", fs.exists(distcp));
    FSDataInputStream is = fs.open(distcp);
    BufferedReader reader = new BufferedReader(new InputStreamReader(is));

    // Copy each file that was exported to the import directory
    String line;
    while (null != (line = reader.readLine())) {
        Path p = new Path(line.substring(5));
        Assert.assertTrue("File doesn't exist: " + p, fs.exists(p));

        Path dest = new Path(importDir, p.getName());
        Assert.assertFalse("Did not expect " + dest + " to exist", fs.exists(dest));
        FileUtil.copy(fs, p, fs, dest, false, fs.getConf());
    }

    reader.close();

    log.info("Import dir: {}", Arrays.toString(fs.listStatus(importDir)));

    // Import the exported data into a new table
    conn.tableOperations().importTable(destTable, importDir.toString());

    // Get the table ID for the table that the importtable command created
    final String tableId = conn.tableOperations().tableIdMap().get(destTable);
    Assert.assertNotNull(tableId);

    // Get all `file` colfams from the metadata table for the new table
    log.info("Imported into table with ID: {}", tableId);
    Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
    s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
    s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
    MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(s);

    // Should find a single entry
    for (Entry<Key, Value> fileEntry : s) {
        Key k = fileEntry.getKey();
        String value = fileEntry.getValue().toString();
        if (k.getColumnFamily().equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME)) {
            // The file should be an absolute URI (file:///...), not a relative path (/b-000.../I000001.rf)
            String fileUri = k.getColumnQualifier().toString();
            Assert.assertFalse("Imported files should have absolute URIs, not relative: " + fileUri,
                    looksLikeRelativePath(fileUri));
        } else if (k.getColumnFamily().equals(MetadataSchema.TabletsSection.ServerColumnFamily.NAME)) {
            Assert.assertFalse("Server directory should have absolute URI, not relative: " + value,
                    looksLikeRelativePath(value));
        } else {
            Assert.fail("Got expected pair: " + k + "=" + fileEntry.getValue());
        }
    }

    // Online the original table before we verify equivalence
    conn.tableOperations().online(srcTable, true);

    verifyTableEquality(conn, srcTable, destTable);
}

From source file:org.apache.accumulo.test.performance.metadata.FastBulkImportIT.java

License:Apache License

@Test
public void test() throws Exception {
    log.info("Creating table");
    final String tableName = getUniqueNames(1)[0];
    final Connector c = getConnector();
    c.tableOperations().create(tableName);
    log.info("Adding splits");
    SortedSet<Text> splits = new TreeSet<>();
    for (int i = 1; i < 0xfff; i += 7) {
        splits.add(new Text(Integer.toHexString(i)));
    }/*from   ww  w .j  a  v  a  2 s . c om*/
    c.tableOperations().addSplits(tableName, splits);

    log.info("Creating lots of bulk import files");
    FileSystem fs = getCluster().getFileSystem();
    Path basePath = getCluster().getTemporaryPath();
    CachedConfiguration.setInstance(fs.getConf());

    Path base = new Path(basePath, "testBulkFail_" + tableName);
    fs.delete(base, true);
    fs.mkdirs(base);
    Path bulkFailures = new Path(base, "failures");
    Path files = new Path(base, "files");
    fs.mkdirs(bulkFailures);
    fs.mkdirs(files);
    for (int i = 0; i < 100; i++) {
        FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder()
                .forFile(files.toString() + "/bulk_" + i + "." + RFile.EXTENSION, fs, fs.getConf())
                .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build();
        writer.startDefaultLocalityGroup();
        for (int j = 0x100; j < 0xfff; j += 3) {
            writer.append(new Key(Integer.toHexString(j)), new Value(new byte[0]));
        }
        writer.close();
    }
    log.info("Waiting for balance");
    c.instanceOperations().waitForBalance();

    log.info("Bulk importing files");
    long now = System.currentTimeMillis();
    c.tableOperations().importDirectory(tableName, files.toString(), bulkFailures.toString(), true);
    double diffSeconds = (System.currentTimeMillis() - now) / 1000.;
    log.info(String.format("Import took %.2f seconds", diffSeconds));
    assertTrue(diffSeconds < 30);
}

From source file:org.apache.accumulo.test.proxy.SimpleProxyBase.java

License:Apache License

@Test
public void importExportTable() throws Exception {
    // Write some data
    String expected[][] = new String[10][];
    for (int i = 0; i < 10; i++) {
        client.updateAndFlush(creds, tableName, mutation("row" + i, "cf", "cq", "" + i));
        expected[i] = new String[] { "row" + i, "cf", "cq", "" + i };
        client.flushTable(creds, tableName, null, null, true);
    }/*from  w ww  .  j ava 2  s  .c om*/
    assertScan(expected, tableName);

    // export/import
    MiniAccumuloClusterImpl cluster = SharedMiniClusterBase.getCluster();
    FileSystem fs = cluster.getFileSystem();
    Path base = cluster.getTemporaryPath();
    Path dir = new Path(base, "test");
    assertTrue(fs.mkdirs(dir));
    Path destDir = new Path(base, "test_dest");
    assertTrue(fs.mkdirs(destDir));
    client.offlineTable(creds, tableName, false);
    client.exportTable(creds, tableName, dir.toString());
    // copy files to a new location
    FSDataInputStream is = fs.open(new Path(dir, "distcp.txt"));
    try (BufferedReader r = new BufferedReader(new InputStreamReader(is, UTF_8))) {
        while (true) {
            String line = r.readLine();
            if (line == null)
                break;
            Path srcPath = new Path(line);
            FileUtil.copy(fs, srcPath, fs, destDir, false, fs.getConf());
        }
    }
    client.deleteTable(creds, tableName);
    client.importTable(creds, "testify", destDir.toString());
    assertScan(expected, "testify");
    client.deleteTable(creds, "testify");

    try {
        // ACCUMULO-1558 a second import from the same dir should fail, the first import moved the files
        client.importTable(creds, "testify2", destDir.toString());
        fail();
    } catch (Exception e) {
    }

    assertFalse(client.listTables(creds).contains("testify2"));
}