Example usage for org.apache.hadoop.fs FileSystem mkdirs

List of usage examples for org.apache.hadoop.fs FileSystem mkdirs

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. mkdirs

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem mkdirs.

Prototype

public boolean mkdirs(Path f) throws IOException

Source Link

Document

Call #mkdirs(Path,FsPermission) with default permission.

Usage

From source file:org.apache.accumulo.proxy.SimpleProxyIT.java

License:Apache License

@Test
public void testTableOperations() throws Exception {
    final String TABLE_TEST = makeTableName();

    client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
    // constraints
    client.addConstraint(creds, TABLE_TEST, NumericValueConstraint.class.getName());
    assertEquals(2, client.listConstraints(creds, TABLE_TEST).size());

    UtilWaitThread.sleep(2000);//from   www .j  a v a2s .c  o  m

    client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "123"));

    try {
        client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x"));
        fail("constraint did not fire");
    } catch (MutationsRejectedException ex) {
    }

    client.removeConstraint(creds, TABLE_TEST, 2);

    UtilWaitThread.sleep(2000);

    assertEquals(1, client.listConstraints(creds, TABLE_TEST).size());

    client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x"));
    assertScan(new String[][] { { "row1", "cf", "cq", "x" } }, TABLE_TEST);
    // splits, merge
    client.addSplits(creds, TABLE_TEST,
            new HashSet<ByteBuffer>(Arrays.asList(s2bb("a"), s2bb("m"), s2bb("z"))));
    List<ByteBuffer> splits = client.listSplits(creds, TABLE_TEST, 1);
    assertEquals(Arrays.asList(s2bb("m")), splits);
    client.mergeTablets(creds, TABLE_TEST, null, s2bb("m"));
    splits = client.listSplits(creds, TABLE_TEST, 10);
    assertEquals(Arrays.asList(s2bb("m"), s2bb("z")), splits);
    client.mergeTablets(creds, TABLE_TEST, null, null);
    splits = client.listSplits(creds, TABLE_TEST, 10);
    List<ByteBuffer> empty = Collections.emptyList();
    assertEquals(empty, splits);
    // iterators
    client.deleteTable(creds, TABLE_TEST);
    client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
    HashMap<String, String> options = new HashMap<String, String>();
    options.put("type", "STRING");
    options.put("columns", "cf");
    IteratorSetting setting = new IteratorSetting(10, TABLE_TEST, SummingCombiner.class.getName(), options);
    client.attachIterator(creds, TABLE_TEST, setting, EnumSet.allOf(IteratorScope.class));
    for (int i = 0; i < 10; i++) {
        client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "1"));
    }
    assertScan(new String[][] { { "row1", "cf", "cq", "10" } }, TABLE_TEST);
    try {
        client.checkIteratorConflicts(creds, TABLE_TEST, setting, EnumSet.allOf(IteratorScope.class));
        fail("checkIteratorConflicts did not throw an exception");
    } catch (Exception ex) {
    }
    client.deleteRows(creds, TABLE_TEST, null, null);
    client.removeIterator(creds, TABLE_TEST, "test", EnumSet.allOf(IteratorScope.class));
    String expected[][] = new String[10][];
    for (int i = 0; i < 10; i++) {
        client.updateAndFlush(creds, TABLE_TEST, mutation("row" + i, "cf", "cq", "" + i));
        expected[i] = new String[] { "row" + i, "cf", "cq", "" + i };
        client.flushTable(creds, TABLE_TEST, null, null, true);
    }
    assertScan(expected, TABLE_TEST);
    // clone
    final String TABLE_TEST2 = makeTableName();
    client.cloneTable(creds, TABLE_TEST, TABLE_TEST2, true, null, null);
    assertScan(expected, TABLE_TEST2);
    client.deleteTable(creds, TABLE_TEST2);

    // don't know how to test this, call it just for fun
    client.clearLocatorCache(creds, TABLE_TEST);

    // compact
    client.compactTable(creds, TABLE_TEST, null, null, null, true, true);
    assertEquals(1, countFiles(TABLE_TEST));
    assertScan(expected, TABLE_TEST);

    // get disk usage
    client.cloneTable(creds, TABLE_TEST, TABLE_TEST2, true, null, null);
    Set<String> tablesToScan = new HashSet<String>();
    tablesToScan.add(TABLE_TEST);
    tablesToScan.add(TABLE_TEST2);
    tablesToScan.add("foo");
    client.createTable(creds, "foo", true, TimeType.MILLIS);
    List<DiskUsage> diskUsage = (client.getDiskUsage(creds, tablesToScan));
    assertEquals(2, diskUsage.size());
    assertEquals(1, diskUsage.get(0).getTables().size());
    assertEquals(2, diskUsage.get(1).getTables().size());
    client.compactTable(creds, TABLE_TEST2, null, null, null, true, true);
    diskUsage = (client.getDiskUsage(creds, tablesToScan));
    assertEquals(3, diskUsage.size());
    assertEquals(1, diskUsage.get(0).getTables().size());
    assertEquals(1, diskUsage.get(1).getTables().size());
    assertEquals(1, diskUsage.get(2).getTables().size());
    client.deleteTable(creds, "foo");
    client.deleteTable(creds, TABLE_TEST2);

    // export/import
    File dir = tempFolder.newFolder("test");
    File destDir = tempFolder.newFolder("test_dest");
    client.offlineTable(creds, TABLE_TEST, false);
    client.exportTable(creds, TABLE_TEST, dir.getAbsolutePath());
    // copy files to a new location
    FileSystem fs = FileSystem.get(new Configuration());
    FSDataInputStream is = fs.open(new Path(dir + "/distcp.txt"));
    BufferedReader r = new BufferedReader(new InputStreamReader(is));
    while (true) {
        String line = r.readLine();
        if (line == null)
            break;
        Path srcPath = new Path(line);
        FileUtils.copyFile(new File(srcPath.toUri().getPath()), new File(destDir, srcPath.getName()));
    }
    client.deleteTable(creds, TABLE_TEST);
    client.importTable(creds, "testify", destDir.getAbsolutePath());
    assertScan(expected, "testify");
    client.deleteTable(creds, "testify");

    try {
        // ACCUMULO-1558 a second import from the same dir should fail, the first import moved the files
        client.importTable(creds, "testify2", destDir.getAbsolutePath());
        fail();
    } catch (Exception e) {
    }

    assertFalse(client.listTables(creds).contains("testify2"));

    // Locality groups
    client.createTable(creds, "test", true, TimeType.MILLIS);
    Map<String, Set<String>> groups = new HashMap<String, Set<String>>();
    groups.put("group1", Collections.singleton("cf1"));
    groups.put("group2", Collections.singleton("cf2"));
    client.setLocalityGroups(creds, "test", groups);
    assertEquals(groups, client.getLocalityGroups(creds, "test"));
    // table properties
    Map<String, String> orig = client.getTableProperties(creds, "test");
    client.setTableProperty(creds, "test", "table.split.threshold", "500M");
    Map<String, String> update = client.getTableProperties(creds, "test");
    assertEquals(update.get("table.split.threshold"), "500M");
    client.removeTableProperty(creds, "test", "table.split.threshold");
    update = client.getTableProperties(creds, "test");
    assertEquals(orig, update);
    // rename table
    Map<String, String> tables = client.tableIdMap(creds);
    client.renameTable(creds, "test", "bar");
    Map<String, String> tables2 = client.tableIdMap(creds);
    assertEquals(tables.get("test"), tables2.get("bar"));
    // table exists
    assertTrue(client.tableExists(creds, "bar"));
    assertFalse(client.tableExists(creds, "test"));
    // bulk import
    String filename = dir + "/bulk/import/rfile.rf";
    FileSKVWriter writer = FileOperations.getInstance().openWriter(filename, fs, fs.getConf(),
            DefaultConfiguration.getInstance());
    writer.startDefaultLocalityGroup();
    writer.append(new org.apache.accumulo.core.data.Key(new Text("a"), new Text("b"), new Text("c")),
            new Value("value".getBytes()));
    writer.close();
    fs.mkdirs(new Path(dir + "/bulk/fail"));
    client.importDirectory(creds, "bar", dir + "/bulk/import", dir + "/bulk/fail", true);
    String scanner = client.createScanner(creds, "bar", null);
    ScanResult more = client.nextK(scanner, 100);
    client.closeScanner(scanner);
    assertEquals(1, more.results.size());
    ByteBuffer maxRow = client.getMaxRow(creds, "bar", null, null, false, null, false);
    assertEquals(s2bb("a"), maxRow);

    assertFalse(client.testTableClassLoad(creds, "bar", "abc123", SortedKeyValueIterator.class.getName()));
    assertTrue(client.testTableClassLoad(creds, "bar", VersioningIterator.class.getName(),
            SortedKeyValueIterator.class.getName()));
}

From source file:org.apache.accumulo.proxy.SimpleTest.java

License:Apache License

@Test
public void testTableOperations() throws Exception {
    final String TABLE_TEST = makeTableName();

    client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
    // constraints
    client.addConstraint(creds, TABLE_TEST, NumericValueConstraint.class.getName());
    assertEquals(2, client.listConstraints(creds, TABLE_TEST).size());
    client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "123"));

    try {/*from   w  w w  .j a v a 2  s  . co m*/
        client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x"));
        fail("constraint did not fire");
    } catch (MutationsRejectedException ex) {
    }

    client.removeConstraint(creds, TABLE_TEST, 2);
    assertEquals(1, client.listConstraints(creds, TABLE_TEST).size());
    client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x"));
    String scanner = client.createScanner(creds, TABLE_TEST, null);
    ScanResult more = client.nextK(scanner, 2);
    client.closeScanner(scanner);
    assertFalse(more.isMore());
    assertEquals(1, more.getResults().size());
    assertEquals(s2bb("x"), more.getResults().get(0).value);
    // splits, merge
    client.addSplits(creds, TABLE_TEST,
            new HashSet<ByteBuffer>(Arrays.asList(s2bb("a"), s2bb("m"), s2bb("z"))));
    List<ByteBuffer> splits = client.listSplits(creds, TABLE_TEST, 1);
    assertEquals(Arrays.asList(s2bb("m")), splits);
    client.mergeTablets(creds, TABLE_TEST, null, s2bb("m"));
    splits = client.listSplits(creds, TABLE_TEST, 10);
    assertEquals(Arrays.asList(s2bb("m"), s2bb("z")), splits);
    client.mergeTablets(creds, TABLE_TEST, null, null);
    splits = client.listSplits(creds, TABLE_TEST, 10);
    List<ByteBuffer> empty = Collections.emptyList();
    assertEquals(empty, splits);
    // iterators
    client.deleteTable(creds, TABLE_TEST);
    client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
    HashMap<String, String> options = new HashMap<String, String>();
    options.put("type", "STRING");
    options.put("columns", "cf");
    IteratorSetting setting = new IteratorSetting(10, TABLE_TEST, SummingCombiner.class.getName(), options);
    client.attachIterator(creds, TABLE_TEST, setting, EnumSet.allOf(IteratorScope.class));
    for (int i = 0; i < 10; i++) {
        client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "1"));
    }
    scanner = client.createScanner(creds, TABLE_TEST, null);
    more = client.nextK(scanner, 2);
    client.closeScanner(scanner);
    assertEquals("10", new String(more.getResults().get(0).getValue()));
    try {
        client.checkIteratorConflicts(creds, TABLE_TEST, setting, EnumSet.allOf(IteratorScope.class));
        fail("checkIteratorConflicts did not throw an exception");
    } catch (Exception ex) {
    }
    client.deleteRows(creds, TABLE_TEST, null, null);
    client.removeIterator(creds, TABLE_TEST, "test", EnumSet.allOf(IteratorScope.class));
    for (int i = 0; i < 10; i++) {
        client.updateAndFlush(creds, TABLE_TEST, mutation("row" + i, "cf", "cq", "" + i));
        client.flushTable(creds, TABLE_TEST, null, null, true);
    }
    scanner = client.createScanner(creds, TABLE_TEST, null);
    more = client.nextK(scanner, 100);
    client.closeScanner(scanner);
    assertEquals(10, more.getResults().size());
    // clone
    final String TABLE_TEST2 = makeTableName();
    client.cloneTable(creds, TABLE_TEST, TABLE_TEST2, true, null, null);
    scanner = client.createScanner(creds, TABLE_TEST2, null);
    more = client.nextK(scanner, 100);
    client.closeScanner(scanner);
    assertEquals(10, more.getResults().size());
    client.deleteTable(creds, TABLE_TEST2);

    // don't know how to test this, call it just for fun
    client.clearLocatorCache(creds, TABLE_TEST);

    // compact
    client.compactTable(creds, TABLE_TEST, null, null, null, true, true);
    assertEquals(1, countFiles(TABLE_TEST));

    // get disk usage
    client.cloneTable(creds, TABLE_TEST, TABLE_TEST2, true, null, null);
    Set<String> tablesToScan = new HashSet<String>();
    tablesToScan.add(TABLE_TEST);
    tablesToScan.add(TABLE_TEST2);
    tablesToScan.add("foo");
    client.createTable(creds, "foo", true, TimeType.MILLIS);
    List<DiskUsage> diskUsage = (client.getDiskUsage(creds, tablesToScan));
    assertEquals(2, diskUsage.size());
    assertEquals(1, diskUsage.get(0).getTables().size());
    assertEquals(2, diskUsage.get(1).getTables().size());
    client.compactTable(creds, TABLE_TEST2, null, null, null, true, true);
    diskUsage = (client.getDiskUsage(creds, tablesToScan));
    assertEquals(3, diskUsage.size());
    assertEquals(1, diskUsage.get(0).getTables().size());
    assertEquals(1, diskUsage.get(1).getTables().size());
    assertEquals(1, diskUsage.get(2).getTables().size());
    client.deleteTable(creds, "foo");
    client.deleteTable(creds, TABLE_TEST2);

    // export/import
    String dir = folder.getRoot() + "/test";
    String destDir = folder.getRoot() + "/test_dest";
    client.offlineTable(creds, TABLE_TEST);
    client.exportTable(creds, TABLE_TEST, dir);
    // copy files to a new location
    FileSystem fs = FileSystem.get(new Configuration());
    FSDataInputStream is = fs.open(new Path(dir + "/distcp.txt"));
    BufferedReader r = new BufferedReader(new InputStreamReader(is));
    while (true) {
        String line = r.readLine();
        if (line == null)
            break;
        Path srcPath = new Path(line);
        FileUtils.copyFile(new File(srcPath.toUri().getPath()), new File(destDir, srcPath.getName()));
    }
    client.deleteTable(creds, TABLE_TEST);
    client.importTable(creds, "testify", destDir);
    scanner = client.createScanner(creds, "testify", null);
    more = client.nextK(scanner, 100);
    client.closeScanner(scanner);
    assertEquals(10, more.results.size());

    try {
        // ACCUMULO-1558 a second import from the same dir should fail, the first import moved the files
        client.importTable(creds, "testify2", destDir);
        fail();
    } catch (Exception e) {
    }

    assertFalse(client.listTables(creds).contains("testify2"));

    // Locality groups
    client.createTable(creds, "test", true, TimeType.MILLIS);
    Map<String, Set<String>> groups = new HashMap<String, Set<String>>();
    groups.put("group1", Collections.singleton("cf1"));
    groups.put("group2", Collections.singleton("cf2"));
    client.setLocalityGroups(creds, "test", groups);
    assertEquals(groups, client.getLocalityGroups(creds, "test"));
    // table properties
    Map<String, String> orig = client.getTableProperties(creds, "test");
    client.setTableProperty(creds, "test", "table.split.threshold", "500M");
    Map<String, String> update = client.getTableProperties(creds, "test");
    for (int i = 0; i < 5; i++) {
        if (update.get("table.split.threshold").equals("500M"))
            break;
        UtilWaitThread.sleep(200);
    }
    assertEquals(update.get("table.split.threshold"), "500M");
    client.removeTableProperty(creds, "test", "table.split.threshold");
    update = client.getTableProperties(creds, "test");
    assertEquals(orig, update);
    // rename table
    Map<String, String> tables = client.tableIdMap(creds);
    client.renameTable(creds, "test", "bar");
    Map<String, String> tables2 = client.tableIdMap(creds);
    assertEquals(tables.get("test"), tables2.get("bar"));
    // table exists
    assertTrue(client.tableExists(creds, "bar"));
    assertFalse(client.tableExists(creds, "test"));
    // bulk import
    String filename = dir + "/bulk/import/rfile.rf";
    FileSKVWriter writer = FileOperations.getInstance().openWriter(filename, fs, fs.getConf(),
            DefaultConfiguration.getInstance());
    writer.startDefaultLocalityGroup();
    writer.append(new org.apache.accumulo.core.data.Key(new Text("a"), new Text("b"), new Text("c")),
            new Value("value".getBytes()));
    writer.close();
    fs.mkdirs(new Path(dir + "/bulk/fail"));
    client.importDirectory(creds, "bar", dir + "/bulk/import", dir + "/bulk/fail", true);
    scanner = client.createScanner(creds, "bar", null);
    more = client.nextK(scanner, 100);
    client.closeScanner(scanner);
    assertEquals(1, more.results.size());
    ByteBuffer maxRow = client.getMaxRow(creds, "bar", null, null, false, null, false);
    assertEquals(s2bb("a"), maxRow);

    assertFalse(client.testTableClassLoad(creds, "bar", "abc123", SortedKeyValueIterator.class.getName()));
    assertTrue(client.testTableClassLoad(creds, "bar", VersioningIterator.class.getName(),
            SortedKeyValueIterator.class.getName()));
}

From source file:org.apache.accumulo.server.fs.VolumeUtilTest.java

License:Apache License

@Test
public void testSame() throws Exception {
    FileSystem fs = FileSystem.getLocal(new Configuration());

    Path subdir1 = new Path(tempFolder.newFolder().toURI());
    Path subdir2 = new Path(tempFolder.newFolder().toURI());
    Path subdir3 = new Path(tempFolder.newFolder().toURI());

    Assert.assertFalse(VolumeUtil.same(fs, subdir1, fs,
            new Path(tempFolder.getRoot().toURI().toString(), "8854339269459287524098238497")));
    Assert.assertFalse(VolumeUtil.same(fs,
            new Path(tempFolder.getRoot().toURI().toString(), "8854339269459287524098238497"), fs, subdir1));
    Assert.assertTrue(VolumeUtil.same(fs, subdir1, fs, subdir1));

    writeFile(fs, subdir1, "abc", "foo");
    writeFile(fs, subdir2, "abc", "bar");
    writeFile(fs, subdir3, "abc", "foo");

    Assert.assertTrue(VolumeUtil.same(fs, subdir1, fs, subdir1));
    Assert.assertFalse(VolumeUtil.same(fs, subdir1, fs, subdir2));
    Assert.assertFalse(VolumeUtil.same(fs, subdir2, fs, subdir1));
    Assert.assertTrue(VolumeUtil.same(fs, subdir1, fs, subdir3));
    Assert.assertTrue(VolumeUtil.same(fs, subdir3, fs, subdir1));

    writeFile(fs, subdir1, "def", "123456");
    writeFile(fs, subdir2, "def", "123456");
    writeFile(fs, subdir3, "def", "123456");

    Assert.assertTrue(VolumeUtil.same(fs, subdir1, fs, subdir1));
    Assert.assertFalse(VolumeUtil.same(fs, subdir1, fs, subdir2));
    Assert.assertFalse(VolumeUtil.same(fs, subdir2, fs, subdir1));
    Assert.assertTrue(VolumeUtil.same(fs, subdir1, fs, subdir3));
    Assert.assertTrue(VolumeUtil.same(fs, subdir3, fs, subdir1));

    writeFile(fs, subdir3, "ghi", "09876");

    Assert.assertFalse(VolumeUtil.same(fs, subdir1, fs, subdir3));
    Assert.assertFalse(VolumeUtil.same(fs, subdir3, fs, subdir1));

    fs.mkdirs(new Path(subdir2, "dir1"));

    try {//w  w  w.  j  a v  a2  s  .c o  m
        VolumeUtil.same(fs, subdir1, fs, subdir2);
        Assert.fail();
    } catch (IllegalArgumentException e) {
    }

    try {
        VolumeUtil.same(fs, subdir2, fs, subdir1);
        Assert.fail();
    } catch (IllegalArgumentException e) {
    }

    try {
        VolumeUtil.same(fs, subdir1, fs, new Path(subdir2, "def"));
        Assert.fail();
    } catch (IllegalArgumentException e) {
    }

    try {
        VolumeUtil.same(fs, new Path(subdir2, "def"), fs, subdir3);
        Assert.fail();
    } catch (IllegalArgumentException e) {
    }

}

From source file:org.apache.accumulo.server.logger.LogArchiver.java

License:Apache License

LogArchiver(AccumuloConfiguration conf, FileSystem src, FileSystem dest, List<String> logDirs, boolean archive)
        throws IOException {
    this.src = src;
    this.dest = dest;
    this.threadPool = Executors.newSingleThreadExecutor();
    this.destDir = conf.get(Property.INSTANCE_DFS_DIR) + "/walogArchive";
    this.archive = archive;
    this.replication = (short) conf.getCount(Property.LOGGER_ARCHIVE_REPLICATION);
    dest.mkdirs(new Path(destDir));

    if (archive) {
        // re-start any previously started archives
        for (String logDir : logDirs) {
            try {
                FileStatus files[] = src.listStatus(new Path(logDir));
                if (files != null) {
                    for (FileStatus file : files) {
                        String name = file.getPath().getName();
                        if (isArchive(name)) {
                            log.info("archiving " + origName(name));
                            threadPool//from  ww  w. j av a2  s .c o m
                                    .execute(new LogArchiveTask(src, dest, file.getPath().toString(), destDir));
                        }
                    }
                }
            } catch (IOException e) {
                log.warn("Failed to process existing files in local archive dir " + logDir, e);
            }
        }
    }
}

From source file:org.apache.accumulo.server.test.BulkImportDirectory.java

License:Apache License

public static void main(String[] args)
        throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException {
    if (args.length != 5)
        throw new RuntimeException("Usage: bin/accumulo " + BulkImportDirectory.class.getName()
                + " <username> <password> <tablename> <sourcedir> <failuredir>");

    final String user = args[0];
    final byte[] pass = args[1].getBytes();
    final String tableName = args[2];
    final String dir = args[3];
    final String failureDir = args[4];
    final Path failureDirPath = new Path(failureDir);
    final FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
    fs.delete(failureDirPath, true);//from ww  w .  j ava2  s  . co  m
    fs.mkdirs(failureDirPath);
    HdfsZooInstance.getInstance().getConnector(user, pass).tableOperations().importDirectory(tableName, dir,
            failureDir, false);
}

From source file:org.apache.accumulo.server.test.functional.FunctionalTest.java

License:Apache License

protected void bulkImport(FileSystem fs, String table, String dir) throws Exception {
    String failDir = dir + "_failures";
    Path failPath = new Path(failDir);
    fs.delete(failPath, true);/*from  w ww.j  av  a2 s .  c o m*/
    fs.mkdirs(failPath);

    getConnector().tableOperations().importDirectory(table, dir, failDir, false);

    if (fs.listStatus(failPath).length > 0) {
        throw new Exception("Some files failed to bulk import");
    }

}

From source file:org.apache.accumulo.server.test.MultipleIndexIterator2.java

License:Apache License

private static ArrayList<Path> reduceFiles(Configuration conf, FileSystem fs, ArrayList<Path> paths,
        int maxFiles, String tmpDir, int pass)
        throws IOException, InstantiationException, IllegalAccessException {
    if (paths.size() <= maxFiles) {
        return paths;
    }/*from w ww  .  j a  v  a  2 s  . c o m*/

    String newDir = String.format("%s/pass_%04d", tmpDir, pass);
    fs.mkdirs(new Path(newDir));

    int start = 0;

    ArrayList<Path> outFiles = new ArrayList<Path>();

    int count = 0;

    while (start < paths.size()) {
        int end = Math.min(maxFiles + start, paths.size());
        List<Path> inFiles = paths.subList(start, end);

        start = end;

        Path outFile = new Path(String.format("%s/index_%04d", newDir, count++));
        outFiles.add(outFile);

        long t1 = System.currentTimeMillis();

        MySequenceFile.Writer writer = MySequenceFile.createWriter(fs, conf, outFile, Key.class,
                LongWritable.class, MySequenceFile.CompressionType.BLOCK);
        MultipleIndexIterator2 mii = new MultipleIndexIterator2(conf, fs, inFiles);

        while (mii.hasNext()) {
            writer.append(mii.next(), new LongWritable(0));
        }

        mii.close();
        writer.close();

        long t2 = System.currentTimeMillis();

        System.out.printf("out : %s  num in : %d   time : %6.2f secs\n", outFile, inFiles.size(),
                (t2 - t1) / 1000.0);
    }

    return reduceFiles(conf, fs, outFiles, maxFiles, tmpDir, pass + 1);
}

From source file:org.apache.accumulo.server.test.MultipleIndexIterator2.java

License:Apache License

private static void createTestData(String dir, int numFiles, int numEntries, int min, int max, Random r)
        throws Exception {
    Configuration conf = CachedConfiguration.getInstance();
    FileSystem fs = FileSystem.get(conf);

    for (int i = 0; i < numFiles; i++) {
        String newDir = String.format("%s/" + MyMapFile.EXTENSION + "_%06d", dir, i);
        fs.mkdirs(new Path(newDir));

        List<Key> keys = new ArrayList<Key>();

        for (int j = 0; j < numEntries; j++) {
            String row = String.format("row_%010d", r.nextInt() % (max - min) + min);
            Key key1 = new Key(new Text(row), new Text(String.format("cf_%03d", r.nextInt() % 100)),
                    new Text(String.format("cf_%05d", r.nextInt() % 10000)));
            keys.add(key1);// w  w w. j  av a  2  s . co  m
        }

        Collections.sort(keys, new CompareKeys());

        MySequenceFile.Writer writer = MySequenceFile.createWriter(fs, conf, new Path(newDir + "/index"),
                Key.class, LongWritable.class, MySequenceFile.CompressionType.BLOCK);

        System.out.println(new Path(newDir + "/index"));

        for (Key key : keys) {
            writer.append(key, new LongWritable(0));
        }

        writer.close();
    }
}

From source file:org.apache.accumulo.server.test.randomwalk.bulk.BulkPlusOne.java

License:Apache License

static void bulkLoadLots(Logger log, State state, Value value) throws Exception {
    final Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString());
    final Path fail = new Path(dir.toString() + "_fail");
    final DefaultConfiguration defaultConfiguration = AccumuloConfiguration.getDefaultConfiguration();
    final Random rand = (Random) state.get("rand");
    final FileSystem fs = (FileSystem) state.get("fs");
    fs.mkdirs(fail);
    final int parts = rand.nextInt(10) + 1;

    TreeSet<Integer> startRows = new TreeSet<Integer>();
    startRows.add(0);// w w  w .  ja va 2 s.c om
    while (startRows.size() < parts)
        startRows.add(rand.nextInt(LOTS));

    List<String> printRows = new ArrayList<String>(startRows.size());
    for (Integer row : startRows)
        printRows.add(String.format(FMT, row));

    String markerColumnQualifier = String.format("%07d", counter.incrementAndGet());
    log.debug("preparing bulk files with start rows " + printRows + " last row " + String.format(FMT, LOTS - 1)
            + " marker " + markerColumnQualifier);

    List<Integer> rows = new ArrayList<Integer>(startRows);
    rows.add(LOTS);

    for (int i = 0; i < parts; i++) {
        String fileName = dir + "/" + String.format("part_%d.", i) + RFile.EXTENSION;
        FileSKVWriter f = FileOperations.getInstance().openWriter(fileName, fs, fs.getConf(),
                defaultConfiguration);
        f.startDefaultLocalityGroup();
        int start = rows.get(i);
        int end = rows.get(i + 1);
        for (int j = start; j < end; j++) {
            Text row = new Text(String.format(FMT, j));
            for (Column col : COLNAMES) {
                f.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier()), value);
            }
            f.append(new Key(row, MARKER_CF, new Text(markerColumnQualifier)), ONE);
        }
        f.close();
    }
    state.getConnector().tableOperations().importDirectory(Setup.getTableName(), dir.toString(),
            fail.toString(), true);
    fs.delete(dir, true);
    FileStatus[] failures = fs.listStatus(fail);
    if (failures != null && failures.length > 0) {
        throw new Exception(failures.length + " failure files found importing files from " + dir);
    }
    fs.delete(fail, true);
    log.debug("Finished bulk import, start rows " + printRows + " last row " + String.format(FMT, LOTS - 1)
            + " marker " + markerColumnQualifier);
}

From source file:org.apache.accumulo.server.test.randomwalk.concurrent.BulkImport.java

License:Apache License

@Override
public void visit(State state, Properties props) throws Exception {
    Connector conn = state.getConnector();

    Random rand = (Random) state.get("rand");

    @SuppressWarnings("unchecked")
    List<String> tableNames = (List<String>) state.get("tables");

    String tableName = tableNames.get(rand.nextInt(tableNames.size()));

    Configuration conf = CachedConfiguration.getInstance();
    FileSystem fs = FileSystem.get(conf);

    String bulkDir = "/tmp/concurrent_bulk/b_" + String.format("%016x", Math.abs(rand.nextLong()));

    fs.mkdirs(new Path(bulkDir));
    fs.mkdirs(new Path(bulkDir + "_f"));

    try {/*from   ww w  .  j a v a 2 s  . c o m*/
        BatchWriter bw = new RFileBatchWriter(conf, fs, bulkDir + "/file01.rf");
        try {
            TreeSet<Long> rows = new TreeSet<Long>();
            int numRows = rand.nextInt(100000);
            for (int i = 0; i < numRows; i++) {
                rows.add(Math.abs(rand.nextLong()));
            }

            for (Long row : rows) {
                Mutation m = new Mutation(String.format("%016x", row));
                long val = Math.abs(rand.nextLong());
                for (int j = 0; j < 10; j++) {
                    m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes()));
                }

                bw.addMutation(m);
            }
        } finally {
            bw.close();
        }

        conn.tableOperations().importDirectory(tableName, bulkDir, bulkDir + "_f", rand.nextBoolean());

        log.debug("BulkImported to " + tableName);
    } catch (TableNotFoundException e) {
        log.debug("BulkImport " + tableName + " failed, doesnt exist");
    } catch (TableOfflineException toe) {
        log.debug("BulkImport " + tableName + " failed, offline");
    } finally {
        fs.delete(new Path(bulkDir), true);
        fs.delete(new Path(bulkDir + "_f"), true);
    }

}