Example usage for org.apache.hadoop.tools DistCp run

Introduction

In this page you can find the example usage for org.apache.hadoop.tools DistCp run.

Prototype

@Override
public int run(String[] argv) 

Source Link

Document

Implementation of Tool::run().

Usage

From source file:org.apache.accumulo.test.ShellServerIT.java

License:Apache License

@Test
public void exporttableImporttable() throws Exception {
    final String table = name.getMethodName(), table2 = table + "2";

    // exporttable / importtable
    ts.exec("createtable " + table + " -evc", true);
    make10();/*from  www .  ja  v a  2s .  c o m*/
    ts.exec("addsplits row5", true);
    ts.exec("config -t " + table + " -s table.split.threshold=345M", true);
    ts.exec("offline " + table, true);
    File exportDir = new File(rootPath, "ShellServerIT.export");
    String exportUri = "file://" + exportDir.toString();
    String localTmp = "file://" + new File(rootPath, "ShellServerIT.tmp").toString();
    ts.exec("exporttable -t " + table + " " + exportUri, true);
    DistCp cp = newDistCp(new Configuration(false));
    String import_ = "file://" + new File(rootPath, "ShellServerIT.import").toString();
    if (getCluster().getClientConfig().getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
        // DistCp bugs out trying to get a fs delegation token to perform the cp. Just copy it ourselves by hand.
        FileSystem fs = getCluster().getFileSystem();
        FileSystem localFs = FileSystem.getLocal(new Configuration(false));

        // Path on local fs to cp into
        Path localTmpPath = new Path(localTmp);
        localFs.mkdirs(localTmpPath);

        // Path in remote fs to importtable from
        Path importDir = new Path(import_);
        fs.mkdirs(importDir);

        // Implement a poor-man's DistCp
        try (BufferedReader reader = new BufferedReader(new FileReader(new File(exportDir, "distcp.txt")))) {
            for (String line; (line = reader.readLine()) != null;) {
                Path exportedFile = new Path(line);
                // There isn't a cp on FileSystem??
                log.info("Copying " + line + " to " + localTmpPath);
                fs.copyToLocalFile(exportedFile, localTmpPath);
                Path tmpFile = new Path(localTmpPath, exportedFile.getName());
                log.info("Moving " + tmpFile + " to the import directory " + importDir);
                fs.moveFromLocalFile(tmpFile, importDir);
            }
        }
    } else {
        String[] distCpArgs = new String[] { "-f", exportUri + "/distcp.txt", import_ };
        assertEquals("Failed to run distcp: " + Arrays.toString(distCpArgs), 0, cp.run(distCpArgs));
    }
    ts.exec("importtable " + table2 + " " + import_, true);
    ts.exec("config -t " + table2 + " -np", true, "345M", true);
    ts.exec("getsplits -t " + table2, true, "row5", true);
    ts.exec("constraint --list -t " + table2, true, "VisibilityConstraint=2", true);
    ts.exec("online " + table, true);
    ts.exec("deletetable -f " + table, true);
    ts.exec("deletetable -f " + table2, true);
}

---

From source file:org.apache.accumulo.test.ShellServerTest.java

License:Apache License

@Test(timeout = 30000)
public void exporttableImporttable() throws Exception {
    // exporttable / importtable
    exec("createtable t -evc", true);
    make10();//from w w w.j  a  v a2s .  co  m
    exec("addsplits row5", true);
    exec("config -t t -s table.split.threshold=345M", true);
    exec("offline t", true);
    String export = folder.newFolder().toString();
    exec("exporttable -t t " + export, true);
    DistCp cp = newDistCp();
    String import_ = folder.newFolder().toString();
    cp.run(new String[] { "-f", export + "/distcp.txt", import_ });
    exec("importtable t2 " + import_, true);
    exec("config -t t2 -np", true, "345M", true);
    exec("getsplits -t t2", true, "row5", true);
    exec("constraint --list -t t2", true, "VisibilityConstraint=1", true);
    exec("onlinetable t", true);
    exec("deletetable -f t", true);
    exec("deletetable -f t2", true);
}

---