Example usage for org.apache.hadoop.fs FileSystem rename

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem rename.

Prototype

public abstract boolean rename(Path src, Path dst) throws IOException;

Source Link

Document

Renames Path src to Path dst.

Usage

From source file:nl.tudelft.graphalytics.mapreducev2.MapReduceV2Platform.java

License:Apache License

public void uploadGraph(Graph graph, String graphFilePath) throws IOException {
    log.entry(graph, graphFilePath);/* ww  w .  j a va  2  s . co  m*/

    String hdfsPathRaw = hdfsDirectory + "/mapreducev2/input/raw-" + graph.getName();
    String hdfsPath = hdfsDirectory + "/mapreducev2/input/" + graph.getName();

    // Establish a connection with HDFS and upload the graph
    Configuration conf = new Configuration();
    FileSystem dfs = FileSystem.get(conf);
    dfs.copyFromLocalFile(new Path(graphFilePath), new Path(hdfsPathRaw));

    // If the graph needs to be preprocessed, do so, otherwise rename it
    if (graph.getGraphFormat().isEdgeBased()) {
        try {
            EdgesToAdjacencyListConversion job = new EdgesToAdjacencyListConversion(hdfsPathRaw, hdfsPath,
                    graph.getGraphFormat().isDirected());
            if (mrConfig.containsKey("mapreducev2.reducer-count"))
                job.withNumberOfReducers(ConfigurationUtil.getInteger(mrConfig, "mapreducev2.reducer-count"));
            job.run();
        } catch (Exception e) {
            throw new IOException("Failed to preprocess graph: ", e);
        }
    } else if (graph.getGraphFormat().isDirected()) {
        try {
            DirectedVertexToAdjacencyListConversion job = new DirectedVertexToAdjacencyListConversion(
                    hdfsPathRaw, hdfsPath);
            if (mrConfig.containsKey("mapreducev2.reducer-count"))
                job.withNumberOfReducers(ConfigurationUtil.getInteger(mrConfig, "mapreducev2.reducer-count"));
            job.run();
        } catch (Exception e) {
            throw new IOException("Failed to preprocess graph: ", e);
        }
    } else {
        // Rename the graph
        dfs.rename(new Path(hdfsPathRaw), new Path(hdfsPath));
    }

    hdfsPathForGraphName.put(graph.getName(), hdfsPath);
    log.exit();
}

---

From source file:org.apache.accumulo.core.data.MapFileTest.java

License:Apache License

public void testMapFileFix() {
    try {//w  ww.j a  va  2 s  . com
        Configuration conf = CachedConfiguration.getInstance();
        FileSystem fs = FileSystem.get(conf);
        conf.setInt("io.seqfile.compress.blocksize", 4000);

        for (CompressionType compressionType : CompressionType.values()) {
            /*****************************
             * write out the test map file
             */
            MyMapFile.Writer mfw = new MyMapFile.Writer(conf, fs, "/tmp/testMapFileIndexingMap", Text.class,
                    BytesWritable.class, compressionType);
            BytesWritable value;
            Random r = new Random();
            byte[] bytes = new byte[1024];
            for (int i = 0; i < 1000; i++) {
                String keyString = Integer.toString(i + 1000000);
                Text key = new Text(keyString);
                r.nextBytes(bytes);
                value = new BytesWritable(bytes);
                mfw.append(key, value);
            }
            mfw.close();

            /************************************
             * move the index file
             */
            fs.rename(new Path("/tmp/testMapFileIndexingMap/index"),
                    new Path("/tmp/testMapFileIndexingMap/oldIndex"));

            /************************************
             * recreate the index
             */
            MyMapFile.fix(fs, new Path("/tmp/testMapFileIndexingMap"), Text.class, BytesWritable.class, false,
                    conf);

            /************************************
             * compare old and new indices
             */
            MySequenceFile.Reader oldIndexReader = new MySequenceFile.Reader(fs,
                    new Path("/tmp/testMapFileIndexingMap/oldIndex"), conf);
            MySequenceFile.Reader newIndexReader = new MySequenceFile.Reader(fs,
                    new Path("/tmp/testMapFileIndexingMap/index"), conf);

            Text oldKey = new Text();
            Text newKey = new Text();
            LongWritable oldValue = new LongWritable();
            LongWritable newValue = new LongWritable();
            while (true) {
                boolean moreKeys = false;
                // check for the same number of records
                assertTrue((moreKeys = oldIndexReader.next(oldKey, oldValue)) == newIndexReader.next(newKey,
                        newValue));
                if (!moreKeys)
                    break;
                assertTrue(oldKey.compareTo(newKey) == 0);
                assertTrue(oldValue.compareTo(newValue) == 0);
            }
            oldIndexReader.close();
            newIndexReader.close();

            fs.delete(new Path("/tmp/testMapFileIndexingMap"), true);
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
}

---

From source file:org.apache.accumulo.server.fs.VolumeManagerImpl.java

License:Apache License

@Override
public boolean rename(Path path, Path newPath) throws IOException {
    Volume srcVolume = getVolumeByPath(path);
    Volume destVolume = getVolumeByPath(newPath);
    FileSystem source = srcVolume.getFileSystem();
    FileSystem dest = destVolume.getFileSystem();
    if (source != dest) {
        throw new NotImplementedException("Cannot rename files across volumes: " + path + " -> " + newPath);
    }//from  w  w  w. j a  va  2s  .  c o  m
    return source.rename(path, newPath);
}

---

From source file:org.apache.accumulo.server.fs.VolumeUtil.java

License:Apache License

private static String decommisionedTabletDir(AccumuloServerContext context, ZooLock zooLock, VolumeManager vm,
        KeyExtent extent, String metaDir) throws IOException {
    Path dir = new Path(metaDir);
    if (isActiveVolume(dir))
        return metaDir;

    if (!dir.getParent().getParent().getName().equals(ServerConstants.TABLE_DIR)) {
        throw new IllegalArgumentException("Unexpected table dir " + dir);
    }/*w  ww  .  j a v  a  2 s . co m*/

    Path newDir = new Path(vm.choose(Optional.of(extent.getTableId()), ServerConstants.getBaseUris())
            + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + dir.getParent().getName()
            + Path.SEPARATOR + dir.getName());

    log.info("Updating directory for " + extent + " from " + dir + " to " + newDir);
    if (extent.isRootTablet()) {
        // the root tablet is special case, its files need to be copied if its dir is changed

        // this code needs to be idempotent

        FileSystem fs1 = vm.getVolumeByPath(dir).getFileSystem();
        FileSystem fs2 = vm.getVolumeByPath(newDir).getFileSystem();

        if (!same(fs1, dir, fs2, newDir)) {
            if (fs2.exists(newDir)) {
                Path newDirBackup = getBackupName(fs2, newDir);
                // never delete anything because were dealing with the root tablet
                // one reason this dir may exist is because this method failed previously
                log.info("renaming " + newDir + " to " + newDirBackup);
                if (!fs2.rename(newDir, newDirBackup)) {
                    throw new IOException("Failed to rename " + newDir + " to " + newDirBackup);
                }
            }

            // do a lot of logging since this is the root tablet
            log.info("copying " + dir + " to " + newDir);
            if (!FileUtil.copy(fs1, dir, fs2, newDir, false, CachedConfiguration.getInstance())) {
                throw new IOException("Failed to copy " + dir + " to " + newDir);
            }

            // only set the new location in zookeeper after a successful copy
            log.info("setting root tablet location to " + newDir);
            MetadataTableUtil.setRootTabletDir(newDir.toString());

            // rename the old dir to avoid confusion when someone looks at filesystem... its ok if we fail here and this does not happen because the location in
            // zookeeper is the authority
            Path dirBackup = getBackupName(fs1, dir);
            log.info("renaming " + dir + " to " + dirBackup);
            fs1.rename(dir, dirBackup);

        } else {
            log.info("setting root tablet location to " + newDir);
            MetadataTableUtil.setRootTabletDir(newDir.toString());
        }

        return newDir.toString();
    } else {
        MetadataTableUtil.updateTabletDir(extent, newDir.toString(), context, zooLock);
        return newDir.toString();
    }
}

---

From source file:org.apache.accumulo.server.tabletserver.BulkFailedCopyProcessor.java

License:Apache License

@Override
public void process(String workID, byte[] data) {

    String paths[] = new String(data).split(",");

    Path orig = new Path(paths[0]);
    Path dest = new Path(paths[1]);
    Path tmp = new Path(dest.getParent(), dest.getName() + ".tmp");

    try {// ww w  .j  ava 2 s  .  c o  m
        FileSystem fs = TraceFileSystem.wrap(org.apache.accumulo.core.file.FileUtil
                .getFileSystem(CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration()));

        FileUtil.copy(fs, orig, fs, tmp, false, true, CachedConfiguration.getInstance());
        fs.rename(tmp, dest);
        log.debug("copied " + orig + " to " + dest);
    } catch (IOException ex) {
        try {
            FileSystem fs = TraceFileSystem.wrap(org.apache.accumulo.core.file.FileUtil.getFileSystem(
                    CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration()));

            fs.create(dest).close();
            log.warn(" marked " + dest + " failed", ex);
        } catch (IOException e) {
            log.error("Unable to create failure flag file " + dest, e);
        }
    }

}

---

From source file:org.apache.accumulo.server.test.functional.BulkFileTest.java

License:Apache License

@Override
public void run() throws Exception {
    Configuration conf = new Configuration();
    FileSystem fs = TraceFileSystem
            .wrap(FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration()));

    String dir = "/tmp/bulk_test_diff_files_89723987592";

    fs.delete(new Path(dir), true);

    FileSKVWriter writer1 = FileOperations.getInstance().openWriter(dir + "/f1." + Constants.MAPFILE_EXTENSION,
            fs, conf, ServerConfiguration.getSystemConfiguration());
    writer1.startDefaultLocalityGroup();
    writeData(writer1, 0, 333);//from   w w  w  . j a v a2  s .c o  m
    writer1.close();

    fs.rename(new Path(dir + "/f1." + Constants.MAPFILE_EXTENSION), new Path(dir + "/f1"));

    FileSKVWriter writer2 = FileOperations.getInstance().openWriter(dir + "/f2." + Constants.MAPFILE_EXTENSION,
            fs, conf, ServerConfiguration.getSystemConfiguration());
    writer2.startDefaultLocalityGroup();
    writeData(writer2, 334, 999);
    writer2.close();

    FileSKVWriter writer3 = FileOperations.getInstance().openWriter(dir + "/f3." + RFile.EXTENSION, fs, conf,
            ServerConfiguration.getSystemConfiguration());
    writer3.startDefaultLocalityGroup();
    writeData(writer3, 1000, 1999);
    writer3.close();

    bulkImport(fs, "bulkFile", dir);

    checkMapFiles("bulkFile", 6, 6, 1, 1);

    verifyData("bulkFile", 0, 1999);

}

---

From source file:org.apache.accumulo.server.test.randomwalk.shard.BulkInsert.java

License:Apache License

private void bulkImport(FileSystem fs, State state, String tableName, String rootDir, String prefix)
        throws Exception {
    while (true) {
        String bulkDir = rootDir + "/" + prefix + "_bulk";
        String failDir = rootDir + "/" + prefix + "_failure";
        Path failPath = new Path(failDir);
        fs.delete(failPath, true);//from   w ww.  j av  a  2s .  c o  m
        fs.mkdirs(failPath);
        state.getConnector().tableOperations().importDirectory(tableName, bulkDir, failDir, true);

        FileStatus[] failures = fs.listStatus(failPath);
        if (failures != null && failures.length > 0) {
            log.warn("Failed to bulk import some files, retrying ");

            for (FileStatus failure : failures) {
                if (!failure.getPath().getName().endsWith(".seq"))
                    fs.rename(failure.getPath(), new Path(new Path(bulkDir), failure.getPath().getName()));
                else
                    log.debug("Ignoring " + failure.getPath());
            }
            UtilWaitThread.sleep(3000);
        } else
            break;
    }
}

---

From source file:org.apache.accumulo.test.randomwalk.shard.BulkInsert.java

License:Apache License

private void bulkImport(FileSystem fs, State state, Environment env, String tableName, String rootDir,
        String prefix) throws Exception {
    while (true) {
        String bulkDir = rootDir + "/" + prefix + "_bulk";
        String failDir = rootDir + "/" + prefix + "_failure";
        Path failPath = new Path(failDir);
        fs.delete(failPath, true);/*w  w  w.j a v a 2 s.c  o  m*/
        fs.mkdirs(failPath);
        env.getConnector().tableOperations().importDirectory(tableName, bulkDir, failDir, true);

        FileStatus[] failures = fs.listStatus(failPath);
        if (failures != null && failures.length > 0) {
            log.warn("Failed to bulk import some files, retrying ");

            for (FileStatus failure : failures) {
                if (!failure.getPath().getName().endsWith(".seq"))
                    fs.rename(failure.getPath(), new Path(new Path(bulkDir), failure.getPath().getName()));
                else
                    log.debug("Ignoring " + failure.getPath());
            }
            sleepUninterruptibly(3, TimeUnit.SECONDS);
        } else
            break;
    }
}

---

From source file:org.apache.accumulo.testing.core.randomwalk.shard.BulkInsert.java

License:Apache License

private void bulkImport(FileSystem fs, State state, RandWalkEnv env, String tableName, String rootDir,
        String prefix) throws Exception {
    while (true) {
        String bulkDir = rootDir + "/" + prefix + "_bulk";
        String failDir = rootDir + "/" + prefix + "_failure";
        Path failPath = new Path(failDir);
        fs.delete(failPath, true);/*from  w  w  w  . j  a  v a 2s  . c o  m*/
        fs.mkdirs(failPath);
        env.getAccumuloConnector().tableOperations().importDirectory(tableName, bulkDir, failDir, true);

        FileStatus[] failures = fs.listStatus(failPath);
        if (failures != null && failures.length > 0) {
            log.warn("Failed to bulk import some files, retrying ");

            for (FileStatus failure : failures) {
                if (!failure.getPath().getName().endsWith(".seq"))
                    fs.rename(failure.getPath(), new Path(new Path(bulkDir), failure.getPath().getName()));
                else
                    log.debug("Ignoring " + failure.getPath());
            }
            sleepUninterruptibly(3, TimeUnit.SECONDS);
        } else
            break;
    }
}

---

From source file:org.apache.accumulo.tserver.BulkFailedCopyProcessor.java

License:Apache License

@Override
public void process(String workID, byte[] data) {

    String paths[] = new String(data, UTF_8).split(",");

    Path orig = new Path(paths[0]);
    Path dest = new Path(paths[1]);
    Path tmp = new Path(dest.getParent(), dest.getName() + ".tmp");

    try {/* w  ww  .j av  a  2s.  c  om*/
        VolumeManager vm = VolumeManagerImpl.get(SiteConfiguration.getInstance());
        FileSystem origFs = vm.getVolumeByPath(orig).getFileSystem();
        FileSystem destFs = vm.getVolumeByPath(dest).getFileSystem();

        FileUtil.copy(origFs, orig, destFs, tmp, false, true, CachedConfiguration.getInstance());
        destFs.rename(tmp, dest);
        log.debug("copied " + orig + " to " + dest);
    } catch (IOException ex) {
        try {
            VolumeManager vm = VolumeManagerImpl.get(SiteConfiguration.getInstance());
            FileSystem destFs = vm.getVolumeByPath(dest).getFileSystem();
            destFs.create(dest).close();
            log.warn(" marked " + dest + " failed", ex);
        } catch (IOException e) {
            log.error("Unable to create failure flag file " + dest, e);
        }
    }

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »