Example usage for org.apache.hadoop.fs FileSystem rename

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem rename.

Prototype

public abstract boolean rename(Path src, Path dst) throws IOException;

Source Link

Document

Renames Path src to Path dst.

Usage

From source file:com.blackberry.logdriver.LockedFs.java

License:Apache License

@SuppressWarnings("deprecation")
public void move(Configuration conf, String[] from, String to) throws IOException {
    FileSystem fs = FileSystem.get(conf);

    List<FileStatus> fromList = new ArrayList<FileStatus>();
    for (String s : from) {
        FileStatus[] statuses = fs.globStatus(new Path(s));
        if (statuses == null) {
            continue;
        }//from   w w w.j  a  v  a2  s  . c  o  m
        for (FileStatus status : statuses) {
            fromList.add(status);
        }
    }

    Path toPath = new Path(to);
    Boolean toExists = fs.exists(toPath);
    FileStatus toFileStatus = null;
    if (toExists) {
        toFileStatus = fs.getFileStatus(toPath);
    }

    // If there is no from, that's a problem.
    if (fromList.isEmpty()) {
        throw new IOException("No input files found");
    }

    // If the to exists, and is a file, that's a problem too.
    if (toExists && !toFileStatus.isDir()) {
        throw new IOException("Destination file exists:" + to);
    }

    // If the destination exists, and is a directory, then ensure that none of
    // the from list names will clash with existing contents of the directory.
    if (toExists && toFileStatus.isDir()) {
        for (FileStatus fromStatus : fromList) {
            String name = fromStatus.getPath().getName();
            if (fs.exists(new Path(toPath, name))) {
                throw new IOException("Destination file exists:" + to + "/" + name);
            }
        }
    }

    // If the destination doesn't exist, but it ends with a slash, then create
    // it as a directory.
    if (!toExists && to.endsWith("/")) {
        fs.mkdirs(toPath);
        toFileStatus = fs.getFileStatus(toPath);
        toExists = true;
    }

    // If the destination doesn't exist, and there is more than one 'from', then
    // create a directory.
    if (!toExists && fromList.size() > 1) {
        fs.mkdirs(toPath);
        toFileStatus = fs.getFileStatus(toPath);
    }

    // If there was only one from, then just rename it to to
    if (fromList.size() == 1) {
        fs.mkdirs(toPath.getParent());
        fs.rename(fromList.get(0).getPath(), toPath);
    }

    // If there was more than one from, then for each file in the from list,
    // move it to the to directory.
    if (fromList.size() > 1) {
        for (FileStatus fromStatus : fromList) {
            String name = fromStatus.getPath().getName();
            fs.rename(fromStatus.getPath(), new Path(toPath, name));
        }
    }
}

---

From source file:com.chinacache.robin.format.action.SimpleMoveFileAction.java

License:Apache License

@Override
public void execute(FileSystem fileSystem, Path filePath) throws IOException {
    String newDestPath = destination;
    newDestPath += "/BILog/src/" + dfs.format(new Date()) + "/" + userName + "/" + channelId + "/";
    Path destPath = new Path(newDestPath);
    LOG.info("Moving file {} to {}", filePath, destPath);
    fileSystem.mkdirs(destPath);/*from   w ww.j a v a 2 s .  c o m*/
    fileSystem.rename(filePath, destPath);
}

---

From source file:com.cloudera.crunch.impl.mr.exec.CrunchJob.java

License:Open Source License

private void handleMultiPaths() throws IOException {
    if (!multiPaths.isEmpty()) {
        // Need to handle moving the data from the output directory of the
        // job to the output locations specified in the paths.
        FileSystem fs = FileSystem.get(job.getConfiguration());
        for (int i = 0; i < multiPaths.size(); i++) {
            Path src = new Path(workingPath, PlanningParameters.MULTI_OUTPUT_PREFIX + i + "-*");
            Path[] srcs = FileUtil.stat2Paths(fs.globStatus(src), src);
            Path dst = multiPaths.get(i);
            if (!fs.exists(dst)) {
                fs.mkdirs(dst);/*from   w w  w. j  a v a  2 s.co  m*/
            }
            int minPartIndex = getMinPartIndex(dst, fs);
            for (Path s : srcs) {
                fs.rename(s, getDestFile(s, dst, minPartIndex++));
            }
        }
    }
}

---

From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.handlers.RENAMEHandler.java

License:Apache License

@Override
protected RENAMEResponse doHandle(NFS4Handler server, Session session, RENAMERequest request)
        throws NFS4Exception, IOException {
    if (session.getCurrentFileHandle() == null || session.getSavedFileHandle() == null) {
        throw new NFS4Exception(NFS4ERR_NOFILEHANDLE);
    }/*from  www .  ja va2 s  . c o  m*/
    if ("".equals(request.getOldName()) || "".equals(request.getNewName())) {
        throw new NFS4Exception(NFS4ERR_INVAL);
    }
    FileSystem fs = session.getFileSystem();
    Path oldParentPath = server.getPath(session.getSavedFileHandle());
    Path oldPath = new Path(oldParentPath, request.getOldName());
    Path newParentPath = server.getPath(session.getCurrentFileHandle());
    Path newPath = new Path(newParentPath, request.getNewName());
    if (!(fs.getFileStatus(oldParentPath).isDir() && fs.getFileStatus(newParentPath).isDir())) {
        throw new NFS4Exception(NFS4ERR_NOTDIR);
    }
    if (!server.fileExists(fs, oldPath)) {
        throw new NFS4Exception(NFS4ERR_NOENT, "Path " + oldPath + " does not exist.");
    }
    if (server.fileExists(fs, newPath)) {
        // TODO according to the RFC we are supposed to check to see if 
        // the entry which exists is compatible (overwrite file and 
        // empty directory if the "old" item is a file or dir respectively.
        throw new NFS4Exception(NFS4ERR_EXIST, "Path " + newPath + " exists.");
    }
    // we won't support renaming files which are open
    // but it happens fairly often that due to tcp/ip
    // the rename request is received before the close.
    // Below we delay the rename if the file is open 
    for (int i = 0; i < 5; i++) {
        if (!server.isFileOpen(oldPath)) {
            break;
        }
        try {
            Thread.sleep(100L);
        } catch (InterruptedException e) {
            throw new IOException("Interrupted while waiting for file to close", e);
        }
    }
    if (server.isFileOpen(oldPath)) {
        throw new NFS4Exception(NFS4ERR_FILE_OPEN);
    }
    LOGGER.info(session.getSessionID() + " Renaming " + oldPath + " to " + newPath);
    long beforeSource = fs.getFileStatus(oldParentPath).getModificationTime();
    long beforeDest = fs.getFileStatus(newParentPath).getModificationTime();
    if (!fs.rename(oldPath, newPath)) {
        throw new NFS4Exception(NFS4ERR_IO);
    }
    long afterSource = fs.getFileStatus(oldParentPath).getModificationTime();
    long afterDest = fs.getFileStatus(newParentPath).getModificationTime();
    RENAMEResponse response = createResponse();
    response.setChangeInfoSource(ChangeInfo.newChangeInfo(true, beforeSource, afterSource));
    response.setChangeInfoDest(ChangeInfo.newChangeInfo(true, beforeDest, afterDest));
    response.setStatus(NFS4_OK);
    return response;
}

---

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testRename() throws Exception {
    FileSystem fs = FileSystem.get(getHadoopConf());
    Path path = new Path(getHadoopTestDir(), "foo");
    fs.mkdirs(path);/*from  w ww.j  a va 2s.  c o  m*/
    fs.close();
    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    fs = FileSystem.get(getJettyURL().toURI(), conf);
    Path oldPath = new Path(path.toUri().getPath());
    Path newPath = new Path(path.getParent(), "bar");
    fs.rename(oldPath, newPath);
    fs.close();
    fs = FileSystem.get(getHadoopConf());
    Assert.assertFalse(fs.exists(oldPath));
    Assert.assertTrue(fs.exists(newPath));
    fs.close();
}

---

From source file:com.cloudera.hoop.fs.FSRename.java

License:Open Source License

/**
 * Executes the filesystem operation./*from   w  w  w . j  a va2  s . c  o  m*/
 *
 * @param fs filesystem instance to use.
 * @return <code>true</code> if the rename operation was successful,
 * <code>false</code> otherwise.
 * @throws IOException thrown if an IO error occured.
 */
@Override
public JSONObject execute(FileSystem fs) throws IOException {
    boolean renamed = fs.rename(path, toPath);
    return FSUtils.toJSON("rename", renamed);
}

---

From source file:com.cloudera.impala.common.FileSystemUtil.java

License:Apache License

/**
 * Moves (renames) the given file to a new location (either another directory or a
 * file. If renameIfAlreadyExists is true, no error will be thrown if a file with the
 * same name already exists in the destination location. Instead, a UUID will be
 * appended to the base file name, preserving the the existing file extension.
 * If renameIfAlreadyExists is false, an IOException will be thrown if there is a
 * file name conflict./*from   ww  w .j  ava  2 s  .c  om*/
 */
public static void moveFile(Path sourceFile, Path dest, boolean renameIfAlreadyExists) throws IOException {
    FileSystem fs = dest.getFileSystem(CONF);

    Path destFile = fs.isDirectory(dest) ? new Path(dest, sourceFile.getName()) : dest;
    // If a file with the same name does not already exist in the destination location
    // then use the same file name. Otherwise, generate a unique file name.
    if (renameIfAlreadyExists && fs.exists(destFile)) {
        Path destDir = fs.isDirectory(dest) ? dest : dest.getParent();
        destFile = new Path(destDir, appendToBaseFileName(destFile.getName(), UUID.randomUUID().toString()));
    }
    LOG.debug(String.format("Moving '%s' to '%s'", sourceFile.toString(), destFile.toString()));
    // Move (rename) the file.
    fs.rename(sourceFile, destFile);
}

---

From source file:com.cloudera.oryx.ml.MLUpdate.java

License:Open Source License

@Override
public void runUpdate(JavaSparkContext sparkContext, long timestamp, JavaPairRDD<Object, M> newKeyMessageData,
        JavaPairRDD<Object, M> pastKeyMessageData, String modelDirString,
        TopicProducer<String, String> modelUpdateTopic) throws IOException, InterruptedException {

    Objects.requireNonNull(newKeyMessageData);

    JavaRDD<M> newData = newKeyMessageData.values();
    JavaRDD<M> pastData = pastKeyMessageData == null ? null : pastKeyMessageData.values();

    if (newData != null) {
        newData.cache();//from   w w w  .jav a 2s. c o m
        // This forces caching of the RDD. This shouldn't be necessary but we see some freezes
        // when many workers try to materialize the RDDs at once. Hence the workaround.
        newData.foreachPartition(p -> {
        });
    }
    if (pastData != null) {
        pastData.cache();
        pastData.foreachPartition(p -> {
        });
    }

    List<HyperParamValues<?>> hyperParamValues = getHyperParameterValues();
    int valuesPerHyperParam = HyperParams.chooseValuesPerHyperParam(hyperParamValues.size(), candidates);
    List<List<?>> hyperParameterCombos = HyperParams.chooseHyperParameterCombos(hyperParamValues, candidates,
            valuesPerHyperParam);

    Path modelDir = new Path(modelDirString);
    Path tempModelPath = new Path(modelDir, ".temporary");
    Path candidatesPath = new Path(tempModelPath, Long.toString(System.currentTimeMillis()));

    FileSystem fs = FileSystem.get(modelDir.toUri(), sparkContext.hadoopConfiguration());
    fs.mkdirs(candidatesPath);

    Path bestCandidatePath = findBestCandidatePath(sparkContext, newData, pastData, hyperParameterCombos,
            candidatesPath);

    Path finalPath = new Path(modelDir, Long.toString(System.currentTimeMillis()));
    if (bestCandidatePath == null) {
        log.info("Unable to build any model");
    } else {
        // Move best model into place
        fs.rename(bestCandidatePath, finalPath);
    }
    // Then delete everything else
    fs.delete(candidatesPath, true);

    if (modelUpdateTopic == null) {
        log.info("No update topic configured, not publishing models to a topic");
    } else {
        // Push PMML model onto update topic, if it exists
        Path bestModelPath = new Path(finalPath, MODEL_FILE_NAME);
        if (fs.exists(bestModelPath)) {
            FileStatus bestModelPathFS = fs.getFileStatus(bestModelPath);
            PMML bestModel = null;
            boolean modelNeededForUpdates = canPublishAdditionalModelData();
            boolean modelNotTooLarge = bestModelPathFS.getLen() <= maxMessageSize;
            if (modelNeededForUpdates || modelNotTooLarge) {
                // Either the model is required for publishAdditionalModelData, or required because it's going to
                // be serialized to Kafka
                try (InputStream in = fs.open(bestModelPath)) {
                    bestModel = PMMLUtils.read(in);
                }
            }

            if (modelNotTooLarge) {
                modelUpdateTopic.send("MODEL", PMMLUtils.toString(bestModel));
            } else {
                modelUpdateTopic.send("MODEL-REF", fs.makeQualified(bestModelPath).toString());
            }

            if (modelNeededForUpdates) {
                publishAdditionalModelData(sparkContext, bestModel, newData, pastData, finalPath,
                        modelUpdateTopic);
            }
        }
    }

    if (newData != null) {
        newData.unpersist();
    }
    if (pastData != null) {
        pastData.unpersist();
    }
}

---

From source file:com.cloudera.RenameTest.java

License:Apache License

public static void testFileSystemRename(URI uri) throws Exception {
    FileSystem fs = FileSystem.get(uri, new Configuration());
    Path testDir = new Path(new Path(uri), "testdir");
    System.out.println("mkdir " + testDir);
    fs.mkdirs(testDir);/*from   www.ja  v a2s  .  c  o m*/
    Path testFile = new Path(new Path(uri), "testfile");
    System.out.println("create " + testFile);
    FSDataOutputStream fos = fs.create(testFile);
    fos.close();
    System.out.println("rename " + testFile + " -> " + testDir);
    fs.rename(testFile, testDir);
}

---

From source file:com.cloudera.sqoop.util.AppendUtils.java

License:Apache License

/**
 * Move files from source to target using a specified starting partition.
 *//*from  w w  w  .  j  a v a 2s.  c o m*/
private void moveFiles(FileSystem fs, Path sourceDir, Path targetDir, int partitionStart) throws IOException {

    NumberFormat numpart = NumberFormat.getInstance();
    numpart.setMinimumIntegerDigits(PARTITION_DIGITS);
    numpart.setGroupingUsed(false);
    Pattern patt = Pattern.compile("part.*-([0-9][0-9][0-9][0-9][0-9]).*");
    FileStatus[] tempFiles = fs.listStatus(sourceDir);

    if (null == tempFiles) {
        // If we've already checked that the dir exists, and now it can't be
        // listed, this is a genuine error (permissions, fs integrity, or other).
        throw new IOException("Could not list files from " + sourceDir);
    }

    // Move and rename files & directories from temporary to target-dir thus
    // appending file's next partition
    for (FileStatus fileStat : tempFiles) {
        if (!fileStat.isDir()) {
            // Move imported data files
            String filename = fileStat.getPath().getName();
            Matcher mat = patt.matcher(filename);
            if (mat.matches()) {
                String name = getFilename(filename);
                String fileToMove = name.concat(numpart.format(partitionStart++));
                String extension = getFileExtension(filename);
                if (extension != null) {
                    fileToMove = fileToMove.concat(extension);
                }
                LOG.debug("Filename: " + filename + " repartitioned to: " + fileToMove);
                fs.rename(fileStat.getPath(), new Path(targetDir, fileToMove));
            }
        } else {
            // Move directories (_logs & any other)
            String dirName = fileStat.getPath().getName();
            Path path = new Path(targetDir, dirName);
            int dirNumber = 0;
            while (fs.exists(path)) {
                path = new Path(targetDir, dirName.concat("-").concat(numpart.format(dirNumber++)));
            }
            LOG.debug("Directory: " + dirName + " renamed to: " + path.getName());
            fs.rename(fileStat.getPath(), path);
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »