Example usage for org.apache.hadoop.fs FileSystem copyToLocalFile

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem copyToLocalFile.

Prototype

public void copyToLocalFile(Path src, Path dst) throws IOException 

Source Link

Document

Copy it a file from the remote filesystem to the local one.

Usage

From source file:org.apache.hive.spark.client.SparkClientUtilities.java

License:Apache License

/**
 * Create a URL from a string representing a path to a local file.
 * The path string can be just a path, or can start with file:/, file:///
 * @param path  path string/*from  w w w  .j  a  v  a2  s.  c  o  m*/
 * @return
 */
private static URL urlFromPathString(String path, Long timeStamp, Configuration conf, File localTmpDir) {
    URL url = null;
    try {
        if (StringUtils.indexOf(path, "file:/") == 0) {
            url = new URL(path);
        } else if (StringUtils.indexOf(path, "hdfs:/") == 0) {
            Path remoteFile = new Path(path);
            Path localFile = new Path(localTmpDir.getAbsolutePath() + File.separator + remoteFile.getName());
            Long currentTS = downloadedFiles.get(path);
            if (currentTS == null) {
                currentTS = -1L;
            }
            if (!new File(localFile.toString()).exists() || currentTS < timeStamp) {
                LOG.info("Copying " + remoteFile + " to " + localFile);
                FileSystem remoteFS = remoteFile.getFileSystem(conf);
                remoteFS.copyToLocalFile(remoteFile, localFile);
                downloadedFiles.put(path, timeStamp);
            }
            return urlFromPathString(localFile.toString(), timeStamp, conf, localTmpDir);
        } else {
            url = new File(path).toURL();
        }
    } catch (Exception err) {
        LOG.error("Bad URL " + path + ", ignoring path", err);
    }
    return url;
}

---

From source file:org.apache.impala.common.FileSystemUtil.java

License:Apache License

/**
 * Copies the source file to a destination path on the local filesystem.
 * Throws IOException on failure.// ww  w  . jav a 2  s .c  om
 */
public static void copyToLocal(Path source, Path dest) throws IOException {
    FileSystem fs = source.getFileSystem(CONF);
    fs.copyToLocalFile(source, dest);
}

---

From source file:org.apache.mahout.cf.taste.hadoop.pseudo.RecommenderReducer.java

License:Apache License

@Override
protected void setup(Context context) {
    Configuration jobConf = context.getConfiguration();
    String dataModelFile = jobConf.get(DATA_MODEL_FILE);
    String recommenderClassName = jobConf.get(RECOMMENDER_CLASS_NAME);
    FileDataModel fileDataModel;/*  w  ww . j ava2 s. co m*/
    try {
        Path dataModelPath = new Path(dataModelFile);
        FileSystem fs = FileSystem.get(dataModelPath.toUri(), jobConf);
        File tempDataFile = File.createTempFile("mahout-taste-hadoop", "txt");
        tempDataFile.deleteOnExit();
        fs.copyToLocalFile(dataModelPath, new Path(tempDataFile.getAbsolutePath()));
        fileDataModel = new FileDataModel(tempDataFile);
    } catch (IOException ioe) {
        throw new IllegalStateException(ioe);
    }
    try {
        Class<? extends Recommender> recommenderClass = Class.forName(recommenderClassName)
                .asSubclass(Recommender.class);
        Constructor<? extends Recommender> constructor = recommenderClass.getConstructor(DataModel.class);
        recommender = constructor.newInstance(fileDataModel);
    } catch (NoSuchMethodException nsme) {
        throw new IllegalStateException(nsme);
    } catch (ClassNotFoundException cnfe) {
        throw new IllegalStateException(cnfe);
    } catch (InstantiationException ie) {
        throw new IllegalStateException(ie);
    } catch (IllegalAccessException iae) {
        throw new IllegalStateException(iae);
    } catch (InvocationTargetException ite) {
        throw new IllegalStateException(ite.getCause());
    }
    recommendationsPerUser = jobConf.getInt(RECOMMENDATIONS_PER_USER, 10);
}

---

From source file:org.apache.mahout.cf.taste.hadoop.RecommenderMapper.java

License:Apache License

@Override
protected void setup(Context context) {
    Configuration jobConf = context.getConfiguration();
    String dataModelFile = jobConf.get(DATA_MODEL_FILE);
    String recommenderClassName = jobConf.get(RECOMMENDER_CLASS_NAME);
    FileDataModel fileDataModel;/*from  w  ww  .  j  a  va 2s  . c  om*/
    try {
        Path dataModelPath = new Path(dataModelFile);
        FileSystem fs = FileSystem.get(dataModelPath.toUri(), jobConf);
        File tempDataFile = File.createTempFile("mahout-taste-hadoop", "txt");
        tempDataFile.deleteOnExit();
        fs.copyToLocalFile(dataModelPath, new Path(tempDataFile.getAbsolutePath()));
        fileDataModel = new FileDataModel(tempDataFile);
    } catch (IOException ioe) {
        throw new RuntimeException(ioe);
    }
    try {
        Class<? extends Recommender> recommenderClass = Class.forName(recommenderClassName)
                .asSubclass(Recommender.class);
        Constructor<? extends Recommender> constructor = recommenderClass.getConstructor(DataModel.class);
        recommender = constructor.newInstance(fileDataModel);
    } catch (NoSuchMethodException nsme) {
        throw new RuntimeException(nsme);
    } catch (ClassNotFoundException cnfe) {
        throw new RuntimeException(cnfe);
    } catch (InstantiationException ie) {
        throw new RuntimeException(ie);
    } catch (IllegalAccessException iae) {
        throw new RuntimeException(iae);
    } catch (InvocationTargetException ite) {
        throw new RuntimeException(ite.getCause());
    }
    recommendationsPerUser = Integer.parseInt(jobConf.get(RECOMMENDATIONS_PER_USER));
}

---

From source file:org.apache.oozie.action.hadoop.GitMain.java

License:Apache License

/**
 * Gathers the Git authentication key from a FileSystem and copies it to a local
 * filesystem location/*from w  ww.ja  va  2  s.  co  m*/
 *
 * @param location where the key is located (an HDFS URI)
 * @return the location to where the key was saved
 */
@VisibleForTesting
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "File created without user input")
File getKeyFromFS(final Path location) throws IOException, URISyntaxException {
    final String keyCopyMsg = "Copied keys to local container!";

    final Configuration conf = new Configuration();
    final FileSystem fs = FileSystem.newInstance(new URI(nameNode), conf);

    final File key = createTempDir("git");

    fs.copyToLocalFile(location, new Path("file:///" + key.getAbsolutePath() + "/privkey"));
    System.out.println(keyCopyMsg);

    return new File(key.getAbsolutePath() + "/privkey");
}

---

From source file:org.apache.oozie.action.hadoop.LauncherMapper.java

License:Apache License

private void setupMainConfiguration() throws IOException {
    Path pathNew = new Path(new Path(actionDir, ACTION_CONF_XML),
            new Path(new File(ACTION_CONF_XML).getAbsolutePath()));
    FileSystem fs = FileSystem.get(pathNew.toUri(), getJobConf());
    fs.copyToLocalFile(new Path(actionDir, ACTION_CONF_XML),
            new Path(new File(ACTION_CONF_XML).getAbsolutePath()));

    System.setProperty("oozie.launcher.job.id", getJobConf().get("mapred.job.id"));
    System.setProperty(OOZIE_JOB_ID, getJobConf().get(OOZIE_JOB_ID));
    System.setProperty(OOZIE_ACTION_ID, getJobConf().get(OOZIE_ACTION_ID));
    System.setProperty("oozie.action.conf.xml", new File(ACTION_CONF_XML).getAbsolutePath());
    System.setProperty(ACTION_PREFIX + ACTION_DATA_EXTERNAL_CHILD_IDS,
            new File(ACTION_DATA_EXTERNAL_CHILD_IDS).getAbsolutePath());
    System.setProperty(ACTION_PREFIX + ACTION_DATA_STATS, new File(ACTION_DATA_STATS).getAbsolutePath());
    System.setProperty(ACTION_PREFIX + ACTION_DATA_NEW_ID, new File(ACTION_DATA_NEW_ID).getAbsolutePath());
    System.setProperty(ACTION_PREFIX + ACTION_DATA_OUTPUT_PROPS,
            new File(ACTION_DATA_OUTPUT_PROPS).getAbsolutePath());
    System.setProperty(ACTION_PREFIX + ACTION_DATA_ERROR_PROPS,
            new File(ACTION_DATA_ERROR_PROPS).getAbsolutePath());
    if (getJobConf().get(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME) != null) {
        System.setProperty(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME,
                getJobConf().get(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME));
    }//from  w w w .j  av a 2s. c o m

    String actionConfigClass = getJobConf().get(OOZIE_ACTION_CONFIG_CLASS);
    if (actionConfigClass != null) {
        System.setProperty(OOZIE_ACTION_CONFIG_CLASS, actionConfigClass);
    }
}

---

From source file:org.apache.phoenix.hive.HiveTestUtil.java

License:Apache License

public void copyDirectoryToLocal(Path src, Path dest) throws Exception {

    FileSystem srcFs = src.getFileSystem(conf);
    FileSystem destFs = dest.getFileSystem(conf);
    if (srcFs.exists(src)) {
        FileStatus[] files = srcFs.listStatus(src);
        for (FileStatus file : files) {
            String name = file.getPath().getName();
            Path dfs_path = file.getPath();
            Path local_path = new Path(dest, name);

            if (file.isDir()) {
                if (!destFs.exists(local_path)) {
                    destFs.mkdirs(local_path);
                }//  w w w.j  a  v  a 2 s.  c om
                copyDirectoryToLocal(dfs_path, local_path);
            } else {
                srcFs.copyToLocalFile(dfs_path, local_path);
            }
        }
    }
}

---

From source file:org.apache.pig.impl.io.FileLocalizer.java

License:Apache License

/**
 * Copies the files from remote to local filesystem.
 * When 'multipleFiles' is set the path could point to multiple files
 * through globs or a directory. In this case, return array contains multiple
 * files, otherwise a single file is returned.
 *
 * If pig.jars.relative.to.dfs is true then a relative path is assumed to be
 * relative to the default filesystem's active directory.
 * Else they are assumed to be relative to the local working directory.
 *
 * @param properties/*from   w  w  w . ja v  a 2s .c  o  m*/
 * @param filePath
 * @param multipleFiles
 * @return
 */
private static FetchFileRet[] fetchFilesInternal(Properties properties, String filePath, boolean multipleFiles)
        throws IOException {

    Path path = new Path(filePath);
    URI uri = path.toUri();
    Configuration conf = new Configuration();
    ConfigurationUtil.mergeConf(conf, ConfigurationUtil.toConfiguration(properties));

    // if there is no schema or if the schema is "local", then it is
    // expected to be a local path.

    FileSystem localFs = FileSystem.getLocal(conf);
    FileSystem srcFs;
    if ((!"true".equals(properties.getProperty("pig.jars.relative.to.dfs")) && uri.getScheme() == null) ||
    // For Windows local files
            (uri.getScheme() == null && uri.getPath().matches("^/[A-Za-z]:.*"))
            || uri.getScheme().equals("local")) {
        srcFs = localFs;
    } else {
        srcFs = path.getFileSystem(conf);
    }

    FileStatus[] files;

    if (multipleFiles) {
        files = srcFs.globStatus(path);
    } else {
        files = new FileStatus[] { srcFs.getFileStatus(path) };
    }
    if (files == null || files.length == 0) {
        throw new ExecException("file '" + filePath + "' does not exist.", 101, PigException.INPUT);
    }

    FetchFileRet[] fetchFiles = new FetchFileRet[files.length];
    int idx = 0;

    for (FileStatus file : files) {
        // should throw an exception if this is not a file?

        String pathname = file.getPath().toUri().getPath();
        String filename = file.getPath().getName();

        if (srcFs == localFs) {
            fetchFiles[idx++] = new FetchFileRet(new File(pathname), false);
        } else {
            // fetch from remote:
            File dest = new File(localTempDir, filename);
            dest.deleteOnExit();
            try {
                srcFs.copyToLocalFile(file.getPath(), new Path(dest.getAbsolutePath()));
            } catch (IOException e) {
                throw new ExecException("Could not copy " + filePath + " to local destination " + dest, 101,
                        PigException.INPUT, e);
            }
            fetchFiles[idx++] = new FetchFileRet(dest, true);
        }
    }

    return fetchFiles;
}

---

From source file:org.apache.rya.accumulo.mr.merge.CopyTool.java

License:Apache License

/**
 * Copies the file from HDFS into the local file system.
 * @param hdfsInputPath the HDFS input {@link Path}.
 * @param localOutputPath the local system output {@link Path}.
 * @param configuration the {@link Configuration} to use.
 * @throws IOException/* ww  w . jav a2  s .  c  o  m*/
 */
public static void copyHdfsToLocal(final Path hdfsInputPath, final Path localOutputPath,
        final Configuration configuration) throws IOException {
    final FileSystem fs = FileSystem.get(configuration);
    fs.copyToLocalFile(hdfsInputPath, localOutputPath);
}

---

From source file:org.apache.tez.analyzer.TestAnalyzer.java

License:Apache License

private DagInfo getDagInfo(String dagId) throws Exception {
    // sleep for a bit to let ATS events be sent from AM
    DagInfo dagInfo = null;/*from  w w  w. ja  v a  2 s  . c  o  m*/
    if (usingATS) {
        //Export the data from ATS
        String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR,
                "--yarnTimelineAddress=" + yarnTimelineAddress };

        int result = ATSImportTool.process(args);
        assertTrue(result == 0);

        //Parse ATS data and verify results
        //Parse downloaded contents
        File downloadedFile = new File(DOWNLOAD_DIR + Path.SEPARATOR + dagId + ".zip");
        ATSFileParser parser = new ATSFileParser(downloadedFile);
        dagInfo = parser.getDAGData(dagId);
        assertTrue(dagInfo.getDagId().equals(dagId));
    } else {
        if (!downloadedSimpleHistoryFile) {
            downloadedSimpleHistoryFile = true;
            TezDAGID tezDAGID = TezDAGID.fromString(dagId);
            ApplicationAttemptId applicationAttemptId = ApplicationAttemptId
                    .newInstance(tezDAGID.getApplicationId(), 1);
            Path historyPath = new Path(miniTezCluster.getConfig().get("fs.defaultFS") + SIMPLE_HISTORY_DIR
                    + HISTORY_TXT + "." + applicationAttemptId);
            FileSystem fs = historyPath.getFileSystem(miniTezCluster.getConfig());

            Path localPath = new Path(DOWNLOAD_DIR, HISTORY_TXT);
            fs.copyToLocalFile(historyPath, localPath);
        }
        //Now parse via SimpleHistory
        File localFile = new File(DOWNLOAD_DIR, HISTORY_TXT);
        SimpleHistoryParser parser = new SimpleHistoryParser(localFile);
        dagInfo = parser.getDAGData(dagId);
        assertTrue(dagInfo.getDagId().equals(dagId));
    }
    return dagInfo;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• »