Example usage for org.apache.hadoop.fs FileSystem copyFromLocalFile

List of usage examples for org.apache.hadoop.fs FileSystem copyFromLocalFile

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. copyFromLocalFile

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem copyFromLocalFile.

Prototype

public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException

Source Link

Document

The src file is on the local disk.

Usage

From source file:org.apache.carbondata.spark.load.CarbonLoaderUtil.java

License:Apache License

public static void copyMergeToHDFS(String databaseName, String tableName, String factTable, String hdfsLocation,
        String localStore, int currentRestructNumber, String mergedLoadName) {
    try {//  w ww .j  a  v  a  2 s  .c  o m
        //If the hdfs store and the local store configured differently, then copy
        if (hdfsLocation != null && !hdfsLocation.equals(localStore)) {
            /**
             * Identify the Load_X folder from the local store folder
             */
            String currentloadedStore = localStore;
            currentloadedStore = currentloadedStore + File.separator + databaseName + File.separator
                    + tableName;

            int rsCounter = currentRestructNumber;

            if (rsCounter == -1) {
                LOGGER.info("Unable to find the local store details (RS_-1) " + currentloadedStore);
                return;
            }
            String localLoadedTable = currentloadedStore + File.separator
                    + CarbonCommonConstants.RESTRUCTRE_FOLDER + rsCounter + File.separator + factTable;

            localLoadedTable = localLoadedTable.replace("\\", "/");

            int loadCounter = CarbonUtil.checkAndReturnCurrentLoadFolderNumber(localLoadedTable);

            if (loadCounter == -1) {
                LOGGER.info("Unable to find the local store details (Load_-1) " + currentloadedStore);

                return;
            }

            String localLoadName = CarbonCommonConstants.LOAD_FOLDER + mergedLoadName;
            String localLoadFolder = localLoadedTable + File.separator + CarbonCommonConstants.LOAD_FOLDER
                    + mergedLoadName;

            LOGGER.info("Local data loaded folder ... = " + localLoadFolder);

            //Identify the Load_X folder in the HDFS store
            String hdfsStoreLocation = hdfsLocation;
            hdfsStoreLocation = hdfsStoreLocation + File.separator + databaseName + File.separator + tableName;

            rsCounter = currentRestructNumber;
            if (rsCounter == -1) {
                rsCounter = 0;
            }

            String hdfsLoadedTable = hdfsStoreLocation + File.separator
                    + CarbonCommonConstants.RESTRUCTRE_FOLDER + rsCounter + File.separator + factTable;

            hdfsLoadedTable = hdfsLoadedTable.replace("\\", "/");

            String hdfsStoreLoadFolder = hdfsLoadedTable + File.separator + localLoadName;

            LOGGER.info("HDFS data load folder ... = " + hdfsStoreLoadFolder);

            // Copy the data created through latest ETL run, to the HDFS store
            LOGGER.info("Copying " + localLoadFolder + " --> " + hdfsStoreLoadFolder);

            hdfsStoreLoadFolder = hdfsStoreLoadFolder.replace("\\", "/");
            Path path = new Path(hdfsStoreLocation);

            FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
            fs.copyFromLocalFile(true, true, new Path(localLoadFolder), new Path(hdfsStoreLoadFolder));

            LOGGER.info("Copying sliceMetaData from " + localLoadedTable + " --> " + hdfsLoadedTable);

        } else {
            LOGGER.info("Separate carbon.storelocation.hdfs is not configured for hdfs store path");
        }
    } catch (RuntimeException e) {
        LOGGER.info(e.getMessage());
    } catch (Exception e) {
        LOGGER.info(e.getMessage());
    }
}

From source file:org.apache.falcon.recipe.RecipeTool.java

License:Apache License

private static void copyFileFromLocalToHdfs(final String localFilePath, final String hdfsFilePath,
        final boolean copyDir, final String hdfsFileDirPath, final FileSystem fs) throws IOException {
    /* If directory already exists and has contents, copyFromLocalFile with overwrite set to yes will fail with
     * "Target is a directory". Delete the directory */
    if (copyDir) {
        Path hdfsPath = new Path(hdfsFileDirPath);
        fs.delete(hdfsPath, true);/*w w  w  . j  a  v a  2  s .  co  m*/
    }

    /* For cases where validation of process entity file fails, the artifacts would have been already copied to
     * HDFS. Set overwrite to true so that next submit recipe copies updated artifacts from local FS to HDFS */
    fs.copyFromLocalFile(false, true, new Path(localFilePath), new Path(hdfsFilePath));
}

From source file:org.apache.falcon.regression.core.util.HadoopUtil.java

License:Apache License

/**
 * Copies files from a source directory to target directories on hdfs.
 * @param fs target filesystem//from   w w  w.  ja v a  2 s. c  o  m
 * @param inputPath source location
 * @param remotePathPrefix prefix for target directories
 * @param remoteLocations target directories
 * @return list of exact locations where data was copied
 * @throws IOException
 */
public static List<String> flattenAndPutDataInFolder(FileSystem fs, String inputPath, String remotePathPrefix,
        List<String> remoteLocations) throws IOException {
    if (StringUtils.isNotEmpty(remotePathPrefix)) {
        deleteDirIfExists(remotePathPrefix, fs);
    }
    LOGGER.info("Creating data in folders: \n" + remoteLocations);
    File input = new File(inputPath);
    File[] files = input.isDirectory() ? input.listFiles() : new File[] { input };
    List<Path> filePaths = new ArrayList<>();
    assert files != null;
    for (final File file : files) {
        if (!file.isDirectory()) {
            final Path filePath = new Path(file.getAbsolutePath());
            filePaths.add(filePath);
        }
    }
    if (!remotePathPrefix.endsWith("/") && !remoteLocations.get(0).startsWith("/")) {
        remotePathPrefix += "/";
    }
    List<String> locations = new ArrayList<>();
    for (String remoteDir : remoteLocations) {
        String remoteLocation = remotePathPrefix + remoteDir;
        remoteLocation = cutProtocol(remoteLocation);
        locations.add(remoteLocation);
        LOGGER.info(String.format("copying to: %s files: %s", fs.getUri() + remoteLocation,
                Arrays.toString(files)));
        if (!fs.exists(new Path(remoteLocation))) {
            fs.mkdirs(new Path(remoteLocation));
        }
        fs.copyFromLocalFile(false, true, filePaths.toArray(new Path[filePaths.size()]),
                new Path(remoteLocation));
    }
    return locations;
}

From source file:org.apache.falcon.resource.AbstractTestBase.java

License:Apache License

@BeforeClass
public void configure() throws Exception {
    StartupProperties.get().setProperty("application.services",
            StartupProperties.get().getProperty("application.services")
                    .replace("org.apache.falcon.service.ProcessSubscriberService", ""));
    String store = StartupProperties.get().getProperty("config.store.uri");
    StartupProperties.get().setProperty("config.store.uri", store + System.currentTimeMillis());
    if (new File("webapp/src/main/webapp").exists()) {
        this.server = new EmbeddedServer(15000, "webapp/src/main/webapp");
    } else if (new File("src/main/webapp").exists()) {
        this.server = new EmbeddedServer(15000, "src/main/webapp");
    } else {/*  ww w.jav  a2  s  .  com*/
        throw new RuntimeException("Cannot run jersey tests");
    }
    ClientConfig config = new DefaultClientConfig();
    Client client = Client.create(config);
    this.service = client.resource(UriBuilder.fromUri(BASE_URL).build());
    this.server.start();

    if (System.getProperty("falcon.test.hadoop.embedded", "true").equals("true")) {
        CLUSTER_FILE_TEMPLATE = "target/cluster-template.xml";
        this.cluster = EmbeddedCluster.newCluster("##cluster##", true);
        Cluster clusterEntity = this.cluster.getCluster();
        FileOutputStream out = new FileOutputStream(CLUSTER_FILE_TEMPLATE);
        marshaller.marshal(clusterEntity, out);
        out.close();
    } else {
        Map<String, String> overlay = new HashMap<String, String>();
        overlay.put("cluster", RandomStringUtils.randomAlphabetic(5));
        String file = overlayParametersOverTemplate(CLUSTER_FILE_TEMPLATE, overlay);
        this.cluster = StandAloneCluster.newCluster(file);
        clusterName = cluster.getCluster().getName();
    }

    cleanupStore();

    // setup dependent workflow and lipath in hdfs
    FileSystem fs = FileSystem.get(this.cluster.getConf());
    fs.mkdirs(new Path("/falcon"), new FsPermission((short) 511));

    Path wfParent = new Path("/falcon/test");
    fs.delete(wfParent, true);
    Path wfPath = new Path(wfParent, "workflow");
    fs.mkdirs(wfPath);
    fs.copyFromLocalFile(false, true, new Path(this.getClass().getResource("/fs-workflow.xml").getPath()),
            new Path(wfPath, "workflow.xml"));
    fs.mkdirs(new Path(wfParent, "input/2012/04/20/00"));
    Path outPath = new Path(wfParent, "output");
    fs.mkdirs(outPath);
    fs.setPermission(outPath, new FsPermission((short) 511));
}

From source file:org.apache.falcon.resource.TestContext.java

License:Apache License

public static void prepare(String clusterTemplate, boolean disableLineage) throws Exception {
    // setup a logged in user
    CurrentUser.authenticate(REMOTE_USER);

    if (disableLineage) {
        // disable recording lineage metadata
        String services = StartupProperties.get().getProperty("application.services");
        StartupProperties.get().setProperty("application.services",
                services.replace("org.apache.falcon.metadata.MetadataMappingService", ""));
    }/*from   ww w  . ja va 2s. c o  m*/

    Map<String, String> overlay = new HashMap<String, String>();
    overlay.put("cluster", RandomStringUtils.randomAlphabetic(5));
    overlay.put("colo", DeploymentUtil.getCurrentColo());
    TestContext.overlayParametersOverTemplate(clusterTemplate, overlay);
    EmbeddedCluster cluster = EmbeddedCluster.newCluster(overlay.get("cluster"), true);

    cleanupStore();

    // setup dependent workflow and lipath in hdfs
    FileSystem fs = FileSystem.get(cluster.getConf());
    mkdir(fs, new Path("/falcon"), new FsPermission((short) 511));

    Path wfParent = new Path("/falcon/test");
    fs.delete(wfParent, true);
    Path wfPath = new Path(wfParent, "workflow");
    mkdir(fs, wfPath);
    mkdir(fs, new Path("/falcon/test/workflow/lib"));
    fs.copyFromLocalFile(false, true, new Path(TestContext.class.getResource("/fs-workflow.xml").getPath()),
            new Path(wfPath, "workflow.xml"));
    mkdir(fs, new Path(wfParent, "input/2012/04/20/00"));
    Path outPath = new Path(wfParent, "output");
    mkdir(fs, outPath, new FsPermission((short) 511));

    // init cluster locations
    initClusterLocations(cluster, fs);
}

From source file:org.apache.falcon.service.SharedLibraryHostingService.java

License:Apache License

@SuppressWarnings("ConstantConditions")
public static void pushLibsToHDFS(FileSystem fs, String src, Path target, FalconPathFilter pathFilter)
        throws IOException, FalconException {
    if (StringUtils.isEmpty(src)) {
        return;/*w ww .j av a  2s . c  o  m*/
    }
    LOG.debug("Copying libs from {}", src);
    createTargetPath(fs, target);

    for (String srcPaths : src.split(",")) {
        File srcFile = new File(srcPaths);
        File[] srcFiles = new File[] { srcFile };
        if (srcFiles != null) {
            if (srcFile.isDirectory()) {
                srcFiles = srcFile.listFiles();
            }
        }

        if (srcFiles != null) {
            for (File file : srcFiles) {
                Path path = new Path(file.getAbsolutePath());
                String jarName = StringUtils.removeEnd(path.getName(), ".jar");
                if (pathFilter != null) {
                    if (!pathFilter.accept(path)) {
                        continue;
                    }
                    jarName = pathFilter.getJarName(path);
                }

                Path targetFile = new Path(target, jarName + ".jar");
                if (fs.exists(targetFile)) {
                    FileStatus fstat = fs.getFileStatus(targetFile);
                    if (fstat.getLen() == file.length()) {
                        continue;
                    }
                }
                fs.copyFromLocalFile(false, true, new Path(file.getAbsolutePath()), targetFile);
                fs.setPermission(targetFile, HadoopClientFactory.READ_EXECUTE_PERMISSION);
                LOG.info("Copied {} to {} in {}", file.getAbsolutePath(), targetFile.toString(), fs.getUri());
            }
        }
    }
}

From source file:org.apache.giraph.utils.DistributedCacheUtils.java

License:Apache License

/**
 * Copy a file to HDFS if it is local. If the path is already in HDFS, this
 * call does nothing.//  w ww . ja va 2  s. c  om
 *
 * @param path path to file
 * @param conf Configuration
 * @return path to file on HDFS.
 */
public static Path copyToHdfs(Path path, Configuration conf) {
    if (path.toString().startsWith("hdfs://")) {
        // Already on HDFS
        return path;
    }

    FileSystem fs = null;
    try {
        fs = FileSystem.get(conf);
    } catch (IOException e) {
        throw new IllegalArgumentException("Failed to get HDFS FileSystem", e);
    }
    String name = getBaseName(path.toString()) + "-" + System.nanoTime();
    Path remotePath = new Path("/tmp/giraph", name);
    LOG.info("copyToHdfsIfNecessary: Copying " + path + " to " + remotePath + " on hdfs " + fs.getUri());
    try {
        fs.copyFromLocalFile(false, true, path, remotePath);
    } catch (IOException e) {
        throw new IllegalArgumentException("Failed to copy jython script from local path " + path
                + " to hdfs path " + remotePath + " on hdfs " + fs.getUri(), e);
    }
    return remotePath;
}

From source file:org.apache.giraph.yarn.GiraphYarnClient.java

License:Apache License

/**
 * Register all local jar files from GiraphConstants.GIRAPH_YARN_LIBJARS
 * in the LocalResources map, copy to HDFS on that same registered path.
 * @param map the LocalResources list to populate.
 *///from w w  w.j av a 2s . co  m
private void addLocalJarsToResourceMap(Map<String, LocalResource> map) throws IOException {
    Set<String> jars = Sets.newHashSet();
    LOG.info("LIB JARS :" + giraphConf.getYarnLibJars());
    String[] libJars = giraphConf.getYarnLibJars().split(",");
    for (String libJar : libJars) {
        jars.add(libJar);
    }
    FileSystem fs = FileSystem.get(giraphConf);
    Path baseDir = YarnUtils.getFsCachePath(fs, appId);
    for (Path jar : YarnUtils.getLocalFiles(jars)) {
        Path dest = new Path(baseDir, jar.getName());
        LOG.info("Made local resource for :" + jar + " to " + dest);
        fs.copyFromLocalFile(false, true, jar, dest);
        YarnUtils.addFileToResourceMap(map, fs, dest);
    }
}

From source file:org.apache.giraph.yarn.YarnUtils.java

License:Apache License

/**
 * Export our populated GiraphConfiguration as an XML file to be used by the
 * ApplicationMaster's exec container, and register it with LocalResources.
 * @param giraphConf the current Configuration object to be published.
 * @param appId the ApplicationId to stamp this app's base HDFS resources dir.
 *///from w  w  w  .j  a  v  a2 s.  c  o  m
public static void exportGiraphConfiguration(GiraphConfiguration giraphConf, ApplicationId appId)
        throws IOException {
    File confFile = new File(System.getProperty("java.io.tmpdir"), GiraphConstants.GIRAPH_YARN_CONF_FILE);
    if (confFile.exists()) {
        if (!confFile.delete()) {
            LOG.warn("Unable to delete file " + confFile);
        }
    }
    String localConfPath = confFile.getAbsolutePath();
    FileOutputStream fos = null;
    try {
        fos = new FileOutputStream(localConfPath);
        giraphConf.writeXml(fos);
        FileSystem fs = FileSystem.get(giraphConf);
        Path hdfsConfPath = new Path(YarnUtils.getFsCachePath(fs, appId),
                GiraphConstants.GIRAPH_YARN_CONF_FILE);
        fos.flush();
        fs.copyFromLocalFile(false, true, new Path(localConfPath), hdfsConfPath);
    } finally {
        if (null != fos) {
            fos.close();
        }
    }
}

From source file:org.apache.hama.bsp.YARNBSPJobClient.java

License:Apache License

private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String fileName,
        Map<String, LocalResource> localResources) throws IOException {
    Path dstPath = new Path(fileDstPath, fileName);
    dstPath = fs.makeQualified(dstPath);
    fs.copyFromLocalFile(false, true, new Path(fileSrcPath), dstPath);
    FileStatus fileStatus = fs.getFileStatus(dstPath);
    LocalResource localRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dstPath.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, fileStatus.getLen(),
            fileStatus.getModificationTime());
    localResources.put(fileName, localRsrc);
}