Example usage for org.apache.hadoop.fs FileSystem copyFromLocalFile

List of usage examples for org.apache.hadoop.fs FileSystem copyFromLocalFile

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. copyFromLocalFile

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem copyFromLocalFile.

Prototype

public void copyFromLocalFile(Path src, Path dst) throws IOException

Source Link

Document

The src file is on the local disk.

Usage

From source file:com.inmobi.conduit.Conduit.java

License:Apache License

private void copyAuditUtilJarToClusterFs(Cluster cluster, String auditUtilSrcJar) throws IOException {
    FileSystem clusterFS = FileSystem.get(cluster.getHadoopConf());
    // create jars path inside /conduit/system/tmp path
    Path jarsPath = new Path(cluster.getTmpPath(), "jars");
    if (!clusterFS.exists(jarsPath)) {
        clusterFS.mkdirs(jarsPath);/*w w  w  . j a  v a 2 s  .  co  m*/
    }
    // copy AuditUtil source jar into /conduit/system/tmp/jars path
    Path AuditUtilJarDestPath = new Path(jarsPath, "messaging-client-core.jar");
    if (clusterFS.exists(AuditUtilJarDestPath)) {
        clusterFS.delete(AuditUtilJarDestPath, true);
    }
    clusterFS.copyFromLocalFile(new Path(auditUtilSrcJar), AuditUtilJarDestPath);
}

From source file:com.inmobi.messaging.consumer.util.HadoopUtil.java

License:Apache License

public static void setUpHadoopFiles(Path streamDirPrefix, Configuration conf, String[] files,
        String[] suffixDirs, Path[] finalFiles, boolean alternateEmptyFiles, Date minuteDirTimeStamp, int index,
        int startIndex) throws Exception {
    FileSystem fs = streamDirPrefix.getFileSystem(conf);
    Path rootDir = streamDirPrefix.getParent();
    Path tmpDataDir = new Path(rootDir, "data");
    boolean emptyFile = false;
    // setup data dirs
    if (files != null) {
        int i = startIndex;
        int j = index;
        for (String file : files) {
            if (alternateEmptyFiles && emptyFile) {
                MessageUtil.createEmptySequenceFile(file, fs, tmpDataDir, conf);
                emptyFile = false;/*w w  w . j  ava2s  .co  m*/
            } else {
                MessageUtil.createMessageSequenceFile(file, fs, tmpDataDir, i, conf);
                emptyFile = true;
                i += 100;
            }
            Path srcPath = new Path(tmpDataDir, file);
            Date commitTime = getCommitDateForFile(file, minuteDirTimeStamp);
            TestUtil.publishMissingPaths(fs, streamDirPrefix, lastCommitTime, commitTime);
            lastCommitTime = commitTime;
            Path targetDateDir = getTargetDateDir(streamDirPrefix, commitTime);
            List<Path> targetDirs = new ArrayList<Path>();
            if (suffixDirs != null) {
                for (String suffixDir : suffixDirs) {
                    targetDirs.add(new Path(targetDateDir, suffixDir));
                }
            } else {
                targetDirs.add(targetDateDir);
            }
            for (Path targetDir : targetDirs) {
                fs.mkdirs(targetDir);
                Path targetPath = new Path(targetDir, file);
                fs.copyFromLocalFile(srcPath, targetPath);
                LOG.info("Copied " + srcPath + " to " + targetPath);
                if (finalFiles != null) {
                    finalFiles[j] = targetPath;
                    j++;
                }
                Thread.sleep(1000);
            }
            fs.delete(srcPath, true);
        }
        TestUtil.publishLastPath(fs, streamDirPrefix, lastCommitTime);
    }
}

From source file:com.intel.hadoop.graphbuilder.util.FsUtil.java

License:Open Source License

/**
 * Archives and distributes the temporary class files generated by {@code IngressJobKeyValueFactory},
 * {@code PreprocessJobKeyValueFactory}, and {@code EdgeTransformJobKeyValueFactory} using Javassist.
 * The temporary class files are located at $currentdir/generatedclass,
 * and the distributed archive is located at hdfs:///tmp/tempclass.jar.
 * @param conf/*from w w  w.j ava2s. c om*/
 */
public static void distributedTempClassToClassPath(JobConf conf) {
    try {
        Process p = Runtime.getRuntime().exec("jar -cf tempclass.jar generatedclass");
        p.waitFor();
        FileSystem fs = FileSystem.get(conf);
        Path src = new Path(new java.io.File(".").getCanonicalPath() + "/tempclass.jar");
        Path dst = new Path("/tmp", "tempclass.jar");
        fs.copyFromLocalFile(src, dst);
        DistributedCache.addFileToClassPath(dst, conf, fs);
    } catch (IOException e) {
        e.printStackTrace();
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
}

From source file:com.linkedin.thirdeye.hadoop.backfill.BackfillControllerAPIs.java

License:Apache License

/**
 * Downloads a segment from the controller, given the table name and segment name
 * @param segmentName//from   ww w  .  j a va2s  .  com
 * @param hdfsSegmentPath
 * @throws IOException
 * @throws ArchiveException
 */
public void downloadSegment(String segmentName, Path hdfsSegmentPath) throws IOException, ArchiveException {

    FileSystem fs = FileSystem.get(new Configuration());
    HttpClient controllerClient = new DefaultHttpClient();
    HttpGet req = new HttpGet(SEGMENTS_ENDPOINT + URLEncoder.encode(tableName, UTF_8) + "/"
            + URLEncoder.encode(segmentName, UTF_8));
    HttpResponse res = controllerClient.execute(controllerHttpHost, req);
    try {
        if (res.getStatusLine().getStatusCode() != 200) {
            throw new IllegalStateException(res.getStatusLine().toString());
        }
        LOGGER.info("Fetching segment {}", segmentName);
        InputStream content = res.getEntity().getContent();

        File tempDir = new File(Files.createTempDir(), "thirdeye_temp");
        tempDir.mkdir();
        LOGGER.info("Creating temporary dir for staging segments {}", tempDir);
        File tempSegmentDir = new File(tempDir, segmentName);
        File tempSegmentTar = new File(tempDir, segmentName + ThirdEyeConstants.TAR_SUFFIX);

        LOGGER.info("Downloading {} to {}", segmentName, tempSegmentTar);
        OutputStream out = new FileOutputStream(tempSegmentTar);
        IOUtils.copy(content, out);
        if (!tempSegmentTar.exists()) {
            throw new IllegalStateException("Download of " + segmentName + " unsuccessful");
        }

        LOGGER.info("Extracting segment {} to {}", tempSegmentTar, tempDir);
        TarGzCompressionUtils.unTar(tempSegmentTar, tempDir);
        File[] files = tempDir.listFiles(new FilenameFilter() {

            @Override
            public boolean accept(File dir, String name) {
                return !name.endsWith(ThirdEyeConstants.TAR_SUFFIX) && new File(dir, name).isDirectory();
            }
        });
        if (files.length == 0) {
            throw new IllegalStateException("Failed to extract " + tempSegmentTar + " to " + tempDir);
        } else if (!files[0].getName().equals(tempSegmentDir.getName())) {
            LOGGER.info("Moving extracted segment to the segment dir {}", tempSegmentDir);
            FileUtils.moveDirectory(files[0], tempSegmentDir);
        }
        if (!tempSegmentDir.exists()) {
            throw new IllegalStateException("Failed to move " + files[0] + " to " + tempSegmentDir);
        }

        LOGGER.info("Copying segment from {} to hdfs {}", tempSegmentDir, hdfsSegmentPath);
        fs.copyFromLocalFile(new Path(tempSegmentDir.toString()), hdfsSegmentPath);
        Path hdfsSegmentDir = new Path(hdfsSegmentPath, segmentName);
        if (!fs.exists(hdfsSegmentDir)) {
            throw new IllegalStateException("Failed to copy segment " + segmentName + " from local path "
                    + tempSegmentDir + " to hdfs path " + hdfsSegmentPath);
        }
    } finally {
        if (res.getEntity() != null) {
            EntityUtils.consume(res.getEntity());
        }
    }
    LOGGER.info("Successfully downloaded segment {} to {}", segmentName, hdfsSegmentPath);
}

From source file:com.mycompany.hdp.hdp.java

private static void copyFilesToDir(String srcDirPath, FileSystem fs, String destDirPath) throws IOException {
    File dir = new File(srcDirPath);
    for (File file : dir.listFiles()) {
        fs.copyFromLocalFile(new Path(file.getPath()), new Path(destDirPath, file.getName()));
    }/*from   w w w.  j  a v  a 2 s .c o  m*/
}

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

private String copyInHDFS(Channel channel, String rfile, String lfile, SSHDataStore remoteServer)
        throws Exception {

    String error = null;//ww  w  .  ja  v a  2  s . c om
    FileSystem fs = NameNodeVar.getFS();

    Map<String, String> p = remoteServer.getProperties(rfile);
    if (p.get(SSHInterface.key_type).equals("file")) {

        String nameRdm = RandomString.getRandomName(20);
        String tmpFileStr = System.getProperty("java.io.tmpdir") + "/" + nameRdm;

        if (channel.isClosed()) {
            channel.connect();
        }
        logger.info("Copy " + rfile + " to " + tmpFileStr);
        ((ChannelSftp) channel).get(rfile, tmpFileStr);
        logger.info("Copy local " + tmpFileStr + " to HDFS " + lfile);
        fs.copyFromLocalFile(new Path(tmpFileStr), new Path(lfile));
        new File(tmpFileStr).delete();

    } else {

        if (!fs.exists(new Path(lfile))) {
            if (!fs.mkdirs(new Path(lfile))) {
                // create the directory
                error = lfile + ": Cannot create such directory";
            }
        } else if (!fs.isDirectory(new Path(lfile))) {
            //already exists as a file
            error = lfile + ": Not a directory";
        }

        if (error == null) {
            logger.info("Create the directory " + lfile);

            Map<String, Map<String, String>> files = remoteServer.getChildrenProperties(rfile);
            logger.debug(files);

            for (String path : files.keySet()) {
                Map<String, String> props = files.get(path);

                logger.debug(props.get("type") + " " + path);

                String fileName = path.replaceFirst(rfile, "");
                //String fileName = path.substring(path.lastIndexOf("/"));
                logger.debug("fileName " + fileName);

                error = copyInHDFS(channel, rfile + fileName, lfile + fileName, remoteServer);
                if (error != null) {
                    break;
                }
            }
        }

    }

    return error;
}

From source file:com.tdunning.plume.local.lazy.MapRedBypassTest.java

License:Apache License

@Test
public void test() throws Exception {
    String outputPath = "/tmp/output-plume-bypasstest";
    String inputPath = "/tmp/input-wordcount.txt";
    // Prepare input for test
    FileSystem system = FileSystem.getLocal(new Configuration());
    system.copyFromLocalFile(new Path(Resources.getResource("simple-text.txt").getPath()), new Path(inputPath));
    // Prepare output for test
    system.delete(new Path(outputPath), true);
    // Prepare workflow
    MapRedBypassWorkflow workFlow = new MapRedBypassWorkflow();
    // Execute it
    MapRedExecutor executor = new MapRedExecutor();
    executor.execute(workFlow, outputPath);

    List<String> str = Files.readLines(new File(outputPath + "/1_1/1-r-00000"), Charsets.UTF_8);
    Map<String, String> m = Maps.newHashMap();
    for (String line : str) {
        m.put(line.split("\t")[0], line.split("\t")[1]); // not super-optimal, but less code
    }//  www . ja va2 s.  c  om
    assertEquals(m.get("To test text processing with some simple-blah"),
            "To test text processing with some simple-bloh");
    assertEquals(m.get("some simple text-blah"), "some simple text-bloh");
    assertEquals(m.get("is is-blah"), "is is-bloh");

    str = Files.readLines(new File(outputPath + "/1_2/2-r-00000"), Charsets.UTF_8);
    m = Maps.newHashMap();
    for (String line : str) {
        m.put(line.split("\t")[0], line.split("\t")[1]); // not super-optimal, but less code
    }
    assertEquals(m.get("To test text processing with some simple"), "foo");
    assertEquals(m.get("some simple text"), "foo");
    assertEquals(m.get("is is"), "foo");
}

From source file:com.tdunning.plume.local.lazy.MapRedFlattenTest.java

License:Apache License

@Test
public void test() throws Exception {
    String outputPath = "/tmp/output-plume-flattentest";
    // Prepare input for test
    FileSystem system = FileSystem.getLocal(new Configuration());
    system.copyFromLocalFile(new Path(Resources.getResource("event2users.txt").getPath()),
            new Path(inputPathEvent2));
    system.copyFromLocalFile(new Path(Resources.getResource("eventslog.txt").getPath()),
            new Path(inputPathLogFile));
    // Prepare output for test
    system.delete(new Path(outputPath), true);
    // Prepare workflow
    MapRedFlattenTestWorkflow workFlow = new MapRedFlattenTestWorkflow();
    // Execute it
    MapRedExecutor executor = new MapRedExecutor();
    executor.execute(workFlow, outputPath);
}

From source file:com.tdunning.plume.local.lazy.MapRedMultipleGroupsTest.java

License:Apache License

@Test
public void test() throws IOException, InterruptedException, ClassNotFoundException {
    String inputPath = "/tmp/input-wordcount.txt";
    String outputPath = "/tmp/output-plume-complex";
    // Prepare input for test
    FileSystem system = FileSystem.getLocal(new Configuration());
    system.copyFromLocalFile(new Path(Resources.getResource("simple-text.txt").getPath()), new Path(inputPath));
    // Prepare output for test
    system.delete(new Path(outputPath), true);
    // Prepare workflow
    MultipleGroupsWorkflow workFlow = new MultipleGroupsWorkflow();
    // Execute it
    MapRedExecutor executor = new MapRedExecutor();
    executor.execute(workFlow, outputPath);

    // Just assert that 3 output files were written and have content
    /**//  w  ww .  j a v a 2  s.c o  m
     * TODO This test has to check the actual results of the 3 outputs
     */
    for (int i = 1; i <= 3; i++) {
        File f = new File(outputPath + "/1_" + i + "/" + i + "-r-00000");
        assertTrue(f.exists());
        assertTrue(f.length() > 64);
    }
}

From source file:com.tdunning.plume.local.lazy.MapRedOnlyFlattensTest.java

License:Apache License

@Test
public void test() throws Exception {
    String outputPath = "/tmp/output-plume-onlyflattentest";
    // Prepare input for test
    FileSystem system = FileSystem.getLocal(new Configuration());
    system.copyFromLocalFile(new Path(Resources.getResource("event2users.txt").getPath()),
            new Path(inputPathEvent2));
    system.copyFromLocalFile(new Path(Resources.getResource("eventslog.txt").getPath()),
            new Path(inputPathLogFile));
    system.copyFromLocalFile(new Path(Resources.getResource("eventslog.txt").getPath()),
            new Path(inputPathLogFile2));
    // Prepare output for test
    system.delete(new Path(outputPath), true);
    // Prepare workflow
    MapRedOnlyFlattensTestWorkflow workFlow = new MapRedOnlyFlattensTestWorkflow();
    // Execute it
    MapRedExecutor executor = new MapRedExecutor();
    executor.execute(workFlow, outputPath);
}