List of usage examples for org.apache.hadoop.fs FileSystem copyFromLocalFile
public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException
From source file:com.mvdb.scratch.HadoopClient.java
License:Apache License
/** * Copy a local sequence file to a remote file on HDFS. * //w w w . ja va2 s. com * @param from Name of the sequence file to copy * @param to Name of the sequence file to copy to * @param remoteHadoopFS HDFS host URI * * @throws IOException */ public static void copySequenceFile(String from, String to, String remoteHadoopFS) throws IOException { conf.set("fs.defaultFS", remoteHadoopFS); FileSystem fs = FileSystem.get(conf); Path localPath = new Path(from); Path hdfsPath = new Path(to); boolean deleteSource = true; fs.copyFromLocalFile(deleteSource, localPath, hdfsPath); logger.info("Copied SequenceFile from: " + from + " to: " + to); }
From source file:com.placeiq.piqconnect.InitialVectorGenerator.java
License:Apache License
private void genCmdFile(long numberOfNodes, int numberOfReducers, Path outputPath) throws IOException { File tmpFile = File.createTempFile("piqconnect_initial_vector", ""); tmpFile.deleteOnExit();/*from www . j a v a 2s. c om*/ try (BufferedWriter out = new BufferedWriter(new FileWriter(tmpFile))) { long step = numberOfNodes / numberOfReducers; long startNode; long endNode; for (int i = 0; i < numberOfReducers; i++) { startNode = i * step; if (i < numberOfReducers - 1) { endNode = step * (i + 1) - 1; } else { endNode = numberOfNodes - 1; } out.write(i + "\t" + startNode + "\t" + endNode + "\n"); } } FileSystem fs = FileSystem.get(getConf()); fs.copyFromLocalFile(true, new Path(tmpFile.getAbsolutePath()), outputPath); }
From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java
License:Open Source License
/** * Copy from local fs to HDFS//from www. j a va2 s. c om * * @param local_path * @param hdfs_path * @return Error message * @throws RemoteException */ @Override public String copyFromLocal(String local_path, String hdfs_path) throws RemoteException { String error = null; Path localP = new Path(local_path), hdfsP = new Path(hdfs_path); File failFile = new File(localP.getParent().toString(), "." + localP.getName() + ".crc"); try { FileChecker hChO = new FileChecker(new File(local_path)); if (hChO.exists()) { FileSystem fs = NameNodeVar.getFS(); if (failFile.exists()) { failFile.delete(); } fs.copyFromLocalFile(false, localP, hdfsP); } else { error = LanguageManagerWF.getText("HdfsInterface.ouputexists"); } } catch (IOException e) { logger.error(e.getMessage()); error = LanguageManagerWF.getText("HdfsInterface.errormove", new Object[] { e.getMessage() }); } if (error != null) { if (failFile.exists()) { failFile.delete(); } logger.debug(error); } return error; }
From source file:fr.jetoile.hadoopunit.component.OozieBootstrap.java
License:Apache License
public void createShareLib() { if (!oozieShareLibCreate) { LOGGER.info("OOZIE: Share Lib Create Disabled... skipping"); } else {/*from w w w . ja v a 2 s .co m*/ try { // Get and extract the oozie release String oozieExtractTempDir = extractOozieTarFileToTempDir( new File(oozieShareLibPath + Path.SEPARATOR + oozieShareLibName)); // Extract the sharelib tarball to a temp dir fullOozieShareLibTarFilePath = oozieExtractTempDir + Path.SEPARATOR + "oozie-" + getOozieVersionFromOozieTarFileName() + Path.SEPARATOR + "oozie-sharelib-" + getOozieVersionFromOozieTarFileName() + ".tar.gz"; ; oozieShareLibExtractTempDir = extractOozieShareLibTarFileToTempDir( new File(fullOozieShareLibTarFilePath)); // Copy the sharelib into HDFS Path destPath = new Path( oozieHdfsShareLibDir + Path.SEPARATOR + SHARE_LIB_PREFIX + getTimestampDirectory()); LOGGER.info("OOZIE: Writing share lib contents to: {}", destPath); FileSystem hdfsFileSystem = null; try { hdfsFileSystem = ((HdfsBootstrap) HadoopBootstrap.INSTANCE.getService(Component.HDFS)) .getHdfsFileSystemHandle(); } catch (Exception e) { LOGGER.error("unable to get hdfs client"); } hdfsFileSystem.copyFromLocalFile(false, new Path(new File(oozieShareLibExtractTempDir).toURI()), destPath); // if (purgeLocalShareLibCache) { // FileUtils.deleteDirectory(new File(shareLibCacheDir)); // } } catch (IOException e) { LOGGER.error("unable to copy oozie sharelib into hdfs"); } } }
From source file:mlbench.pagerank.PagerankMerge.java
License:Apache License
private static void reduceDiffs(int local_diffs, int rank) { int diffs[] = { 0 }; int bs[] = { local_diffs }; try {/*from w w w.j a v a2 s. c o m*/ MPI.COMM_WORLD.Reduce(bs, 0, diffs, 0, 1, MPI.INT, MPI.SUM, 0); MPI.COMM_WORLD.Barrier(); if (rank == 0) { LOG.info("Uncoveraged diffs:" + diffs[0]); FileWriter output = new FileWriter("var.tmp"); output.write(String.valueOf(diffs[0])); output.close(); JobConf conf = new JobConf(confPath); final FileSystem fs = FileSystem.get(conf); fs.copyFromLocalFile(true, new Path("./var.tmp"), new Path("var.tmp")); } } catch (MPIException | IOException e) { e.printStackTrace(); } }
From source file:org.anon.smart.d2cache.store.fileStore.hadoop.HadoopFileStoreTransaction.java
License:Open Source License
@Override public void commit() throws CtxException { FileSystem hdfs = ((HadoopFileStoreConnection) _connection).getHadoopFS(); assertion().assertNotNull(hdfs, "Hadoop FileSystem is null"); String repo = hdfs.getWorkingDirectory().toUri().toString(); for (Object fi : files.keySet()) { try {/*from w w w. jav a2 s .com*/ String filePath = (String) fi; String[] tmp = filePath.split("/"); String fileName = tmp[tmp.length - 1]; Path fldr = new Path(files.get(fi)); if (!hdfs.exists(fldr)) hdfs.mkdirs(fldr); hdfs.copyFromLocalFile(true, new Path(filePath), new Path(files.get(fi) + "/" + fileName)); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } }
From source file:org.apache.oozie.tools.OozieSharelibCLI.java
License:Apache License
public synchronized int run(String[] args) throws Exception { if (used) {/*from ww w . java 2 s.c om*/ throw new IllegalStateException("CLI instance already used"); } used = true; CLIParser parser = new CLIParser("oozie-setup.sh", HELP_INFO); String oozieHome = System.getProperty(OOZIE_HOME); parser.addCommand(HELP_CMD, "", "display usage for all commands or specified command", new Options(), false); parser.addCommand(CREATE_CMD, "", "create a new timestamped version of oozie sharelib", createUpgradeOptions(CREATE_CMD), false); parser.addCommand(UPGRADE_CMD, "", "[deprecated][use command \"create\" to create new version] upgrade oozie sharelib \n", createUpgradeOptions(UPGRADE_CMD), false); try { final CLIParser.Command command = parser.parse(args); String sharelibAction = command.getName(); if (sharelibAction.equals(HELP_CMD)) { parser.showHelp(command.getCommandLine()); return 0; } if (!command.getCommandLine().hasOption(FS_OPT)) { throw new Exception("-fs option must be specified"); } int threadPoolSize = Integer.valueOf(command.getCommandLine().getOptionValue(CONCURRENCY_OPT, "1")); File srcFile = null; //Check whether user provided locallib if (command.getCommandLine().hasOption(LIB_OPT)) { srcFile = new File(command.getCommandLine().getOptionValue(LIB_OPT)); } else { //Since user did not provide locallib, find the default one under oozie home dir Collection<File> files = FileUtils.listFiles(new File(oozieHome), new WildcardFileFilter("oozie-sharelib*.tar.gz"), null); if (files.size() > 1) { throw new IOException("more than one sharelib tar found at " + oozieHome); } if (files.isEmpty()) { throw new IOException("default sharelib tar not found in oozie home dir: " + oozieHome); } srcFile = files.iterator().next(); } File temp = File.createTempFile("oozie", ".dir"); temp.delete(); temp.mkdir(); temp.deleteOnExit(); //Check whether the lib is a tar file or folder if (!srcFile.isDirectory()) { FileUtil.unTar(srcFile, temp); srcFile = new File(temp.toString() + "/share/lib"); } else { //Get the lib directory since it's a folder srcFile = new File(srcFile, "lib"); } String hdfsUri = command.getCommandLine().getOptionValue(FS_OPT); Path srcPath = new Path(srcFile.toString()); Services services = new Services(); services.getConf().set(Services.CONF_SERVICE_CLASSES, "org.apache.oozie.service.LiteWorkflowAppService, org.apache.oozie.service.HadoopAccessorService"); services.getConf().set(Services.CONF_SERVICE_EXT_CLASSES, ""); services.init(); WorkflowAppService lwas = services.get(WorkflowAppService.class); HadoopAccessorService has = services.get(HadoopAccessorService.class); Path dstPath = lwas.getSystemLibPath(); if (sharelibAction.equals(CREATE_CMD) || sharelibAction.equals(UPGRADE_CMD)) { dstPath = new Path( dstPath.toString() + Path.SEPARATOR + SHARE_LIB_PREFIX + getTimestampDirectory()); } System.out.println("the destination path for sharelib is: " + dstPath); URI uri = new Path(hdfsUri).toUri(); Configuration fsConf = has.createJobConf(uri.getAuthority()); FileSystem fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf); if (!srcFile.exists()) { throw new IOException(srcPath + " cannot be found"); } if (threadPoolSize > 1) { concurrentCopyFromLocal(fs, threadPoolSize, srcFile, dstPath); } else { fs.copyFromLocalFile(false, srcPath, dstPath); } services.destroy(); FileUtils.deleteDirectory(temp); return 0; } catch (ParseException ex) { System.err.println("Invalid sub-command: " + ex.getMessage()); System.err.println(); System.err.println(parser.shortHelp()); return 1; } catch (Exception ex) { logError(ex.getMessage(), ex); return 1; } }
From source file:org.apache.sentry.tests.e2e.solr.AbstractSolrSentryTestBase.java
License:Apache License
public static File setupSentry() throws Exception { File sentrySite = File.createTempFile("sentry-site", "xml"); sentrySite.deleteOnExit();/*from w ww .j a va 2s. c o m*/ File authProviderDir = new File(RESOURCES_DIR, "sentry"); String authProviderName = "test-authz-provider.ini"; FileSystem clusterFs = dfsCluster.getFileSystem(); clusterFs.copyFromLocalFile(false, new Path(authProviderDir.toString(), authProviderName), new Path(authProviderName)); // need to write sentry-site at execution time because we don't know // the location of sentry.solr.provider.resource beforehand StringBuilder sentrySiteData = new StringBuilder(); sentrySiteData.append("<configuration>\n"); addPropertyToSentry(sentrySiteData, "sentry.provider", "org.apache.sentry.provider.file.LocalGroupResourceAuthorizationProvider"); addPropertyToSentry(sentrySiteData, "sentry.solr.provider.resource", clusterFs.getWorkingDirectory() + File.separator + authProviderName); sentrySiteData.append("</configuration>\n"); FileUtils.writeStringToFile(sentrySite, sentrySiteData.toString()); return sentrySite; }
From source file:org.apache.sentry.tests.e2e.solr.db.integration.AbstractSolrSentryTestWithDbProvider.java
License:Apache License
public static void writePolicyFile() throws Exception { policyFile.write(policyFilePath);//w ww . ja va 2 s . com FileSystem clusterFs = dfsCluster.getFileSystem(); clusterFs.copyFromLocalFile(false, new Path(policyFilePath.getPath()), new Path(policyFilePath.getPath())); }
From source file:org.apache.tinkerpop.gremlin.hadoop.process.computer.AbstractHadoopGraphComputerTest.java
License:Apache License
@Test public void shouldCopyDirectoriesCorrectly() throws Exception { final String hdfsName = this.getClass().getSimpleName() + "-hdfs"; final String localName = this.getClass().getSimpleName() + "-local"; final FileSystem fs = FileSystem.get(new Configuration()); if (!new File(System.getProperty("java.io.tmpdir") + "/" + localName).exists()) assertTrue(new File(System.getProperty("java.io.tmpdir") + "/" + localName).mkdir()); File tempFile1 = new File(System.getProperty("java.io.tmpdir") + "/" + localName + "/test1.txt"); File tempFile2 = new File(System.getProperty("java.io.tmpdir") + "/" + localName + "/test2.txt"); assertTrue(tempFile1.createNewFile()); assertTrue(tempFile2.createNewFile()); assertTrue(tempFile1.exists());/*from w ww . j av a 2 s . c o m*/ assertTrue(tempFile2.exists()); if (fs.exists(new Path("target/" + hdfsName))) assertTrue(fs.delete(new Path("target/" + hdfsName), true)); fs.copyFromLocalFile(true, new Path(tempFile1.getAbsolutePath()), new Path("target/" + hdfsName + "/test1.dat")); fs.copyFromLocalFile(true, new Path(tempFile2.getAbsolutePath()), new Path("target/" + hdfsName + "/test2.dat")); assertTrue(fs.exists(new Path("target/" + hdfsName + "/test1.dat"))); assertTrue(fs.exists(new Path("target/" + hdfsName + "/test2.dat"))); assertTrue(fs.exists(new Path("target/" + hdfsName))); assertTrue(fs.isDirectory(new Path("target/" + hdfsName))); assertFalse(tempFile1.exists()); assertFalse(tempFile2.exists()); assertTrue(new File(System.getProperty("java.io.tmpdir") + "/" + localName).exists()); assertTrue(new File(System.getProperty("java.io.tmpdir") + "/" + localName).delete()); assertTrue(fs.exists(new Path("target/" + hdfsName + "/test1.dat"))); assertTrue(fs.exists(new Path("target/" + hdfsName + "/test2.dat"))); assertTrue(fs.exists(new Path("target/" + hdfsName))); assertTrue(fs.isDirectory(new Path("target/" + hdfsName))); ///// final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs"; final File localDirectory = new File(System.getProperty("java.io.tmpdir") + "/" + hadoopGremlinLibsRemote); final File localLibDirectory = new File(localDirectory.getAbsolutePath() + "/" + hdfsName); if (localLibDirectory.exists()) { Stream.of(localLibDirectory.listFiles()).forEach(File::delete); assertTrue(localLibDirectory.delete()); } assertFalse(localLibDirectory.exists()); assertEquals(localLibDirectory, AbstractHadoopGraphComputer.copyDirectoryIfNonExistent(fs, "target/" + hdfsName)); assertTrue(localLibDirectory.exists()); assertTrue(localLibDirectory.isDirectory()); assertEquals(2, Stream.of(localLibDirectory.listFiles()).filter(file -> file.getName().endsWith(".dat")).count()); }