List of usage examples for org.apache.hadoop.fs FileSystem copyToLocalFile
public void copyToLocalFile(Path src, Path dst) throws IOException
From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java
License:Open Source License
private String copyFileInRemote(Channel channel, String lfile, String rfile) { FSDataInputStream fisd = null;//from ww w . ja va2s .c o m String error = null; boolean ptimestamp = true; try { String nameRdm = RandomString.getRandomName(20); String tmpFileStr = System.getProperty("java.io.tmpdir") + "/" + nameRdm; FileSystem fs = NameNodeVar.getFS(); fs.copyToLocalFile(new Path(lfile), new Path(tmpFileStr)); channel = channel.getSession().openChannel("sftp"); channel.connect(); ChannelSftp sftpChannel = (ChannelSftp) channel; sftpChannel.put(tmpFileStr, rfile); sftpChannel.exit(); new File(tmpFileStr).delete(); } catch (Exception e) { error = "Unexpected error when copying file accross: " + e.getMessage(); logger.error(error, e); } return error; }
From source file:com.redsqirl.workflow.server.Workflow.java
License:Open Source License
/** * Reads the xml part of a workflow.//w ww. j a v a 2 s.c om * * @param filePath * the xml file path to read from. * @return null if OK, or a description of the error. */ public String read(String filePath) { String error = null; try { String[] path = filePath.split("/"); String fileName = path[path.length - 1]; String tempPath = WorkflowPrefManager.getPathtmpfolder() + "/" + fileName + "_" + RandomString.getRandomName(4); FileSystem fs = NameNodeVar.getFS(); if (!fs.isFile(new Path(filePath))) { return "'" + filePath + "' is not a file."; } logger.debug("filePath " + filePath); logger.debug("tempPath " + tempPath); fs.copyToLocalFile(new Path(filePath), new Path(tempPath)); File xmlFile = new File(tempPath); error = readFromLocal(xmlFile); // clean temporary files xmlFile.delete(); if (filePath.startsWith(WorkflowPrefManager.getBackupPath())) { saved = false; this.path = null; } else { this.path = filePath; } } catch (Exception e) { error = LanguageManagerWF.getText("workflow.read_failXml"); logger.error(error, e); } return error; }
From source file:com.senseidb.util.HDFSIndexCopier.java
License:Apache License
public boolean copy(String src, String dest) { try {/*www.j a v a 2 s . co m*/ URI srcUri = new URI(src), destUri = new URI(dest); Configuration config = new Configuration(); config.set("fs.default.name", srcUri.resolve("/").toString()); FileSystem dfs = FileSystem.get(config); Path destPath = new Path(destUri.toString()); FileStatus[] files = dfs.listStatus(new Path(srcUri.toString())); if (files == null || files.length == 0) return false; for (FileStatus f : files) { log.info("Copying " + f.getPath().toString()); dfs.copyToLocalFile(f.getPath(), destPath); } return true; } catch (Exception e) { log.error(e.getMessage(), e); return false; } }
From source file:com.splunk.shuttl.testutil.TUtilsFileSystem.java
License:Apache License
/** * @return The file on specified path from specified filessystem. *//*from w w w . j a v a 2 s . c om*/ public static File getFileFromFileSystem(FileSystem fileSystem, Path pathOftheFileOnRemote) { File retrivedFile = TUtilsFile.createFilePath(); Path localFilePath = new Path(retrivedFile.toURI()); try { fileSystem.copyToLocalFile(pathOftheFileOnRemote, localFilePath); } catch (IOException e) { TUtilsTestNG.failForException("Can't retrive the file from remote filesystem", e); } return retrivedFile; }
From source file:com.talis.hadoop.rdf.merge.IndexMergeReducer.java
License:Apache License
@Override public void reduce(LongWritable key, Iterable<Text> value, final Context context) throws IOException, InterruptedException { Runnable reporter = new Runnable() { @Override/*w w w . j ava 2 s . c om*/ public void run() { context.progress(); } }; ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); ScheduledFuture<?> task = scheduler.scheduleAtFixedRate(reporter, 60, 60, TimeUnit.SECONDS); LOG.debug("Scheduled progress reporter, combining index shards"); FileSystem shardsFs = null; for (Text remoteShard : value) { Path remote = new Path(remoteShard.toString()); if (null == shardsFs) { shardsFs = FileSystem.get(remote.toUri(), context.getConfiguration()); } LOG.debug("Copying shard from {} to {}", remote, localShards); shardsFs.copyToLocalFile(remote, localShards); LOG.debug("Copy complete"); } Directory[] shards = getDirectories(); LOG.debug("About to combine {} shards", shards.length); writer.addIndexesNoOptimize(shards); LOG.debug("Combined index built, terminating reporter"); task.cancel(true); }
From source file:com.tripadvisor.hadoop.BackupHdfs.java
License:Apache License
/** * Method to move files from HDFS to local filesystem * * localPath: Path on the machines filesystem * fs:FileSystem object from HDFS// ww w . j a va2s .c om * pathList:List of paths for files that might need to be backed * up * size:max size in bytes to be backed up * * ReturnsDate of the last files backed up if reached size limit, * else, zero **/ public long backupFiles(String localPath, String preservePath, FileSystem fs, ArrayList<Path> pathList, long size) { Path fsPath; long tmpSize = 0; long tmpDate = 0; // Start iterating over all paths for (Path hdfsPath : pathList) { try { long nFileSize = fs.getContentSummary(hdfsPath).getLength(); tmpSize = tmpSize + nFileSize; if ((tmpSize <= size) || (size == 0)) { FileStatus stat = fs.getFileStatus(hdfsPath); System.err.println("File " + hdfsPath.toUri().getPath() + " " + nFileSize + " bytes, " + "perms: " + stat.getOwner() + "/" + stat.getGroup() + ", " + stat.getPermission().toString()); tmpDate = stat.getModificationTime() / 1000; String sFsPath = localPath + hdfsPath.toUri().getPath(); fsPath = new Path(sFsPath); File f = new File(sFsPath); // COMMENTED OUT: until a few backup cycles run // and the mtime gets in fact set on all copied // files. // // ignore it if the file exists and has the same mtime // if (f.exists() && f.isFile() && f.lastModified() == stat.getModificationTime()) // { // System.out.println("no need to backup " + f.toString() + ", mtime matches hdfs"); // continue; // } if (false == m_bDryRun) { // check if we need to back up the local file // (not directory), if it already exists. if (f.exists() && f.isFile()) { // ignore files with substrings in the // no-preserve file if (true == doPreserveFile(sFsPath)) { // move it to the backup path String sNewPath = preservePath + hdfsPath.toUri().getPath(); File newFile = new File(sNewPath); // create directory structure for new file? if (false == newFile.getParentFile().exists()) { if (false == newFile.getParentFile().mkdirs()) { System.err .println("Failed to mkdirs " + newFile.getParentFile().toString()); System.exit(1); } } // rename existing file to new location if (false == f.renameTo(newFile)) { System.err.println( "Failed to renameTo " + f.toString() + " to " + newFile.toString()); System.exit(1); } System.out.println("preserved " + f.toString() + " into " + newFile.toString()); } else { System.out.println("skipped preservation of " + f.toString()); } } // copy from hdfs to local filesystem fs.copyToLocalFile(hdfsPath, fsPath); // set the mtime to match hdfs file f.setLastModified(stat.getModificationTime()); // compare checksums on both files compareChecksums(fs, hdfsPath, sFsPath); } // don't print the progress after every file -- go // by at least 1% increments long nPercentDone = (long) (100 * tmpSize / m_nTotalBytes); if (nPercentDone > m_nLastPercentBytesDone) { System.out.println("progress: copied " + prettyPrintBytes(tmpSize) + ", " + nPercentDone + "% done" + ", tstamp=" + tmpDate); m_nLastPercentBytesDone = nPercentDone; } if (m_nSleepSeconds > 0) { try { Thread.sleep(1000 * m_nSleepSeconds); } catch (Exception e2) { // ignore } } } else { return tmpDate; } } catch (IOException e) { System.err.println("FATAL ERROR: Something wrong with the file"); System.err.println(e); System.out.println(tmpDate); System.exit(1); return 0; } } return 0; }
From source file:eagle.dataproc.impl.storm.hdfs.DataCollectionHDFSSpout.java
License:Apache License
public void copyFiles() { LOG.info("Inside listFiles()"); Configuration conf = new Configuration(); // _____________ TO TEST THAT CORRECT HADOOP JARs ARE INCLUDED __________________ ClassLoader cl = ClassLoader.getSystemClassLoader(); URL[] urls = ((URLClassLoader) cl).getURLs(); if (LOG.isDebugEnabled()) { for (URL url : urls) { LOG.debug(url.getFile());//from www . j av a 2 s.c o m } } // _________________________________________ String hdfsConnectionStr = configContext.getString("dataSourceConfig.hdfsConnnection"); LOG.info("HDFS connection string: " + hdfsConnectionStr); String hdfsPath = configContext.getString("dataSourceConfig.hdfsPath"); LOG.info("HDFS path: " + hdfsPath); String copyToPath = configContext.getString("dataSourceConfig.copyToPath"); LOG.info("copyToPath: " + copyToPath); String srcPathStr = new String("hdfs://" + hdfsConnectionStr + hdfsPath); Path srcPath = new Path(srcPathStr); LOG.info("listFiles called"); LOG.info("srcPath: " + srcPath); try { FileSystem fs = srcPath.getFileSystem(conf); /*CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(srcPath); DataInputStream inputStream = new DataInputStream(codec.createInputStream(fs.open(srcPath))); */ Path destPath = new Path(copyToPath); LOG.info("Destination path: " + destPath); fs.copyToLocalFile(srcPath, destPath); LOG.info("Copy to local succeed"); fs.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
From source file:eagle.dataproc.impl.storm.hdfs.UserProfileGenerationHDFSSpout.java
License:Apache License
public void copyFiles() { LOG.info("Inside listFiles()"); //Configuration conf = new Configuration(); JobConf conf = new JobConf(); // _____________ TO TEST THAT CORRECT HADOOP JARs ARE INCLUDED __________________ ClassLoader cl = ClassLoader.getSystemClassLoader(); URL[] urls = ((URLClassLoader) cl).getURLs(); if (LOG.isDebugEnabled()) { for (URL url : urls) { LOG.debug(url.getFile());//from w w w. java 2 s. co m } } // _________________________________________ String hdfsConnectionStr = configContext.getString("dataSourceConfig.hdfsConnection"); LOG.info("HDFS connection string: " + hdfsConnectionStr); String hdfsPath = configContext.getString("dataSourceConfig.hdfsPath"); LOG.info("HDFS path: " + hdfsPath); String copyToPath = configContext.getString("dataSourceConfig.copyToPath"); LOG.info("copyToPath: " + copyToPath); String srcPathStr = new String("hdfs://" + hdfsConnectionStr + hdfsPath); Path srcPath = new Path(srcPathStr); LOG.info("listFiles called"); LOG.info("srcPath: " + srcPath); try { FileSystem fs = srcPath.getFileSystem(conf); /*CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(srcPath); DataInputStream inputStream = new DataInputStream(codec.createInputStream(fs.open(srcPath))); */ Path destPath = new Path(copyToPath); LOG.info("Destination path: " + destPath); String userListFileName = configContext.getString("dataSourceConfig.userList"); //loggerHDFSSpout.info("userListFileName: " + userListFileName); List<String> userList = getUser(userListFileName); for (String user : userList) { Path finalSrcPath = new Path(srcPath.getName() + "/" + user); fs.copyToLocalFile(finalSrcPath, destPath); } LOG.info("Copy to local succeed"); fs.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
From source file:edu.uci.ics.asterix.aoya.Utils.java
License:Apache License
public static String getCCHostname(String instanceName, Configuration conf) throws YarnException { try {/*from w w w .j av a 2 s . co m*/ FileSystem fs = FileSystem.get(conf); String instanceFolder = instanceName + "/"; String pathSuffix = CONF_DIR_REL + instanceFolder + "cluster-config.xml"; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); File tmp = File.createTempFile("cluster-config", "xml"); tmp.deleteOnExit(); fs.copyToLocalFile(dstConf, new Path(tmp.getPath())); JAXBContext clusterConf = JAXBContext.newInstance(Cluster.class); Unmarshaller unm = clusterConf.createUnmarshaller(); Cluster cl = (Cluster) unm.unmarshal(tmp); String ccIp = cl.getMasterNode().getClientIp(); return ccIp; } catch (IOException | JAXBException e) { throw new YarnException(e); } }
From source file:edu.uci.ics.hyracks.hdfs.dataflow.DataflowTest.java
License:Apache License
/** * Check if the results are correct/*from w ww.j av a 2 s .c o m*/ * * @return true if correct * @throws Exception */ private boolean checkResults() throws Exception { FileSystem dfs = FileSystem.get(conf); Path result = new Path(HDFS_OUTPUT_PATH); Path actual = new Path(ACTUAL_RESULT_DIR); dfs.copyToLocalFile(result, actual); TestUtils.compareWithResult(new File(EXPECTED_RESULT_PATH + File.separator + "part-0"), new File(ACTUAL_RESULT_DIR + File.separator + "customer_result" + File.separator + "part-0")); return true; }