List of usage examples for org.apache.hadoop.fs Path getName
public String getName()
From source file:com.asakusafw.testdriver.FlowPartTestDriver.java
License:Apache License
private boolean isSystemFile(Path path) { assert path != null; String name = path.getName(); return name.equals(FileOutputCommitter.SUCCEEDED_FILE_NAME) || name.equals("_logs"); }
From source file:com.asakusafw.windgate.hadoopfs.ssh.AbstractSshHadoopFsMirrorTest.java
License:Apache License
private Map<String, List<String>> read(File file) throws IOException { List<File> files = new ArrayList<>(); try (FileInputStream in = new FileInputStream(file); FileList.Reader reader = FileList.createReader(in)) { while (reader.next()) { Path path = reader.getCurrentPath(); File entry = folder.newFile(path.getName()); try (FileOutputStream dst = new FileOutputStream(entry); InputStream src = reader.openContent()) { byte[] buf = new byte[256]; while (true) { int read = src.read(buf); if (read < 0) { break; }//from w w w .jav a2 s .co m dst.write(buf, 0, read); } } files.add(entry); } } Configuration conf = new Configuration(); Map<String, List<String>> results = new HashMap<>(); Text text = new Text(); for (File entry : files) { List<String> lines = new ArrayList<>(); results.put(entry.getName(), lines); try (ModelInput<Text> input = TemporaryStorage.openInput(conf, Text.class, new Path(entry.toURI()))) { while (input.readTo(text)) { lines.add(text.toString()); } } } return results; }
From source file:com.atlantbh.jmeter.plugins.hadooputilities.hdfsoperations.HdfsOperations.java
License:Apache License
private void copyFileToHDFS() throws IOException { Path localPath = new Path(this.getInputFilePath()); Path hdfsPath = new Path(this.getOutputFilePath()); String fileName = localPath.getName(); Path hdfsFileLocation = new Path(this.getOutputFilePath() + "/" + fileName); if (!hdfs.exists(hdfsFileLocation)) { hdfs.copyFromLocalFile(localPath, hdfsPath); this.setUploadSuccess("File " + fileName + " copied to HDFS on location: " + hdfsPath); } else {// w w w . j a v a2s.co m this.setUploadSuccess("File " + fileName + " already exists on HDFS on location: " + hdfsPath); } hdfs.close(); }
From source file:com.awcoleman.StandaloneJava.AvroCombinerByRecord.java
License:Apache License
public AvroCombinerByRecord(String inDirStr, String outDirStr) throws IOException { //Get list of input files ArrayList<FileStatus> inputFileList = new ArrayList<FileStatus>(); Configuration conf = new Configuration(); conf.addResource(new Path("/etc/hadoop/conf/core-site.xml")); FileSystem hdfs = FileSystem.get(conf); //Check if input and output dirs exist Path inDir = new Path(inDirStr); Path outDir = new Path(outDirStr); if (!(hdfs.exists(inDir) || hdfs.isDirectory(inDir))) { System.out.println("Input directory ( " + inDirStr + " ) not found or is not directory. Exiting."); System.exit(1);//from w w w . j ava 2s.c om } if (!(hdfs.exists(outDir) || hdfs.isDirectory(outDir))) { if (hdfs.exists(outDir)) { //outDir exists and is a symlink or file, must die System.out.println("Requested output directory name ( " + outDirStr + " ) exists but is not a directory. Exiting."); System.exit(1); } else { hdfs.mkdirs(outDir); } } RemoteIterator<LocatedFileStatus> fileStatusListIterator = hdfs.listFiles(inDir, true); while (fileStatusListIterator.hasNext()) { LocatedFileStatus fileStatus = fileStatusListIterator.next(); if (fileStatus.isFile()) { inputFileList.add((FileStatus) fileStatus); } } if (inputFileList.size() <= 1) { System.out.println("Only one or zero files found in input directory ( " + inDirStr + " ). Exiting."); System.exit(1); } //Get Schema and Compression Codec from seed file since we need it for the writer Path firstFile = inputFileList.get(0).getPath(); FsInput fsin = new FsInput(firstFile, conf); DataFileReader<Object> dfrFirstFile = new DataFileReader<Object>(fsin, new GenericDatumReader<Object>()); Schema fileSchema = dfrFirstFile.getSchema(); String compCodecName = dfrFirstFile.getMetaString("avro.codec"); dfrFirstFile.close(); //Create Empty HDFS file in output dir Path seedFile = new Path(outDirStr + "/combinedByRecord.avro"); FSDataOutputStream hdfsdos = hdfs.create(seedFile, false); //Append other files GenericDatumWriter gdw = new GenericDatumWriter(fileSchema); DataFileWriter dfwBase = new DataFileWriter(gdw); //Set compression to that found in the first file dfwBase.setCodec(CodecFactory.fromString(compCodecName)); DataFileWriter dfw = dfwBase.create(fileSchema, hdfsdos); for (FileStatus thisFileStatus : inputFileList) { DataFileStream<Object> avroStream = null; FSDataInputStream inStream = hdfs.open(thisFileStatus.getPath()); GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); avroStream = new DataFileStream<Object>(inStream, reader); long recordCounter = 0; while (avroStream.hasNext()) { dfw.append(avroStream.next()); recordCounter++; } avroStream.close(); inStream.close(); System.out.println("Appended " + recordCounter + " records from " + thisFileStatus.getPath().getName() + " to " + seedFile.getName()); } dfw.close(); dfwBase.close(); }
From source file:com.blackberry.logtools.LogTools.java
License:Apache License
public void runPigRemote(Map<String, String> params, String out, String tmp, boolean quiet, boolean silent, Configuration conf, String queue_name, String additional_jars, File pig_tmp, ArrayList<String> D_options, String PIG_DIR, FileSystem fs) { //Set input parameter for pig job - calling Pig directly params.put("tmpdir", StringEscapeUtils.escapeJava(tmp)); //Check for an out of '-', meaning write to stdout String pigout;/*from w w w . jav a 2 s .c om*/ if (out.equals("-")) { params.put("out", tmp + "/final"); pigout = tmp + "/final"; } else { params.put("out", StringEscapeUtils.escapeJava(out)); pigout = StringEscapeUtils.escapeJava(out); } try { logConsole(quiet, silent, info, "Running PIG Command"); conf.set("mapred.job.queue.name", queue_name); conf.set("pig.additional.jars", additional_jars); conf.set("pig.exec.reducers.bytes.per.reducer", Integer.toString(100 * 1000 * 1000)); conf.set("pig.logfile", pig_tmp.toString()); conf.set("hadoopversion", "23"); //PIG temp directory set to be able to delete all temp files/directories conf.set("pig.temp.dir", tmp); //Setting output separator for logdriver String DEFAULT_OUTPUT_SEPARATOR = "\t"; Charset UTF_8 = Charset.forName("UTF-8"); String outputSeparator = conf.get("logdriver.output.field.separator", DEFAULT_OUTPUT_SEPARATOR); byte[] bytes = outputSeparator.getBytes(UTF_8); if (bytes.length != 1) { logConsole(true, true, error, "The output separator must be a single byte in UTF-8."); System.exit(1); } conf.set("logdriver.output.field.separator", Byte.toString(bytes[0])); dOpts(D_options, silent, out, conf); PigServer pigServer = new PigServer(ExecType.MAPREDUCE, conf); pigServer.registerScript(PIG_DIR + "/formatAndSort.pg", params); } catch (Exception e) { e.printStackTrace(); System.exit(1); } logConsole(quiet, silent, warn, "PIG Job Completed."); if (out.equals("-")) { System.out.println(";#################### DATA RESULTS ####################"); try { //Create filter to find files with the results from PIG job PathFilter filter = new PathFilter() { public boolean accept(Path file) { return file.getName().contains("part-"); } }; //Find the files in the directory, open and printout results FileStatus[] status = fs.listStatus(new Path(tmp + "/final"), filter); for (int i = 0; i < status.length; i++) { BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(status[i].getPath()))); String line; line = br.readLine(); while (line != null) { System.out.println(line); line = br.readLine(); } } System.out.println(";#################### END OF RESULTS ####################"); } catch (IOException e) { e.printStackTrace(); System.exit(1); } } else { System.out.println( ";#################### Done. Search results are in " + pigout + " ####################"); } }
From source file:com.blm.orc.OrcInputFormat.java
License:Apache License
@Override public RawReader<OrcStruct> getRawReader(Configuration conf, boolean collapseEvents, int bucket, ValidTxnList validTxnList, Path baseDirectory, Path[] deltaDirectory) throws IOException { Reader reader = null;// w w w . ja v a 2 s . c om boolean isOriginal = false; if (baseDirectory != null) { Path bucketFile; if (baseDirectory.getName().startsWith(AcidUtils.BASE_PREFIX)) { bucketFile = AcidUtils.createBucketFile(baseDirectory, bucket); } else { isOriginal = true; bucketFile = findOriginalBucket(baseDirectory.getFileSystem(conf), baseDirectory, bucket); } reader = OrcFile.createReader(bucketFile, OrcFile.readerOptions(conf)); } return new OrcRawRecordMerger(conf, collapseEvents, reader, isOriginal, bucket, validTxnList, new Reader.Options(), deltaDirectory); }
From source file:com.btoddb.chronicle.apps.AvroTools.java
License:Open Source License
private void go(String srcDir) throws URISyntaxException, IOException { hdfsFs = FileSystem.get(new URI(srcDir), hdfsConfig); System.out.println();/*from w w w . ja va 2s . co m*/ System.out.println("Processing files from " + srcDir); System.out.println(); logger.debug("Searching for files in {}", srcDir); Path path = new Path(srcDir); if (!hdfsFs.exists(path)) { System.out.println("The path does not exist - cannot continue : " + path.toString()); return; } FileStatus[] statuses = hdfsFs.listStatus(path, new PathFilter() { @Override public boolean accept(Path path) { String name = path.getName(); return !name.startsWith("_") && name.endsWith(".avro"); } }); for (FileStatus fs : statuses) { try { Path inPath = fs.getPath(); long fileSize = hdfsFs.getFileStatus(inPath).getLen(); System.out.println(String.format("Processing file, %s (%d)", inPath.toString(), fileSize)); testFileAndFix(inPath); } catch (Exception e) { // don't care about the cause, the test should be able to read all files it cares about e.printStackTrace(); } } }
From source file:com.ceph.rados.fs.hdfs.RadosFileSystem.java
License:Apache License
@Override public boolean rename(Path src, Path dst) throws IOException { Path absoluteSrc = makeAbsolute(src); final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - "; INode srcINode = store.retrieveINode(absoluteSrc); boolean debugEnabled = LOG.isDebugEnabled(); if (srcINode == null) { // src path doesn't exist if (debugEnabled) { LOG.debug(debugPreamble + "returning false as src does not exist"); }// w w w . j a v a 2 s . c om return false; } Path absoluteDst = makeAbsolute(dst); //validate the parent dir of the destination Path dstParent = absoluteDst.getParent(); if (dstParent != null) { //if the dst parent is not root, make sure it exists INode dstParentINode = store.retrieveINode(dstParent); if (dstParentINode == null) { // dst parent doesn't exist if (debugEnabled) { LOG.debug(debugPreamble + "returning false as dst parent does not exist"); } return false; } if (dstParentINode.isFile()) { // dst parent exists but is a file if (debugEnabled) { LOG.debug(debugPreamble + "returning false as dst parent exists and is a file"); } return false; } } //get status of source boolean srcIsFile = srcINode.isFile(); INode dstINode = store.retrieveINode(absoluteDst); boolean destExists = dstINode != null; boolean destIsDir = destExists && !dstINode.isFile(); if (srcIsFile) { //source is a simple file if (destExists) { if (destIsDir) { //outcome #1 dest exists and is dir -filename to subdir of dest if (debugEnabled) { LOG.debug(debugPreamble + "copying src file under dest dir to " + absoluteDst); } absoluteDst = new Path(absoluteDst, absoluteSrc.getName()); } else { //outcome #2 dest it's a file: fail iff different from src boolean renamingOnToSelf = absoluteSrc.equals(absoluteDst); if (debugEnabled) { LOG.debug(debugPreamble + "copying file onto file, outcome is " + renamingOnToSelf); } return renamingOnToSelf; } } else { // #3 dest does not exist: use dest as path for rename if (debugEnabled) { LOG.debug(debugPreamble + "copying file onto file"); } } } else { //here the source exists and is a directory // outcomes (given we know the parent dir exists if we get this far) // #1 destination is a file: fail // #2 destination is a directory: create a new dir under that one // #3 destination doesn't exist: create a new dir with that name // #3 and #4 are only allowed if the dest path is not == or under src if (destExists) { if (!destIsDir) { // #1 destination is a file: fail if (debugEnabled) { LOG.debug(debugPreamble + "returning false as src is a directory, but not dest"); } return false; } else { // the destination dir exists // destination for rename becomes a subdir of the target name absoluteDst = new Path(absoluteDst, absoluteSrc.getName()); if (debugEnabled) { LOG.debug(debugPreamble + "copying src dir under dest dir to " + absoluteDst); } } } //the final destination directory is now know, so validate it for //illegal moves if (absoluteSrc.equals(absoluteDst)) { //you can't rename a directory onto itself if (debugEnabled) { LOG.debug(debugPreamble + "Dest==source && isDir -failing"); } return false; } if (absoluteDst.toString().startsWith(absoluteSrc.toString() + "/")) { //you can't move a directory under itself if (debugEnabled) { LOG.debug(debugPreamble + "dst is equal to or under src dir -failing"); } return false; } } //here the dest path is set up -so rename return renameRecursive(absoluteSrc, absoluteDst); }
From source file:com.cloudera.cdk.data.filesystem.FileSystemDataset.java
License:Apache License
@SuppressWarnings("unchecked") private PartitionKey fromDirectoryName(Path dir) { final FieldPartitioner fp = partitionStrategy.getFieldPartitioners().get(0); final List<Object> values = Lists.newArrayList(); if (partitionKey != null) { values.addAll(partitionKey.getValues()); }//from ww w . j a va 2 s. co m values.add(convert.valueForDirname(fp, dir.getName())); return Accessor.getDefault().newPartitionKey(values.toArray()); }
From source file:com.cloudera.cdk.data.filesystem.FileSystemDatasetWriter.java
License:Apache License
public FileSystemDatasetWriter(FileSystem fileSystem, Path path, Schema schema, boolean enableCompression) { this.fileSystem = fileSystem; this.path = path; this.pathTmp = new Path(path.getParent(), "." + path.getName() + ".tmp"); this.schema = schema; this.enableCompression = enableCompression; this.state = ReaderWriterState.NEW; }