List of usage examples for org.apache.hadoop.fs FileStatus isFile
public boolean isFile()
From source file:org.bgi.flexlab.gaea.tools.mapreduce.annotator.AnnotationMapper.java
License:Open Source License
@Override protected void setup(Context context) throws IOException, InterruptedException { resultKey = new Text(); resultValue = new VcfLineWritable(); conf = context.getConfiguration();//www . j av a 2s .c om vcfCodecs = new HashMap<>(); Path inputPath = new Path(conf.get("inputFilePath")); FileSystem fs = inputPath.getFileSystem(conf); FileStatus[] files = fs.listStatus(inputPath); for (FileStatus file : files) { //System.out.println(file.getPath()); if (file.isFile()) { SingleVCFHeader singleVcfHeader = new SingleVCFHeader(); singleVcfHeader.readHeaderFrom(file.getPath(), fs); VCFHeader vcfHeader = singleVcfHeader.getHeader(); VCFHeaderVersion vcfVersion = singleVcfHeader.getVCFVersion(vcfHeader); VCFCodec vcfcodec = new VCFCodec(); vcfcodec.setVCFHeader(vcfHeader, vcfVersion); vcfCodecs.put(file.getPath().getName(), vcfcodec); } } }
From source file:org.bgi.flexlab.gaea.tools.mapreduce.annotator.AnnotationReducer.java
License:Open Source License
@Override protected void setup(Context context) throws IOException, InterruptedException { long setupStart = System.currentTimeMillis(); conf = context.getConfiguration();//from w w w. j a v a 2s . c o m long start = System.currentTimeMillis(); ReferenceShare genomeShare = new ReferenceShare(); genomeShare.loadChromosomeList(); System.err.println("genomeShare" + (System.currentTimeMillis() - start) + ""); Config userConfig = new Config(conf, genomeShare); userConfig.setVerbose(conf.getBoolean("verbose", false)); userConfig.setDebug(conf.getBoolean("debug", false)); start = System.currentTimeMillis(); AnnotatorBuild annoBuild = new AnnotatorBuild(userConfig); userConfig.setSnpEffectPredictor(annoBuild.createSnpEffPredictor()); annoBuild.buildForest(); System.err.println("build SnpEffectPredictor" + (System.currentTimeMillis() - start) + ""); sampleNames = new ArrayList<>(); vcfCodecs = new HashMap<>(); Path inputPath = new Path(conf.get("inputFilePath")); FileSystem fs = inputPath.getFileSystem(conf); FileStatus[] files = fs.listStatus(inputPath); for (FileStatus file : files) { System.out.println(file.getPath()); if (file.isFile()) { SingleVCFHeader singleVcfHeader = new SingleVCFHeader(); singleVcfHeader.readHeaderFrom(file.getPath(), fs); VCFHeader vcfHeader = singleVcfHeader.getHeader(); VCFHeaderVersion vcfVersion = singleVcfHeader.getVCFVersion(vcfHeader); VCFCodec vcfcodec = new VCFCodec(); vcfcodec.setVCFHeader(vcfHeader, vcfVersion); vcfCodecs.put(file.getPath().getName(), vcfcodec); System.out.println("getname: " + file.getPath().getName()); sampleNames.addAll(vcfHeader.getSampleNamesInOrder()); System.out.println(sampleNames.toString()); } } multipleOutputs = new MultipleOutputs(context); System.err.println("getVCFHeader" + (System.currentTimeMillis() - start) + ""); vcfAnnotator = new VcfAnnotator(userConfig); start = System.currentTimeMillis(); //?? dbAnnotator = new DBAnnotator(userConfig); try { dbAnnotator.connection(); } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { e.printStackTrace(); } System.err.println("dbAnnotator.connection" + (System.currentTimeMillis() - start) + ""); // header? resultValue.set(userConfig.getHeader()); for (int i = 0; i < sampleNames.size(); i++) { multipleOutputs.write(SampleNameModifier.modify(sampleNames.get(i)), NullWritable.get(), resultValue, sampleNames.get(i) + "/part"); } System.err.println("mapper.setup" + (System.currentTimeMillis() - setupStart) + ""); }
From source file:org.bgi.flexlab.gaea.tools.mapreduce.annotator.Annotator.java
License:Open Source License
public int runAnnotator(String[] arg0) throws Exception { Configuration conf = new Configuration(); String[] remainArgs = remainArgs(arg0, conf); AnnotatorOptions options = new AnnotatorOptions(); options.parse(remainArgs);//ww w.j a va2 s .c om options.setHadoopConf(remainArgs, conf); System.out.println("inputFilePath: " + conf.get("inputFilePath")); BioJob job = BioJob.getInstance(conf); if (options.isCachedRef()) System.err.println("--------- isCachedRef --------"); ReferenceShare.distributeCache(options.getReferenceSequencePath(), job); job.setHeader(new Path(options.getInput()), new Path(options.getOutput())); job.setJobName("GaeaAnnotator"); job.setJarByClass(this.getClass()); job.setMapperClass(AnnotationMapper.class); job.setReducerClass(AnnotationReducer.class); job.setNumReduceTasks(options.getReducerNum()); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(VcfLineWritable.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(Text.class); job.setInputFormatClass(MNLineInputFormat.class); List<String> sampleNames = new ArrayList<>(); Path inputPath = new Path(conf.get("inputFilePath")); FileSystem fs = inputPath.getFileSystem(conf); FileStatus[] files = fs.listStatus(inputPath); for (FileStatus file : files) {//sample names System.out.println(file.getPath()); if (file.isFile()) { SingleVCFHeader singleVcfHeader = new SingleVCFHeader(); singleVcfHeader.readHeaderFrom(file.getPath(), fs); VCFHeader vcfHeader = singleVcfHeader.getHeader(); sampleNames.addAll(vcfHeader.getSampleNamesInOrder()); } } MNLineInputFormat.addInputPath(job, new Path(options.getInputFilePath())); MNLineInputFormat.setMinNumLinesToSplit(job, 1000); //???? MNLineInputFormat.setMapperNum(job, options.getMapperNum()); Path partTmp = new Path(options.getTmpPath()); FileOutputFormat.setOutputPath(job, partTmp); for (int i = 0; i < sampleNames.size(); i++)//?sample name? { System.out.println("sampleName " + i + ":" + SampleNameModifier.modify(sampleNames.get(i))); MultipleOutputs.addNamedOutput(job, SampleNameModifier.modify(sampleNames.get(i)), TextOutputFormat.class, NullWritable.class, Text.class); } if (job.waitForCompletion(true)) { for (int i = 0; i < sampleNames.size(); i++) {//?? GZIPOutputStream os = new GZIPOutputStream( new FileOutputStream(options.getOutputPath() + "/" + sampleNames.get(i) + ".tsv.gz")); final FileStatus[] parts = partTmp.getFileSystem(conf).globStatus(new Path(options.getTmpPath() + "/" + sampleNames.get(i) + "/part" + "-*-[0-9][0-9][0-9][0-9][0-9]*")); boolean writeHeader = true; for (FileStatus p : parts) { FSDataInputStream dis = p.getPath().getFileSystem(conf).open(p.getPath()); BufferedReader reader = new BufferedReader(new InputStreamReader(dis)); String line; while ((line = reader.readLine()) != null) { if (line.startsWith("#")) { if (writeHeader) { os.write(line.getBytes()); os.write('\n'); writeHeader = false; } continue; } os.write(line.getBytes()); os.write('\n'); } } os.close(); } partTmp.getFileSystem(conf).delete(partTmp, true); return 0; } else { return 1; } }
From source file:org.bgi.flexlab.gaea.tools.mapreduce.realigner.Realigner.java
License:Open Source License
private ArrayList<Path> inputFilter(String path, Configuration conf, NonRecalibratorPathFilter filter) { ArrayList<Path> lists = new ArrayList<Path>(); Path p = new Path(path); FileSystem fs = HdfsFileManager.getFileSystem(p, conf); FileStatus status = null; try {// w w w . j a va2 s.c o m status = fs.getFileStatus(p); } catch (IOException e2) { throw new FileNotExistException(p.getName()); } if (status.isFile()) { if (filter.accept(status.getPath())) lists.add(status.getPath()); } else { FileStatus[] stats = null; try { stats = fs.listStatus(p, filter); } catch (IOException e) { throw new RuntimeException(e.toString()); } for (FileStatus file : stats) { if (!file.isFile()) { throw new RuntimeException("input directory cann't contains sub directory!!"); } else { if (filter.accept(file.getPath())) lists.add(file.getPath()); } } } if (lists.size() == 0) return null; return lists; }
From source file:org.datacleaner.windows.HdfsUrlChooser.java
License:Open Source License
private void selectOrBrowsePath(final boolean selectDirectory) { // Double-click detected final FileStatus element = _fileList.getModel().getElementAt(_fileList.getSelectedIndex()); if (element.isSymlink()) { try {//from ww w.j a va 2s .co m _currentDirectory = element.getSymlink(); } catch (final IOException e) { logger.warn("Could not get the symlink value for element {}", element, e); } } else if (element.isFile() || (element.isDirectory() && selectDirectory)) { _selectedFile = element.getPath(); logger.info("Selected: " + _selectedFile); _dialog.dispose(); } else if (element.isDirectory()) { _currentDirectory = element.getPath(); _directoryComboBoxModel.updateDirectories(); } ((HdfsDirectoryModel) _fileList.getModel()).updateFileList(); }
From source file:org.exem.flamingo.agent.nn.hdfs.HdfsFileInfo.java
License:Apache License
public HdfsFileInfo(FileStatus fileStatus, ContentSummary contentSummary) { this.fullyQualifiedPath = fileStatus.getPath().toUri().getPath(); this.filename = isEmpty(getFilename(fullyQualifiedPath)) ? getDirectoryName(fullyQualifiedPath) : getFilename(fullyQualifiedPath); this.length = fileStatus.isFile() ? fileStatus.getLen() : contentSummary.getLength(); this.path = getPath(fullyQualifiedPath); this.directory = fileStatus.isDirectory(); this.file = !fileStatus.isDirectory(); this.owner = fileStatus.getOwner(); this.group = fileStatus.getGroup(); this.blockSize = fileStatus.getBlockSize(); this.replication = fileStatus.getReplication(); this.modificationTime = fileStatus.getModificationTime(); if (contentSummary != null) { this.spaceConsumed = contentSummary.getSpaceConsumed(); this.spaceQuota = contentSummary.getSpaceQuota(); this.quota = contentSummary.getQuota(); this.directoryCount = contentSummary.getDirectoryCount(); this.fileCount = contentSummary.getFileCount(); }//from w ww . j a v a2 s . c om this.accessTime = fileStatus.getAccessTime(); this.permission = fileStatus.getPermission().toString(); }
From source file:org.gridgain.grid.kernal.ggfs.hadoop.GridGgfsHadoopFileSystemWrapper.java
License:Open Source License
/** {@inheritDoc} */ @Override/*from w w w. ja v a2s . c o m*/ public GridGgfsFile info(final GridGgfsPath path) throws GridException { try { final FileStatus status = fileSys.getFileStatus(convert(path)); if (status == null) return null; final Map<String, String> props = properties(status); return new GridGgfsFile() { @Override public GridGgfsPath path() { return path; } @Override public boolean isFile() { return status.isFile(); } @Override public boolean isDirectory() { return status.isDirectory(); } @Override public int blockSize() { return (int) status.getBlockSize(); } @Override public long groupBlockSize() { return status.getBlockSize(); } @Override public long accessTime() { return status.getAccessTime(); } @Override public long modificationTime() { return status.getModificationTime(); } @Override public String property(String name) throws IllegalArgumentException { String val = props.get(name); if (val == null) throw new IllegalArgumentException( "File property not found [path=" + path + ", name=" + name + ']'); return val; } @Nullable @Override public String property(String name, @Nullable String dfltVal) { String val = props.get(name); return val == null ? dfltVal : val; } @Override public long length() { return status.getLen(); } /** {@inheritDoc} */ @Override public Map<String, String> properties() { return props; } }; } catch (FileNotFoundException ignore) { return null; } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]"); } }
From source file:org.opencloudengine.garuda.backend.hdfs.HdfsServiceImpl.java
License:Open Source License
@Override public void downloadFile(String path, HttpServletResponse response) throws Exception { this.mustExists(path); FileSystem fs = fileSystemFactory.getFileSystem(); Path fsPath = new Path(path); FileStatus fileStatus = fs.getFileStatus(fsPath); if (!fileStatus.isFile()) { this.notFileException(fsPath.toString()); }//from w ww . j a v a 2 s.c o m HdfsFileInfo fileInfo = new HdfsFileInfo(fileStatus, fs.getContentSummary(fsPath)); FSDataInputStream in = fs.open(fsPath); String filename = fileInfo.getFilename(); response.setHeader("Content-Length", "" + fileInfo.getLength()); response.setHeader("Content-Transfer-Encoding", "binary"); response.setHeader("Content-Type", "application/force-download"); response.setHeader("Content-Disposition", MessageFormatter .format("attachment; fullyQualifiedPath={}; filename={};", URLEncoder.encode(fileInfo.getFullyQualifiedPath(), "UTF-8"), filename) .getMessage()); response.setStatus(200); ServletOutputStream out = response.getOutputStream(); byte[] b = new byte[1024]; int numBytes = 0; while ((numBytes = in.read(b)) > 0) { out.write(b, 0, numBytes); } in.close(); out.close(); fs.close(); }
From source file:org.opencloudengine.garuda.backend.hdfs.HdfsServiceImpl.java
License:Open Source License
@Override public void appendFile(String path, InputStream is) throws Exception { this.mustExists(path); FileSystem fs = fileSystemFactory.getFileSystem(); Path fsPath = new Path(path); FileStatus fileStatus = fs.getFileStatus(fsPath); if (!fileStatus.isFile()) { this.notFileException(fsPath.toString()); }/*from w w w . j a va 2 s .com*/ this._appendFile(path, is); }
From source file:org.shaf.core.io.Location.java
License:Apache License
/** * Returns files selected in this I/O location. * // ww w . j av a 2 s .c o m * @return the selected files. * @throws IOException * if an I/O error occurs. * @throws FileNotFoundException * if a file is not found. */ public final Path[] getFiles() throws FileNotFoundException, IOException { Set<Path> content = new TreeSet<>(); for (FileStatus status : this.fs.listStatus(this.getAbsolutePath(), this.filter)) { if (status.isFile()) { content.add(new Path(IOUtils.normalizePath(status.getPath()).toString() .substring(this.getAbsolutePath().toString().length()))); } } return content.toArray(new Path[content.size()]); }