List of usage examples for org.apache.hadoop.fs FileStatus getBlockSize
public long getBlockSize()
From source file:org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.java
License:Apache License
/** * Convert a file status obtained from the secondary file system to a status of the primary file system. * * @param status Secondary file system status. * @return Primary file system status.// w w w .java2 s . co m */ private FileStatus toPrimary(FileStatus status) { return status != null ? new FileStatus(status.getLen(), status.isDirectory(), status.getReplication(), status.getBlockSize(), status.getModificationTime(), status.getAccessTime(), status.getPermission(), status.getOwner(), status.getGroup(), toPrimary(status.getPath())) : null; }
From source file:org.apache.ignite.internal.igfs.hadoop.IgfsHadoopFileSystemWrapper.java
License:Apache License
/** {@inheritDoc} */ @Override/*from w ww .j a v a2 s.com*/ public Collection<IgfsFile> listFiles(IgfsPath path) { try { FileStatus[] statuses = fileSys.listStatus(convert(path)); if (statuses == null) throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path); Collection<IgfsFile> res = new ArrayList<>(statuses.length); for (FileStatus status : statuses) { IgfsFileInfo fsInfo = status.isDirectory() ? new IgfsFileInfo(true, properties(status)) : new IgfsFileInfo((int) status.getBlockSize(), status.getLen(), null, null, false, properties(status)); res.add(new IgfsFileImpl(new IgfsPath(path, status.getPath().getName()), fsInfo, 1)); } return res; } catch (FileNotFoundException ignored) { throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path); } }
From source file:org.apache.ignite.internal.igfs.hadoop.IgfsHadoopFileSystemWrapper.java
License:Apache License
/** {@inheritDoc} */ @Override//from ww w. j a va 2s . c o m public IgfsFile info(final IgfsPath path) { try { final FileStatus status = fileSys.getFileStatus(convert(path)); if (status == null) return null; final Map<String, String> props = properties(status); return new IgfsFile() { @Override public IgfsPath path() { return path; } @Override public boolean isFile() { return status.isFile(); } @Override public boolean isDirectory() { return status.isDirectory(); } @Override public int blockSize() { return (int) status.getBlockSize(); } @Override public long groupBlockSize() { return status.getBlockSize(); } @Override public long accessTime() { return status.getAccessTime(); } @Override public long modificationTime() { return status.getModificationTime(); } @Override public String property(String name) throws IllegalArgumentException { String val = props.get(name); if (val == null) throw new IllegalArgumentException( "File property not found [path=" + path + ", name=" + name + ']'); return val; } @Nullable @Override public String property(String name, @Nullable String dfltVal) { String val = props.get(name); return val == null ? dfltVal : val; } @Override public long length() { return status.getLen(); } /** {@inheritDoc} */ @Override public Map<String, String> properties() { return props; } }; } catch (FileNotFoundException ignore) { return null; } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]"); } }
From source file:org.apache.ignite.internal.processors.hadoop.impl.delegate.HadoopIgfsSecondaryFileSystemDelegateImpl.java
License:Apache License
/** {@inheritDoc} */ @Override/*from w w w.ja va2 s . co m*/ public Collection<IgfsFile> listFiles(IgfsPath path) { try { FileStatus[] statuses = fileSystemForUser().listStatus(convert(path)); if (statuses == null) throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path); Collection<IgfsFile> res = new ArrayList<>(statuses.length); for (FileStatus s : statuses) { IgfsEntryInfo fsInfo = s.isDirectory() ? IgfsUtils.createDirectory(IgniteUuid.randomUuid(), null, properties(s), s.getAccessTime(), s.getModificationTime()) : IgfsUtils.createFile(IgniteUuid.randomUuid(), (int) s.getBlockSize(), s.getLen(), null, null, false, properties(s), s.getAccessTime(), s.getModificationTime()); res.add(new IgfsFileImpl(new IgfsPath(path, s.getPath().getName()), fsInfo, 1)); } return res; } catch (FileNotFoundException ignored) { throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path); } }
From source file:org.apache.ignite.internal.processors.hadoop.impl.delegate.HadoopIgfsSecondaryFileSystemDelegateImpl.java
License:Apache License
/** {@inheritDoc} */ @Override//from w ww .j a v a 2s .co m public IgfsFile info(final IgfsPath path) { try { final FileStatus status = fileSystemForUser().getFileStatus(convert(path)); if (status == null) return null; final Map<String, String> props = properties(status); return new IgfsFileImpl(new IgfsFile() { @Override public IgfsPath path() { return path; } @Override public boolean isFile() { return status.isFile(); } @Override public boolean isDirectory() { return status.isDirectory(); } @Override public int blockSize() { // By convention directory has blockSize == 0, while file has blockSize > 0: return isDirectory() ? 0 : (int) status.getBlockSize(); } @Override public long groupBlockSize() { return status.getBlockSize(); } @Override public long accessTime() { return status.getAccessTime(); } @Override public long modificationTime() { return status.getModificationTime(); } @Override public String property(String name) throws IllegalArgumentException { String val = props.get(name); if (val == null) throw new IllegalArgumentException( "File property not found [path=" + path + ", name=" + name + ']'); return val; } @Nullable @Override public String property(String name, @Nullable String dfltVal) { String val = props.get(name); return val == null ? dfltVal : val; } @Override public long length() { return status.getLen(); } /** {@inheritDoc} */ @Override public Map<String, String> properties() { return props; } }, 0); } catch (FileNotFoundException ignore) { return null; } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]"); } }
From source file:org.apache.oozie.action.hadoop.FsELFunctions.java
License:Apache License
/** * Return the file block size in bytes.//from w ww . j a v a 2 s . c o m * * @param pathUri file system path uri. * @return the block size of the file in bytes, -1 if the file does not exist or if it is a directory. * @throws Exception */ public static long fs_blockSize(String pathUri) throws Exception { long blockSize = -1; FileStatus fileStatus = getFileStatus(pathUri); if (fileStatus != null) { blockSize = fileStatus.getBlockSize(); } return blockSize; }
From source file:org.apache.orc.tools.FileDump.java
License:Apache License
private static void recoverFile(final Path corruptPath, final FileSystem fs, final Configuration conf, final List<Long> footerOffsets, final String backup) throws IOException { // first recover the file to .recovered file and then once successful rename it to actual file Path recoveredPath = getRecoveryFile(corruptPath); // make sure that file does not exist if (fs.exists(recoveredPath)) { fs.delete(recoveredPath, false); }/* w w w. j ava 2s .co m*/ // if there are no valid footers, the file should still be readable so create an empty orc file if (footerOffsets == null || footerOffsets.isEmpty()) { System.err.println("No readable footers found. Creating empty orc file."); TypeDescription schema = TypeDescription.createStruct(); Writer writer = OrcFile.createWriter(recoveredPath, OrcFile.writerOptions(conf).setSchema(schema)); writer.close(); } else { FSDataInputStream fdis = fs.open(corruptPath); FileStatus fileStatus = fs.getFileStatus(corruptPath); // read corrupt file and copy it to recovered file until last valid footer FSDataOutputStream fdos = fs.create(recoveredPath, true, conf.getInt("io.file.buffer.size", 4096), fileStatus.getReplication(), fileStatus.getBlockSize()); try { long fileLen = footerOffsets.get(footerOffsets.size() - 1); long remaining = fileLen; while (remaining > 0) { int toRead = (int) Math.min(DEFAULT_BLOCK_SIZE, remaining); byte[] data = new byte[toRead]; long startPos = fileLen - remaining; fdis.readFully(startPos, data, 0, toRead); fdos.write(data); System.err.println("Copying data to recovery file - startPos: " + startPos + " toRead: " + toRead + " remaining: " + remaining); remaining = remaining - toRead; } } catch (Exception e) { fs.delete(recoveredPath, false); throw new IOException(e); } finally { fdis.close(); fdos.close(); } } // validate the recovered file once again and start moving corrupt files to backup folder if (isReadable(recoveredPath, conf, Long.MAX_VALUE)) { Path backupDataPath; String scheme = corruptPath.toUri().getScheme(); String authority = corruptPath.toUri().getAuthority(); String filePath = corruptPath.toUri().getPath(); // use the same filesystem as corrupt file if backup-path is not explicitly specified if (backup.equals(DEFAULT_BACKUP_PATH)) { backupDataPath = new Path(scheme, authority, DEFAULT_BACKUP_PATH + filePath); } else { backupDataPath = Path.mergePaths(new Path(backup), corruptPath); } // Move data file to backup path moveFiles(fs, corruptPath, backupDataPath); // Move side file to backup path Path sideFilePath = OrcAcidUtils.getSideFile(corruptPath); Path backupSideFilePath = new Path(backupDataPath.getParent(), sideFilePath.getName()); moveFiles(fs, sideFilePath, backupSideFilePath); // finally move recovered file to actual file moveFiles(fs, recoveredPath, corruptPath); // we are done recovering, backing up and validating System.err.println("Validation of recovered file successful!"); } }
From source file:org.apache.pig.backend.hadoop.datastorage.HPath.java
License:Apache License
public Map<String, Object> getStatistics() throws IOException { HashMap<String, Object> props = new HashMap<String, Object>(); FileStatus fileStatus = fs.getHFS().getFileStatus(path); props.put(BLOCK_SIZE_KEY, fileStatus.getBlockSize()); props.put(BLOCK_REPLICATION_KEY, fileStatus.getReplication()); props.put(LENGTH_KEY, fileStatus.getLen()); props.put(MODIFICATION_TIME_KEY, fileStatus.getModificationTime()); return props; }
From source file:org.apache.tajo.storage.AbstractStorageManager.java
License:Apache License
/** * Generate the list of files and make them into FileSplits. * * @throws IOException/*from w w w . ja v a2 s . c o m*/ */ public List<FileFragment> getSplits(String tableName, TableMeta meta, Schema schema, Path... inputs) throws IOException { // generate splits' List<FileFragment> splits = Lists.newArrayList(); List<FileFragment> volumeSplits = Lists.newArrayList(); List<BlockLocation> blockLocations = Lists.newArrayList(); for (Path p : inputs) { FileSystem fs = p.getFileSystem(conf); ArrayList<FileStatus> files = Lists.newArrayList(); if (fs.isFile(p)) { files.addAll(Lists.newArrayList(fs.getFileStatus(p))); } else { files.addAll(listStatus(p)); } int previousSplitSize = splits.size(); for (FileStatus file : files) { Path path = file.getPath(); long length = file.getLen(); if (length > 0) { // Get locations of blocks of file BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length); boolean splittable = isSplittable(meta, schema, path, file); if (blocksMetadataEnabled && fs instanceof DistributedFileSystem) { if (splittable) { for (BlockLocation blockLocation : blkLocations) { volumeSplits.add(makeSplit(tableName, path, blockLocation)); } blockLocations.addAll(Arrays.asList(blkLocations)); } else { // Non splittable long blockSize = blkLocations[0].getLength(); if (blockSize >= length) { blockLocations.addAll(Arrays.asList(blkLocations)); for (BlockLocation blockLocation : blkLocations) { volumeSplits.add(makeSplit(tableName, path, blockLocation)); } } else { splits.add(makeNonSplit(tableName, path, 0, length, blkLocations)); } } } else { if (splittable) { long minSize = Math.max(getMinSplitSize(), 1); long blockSize = file.getBlockSize(); // s3n rest api contained block size but blockLocations is one long splitSize = Math.max(minSize, blockSize); long bytesRemaining = length; // for s3 while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) { int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); splits.add(makeSplit(tableName, path, length - bytesRemaining, splitSize, blkLocations[blkIndex].getHosts())); bytesRemaining -= splitSize; } if (bytesRemaining > 0) { int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); splits.add(makeSplit(tableName, path, length - bytesRemaining, bytesRemaining, blkLocations[blkIndex].getHosts())); } } else { // Non splittable splits.add(makeNonSplit(tableName, path, 0, length, blkLocations)); } } } else { //for zero length files splits.add(makeSplit(tableName, path, 0, length)); } } if (LOG.isDebugEnabled()) { LOG.debug("# of splits per partition: " + (splits.size() - previousSplitSize)); } } // Combine original fileFragments with new VolumeId information setVolumeMeta(volumeSplits, blockLocations); splits.addAll(volumeSplits); LOG.info("Total # of splits: " + splits.size()); return splits; }
From source file:org.apache.tajo.storage.FileStorageManager.java
License:Apache License
/** * Generate the list of files and make them into FileSplits. * * @throws IOException//w w w . jav a 2s . c om */ public List<Fragment> getSplits(String tableName, TableMeta meta, Schema schema, Path... inputs) throws IOException { // generate splits' List<Fragment> splits = Lists.newArrayList(); List<Fragment> volumeSplits = Lists.newArrayList(); List<BlockLocation> blockLocations = Lists.newArrayList(); for (Path p : inputs) { FileSystem fs = p.getFileSystem(conf); ArrayList<FileStatus> files = Lists.newArrayList(); if (fs.isFile(p)) { files.addAll(Lists.newArrayList(fs.getFileStatus(p))); } else { files.addAll(listStatus(p)); } int previousSplitSize = splits.size(); for (FileStatus file : files) { Path path = file.getPath(); long length = file.getLen(); if (length > 0) { // Get locations of blocks of file BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length); boolean splittable = isSplittable(meta, schema, path, file); if (blocksMetadataEnabled && fs instanceof DistributedFileSystem) { if (splittable) { for (BlockLocation blockLocation : blkLocations) { volumeSplits.add(makeSplit(tableName, path, blockLocation)); } blockLocations.addAll(Arrays.asList(blkLocations)); } else { // Non splittable long blockSize = blkLocations[0].getLength(); if (blockSize >= length) { blockLocations.addAll(Arrays.asList(blkLocations)); for (BlockLocation blockLocation : blkLocations) { volumeSplits.add(makeSplit(tableName, path, blockLocation)); } } else { splits.add(makeNonSplit(tableName, path, 0, length, blkLocations)); } } } else { if (splittable) { long minSize = Math.max(getMinSplitSize(), 1); long blockSize = file.getBlockSize(); // s3n rest api contained block size but blockLocations is one long splitSize = Math.max(minSize, blockSize); long bytesRemaining = length; // for s3 while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) { int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); splits.add(makeSplit(tableName, path, length - bytesRemaining, splitSize, blkLocations[blkIndex].getHosts())); bytesRemaining -= splitSize; } if (bytesRemaining > 0) { int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); splits.add(makeSplit(tableName, path, length - bytesRemaining, bytesRemaining, blkLocations[blkIndex].getHosts())); } } else { // Non splittable splits.add(makeNonSplit(tableName, path, 0, length, blkLocations)); } } } else { //for zero length files splits.add(makeSplit(tableName, path, 0, length)); } } if (LOG.isDebugEnabled()) { LOG.debug("# of splits per partition: " + (splits.size() - previousSplitSize)); } } // Combine original fileFragments with new VolumeId information setVolumeMeta(volumeSplits, blockLocations); splits.addAll(volumeSplits); LOG.info("Total # of splits: " + splits.size()); return splits; }