List of usage examples for org.apache.hadoop.fs FileStatus getModificationTime
public long getModificationTime()
From source file:org.apache.drill.exec.planner.sql.handlers.ShowFileHandler.java
License:Apache License
@Override public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException { SqlIdentifier from = ((SqlShowFiles) sqlNode).getDb(); DrillFileSystem fs = null;/*from w ww . j ava 2s .c o m*/ String defaultLocation = null; String fromDir = "./"; SchemaPlus defaultSchema = config.getConverter().getDefaultSchema(); SchemaPlus drillSchema = defaultSchema; // Show files can be used without from clause, in which case we display the files in the default schema if (from != null) { // We are not sure if the full from clause is just the schema or includes table name, // first try to see if the full path specified is a schema drillSchema = SchemaUtilites.findSchema(defaultSchema, from.names); if (drillSchema == null) { // Entire from clause is not a schema, try to obtain the schema without the last part of the specified clause. drillSchema = SchemaUtilites.findSchema(defaultSchema, from.names.subList(0, from.names.size() - 1)); fromDir = fromDir + from.names.get((from.names.size() - 1)); } if (drillSchema == null) { throw UserException.validationError().message("Invalid FROM/IN clause [%s]", from.toString()) .build(logger); } } WorkspaceSchema wsSchema; try { wsSchema = (WorkspaceSchema) drillSchema.unwrap(AbstractSchema.class).getDefaultSchema(); } catch (ClassCastException e) { throw UserException.validationError().message( "SHOW FILES is supported in workspace type schema only. Schema [%s] is not a workspace schema.", SchemaUtilites.getSchemaPath(drillSchema)).build(logger); } // Get the file system object fs = wsSchema.getFS(); // Get the default path defaultLocation = wsSchema.getDefaultLocation(); List<ShowFilesCommandResult> rows = new ArrayList<>(); for (FileStatus fileStatus : fs.list(false, new Path(defaultLocation, fromDir))) { ShowFilesCommandResult result = new ShowFilesCommandResult(fileStatus.getPath().getName(), fileStatus.isDir(), !fileStatus.isDir(), fileStatus.getLen(), fileStatus.getOwner(), fileStatus.getGroup(), fileStatus.getPermission().toString(), fileStatus.getAccessTime(), fileStatus.getModificationTime()); rows.add(result); } return DirectPlan.createDirectPlan(context.getCurrentEndpoint(), rows.iterator(), ShowFilesCommandResult.class); }
From source file:org.apache.drill.exec.store.image.GenericMetadataReader.java
License:Apache License
public void read(@NotNull FileType fileType, @NotNull FileStatus fileStatus, @NotNull Metadata metadata) { GenericMetadataDirectory directory = new GenericMetadataDirectory(); boolean skipEPSPreview = false; directory.setLong(GenericMetadataDirectory.TAG_FILE_SIZE, fileStatus.getLen()); // Add local time zone offset to store the last modified time as local time // just like TO_TIMESTAMP(UNIX_TIMESTAMP()) returns local time directory.setDate(GenericMetadataDirectory.TAG_FILE_DATE_TIME, new Date(fileStatus.getModificationTime() + TimeZone.getDefault().getRawOffset())); directory.setString(GenericMetadataDirectory.TAG_FORMAT, fileType.name().toUpperCase()); for (Directory dir : metadata.getDirectories()) { if (dir instanceof JpegDirectory) { final JpegDirectory jpegDir = (JpegDirectory) dir; directory.setPixelWidth(jpegDir, JpegDirectory.TAG_IMAGE_WIDTH); directory.setPixelHeight(jpegDir, JpegDirectory.TAG_IMAGE_HEIGHT); directory.setBitPerPixel(jpegDir, JpegDirectory.TAG_DATA_PRECISION, JpegDirectory.TAG_NUMBER_OF_COMPONENTS); continue; }/*from w w w .ja v a 2 s .co m*/ if (dir instanceof JfifDirectory) { final JfifDirectory jfifDir = (JfifDirectory) dir; try { final int unit = jfifDir.getResUnits(); if (unit == 1 || unit == 2) { directory.setDPIWidth(jfifDir, JfifDirectory.TAG_RESX, unit == 1 ? 1.0 : 2.54); directory.setDPIHeight(jfifDir, JfifDirectory.TAG_RESY, unit == 1 ? 1.0 : 2.54); } } catch (MetadataException e) { // Nothing needs to be done } continue; } if (dir instanceof ExifIFD0Directory) { if (skipEPSPreview) { skipEPSPreview = false; continue; } final ExifIFD0Directory ifd0Dir = (ExifIFD0Directory) dir; directory.setPixelWidth(ifd0Dir, ExifIFD0Directory.TAG_IMAGE_WIDTH); directory.setPixelHeight(ifd0Dir, ExifIFD0Directory.TAG_IMAGE_HEIGHT); directory.setOrientation(ifd0Dir, ExifIFD0Directory.TAG_ORIENTATION); try { final int unit = ifd0Dir.getInt(ExifIFD0Directory.TAG_RESOLUTION_UNIT); if (unit == 2 || unit == 3) { directory.setDPIWidth(ifd0Dir, ExifIFD0Directory.TAG_X_RESOLUTION, unit == 2 ? 1.0 : 2.54); directory.setDPIHeight(ifd0Dir, ExifIFD0Directory.TAG_Y_RESOLUTION, unit == 2 ? 1.0 : 2.54); } } catch (MetadataException e) { // Nothing needs to be done } int[] bitPerSample = ifd0Dir.getIntArray(ExifIFD0Directory.TAG_BITS_PER_SAMPLE); if (bitPerSample != null) { int bitsPerPixel = 0; for (int n : bitPerSample) { bitsPerPixel += n; } directory.setBitPerPixel(bitsPerPixel); } continue; } if (dir instanceof ExifSubIFDDirectory) { final ExifSubIFDDirectory subIFDDir = (ExifSubIFDDirectory) dir; directory.setPixelWidth(subIFDDir, ExifSubIFDDirectory.TAG_EXIF_IMAGE_WIDTH); directory.setPixelHeight(subIFDDir, ExifSubIFDDirectory.TAG_EXIF_IMAGE_HEIGHT); continue; } if (dir instanceof PsdHeaderDirectory) { final PsdHeaderDirectory psdDir = (PsdHeaderDirectory) dir; directory.setPixelWidth(psdDir, PsdHeaderDirectory.TAG_IMAGE_WIDTH); directory.setPixelHeight(psdDir, PsdHeaderDirectory.TAG_IMAGE_HEIGHT); directory.setBitPerPixel(psdDir, PsdHeaderDirectory.TAG_BITS_PER_CHANNEL, PsdHeaderDirectory.TAG_CHANNEL_COUNT); directory.setColorMode(psdDir, PsdHeaderDirectory.TAG_COLOR_MODE); continue; } if (dir instanceof PngDirectory) { final PngDirectory pngDir = (PngDirectory) dir; if (pngDir.getPngChunkType() == PngChunkType.IHDR) { directory.setPixelWidth(pngDir, PngDirectory.TAG_IMAGE_WIDTH); directory.setPixelHeight(pngDir, PngDirectory.TAG_IMAGE_HEIGHT); try { int numOfComponent = 1; int colorType = pngDir.getInt(PngDirectory.TAG_COLOR_TYPE); if (colorType == PngColorType.IndexedColor.getNumericValue()) { directory.setColorMode("Indexed"); } else if (colorType == PngColorType.Greyscale.getNumericValue()) { directory.setColorMode("Grayscale"); } else if (colorType == PngColorType.GreyscaleWithAlpha.getNumericValue()) { numOfComponent = 2; directory.setColorMode("Grayscale"); directory.setAlpha(true); } else if (colorType == PngColorType.TrueColor.getNumericValue()) { numOfComponent = 3; } else if (colorType == PngColorType.TrueColorWithAlpha.getNumericValue()) { numOfComponent = 4; directory.setAlpha(true); } directory.setBitPerPixel(pngDir.getInt(PngDirectory.TAG_BITS_PER_SAMPLE) * numOfComponent); } catch (MetadataException e) { // Nothing needs to be done } continue; } if (pngDir.getPngChunkType() == PngChunkType.pHYs) { try { final int unit = pngDir.getInt(PngDirectory.TAG_UNIT_SPECIFIER); if (unit == 1) { directory.setDPIWidth(pngDir, PngDirectory.TAG_PIXELS_PER_UNIT_X, 0.0254); directory.setDPIHeight(pngDir, PngDirectory.TAG_PIXELS_PER_UNIT_Y, 0.0254); } } catch (MetadataException e) { // Nothing needs to be done } continue; } if (pngDir.getPngChunkType() == PngChunkType.tRNS) { directory.setAlpha(true); continue; } continue; } if (dir instanceof BmpHeaderDirectory) { final BmpHeaderDirectory bmpDir = (BmpHeaderDirectory) dir; directory.setPixelWidth(bmpDir, BmpHeaderDirectory.TAG_IMAGE_WIDTH); directory.setPixelHeight(bmpDir, BmpHeaderDirectory.TAG_IMAGE_HEIGHT); directory.setDPIWidth(bmpDir, BmpHeaderDirectory.TAG_X_PIXELS_PER_METER, 0.0254); directory.setDPIHeight(bmpDir, BmpHeaderDirectory.TAG_Y_PIXELS_PER_METER, 0.0254); try { final int bitsPerPixel = bmpDir.getInt(BmpHeaderDirectory.TAG_BITS_PER_PIXEL); if (bitsPerPixel <= 8) { directory.setColorMode("Indexed"); } directory.setBitPerPixel(bitsPerPixel); } catch (MetadataException e) { // Nothing needs to be done } continue; } if (dir instanceof GifHeaderDirectory) { final GifHeaderDirectory gifDir = (GifHeaderDirectory) dir; directory.setPixelWidth(gifDir, GifHeaderDirectory.TAG_IMAGE_WIDTH); directory.setPixelHeight(gifDir, GifHeaderDirectory.TAG_IMAGE_HEIGHT); directory.setColorMode("Indexed"); directory.setBitPerPixel(gifDir, GifHeaderDirectory.TAG_BITS_PER_PIXEL); continue; } if (dir instanceof GifControlDirectory) { final GifControlDirectory gifControlDir = (GifControlDirectory) dir; directory.setAlpha(gifControlDir, GifControlDirectory.TAG_TRANSPARENT_COLOR_FLAG); continue; } if (dir instanceof IcoDirectory) { final IcoDirectory icoDir = (IcoDirectory) dir; directory.setPixelWidth(icoDir, IcoDirectory.TAG_IMAGE_WIDTH); directory.setPixelHeight(icoDir, IcoDirectory.TAG_IMAGE_HEIGHT); try { if (icoDir.getInt(IcoDirectory.TAG_COLOUR_PALETTE_SIZE) != 0) { directory.setColorMode("Indexed"); } } catch (MetadataException e) { // Nothing needs to be done } directory.setBitPerPixel(icoDir, IcoDirectory.TAG_BITS_PER_PIXEL); directory.setAlpha(true); continue; } if (dir instanceof PcxDirectory) { final PcxDirectory pcxDir = (PcxDirectory) dir; try { directory.setPixelWidth( pcxDir.getInt(PcxDirectory.TAG_XMAX) - pcxDir.getInt(PcxDirectory.TAG_XMIN) + 1); } catch (MetadataException e) { // Nothing needs to be done } try { directory.setPixelHeight( pcxDir.getInt(PcxDirectory.TAG_YMAX) - pcxDir.getInt(PcxDirectory.TAG_YMIN) + 1); } catch (MetadataException e) { // Nothing needs to be done } directory.setDPIWidth(pcxDir, PcxDirectory.TAG_HORIZONTAL_DPI); directory.setDPIHeight(pcxDir, PcxDirectory.TAG_VERTICAL_DPI); directory.setBitPerPixel(pcxDir, PcxDirectory.TAG_BITS_PER_PIXEL, PcxDirectory.TAG_COLOR_PLANES); try { int colorPlanes = pcxDir.getInt(PcxDirectory.TAG_COLOR_PLANES); if (colorPlanes == 1) { if (pcxDir.getInt(PcxDirectory.TAG_PALETTE_TYPE) == 2) { directory.setColorMode("Grayscale"); } else { directory.setColorMode("Indexed"); } } directory.setAlpha(colorPlanes == 4); } catch (MetadataException e) { // Nothing needs to be done } continue; } if (dir instanceof WavDirectory) { final WavDirectory wavDir = (WavDirectory) dir; directory.setColorMode("N/A"); directory.setDuration(wavDir, WavDirectory.TAG_DURATION); directory.setAudioCodec(wavDir, WavDirectory.TAG_FORMAT); directory.setAudioSampleSize(wavDir, WavDirectory.TAG_BITS_PER_SAMPLE); directory.setAudioSampleRate(wavDir, WavDirectory.TAG_SAMPLES_PER_SEC); } if (dir instanceof AviDirectory) { final AviDirectory aviDir = (AviDirectory) dir; directory.setPixelWidth(aviDir, AviDirectory.TAG_WIDTH); directory.setPixelHeight(aviDir, AviDirectory.TAG_HEIGHT); directory.setDuration(aviDir, AviDirectory.TAG_DURATION); directory.setVideoCodec(aviDir, AviDirectory.TAG_VIDEO_CODEC); directory.setFrameRate(aviDir, AviDirectory.TAG_FRAMES_PER_SECOND); directory.setAudioCodec(aviDir, AviDirectory.TAG_AUDIO_CODEC); directory.setAudioSampleRate(aviDir, AviDirectory.TAG_SAMPLES_PER_SECOND); continue; } if (dir instanceof WebpDirectory) { final WebpDirectory webpDir = (WebpDirectory) dir; directory.setPixelWidth(webpDir, WebpDirectory.TAG_IMAGE_WIDTH); directory.setPixelHeight(webpDir, WebpDirectory.TAG_IMAGE_HEIGHT); directory.setAlpha(webpDir, WebpDirectory.TAG_HAS_ALPHA); continue; } if (dir instanceof QuickTimeVideoDirectory) { final QuickTimeVideoDirectory qtVideoDir = (QuickTimeVideoDirectory) dir; directory.setPixelWidth(qtVideoDir, QuickTimeVideoDirectory.TAG_WIDTH); directory.setPixelHeight(qtVideoDir, QuickTimeVideoDirectory.TAG_HEIGHT); directory.setDPIWidth(qtVideoDir, QuickTimeVideoDirectory.TAG_HORIZONTAL_RESOLUTION); directory.setDPIHeight(qtVideoDir, QuickTimeVideoDirectory.TAG_VERTICAL_RESOLUTION); try { int bitsPerPixel = qtVideoDir.getInt(QuickTimeVideoDirectory.TAG_DEPTH) % 32; directory.setBitPerPixel(bitsPerPixel); } catch (MetadataException e) { // Nothing needs to be done } directory.setDuration(qtVideoDir, QuickTimeVideoDirectory.TAG_DURATION); directory.setVideoCodec(qtVideoDir, QuickTimeVideoDirectory.TAG_COMPRESSION_TYPE); directory.setFrameRate(qtVideoDir, QuickTimeVideoDirectory.TAG_FRAME_RATE); continue; } if (dir instanceof QuickTimeSoundDirectory) { final QuickTimeSoundDirectory qtSoundDir = (QuickTimeSoundDirectory) dir; directory.setAudioCodec(qtSoundDir, QuickTimeSoundDirectory.TAG_AUDIO_FORMAT); directory.setAudioSampleSize(qtSoundDir, QuickTimeSoundDirectory.TAG_AUDIO_SAMPLE_SIZE); directory.setAudioSampleRate(qtSoundDir, QuickTimeSoundDirectory.TAG_AUDIO_SAMPLE_RATE); continue; } if (dir instanceof QuickTimeDirectory) { final QuickTimeDirectory qtDir = (QuickTimeDirectory) dir; directory.setDuration(qtDir, QuickTimeDirectory.TAG_DURATION); continue; } if (dir instanceof Mp4VideoDirectory) { final Mp4VideoDirectory mp4VideoDir = (Mp4VideoDirectory) dir; directory.setPixelWidth(mp4VideoDir, Mp4VideoDirectory.TAG_WIDTH); directory.setPixelHeight(mp4VideoDir, Mp4VideoDirectory.TAG_HEIGHT); directory.setDPIWidth(mp4VideoDir, Mp4VideoDirectory.TAG_HORIZONTAL_RESOLUTION); directory.setDPIHeight(mp4VideoDir, Mp4VideoDirectory.TAG_VERTICAL_RESOLUTION); try { int bitsPerPixel = mp4VideoDir.getInt(Mp4VideoDirectory.TAG_DEPTH) % 32; directory.setBitPerPixel(bitsPerPixel); } catch (MetadataException e) { // Nothing needs to be done } directory.setDuration(mp4VideoDir, Mp4VideoDirectory.TAG_DURATION); directory.setVideoCodec(mp4VideoDir, Mp4VideoDirectory.TAG_COMPRESSION_TYPE); directory.setFrameRate(mp4VideoDir, Mp4VideoDirectory.TAG_FRAME_RATE); continue; } if (dir instanceof Mp4SoundDirectory) { final Mp4SoundDirectory mp4SoundDir = (Mp4SoundDirectory) dir; directory.setAudioCodec(mp4SoundDir, Mp4SoundDirectory.TAG_AUDIO_FORMAT); directory.setAudioSampleSize(mp4SoundDir, Mp4SoundDirectory.TAG_AUDIO_SAMPLE_SIZE); directory.setAudioSampleRate(mp4SoundDir, Mp4SoundDirectory.TAG_AUDIO_SAMPLE_RATE); continue; } if (dir instanceof Mp4Directory) { final Mp4Directory mp4Dir = (Mp4Directory) dir; directory.setDuration(mp4Dir, Mp4Directory.TAG_DURATION); continue; } if (dir instanceof EpsDirectory) { final EpsDirectory epsDir = (EpsDirectory) dir; directory.setPixelWidth(epsDir, EpsDirectory.TAG_IMAGE_WIDTH); directory.setPixelHeight(epsDir, EpsDirectory.TAG_IMAGE_HEIGHT); try { int bitsPerPixel = 24; int colorType = epsDir.getInt(EpsDirectory.TAG_COLOR_TYPE); if (colorType == 1) { String imageData = epsDir.getString(EpsDirectory.TAG_IMAGE_DATA); if (imageData != null && imageData.split(" ")[2].equals("1")) { bitsPerPixel = 1; directory.setColorMode("Bitmap"); } else { bitsPerPixel = 8; directory.setColorMode("Grayscale"); } } else if (colorType == 2) { directory.setColorMode("Lab"); } else if (colorType == 4) { bitsPerPixel = 32; directory.setColorMode("CMYK"); } directory.setBitPerPixel(bitsPerPixel); skipEPSPreview = epsDir.containsTag(EpsDirectory.TAG_TIFF_PREVIEW_SIZE); } catch (MetadataException e) { // Nothing needs to be done } continue; } } // Set default value if empty directory.setPixelWidth(0); directory.setPixelHeight(0); directory.setOrientation(0); directory.setDPIWidth(0.0); directory.setDPIHeight(0.0); directory.setColorMode("RGB"); directory.setBitPerPixel(0); directory.setAlpha(false); directory.setDuration(0); directory.setVideoCodec("Unknown"); directory.setFrameRate(0.0); directory.setAudioCodec("Unknown"); directory.setAudioSampleSize(0); directory.setAudioSampleRate(0.0); metadata.addDirectory(directory); }
From source file:org.apache.drill.exec.store.parquet.metadata.Metadata.java
License:Apache License
/** * Check if the parquet metadata needs to be updated by comparing the modification time of the directories with * the modification time of the metadata file * * @param directories List of directories * @param metaFilePath path of parquet metadata cache file * @return true if metadata needs to be updated, false otherwise * @throws IOException if some resources are not accessible *//*from ww w .j a va 2 s .c o m*/ private boolean tableModified(List<Path> directories, Path metaFilePath, Path parentDir, MetadataContext metaContext, FileSystem fs) throws IOException { Stopwatch timer = logger.isDebugEnabled() ? Stopwatch.createStarted() : null; metaContext.setStatus(parentDir); long metaFileModifyTime = fs.getFileStatus(metaFilePath).getModificationTime(); FileStatus directoryStatus = fs.getFileStatus(parentDir); int numDirs = 1; if (directoryStatus.getModificationTime() > metaFileModifyTime) { if (timer != null) { logger.debug("Directory {} was modified. Took {} ms to check modification time of {} directories", directoryStatus.getPath().toString(), timer.elapsed(TimeUnit.MILLISECONDS), numDirs); timer.stop(); } return true; } for (Path directory : directories) { numDirs++; metaContext.setStatus(directory); directoryStatus = fs.getFileStatus(directory); if (directoryStatus.getModificationTime() > metaFileModifyTime) { if (timer != null) { logger.debug( "Directory {} was modified. Took {} ms to check modification time of {} directories", directoryStatus.getPath().toString(), timer.elapsed(TimeUnit.MILLISECONDS), numDirs); timer.stop(); } return true; } } if (timer != null) { logger.debug("No directories were modified. Took {} ms to check modification time of {} directories", timer.elapsed(TimeUnit.MILLISECONDS), numDirs); timer.stop(); } return false; }
From source file:org.apache.drill.exec.store.parquet.Metadata.java
License:Apache License
/** * Check if the parquet metadata needs to be updated by comparing the modification time of the directories with * the modification time of the metadata file * * @param tableMetadata/*w w w . jav a 2 s .co m*/ * @param metaFilePath * @return * @throws IOException */ private boolean tableModified(List<String> directories, Path metaFilePath, Path parentDir, MetadataContext metaContext) throws IOException { Stopwatch timer = Stopwatch.createStarted(); if (metaContext != null) { metaContext.setStatus(parentDir.toString()); } long metaFileModifyTime = fs.getFileStatus(metaFilePath).getModificationTime(); FileStatus directoryStatus = fs.getFileStatus(parentDir); int numDirs = 1; if (directoryStatus.getModificationTime() > metaFileModifyTime) { logger.info("Directory {} was modified. Took {} ms to check modification time of {} directories", directoryStatus.getPath().toString(), timer.elapsed(TimeUnit.MILLISECONDS), numDirs); timer.stop(); return true; } for (String directory : directories) { numDirs++; if (metaContext != null) { metaContext.setStatus(directory); } directoryStatus = fs.getFileStatus(new Path(directory)); if (directoryStatus.getModificationTime() > metaFileModifyTime) { logger.info("Directory {} was modified. Took {} ms to check modification time of {} directories", directoryStatus.getPath().toString(), timer.elapsed(TimeUnit.MILLISECONDS), numDirs); timer.stop(); return true; } } logger.info("No directories were modified. Took {} ms to check modification time of {} directories", timer.elapsed(TimeUnit.MILLISECONDS), numDirs); timer.stop(); return false; }
From source file:org.apache.drill.yarn.core.DfsFacade.java
License:Apache License
/** * Create a local resource definition for YARN. A local resource is one that * must be localized onto the remote node prior to running a command on that * node.//from w w w . j a v a 2 s .c o m * <p> * YARN uses the size and timestamp are used to check if the file has changed * on HDFS to check if YARN can use an existing copy, if any. * <p> * Resources are made public. * * @param conf * Configuration created from the Hadoop config files, in this case, * identifies the target file system. * @param resourcePath * the path (relative or absolute) to the file on the configured file * system (usually HDFS). * @return a YARN local resource records that contains information about path, * size, type, resource and so on that YARN requires. * @throws IOException * if the resource does not exist on the configured file system */ public LocalResource makeResource(Path dfsPath, FileStatus dfsFileStatus, LocalResourceType type, LocalResourceVisibility visibility) throws DfsFacadeException { URL destUrl; try { destUrl = ConverterUtils.getYarnUrlFromPath(FileContext.getFileContext().makeQualified(dfsPath)); } catch (UnsupportedFileSystemException e) { throw new DfsFacadeException("Unable to convert dfs file to a URL: " + dfsPath.toString(), e); } LocalResource resource = LocalResource.newInstance(destUrl, type, visibility, dfsFileStatus.getLen(), dfsFileStatus.getModificationTime()); return resource; }
From source file:org.apache.druid.indexer.JobHelper.java
License:Apache License
/** * Rename the files. This works around some limitations of both FileContext (no s3n support) and NativeS3FileSystem.rename * which will not overwrite/*from ww w . j a va 2 s . c o m*/ * * @param outputFS The output fs * @param indexZipFilePath The original file path * @param finalIndexZipFilePath The to rename the original file to * * @return False if a rename failed, true otherwise (rename success or no rename needed) */ private static boolean renameIndexFiles(final FileSystem outputFS, final Path indexZipFilePath, final Path finalIndexZipFilePath) { try { return RetryUtils.retry(() -> { final boolean needRename; if (outputFS.exists(finalIndexZipFilePath)) { // NativeS3FileSystem.rename won't overwrite, so we might need to delete the old index first final FileStatus zipFile = outputFS.getFileStatus(indexZipFilePath); final FileStatus finalIndexZipFile = outputFS.getFileStatus(finalIndexZipFilePath); if (zipFile.getModificationTime() >= finalIndexZipFile.getModificationTime() || zipFile.getLen() != finalIndexZipFile.getLen()) { log.info("File[%s / %s / %sB] existed, but wasn't the same as [%s / %s / %sB]", finalIndexZipFile.getPath(), DateTimes.utc(finalIndexZipFile.getModificationTime()), finalIndexZipFile.getLen(), zipFile.getPath(), DateTimes.utc(zipFile.getModificationTime()), zipFile.getLen()); outputFS.delete(finalIndexZipFilePath, false); needRename = true; } else { log.info("File[%s / %s / %sB] existed and will be kept", finalIndexZipFile.getPath(), DateTimes.utc(finalIndexZipFile.getModificationTime()), finalIndexZipFile.getLen()); needRename = false; } } else { needRename = true; } if (needRename) { log.info("Attempting rename from [%s] to [%s]", indexZipFilePath, finalIndexZipFilePath); return outputFS.rename(indexZipFilePath, finalIndexZipFilePath); } else { return true; } }, FileUtils.IS_EXCEPTION, NUM_RETRIES); } catch (Exception e) { throw new RuntimeException(e); } }
From source file:org.apache.druid.indexer.path.GranularUnprocessedPathSpec.java
License:Apache License
@Override public Job addInputPaths(HadoopDruidIndexerConfig config, Job job) throws IOException { // This PathSpec breaks so many abstractions that we might as break some more Preconditions.checkState(config.getGranularitySpec() instanceof UniformGranularitySpec, StringUtils.format("Cannot use %s without %s", GranularUnprocessedPathSpec.class.getSimpleName(), UniformGranularitySpec.class.getSimpleName())); final Path betaInput = new Path(getInputPath()); final FileSystem fs = betaInput.getFileSystem(job.getConfiguration()); final Granularity segmentGranularity = config.getGranularitySpec().getSegmentGranularity(); Map<Long, Long> inputModifiedTimes = new TreeMap<>(Ordering.natural().reverse()); for (FileStatus status : FSSpideringIterator.spiderIterable(fs, betaInput)) { final DateTime key = segmentGranularity.toDate(status.getPath().toString()); final Long currVal = inputModifiedTimes.get(key.getMillis()); final long mTime = status.getModificationTime(); inputModifiedTimes.put(key.getMillis(), currVal == null ? mTime : Math.max(currVal, mTime)); }// w w w. j av a 2 s . c om Set<Interval> bucketsToRun = new TreeSet<>(Comparators.intervals()); for (Map.Entry<Long, Long> entry : inputModifiedTimes.entrySet()) { DateTime timeBucket = DateTimes.utc(entry.getKey()); long mTime = entry.getValue(); String bucketOutput = StringUtils.format("%s/%s", config.getSchema().getIOConfig().getSegmentOutputPath(), segmentGranularity.toPath(timeBucket)); for (FileStatus fileStatus : FSSpideringIterator.spiderIterable(fs, new Path(bucketOutput))) { if (fileStatus.getModificationTime() > mTime) { bucketsToRun.add(new Interval(timeBucket, segmentGranularity.increment(timeBucket))); break; } } if (bucketsToRun.size() >= maxBuckets) { break; } } config.setGranularitySpec( new UniformGranularitySpec(segmentGranularity, config.getGranularitySpec().getQueryGranularity(), config.getGranularitySpec().isRollup(), Lists.newArrayList(bucketsToRun) )); return super.addInputPaths(config, job); }
From source file:org.apache.eagle.jpm.mr.history.crawler.JobHistoryDAOImpl.java
License:Apache License
@SuppressWarnings("deprecation") @Override/*from w w w . j a va 2 s . com*/ public List<Pair<Long, String>> readFileNames(int year, int month, int day, int serialNumber) throws Exception { LOG.info("crawl file names under one serial number : " + year + "/" + month + "/" + day + ":" + serialNumber); List<Pair<Long, String>> jobFileNames = new ArrayList<>(); String serialPath = buildWholePathToSerialNumber(year, month, day, serialNumber); try { Path hdfsFile = new Path(serialPath); // filter those files which is job configuration file in xml format FileStatus[] files = hdfs.listStatus(hdfsFile, new PathFilter() { @Override public boolean accept(Path path) { if (path.getName().endsWith(".xml")) { return false; } return true; } }); for (FileStatus fs : files) { if (!fs.isDir()) { jobFileNames.add(Pair.of(fs.getModificationTime(), fs.getPath().getName())); } } if (LOG.isDebugEnabled()) { StringBuilder sb = new StringBuilder(); for (Pair<Long, String> sn : jobFileNames) { sb.append(sn.getRight()); sb.append(","); } LOG.debug("crawled: " + sb); } } catch (Exception ex) { LOG.error("fail reading job history file names under serial number " + serialPath, ex); throw ex; } return jobFileNames; }
From source file:org.apache.falcon.cleanup.AbstractCleanupHandler.java
License:Apache License
private void deleteInternal(FileSystem fs, Cluster cluster, Entity entity, long retention, FileStatus[] logs) throws FalconException { if (logs == null || logs.length == 0) { LOG.info("Nothing to delete for cluster: {}, entity: {}", cluster.getName(), entity.getName()); return;/*from w ww . j a v a 2 s . co m*/ } long now = System.currentTimeMillis(); for (FileStatus log : logs) { if (now - log.getModificationTime() > retention) { try { boolean isDeleted = fs.delete(log.getPath(), true); LOG.error(isDeleted ? "Deleted path: {}" : "Unable to delete path: {}", log.getPath()); deleteParentIfEmpty(fs, log.getPath().getParent()); } catch (IOException e) { throw new FalconException(" Unable to delete log file : " + log.getPath() + " for entity " + entity.getName() + " for cluster: " + cluster.getName(), e); } } else { LOG.info("Retention limit: {} is less than modification {} for path: {}", retention, (now - log.getModificationTime()), log.getPath()); } } }
From source file:org.apache.falcon.entity.FileSystemStorage.java
License:Apache License
@Override @SuppressWarnings("MagicConstant") public List<FeedInstanceStatus> getListing(Feed feed, String clusterName, LocationType locationType, Date start, Date end) throws FalconException { Calendar calendar = Calendar.getInstance(); List<Location> clusterSpecificLocation = FeedHelper.getLocations(FeedHelper.getCluster(feed, clusterName), feed);// www .ja va 2s . c o m Location location = getLocation(clusterSpecificLocation, locationType); try { FileSystem fileSystem = HadoopClientFactory.get().createProxiedFileSystem(getConf()); Cluster cluster = ClusterHelper.getCluster(clusterName); Properties baseProperties = FeedHelper.getClusterProperties(cluster); baseProperties.putAll(FeedHelper.getFeedProperties(feed)); List<FeedInstanceStatus> instances = new ArrayList<FeedInstanceStatus>(); Date feedStart = FeedHelper.getCluster(feed, clusterName).getValidity().getStart(); TimeZone tz = feed.getTimezone(); Date alignedStart = EntityUtil.getNextStartTime(feedStart, feed.getFrequency(), tz, start); String basePath = location.getPath(); while (!end.before(alignedStart)) { Properties allProperties = ExpressionHelper.getTimeVariables(alignedStart, tz); allProperties.putAll(baseProperties); String feedInstancePath = ExpressionHelper.substitute(basePath, allProperties); FileStatus fileStatus = getFileStatus(fileSystem, new Path(feedInstancePath)); FeedInstanceStatus instance = new FeedInstanceStatus(feedInstancePath); Date date = FeedHelper.getDate(basePath, new Path(feedInstancePath), tz); instance.setInstance(SchemaHelper.formatDateUTC(date)); if (fileStatus != null) { instance.setCreationTime(fileStatus.getModificationTime()); ContentSummary contentSummary = fileSystem.getContentSummary(fileStatus.getPath()); if (contentSummary != null) { long size = contentSummary.getSpaceConsumed(); instance.setSize(size); if (!StringUtils.isEmpty(feed.getAvailabilityFlag())) { FileStatus doneFile = getFileStatus(fileSystem, new Path(fileStatus.getPath(), feed.getAvailabilityFlag())); if (doneFile != null) { instance.setStatus(FeedInstanceStatus.AvailabilityStatus.AVAILABLE); } else { instance.setStatus(FeedInstanceStatus.AvailabilityStatus.PARTIAL); } } else { instance.setStatus(size > 0 ? FeedInstanceStatus.AvailabilityStatus.AVAILABLE : FeedInstanceStatus.AvailabilityStatus.EMPTY); } } } instances.add(instance); calendar.setTime(alignedStart); calendar.add(feed.getFrequency().getTimeUnit().getCalendarUnit(), feed.getFrequency().getFrequencyAsInt()); alignedStart = calendar.getTime(); } return instances; } catch (IOException e) { LOG.error("Unable to retrieve listing for {}:{}", locationType, getStorageUrl(), e); throw new FalconException("Unable to retrieve listing for (URI " + getStorageUrl() + ")", e); } }