List of usage examples for java.nio.file.attribute BasicFileAttributes creationTime
FileTime creationTime();
From source file:gobblin.example.simplejson.SimpleJsonSource.java
@Override public List<WorkUnit> getWorkunits(SourceState state) { List<WorkUnit> workUnits = Lists.newArrayList(); if (!state.contains(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL)) { return workUnits; }/*from w w w .jav a 2s. c om*/ // Create a single snapshot-type extract for all files Extract extract = new Extract(state, Extract.TableType.SNAPSHOT_ONLY, state.getProp(ConfigurationKeys.EXTRACT_NAMESPACE_NAME_KEY, "ExampleNamespace"), "ExampleTable"); String filesToPull = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL); File tempFileDir = new File("test_temp/"); // TODO: Delete the dir after completion. tempFileDir.mkdir(); String tempFileDirAbsolute = ""; try { tempFileDirAbsolute = tempFileDir.getCanonicalPath(); // Retrieve absolute path of temp folder } catch (IOException e) { e.printStackTrace(); } int nameCount = 0; int csvCount = 0; for (String file : Splitter.on(',').omitEmptyStrings().split(filesToPull)) { Iterator it = FileUtils.iterateFiles(new File(file), null, true); while (it.hasNext()) { try { File newFile = (File) it.next(); String basePath = newFile.getCanonicalPath(); // Retrieve absolute path of source Path path = newFile.toPath(); //call to rest api:, provide with file basePath String extension = ""; System.out.println("basePath is" + basePath); int i = basePath.lastIndexOf('.'); System.out.println("i"); if (i > 0) { extension = basePath.substring(i + 1); } String url_file_name = ""; int j = basePath.lastIndexOf('/'); if (j > 0) { url_file_name = basePath.substring(j + 1); } //hand off to rest api if (extension.equals("csv")) { System.out.println("CSVCSVCSV"); csvCount += 1; System.out.println("CURL________________________________________"); //Include basePath, filename, location you want to store file System.out.println( "curl http://localhost:8080" + basePath + "/" + Integer.toString(nameCount)); //10.0.2.2 is localhost from vagrant // Insert the nameCount so that it can be joined back later. Process p = Runtime.getRuntime() .exec("curl http://localhost:8080" + basePath + "/" + Integer.toString(nameCount)); // String myUrl = "http://localhost:8080/parse" + basePath + "&" + url_file_name + "&" + tempFileDirAbsolute; // System.out.println("------------------------------"); // System.out.println(myUrl); // try { // URL url = new URL(myUrl); // HttpURLConnection connection = (HttpURLConnection) url.openConnection(); // connection.setRequestMethod("GET"); // connection.connect(); // } catch (Exception e) { // e.printStackTrace(); // } } // Print filename and associated metadata System.out.println(basePath); BasicFileAttributes attr = Files.readAttributes(path, BasicFileAttributes.class); // System.out.println(" creationTime: " + attr.creationTime()); // System.out.println(" lastAccessTime: " + attr.lastAccessTime()); // System.out.println(" lastModifiedTime: " + attr.lastModifiedTime()); // System.out.println(" isDirectory: " + attr.isDirectory()); // System.out.println(" isOther: " + attr.isOther()); // System.out.println(" isRegularFile: " + attr.isRegularFile()); // System.out.println(" isSymbolicLink: " + attr.isSymbolicLink()); // System.out.println(" size: " + attr.size()); // System.out.println(" "); //creating intermediate JSON JSONObject intermediate = new JSONObject(); intermediate.put("filename", basePath); intermediate.put("timestamp", String.valueOf((new Date()).getTime())); intermediate.put("namespace", getMacAddress()); intermediate.put("creationTime", String.valueOf(attr.creationTime())); intermediate.put("lastAccessTime", String.valueOf(attr.lastAccessTime())); intermediate.put("lastModifiedTime", String.valueOf(attr.lastModifiedTime())); intermediate.put("isDirectory", String.valueOf(attr.isDirectory())); intermediate.put("isOther", String.valueOf(attr.isOther())); intermediate.put("isRegularFile", String.valueOf(attr.isRegularFile())); intermediate.put("isSymbolicLink", String.valueOf(attr.isSymbolicLink())); intermediate.put("size", attr.size()); // Create intermediate temp file nameCount += 1; String intermediateName = "/generated" + String.valueOf(nameCount) + ".json"; String finalName = tempFileDirAbsolute + intermediateName; FileWriter generated = new FileWriter(finalName); generated.write(intermediate.toJSONString()); generated.flush(); generated.close(); // Create one work unit for each file to pull WorkUnit workUnit = new WorkUnit(state, extract); workUnit.setProp(SOURCE_FILE_KEY, finalName); workUnits.add(workUnit); } catch (IOException e) { e.printStackTrace(); } } // write out number of files found to temp file try { FileWriter numCsvFiles = new FileWriter(tempFileDirAbsolute + "/numCsvFiles.txt"); numCsvFiles.write("" + csvCount); numCsvFiles.flush(); numCsvFiles.close(); FileWriter numFiles = new FileWriter(tempFileDirAbsolute + "/numFiles.txt"); numFiles.write("" + nameCount); numFiles.flush(); numFiles.close(); } catch (IOException e) { e.printStackTrace(); } } return workUnits; }
From source file:com.nridge.connector.fs.con_fs.core.FileCrawler.java
private void processFile(Path aPath, BasicFileAttributes aFileAttributes) throws IOException { Logger appLogger = mAppMgr.getLogger(this, "processFile"); appLogger.trace(mAppMgr.LOGMSG_TRACE_ENTER); StopWatch stopWatch = new StopWatch(); stopWatch.start();/*ww w.j a v a2 s . c o m*/ File fsFile = aPath.toFile(); String docId = generateDocumentId(aPath); String pathFileName = aPath.toAbsolutePath().toString(); appLogger.debug(String.format("Processing File (%s): %s", docId, pathFileName)); boolean isFileFlat = true; Document fsDocument = new Document(Constants.FS_DOCUMENT_TYPE, mBag); DataBag fileBag = fsDocument.getBag(); fileBag.resetValuesWithDefaults(); fileBag.setValueByName("nsd_id", docId); String fileName = fsFile.getName(); fileBag.setValueByName("nsd_url", fsFile.toURI().toURL().toString()); String viewURL = createViewURL(docId); fileBag.setValueByName("nsd_url_view", viewURL); fileBag.setValueByName("nsd_url_display", viewURL); fileBag.setValueByName("nsd_name", fileName); fileBag.setValueByName("nsd_file_name", fileName); fileBag.setValueByName("nsd_file_size", aFileAttributes.size()); FileTime creationTime = aFileAttributes.creationTime(); Date cDate = new Date(creationTime.toMillis()); fileBag.setValueByName("nsd_doc_created_ts", cDate); FileTime lastModifiedTime = aFileAttributes.lastModifiedTime(); Date lmDate = new Date(lastModifiedTime.toMillis()); fileBag.setValueByName("nsd_doc_modified_ts", lmDate); fileBag.setValueByName("nsd_crawl_type", mCrawlQueue.getCrawlType()); DataField dataField = fileBag.getFirstFieldByFeatureName(Field.FEATURE_IS_CONTENT); if (dataField != null) { ContentExtractor contentExtractor = new ContentExtractor(mAppMgr); contentExtractor.setCfgPropertyPrefix(Constants.CFG_PROPERTY_PREFIX + ".extract"); try { String mimeType = contentExtractor.detectType(fsFile); if (StringUtils.isNotEmpty(mimeType)) fileBag.setValueByName("nsd_mime_type", mimeType); if (isExpandableCSVFile(mimeType)) { isFileFlat = false; processCSVFile(aPath, aFileAttributes, viewURL); } else contentExtractor.process(pathFileName, dataField); } catch (NSException e) { String msgStr = String.format("%s: %s", pathFileName, e.getMessage()); appLogger.error(msgStr); } } if (isFileFlat) { fileBag.setValueByName("nsd_doc_hash", fsDocument.generateUniqueHash(false)); saveAddQueueDocument(fsDocument, stopWatch); } appLogger.trace(mAppMgr.LOGMSG_TRACE_DEPART); }
From source file:com.nridge.connector.fs.con_fs.core.FileCrawler.java
private void processCSVFile(Path aPath, BasicFileAttributes aFileAttributes, String aViewURL) throws IOException { String docId;/*from w w w. jav a 2s .co m*/ StopWatch stopWatch; Document fsDocument; Logger appLogger = mAppMgr.getLogger(this, "processCSVFile"); appLogger.trace(mAppMgr.LOGMSG_TRACE_ENTER); File fsFile = aPath.toFile(); String pathFileName = aPath.toAbsolutePath().toString(); appLogger.debug(String.format("Processing CSV File: %s", pathFileName)); CSVDocument csvDocument = new CSVDocument(mAppMgr, mBag); csvDocument.open(pathFileName); int row = 1; DataBag csvBag = csvDocument.extractNext(); while (csvBag != null) { stopWatch = new StopWatch(); stopWatch.start(); docId = csvBag.generateUniqueHash(true); appLogger.debug(String.format(" Expanding Row [%d]: %s", row++, docId)); csvBag.setValueByName("nsd_id", mIdValuePrefix + docId); csvBag.setValueByName("nsd_url", fsFile.toURI().toURL().toString()); csvBag.setValueByName("nsd_url_view", aViewURL); csvBag.setValueByName("nsd_url_display", aViewURL); csvBag.setValueByName("nsd_file_name", fsFile.getName()); csvBag.setValueByName("nsd_mime_type", Content.CONTENT_TYPE_TXT_CSV); FileTime creationTime = aFileAttributes.creationTime(); Date cDate = new Date(creationTime.toMillis()); csvBag.setValueByName("nsd_doc_created_ts", cDate); FileTime lastModifiedTime = aFileAttributes.lastModifiedTime(); Date lmDate = new Date(lastModifiedTime.toMillis()); csvBag.setValueByName("nsd_doc_modified_ts", lmDate); csvBag.setValueByName("nsd_crawl_type", mCrawlQueue.getCrawlType()); fsDocument = new Document(Constants.FS_DOCUMENT_TYPE, csvBag); csvBag.setValueByName("nsd_doc_hash", fsDocument.generateUniqueHash(false)); saveAddQueueDocument(fsDocument, stopWatch); csvBag = csvDocument.extractNext(); } csvDocument.close(); appLogger.trace(mAppMgr.LOGMSG_TRACE_DEPART); }
From source file:net.mozq.picto.core.ProcessCore.java
public static void findFiles(ProcessCondition processCondition, Consumer<ProcessData> processDataSetter, BooleanSupplier processStopper) throws IOException { Set<FileVisitOption> fileVisitOptionSet; if (processCondition.isFollowLinks()) { fileVisitOptionSet = EnumSet.of(FileVisitOption.FOLLOW_LINKS); } else {//from www .ja va 2s.com fileVisitOptionSet = Collections.emptySet(); } Files.walkFileTree(processCondition.getSrcRootPath(), fileVisitOptionSet, processCondition.getDept(), new SimpleFileVisitor<Path>() { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { if (processStopper.getAsBoolean()) { return FileVisitResult.TERMINATE; } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (attrs.isDirectory()) { return FileVisitResult.SKIP_SUBTREE; } if (processStopper.getAsBoolean()) { return FileVisitResult.TERMINATE; } if (!processCondition.getPathFilter().accept(file, attrs)) { return FileVisitResult.SKIP_SUBTREE; } Path rootRelativeSubPath = processCondition.getSrcRootPath().relativize(file.getParent()); ImageMetadata imageMetadata = getImageMetadata(file); Date baseDate; if (processCondition.isChangeFileCreationDate() || processCondition.isChangeFileModifiedDate() || processCondition.isChangeFileAccessDate() || processCondition.isChangeExifDate()) { baseDate = getBaseDate(processCondition, file, attrs, imageMetadata); } else { baseDate = null; } String destSubPathname = processCondition.getDestSubPathFormat().format(varName -> { try { switch (varName) { case "Now": return new Date(); case "ParentSubPath": return rootRelativeSubPath.toString(); case "FileName": return file.getFileName().toString(); case "BaseName": return FileUtilz.getBaseName(file.getFileName().toString()); case "Extension": return FileUtilz.getExt(file.getFileName().toString()); case "Size": return Long.valueOf(Files.size(file)); case "CreationDate": return (processCondition.isChangeFileCreationDate()) ? baseDate : new Date(attrs.creationTime().toMillis()); case "ModifiedDate": return (processCondition.isChangeFileModifiedDate()) ? baseDate : new Date(attrs.lastModifiedTime().toMillis()); case "AccessDate": return (processCondition.isChangeFileAccessDate()) ? baseDate : new Date(attrs.lastAccessTime().toMillis()); case "PhotoTakenDate": return (processCondition.isChangeExifDate()) ? baseDate : getPhotoTakenDate(file, imageMetadata); case "Width": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_EXIF_IMAGE_WIDTH); case "Height": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_EXIF_IMAGE_LENGTH); case "FNumber": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_FNUMBER); case "Aperture": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_APERTURE_VALUE); case "MaxAperture": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_MAX_APERTURE_VALUE); case "ISO": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_ISO); case "FocalLength": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_FOCAL_LENGTH); // ? case "FocalLength35mm": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_FOCAL_LENGTH_IN_35MM_FORMAT); case "ShutterSpeed": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_SHUTTER_SPEED_VALUE); case "Exposure": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_EXPOSURE); // case "ExposureTime": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_EXPOSURE_TIME); // case "ExposureMode": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_EXPOSURE_MODE); case "ExposureProgram": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_EXPOSURE_PROGRAM); case "Brightness": return getEXIFDoubleValue(imageMetadata, ExifTagConstants.EXIF_TAG_BRIGHTNESS_VALUE); case "WhiteBalance": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_WHITE_BALANCE_1); case "LightSource": return getEXIFIntValue(imageMetadata, ExifTagConstants.EXIF_TAG_LIGHT_SOURCE); case "Lens": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_LENS); case "LensMake": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_LENS_MAKE); case "LensModel": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_LENS_MODEL); case "LensSerialNumber": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_LENS_SERIAL_NUMBER); case "Make": return getEXIFStringValue(imageMetadata, TiffTagConstants.TIFF_TAG_MAKE); case "Model": return getEXIFStringValue(imageMetadata, TiffTagConstants.TIFF_TAG_MODEL); case "SerialNumber": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_SERIAL_NUMBER); case "Software": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_SOFTWARE); case "ProcessingSoftware": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_PROCESSING_SOFTWARE); case "OwnerName": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_OWNER_NAME); case "CameraOwnerName": return getEXIFStringValue(imageMetadata, ExifTagConstants.EXIF_TAG_CAMERA_OWNER_NAME); case "GPSLat": return getEXIFGpsLat(imageMetadata); case "GPSLatDeg": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LATITUDE, 0); case "GPSLatMin": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LATITUDE, 1); case "GPSLatSec": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LATITUDE, 2); case "GPSLatRef": return getEXIFStringValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LATITUDE_REF); case "GPSLon": return getEXIFGpsLon(imageMetadata); case "GPSLonDeg": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LONGITUDE, 0); case "GPSLonMin": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LONGITUDE, 1); case "GPSLonSec": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LONGITUDE, 2); case "GPSLonRef": return getEXIFStringValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_LONGITUDE_REF); case "GPSAlt": return getEXIFDoubleValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_ALTITUDE); case "GPSAltRef": return getEXIFIntValue(imageMetadata, GpsTagConstants.GPS_TAG_GPS_ALTITUDE_REF); default: throw new PictoInvalidDestinationPathException(Messages .getString("message.warn.invalid.destSubPath.varName", varName)); } } catch (PictoException e) { throw e; } catch (Exception e) { throw new PictoInvalidDestinationPathException( Messages.getString("message.warn.invalid.destSubPath.pattern"), e); } }); Path destSubPath = processCondition.getDestRootPath().resolve(destSubPathname).normalize(); if (!destSubPath.startsWith(processCondition.getDestRootPath())) { throw new PictoInvalidDestinationPathException( Messages.getString("message.warn.invalid.destination.path", destSubPath)); } ProcessData processData = new ProcessData(); processData.setSrcPath(file); processData.setSrcFileAttributes(attrs); processData.setDestPath(destSubPath); processData.setBaseDate(baseDate); processDataSetter.accept(processData); return FileVisitResult.CONTINUE; } }); }