List of usage examples for java.nio.file Path getFileName
Path getFileName();
From source file:org.ow2.authzforce.pap.dao.flatfile.FlatFileBasedDomainsDAO.java
/** * Creates instance//from w w w . ja v a 2 s.com * * @param domainsRoot * root directory of the configuration data of security domains, * one subdirectory per domain * @param domainTmpl * domain template directory; directories of new domains are * created from this template * @param domainsSyncIntervalSec * how often (in seconds) the synchronization of managed domains * (in memory) with the domain subdirectories in the * <code>domainsRoot</code> directory (on disk) is done. If * <code>domainSyncInterval</code> > 0, every * <code>domainSyncInterval</code>, the managed domains (loaded * in memory) are updated if any change has been detected in the * <code>domainsRoot</code> directory in this interval (since * last sync). To be more specific, <i>any change</i> here means * any creation/deletion/modification of a domain folder * (modification means: any file changed within the folder). If * <code>domainSyncInterval</code> <= 0, synchronization is * disabled. * @param pdpModelHandler * PDP configuration model handler * @param useRandomAddressBasedUUID * true iff a random multicast address must be used as node field * of generated UUIDs (Version 1), else the MAC address of one of * the network interfaces is used. Setting this to 'true' is NOT * recommended unless the host is disconnected from the network. * These generated UUIDs are used for domain IDs. * @param domainDAOClientFactory * domain DAO client factory * @throws IOException * I/O error occurred scanning existing domain folders in * {@code domainsRoot} for loading. */ @ConstructorProperties({ "domainsRoot", "domainTmpl", "domainsSyncIntervalSec", "pdpModelHandler", "useRandomAddressBasedUUID", "domainDAOClientFactory" }) public FlatFileBasedDomainsDAO(final Resource domainsRoot, final Resource domainTmpl, final int domainsSyncIntervalSec, final PdpModelHandler pdpModelHandler, final boolean useRandomAddressBasedUUID, final DomainDAOClient.Factory<VERSION_DAO_CLIENT, POLICY_DAO_CLIENT, FlatFileBasedDomainDAO<VERSION_DAO_CLIENT, POLICY_DAO_CLIENT>, DOMAIN_DAO_CLIENT> domainDAOClientFactory) throws IOException { if (domainsRoot == null || domainTmpl == null || pdpModelHandler == null || domainDAOClientFactory == null) { throw ILLEGAL_CONSTRUCTOR_ARGS_EXCEPTION; } this.domainDAOClientFactory = domainDAOClientFactory; this.policyDAOClientFactory = domainDAOClientFactory.getPolicyDAOClientFactory(); this.policyVersionDAOClientFactory = policyDAOClientFactory.getVersionDAOClientFactory(); this.uuidGen = initUUIDGenerator(useRandomAddressBasedUUID); this.pdpModelHandler = pdpModelHandler; // Validate domainsRoot arg if (!domainsRoot.exists()) { throw new IllegalArgumentException( "'domainsRoot' resource does not exist: " + domainsRoot.getDescription()); } final String ioExMsg = "Cannot resolve 'domainsRoot' resource '" + domainsRoot.getDescription() + "' as a file on the file system"; File domainsRootFile = null; try { domainsRootFile = domainsRoot.getFile(); } catch (final IOException e) { throw new IllegalArgumentException(ioExMsg, e); } this.domainsRootDir = domainsRootFile.toPath(); FlatFileDAOUtils.checkFile("File defined by SecurityDomainManager parameter 'domainsRoot'", domainsRootDir, true, true); // Validate domainTmpl directory arg if (!domainTmpl.exists()) { throw new IllegalArgumentException( "'domainTmpl' resource does not exist: " + domainTmpl.getDescription()); } final String ioExMsg2 = "Cannot resolve 'domainTmpl' resource '" + domainTmpl.getDescription() + "' as a file on the file system"; File domainTmplFile = null; try { domainTmplFile = domainTmpl.getFile(); } catch (final IOException e) { throw new IllegalArgumentException(ioExMsg2, e); } this.domainTmplDirPath = domainTmplFile.toPath(); FlatFileDAOUtils.checkFile("File defined by SecurityDomainManager parameter 'domainTmpl'", domainTmplDirPath, true, false); LOGGER.debug("Looking for domain sub-directories in directory {}", domainsRootDir); try (final DirectoryStream<Path> dirStream = Files.newDirectoryStream(domainsRootDir)) { for (final Path domainPath : dirStream) { LOGGER.debug("Checking domain in file {}", domainPath); if (!Files.isDirectory(domainPath)) { LOGGER.warn("Ignoring invalid domain file {} (not a directory)", domainPath); continue; } // domain folder name is the domain ID final Path lastPathSegment = domainPath.getFileName(); if (lastPathSegment == null) { throw new RuntimeException("Invalid Domain folder path '" + domainPath + "': no filename"); } final String domainId = lastPathSegment.toString(); FlatFileBasedDomainDAO<VERSION_DAO_CLIENT, POLICY_DAO_CLIENT> domainDAO = null; try { domainDAO = new FileBasedDomainDAOImpl(domainPath, null); } catch (final IllegalArgumentException e) { throw new RuntimeException("Invalid domain data for domain '" + domainId + "'", e); } final DOMAIN_DAO_CLIENT domain = domainDAOClientFactory.getInstance(domainId, domainDAO); domainMap.put(domainId, domain); } } catch (final IOException e) { throw new IOException("Failed to scan files in the domains root directory '" + domainsRootDir + "' looking for domain directories", e); } this.domainDirToMemSyncIntervalSec = Integer.valueOf(domainsSyncIntervalSec).longValue(); }
From source file:eu.itesla_project.dymola.DymolaImpactAnalysis.java
private List<String> writeDymolaInputs(Path workingDir, List<Contingency> contingencies) throws IOException { LOGGER.info(" Start writing dymola inputs"); List<String> retList = new ArrayList<>(); DdbConfig ddbConfig = DdbConfig.load(); String jbossHost = ddbConfig.getJbossHost(); String jbossPort = ddbConfig.getJbossPort(); String jbossUser = ddbConfig.getJbossUser(); String jbossPassword = ddbConfig.getJbossPassword(); Path dymolaExportPath = workingDir.resolve(MO_EXPORT_DIRECTORY); if (!Files.exists(dymolaExportPath)) { Files.createDirectory(dymolaExportPath); }//from ww w .jav a 2 s. co m //retrieve modelica export parameters from configuration String modelicaVersion = config.getModelicaVersion(); String sourceEngine = config.getSourceEngine(); String sourceVersion = config.getSourceEngineVersion(); Path modelicaPowerSystemLibraryPath = Paths.get(config.getModelicaPowerSystemLibraryFile()); //write the modelica events file, to feed the modelica exporter Path eventsPath = workingDir.resolve(MODELICA_EVENTS_CSV_FILENAME); writeModelicaExporterContingenciesFile(eventsPath, contingencies); //these are only optional params needed if the source is eurostag Path modelicaLibPath = null; String slackId = config.getSlackId(); if ("".equals(slackId)) { slackId = null; // null when not specified () } LoadFlowFactory loadFlowFactory; try { loadFlowFactory = config.getLoadFlowFactoryClass().newInstance(); } catch (InstantiationException | IllegalAccessException e) { throw new RuntimeException(e); } LOGGER.info("Exporting modelica data for network {}, working state-id {} ", network, network.getStateManager().getWorkingStateId()); ModelicaMainExporter exporter = new ModelicaMainExporter(network, slackId, jbossHost, jbossPort, jbossUser, jbossPassword, modelicaVersion, sourceEngine, sourceVersion, modelicaLibPath, loadFlowFactory); exporter.export(dymolaExportPath); ModEventsExport eventsExporter = new ModEventsExport( dymolaExportPath.resolve(network.getId() + ".mo").toFile(), eventsPath.toFile()); eventsExporter.export(dymolaExportPath); LOGGER.info(" modelica data exported."); // now assemble the input files to feed dymola // one .zip per contingency; in the zip, the .mo file and the powersystem library //TODO here it is assumed that contingencies ids in csv file start from 0 (i.e. 0 is the first contingency); id should be decoupled from the implementation try (final Stream<Path> pathStream = Files.walk(dymolaExportPath)) { pathStream.filter((p) -> !p.toFile().isDirectory() && p.toFile().getAbsolutePath().contains("events_") && p.toFile().getAbsolutePath().endsWith(".mo")).forEach(p -> { GenericArchive archive = ShrinkWrap.createDomain().getArchiveFactory() .create(GenericArchive.class); try (FileSystem fileSystem = ShrinkWrapFileSystems.newFileSystem(archive)) { Path rootDir = fileSystem.getPath("/"); Files.copy(modelicaPowerSystemLibraryPath, rootDir.resolve(modelicaPowerSystemLibraryPath.getFileName())); Files.copy(Paths.get(p.toString()), rootDir.resolve(DymolaUtil.DYMOLA_SIM_MODEL_INPUT_PREFIX + ".mo")); String[] c = p.getFileName().toString().replace(".mo", "").split("_"); try (OutputStream os = Files.newOutputStream(dymolaExportPath.getParent().resolve( DymolaUtil.DYMOLAINPUTZIPFILENAMEPREFIX + "_" + c[c.length - 1] + ".zip"))) { archive.as(ZipExporter.class).exportTo(os); retList.add(new String(c[c.length - 1])); } catch (IOException e) { //e.printStackTrace(); throw new RuntimeException(e); } } catch (IOException e) { throw new RuntimeException(e); } }); } retList.sort(Comparator.<String>naturalOrder()); //prepare param inputs for indexes from indexes properties file LOGGER.info("writing input indexes parameters in .mat format - start "); try { Path baseWp43ConfigFile = PlatformConfig.CONFIG_DIR.resolve(WP43_CONFIG_FILE_NAME); HierarchicalINIConfiguration configuration = new HierarchicalINIConfiguration( baseWp43ConfigFile.toFile()); //fix params for smallsignal index (cfr EurostagImpactAnalysis sources) SubnodeConfiguration node = configuration.getSection("smallsignal"); node.setProperty("f_instant", Double.toString(parameters.getFaultEventInstant())); for (int i = 0; i < contingencies.size(); i++) { Contingency contingency = contingencies.get(i); if (contingency.getElements().isEmpty()) { throw new AssertionError("Empty contingency " + contingency.getId()); } Iterator<ContingencyElement> it = contingency.getElements().iterator(); // compute the maximum fault duration double maxDuration = getFaultDuration(it.next()); while (it.hasNext()) { maxDuration = Math.max(maxDuration, getFaultDuration(it.next())); } node.setProperty("f_duration", Double.toString(maxDuration)); } DymolaAdaptersMatParamsWriter writer = new DymolaAdaptersMatParamsWriter(configuration); for (String cId : retList) { String parFileNamePrefix = DymolaUtil.DYMOLA_SIM_MAT_OUTPUT_PREFIX + "_" + cId + "_wp43_"; String parFileNameSuffix = "_pars.mat"; String zippedParFileNameSuffix = "_pars.zip"; try (OutputStream os = Files.newOutputStream(dymolaExportPath.getParent() .resolve(DymolaUtil.DYMOLAINPUTZIPFILENAMEPREFIX + "_" + cId + zippedParFileNameSuffix))) { JavaArchive archive = ShrinkWrap.create(JavaArchive.class); Path sfile1 = ShrinkWrapFileSystems.newFileSystem(archive).getPath("/"); Arrays.asList(config.getIndexesNames()).forEach(indexName -> writer.write(indexName, sfile1.resolve(parFileNamePrefix + indexName + parFileNameSuffix))); archive.as(ZipExporter.class).exportTo(os); } catch (Exception e) { throw new RuntimeException(e); } } } catch (ConfigurationException exc) { throw new RuntimeException(exc); } LOGGER.info("writing input indexes parameters in .mat format - end - {}", retList); return retList; }
From source file:com.github.podd.resources.UploadArtifactResourceImpl.java
private InferredOWLOntologyID uploadFileAndLoadArtifactIntoPodd(final Representation entity) throws ResourceException { List<FileItem> items;// w w w . j a v a 2s . c om Path filePath = null; String contentType = null; // 1: Create a factory for disk-based file items final DiskFileItemFactory factory = new DiskFileItemFactory(1000240, this.tempDirectory.toFile()); // 2: Create a new file upload handler final RestletFileUpload upload = new RestletFileUpload(factory); final Map<String, String> props = new HashMap<String, String>(); try { // 3: Request is parsed by the handler which generates a list of // FileItems items = upload.parseRequest(this.getRequest()); for (final FileItem fi : items) { final String name = fi.getName(); if (name == null) { props.put(fi.getFieldName(), new String(fi.get(), StandardCharsets.UTF_8)); } else { // FIXME: Strip everything up to the last . out of the // filename so that // the filename can be used for content type determination // where // possible. // InputStream uploadedFileInputStream = // fi.getInputStream(); try { // Note: These are Java-7 APIs contentType = fi.getContentType(); props.put("Content-Type", fi.getContentType()); filePath = Files.createTempFile(this.tempDirectory, "ontologyupload-", name); final File file = filePath.toFile(); file.deleteOnExit(); fi.write(file); } catch (final IOException ioe) { throw ioe; } catch (final Exception e) { // avoid throwing a generic exception just because the // apache // commons library throws Exception throw new IOException(e); } } } } catch (final IOException | FileUploadException e) { throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST, e); } this.log.info("props={}", props.toString()); if (filePath == null) { throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST, "Did not submit a valid file and filename"); } this.log.info("filename={}", filePath.toAbsolutePath().toString()); this.log.info("contentType={}", contentType); RDFFormat format = null; // If the content type was application/octet-stream then use the file // name instead // Browsers attach this content type when they are not sure what the // real type is if (MediaType.APPLICATION_OCTET_STREAM.getName().equals(contentType)) { format = Rio.getParserFormatForFileName(filePath.getFileName().toString()); this.log.info("octet-stream contentType filename format={}", format); } // Otherwise use the content type directly in preference to using the // filename else if (contentType != null) { format = Rio.getParserFormatForMIMEType(contentType); this.log.info("non-octet-stream contentType format={}", format); } // If the content type choices failed to resolve the type, then try the // filename if (format == null) { format = Rio.getParserFormatForFileName(filePath.getFileName().toString()); this.log.info("non-content-type filename format={}", format); } // Or fallback to RDF/XML which at minimum is able to detect when the // document is // structurally invalid if (format == null) { this.log.warn("Could not determine RDF format from request so falling back to RDF/XML"); format = RDFFormat.RDFXML; } try (final InputStream inputStream = new BufferedInputStream( Files.newInputStream(filePath, StandardOpenOption.READ));) { return this.uploadFileAndLoadArtifactIntoPodd(inputStream, format, DanglingObjectPolicy.REPORT, DataReferenceVerificationPolicy.DO_NOT_VERIFY); } catch (final IOException e) { throw new ResourceException(Status.SERVER_ERROR_INTERNAL, "File IO error occurred", e); } }
From source file:net.rptools.tokentool.controller.ManageOverlays_Controller.java
private void loadImages(File dir) { // Clear Details panel clearDetails();/*from w w w . j av a 2s.co m*/ currentDirectory = dir; File[] files = dir.listFiles(ImageUtil.SUPPORTED_FILENAME_FILTER); Task<Void> task = new Task<Void>() { @Override public Void call() { for (File file : files) { Path filePath = file.toPath(); if (loadOverlaysThread.isInterrupted()) { Platform.runLater(() -> overlayViewFlowPane.getChildren().clear()); break; } try { ToggleButton overlayButton = new ToggleButton(); ImageView imageViewNode = ImageUtil.getOverlayThumb(new ImageView(), filePath); overlayButton.getStyleClass().add("overlay-toggle-button"); overlayButton.setGraphic(imageViewNode); overlayButton.setUserData(file); overlayButton.setToggleGroup(overlayToggleGroup); overlayButton.addEventHandler(ActionEvent.ACTION, event -> { // No modifier keys used so add toggle group back to all buttons resetToggleGroup(); // Also set button to selected due to resetting toggle groups & no unselecting needed, makes for better interface IMO overlayButton.setSelected(true); // Update the Details panel with the last selected overlay File overlayFile = (File) overlayButton.getUserData(); updateDetails(overlayFile, (ImageView) overlayButton.getGraphic(), overlayButton.isSelected()); // Consume the event, no more logic needed event.consume(); }); overlayButton.setOnMouseClicked(new EventHandler<MouseEvent>() { @Override public void handle(MouseEvent event) { // Allow multiple selections if shortcutKey+left_mouse is pressed if (event.getButton().equals(MouseButton.PRIMARY) && event.isShortcutDown()) { // Update the Details panel with the last selected overlay File overlayFile = (File) overlayButton.getUserData(); updateDetails(overlayFile, (ImageView) overlayButton.getGraphic(), true); // Remove the toggle group to allow multiple toggle button selection overlayButton.setToggleGroup(null); // Select the button overlayButton.setSelected(true); // Consume the event, no more logic needed event.consume(); } } }); Platform.runLater(() -> overlayViewFlowPane.getChildren().add(overlayButton)); } catch (IOException e) { log.error("Loading image: " + filePath.getFileName(), e); } } return null; } }; loadOverlaysThread.interrupt(); executorService.execute(task); }
From source file:org.ow2.authzforce.pap.dao.flatfile.FlatFileBasedDomainsDAO.java
@Override public Set<String> getDomainIDs(final String externalId) throws IOException { synchronized (domainsRootDir) { if (externalId != null) { // externalId not null final String domainId = domainIDsByExternalId.get(externalId); if (domainId == null) { return Collections.<String>emptySet(); }//from w w w . ja v a2s . c om // domainId not null, check if domain is still there in the // repository final Path domainDirPath = this.domainsRootDir.resolve(domainId); if (Files.exists(domainDirPath, LinkOption.NOFOLLOW_LINKS)) { return Collections.<String>singleton(domainId); } // domain directory no longer exists, remove from map and so on removeDomainFromCache(domainId); return Collections.<String>emptySet(); } // externalId == null /* * All changes to domainMap are synchronized by 'domainsRootDir'. So * we can iterate and change if necessary for synchronizing the * domains root directory with the domainMap (Using a domainMap is * necessary for quick access to domains' PDPs.) */ final Set<String> oldDomainIDs = new HashSet<>(domainMap.keySet()); final Set<String> newDomainIDs = new HashSet<>(); try (final DirectoryStream<Path> dirStream = Files.newDirectoryStream(domainsRootDir)) { for (final Path domainDirPath : dirStream) { LOGGER.debug("Checking domain in file {}", domainDirPath); if (!Files.isDirectory(domainDirPath)) { LOGGER.warn("Ignoring invalid domain file {} (not a directory)", domainDirPath); continue; } // domain folder name is the domain ID final Path lastPathSegment = domainDirPath.getFileName(); if (lastPathSegment == null) { throw new RuntimeException( "Invalid Domain folder path '" + domainDirPath + "': no filename"); } final String domainId = lastPathSegment.toString(); newDomainIDs.add(domainId); if (oldDomainIDs.remove(domainId)) { // not new domain, but directory may have changed -> // sync final DOMAIN_DAO_CLIENT domain = domainMap.get(domainId); if (domain != null) { domain.getDAO().sync(); } } else { // new domain directory addDomainToCacheAfterDirectoryCreated(domainId, domainDirPath, null); } } } catch (final IOException e) { throw new IOException("Failed to scan files in the domains root directory '" + domainsRootDir + "' looking for domain directories", e); } if (!oldDomainIDs.isEmpty()) { // old domains remaining in cache that don't match directories // -> removed // -> remove from cache for (final String domainId : oldDomainIDs) { removeDomainFromCache(domainId); } } return newDomainIDs; } }
From source file:com.pianobakery.complsa.MainGui.java
public void createNewProjectFolder() { try {/*w w w . ja va 2 s . c o m*/ JFrame frame = new JFrame(); JFileChooser chooser = new JFileChooser(); chooser.setCurrentDirectory(openFolder); chooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); //chooser.setCurrentDirectory(new java.io.File(System.getProperty("user.home"))); chooser.setDialogTitle("Create Folder"); chooser.setFileHidingEnabled(Boolean.TRUE); chooser.setMultiSelectionEnabled(false); chooser.setAcceptAllFileFilterUsed(false); chooser.setDialogType(JFileChooser.SAVE_DIALOG); chooser.setSelectedFile(new File("Workingfile")); frame.getContentPane().add(chooser); chooser.setApproveButtonText("Choose"); //Disable Save as ArrayList<JPanel> jpanels = new ArrayList<JPanel>(); for (Component c : chooser.getComponents()) { if (c instanceof JPanel) { jpanels.add((JPanel) c); } } jpanels.get(0).getComponent(0).setVisible(false); frame.pack(); frame.setLocationRelativeTo(null); int whatChoose = chooser.showSaveDialog(null); if (whatChoose == JFileChooser.APPROVE_OPTION) { File selFile = chooser.getSelectedFile(); File currDir = chooser.getCurrentDirectory(); Path parentDir = Paths.get(chooser.getCurrentDirectory().getParent()); String parentDirName = parentDir.getFileName().toString(); logger.debug("Chooser SelectedFile: " + selFile.toString()); logger.debug("getCurrentDirectory(): " + currDir.toString()); logger.debug("Chooser parentdir: " + parentDir); logger.debug("Parentdirname: " + parentDirName); if (selFile.getName().equals(parentDirName)) { wDir = currDir; } else { wDir = chooser.getSelectedFile(); } logger.debug("WDIR is: " + wDir.toString()); wDirText.setText(wDir.toString()); enableUIElements(true); } } catch (Exception ex) { JOptionPane.showMessageDialog(null, "Falsche Eingabe"); logger.debug("Exeption: " + ex.toString()); } }
From source file:divconq.tool.release.Main.java
@Override public void run(Scanner scan, ApiSession api) { Path relpath = null;// ww w . j a v a2s .c o m Path gitpath = null; Path wikigitpath = null; XElement fldset = Hub.instance.getConfig().selectFirst("CommandLine/Settings"); if (fldset != null) { relpath = Paths.get(fldset.getAttribute("ReleasePath")); gitpath = Paths.get(fldset.getAttribute("GitPath")); wikigitpath = Paths.get(fldset.getAttribute("WikiGitPath")); } boolean running = true; while (running) { try { System.out.println(); System.out.println("-----------------------------------------------"); System.out.println(" Release Builder Menu"); System.out.println("-----------------------------------------------"); System.out.println("0) Exit"); if (relpath != null) System.out.println("1) Build release package from Settings File"); System.out.println("2) Build custom release package [under construction]"); System.out.println("4) Pack the .jar files"); if (gitpath != null) System.out.println("5) Copy Source to GitHub folder"); System.out.println("6) Update AWWW"); String opt = scan.nextLine(); Long mopt = StringUtil.parseInt(opt); if (mopt == null) continue; switch (mopt.intValue()) { case 0: running = false; break; case 1: { ReleasesHelper releases = new ReleasesHelper(); if (!releases.init(relpath)) break; System.out.println("Select a release to build"); System.out.println("0) None"); List<String> rnames = releases.names(); for (int i = 0; i < rnames.size(); i++) System.out.println((i + 1) + ") " + rnames.get(i)); System.out.println("Option #: "); opt = scan.nextLine(); mopt = StringUtil.parseInt(opt); if ((mopt == null) || (mopt == 0)) break; XElement relchoice = releases.get(mopt.intValue() - 1); if (relchoice == null) { System.out.println("Invalid option"); break; } PackagesHelper availpackages = new PackagesHelper(); availpackages.init(); InstallHelper inst = new InstallHelper(); if (!inst.init(availpackages, relchoice)) break; XElement prindesc = availpackages.get(inst.prinpackage); XElement prininst = prindesc.find("Install"); if (prininst == null) { System.out.println("Principle package: " + inst.prinpackagenm + " cannot be released directly, it must be part of another package."); break; } String relvers = prindesc.getAttribute("Version"); System.out.println("Building release version " + relvers); if (prindesc.hasAttribute("LastVersion")) System.out.println("Previous release version " + prindesc.getAttribute("LastVersion")); String rname = relchoice.getAttribute("Name"); Path destpath = relpath.resolve(rname + "/" + rname + "-" + relvers + "-bin.zip"); if (Files.exists(destpath)) { System.out.println("Version " + relvers + " already exists, overwrite? (y/n): "); if (!scan.nextLine().toLowerCase().startsWith("y")) break; Files.delete(destpath); } System.out.println("Preparing zip files"); AtomicBoolean errored = new AtomicBoolean(); Path tempfolder = FileUtil.allocateTempFolder2(); ListStruct ignorepaths = new ListStruct(); Set<String> nolongerdepends = new HashSet<>(); Set<String> dependson = new HashSet<>(); // put all the release files into a temp folder inst.instpkgs.forEach(pname -> { availpackages.get(pname).selectAll("DependsOn").stream() .filter(doel -> !doel.hasAttribute("Option") || inst.relopts.contains(doel.getAttribute("Option"))) .forEach(doel -> { // copy all libraries we rely on doel.selectAll("Library").forEach(libel -> { dependson.add(libel.getAttribute("File")); Path src = Paths.get("./lib/" + libel.getAttribute("File")); Path dest = tempfolder.resolve("lib/" + libel.getAttribute("File")); try { Files.createDirectories(dest.getParent()); if (Files.notExists(dest)) Files.copy(src, dest, StandardCopyOption.COPY_ATTRIBUTES); } catch (Exception x) { errored.set(true); System.out.println("Unable to copy file: " + src); } }); // copy all files we rely on doel.selectAll("File").forEach(libel -> { Path src = Paths.get("./" + libel.getAttribute("Path")); Path dest = tempfolder.resolve(libel.getAttribute("Path")); try { Files.createDirectories(dest.getParent()); if (Files.notExists(dest)) Files.copy(src, dest, StandardCopyOption.COPY_ATTRIBUTES); } catch (Exception x) { errored.set(true); System.out.println("Unable to copy file: " + src); } }); // copy all folders we rely on doel.selectAll("Folder").forEach(libel -> { Path src = Paths.get("./" + libel.getAttribute("Path")); Path dest = tempfolder.resolve(libel.getAttribute("Path")); try { Files.createDirectories(dest.getParent()); } catch (Exception x) { errored.set(true); System.out.println("Unable to copy file: " + src); } OperationResult cres = FileUtil.copyFileTree(src, dest); if (cres.hasErrors()) errored.set(true); }); }); availpackages.get(pname).selectAll("IgnorePaths/Ignore") .forEach(doel -> ignorepaths.addItem(doel.getAttribute("Path"))); // NoLongerDependsOn functionally currently only applies to libraries availpackages.get(pname).selectAll("NoLongerDependsOn/Library") .forEach(doel -> nolongerdepends.add(doel.getAttribute("File"))); // copy the released packages folders Path src = Paths.get("./packages/" + pname); Path dest = tempfolder.resolve("packages/" + pname); try { Files.createDirectories(dest.getParent()); } catch (Exception x) { errored.set(true); System.out.println("Unable to copy file: " + src); } // we may wish to enhance filter to allow .JAR sometimes, but this is meant to prevent copying of packages/pname/lib/abc.lib.jar files OperationResult cres = FileUtil.copyFileTree(src, dest, path -> !path.toString().endsWith(".jar")); if (cres.hasErrors()) errored.set(true); // copy the released packages libraries Path libsrc = Paths.get("./packages/" + pname + "/lib"); Path libdest = tempfolder.resolve("lib"); if (Files.exists(libsrc)) { cres = FileUtil.copyFileTree(libsrc, libdest); if (cres.hasErrors()) errored.set(true); } }); if (errored.get()) { System.out.println("Error with assembling package"); break; } // copy the principle config Path csrc = Paths.get("./packages/" + inst.prinpackage + "/config"); Path cdest = tempfolder.resolve("config/" + inst.prinpackagenm); if (Files.exists(csrc)) { Files.createDirectories(cdest); OperationResult cres = FileUtil.copyFileTree(csrc, cdest); if (cres.hasErrors()) { System.out.println("Error with prepping config"); break; } } boolean configpassed = true; // copy packages with config = true for (XElement pkg : relchoice.selectAll("Package")) { if (!"true".equals(pkg.getAttribute("Config"))) break; String pname = pkg.getAttribute("Name"); int pspos = pname.lastIndexOf('/'); String pnm = (pspos != -1) ? pname.substring(pspos + 1) : pname; csrc = Paths.get("./packages/" + pname + "/config"); cdest = tempfolder.resolve("config/" + pnm); if (Files.exists(csrc)) { Files.createDirectories(cdest); OperationResult cres = FileUtil.copyFileTree(csrc, cdest); if (cres.hasErrors()) { System.out.println("Error with prepping extra config"); configpassed = false; break; } } } if (!configpassed) break; // also copy installer config if being used if (inst.includeinstaller) { csrc = Paths.get("./packages/dc/dcInstall/config"); cdest = tempfolder.resolve("config/dcInstall"); if (Files.exists(csrc)) { Files.createDirectories(cdest); OperationResult cres = FileUtil.copyFileTree(csrc, cdest); if (cres.hasErrors()) { System.out.println("Error with prepping install config"); break; } } } // write out the deployed file RecordStruct deployed = new RecordStruct(); deployed.setField("Version", relvers); deployed.setField("PackageFolder", relpath.resolve(rname)); deployed.setField("PackagePrefix", rname); OperationResult d1res = IOUtil.saveEntireFile(tempfolder.resolve("config/deployed.json"), deployed.toPrettyString()); if (d1res.hasErrors()) { System.out.println("Error with prepping deployed"); break; } RecordStruct deployment = new RecordStruct(); deployment.setField("Version", relvers); if (prindesc.hasAttribute("LastVersion")) deployment.setField("DependsOn", prindesc.getAttribute("LastVersion")); deployment.setField("UpdateMessage", "This update is complete, you may accept this update as runnable."); nolongerdepends.removeAll(dependson); ListStruct deletefiles = new ListStruct(); nolongerdepends.forEach(fname -> deletefiles.addItem("lib/" + fname)); deployment.setField("DeleteFiles", deletefiles); deployment.setField("IgnorePaths", ignorepaths); d1res = IOUtil.saveEntireFile(tempfolder.resolve("deployment.json"), deployment.toPrettyString()); if (d1res.hasErrors()) { System.out.println("Error with prepping deployment"); break; } // write env file d1res = IOUtil.saveEntireFile(tempfolder.resolve("env.bat"), "set mem=" + relchoice.getAttribute("Memory", "2048") + "\r\n" + "SET project=" + inst.prinpackagenm + "\r\n" + "SET service=" + relchoice.getAttribute("Service", inst.prinpackagenm) + "\r\n" + "SET servicename=" + relchoice.getAttribute("ServiceName", inst.prinpackagenm + " Service") + "\r\n"); if (d1res.hasErrors()) { System.out.println("Error with prepping env"); break; } System.out.println("Packing Release file."); Path relbin = relpath.resolve(rname + "/" + rname + "-" + relvers + "-bin.zip"); if (Files.notExists(relbin.getParent())) Files.createDirectories(relbin.getParent()); ZipArchiveOutputStream zipout = new ZipArchiveOutputStream(relbin.toFile()); try { Files.walkFileTree(tempfolder, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { ZipArchiveEntry entry = new ZipArchiveEntry( tempfolder.relativize(file).toString()); entry.setSize(Files.size(file)); zipout.putArchiveEntry(entry); zipout.write(Files.readAllBytes(file)); zipout.closeArchiveEntry(); return FileVisitResult.CONTINUE; } }); } catch (IOException x) { System.out.println("Error building zip: " + x); } zipout.close(); System.out.println("Release file written"); FileUtil.deleteDirectory(tempfolder); break; } // end case 1 case 3: { System.out.println("Note these utilities are only good from the main console,"); System.out.println("if you are using a remote connection then the encryption will"); System.out.println("not work as expected. [we do not have access the master keys]"); System.out.println(); Foreground.utilityMenu(scan); break; } case 4: { System.out.println("Packing jar library files."); String[] packlist = new String[] { "divconq.core", "divconq.interchange", "divconq.web", "divconq.tasks", "divconq.tasks.api", "ncc.uploader.api", "ncc.uploader.core", "ncc.workflow", "sd.core" }; String[] packnames = new String[] { "dcCore", "dcInterchange", "dcWeb", "dcTasks", "dcTasksApi", "nccUploaderApi", "nccUploader", "nccWorkflow", "sd/sdBackend" }; for (int i = 0; i < packlist.length; i++) { String lib = packlist[i]; String pname = packnames[i]; Path relbin = Paths.get("./ext/" + lib + ".jar"); Path srcbin = Paths.get("./" + lib + "/bin"); Path packbin = Paths.get("./packages/" + pname + "/lib/" + lib + ".jar"); if (Files.notExists(relbin.getParent())) Files.createDirectories(relbin.getParent()); Files.deleteIfExists(relbin); ZipArchiveOutputStream zipout = new ZipArchiveOutputStream(relbin.toFile()); try { Files.walkFileTree(srcbin, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { ZipArchiveEntry entry = new ZipArchiveEntry( srcbin.relativize(file).toString()); entry.setSize(Files.size(file)); zipout.putArchiveEntry(entry); zipout.write(Files.readAllBytes(file)); zipout.closeArchiveEntry(); return FileVisitResult.CONTINUE; } }); } catch (IOException x) { System.out.println("Error building zip: " + x); } zipout.close(); Files.copy(relbin, packbin, StandardCopyOption.REPLACE_EXISTING); } System.out.println("Done"); break; } case 5: { System.out.println("Copying Source Files"); System.out.println("Cleaning folders"); OperationResult or = FileUtil.deleteDirectory(gitpath.resolve("divconq.core/src/main/java")); if (or.hasErrors()) { System.out.println("Error deleting files"); break; } or = FileUtil.deleteDirectory(gitpath.resolve("divconq.core/src/main/resources")); if (or.hasErrors()) { System.out.println("Error deleting files"); break; } or = FileUtil.deleteDirectory(gitpath.resolve("divconq.interchange/src/main/java")); if (or.hasErrors()) { System.out.println("Error deleting files"); break; } or = FileUtil.deleteDirectory(gitpath.resolve("divconq.tasks/src/main/java")); if (or.hasErrors()) { System.out.println("Error deleting files"); break; } or = FileUtil.deleteDirectory(gitpath.resolve("divconq.tasks.api/src/main/java")); if (or.hasErrors()) { System.out.println("Error deleting files"); break; } or = FileUtil.deleteDirectory(gitpath.resolve("divconq.web/src/main/java")); if (or.hasErrors()) { System.out.println("Error deleting files"); break; } or = FileUtil.deleteDirectory(gitpath.resolve("packages")); if (or.hasErrors()) { System.out.println("Error deleting files"); break; } or = FileUtil.deleteDirectoryContent(wikigitpath, ".git"); if (or.hasErrors()) { System.out.println("Error deleting wiki files"); break; } System.out.println("Copying folders"); System.out.println("Copy tree ./divconq.core/src"); or = FileUtil.copyFileTree(Paths.get("./divconq.core/src/divconq"), gitpath.resolve("divconq.core/src/main/java/divconq"), new Predicate<Path>() { @Override public boolean test(Path file) { return file.getFileName().toString().endsWith(".java"); } }); if (or.hasErrors()) { System.out.println("Error copying files"); break; } or = FileUtil.copyFileTree(Paths.get("./divconq.core/src/org"), gitpath.resolve("divconq.core/src/main/java/org"), new Predicate<Path>() { @Override public boolean test(Path file) { return file.getFileName().toString().endsWith(".java"); } }); if (or.hasErrors()) { System.out.println("Error copying files"); break; } or = FileUtil.copyFileTree(Paths.get("./divconq.core/src/localize"), gitpath.resolve("divconq.core/src/main/resources/localize"), new Predicate<Path>() { @Override public boolean test(Path file) { return file.getFileName().toString().endsWith(".xml"); } }); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./divconq.interchange/src"); or = FileUtil.copyFileTree(Paths.get("./divconq.interchange/src"), gitpath.resolve("divconq.interchange/src/main/java")); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./divconq.tasks/src"); or = FileUtil.copyFileTree(Paths.get("./divconq.tasks/src"), gitpath.resolve("divconq.tasks/src/main/java")); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./divconq.tasks.api/src"); or = FileUtil.copyFileTree(Paths.get("./divconq.tasks.api/src"), gitpath.resolve("divconq.tasks.api/src/main/java")); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./divconq.web/src"); or = FileUtil.copyFileTree(Paths.get("./divconq.web/src"), gitpath.resolve("divconq.web/src/main/java")); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./packages/dcCore"); or = FileUtil.copyFileTree(Paths.get("./packages/dcCore"), gitpath.resolve("packages/dcCore")); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./packages/dcCorePublic"); or = FileUtil.copyFileTree(Paths.get("./packages/dcCorePublic"), gitpath.resolve("packages/dcCorePublic")); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./packages/dcInterchange"); or = FileUtil.copyFileTree(Paths.get("./packages/dcInterchange"), gitpath.resolve("packages/dcInterchange")); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./packages/dcTasks"); or = FileUtil.copyFileTree(Paths.get("./packages/dcTasks"), gitpath.resolve("packages/dcTasks")); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./packages/dcTasksApi"); or = FileUtil.copyFileTree(Paths.get("./packages/dcTasksApi"), gitpath.resolve("packages/dcTasksApi")); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./packages/dcTasksWeb"); or = FileUtil.copyFileTree(Paths.get("./packages/dcTasksWeb"), gitpath.resolve("packages/dcTasksWeb")); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./packages/dcTest"); or = FileUtil.copyFileTree(Paths.get("./packages/dcTest"), gitpath.resolve("packages/dcTest")); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./packages/dcWeb"); or = FileUtil.copyFileTree(Paths.get("./packages/dcWeb"), gitpath.resolve("packages/dcWeb")); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copy tree ./divconq.wiki/public"); or = FileUtil.copyFileTree(Paths.get("./divconq.wiki/public"), wikigitpath); if (or.hasErrors()) { System.out.println("Error copying files"); break; } System.out.println("Copying files"); Files.copy(Paths.get("./README.md"), gitpath.resolve("README.md"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); Files.copy(Paths.get("./RELEASE_NOTES.md"), gitpath.resolve("RELEASE_NOTES.md"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); Files.copy(Paths.get("./NOTICE.txt"), gitpath.resolve("NOTICE.txt"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); Files.copy(Paths.get("./LICENSE.txt"), gitpath.resolve("LICENSE.txt"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); System.out.println("Done"); break; } case 6: { System.out.println("Are you sure you want to update AWWW Server? (y/n): "); if (!scan.nextLine().toLowerCase().startsWith("y")) break; ReleasesHelper releases = new ReleasesHelper(); if (!releases.init(relpath)) break; XElement relchoice = releases.get("AWWWServer"); if (relchoice == null) { System.out.println("Invalid option"); break; } PackagesHelper availpackages = new PackagesHelper(); availpackages.init(); InstallHelper inst = new InstallHelper(); if (!inst.init(availpackages, relchoice)) break; ServerHelper ssh = new ServerHelper(); if (!ssh.init(relchoice.find("SSH"))) break; ChannelSftp sftp = null; try { Channel channel = ssh.session().openChannel("sftp"); channel.connect(); sftp = (ChannelSftp) channel; // go to routines folder sftp.cd("/usr/local/bin/dc/AWWWServer"); FileRepositoryBuilder builder = new FileRepositoryBuilder(); Repository repository = builder.setGitDir(new File(".git")).findGitDir() // scan up the file system tree .build(); String lastsync = releases.getData("AWWWServer").getFieldAsString("LastCommitSync"); RevWalk rw = new RevWalk(repository); ObjectId head1 = repository.resolve(Constants.HEAD); RevCommit commit1 = rw.parseCommit(head1); releases.getData("AWWWServer").setField("LastCommitSync", head1.name()); ObjectId rev2 = repository.resolve(lastsync); RevCommit parent = rw.parseCommit(rev2); //RevCommit parent2 = rw.parseCommit(parent.getParent(0).getId()); DiffFormatter df = new DiffFormatter(DisabledOutputStream.INSTANCE); df.setRepository(repository); df.setDiffComparator(RawTextComparator.DEFAULT); df.setDetectRenames(true); // list oldest first or change types are all wrong!! List<DiffEntry> diffs = df.scan(parent.getTree(), commit1.getTree()); for (DiffEntry diff : diffs) { String gnpath = diff.getNewPath(); String gopath = diff.getOldPath(); Path npath = Paths.get("./" + gnpath); Path opath = Paths.get("./" + gopath); if (diff.getChangeType() == ChangeType.DELETE) { if (inst.containsPathExtended(opath)) { System.out.println("- " + diff.getChangeType().name() + " - " + opath); try { sftp.rm(opath.toString()); System.out.println("deleted!!"); } catch (SftpException x) { System.out.println( " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); System.out.println("Sftp Error: " + x); System.out.println( " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); } } else { System.out.println("/ " + diff.getChangeType().name() + " - " + gopath + " !!!!!!!!!!!!!!!!!!!!!!!!!"); } } else if ((diff.getChangeType() == ChangeType.ADD) || (diff.getChangeType() == ChangeType.MODIFY) || (diff.getChangeType() == ChangeType.COPY)) { if (inst.containsPathExtended(npath)) { System.out.println("+ " + diff.getChangeType().name() + " - " + npath); try { ssh.makeDirSftp(sftp, npath.getParent()); sftp.put(npath.toString(), npath.toString(), ChannelSftp.OVERWRITE); sftp.chmod(npath.endsWith(".sh") ? 484 : 420, npath.toString()); // 644 octal = 420 dec, 744 octal = 484 dec System.out.println("uploaded!!"); } catch (SftpException x) { System.out.println( " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); System.out.println("Sftp Error: " + x); System.out.println( " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); } } else { System.out.println("> " + diff.getChangeType().name() + " - " + gnpath + " !!!!!!!!!!!!!!!!!!!!!!!!!"); } } else if (diff.getChangeType() == ChangeType.RENAME) { // remove the old if (inst.containsPathExtended(opath)) { System.out.println("- " + diff.getChangeType().name() + " - " + opath); try { sftp.rm(opath.toString()); System.out.println("deleted!!"); } catch (SftpException x) { System.out.println( " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); System.out.println("Sftp Error: " + x); System.out.println( " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); } } else { System.out.println("/ " + diff.getChangeType().name() + " - " + gopath + " !!!!!!!!!!!!!!!!!!!!!!!!!"); } // add the new path if (inst.containsPathExtended(npath)) { System.out.println("+ " + diff.getChangeType().name() + " - " + npath); try { ssh.makeDirSftp(sftp, npath.getParent()); sftp.put(npath.toString(), npath.toString(), ChannelSftp.OVERWRITE); sftp.chmod(npath.endsWith(".sh") ? 484 : 420, npath.toString()); // 644 octal = 420 dec, 744 octal = 484 dec System.out.println("uploaded!!"); } catch (SftpException x) { System.out.println( " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); System.out.println("Sftp Error: " + x); System.out.println( " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); } } else { System.out.println("> " + diff.getChangeType().name() + " - " + gnpath + " !!!!!!!!!!!!!!!!!!!!!!!!!"); } } else { System.out.println("??????????????????????????????????????????????????????????"); System.out.println(": " + diff.getChangeType().name() + " - " + gnpath + " ?????????????????????????"); System.out.println("??????????????????????????????????????????????????????????"); } } rw.dispose(); repository.close(); releases.saveData(); } catch (JSchException x) { System.out.println("Sftp Error: " + x); } finally { if (sftp.isConnected()) sftp.exit(); ssh.close(); } break; } case 7: { Path sfolder = Paths.get("/Work/Projects/awww-current/dairy-graze/poly"); Path dfolder = Paths.get("/Work/Projects/awww-current/dairy-graze/poly-js"); Files.list(sfolder).forEach(file -> { String fname = file.getFileName().toString(); if (!fname.endsWith(".xml")) return; FuncResult<XElement> lres = XmlReader.loadFile(file, false); if (lres.isEmptyResult()) { System.out.println("Unable to parse: " + file); return; } String zc = fname.substring(5, 8); String code = "zipsData['" + zc + "'] = "; XElement root = lres.getResult(); /* <polyline1 lng="-90.620897" lat="45.377447"/> <polyline1 lng="-90.619327" lat="45.3805"/> [-71.196845,41.67757],[-71.120168,41.496831],[-71.317338,41.474923],[-71.196845,41.67757] */ ListStruct center = new ListStruct(); ListStruct cords = new ListStruct(); ListStruct currentPoly = null; //String currentName = null; for (XElement child : root.selectAll("*")) { String cname = child.getName(); if (cname.startsWith("marker")) { // not always accurate if (center.isEmpty()) center.addItem(Struct.objectToDecimal(child.getAttribute("lng")), Struct.objectToDecimal(child.getAttribute("lat"))); currentPoly = new ListStruct(); cords.addItem(new ListStruct(currentPoly)); continue; } /* if (cname.startsWith("info")) { System.out.println("areas: " + child.getAttribute("areas")); continue; } */ if (!cname.startsWith("polyline")) continue; if (currentPoly == null) { //if (!cname.equals(currentName)) { //if (currentName == null) { // currentName = cname; // System.out.println("new poly: " + cname); currentPoly = new ListStruct(); cords.addItem(new ListStruct(currentPoly)); } currentPoly.addItem(new ListStruct(Struct.objectToDecimal(child.getAttribute("lng")), Struct.objectToDecimal(child.getAttribute("lat")))); } RecordStruct feat = new RecordStruct().withField("type", "Feature") .withField("id", "zip" + zc) .withField("properties", new RecordStruct().withField("name", "Prefix " + zc).withField("alias", zc)) .withField("geometry", new RecordStruct().withField("type", "MultiPolygon") .withField("coordinates", cords)); RecordStruct entry = new RecordStruct().withField("code", zc).withField("geo", feat) .withField("center", center); IOUtil.saveEntireFile2(dfolder.resolve("us-zips-" + zc + ".js"), code + entry.toPrettyString() + ";"); }); break; } } } catch (Exception x) { System.out.println("CLI error: " + x); } } }
From source file:edu.harvard.iq.dataverse.ingest.IngestServiceBean.java
public List<DataFile> createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType) throws IOException { List<DataFile> datafiles = new ArrayList<DataFile>(); String warningMessage = null; // save the file, in the temporary location for now: Path tempFile = null; if (getFilesTempDirectory() != null) { tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload"); // "temporary" location is the key here; this is why we are not using // the DataStore framework for this - the assumption is that // temp files will always be stored on the local filesystem. // -- L.A. Jul. 2014 logger.fine("Will attempt to save the file as: " + tempFile.toString()); Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING); } else {//from w w w . j ava 2 s.c om throw new IOException("Temp directory is not configured."); } logger.fine("mime type supplied: " + suppliedContentType); // Let's try our own utilities (Jhove, etc.) to determine the file type // of the uploaded file. (We may already have a mime type supplied for this // file - maybe the type that the browser recognized on upload; or, if // it's a harvest, maybe the remote server has already given us the type // for this file... with our own type utility we may or may not do better // than the type supplied: // -- L.A. String recognizedType = null; String finalType = null; try { recognizedType = FileUtil.determineFileType(tempFile.toFile(), fileName); logger.fine("File utility recognized the file as " + recognizedType); if (recognizedType != null && !recognizedType.equals("")) { // is it any better than the type that was supplied to us, // if any? // This is not as trivial a task as one might expect... // We may need a list of "good" mime types, that should always // be chosen over other choices available. Maybe it should // even be a weighed list... as in, "application/foo" should // be chosen over "application/foo-with-bells-and-whistles". // For now the logic will be as follows: // // 1. If the contentType supplied (by the browser, most likely) // is some form of "unknown", we always discard it in favor of // whatever our own utilities have determined; // 2. We should NEVER trust the browser when it comes to the // following "ingestable" types: Stata, SPSS, R; // 2a. We are willing to TRUST the browser when it comes to // the CSV and XSLX ingestable types. // 3. We should ALWAYS trust our utilities when it comes to // ingestable types. if (suppliedContentType == null || suppliedContentType.equals("") || suppliedContentType.equalsIgnoreCase(MIME_TYPE_UNDETERMINED_DEFAULT) || suppliedContentType.equalsIgnoreCase(MIME_TYPE_UNDETERMINED_BINARY) || (ingestableAsTabular(suppliedContentType) && !suppliedContentType.equalsIgnoreCase(MIME_TYPE_CSV) && !suppliedContentType.equalsIgnoreCase(MIME_TYPE_CSV_ALT) && !suppliedContentType.equalsIgnoreCase(MIME_TYPE_XLSX)) || ingestableAsTabular(recognizedType) || recognizedType.equals("application/fits-gzipped") || recognizedType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE) || recognizedType.equals(MIME_TYPE_ZIP)) { finalType = recognizedType; } } } catch (Exception ex) { logger.warning("Failed to run the file utility mime type check on file " + fileName); } if (finalType == null) { finalType = (suppliedContentType == null || suppliedContentType.equals("")) ? MIME_TYPE_UNDETERMINED_DEFAULT : suppliedContentType; } // A few special cases: // if this is a gzipped FITS file, we'll uncompress it, and ingest it as // a regular FITS file: if (finalType.equals("application/fits-gzipped")) { InputStream uncompressedIn = null; String finalFileName = fileName; // if the file name had the ".gz" extension, remove it, // since we are going to uncompress it: if (fileName != null && fileName.matches(".*\\.gz$")) { finalFileName = fileName.replaceAll("\\.gz$", ""); } DataFile datafile = null; try { uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile())); datafile = createSingleDataFile(version, uncompressedIn, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT); } catch (IOException ioex) { datafile = null; } finally { if (uncompressedIn != null) { try { uncompressedIn.close(); } catch (IOException e) { } } } // If we were able to produce an uncompressed file, we'll use it // to create and return a final DataFile; if not, we're not going // to do anything - and then a new DataFile will be created further // down, from the original, uncompressed file. if (datafile != null) { // remove the compressed temp file: try { tempFile.toFile().delete(); } catch (SecurityException ex) { // (this is very non-fatal) logger.warning("Failed to delete temporary file " + tempFile.toString()); } datafiles.add(datafile); return datafiles; } // If it's a ZIP file, we are going to unpack it and create multiple // DataFile objects from its contents: } else if (finalType.equals("application/zip")) { ZipInputStream unZippedIn = null; ZipEntry zipEntry = null; int fileNumberLimit = systemConfig.getZipUploadFilesLimit(); try { Charset charset = null; /* TODO: (?) We may want to investigate somehow letting the user specify the charset for the filenames in the zip file... - otherwise, ZipInputStream bails out if it encounteres a file name that's not valid in the current charest (i.e., UTF-8, in our case). It would be a bit trickier than what we're doing for SPSS tabular ingests - with the lang. encoding pulldown menu - because this encoding needs to be specified *before* we upload and attempt to unzip the file. -- L.A. 4.0 beta12 logger.info("default charset is "+Charset.defaultCharset().name()); if (Charset.isSupported("US-ASCII")) { logger.info("charset US-ASCII is supported."); charset = Charset.forName("US-ASCII"); if (charset != null) { logger.info("was able to obtain charset for US-ASCII"); } } */ if (charset != null) { unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()), charset); } else { unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile())); } while (true) { try { zipEntry = unZippedIn.getNextEntry(); } catch (IllegalArgumentException iaex) { // Note: // ZipInputStream documentation doesn't even mention that // getNextEntry() throws an IllegalArgumentException! // but that's what happens if the file name of the next // entry is not valid in the current CharSet. // -- L.A. warningMessage = "Failed to unpack Zip file. (Unknown Character Set used in a file name?) Saving the file as is."; logger.warning(warningMessage); throw new IOException(); } if (zipEntry == null) { break; } // Note that some zip entries may be directories - we // simply skip them: if (!zipEntry.isDirectory()) { if (datafiles.size() > fileNumberLimit) { logger.warning("Zip upload - too many files."); warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit + "); please upload a zip archive with fewer files, if you want them to be ingested " + "as individual DataFiles."; throw new IOException(); } String fileEntryName = zipEntry.getName(); logger.fine("ZipEntry, file: " + fileEntryName); if (fileEntryName != null && !fileEntryName.equals("")) { String shortName = fileEntryName.replaceFirst("^.*[\\/]", ""); // Check if it's a "fake" file - a zip archive entry // created for a MacOS X filesystem element: (these // start with "._") if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) { // OK, this seems like an OK file entry - we'll try // to read it and create a DataFile with it: DataFile datafile = createSingleDataFile(version, unZippedIn, shortName, MIME_TYPE_UNDETERMINED_DEFAULT, false); if (!fileEntryName.equals(shortName)) { String categoryName = fileEntryName.replaceFirst("[\\/][^\\/]*$", ""); if (!"".equals(categoryName)) { logger.fine("setting category to " + categoryName); //datafile.getFileMetadata().setCategory(categoryName.replaceAll("[\\/]", "-")); datafile.getFileMetadata() .addCategoryByName(categoryName.replaceAll("[\\/]", "-")); } } if (datafile != null) { // We have created this datafile with the mime type "unknown"; // Now that we have it saved in a temporary location, // let's try and determine its real type: String tempFileName = getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(); try { recognizedType = FileUtil.determineFileType(new File(tempFileName), shortName); logger.fine("File utility recognized unzipped file as " + recognizedType); if (recognizedType != null && !recognizedType.equals("")) { datafile.setContentType(recognizedType); } } catch (Exception ex) { logger.warning("Failed to run the file utility mime type check on file " + fileName); } datafiles.add(datafile); } } } } unZippedIn.closeEntry(); } } catch (IOException ioex) { // just clear the datafiles list and let // ingest default to creating a single DataFile out // of the unzipped file. logger.warning("Unzipping failed; rolling back to saving the file as is."); if (warningMessage == null) { warningMessage = "Failed to unzip the file. Saving the file as is."; } datafiles.clear(); } finally { if (unZippedIn != null) { try { unZippedIn.close(); } catch (Exception zEx) { } } } if (datafiles.size() > 0) { // link the data files to the dataset/version: Iterator<DataFile> itf = datafiles.iterator(); while (itf.hasNext()) { DataFile datafile = itf.next(); datafile.setOwner(version.getDataset()); if (version.getFileMetadatas() == null) { version.setFileMetadatas(new ArrayList()); } version.getFileMetadatas().add(datafile.getFileMetadata()); datafile.getFileMetadata().setDatasetVersion(version); /* TODO!! // re-implement this in some way that does not use the // deprecated .getCategory() on FileMeatadata: if (datafile.getFileMetadata().getCategory() != null) { datafile.getFileMetadata().addCategoryByName(datafile.getFileMetadata().getCategory()); datafile.getFileMetadata().setCategory(null); -- done? see above? } */ version.getDataset().getFiles().add(datafile); } // remove the uploaded zip file: try { Files.delete(tempFile); } catch (IOException ioex) { // do nothing - it's just a temp file. logger.warning("Could not remove temp file " + tempFile.getFileName().toString()); } // and return: return datafiles; } } else if (finalType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)) { // Shape files may have to be split into multiple files, // one zip archive per each complete set of shape files: //File rezipFolder = new File(this.getFilesTempDirectory()); File rezipFolder = this.getShapefileUnzipTempDirectory(); IngestServiceShapefileHelper shpIngestHelper; shpIngestHelper = new IngestServiceShapefileHelper(tempFile.toFile(), rezipFolder); boolean didProcessWork = shpIngestHelper.processFile(); if (!(didProcessWork)) { logger.severe("Processing of zipped shapefile failed."); return null; } for (File finalFile : shpIngestHelper.getFinalRezippedFiles()) { FileInputStream finalFileInputStream = new FileInputStream(finalFile); finalType = this.getContentType(finalFile); if (finalType == null) { logger.warning("Content type is null; but should default to 'MIME_TYPE_UNDETERMINED_DEFAULT'"); continue; } DataFile new_datafile = createSingleDataFile(version, finalFileInputStream, finalFile.getName(), finalType); if (new_datafile != null) { datafiles.add(new_datafile); } else { logger.severe("Could not add part of rezipped shapefile. new_datafile was null: " + finalFile.getName()); } finalFileInputStream.close(); } // Delete the temp directory used for unzipping /* logger.fine("Delete temp shapefile unzip directory: " + rezipFolder.getAbsolutePath()); FileUtils.deleteDirectory(rezipFolder); // Delete rezipped files for (File finalFile : shpIngestHelper.getFinalRezippedFiles()){ if (finalFile.isFile()){ finalFile.delete(); } } */ if (datafiles.size() > 0) { return datafiles; } else { logger.severe("No files added from directory of rezipped shapefiles"); } return null; } // Finally, if none of the special cases above were applicable (or // if we were unable to unpack an uploaded file, etc.), we'll just // create and return a single DataFile: // (Note that we are passing null for the InputStream; that's because // we already have the file saved; we'll just need to rename it, below) DataFile datafile = createSingleDataFile(version, null, fileName, finalType); if (datafile != null) { fileService.generateStorageIdentifier(datafile); if (!tempFile.toFile() .renameTo(new File(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier()))) { return null; } // MD5: MD5Checksum md5Checksum = new MD5Checksum(); try { datafile.setmd5( md5Checksum.CalculateMD5(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier())); } catch (Exception md5ex) { logger.warning("Could not calculate MD5 signature for new file " + fileName); } if (warningMessage != null) { createIngestFailureReport(datafile, warningMessage); datafile.SetIngestProblem(); } datafiles.add(datafile); return datafiles; } return null; }