List of usage examples for org.apache.commons.io FileUtils copyURLToFile
public static void copyURLToFile(URL source, File destination) throws IOException
source
to a file destination
. From source file:net.dries007.coremod.Module.java
/** * Check to see if all module files exist. Checks hash, downloads new one if necessary. * * @throws IOException/*from ww w .j av a 2 s .c o m*/ */ public void checkJarFiles() throws IOException { for (final ModuleFile mFile : this.files) { String sum; if (mFile.file.exists() && mFile.hash != null && (sum = Coremod.getChecksum(mFile.file)) != null) if (!sum.equals(mFile.hash)) { System.out.println("[" + Data.NAME + "] Module " + this.name + "'s file " + mFile.file.getName() + " has wrong hash. Removing."); mFile.file.delete(); } if (!mFile.file.exists()) { System.out.println("[" + Data.NAME + "] Module " + this.name + "'s file " + mFile.file.getName() + " is downloading."); FileUtils.copyURLToFile(mFile.url, mFile.file); } } }
From source file:eu.sisob.uma.extractors.adhoc.websearchers.WebSearchersExtractor.java
/** * * @param nextLine// w w w. j a v a 2s . com * @param idStaffIdentifier * @param idName * @param idFirstName * @param idLastName * @param idInitials * @param idSubject * @param idInstitutionName * @param idWebAddress * @param expression * @param params * @return */ @Override protected String get_result(String[] nextLine, int idStaffIdentifier, int idName, int idFirstName, int idLastName, int idInitials, int idSubject, int idInstitutionName, int idWebAddress, String expression, Object[] params) { String keywords = " (PROFILE OR PHD OR RESEARCHER OR FACULTY OR PROFESSOR OR RESEARCH) AND "; keywords = ""; String domain = clean_site(nextLine[idWebAddress]); String subject = nextLine[idSubject]; String and_institution_name = (idInstitutionName != -1 ? " AND " + nextLine[idInstitutionName] : ""); String expression_subject = expression + " AND " + subject; String expression_site = expression + " site: " + domain; String expression_inst_name = expression + and_institution_name; String expression_inst_name_and_subject = expression + and_institution_name + " AND " + subject; String url = ""; switch (search_patterns) { case P1: url = "https://duckduckgo.com/html/?q=" + keywords + expression; break; case P2: url = "https://duckduckgo.com/html/?q=" + keywords + expression_subject; break; case P3: url = "https://duckduckgo.com/html/?q=" + keywords + expression_site; break; case P4: url = "https://duckduckgo.com/html/?q=" + keywords + expression_inst_name; break; case P5: url = "https://duckduckgo.com/html/?q=" + keywords + expression_inst_name_and_subject; break; default: url = "https://duckduckgo.com/html/?q=" + keywords + expression_subject; break; } Logger.getRootLogger().info("Go with " + url); boolean again = false; Document doc = null; do { doc = getDocumentFromPage(url, 10, 1000, 5000); if (doc != null && doc.text().contains("If this error persists, please let us know")) { try { Thread.sleep(30000); } catch (InterruptedException ex) { } again = true; } else { again = false; } } while (again); String final_result = ""; if (doc != null && doc.select("div[class*=links_main] > a").size() > 0) { /* Write resercher founded */ Elements elements = doc.select("div[class*=links_main] > a"); /* We will take the first html page and the first pdf */ HashMap<String, String> results = new HashMap<String, String>(); int max_results = 2; int i_result = 0; for (Element e : elements) { if ((e.text().startsWith("[") //&& !e.text().startsWith("[PDF]") ) || e.absUrl("href").contains("duckduckgo.com/y.js") || e.absUrl("href").contains("wikipedia.") || e.absUrl("href").contains("facebook.com") || e.absUrl("href").contains("microsoft.com") || e.absUrl("href").contains("google.com") || e.absUrl("href").contains("linkedin") || e.absUrl("href").contains("www.biography.com") || e.absUrl("href").contains("biomedexperts.com") || e.absUrl("href").contains("www.experts.scival.com") || e.absUrl("href").contains("ratemyprofessors.com") || e.absUrl("href").contains("flickr.com") || e.absUrl("href").endsWith(".txt") || e.absUrl("href").endsWith(".csv") || e.absUrl("href").endsWith(".xml") || e.absUrl("href").endsWith(".doc") || e.absUrl("href").endsWith(".docx") || e.absUrl("href").endsWith(".xls") || e.absUrl("href").endsWith(".xlxs") || e.absUrl("href").contains("www.amazon")) { max_results++; continue; } boolean add = false; String score = ""; String ext = ""; if (!results.containsKey("HTML") && !e.text().startsWith("[")) { //results.put("html", ) File temp; try { temp = File.createTempFile("temp-file-name", ".tmp"); URL fetched_url = Downloader.fetchURL(e.absUrl("href")); FileUtils.copyURLToFile(fetched_url, temp); long sizeInBytes = temp.length(); long sizeInMb = sizeInBytes / (1024 * 1024); if (sizeInMb > 100) { score = "B"; } else { String content = FileUtils.readFileToString(temp); if (content.contains(nextLine[idLastName])) { score = "A"; } else { score = "B"; } } } catch (IOException ex) { score = "B"; } ext = "HTML"; add = true; } //if(!results.containsKey("PDF") && e.text().startsWith("[PDF]")){ // score = "A"; // ext = "PDF"; // add = true; //} if (add) { String result = ""; result += "\"" + nextLine[idStaffIdentifier] + "\";"; result += "\"" + nextLine[idLastName] + "\";"; result += "\"" + nextLine[idInitials] + "\";"; if (idFirstName != -1) result += "\"" + nextLine[idFirstName] + "\";"; if (idName != -1) result += "\"" + nextLine[idName] + "\";"; result += "\"" + e.absUrl("href") + "\";"; result += "\"" + ext + "\";"; result += "\"" + "CV" + "\";"; result += "\"" + score + "\""; result += "\r\n"; results.put(ext, result); Logger.getRootLogger().info("Select " + e.absUrl("href") + " - " + e.text()); } // if(results.containsKey("PDF") && results.containsKey("HTML")){ // break; // } i_result++; if (max_results <= i_result) { break; } } // if(results.containsKey("PDF")) // final_result = results.get("PDF"); // else if (results.containsKey("HTML")) final_result = results.get("HTML"); else final_result = ""; } return final_result; }
From source file:hudson.cli.CLITest.java
License:asdf
private void grabCliJar() throws IOException { jar = tmp.newFile("jenkins-cli.jar"); FileUtils.copyURLToFile(r.jenkins.getJnlpJars("jenkins-cli.jar").getURL(), jar); }
From source file:ffx.numerics.fft.Real3DCuda.java
/** * {@inheritDoc}//from ww w . j av a2 s .c o m */ @Override public void run() { JCudaDriver.setExceptionsEnabled(true); JCudaDriver.setLogLevel(LogLevel.LOG_ERROR); JCufft.setExceptionsEnabled(true); JCufft.setLogLevel(LogLevel.LOG_ERROR); // Initialize the driver and create a context for the first device. cuInit(0); CUcontext pctx = new CUcontext(); CUdevice dev = new CUdevice(); CUdevprop prop = new CUdevprop(); cuDeviceGetProperties(prop, dev); logger.info(" CUDA " + prop.toFormattedString()); cuDeviceGet(dev, 0); cuCtxCreate(pctx, 0, dev); // Load the CUBIN file and obtain the "recipSummation" function. try { String bit = System.getProperty("sun.arch.data.model").trim(); URL source = getClass().getClassLoader() .getResource("ffx/numerics/fft/recipSummation-" + bit + ".cubin"); File cubinFile = File.createTempFile("recipSummation", "cubin"); FileUtils.copyURLToFile(source, cubinFile); module = new CUmodule(); cuModuleLoad(module, cubinFile.getCanonicalPath()); function = new CUfunction(); cuModuleGetFunction(function, module, "recipSummation"); } catch (Exception e) { String message = "Error loading the reciprocal summation kernel"; logger.log(Level.SEVERE, message, e); } // Copy the data array to the device. dataDevice = new CUdeviceptr(); cuMemAlloc(dataDevice, len * Sizeof.FLOAT); dataPtr = Pointer.to(data); cuMemcpyHtoD(dataDevice, dataPtr, len * Sizeof.FLOAT); // Copy the recip array to the device. recipDevice = new CUdeviceptr(); cuMemAlloc(recipDevice, len * Sizeof.FLOAT); recipPtr = Pointer.to(recip); cuMemcpyHtoD(recipDevice, recipPtr, len * Sizeof.FLOAT); // Create a Real to Complex CUFFT plan planR2C = new cufftHandle(); cufftPlan3d(planR2C, nX, nY, nZ, cufftType.CUFFT_R2C); cufftSetCompatibilityMode(planR2C, cufftCompatibility.CUFFT_COMPATIBILITY_FFTW_ALL); // Create a Complex to Real CUFFT plan planC2R = new cufftHandle(); cufftPlan3d(planC2R, nX, nY, nZ, cufftType.CUFFT_C2R); cufftSetCompatibilityMode(planC2R, cufftCompatibility.CUFFT_COMPATIBILITY_FFTW_ALL); dataDevicePtr = Pointer.to(dataDevice); recipDevicePtr = Pointer.to(recipDevice); int threads = 512; int nBlocks = len / threads + (len % threads == 0 ? 0 : 1); int gridSize = (int) Math.floor(Math.sqrt(nBlocks)) + 1; logger.info(format(" CUDA thread initialized with %d threads per block", threads)); logger.info(format(" Grid Size: (%d x %d x 1).", gridSize, gridSize)); assert (gridSize * gridSize * threads >= len); synchronized (this) { while (!free) { if (doConvolution) { cuMemcpyHtoD(dataDevice, dataPtr, len * Sizeof.FLOAT); int ret = cufftExecR2C(planR2C, dataDevice, dataDevice); if (ret != cufftResult.CUFFT_SUCCESS) { logger.warning("R2C Result " + cufftResult.stringFor(ret)); } // Set up the execution parameters for the kernel cuFuncSetBlockShape(function, threads, 1, 1); int offset = 0; offset = align(offset, Sizeof.POINTER); cuParamSetv(function, offset, dataDevicePtr, Sizeof.POINTER); offset += Sizeof.POINTER; offset = align(offset, Sizeof.POINTER); cuParamSetv(function, offset, recipDevicePtr, Sizeof.POINTER); offset += Sizeof.POINTER; offset = align(offset, Sizeof.INT); cuParamSeti(function, offset, len / 2); offset += Sizeof.INT; cuParamSetSize(function, offset); // Call the kernel function. cuLaunchGrid(function, gridSize, gridSize); ret = cufftExecC2R(planC2R, dataDevice, dataDevice); if (ret != cufftResult.CUFFT_SUCCESS) { logger.warning("C2R Result " + cufftResult.stringFor(ret)); } ret = cuMemcpyDtoH(dataPtr, dataDevice, len * Sizeof.FLOAT); doConvolution = false; notify(); } try { wait(); } catch (InterruptedException e) { logger.severe(e.toString()); } } cufftDestroy(planR2C); cufftDestroy(planC2R); cuMemFree(dataDevice); cuMemFree(recipDevice); dead = true; notify(); } logger.info(" CUDA Thread Done!"); }
From source file:fbpwn.plugins.core.DumpImagesTask.java
/** * Dump Album Images with Comments//from ww w. j a va2 s . c o m * @param albumPage Mobile album page containing Images with comments * @param albumIndex Album index that identify it's folder */ private void processAlbum(HtmlPage albumPage, int albumIndex) throws FileNotFoundException, UnsupportedEncodingException, IOException { ArrayList<String> photos = new ArrayList<String>(); //Array of Images links DomNodeList<HtmlElement> anchors; //All anchors in album page while (true) { // Extracting images links HtmlElement body = albumPage.getElementById("root"); anchors = body.getElementsByTagName("a"); for (int j = 0; j < anchors.size(); j++) { if (anchors.get(j).getAttribute("href").contains("fbid")) { photos.add("http://m.facebook.com" + anchors.get(j).getAttribute("href")); } } //checking for Additional images in this album try { albumPage = getAuthenticatedProfile().getBrowser().getPage("http://m.facebook.com" + albumPage .getElementById("m_more_item").getElementsByTagName("a").get(0).getAttribute("href")); } catch (Exception ex) { break; } if (checkForCancel()) { return; } } //dumping Images with comments for (int j = 0; j < photos.size(); j++) { //opening Image Page that containg it's comments HtmlPage photoPage = getAuthenticatedProfile().getBrowser().getPage(photos.get(j)); //writing image to file FileUtils.copyURLToFile(new URL(photoPage.getElementsByTagName("img").get(1).getAttribute("src")), new File(getDirectory().getAbsolutePath() + System.getProperty("file.separator") + "Album-" + albumIndex + System.getProperty("file.separator") + "Image-" + (j + 1) + ".jpg")); //Initializing html file to look like facebook interface PrintWriter commentWriter = new PrintWriter(new File( getDirectory().getAbsolutePath() + System.getProperty("file.separator") + "Album-" + albumIndex + System.getProperty("file.separator") + "Comments-on-Image-" + (j + 1) + ".html"), "UTF-8"); commentWriter.println("<?xml version=\"1.0\" encoding=\"utf-8\"?>"); commentWriter.println( "<!DOCTYPE html PUBLIC \"-//WAPFORUM//DTD XHTML Mobile 1.0//EN\" \"http://www.wapforum.org/DTD/xhtml-mobile10.dtd\">"); commentWriter.println(albumPage.getElementsByTagName("head").get(0).asXml()); //checking for previous comments on this image try { //dumping previous comments dumpComments((HtmlPage) getAuthenticatedProfile().getBrowser() .getPage("http://m.facebook.com" + photoPage.getElementById("see_prev") .getElementsByTagName("a").get(0).getAttribute("href")), commentWriter); } catch (Exception ex) { } //dumping latest comments dumpComments(photoPage, commentWriter); commentWriter.flush(); commentWriter.close(); setPercentage((double) (j + 1) / photos.size() * 100); getFacebookGUI().updateTaskProgress(this); if (checkForCancel()) { return; } } //dumping Album name in text file PrintWriter nameWriter = new PrintWriter( new File(getDirectory().getAbsolutePath() + System.getProperty("file.separator") + "Album-" + albumIndex + System.getProperty("file.separator") + "Album Name.txt"), "UTF-8"); nameWriter.println(albumPage.getTitleText()); nameWriter.flush(); nameWriter.close(); }
From source file:fr.inria.eventcloud.deployment.cli.launchers.EventCloudsManagementServiceDeployer.java
private static void downloadLibs(String libsUrl) throws IOException { libDirPath = System.getProperty("java.io.tmpdir") + File.separator + "eventcloud-libs"; File tmpLibDir = new File(libDirPath); tmpLibDir.mkdir();// w w w . ja va 2 s . c om File readme = new File(tmpLibDir, "README"); FileUtils.copyURLToFile(new URL(libsUrl + "README"), readme); String[] libNames = FileUtils.readFileToString(readme).split("\n"); for (String libName : libNames) { FileUtils.copyURLToFile(new URL(libsUrl + libName), new File(tmpLibDir, libName)); } }
From source file:model.Modele.java
/** * Methode qui tlcharger le fichier des donnes du site directement * * @param jour jour du fichier telecharger * @param mois mois du fichier telecharger * @param annee annee du fichier telecharger * @param heure heure du fichier telecharger * @return nom du fichier tlcharger//ww w . j a v a 2 s.c o m * @throws MalformedURLException * @throws IOException */ public String Download_File(String jour, String mois, String annee, String heure) throws MalformedURLException, IOException { if (jour.equals("NULL") && heure.equals("NULL")) { URL url = new URL("https://donneespubliques.meteofrance.fr/donnees_libres/Txt/Synop/Archive/synop." + annee + mois + ".csv.gz"); File file = new File("synop." + annee + mois + ".csv.gz"); FileUtils.copyURLToFile(url, file); FileInputStream in = new FileInputStream("synop." + annee + mois + ".csv.gz"); GZIPInputStream zipin = new GZIPInputStream(in); byte[] buffer = new byte[8192]; FileOutputStream out = new FileOutputStream("synop." + annee + mois + ".csv.gz.txt"); int length; while ((length = zipin.read(buffer, 0, 8192)) != -1) { out.write(buffer, 0, length); } out.close(); zipin.close(); File f = new File("synop." + annee + mois + ".csv.gz.txt"); if (f.exists()) { nom_fichier = Save_data(f); } else { JOptionPane.showMessageDialog(null, "Fichier inexistant ou n'a pas pu tre tlcharg !"); } } else { if (annee.equals("2017")) { URL url = new URL("https://donneespubliques.meteofrance.fr/donnees_libres/Txt/Synop/Archive/synop." + annee + mois + jour + heure + ".csv.gz"); File file = new File("synop." + annee + mois + jour + heure + ".csv"); FileUtils.copyURLToFile(url, file); File f = new File("synop." + annee + mois + jour + heure + ".csv"); if (f.exists()) { nom_fichier = Save_data(f); } else { JOptionPane.showMessageDialog(null, "Fichier inexistant ou n'a pas pu tre tlcharg !"); } } else { JOptionPane.showMessageDialog(null, "Seulement l'anne 2017 disponible pour cette option !"); } } return nom_fichier; }
From source file:gmusic.api.impl.GoogleMusicAPI.java
protected File downloadTune(Tune tune) throws MalformedURLException, ClientProtocolException, IOException, URISyntaxException { File file = new File(storageDirectory.getAbsolutePath() + tune.getId() + ".mp3"); if (!file.exists()) { FileUtils.copyURLToFile(getTuneURL(tune).toURL(), file); populateFileWithTuneTags(file, tune); }/*from www. j ava 2 s . c om*/ return file; }
From source file:gov.nasa.jpl.mudrod.main.MudrodEngine.java
private String decompressSVMWithSGDModel(String archiveName) throws IOException { URL scmArchive = getClass().getClassLoader().getResource(archiveName); if (scmArchive == null) { throw new IOException("Unable to locate " + archiveName + " as a classpath resource."); }//from www .j a va2 s . c om File tempDir = Files.createTempDirectory("mudrod").toFile(); assert tempDir.setWritable(true); File archiveFile = new File(tempDir, archiveName); FileUtils.copyURLToFile(scmArchive, archiveFile); // Decompress archive int BUFFER_SIZE = 512000; ZipInputStream zipIn = new ZipInputStream(new FileInputStream(archiveFile)); ZipEntry entry; while ((entry = zipIn.getNextEntry()) != null) { File f = new File(tempDir, entry.getName()); // If the entry is a directory, create the directory. if (entry.isDirectory() && !f.exists()) { boolean created = f.mkdirs(); if (!created) { LOG.error("Unable to create directory '{}', during extraction of archive contents.", f.getAbsolutePath()); } } else if (!entry.isDirectory()) { boolean created = f.getParentFile().mkdirs(); if (!created && !f.getParentFile().exists()) { LOG.error("Unable to create directory '{}', during extraction of archive contents.", f.getParentFile().getAbsolutePath()); } int count; byte data[] = new byte[BUFFER_SIZE]; FileOutputStream fos = new FileOutputStream(new File(tempDir, entry.getName()), false); try (BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE)) { while ((count = zipIn.read(data, 0, BUFFER_SIZE)) != -1) { dest.write(data, 0, count); } } } } return new File(tempDir, StringUtils.removeEnd(archiveName, ".zip")).toURI().toString(); }
From source file:com.comcast.magicwand.spells.web.iexplore.IePhoenixDriver.java
/** * {@inheritDoc}//from w w w. jav a 2 s .co m */ public boolean verify(PhoenixDriverIngredients i) { boolean rv = false; Map<String, Object> configs = i.getDriverConfigs(); LOGGER.debug("[version_property, arch_property] = [{}, {}]", (String) configs.get(IE_DRIVER_VERSION_KEY), (String) configs.get(IE_DRIVER_ARCH_KEY)); IePlatformSpecifics ips = this.createIePlatformSpecifics((String) configs.get(IE_DRIVER_VERSION_KEY), (String) configs.get(IE_DRIVER_ARCH_KEY)); if (!ips.isValid()) { LOGGER.error("The IePlatformSpecifics retrieved are not valid."); return false; } try { XPath xpath = XPathFactory.newInstance().newXPath(); String pwd = System.getProperty("user.dir"); String version = ips.getVersion(); String arch = ips.getArch(); Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() .parse(IE_XML_DRIVER_LISTING_URL); NodeList nodes = (NodeList) xpath.evaluate(IE_XML_DRIVER_LISTING_XPATH, doc, XPathConstants.NODESET); String highestVersion = null; String needle = version + "/IEDriverServer_" + arch + "_"; for (int idx = 0; idx < nodes.getLength(); idx++) { Node n = nodes.item(idx); String text = n.getTextContent(); if (text.startsWith(needle)) { text = text.substring(needle.length(), text.length()); if (IePhoenixDriver.versionGreater(highestVersion, text)) { highestVersion = text; } } } if (null != highestVersion) { highestVersion = FilenameUtils.removeExtension(highestVersion); URL url = new URL(String.format(IE_HTTP_DRIVER_PATH_FORMAT, version, arch, highestVersion)); String zipName = String.format(IE_ZIP_FILE_FORMAT, arch, highestVersion); File ieSaveDir = new File(Paths.get(pwd, "target", "drivers", "iexplore", version).toString()); LOGGER.debug("Will read from \"{}\"", url); File zipFile = new File( Paths.get(pwd, "target", "drivers", "iexplore", version, zipName).toString()); FileUtils.copyURLToFile(url, zipFile); extract(zipFile, ieSaveDir.getAbsolutePath()); File exe = Paths.get(pwd, "target", "drivers", "iexplore", version, IE_EXE_FILE_NAME).toFile(); if (exe.exists()) { exe.setExecutable(true); systemSetProperty("webdriver.ie.driver", exe.getAbsolutePath()); this.webDriver = this.createDriver(i.getDriverCapabilities()); rv = true; } else { LOGGER.error("Extracted zip archive did nto contain \"{}\".", IE_EXE_FILE_NAME); } } else { LOGGER.error("Unable to find any IE Drivers from [{}]", IE_XML_DRIVER_LISTING_XPATH); } } catch (ParserConfigurationException | SAXException | XPathExpressionException err) { throw new RuntimeException(err); } catch (IOException ioe) { LOGGER.error("IO failure: {}", ioe); } return rv; }