List of usage examples for java.io File setReadable
public boolean setReadable(boolean readable)
From source file:alluxio.shell.command.CpCommandIntegrationTest.java
@Test public void copyFromLocalDirNotReadable() throws Exception { // Copy a directory from local to Alluxio filesystem, which the destination uri was not created // before./*from www . jav a 2 s . c o m*/ File srcOuterDir = new File(mLocalAlluxioCluster.getAlluxioHome() + "/outerDir"); File srcInnerDir = new File(mLocalAlluxioCluster.getAlluxioHome() + "/outerDir/innerDir"); File emptyDir = new File(mLocalAlluxioCluster.getAlluxioHome() + "/outerDir/emptyDir"); srcOuterDir.mkdir(); srcInnerDir.mkdir(); emptyDir.mkdir(); generateFileContent("/outerDir/srcFile1", BufferUtils.getIncreasingByteArray(10)); generateFileContent("/outerDir/innerDir/srcFile2", BufferUtils.getIncreasingByteArray(10)); srcOuterDir.setReadable(false); int ret = mFsShell.run("cp", "file://" + srcOuterDir.getPath() + "/", "/dstDir"); Assert.assertEquals(-1, ret); Assert.assertEquals("Failed to list files for directory " + srcOuterDir.getAbsolutePath() + "\n", mOutput.toString()); }
From source file:alluxio.shell.command.CpCommandTest.java
@Test public void copyFromLocalDirNotReadable() throws IOException, AlluxioException { // Copy a directory from local to Alluxio filesystem, which the destination uri was not created // before./* w w w . ja v a 2 s . co m*/ File srcOuterDir = new File(mLocalAlluxioCluster.getAlluxioHome() + "/outerDir"); File srcInnerDir = new File(mLocalAlluxioCluster.getAlluxioHome() + "/outerDir/innerDir"); File emptyDir = new File(mLocalAlluxioCluster.getAlluxioHome() + "/outerDir/emptyDir"); srcOuterDir.mkdir(); srcInnerDir.mkdir(); emptyDir.mkdir(); generateFileContent("/outerDir/srcFile1", BufferUtils.getIncreasingByteArray(10)); generateFileContent("/outerDir/innerDir/srcFile2", BufferUtils.getIncreasingByteArray(10)); srcOuterDir.setReadable(false); int ret = mFsShell.run("cp", "file://" + srcOuterDir.getPath() + "/", "/dstDir"); Assert.assertEquals(-1, ret); Assert.assertEquals("Failed to list files for directory " + srcOuterDir.getAbsolutePath() + "\n", mOutput.toString()); }
From source file:org.apache.hive.spark.client.SparkClientImpl.java
private Thread startDriver(final RpcServer rpcServer, final String clientId, final String secret) throws IOException { Runnable runnable;/*from w w w.j a va2s . c o m*/ final String serverAddress = rpcServer.getAddress(); final String serverPort = String.valueOf(rpcServer.getPort()); if (conf.containsKey(SparkClientFactory.CONF_KEY_IN_PROCESS)) { // Mostly for testing things quickly. Do not do this in production. // when invoked in-process it inherits the environment variables of the parent LOG.warn("!!!! Running remote driver in-process. !!!!"); runnable = new Runnable() { @Override public void run() { List<String> args = Lists.newArrayList(); args.add("--remote-host"); args.add(serverAddress); args.add("--remote-port"); args.add(serverPort); args.add("--client-id"); args.add(clientId); args.add("--secret"); args.add(secret); for (Map.Entry<String, String> e : conf.entrySet()) { args.add("--conf"); args.add(String.format("%s=%s", e.getKey(), conf.get(e.getKey()))); } try { RemoteDriver.main(args.toArray(new String[args.size()])); } catch (Exception e) { LOG.error("Error running driver.", e); } } }; } else { // If a Spark installation is provided, use the spark-submit script. Otherwise, call the // SparkSubmit class directly, which has some caveats (like having to provide a proper // version of Guava on the classpath depending on the deploy mode). String sparkHome = Strings.emptyToNull(conf.get(SPARK_HOME_KEY)); if (sparkHome == null) { sparkHome = Strings.emptyToNull(System.getenv(SPARK_HOME_ENV)); } if (sparkHome == null) { sparkHome = Strings.emptyToNull(System.getProperty(SPARK_HOME_KEY)); } String sparkLogDir = conf.get("hive.spark.log.dir"); if (sparkLogDir == null) { if (sparkHome == null) { sparkLogDir = "./target/"; } else { sparkLogDir = sparkHome + "/logs/"; } } String osxTestOpts = ""; if (Strings.nullToEmpty(System.getProperty("os.name")).toLowerCase().contains("mac")) { osxTestOpts = Strings.nullToEmpty(System.getenv(OSX_TEST_OPTS)); } String driverJavaOpts = Joiner.on(" ").skipNulls().join("-Dhive.spark.log.dir=" + sparkLogDir, osxTestOpts, conf.get(DRIVER_OPTS_KEY)); String executorJavaOpts = Joiner.on(" ").skipNulls().join("-Dhive.spark.log.dir=" + sparkLogDir, osxTestOpts, conf.get(EXECUTOR_OPTS_KEY)); // Create a file with all the job properties to be read by spark-submit. Change the // file's permissions so that only the owner can read it. This avoid having the // connection secret show up in the child process's command line. File properties = File.createTempFile("spark-submit.", ".properties"); if (!properties.setReadable(false) || !properties.setReadable(true, true)) { throw new IOException("Cannot change permissions of job properties file."); } properties.deleteOnExit(); Properties allProps = new Properties(); // first load the defaults from spark-defaults.conf if available try { URL sparkDefaultsUrl = Thread.currentThread().getContextClassLoader() .getResource("spark-defaults.conf"); if (sparkDefaultsUrl != null) { LOG.info("Loading spark defaults: " + sparkDefaultsUrl); allProps.load(new ByteArrayInputStream(Resources.toByteArray(sparkDefaultsUrl))); } } catch (Exception e) { String msg = "Exception trying to load spark-defaults.conf: " + e; throw new IOException(msg, e); } // then load the SparkClientImpl config for (Map.Entry<String, String> e : conf.entrySet()) { allProps.put(e.getKey(), conf.get(e.getKey())); } allProps.put(SparkClientFactory.CONF_CLIENT_ID, clientId); allProps.put(SparkClientFactory.CONF_KEY_SECRET, secret); allProps.put(DRIVER_OPTS_KEY, driverJavaOpts); allProps.put(EXECUTOR_OPTS_KEY, executorJavaOpts); String isTesting = conf.get("spark.testing"); if (isTesting != null && isTesting.equalsIgnoreCase("true")) { String hiveHadoopTestClasspath = Strings.nullToEmpty(System.getenv("HIVE_HADOOP_TEST_CLASSPATH")); if (!hiveHadoopTestClasspath.isEmpty()) { String extraDriverClasspath = Strings .nullToEmpty((String) allProps.get(DRIVER_EXTRA_CLASSPATH)); if (extraDriverClasspath.isEmpty()) { allProps.put(DRIVER_EXTRA_CLASSPATH, hiveHadoopTestClasspath); } else { extraDriverClasspath = extraDriverClasspath.endsWith(File.pathSeparator) ? extraDriverClasspath : extraDriverClasspath + File.pathSeparator; allProps.put(DRIVER_EXTRA_CLASSPATH, extraDriverClasspath + hiveHadoopTestClasspath); } String extraExecutorClasspath = Strings .nullToEmpty((String) allProps.get(EXECUTOR_EXTRA_CLASSPATH)); if (extraExecutorClasspath.isEmpty()) { allProps.put(EXECUTOR_EXTRA_CLASSPATH, hiveHadoopTestClasspath); } else { extraExecutorClasspath = extraExecutorClasspath.endsWith(File.pathSeparator) ? extraExecutorClasspath : extraExecutorClasspath + File.pathSeparator; allProps.put(EXECUTOR_EXTRA_CLASSPATH, extraExecutorClasspath + hiveHadoopTestClasspath); } } } Writer writer = new OutputStreamWriter(new FileOutputStream(properties), Charsets.UTF_8); try { allProps.store(writer, "Spark Context configuration"); } finally { writer.close(); } // Define how to pass options to the child process. If launching in client (or local) // mode, the driver options need to be passed directly on the command line. Otherwise, // SparkSubmit will take care of that for us. String master = conf.get("spark.master"); Preconditions.checkArgument(master != null, "spark.master is not defined."); String deployMode = conf.get("spark.submit.deployMode"); List<String> argv = Lists.newLinkedList(); if (sparkHome != null) { argv.add(new File(sparkHome, "bin/spark-submit").getAbsolutePath()); } else { LOG.info("No spark.home provided, calling SparkSubmit directly."); argv.add(new File(System.getProperty("java.home"), "bin/java").getAbsolutePath()); if (master.startsWith("local") || master.startsWith("mesos") || SparkClientUtilities.isYarnClientMode(master, deployMode) || master.startsWith("spark")) { String mem = conf.get("spark.driver.memory"); if (mem != null) { argv.add("-Xms" + mem); argv.add("-Xmx" + mem); } String cp = conf.get("spark.driver.extraClassPath"); if (cp != null) { argv.add("-classpath"); argv.add(cp); } String libPath = conf.get("spark.driver.extraLibPath"); if (libPath != null) { argv.add("-Djava.library.path=" + libPath); } String extra = conf.get(DRIVER_OPTS_KEY); if (extra != null) { for (String opt : extra.split("[ ]")) { if (!opt.trim().isEmpty()) { argv.add(opt.trim()); } } } } argv.add("org.apache.spark.deploy.SparkSubmit"); } if (SparkClientUtilities.isYarnClusterMode(master, deployMode)) { String executorCores = conf.get("spark.executor.cores"); if (executorCores != null) { argv.add("--executor-cores"); argv.add(executorCores); } String executorMemory = conf.get("spark.executor.memory"); if (executorMemory != null) { argv.add("--executor-memory"); argv.add(executorMemory); } String numOfExecutors = conf.get("spark.executor.instances"); if (numOfExecutors != null) { argv.add("--num-executors"); argv.add(numOfExecutors); } } // The options --principal/--keypad do not work with --proxy-user in spark-submit.sh // (see HIVE-15485, SPARK-5493, SPARK-19143), so Hive could only support doAs or // delegation token renewal, but not both. Since doAs is a more common case, if both // are needed, we choose to favor doAs. So when doAs is enabled, we use kinit command, // otherwise, we pass the principal/keypad to spark to support the token renewal for // long-running application. if ("kerberos".equals(hiveConf.get(HADOOP_SECURITY_AUTHENTICATION))) { String principal = SecurityUtil .getServerPrincipal(hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL), "0.0.0.0"); String keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB); if (StringUtils.isNotBlank(principal) && StringUtils.isNotBlank(keyTabFile)) { if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS)) { List<String> kinitArgv = Lists.newLinkedList(); kinitArgv.add("kinit"); kinitArgv.add(principal); kinitArgv.add("-k"); kinitArgv.add("-t"); kinitArgv.add(keyTabFile + ";"); kinitArgv.addAll(argv); argv = kinitArgv; } else { // if doAs is not enabled, we pass the principal/keypad to spark-submit in order to // support the possible delegation token renewal in Spark argv.add("--principal"); argv.add(principal); argv.add("--keytab"); argv.add(keyTabFile); } } } if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS)) { try { String currentUser = Utils.getUGI().getShortUserName(); // do not do impersonation in CLI mode if (!currentUser.equals(System.getProperty("user.name"))) { LOG.info("Attempting impersonation of " + currentUser); argv.add("--proxy-user"); argv.add(currentUser); } } catch (Exception e) { String msg = "Cannot obtain username: " + e; throw new IllegalStateException(msg, e); } } argv.add("--properties-file"); argv.add(properties.getAbsolutePath()); argv.add("--class"); argv.add(RemoteDriver.class.getName()); String jar = "spark-internal"; if (SparkContext.jarOfClass(this.getClass()).isDefined()) { jar = SparkContext.jarOfClass(this.getClass()).get(); } argv.add(jar); argv.add("--remote-host"); argv.add(serverAddress); argv.add("--remote-port"); argv.add(serverPort); //hive.spark.* keys are passed down to the RemoteDriver via --conf, //as --properties-file contains the spark.* keys that are meant for SparkConf object. for (String hiveSparkConfKey : RpcConfiguration.HIVE_SPARK_RSC_CONFIGS) { String value = RpcConfiguration.getValue(hiveConf, hiveSparkConfKey); argv.add("--conf"); argv.add(String.format("%s=%s", hiveSparkConfKey, value)); } String cmd = Joiner.on(" ").join(argv); LOG.info("Running client driver with argv: {}", cmd); ProcessBuilder pb = new ProcessBuilder("sh", "-c", cmd); // Prevent hive configurations from being visible in Spark. pb.environment().remove("HIVE_HOME"); pb.environment().remove("HIVE_CONF_DIR"); // Add credential provider password to the child process's environment // In case of Spark the credential provider location is provided in the jobConf when the job is submitted String password = getSparkJobCredentialProviderPassword(); if (password != null) { pb.environment().put(Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, password); } if (isTesting != null) { pb.environment().put("SPARK_TESTING", isTesting); } final Process child = pb.start(); String threadName = Thread.currentThread().getName(); final List<String> childErrorLog = Collections.synchronizedList(new ArrayList<String>()); redirect("RemoteDriver-stdout-redir-" + threadName, new Redirector(child.getInputStream())); redirect("RemoteDriver-stderr-redir-" + threadName, new Redirector(child.getErrorStream(), childErrorLog)); runnable = new Runnable() { @Override public void run() { try { int exitCode = child.waitFor(); if (exitCode != 0) { StringBuilder errStr = new StringBuilder(); synchronized (childErrorLog) { Iterator iter = childErrorLog.iterator(); while (iter.hasNext()) { errStr.append(iter.next()); errStr.append('\n'); } } LOG.warn("Child process exited with code {}", exitCode); rpcServer.cancelClient(clientId, "Child process (spark-submit) exited before connecting back with error log " + errStr.toString()); } } catch (InterruptedException ie) { LOG.warn( "Thread waiting on the child process (spark-submit) is interrupted, killing the child process."); rpcServer.cancelClient(clientId, "Thread waiting on the child porcess (spark-submit) is interrupted"); Thread.interrupted(); child.destroy(); } catch (Exception e) { String errMsg = "Exception while waiting for child process (spark-submit)"; LOG.warn(errMsg, e); rpcServer.cancelClient(clientId, errMsg); } } }; } Thread thread = new Thread(runnable); thread.setDaemon(true); thread.setName("Driver"); thread.start(); return thread; }
From source file:org.fuin.esmp.EventStoreDownloadMojo.java
private void applyFileMode(final File file, final FileMode fileMode) throws MojoExecutionException { if (OS.isFamilyUnix() || OS.isFamilyMac()) { final String smode = fileMode.toChmodStringFull(); final CommandLine cmdLine = new CommandLine("chmod"); cmdLine.addArgument(smode);// w w w. j a v a2 s. c om cmdLine.addArgument(file.getAbsolutePath()); final Executor executor = new DefaultExecutor(); try { final int result = executor.execute(cmdLine); if (result != 0) { throw new MojoExecutionException("Error # " + result + " while trying to set mode \"" + smode + "\" for file: " + file.getAbsolutePath()); } } catch (final IOException ex) { throw new MojoExecutionException( "Error while trying to set mode \"" + smode + "\" for file: " + file.getAbsolutePath(), ex); } } else { file.setReadable(fileMode.isUr() || fileMode.isGr() || fileMode.isOr()); file.setWritable(fileMode.isUw() || fileMode.isGw() || fileMode.isOw()); file.setExecutable(fileMode.isUx() || fileMode.isGx() || fileMode.isOx()); } }
From source file:com.thinkbiganalytics.feedmgr.rest.controller.FeedRestController.java
@POST @Path("/{feedId}/upload-file") @Consumes(MediaType.MULTIPART_FORM_DATA) @Produces(MediaType.APPLICATION_JSON)/*from w w w .j ava 2 s.c o m*/ @ApiOperation("Uploads a file to be ingested by a feed.") @ApiResponses({ @ApiResponse(code = 200, message = "The file is ready to be ingested."), @ApiResponse(code = 500, message = "The file could not be saved.", response = RestResponseStatus.class) }) public Response uploadFile(@PathParam("feedId") String feedId, @FormDataParam("file") InputStream fileInputStream, @FormDataParam("file") FormDataContentDisposition fileMetaData) throws Exception { FeedMetadata feed = getMetadataService().getFeedById(feedId, false); // Derive path and file List<NifiProperty> properties = feed.getProperties(); String dropzone = null; String regexFileFilter = null; for (NifiProperty property : properties) { if (property.getProcessorType().equals("org.apache.nifi.processors.standard.GetFile")) { if (property.getKey().equals("File Filter")) { regexFileFilter = property.getValue(); } else if (property.getKey().equals("Input Directory")) { dropzone = property.getValue(); } } } if (StringUtils.isEmpty(regexFileFilter) || StringUtils.isEmpty(dropzone)) { throw new IOException("Unable to upload file with empty dropzone and file"); } File tempTarget = File.createTempFile("kylo-upload", ""); String fileName = ""; try { Generex fileNameGenerator = new Generex(regexFileFilter); fileName = fileNameGenerator.random(); // Cleanup oddball characters generated by generex fileName = fileName.replaceAll("[^A-Za-z0-9\\.\\_\\+\\%\\-\\|]+", "\\."); java.nio.file.Path dropZoneTarget = Paths.get(dropzone, fileName); File dropZoneFile = dropZoneTarget.toFile(); if (dropZoneFile.exists()) { throw new IOException("File with the name [" + fileName + "] already exists in [" + dropzone + "]"); } Files.copy(fileInputStream, tempTarget.toPath(), StandardCopyOption.REPLACE_EXISTING); Files.move(tempTarget.toPath(), dropZoneTarget); // Set read, write dropZoneFile.setReadable(true); dropZoneFile.setWritable(true); } catch (AccessDeniedException e) { String errTemplate = "Permission denied attempting to write file [%s] to [%s]. Check with system administrator to ensure this application has write permissions to folder"; String err = String.format(errTemplate, fileName, dropzone); log.error(err); throw new InternalServerErrorException(err); } catch (Exception e) { String errTemplate = "Unexpected exception writing file [%s] to [%s]."; String err = String.format(errTemplate, fileName, dropzone); log.error(err); throw new InternalServerErrorException(err); } return Response.ok("").build(); }
From source file:lc.kra.servlet.FileManagerServlet.java
/** * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) */// w ww. jav a 2 s.co m protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { Files files = null; File file = null, parent; String path = request.getParameter("path"), type = request.getContentType(), search = request.getParameter("search"), mode; if (path == null || !(file = new File(path)).exists()) files = new Roots(); else if (request.getParameter("zip") != null) { File zipFile = File.createTempFile(file.getName() + "-", ".zip"); if (file.isFile()) ZipUtil.addEntry(zipFile, file.getName(), file); else if (file.isDirectory()) ZipUtil.pack(file, zipFile); downloadFile(response, zipFile, permamentName(zipFile.getName()), "application/zip"); } else if (request.getParameter("delete") != null) { if (file.isFile()) file.delete(); else if (file.isDirectory()) { java.nio.file.Files.walkFileTree(file.toPath(), new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { java.nio.file.Files.delete(file); return FileVisitResult.CONTINUE; } @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { java.nio.file.Files.delete(dir); return FileVisitResult.CONTINUE; } }); } } else if ((mode = request.getParameter("mode")) != null) { boolean add = mode.startsWith("+"); if (mode.indexOf('r') > -1) file.setReadable(add); if (mode.indexOf('w') > -1) file.setWritable(add); if (mode.indexOf('x') > -1) file.setExecutable(add); } else if (file.isFile()) downloadFile(response, file); else if (file.isDirectory()) { if (search != null && !search.isEmpty()) files = new Search(file.toPath(), search); else if (type != null && type.startsWith("multipart/form-data")) { for (Part part : request.getParts()) { String name; if ((name = partFileName(part)) == null) //retrieves <input type="file" name="...">, no other (e.g. input) form fields continue; if (request.getParameter("unzip") == null) try (OutputStream output = new FileOutputStream(new File(file, name))) { copyStream(part.getInputStream(), output); } else ZipUtil.unpack(part.getInputStream(), file); } } else files = new Directory(file); } else throw new ServletException("Unknown type of file or folder."); if (files != null) { final PrintWriter writer = response.getWriter(); writer.println( "<!DOCTYPE html><html><head><style>*,input[type=\"file\"]::-webkit-file-upload-button{font-family:monospace}</style></head><body>"); writer.println("<p>Current directory: " + files + "</p><pre>"); if (!(files instanceof Roots)) { writer.print( "<form method=\"post\"><label for=\"search\">Search Files:</label> <input type=\"text\" name=\"search\" id=\"search\" value=\"" + (search != null ? search : "") + "\"> <button type=\"submit\">Search</button></form>"); writer.print( "<form method=\"post\" enctype=\"multipart/form-data\"><label for=\"upload\">Upload Files:</label> <button type=\"submit\">Upload</button> <button type=\"submit\" name=\"unzip\">Upload & Unzip</button> <input type=\"file\" name=\"upload[]\" id=\"upload\" multiple></form>"); writer.println(); } if (files instanceof Directory) { writer.println("+ <a href=\"?path=" + URLEncoder.encode(path, ENCODING) + "\">.</a>"); if ((parent = file.getParentFile()) != null) writer.println("+ <a href=\"?path=" + URLEncoder.encode(parent.getAbsolutePath(), ENCODING) + "\">..</a>"); else writer.println("+ <a href=\"?path=\">..</a>"); } for (File child : files.listFiles()) { writer.print(child.isDirectory() ? "+ " : " "); writer.print("<a href=\"?path=" + URLEncoder.encode(child.getAbsolutePath(), ENCODING) + "\" title=\"" + child.getAbsolutePath() + "\">" + child.getName() + "</a>"); if (child.isDirectory()) writer.print(" <a href=\"?path=" + URLEncoder.encode(child.getAbsolutePath(), ENCODING) + "&zip\" title=\"download\">⇩</a>"); if (search != null && !search.isEmpty()) writer.print(" <a href=\"?path=" + URLEncoder.encode(child.getParentFile().getAbsolutePath(), ENCODING) + "\" title=\"go to parent folder\">🗁</a>"); writer.println(); } writer.print("</pre></body></html>"); writer.flush(); } }
From source file:com.github.jarlakxen.embedphantomjs.PhantomJSReference.java
private void downloadPhantomJS(File binaryFile) throws IOException { Properties properties = new Properties(); properties.load(this.getClass().getClassLoader().getResourceAsStream(PHANTOMJS_DATA_FILE)); String name = properties.getProperty(this.getVersion().getDescription() + "." + this.getHostOs() + ".name"); String architecture = this.getArchitecture().indexOf("64") >= 0 ? "x86_64" : "i686"; LOGGER.debug("System Data: Arch [" + architecture + "] - OS [" + this.getHostOs() + "]"); if (this.getHostOs().equals("linux")) { name = String.format(name, architecture); }/*from w w w.j ava2 s .co m*/ // Download PhantomJS URL downloadPath = new URL(this.getDownloadUrl() + name); File phantomJsCompressedFile = new File(System.getProperty("java.io.tmpdir") + "/" + name); LOGGER.info("Downloading " + downloadPath.getPath() + " ..."); FileUtils.copyURLToFile(downloadPath, phantomJsCompressedFile); ArchiveInputStream archiveInputStream = null; if (phantomJsCompressedFile.getName().endsWith(".zip")) { archiveInputStream = new ZipArchiveInputStream(new FileInputStream(phantomJsCompressedFile)); } else if (phantomJsCompressedFile.getName().endsWith(".bz2")) { archiveInputStream = new TarArchiveInputStream( new BZip2CompressorInputStream(new FileInputStream(phantomJsCompressedFile))); } else if (phantomJsCompressedFile.getName().endsWith(".gz")) { archiveInputStream = new TarArchiveInputStream( new GzipCompressorInputStream(new FileInputStream(phantomJsCompressedFile))); } ArchiveEntry entry; while ((entry = archiveInputStream.getNextEntry()) != null) { if (entry.getName().endsWith(PHANTOMJS_DOWNLOAD_BINARY_PATH) || entry.getName().toLowerCase().endsWith("phantomjs.exe")) { // Create target folder new File(this.getTargetInstallationFolder() + "/" + this.getVersion().getDescription()).mkdirs(); FileUtils.forceMkdir(new File(binaryFile.getParent())); if (!binaryFile.exists()) { binaryFile.createNewFile(); } binaryFile.setExecutable(true); binaryFile.setReadable(true); // Untar the binary file FileOutputStream outputBinary = new FileOutputStream(binaryFile); LOGGER.info("Un-compress download to " + downloadPath.getPath() + " ..."); IOUtils.copy(archiveInputStream, outputBinary); outputBinary.close(); } } archiveInputStream.close(); }
From source file:com.storm.function.GsxtFunction.java
@SuppressWarnings("deprecation") public String getSerializedAllIn(WebParam webParam) throws Exception { ChannelLogger LOGGER = ChannelLoggerFactory.getLogger(GsxtFunction.class, webParam.getLogback()); Gson gson = new GsonBuilder().create(); Map<String, String> resultMap = new LinkedHashMap<String, String>(); try {/*from w w w . j av a2s .c o m*/ LOGGER.info("==================GsxtFunction.getSerializedAllIn start!======================"); SerializedAllIn serializedAllIn = new SerializedAllIn(); // WebClient webClient = WebCrawler.getInstance().getWebClient(); String searchPageUrl = webParam.getSearchPage(); if (StringUtils.isEmpty(searchPageUrl)) { LOGGER.error("The searchPageUrl is not defined!"); return null; } //?cookie clearCookies(webClient, new URL(searchPageUrl)); WebRequest webRequest = new WebRequest(new URL(searchPageUrl), HttpMethod.GET); HtmlPage searchPage = webClient.getPage(webRequest); String searchPageHtml = searchPage.asXml(); LOGGER.info("searchPageHtml:" + searchPageHtml); if (searchPageHtml.contains("???") || searchPageHtml.contains("?")) { //???? ?? resultMap.put("statusCodeDef", StatusCodeDef.FREQUENCY_LIMITED); resultMap.put("searchPageHtml", IPUtil.getHostAndIpStr() + searchPageHtml); return gson.toJson(resultMap); } //imageUrl HtmlImage image = searchPage.getFirstByXPath(webParam.getCodeImageId()); Map<String, String> webParamParams = webParam.getParams(); File parentDirFile = new File(StormTopologyConfig.getNfs_filepath()); parentDirFile.setReadable(true); // parentDirFile.setWritable(true); // if (!parentDirFile.exists()) { parentDirFile.mkdirs(); } String imageName = UUID.randomUUID() + ".jpg"; File codeImageFile = new File(StormTopologyConfig.getNfs_filepath() + "/" + imageName); codeImageFile.setReadable(true); // codeImageFile.setWritable(true); // LOGGER.info( "The codeImageFile of GsxtFunction.getSerializedAllIn is:" + codeImageFile.getAbsolutePath()); if (searchPageUrl.contains("gsxt.cqgs.gov.cn")) { //? Page page = webClient.getPage("http://gsxt.cqgs.gov.cn/sc.action?width=130&height=40&fs=23"); InputStream is = page.getWebResponse().getContentAsStream(); FileUtils.copyInputStreamToFile(is, codeImageFile); LOGGER.info("----codeImageFile saved!----"); LOGGER.info("The codeImageFile of GsxtFunction.getSerializedAllIn is:{}", codeImageFile.getAbsolutePath()); serializedAllIn .setImageUrl("http://" + StormTopologyConfig.getNfs_nginx_server() + "/" + imageName); LOGGER.info("The serializedAllIn.imageUrl is: " + serializedAllIn.getImageUrl()); } else { if (image == null && webParamParams != null && !StringUtils.isEmpty(webParamParams.get("imagecodeIframeSrc"))) { //?? ???iframe-parent HtmlPage imagecodeIframePage = webClient.getPage(webParamParams.get("imagecodeIframeSrc")); image = imagecodeIframePage.getFirstByXPath(webParam.getCodeImageId()); } else if (image != null && StringUtils.isEmpty(image.getAttribute("src"))) { //???src??? image.click(); HtmlElement hyzBtn1 = searchPage.getFirstByXPath("//div[@id='woaicss_con1']/ul/li[2]/div[2]/a");//??? HtmlElement hyzBtn2 = searchPage .getFirstByXPath("//div[@id='codeWindow']/div/ul/li[2]/div[2]/a");//? if (StringUtils.isEmpty(image.getAttribute("src")) && hyzBtn1 != null) { //?? hyzBtn1.click(); } else if (StringUtils.isEmpty(image.getAttribute("src")) && hyzBtn2 != null) { hyzBtn2.click(); } } if (searchPageHtml.contains("??")) { // LOGGER.info("====================?===================="); HtmlElement hyzBtn3 = searchPage.getFirstByXPath("//a[@id='kaptchaText']"); //? hyzBtn3.click(); } if (image == null) { resultMap.put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); resultMap.put("searchPageHtml", IPUtil.getHostAndIpStr() + searchPageHtml); resultMap.put("isImageNull", "true"); return gson.toJson(resultMap); } try { image.saveAs(codeImageFile); LOGGER.info("----codeImageFile saved!----"); LOGGER.info("The codeImageFile of GsxtFunction.getSerializedAllIn is:{}", codeImageFile.getAbsolutePath()); serializedAllIn .setImageUrl("http://" + StormTopologyConfig.getNfs_nginx_server() + "/" + imageName); LOGGER.info("The serializedAllIn.imageUrl is: " + serializedAllIn.getImageUrl()); } catch (Exception e) { e.printStackTrace(); LOGGER.info("----codeImageFile saveAs fail----"); resultMap.put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); resultMap.put("searchPageHtml", IPUtil.getHostAndIpStr() + "?saveAs?"); return gson.toJson(resultMap); } } //cookies Set<Cookie> cookies = webClient.getCookieManager().getCookies(new URL(searchPageUrl)); serializedAllIn.setCookies(cookies); //webResponse WebResponse webResponse = searchPage.getWebResponse(); serializedAllIn.setWebResponse(webResponse); //?? String serializedAllInFileName = UUID.randomUUID().toString() + StatusCodeDef.SERIALIZED_FILE_SUFFIX; File serializedAllInFile = new File( StormTopologyConfig.getNfs_filepath() + "/" + serializedAllInFileName); ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(serializedAllInFile)); oos.writeObject(serializedAllIn); oos.close(); LOGGER.info("The serializedAllInFileName is: " + serializedAllInFileName); // ??? ??URL resultMap.put("ip", ip); resultMap.put("hostName", hostName); resultMap.put("statusCodeDef", StatusCodeDef.SCCCESS); resultMap.put("codeImageUrl", serializedAllIn.getImageUrl()); resultMap.put("serializedFileName", serializedAllInFileName); } finally { LOGGER.returnRedisResource(); } return gson.toJson(resultMap); }
From source file:com.storm.function.GsxtFunction.java
private HtmlPage getSearchPageByUrl(WebClient webClient, WebParam webParam) throws Exception { ChannelLogger LOGGER = ChannelLoggerFactory.getLogger(GsxtFunction.class, webParam.getLogback()); HtmlPage searchPage = null;//w ww . j av a 2 s . c o m try { String searchPageUrl = webParam.getSearchPage(); if (StringUtils.isEmpty(searchPageUrl)) { LOGGER.error("The searchPageUrl is not defined!"); return null; } //?cookie clearCookies(webClient, new URL(searchPageUrl)); String searchPageHtml = ""; WebRequest webRequest = new WebRequest(new URL(searchPageUrl), HttpMethod.GET); if ("shanxi".equals(webParam.getParams().get("area").toString())) { WebWindow window = webClient.openWindow(new URL(searchPageUrl), "shanxi"); searchPage = webClient.getPage(window, webRequest); } else { searchPage = webClient.getPage(webRequest); } searchPageHtml = searchPage.asXml(); LOGGER.info("searchPageHtml:" + searchPageHtml); if (searchPageHtml.contains("???")) { //???? ?? webParam.getParams().put("statusCodeDef", StatusCodeDef.FREQUENCY_LIMITED); webParam.getParams().put("searchPageHtml", IPUtil.getHostAndIpStr() + searchPageHtml); return searchPage; } // ??? if (!StringUtils.isEmpty(webParam.getParams().get("area")) && "ningxia".equals(webParam.getParams().get("area")) && searchPageHtml.contains("?")) { webParam.getParams().put("statusCodeDef", StatusCodeDef.FREQUENCY_LIMITED); webParam.getParams().put("searchPageHtml", IPUtil.getHostAndIpStr() + searchPageHtml); return searchPage; } //imageUrl HtmlImage image = searchPage.getFirstByXPath(webParam.getCodeImageId()); Map<String, String> webParamParams = webParam.getParams(); File parentDirFile = new File(StormTopologyConfig.getNfs_filepath()); parentDirFile.setReadable(true); // parentDirFile.setWritable(true); // if (!parentDirFile.exists()) { parentDirFile.mkdirs(); } String imageName = null; if ("chongqing".equals(webParam.getParams().get("area").toString())) { imageName = UUID.randomUUID() + ".png"; } else { imageName = UUID.randomUUID() + ".jpg"; } File codeImageFile = new File(StormTopologyConfig.getNfs_filepath() + "/" + imageName); codeImageFile.setReadable(true); // codeImageFile.setWritable(true); // if (searchPageUrl.contains("gsxt.cqgs.gov.cn")) { //? Page page = webClient.getPage("http://gsxt.cqgs.gov.cn/sc.action?width=130&height=40&fs=23"); InputStream is = null; try { is = page.getWebResponse().getContentAsStream(); FileUtils.copyInputStreamToFile(is, codeImageFile); LOGGER.info("----codeImageFile saved!----"); LOGGER.info("The codeImageFile of GsxtFunction.getSearchPageByUrl is:{}", codeImageFile.getAbsolutePath()); String imageUrl = "http://" + StormTopologyConfig.getNfs_nginx_server() + "/" + imageName; LOGGER.info("The imageUrl is: " + imageUrl); webParam.getParams().put("imageUrl", imageUrl); webParam.getParams().put("statusCodeDef", StatusCodeDef.SCCCESS); } catch (Exception e) { e.printStackTrace(); LOGGER.info("----codeImageFile saved fail!----"); webParam.getParams().put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); webParam.getParams().put("searchPageHtml", IPUtil.getHostAndIpStr() + "?copyInputStreamToFile?"); } } else if (searchPageUrl.contains("fjaic.gov.cn")) { //? Page page = webClient.getPage( "http://wsgs.fjaic.gov.cn/creditpub/captcha?preset=str-01,math-01&ra=" + Math.random()); InputStream is = null; try { is = page.getWebResponse().getContentAsStream(); FileUtils.copyInputStreamToFile(is, codeImageFile); LOGGER.info("----codeImageFile saved!----"); LOGGER.info("The codeImageFile of GsxtFunction.getSearchPageByUrl is:{}", codeImageFile.getAbsolutePath()); String imageUrl = "http://" + StormTopologyConfig.getNfs_nginx_server() + "/" + imageName; LOGGER.info("The imageUrl is: " + imageUrl); webParam.getParams().put("imageUrl", imageUrl); webParam.getParams().put("statusCodeDef", StatusCodeDef.SCCCESS); } catch (Exception e) { e.printStackTrace(); LOGGER.info("----codeImageFile saved fail!----"); webParam.getParams().put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); webParam.getParams().put("searchPageHtml", IPUtil.getHostAndIpStr() + "?copyInputStreamToFile?"); } } else if (searchPageUrl.contains("gsxt.hnaic.gov.cn")) { //? Page page = webClient .getPage("http://gsxt.hnaic.gov.cn/notice/captcha?preset=&ra=" + Math.random()); InputStream is = null; try { is = page.getWebResponse().getContentAsStream(); FileUtils.copyInputStreamToFile(is, codeImageFile); LOGGER.info("----codeImageFile saved!----"); LOGGER.info("The codeImageFile of GsxtFunction.getSearchPageByUrl is:{}", codeImageFile.getAbsolutePath()); String imageUrl = "http://" + StormTopologyConfig.getNfs_nginx_server() + "/" + imageName; LOGGER.info("The imageUrl is: " + imageUrl); webParam.getParams().put("imageUrl", imageUrl); webParam.getParams().put("statusCodeDef", StatusCodeDef.SCCCESS); } catch (Exception e) { e.printStackTrace(); LOGGER.info("----codeImageFile saved fail!----"); webParam.getParams().put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); webParam.getParams().put("searchPageHtml", IPUtil.getHostAndIpStr() + "?copyInputStreamToFile?"); } } else if (searchPageUrl.contains("218.95.241.36")) { //? Page page = webClient .getPage("http://218.95.241.36/validateCode.jspx?type=0&id=0.9730435636142166"); InputStream is = null; try { is = page.getWebResponse().getContentAsStream(); FileUtils.copyInputStreamToFile(is, codeImageFile); LOGGER.info("----codeImageFile saved!----"); LOGGER.info("The codeImageFile of GsxtFunction.getSearchPageByUrl is:{}", codeImageFile.getAbsolutePath()); String imageUrl = "http://" + StormTopologyConfig.getNfs_nginx_server() + "/" + imageName; LOGGER.info("The imageUrl is: " + imageUrl); webParam.getParams().put("imageUrl", imageUrl); webParam.getParams().put("statusCodeDef", StatusCodeDef.SCCCESS); } catch (Exception e) { e.printStackTrace(); LOGGER.info("----codeImageFile saved fail!----"); webParam.getParams().put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); webParam.getParams().put("searchPageHtml", IPUtil.getHostAndIpStr() + "?copyInputStreamToFile?"); } } else { if (image == null && webParamParams != null && !StringUtils.isEmpty(webParamParams.get("imagecodeIframeSrc"))) { //?? ???iframe-parent HtmlPage imagecodeIframePage = webClient.getPage(webParamParams.get("imagecodeIframeSrc")); image = imagecodeIframePage.getFirstByXPath(webParam.getCodeImageId()); } else if (image != null && StringUtils.isEmpty(image.getAttribute("src"))) { //???src??? image.click(); HtmlElement hyzBtn1 = searchPage.getFirstByXPath("//div[@id='woaicss_con1']/ul/li[2]/div[2]/a");//??? HtmlElement hyzBtn2 = searchPage .getFirstByXPath("//div[@id='codeWindow']/div/ul/li[2]/div[2]/a");//? if (StringUtils.isEmpty(image.getAttribute("src")) && hyzBtn1 != null) { //?? hyzBtn1.click(); } else if (StringUtils.isEmpty(image.getAttribute("src")) && hyzBtn2 != null) { hyzBtn2.click(); } } else if (searchPageUrl.contains("218.57.139.24")) { //? HtmlElement zdmBtn = searchPage.getFirstByXPath("//a[@onclick='zdm()']"); if (zdmBtn != null) { zdmBtn.click(); } } if (image == null) { webParam.getParams().put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); webParam.getParams().put("searchPageHtml", IPUtil.getHostAndIpStr() + searchPageHtml); webParam.getParams().put("isImageNull", "true"); return searchPage; } try { image.saveAs(codeImageFile); LOGGER.info("----codeImageFile saved!----"); LOGGER.info("The codeImageFile of GsxtFunction.getSearchPageByUrl is:" + codeImageFile.getAbsolutePath()); String imageUrl = "http://" + StormTopologyConfig.getNfs_nginx_server() + "/" + imageName; LOGGER.info("The imageUrl is: " + imageUrl); webParam.getParams().put("imageUrl", imageUrl); webParam.getParams().put("statusCodeDef", StatusCodeDef.SCCCESS); } catch (Exception e) { e.printStackTrace(); LOGGER.info("----codeImageFile saveAs fail!----"); webParam.getParams().put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); webParam.getParams().put("searchPageHtml", IPUtil.getHostAndIpStr() + "?saveAs?"); } } } finally { LOGGER.returnRedisResource(); } return searchPage; }