List of usage examples for org.apache.commons.vfs2.provider.http HttpFileSystemConfigBuilder getInstance
public static HttpFileSystemConfigBuilder getInstance()
From source file:maspack.fileutil.FileCacher.java
public static void setDefaultFsOptions(FileSystemOptions opts) throws FileSystemException { // SSH Defaults // Don't check host key // Use paths relative to root (as opposed to the user's home dir) // 10 second timeout SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no"); SftpFileSystemConfigBuilder.getInstance().setUserDirIsRoot(opts, true); SftpFileSystemConfigBuilder.getInstance().setTimeout(opts, 10000); /**/*from w w w . j a v a 2 s . com*/ * Allow connection to silly UBC servers who don't update their credentials */ TrustStrategy[] ts = { new UnsafeTrustStrategy() }; HttpFileSystemConfigBuilder httpBuilder = HttpFileSystemConfigBuilder.getInstance(); WebdavFileSystemConfigBuilder webdavBuilder = WebdavFileSystemConfigBuilder.getInstance(); // allow all SSL connections httpBuilder.setTrustStrategies(opts, ts); webdavBuilder.setTrustStrategies(opts, ts); // silly deprecated UBC cipher suite String[] ciphers = httpBuilder.getDefaultSSLCipherSuites(); ciphers = Arrays.copyOf(ciphers, ciphers.length + 1); ciphers[ciphers.length - 1] = "SSL_RSA_WITH_RC4_128_SHA"; httpBuilder.setEnabledSSLCipherSuites(opts, ciphers); webdavBuilder.setEnabledSSLCipherSuites(opts, ciphers); }
From source file:net.sf.jabb.util.vfs.VfsUtility.java
/** * Configure FileSystemOptions for HttpFileSystem * @param fsOptions//from w w w . j a v a 2 s .co m * @param webProxyHost * @param webProxyPort * @param webProxyUserName * @param webProxyPassword */ static public void configHttpFileSystemProxy(FileSystemOptions fsOptions, String webProxyHost, Integer webProxyPort, String webProxyUserName, String webProxyPassword) { if (webProxyHost != null && webProxyPort != null) { HttpFileSystemConfigBuilder.getInstance().setProxyHost(fsOptions, webProxyHost); HttpFileSystemConfigBuilder.getInstance().setProxyPort(fsOptions, webProxyPort); if (webProxyUserName != null) { StaticUserAuthenticator auth = new StaticUserAuthenticator(webProxyUserName, webProxyPassword, null); HttpFileSystemConfigBuilder.getInstance().setProxyAuthenticator(fsOptions, auth); } } }
From source file:dslab.crawler.pack.CrawlerPack.java
public String getFromRemote(String uri) { // clear cache fileSystem.getFilesCache().close();/*from w w w . j a va2s . com*/ String remoteContent; String remoteEncoding = "utf-8"; log.debug("Loading remote URI:" + uri); FileContent fileContent; try { // set cookie if cookies set if (0 < this.cookies.size()) { FileSystemOptions fsOptions = new FileSystemOptions(); HttpFileSystemConfigBuilder.getInstance().setCookies(fsOptions, getCookies(uri)); fileContent = fileSystem.resolveFile(uri, fsOptions).getContent(); } else fileContent = fileSystem.resolveFile(uri).getContent(); // 2016-03-22 only pure http/https auto detect encoding if ("http".equalsIgnoreCase(uri.substring(0, 4))) { fileContent.getSize(); // pass a bug {@link https://issues.apache.org/jira/browse/VFS-427} remoteEncoding = fileContent.getContentInfo().getContentEncoding(); } if (null == remoteEncoding) remoteEncoding = "utf-8"; if (!"utf".equalsIgnoreCase(remoteEncoding.substring(0, 3))) { log.debug("remote content encoding: " + remoteEncoding); // force charset encoding if setRemoteEncoding set if (!"utf".equalsIgnoreCase(encoding.substring(0, 3))) { remoteEncoding = encoding; } else { // auto detecting encoding remoteEncoding = detectCharset(IOUtils.toByteArray(fileContent.getInputStream())); log.info("real encoding: " + remoteEncoding); } } // 2016-02-29 fixed remoteContent = IOUtils.toString(fileContent.getInputStream(), remoteEncoding); } catch (FileSystemException fse) { log.warn(fse.getMessage()); remoteContent = null; } catch (IOException ioe) { // return empty log.warn(ioe.getMessage()); remoteContent = null; } catch (StringIndexOutOfBoundsException stre) { log.warn("uri: " + uri); log.warn(stre.getMessage()); remoteContent = null; } clearCookies(); // any exception will return "null" return remoteContent; }
From source file:com.github.abola.crawler.CrawlerPack.java
/** * ?? Apache Common VFS ???/*from ww w . j a va 2s . c o m*/ * * ?? * @see <a href="https://commons.apache.org/proper/commons-vfs/filesystems.html">commons-vfs filesystems</a> */ public String getFromRemote(String uri) { // clear cache fileSystem.getFilesCache().close(); String remoteContent; String remoteEncoding = "utf-8"; log.debug("getFromRemote: Loading remote URI=" + uri); FileContent fileContent; try { FileSystemOptions fsOptions = new FileSystemOptions(); // set userAgent HttpFileSystemConfigBuilder.getInstance().setUserAgent(fsOptions, userAgent); // set cookie if cookies set if (0 < this.cookies.size()) { HttpFileSystemConfigBuilder.getInstance().setCookies(fsOptions, getCookies(uri)); } log.debug("getFromRemote: userAgent=" + userAgent); log.debug("getFromRemote: cookieSize=" + cookies.size()); log.debug("getFromRemote: cookies=" + cookies.toString()); fileContent = fileSystem.resolveFile(uri, fsOptions).getContent(); // 2016-03-22 only pure http/https auto detect encoding if ("http".equalsIgnoreCase(uri.substring(0, 4))) { fileContent.getSize(); // pass a bug {@link https://issues.apache.org/jira/browse/VFS-427} remoteEncoding = fileContent.getContentInfo().getContentEncoding(); } log.debug("getFromRemote: remoteEncoding=" + remoteEncoding + "(auto detect) "); // 2016-03-21 zip file getContentEncoding null if (null == remoteEncoding) remoteEncoding = "utf-8"; if (!"utf".equalsIgnoreCase(remoteEncoding.substring(0, 3))) { log.debug("getFromRemote: remote content encoding=" + remoteEncoding); // force charset encoding if setRemoteEncoding set if (!"utf".equalsIgnoreCase(encoding.substring(0, 3))) { remoteEncoding = encoding; } else { // auto detecting encoding remoteEncoding = detectCharset(IOUtils.toByteArray(fileContent.getInputStream())); log.debug("getFromRemote: real encoding=" + remoteEncoding); } } // ?? Apache VFS ?? // 2016-02-29 fixed remoteContent = IOUtils.toString(fileContent.getInputStream(), remoteEncoding); } catch (FileSystemException fse) { log.warn("getFromRemote: FileSystemException=" + fse.getMessage()); remoteContent = null; } catch (IOException ioe) { // return empty log.warn("getFromRemote: IOException=" + ioe.getMessage()); remoteContent = null; } catch (StringIndexOutOfBoundsException stre) { log.warn("getFromRemote: StringIndexOutOfBoundsException=" + stre.getMessage()); log.warn("getFromRemote: uri=" + uri); log.warn(stre.getMessage()); remoteContent = null; } clearCookies(); log.debug("getFromRemote: remoteContent=\n" + remoteContent); // any exception will return "null" return remoteContent; }
From source file:org.kalypso.commons.io.VFSUtilities.java
/** * This function will check the string for protocol, and if neccessary applys an proxy object to it. * * @param absoluteFile//from w ww . j ava2 s .c o m * The absolute file path to the file. It should be absolute, because this function was not testet against * relative files. * @return The file object. */ public static FileObject checkProxyFor(final String absoluteFile, final FileSystemManager fsManager) throws FileSystemException { final Proxy proxy = ProxyUtilities.getProxy(); KalypsoCommonsDebug.DEBUG.printf("Should use proxy: %s%n", String.valueOf(proxy.useProxy())); //$NON-NLS-1$ // TODO: VFS usually accepts file-pathes (without protocoll), but creating an url here will prohibit this. // TODO: handle file pathes differently if (proxy.useProxy() && !ProxyUtilities.isNonProxyHost(absoluteFile)) { final String proxyHost = proxy.getProxyHost(); final int proxyPort = proxy.getProxyPort(); KalypsoCommonsDebug.DEBUG.printf("Proxy host: %s%n", proxyHost); //$NON-NLS-1$ KalypsoCommonsDebug.DEBUG.printf("Proxy port: %s%n", String.valueOf(proxyPort)); //$NON-NLS-1$ /* Get the credentials. */ final String user = proxy.getUser(); final String password = proxy.getPassword(); final Pattern p = Pattern.compile("(.+)://.+"); //$NON-NLS-1$ final Matcher m = p.matcher(absoluteFile); if (m.find() == true) { KalypsoCommonsDebug.DEBUG.printf("File: %s%n", absoluteFile); //$NON-NLS-1$ KalypsoCommonsDebug.DEBUG.printf("Protocol: %s%n", m.group(1)); //$NON-NLS-1$ if (m.group(1).equals("webdav")) //$NON-NLS-1$ { WebdavFileSystemConfigBuilder.getInstance().setProxyHost(THE_WEBDAV_OPTIONS, proxyHost); WebdavFileSystemConfigBuilder.getInstance().setProxyPort(THE_WEBDAV_OPTIONS, proxyPort); /* If there are credentials given, set them. */ if (user != null && password != null) { final UserAuthenticator authenticator = new StaticUserAuthenticator(null, user, password); WebdavFileSystemConfigBuilder.getInstance().setProxyAuthenticator(THE_WEBDAV_OPTIONS, authenticator); } return fsManager.resolveFile(absoluteFile, THE_WEBDAV_OPTIONS); } else if (m.group(1).equals("http")) //$NON-NLS-1$ { HttpFileSystemConfigBuilder.getInstance().setProxyHost(THE_HTTP_OPTIONS, proxyHost); HttpFileSystemConfigBuilder.getInstance().setProxyPort(THE_HTTP_OPTIONS, proxyPort); /* If there are credentials given, set them. */ if (user != null && password != null) { final UserAuthenticator authenticator = new StaticUserAuthenticator(null, user, password); HttpFileSystemConfigBuilder.getInstance().setProxyAuthenticator(THE_HTTP_OPTIONS, authenticator); } return fsManager.resolveFile(absoluteFile, THE_HTTP_OPTIONS); } else if (m.group(1).equals("https")) //$NON-NLS-1$ { HttpFileSystemConfigBuilder.getInstance().setProxyHost(THE_HTTPS_OPTIONS, proxyHost); HttpFileSystemConfigBuilder.getInstance().setProxyPort(THE_HTTPS_OPTIONS, proxyPort); /* If there are credentials given, set them. */ if (user != null && password != null) { final UserAuthenticator authenticator = new StaticUserAuthenticator(null, user, password); HttpFileSystemConfigBuilder.getInstance().setProxyAuthenticator(THE_HTTPS_OPTIONS, authenticator); } return fsManager.resolveFile(absoluteFile, THE_HTTPS_OPTIONS); } } } return fsManager.resolveFile(absoluteFile); }