List of usage examples for org.apache.commons.net.ftp FTP BINARY_FILE_TYPE
int BINARY_FILE_TYPE
To view the source code for org.apache.commons.net.ftp FTP BINARY_FILE_TYPE.
Click Source Link
From source file:org.apache.hadoop.fs.ftp.FTPFileSystem.java
/** * Connect to the FTP server using configuration parameters * * // w w w. ja va 2 s .c o m * @return An FTPClient instance * @throws IOException */ private FTPClient connect() throws IOException { FTPClient client = null; Configuration conf = getConf(); String host = conf.get("fs.ftp.host"); int port = conf.getInt("fs.ftp.host.port", FTP.DEFAULT_PORT); String user = conf.get("fs.ftp.user." + host); String password = conf.get("fs.ftp.password." + host); client = new FTPClient(); client.connect(host, port); int reply = client.getReplyCode(); if (!FTPReply.isPositiveCompletion(reply)) { throw new IOException("Server - " + host + " refused connection on port - " + port); } else if (client.login(user, password)) { client.setFileTransferMode(FTP.BLOCK_TRANSFER_MODE); client.setFileType(FTP.BINARY_FILE_TYPE); client.setBufferSize(DEFAULT_BUFFER_SIZE); } else { throw new IOException("Login failed on server - " + host + ", port - " + port); } return client; }
From source file:org.apache.jmeter.protocol.ftp.sampler.FTPSampler.java
@Override public SampleResult sample(Entry e) { SampleResult res = new SampleResult(); res.setSuccessful(false); // Assume failure String remote = getRemoteFilename(); String local = getLocalFilename(); boolean binaryTransfer = isBinaryMode(); res.setSampleLabel(getName());// w w w .java 2 s . co m final String label = getLabel(); res.setSamplerData(label); try { res.setURL(new URL(label)); } catch (MalformedURLException e1) { log.warn("Cannot set URL: " + e1.getLocalizedMessage()); } InputStream input = null; OutputStream output = null; res.sampleStart(); FTPClient ftp = new FTPClient(); try { savedClient = ftp; final int port = getPortAsInt(); if (port > 0) { ftp.connect(getServer(), port); } else { ftp.connect(getServer()); } res.latencyEnd(); int reply = ftp.getReplyCode(); if (FTPReply.isPositiveCompletion(reply)) { if (ftp.login(getUsername(), getPassword())) { if (binaryTransfer) { ftp.setFileType(FTP.BINARY_FILE_TYPE); } ftp.enterLocalPassiveMode();// should probably come from the setup dialog boolean ftpOK = false; if (isUpload()) { String contents = getLocalFileContents(); if (contents.length() > 0) { byte[] bytes = contents.getBytes(); // TODO - charset? input = new ByteArrayInputStream(bytes); res.setBytes(bytes.length); } else { File infile = new File(local); res.setBytes((int) infile.length()); input = new BufferedInputStream(new FileInputStream(infile)); } ftpOK = ftp.storeFile(remote, input); } else { final boolean saveResponse = isSaveResponse(); ByteArrayOutputStream baos = null; // No need to close this OutputStream target = null; // No need to close this if (saveResponse) { baos = new ByteArrayOutputStream(); target = baos; } if (local.length() > 0) { output = new FileOutputStream(local); if (target == null) { target = output; } else { target = new TeeOutputStream(output, baos); } } if (target == null) { target = new NullOutputStream(); } input = ftp.retrieveFileStream(remote); if (input == null) {// Could not access file or other error res.setResponseCode(Integer.toString(ftp.getReplyCode())); res.setResponseMessage(ftp.getReplyString()); } else { long bytes = IOUtils.copy(input, target); ftpOK = bytes > 0; if (saveResponse && baos != null) { res.setResponseData(baos.toByteArray()); if (!binaryTransfer) { res.setDataType(SampleResult.TEXT); } } else { res.setBytes((int) bytes); } } } if (ftpOK) { res.setResponseCodeOK(); res.setResponseMessageOK(); res.setSuccessful(true); } else { res.setResponseCode(Integer.toString(ftp.getReplyCode())); res.setResponseMessage(ftp.getReplyString()); } } else { res.setResponseCode(Integer.toString(ftp.getReplyCode())); res.setResponseMessage(ftp.getReplyString()); } } else { res.setResponseCode("501"); // TODO res.setResponseMessage("Could not connect"); //res.setResponseCode(Integer.toString(ftp.getReplyCode())); res.setResponseMessage(ftp.getReplyString()); } } catch (IOException ex) { res.setResponseCode("000"); // TODO res.setResponseMessage(ex.toString()); } finally { savedClient = null; if (ftp.isConnected()) { try { ftp.logout(); } catch (IOException ignored) { } try { ftp.disconnect(); } catch (IOException ignored) { } } IOUtils.closeQuietly(input); IOUtils.closeQuietly(output); } res.sampleEnd(); return res; }
From source file:org.apache.maven.wagon.providers.ftp.FtpWagon.java
protected void openConnectionInternal() throws ConnectionException, AuthenticationException { AuthenticationInfo authInfo = getAuthenticationInfo(); if (authInfo == null) { throw new IllegalArgumentException("Authentication Credentials cannot be null for FTP protocol"); }/*from ww w .j a va2s .c o m*/ if (authInfo.getUserName() == null) { authInfo.setUserName(System.getProperty("user.name")); } String username = authInfo.getUserName(); String password = authInfo.getPassword(); if (username == null) { throw new AuthenticationException("Username not specified for repository " + getRepository().getId()); } if (password == null) { throw new AuthenticationException("Password not specified for repository " + getRepository().getId()); } String host = getRepository().getHost(); ftp = new FTPClient(); ftp.setDefaultTimeout(getTimeout()); ftp.setDataTimeout(getTimeout()); ftp.setControlEncoding(getControlEncoding()); ftp.addProtocolCommandListener(new PrintCommandListener(this)); try { if (getRepository().getPort() != WagonConstants.UNKNOWN_PORT) { ftp.connect(host, getRepository().getPort()); } else { ftp.connect(host); } // After connection attempt, you should check the reply code to // verify // success. int reply = ftp.getReplyCode(); if (!FTPReply.isPositiveCompletion(reply)) { ftp.disconnect(); throw new AuthenticationException("FTP server refused connection."); } } catch (IOException e) { if (ftp.isConnected()) { try { fireSessionError(e); ftp.disconnect(); } catch (IOException f) { // do nothing } } throw new AuthenticationException("Could not connect to server.", e); } try { if (!ftp.login(username, password)) { throw new AuthenticationException("Cannot login to remote system"); } fireSessionDebug("Remote system is " + ftp.getSystemName()); // Set to binary mode. ftp.setFileType(FTP.BINARY_FILE_TYPE); ftp.setListHiddenFiles(true); // Use passive mode as default because most of us are // behind firewalls these days. if (isPassiveMode()) { ftp.enterLocalPassiveMode(); } } catch (IOException e) { throw new ConnectionException("Cannot login to remote system", e); } }
From source file:org.apache.maven.wagon.providers.ftp.FtpWagonInclRmdir.java
protected void openConnectionInternal() throws ConnectionException, AuthenticationException { logger.info("Open ftp connection ..."); this.remoteRemovedDirectories = new LinkedList<String>(); AuthenticationInfo authInfo = getAuthenticationInfo(); if (authInfo == null) { throw new IllegalArgumentException("Authentication Credentials cannot be null for FTP protocol"); }/*from ww w. j a v a 2s .c o m*/ if (authInfo.getUserName() == null) { authInfo.setUserName(System.getProperty("user.name")); } String username = authInfo.getUserName(); String password = authInfo.getPassword(); if (username == null) { throw new AuthenticationException("Username not specified for repository " + getRepository().getId()); } if (password == null) { throw new AuthenticationException("Password not specified for repository " + getRepository().getId()); } String host = getRepository().getHost(); ftp = new FTPClient(); ftp.setDefaultTimeout(getTimeout()); ftp.setDataTimeout(getTimeout()); ftp.setControlEncoding(getControlEncoding()); ftp.addProtocolCommandListener(new PrintCommandListener(this)); try { if (getRepository().getPort() != WagonConstants.UNKNOWN_PORT) { ftp.connect(host, getRepository().getPort()); } else { ftp.connect(host); } // After connection attempt, you should check the reply code to // verify // success. int reply = ftp.getReplyCode(); if (!FTPReply.isPositiveCompletion(reply)) { ftp.disconnect(); throw new AuthenticationException("FTP server refused connection."); } } catch (IOException e) { if (ftp.isConnected()) { try { fireSessionError(e); ftp.disconnect(); } catch (IOException f) { // do nothing } } throw new AuthenticationException("Could not connect to server.", e); } try { if (!ftp.login(username, password)) { throw new AuthenticationException("Cannot login to remote system"); } fireSessionDebug("Remote system is " + ftp.getSystemName()); // Set to binary mode. ftp.setFileType(FTP.BINARY_FILE_TYPE); ftp.setListHiddenFiles(true); // Use passive mode as default because most of us are // behind firewalls these days. if (isPassiveMode()) { ftp.enterLocalPassiveMode(); } } catch (IOException e) { throw new ConnectionException("Cannot login to remote system", e); } }
From source file:org.apache.nutch.protocol.ftp.FtpResponse.java
public FtpResponse(URL url, CrawlDatum datum, Ftp ftp, Configuration conf) throws FtpException, IOException { this.orig = url.toString(); this.base = url.toString(); this.ftp = ftp; this.conf = conf; if (!"ftp".equals(url.getProtocol())) throw new FtpException("Not a ftp url:" + url); if (url.getPath() != url.getFile()) { if (Ftp.LOG.isWarnEnabled()) { Ftp.LOG.warn("url.getPath() != url.getFile(): " + url); }/*from w w w. j av a 2s . co m*/ } String path = "".equals(url.getPath()) ? "/" : url.getPath(); try { if (ftp.followTalk) { if (Ftp.LOG.isInfoEnabled()) { Ftp.LOG.info("fetching " + url); } } else { if (Ftp.LOG.isTraceEnabled()) { Ftp.LOG.trace("fetching " + url); } } InetAddress addr = InetAddress.getByName(url.getHost()); if (addr != null && conf.getBoolean("store.ip.address", false) == true) { headers.add("_ip_", addr.getHostAddress()); } // idled too long, remote server or ourselves may have timed out, // should start anew. if (ftp.client != null && ftp.keepConnection && ftp.renewalTime < System.currentTimeMillis()) { if (Ftp.LOG.isInfoEnabled()) { Ftp.LOG.info("delete client because idled too long"); } ftp.client = null; } // start anew if needed if (ftp.client == null) { if ((ftp.followTalk) && (Ftp.LOG.isInfoEnabled())) { Ftp.LOG.info("start client"); } // the real client ftp.client = new Client(); // when to renew, take the lesser // ftp.renewalTime = System.currentTimeMillis() // + ((ftp.timeout<ftp.serverTimeout) ? ftp.timeout : // ftp.serverTimeout); // timeout for control connection ftp.client.setDefaultTimeout(ftp.timeout); // timeout for data connection ftp.client.setDataTimeout(ftp.timeout); // follow ftp talk? if (ftp.followTalk) ftp.client.addProtocolCommandListener(new PrintCommandListener(Ftp.LOG)); } // quit from previous site if at a different site now if (ftp.client.isConnected()) { InetAddress remoteAddress = ftp.client.getRemoteAddress(); if (!addr.equals(remoteAddress)) { if ((ftp.followTalk) && (Ftp.LOG.isInfoEnabled())) { Ftp.LOG.info("disconnect from " + remoteAddress + " before connect to " + addr); } // quit from current site ftp.client.logout(); ftp.client.disconnect(); } } // connect to current site if needed if (!ftp.client.isConnected()) { if ((ftp.followTalk) && (Ftp.LOG.isInfoEnabled())) { Ftp.LOG.info("connect to " + addr); } ftp.client.connect(addr); if (!FTPReply.isPositiveCompletion(ftp.client.getReplyCode())) { ftp.client.disconnect(); if (Ftp.LOG.isWarnEnabled()) { Ftp.LOG.warn("ftp.client.connect() failed: " + addr + " " + ftp.client.getReplyString()); } this.code = 500; // http Internal Server Error return; } if ((ftp.followTalk) && (Ftp.LOG.isInfoEnabled())) { Ftp.LOG.info("log into " + addr); } if (!ftp.client.login(ftp.userName, ftp.passWord)) { // login failed. // please note that some server may return 421 immediately // after USER anonymous, thus ftp.client.login() won't return false, // but throw exception, which then will be handled by caller // (not dealt with here at all) . ftp.client.disconnect(); if (Ftp.LOG.isWarnEnabled()) { Ftp.LOG.warn("ftp.client.login() failed: " + addr); } this.code = 401; // http Unauthorized return; } // insist on binary file type if (!ftp.client.setFileType(FTP.BINARY_FILE_TYPE)) { ftp.client.logout(); ftp.client.disconnect(); if (Ftp.LOG.isWarnEnabled()) { Ftp.LOG.warn("ftp.client.setFileType() failed: " + addr); } this.code = 500; // http Internal Server Error return; } if ((ftp.followTalk) && (Ftp.LOG.isInfoEnabled())) { Ftp.LOG.info("set parser for " + addr); } // SYST is valid only after login try { ftp.parser = null; String parserKey = ftp.client.getSystemName(); // some server reports as UNKNOWN Type: L8, but in fact UNIX Type: L8 if (parserKey.startsWith("UNKNOWN Type: L8")) parserKey = "UNIX Type: L8"; ftp.parser = (new DefaultFTPFileEntryParserFactory()).createFileEntryParser(parserKey); } catch (FtpExceptionBadSystResponse e) { if (Ftp.LOG.isWarnEnabled()) { Ftp.LOG.warn("ftp.client.getSystemName() failed: " + addr + " " + e); } ftp.parser = null; } catch (ParserInitializationException e) { // ParserInitializationException is RuntimeException defined in // org.apache.commons.net.ftp.parser.ParserInitializationException if (Ftp.LOG.isWarnEnabled()) { Ftp.LOG.warn("createFileEntryParser() failed. " + addr + " " + e); } ftp.parser = null; } finally { if (ftp.parser == null) { // do not log as severe, otherwise // FetcherThread/RequestScheduler will abort if (Ftp.LOG.isWarnEnabled()) { Ftp.LOG.warn("ftp.parser is null: " + addr); } ftp.client.logout(); ftp.client.disconnect(); this.code = 500; // http Internal Server Error return; } } } else { if ((ftp.followTalk) && (Ftp.LOG.isInfoEnabled())) { Ftp.LOG.info("use existing connection"); } } this.content = null; path = java.net.URLDecoder.decode(path, "UTF-8"); if (path.endsWith("/")) { getDirAsHttpResponse(path, datum.getModifiedTime()); } else { getFileAsHttpResponse(path, datum.getModifiedTime()); } // reset next renewalTime, take the lesser if (ftp.client != null && ftp.keepConnection) { ftp.renewalTime = System.currentTimeMillis() + ((ftp.timeout < ftp.serverTimeout) ? ftp.timeout : ftp.serverTimeout); if ((ftp.followTalk) && (Ftp.LOG.isInfoEnabled())) { Ftp.LOG.info("reset renewalTime to " + HttpDateFormat.toString(ftp.renewalTime)); } } // getDirAsHttpResponse() or getFileAsHttpResponse() above // may have deleted ftp.client if (ftp.client != null && !ftp.keepConnection) { if ((ftp.followTalk) && (Ftp.LOG.isInfoEnabled())) { Ftp.LOG.info("disconnect from " + addr); } ftp.client.logout(); ftp.client.disconnect(); } } catch (Exception e) { if (Ftp.LOG.isWarnEnabled()) { Ftp.LOG.warn("Error: ", e); } // for any un-foreseen exception (run time exception or not), // do ultimate clean and leave ftp.client for garbage collection if ((ftp.followTalk) && (Ftp.LOG.isInfoEnabled())) { Ftp.LOG.info("delete client due to exception"); } ftp.client = null; // or do explicit garbage collection? // System.gc(); // can we be less dramatic, using the following instead? // probably unnecessary for our practical purpose here // try { // ftp.client.logout(); // ftp.client.disconnect(); // } throw new FtpException(e); // throw e; } }
From source file:org.apache.ofbiz.common.FtpServices.java
public static Map<String, Object> putFile(DispatchContext dctx, Map<String, ?> context) { Locale locale = (Locale) context.get("locale"); Debug.logInfo("[putFile] starting...", module); InputStream localFile = null; try {/* www.jav a 2 s. c o m*/ localFile = new FileInputStream((String) context.get("localFilename")); } catch (IOException ioe) { Debug.logError(ioe, "[putFile] Problem opening local file", module); return ServiceUtil .returnError(UtilProperties.getMessage(resource, "CommonFtpFileCannotBeOpen", locale)); } List<String> errorList = new LinkedList<String>(); FTPClient ftp = new FTPClient(); try { Integer defaultTimeout = (Integer) context.get("defaultTimeout"); if (UtilValidate.isNotEmpty(defaultTimeout)) { Debug.logInfo("[putFile] set default timeout to: " + defaultTimeout.intValue() + " milliseconds", module); ftp.setDefaultTimeout(defaultTimeout.intValue()); } Debug.logInfo("[putFile] connecting to: " + (String) context.get("hostname"), module); ftp.connect((String) context.get("hostname")); if (!FTPReply.isPositiveCompletion(ftp.getReplyCode())) { Debug.logInfo("[putFile] Server refused connection", module); errorList.add(UtilProperties.getMessage(resource, "CommonFtpConnectionRefused", locale)); } else { String username = (String) context.get("username"); String password = (String) context.get("password"); Debug.logInfo("[putFile] logging in: username=" + username + ", password=" + password, module); if (!ftp.login(username, password)) { Debug.logInfo("[putFile] login failed", module); errorList.add(UtilProperties.getMessage(resource, "CommonFtpLoginFailure", UtilMisc.toMap("username", username, "password", password), locale)); } else { Boolean binaryTransfer = (Boolean) context.get("binaryTransfer"); boolean binary = (binaryTransfer == null) ? false : binaryTransfer.booleanValue(); if (binary) { ftp.setFileType(FTP.BINARY_FILE_TYPE); } Boolean passiveMode = (Boolean) context.get("passiveMode"); boolean passive = (passiveMode == null) ? true : passiveMode.booleanValue(); if (passive) { ftp.enterLocalPassiveMode(); } Debug.logInfo("[putFile] storing local file remotely as: " + context.get("remoteFilename"), module); if (!ftp.storeFile((String) context.get("remoteFilename"), localFile)) { Debug.logInfo("[putFile] store was unsuccessful", module); errorList.add(UtilProperties.getMessage(resource, "CommonFtpFileNotSentSuccesfully", UtilMisc.toMap("replyString", ftp.getReplyString()), locale)); } else { Debug.logInfo("[putFile] store was successful", module); List<String> siteCommands = checkList(context.get("siteCommands"), String.class); if (siteCommands != null) { for (String command : siteCommands) { Debug.logInfo("[putFile] sending SITE command: " + command, module); if (!ftp.sendSiteCommand(command)) { errorList.add(UtilProperties.getMessage(resource, "CommonFtpSiteCommandFailed", UtilMisc.toMap("command", command, "replyString", ftp.getReplyString()), locale)); } } } } } ftp.logout(); } } catch (IOException ioe) { Debug.logWarning(ioe, "[putFile] caught exception: " + ioe.getMessage(), module); errorList.add(UtilProperties.getMessage(resource, "CommonFtpProblemWithTransfer", UtilMisc.toMap("errorString", ioe.getMessage()), locale)); } finally { try { if (ftp.isConnected()) { ftp.disconnect(); } } catch (Exception e) { Debug.logWarning(e, "[putFile] Problem with FTP disconnect: ", module); } try { localFile.close(); } catch (Exception e) { Debug.logWarning(e, "[putFile] Problem closing local file: ", module); } } if (errorList.size() > 0) { Debug.logError("[putFile] The following error(s) (" + errorList.size() + ") occurred: " + errorList, module); return ServiceUtil.returnError(errorList); } Debug.logInfo("[putFile] finished successfully", module); return ServiceUtil.returnSuccess(); }
From source file:org.apache.ofbiz.common.FtpServices.java
public static Map<String, Object> getFile(DispatchContext dctx, Map<String, ?> context) { Locale locale = (Locale) context.get("locale"); String localFilename = (String) context.get("localFilename"); OutputStream localFile = null; try {/* ww w . ja va 2 s . c om*/ localFile = new FileOutputStream(localFilename); } catch (IOException ioe) { Debug.logError(ioe, "[getFile] Problem opening local file", module); return ServiceUtil .returnError(UtilProperties.getMessage(resource, "CommonFtpFileCannotBeOpen", locale)); } List<String> errorList = new LinkedList<String>(); FTPClient ftp = new FTPClient(); try { Integer defaultTimeout = (Integer) context.get("defaultTimeout"); if (UtilValidate.isNotEmpty(defaultTimeout)) { Debug.logInfo("[getFile] Set default timeout to: " + defaultTimeout.intValue() + " milliseconds", module); ftp.setDefaultTimeout(defaultTimeout.intValue()); } ftp.connect((String) context.get("hostname")); if (!FTPReply.isPositiveCompletion(ftp.getReplyCode())) { errorList.add(UtilProperties.getMessage(resource, "CommonFtpConnectionRefused", locale)); } else { String username = (String) context.get("username"); String password = (String) context.get("password"); if (!ftp.login(username, password)) { errorList.add(UtilProperties.getMessage(resource, "CommonFtpLoginFailure", UtilMisc.toMap("username", username, "password", password), locale)); } else { Boolean binaryTransfer = (Boolean) context.get("binaryTransfer"); boolean binary = (binaryTransfer == null) ? false : binaryTransfer.booleanValue(); if (binary) { ftp.setFileType(FTP.BINARY_FILE_TYPE); } Boolean passiveMode = (Boolean) context.get("passiveMode"); boolean passive = (passiveMode == null) ? false : passiveMode.booleanValue(); if (passive) { ftp.enterLocalPassiveMode(); } if (!ftp.retrieveFile((String) context.get("remoteFilename"), localFile)) { errorList.add(UtilProperties.getMessage(resource, "CommonFtpFileNotSentSuccesfully", UtilMisc.toMap("replyString", ftp.getReplyString()), locale)); } } ftp.logout(); } } catch (IOException ioe) { Debug.logWarning(ioe, "[getFile] caught exception: " + ioe.getMessage(), module); errorList.add(UtilProperties.getMessage(resource, "CommonFtpProblemWithTransfer", UtilMisc.toMap("errorString", ioe.getMessage()), locale)); } finally { try { if (ftp.isConnected()) { ftp.disconnect(); } } catch (Exception e) { Debug.logWarning(e, "[getFile] Problem with FTP disconnect: ", module); } try { localFile.close(); } catch (Exception e) { Debug.logWarning(e, "[getFile] Problem closing local file: ", module); } } if (errorList.size() > 0) { Debug.logError("[getFile] The following error(s) (" + errorList.size() + ") occurred: " + errorList, module); return ServiceUtil.returnError(errorList); } return ServiceUtil.returnSuccess(); }
From source file:org.archive.crawler.fetcher.FetchFTP.java
/** * Fetches a document from an FTP server. * //from w ww . ja v a 2 s. co m * @param curi the URI of the document to fetch * @param client the FTPClient to use for the fetch * @param recorder the recorder to preserve the document in * @throws IOException if a network or protocol error occurs * @throws InterruptedException if the thread is interrupted */ private void fetch(CrawlURI curi, ClientFTP client, HttpRecorder recorder) throws IOException, InterruptedException { // Connect to the FTP server. UURI uuri = curi.getUURI(); int port = uuri.getPort(); if (port == -1) { port = 21; } client.connect(uuri.getHost(), port); // Authenticate. String[] auth = getAuth(curi); client.login(auth[0], auth[1]); // The given resource may or may not be a directory. // To figure out which is which, execute a CD command to // the UURI's path. If CD works, it's a directory. boolean isDirectory = client.changeWorkingDirectory(uuri.getPath()); // Get a data socket. This will either be the result of a NLST // command for a directory, or a RETR command for a file. int command; String path; if (isDirectory) { curi.addAnnotation("ftpDirectoryList"); command = FTPCommand.NLST; client.setFileType(FTP.ASCII_FILE_TYPE); path = "."; } else { command = FTPCommand.RETR; client.setFileType(FTP.BINARY_FILE_TYPE); path = uuri.getPath(); } client.enterLocalPassiveMode(); Socket socket = null; try { socket = client.openDataConnection(command, path); // if "227 Entering Passive Mode" these will get reset later curi.setFetchStatus(client.getReplyCode()); curi.putString(A_FTP_FETCH_STATUS, client.getReplyStrings()[0]); } catch (IOException e) { // try it again, see AbstractFrontier.needsRetrying() curi.setFetchStatus(S_CONNECT_LOST); } // Save the streams in the CURI, where downstream processors // expect to find them. if (socket != null) { // Shall we get a digest on the content downloaded? boolean digestContent = ((Boolean) getUncheckedAttribute(curi, FetchHTTP.ATTR_DIGEST_CONTENT)) .booleanValue(); String algorithm = null; if (digestContent) { algorithm = ((String) getUncheckedAttribute(curi, FetchHTTP.ATTR_DIGEST_ALGORITHM)); recorder.getRecordedInput().setDigest(algorithm); recorder.getRecordedInput().startDigest(); } else { // clear recorder.getRecordedInput().setDigest((MessageDigest) null); } try { saveToRecorder(curi, socket, recorder); } finally { recorder.close(); client.closeDataConnection(); // does socket.close() curi.setContentSize(recorder.getRecordedInput().getSize()); // "226 Transfer complete." client.getReply(); curi.setFetchStatus(client.getReplyCode()); curi.putString(A_FTP_FETCH_STATUS, client.getReplyStrings()[0]); if (isDirectory) { curi.setContentType("text/plain"); } else { curi.setContentType("application/octet-stream"); } if (logger.isLoggable(Level.INFO)) { logger.info("read " + recorder.getRecordedInput().getSize() + " bytes from ftp data socket"); } if (digestContent) { curi.setContentDigest(algorithm, recorder.getRecordedInput().getDigestValue()); } } if (isDirectory) { extract(curi, recorder); } } addParent(curi); }
From source file:org.archive.modules.fetcher.FetchFTP.java
/** * Fetches a document from an FTP server. * /*w ww.j ava2 s. co m*/ * @param curi the URI of the document to fetch * @param client the FTPClient to use for the fetch * @param recorder the recorder to preserve the document in * @throws IOException if a network or protocol error occurs * @throws InterruptedException if the thread is interrupted */ private void fetch(CrawlURI curi, ClientFTP client, Recorder recorder) throws IOException, InterruptedException { // Connect to the FTP server. UURI uuri = curi.getUURI(); int port = uuri.getPort(); if (port == -1) { port = 21; } if (socketFactory == null) { socketFactory = new SocketFactoryWithTimeout(); } socketFactory.setConnectTimeoutMs(getSoTimeoutMs()); client.setSocketFactory(socketFactory); client.setConnectTimeout(getSoTimeoutMs()); client.setDefaultTimeout(getSoTimeoutMs()); client.setDataTimeout(getSoTimeoutMs()); client.connect(uuri.getHost(), port); client.setSoTimeout(getSoTimeoutMs()); // must be after connect() // Authenticate. String[] auth = getAuth(curi); client.login(auth[0], auth[1]); // The given resource may or may not be a directory. // To figure out which is which, execute a CD command to // the UURI's path. If CD works, it's a directory. boolean isDirectory = client.changeWorkingDirectory(uuri.getPath()); // Get a data socket. This will either be the result of a NLST // command for a directory, or a RETR command for a file. int command; String path; if (isDirectory) { curi.getAnnotations().add("ftpDirectoryList"); command = FTPCommand.NLST; client.setFileType(FTP.ASCII_FILE_TYPE); path = "."; } else { command = FTPCommand.RETR; client.setFileType(FTP.BINARY_FILE_TYPE); path = uuri.getPath(); } client.enterLocalPassiveMode(); Socket socket = null; try { socket = client.openDataConnection(command, path); // if "227 Entering Passive Mode" these will get reset later curi.setFetchStatus(client.getReplyCode()); curi.getData().put(A_FTP_FETCH_STATUS, client.getReplyStrings()[0]); } catch (IOException e) { // try it again, see AbstractFrontier.needsRetrying() curi.setFetchStatus(FetchStatusCodes.S_CONNECT_LOST); } // Save the streams in the CURI, where downstream processors // expect to find them. if (socket != null) { if (socket.getSoTimeout() != getSoTimeoutMs()) { logger.warning("data socket timeout " + socket.getSoTimeout() + "ms is not expected value " + getSoTimeoutMs() + "ms"); } // Shall we get a digest on the content downloaded? boolean digestContent = getDigestContent(); String algorithm = null; if (digestContent) { algorithm = getDigestAlgorithm(); recorder.getRecordedInput().setDigest(algorithm); recorder.getRecordedInput().startDigest(); } else { // clear recorder.getRecordedInput().setDigest((MessageDigest) null); } try { saveToRecorder(curi, socket, recorder); } finally { recorder.close(); client.closeDataConnection(); // does socket.close() curi.setContentSize(recorder.getRecordedInput().getSize()); // "226 Transfer complete." client.getReply(); curi.setFetchStatus(client.getReplyCode()); curi.getData().put(A_FTP_FETCH_STATUS, client.getReplyStrings()[0]); if (isDirectory) { curi.setContentType("text/plain"); } else { curi.setContentType("application/octet-stream"); } if (logger.isLoggable(Level.FINE)) { logger.fine("read " + recorder.getRecordedInput().getSize() + " bytes from ftp data socket"); } if (digestContent) { curi.setContentDigest(algorithm, recorder.getRecordedInput().getDigestValue()); } } if (isDirectory) { extract(curi, recorder); } } else { // no data - without this, content size is -1 curi.setContentSize(0); } addParent(curi); }
From source file:org.codelibs.fess.crawler.client.ftp.FtpClient.java
protected FTPClient getClient(final FtpInfo info) throws IOException { FTPClient ftpClient = ftpClientQueue.poll(); if (ftpClient != null) { if (ftpClient.isAvailable()) { return ftpClient; }// ww w.j a v a 2 s . co m try { ftpClient.disconnect(); } catch (final Exception e) { logger.debug("Failed to disconnect " + info.toUrl(), e); } } try { ftpClient = new FTPClient(); if (activeExternalHost != null) { ftpClient.setActiveExternalIPAddress(activeExternalHost); } if (passiveLocalHost != null) { ftpClient.setPassiveLocalIPAddress(passiveLocalHost); } if (reportActiveExternalHost != null) { ftpClient.setReportActiveExternalIPAddress(reportActiveExternalHost); } if (activeMinPort != -1 && activeMaxPort != -1) { ftpClient.setActivePortRange(activeMinPort, activeMaxPort); } ftpClient.setAutodetectUTF8(autodetectEncoding); ftpClient.setConnectTimeout(connectTimeout); ftpClient.setDataTimeout(dataTimeout); ftpClient.setControlEncoding(controlEncoding); ftpClient.setBufferSize(bufferSize); ftpClient.setPassiveNatWorkaround(passiveNatWorkaround); ftpClient.setUseEPSVwithIPv4(useEPSVwithIPv4); ftpClient.configure(ftpClientConfig); ftpClient.connect(info.getHost(), info.getPort()); validateRequest(ftpClient); final FtpAuthentication auth = ftpAuthenticationHolder.get(info.toUrl()); if (auth != null) { if (!ftpClient.login(auth.getUsername(), auth.getPassword())) { throw new CrawlerLoginFailureException( "Login Failure: " + auth.getUsername() + " for " + info.toUrl()); } } ftpClient.setFileType(FTP.BINARY_FILE_TYPE); return ftpClient; } catch (final IOException e) { if (ftpClient != null) { try { ftpClient.disconnect(); } catch (final Exception e1) { logger.debug("Failed to disconnect " + info.toUrl(), e); } } throw e; } }