src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java - nutch - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.nutch.protocol.ftp;

 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;

 import java.net.InetAddress;
 import java.net.Socket;

 import java.util.List;

 import org.apache.commons.net.MalformedServerReplyException;

 import org.apache.commons.net.ftp.FTP;
 import org.apache.commons.net.ftp.FTPCommand;
 import org.apache.commons.net.ftp.FTPFile;
 import org.apache.commons.net.ftp.FTPFileEntryParser;
 import org.apache.commons.net.ftp.FTPReply;

 import org.apache.commons.net.ftp.FTPConnectionClosedException;

 /***********************************************
  * Client.java encapsulates functionalities necessary for nutch to get dir list
  * and retrieve file from an FTP server. This class takes care of all low level
  * details of interacting with an FTP server and provides a convenient higher
  * level interface.
  *
  * Modified from FtpClient.java in apache commons-net.
  *
  * Notes by John Xing: ftp server implementations are hardly uniform and none
  * seems to follow RFCs whole-heartedly. We have no choice, but assume common
  * denominator as following: (1) Use stream mode for data transfer. Block mode
  * will be better for multiple file downloading and partial file downloading.
  * However not every ftpd has block mode support. (2) Use passive mode for data
  * connection. So Nutch will work if we run behind firewall. (3) Data connection
  * is opened/closed per ftp command for the reasons listed in (1). There are ftp
  * servers out there, when partial downloading is enforced by closing data
  * channel socket on our client side, the server side immediately closes control
  * channel (socket). Our codes deal with such a bad behavior. (4) LIST is used
  * to obtain remote file attributes if possible. MDTM and SIZE would be nice, but
  * not as ubiquitously implemented as LIST. (5) Avoid using ABOR in single
  * thread? Do not use it at all.
  *
  * About exceptions: Some specific exceptions are re-thrown as one of
  * FtpException*.java In fact, each function throws FtpException*.java or pass
  * IOException.
  *
  * @author John Xing
  ***********************************************/

 public class Client extends FTP {
   private int __dataTimeout;
   private int __passivePort;
   private String __passiveHost;
   // private int __fileType, __fileFormat;
   private boolean __remoteVerificationEnabled;
   // private FTPFileEntryParser __entryParser;
   private String __systemName;

   /** Public default constructor */
   public Client() {
     __initDefaults();
     __dataTimeout = -1;
     __remoteVerificationEnabled = true;
   }

   // defaults when initialize
   private void __initDefaults() {
     __passiveHost = null;
     __passivePort = -1;
     __systemName = null;
     // __fileType = FTP.ASCII_FILE_TYPE;
     // __fileFormat = FTP.NON_PRINT_TEXT_FORMAT;
     // __entryParser = null;
   }

   // parse reply for pass()
   private void __parsePassiveModeReply(String reply)
       throws MalformedServerReplyException {
     int i, index, lastIndex;
     String octet1, octet2;
     StringBuffer host;

     reply = reply.substring(reply.indexOf('(') + 1, reply.indexOf(')')).trim();

     host = new StringBuffer(24);
     lastIndex = 0;
     index = reply.indexOf(',');
     host.append(reply.substring(lastIndex, index));

     for (i = 0; i < 3; i++) {
       host.append('.');
       lastIndex = index + 1;
       index = reply.indexOf(',', lastIndex);
       host.append(reply.substring(lastIndex, index));
     }

     lastIndex = index + 1;
     index = reply.indexOf(',', lastIndex);

     octet1 = reply.substring(lastIndex, index);
     octet2 = reply.substring(index + 1);

     // index and lastIndex now used as temporaries
     try {
       index = Integer.parseInt(octet1);
       lastIndex = Integer.parseInt(octet2);
     } catch (NumberFormatException e) {
       throw new MalformedServerReplyException(
           "Could not parse passive host information.\nServer Reply: " + reply);
     }

     index <<= 8;
     index |= lastIndex;

     __passiveHost = host.toString();
     __passivePort = index;
   }

   /**
    * open a passive data connection socket
    *
    * @param command
    * @param arg
    * @return
    * @throws IOException
    * @throws FtpExceptionCanNotHaveDataConnection
    */
   protected Socket __openPassiveDataConnection(int command, String arg)
       throws IOException, FtpExceptionCanNotHaveDataConnection {
     Socket socket;

     // // 20040317, xing, accommodate ill-behaved servers, see below
     // int port_previous = __passivePort;

     if (pasv() != FTPReply.ENTERING_PASSIVE_MODE)
       throw new FtpExceptionCanNotHaveDataConnection("pasv() failed. "
           + getReplyString());

     try {
       __parsePassiveModeReply(getReplyStrings()[0]);
     } catch (MalformedServerReplyException e) {
       throw new FtpExceptionCanNotHaveDataConnection(e.getMessage());
     }

     // // 20040317, xing, accommodate ill-behaved servers, see above
     // int count = 0;
     // System.err.println("__passivePort "+__passivePort);
     // System.err.println("port_previous "+port_previous);
     // while (__passivePort == port_previous) {
     // // just quit if too many tries. make it an exception here?
     // if (count++ > 10)
     // return null;
     // // slow down further for each new try
     // Thread.sleep(500*count);
     // if (pasv() != FTPReply.ENTERING_PASSIVE_MODE)
     // throw new FtpExceptionCanNotHaveDataConnection(
     // "pasv() failed. " + getReplyString());
     // //return null;
     // try {
     // __parsePassiveModeReply(getReplyStrings()[0]);
     // } catch (MalformedServerReplyException e) {
     // throw new FtpExceptionCanNotHaveDataConnection(e.getMessage());
     // }
     // }

     socket = _socketFactory_.createSocket(__passiveHost, __passivePort);

     if (!FTPReply.isPositivePreliminary(sendCommand(command, arg))) {
       socket.close();
       return null;
     }

     if (__remoteVerificationEnabled && !verifyRemote(socket)) {
       InetAddress host1, host2;

       host1 = socket.getInetAddress();
       host2 = getRemoteAddress();

       socket.close();

       // our precaution
       throw new FtpExceptionCanNotHaveDataConnection(
           "Host attempting data connection " + host1.getHostAddress()
               + " is not same as server " + host2.getHostAddress()
               + " So we intentionally close it for security precaution.");
     }

     if (__dataTimeout >= 0)
       socket.setSoTimeout(__dataTimeout);

     return socket;
   }

   /***
    * Sets the timeout in milliseconds to use for data connection. set
    * immediately after opening the data connection.
    ***/
   public void setDataTimeout(int timeout) {
     __dataTimeout = timeout;
   }

   /***
    * Closes the connection to the FTP server and restores connection parameters
    * to the default values.
    * <p>
    *
    * @exception IOException
    *              If an error occurs while disconnecting.
    ***/
   public void disconnect() throws IOException {
     __initDefaults();
     super.disconnect();
     // no worry for data connection, since we always close it
     // in every ftp command that invloves data connection
   }

   /***
    * Enable or disable verification that the remote host taking part of a data
    * connection is the same as the host to which the control connection is
    * attached. The default is for verification to be enabled. You may set this
    * value at any time, whether the FTPClient is currently connected or not.
    * <p>
    *
    * @param enable
    *          True to enable verification, false to disable verification.
    ***/
   public void setRemoteVerificationEnabled(boolean enable) {
     __remoteVerificationEnabled = enable;
   }

   /***
    * Return whether or not verification of the remote host participating in data
    * connections is enabled. The default behavior is for verification to be
    * enabled.
    * <p>
    *
    * @return True if verification is enabled, false if not.
    ***/
   public boolean isRemoteVerificationEnabled() {
     return __remoteVerificationEnabled;
   }

   /***
    * Login to the FTP server using the provided username and password.
    * <p>
    *
    * @param username
    *          The username to login under.
    * @param password
    *          The password to use.
    * @return True if successfully completed, false if not.
    * @exception FTPConnectionClosedException
    *              If the FTP server prematurely closes the connection as a
    *              result of the client being idle or some other reason causing
    *              the server to send FTP reply code 421. This exception may be
    *              caught either as an IOException or independently as itself.
    * @exception IOException
    *              If an I/O error occurs while either sending a command to the
    *              server or receiving a reply from the server.
    ***/
   public boolean login(String username, String password) throws IOException {
     user(username);

     if (FTPReply.isPositiveCompletion(getReplyCode()))
       return true;

     // If we get here, we either have an error code, or an intermmediate
     // reply requesting password.
     if (!FTPReply.isPositiveIntermediate(getReplyCode()))
       return false;

     return FTPReply.isPositiveCompletion(pass(password));
   }

   /***
    * Logout of the FTP server by sending the QUIT command.
    * <p>
    *
    * @return True if successfully completed, false if not.
    * @exception FTPConnectionClosedException
    *              If the FTP server prematurely closes the connection as a
    *              result of the client being idle or some other reason causing
    *              the server to send FTP reply code 421. This exception may be
    *              caught either as an IOException or independently as itself.
    * @exception IOException
    *              If an I/O error occurs while either sending a command to the
    *              server or receiving a reply from the server.
    ***/
   public boolean logout() throws IOException {
     return FTPReply.isPositiveCompletion(quit());
   }

   /**
    * retrieve list reply for path
    *
    * @param path
    * @param entries
    * @param limit
    * @param parser
    * @throws IOException
    * @throws FtpExceptionCanNotHaveDataConnection
    * @throws FtpExceptionUnknownForcedDataClose
    * @throws FtpExceptionControlClosedByForcedDataClose
    */
   public void retrieveList(String path, List<FTPFile> entries, int limit,
       FTPFileEntryParser parser) throws IOException,
       FtpExceptionCanNotHaveDataConnection, FtpExceptionUnknownForcedDataClose,
       FtpExceptionControlClosedByForcedDataClose {
     Socket socket = __openPassiveDataConnection(FTPCommand.LIST, path);

     if (socket == null)
       throw new FtpExceptionCanNotHaveDataConnection("LIST "
           + ((path == null) ? "" : path));

     BufferedReader reader = new BufferedReader(new InputStreamReader(
         socket.getInputStream()));

     // force-close data channel socket, when download limit is reached
     // boolean mandatory_close = false;

     // List entries = new LinkedList();
     int count = 0;
     String line = parser.readNextEntry(reader);
     while (line != null) {
       FTPFile ftpFile = parser.parseFTPEntry(line);
       // skip non-formatted lines
       if (ftpFile == null) {
         line = parser.readNextEntry(reader);
         continue;
       }
       entries.add(ftpFile);
       count += line.length();
       // impose download limit if limit >= 0, otherwise no limit
       // here, cut off is up to the line when total bytes is just over limit
       if (limit >= 0 && count > limit) {
         // mandatory_close = true;
         break;
       }
       line = parser.readNextEntry(reader);
     }

     // if (mandatory_close)
     // you always close here, no matter mandatory_close or not.
     // however different ftp servers respond differently, see below.
     socket.close();

     // scenarios:
     // (1) mandatory_close is false, download limit not reached
     // no special care here
     // (2) mandatory_close is true, download limit is reached
     // different servers have different reply codes:

     try {
       int reply = getReply();
       if (!_notBadReply(reply))
         throw new FtpExceptionUnknownForcedDataClose(getReplyString());
     } catch (FTPConnectionClosedException e) {
       // some ftp servers will close control channel if data channel socket
       // is closed by our end before all data has been read out. Check:
       // tux414.q-tam.hp.com FTP server (hp.com version whp02)
       // so must catch FTPConnectionClosedException thrown by getReply() above
       // disconnect();
       throw new FtpExceptionControlClosedByForcedDataClose(e.getMessage());
     }

   }

   /**
    * retrieve file for path
    *
    * @param path
    * @param os
    * @param limit
    * @throws IOException
    * @throws FtpExceptionCanNotHaveDataConnection
    * @throws FtpExceptionUnknownForcedDataClose
    * @throws FtpExceptionControlClosedByForcedDataClose
    */
   public void retrieveFile(String path, OutputStream os, int limit)
       throws IOException, FtpExceptionCanNotHaveDataConnection,
       FtpExceptionUnknownForcedDataClose,
       FtpExceptionControlClosedByForcedDataClose {

     Socket socket = __openPassiveDataConnection(FTPCommand.RETR, path);

     if (socket == null)
       throw new FtpExceptionCanNotHaveDataConnection("RETR "
           + ((path == null) ? "" : path));

     InputStream input = socket.getInputStream();

     // 20040318, xing, treat everything as BINARY_FILE_TYPE for now
     // do we ever need ASCII_FILE_TYPE?
     // if (__fileType == ASCII_FILE_TYPE)
     // input = new FromNetASCIIInputStream(input);

     // fixme, should we instruct server here for binary file type?

     // force-close data channel socket
     // boolean mandatory_close = false;

     int len;
     int count = 0;
     byte[] buf = new byte[org.apache.commons.net.io.Util.DEFAULT_COPY_BUFFER_SIZE];
     while ((len = input.read(buf, 0, buf.length)) != -1) {
       count += len;
       // impose download limit if limit >= 0, otherwise no limit
       // here, cut off is exactly of limit bytes
       if (limit >= 0 && count > limit) {
         os.write(buf, 0, len - (count - limit));
         // mandatory_close = true;
         break;
       }
       os.write(buf, 0, len);
       os.flush();
     }

     // if (mandatory_close)
     // you always close here, no matter mandatory_close or not.
     // however different ftp servers respond differently, see below.
     socket.close();

     // scenarios:
     // (1) mandatory_close is false, download limit not reached
     // no special care here
     // (2) mandatory_close is true, download limit is reached
     // different servers have different reply codes:

     // do not need this
     // sendCommand("ABOR");

     try {
       int reply = getReply();
       if (!_notBadReply(reply))
         throw new FtpExceptionUnknownForcedDataClose(getReplyString());
     } catch (FTPConnectionClosedException e) {
       // some ftp servers will close control channel if data channel socket
       // is closed by our end before all data has been read out. Check:
       // tux414.q-tam.hp.com FTP server (hp.com version whp02)
       // so must catch FTPConnectionClosedException thrown by getReply() above
       // disconnect();
       throw new FtpExceptionControlClosedByForcedDataClose(e.getMessage());
     }

   }

   /**
    * reply check after closing data connection
    *
    * @param reply
    * @return
    */
   private boolean _notBadReply(int reply) {

     if (FTPReply.isPositiveCompletion(reply)) {
       // do nothing
     } else if (reply == 426) { // FTPReply.TRANSFER_ABORTED
       // some ftp servers reply 426, e.g.,
       // foggy FTP server (Version wu-2.6.2(2)
       // there is second reply witing? no!
       // getReply();
     } else if (reply == 450) { // FTPReply.FILE_ACTION_NOT_TAKEN
       // some ftp servers reply 450, e.g.,
       // ProFTPD [ftp.kernel.org]
       // there is second reply witing? no!
       // getReply();
     } else if (reply == 451) { // FTPReply.ACTION_ABORTED
       // some ftp servers reply 451, e.g.,
       // ProFTPD [ftp.kernel.org]
       // there is second reply witing? no!
       // getReply();
     } else {
       // what other kind of ftp server out there?
       return false;
     }

     return true;
   }

   /***
    * Sets the file type to be transferred. This should be one of
    * <code> FTP.ASCII_FILE_TYPE </code>, <code> FTP.IMAGE_FILE_TYPE </code>,
    * etc. The file type only needs to be set when you want to change the type.
    * After changing it, the new type stays in effect until you change it again.
    * The default file type is <code> FTP.ASCII_FILE_TYPE </code> if this method
    * is never called.
    * <p>
    *
    * @param fileType
    *          The <code> _FILE_TYPE </code> constant indcating the type of file.
    * @return True if successfully completed, false if not.
    * @exception FTPConnectionClosedException
    *              If the FTP server prematurely closes the connection as a
    *              result of the client being idle or some other reason causing
    *              the server to send FTP reply code 421. This exception may be
    *              caught either as an IOException or independently as itself.
    * @exception IOException
    *              If an I/O error occurs while either sending a command to the
    *              server or receiving a reply from the server.
    ***/
   public boolean setFileType(int fileType) throws IOException {
     if (FTPReply.isPositiveCompletion(type(fileType))) {
       /*
        * __fileType = fileType; __fileFormat = FTP.NON_PRINT_TEXT_FORMAT;
        */
       return true;
     }
     return false;
   }

   /***
    * Fetches the system type name from the server and returns the string. This
    * value is cached for the duration of the connection after the first call to
    * this method. In other words, only the first time that you invoke this
    * method will it issue a SYST command to the FTP server. FTPClient will
    * remember the value and return the cached value until a call to disconnect.
    * <p>
    *
    * @return The system type name obtained from the server. null if the
    *         information could not be obtained.
    * @exception FTPConnectionClosedException
    *              If the FTP server prematurely closes the connection as a
    *              result of the client being idle or some other reason causing
    *              the server to send FTP reply code 421. This exception may be
    *              caught either as an IOException or independently as itself.
    * @exception IOException
    *              If an I/O error occurs while either sending a command to the
    *              server or receiving a reply from the server.
    ***/
   public String getSystemName() throws IOException, FtpExceptionBadSystResponse {
     // if (syst() == FTPReply.NAME_SYSTEM_TYPE)
     // Technically, we should expect a NAME_SYSTEM_TYPE response, but
     // in practice FTP servers deviate, so we soften the condition to
     // a positive completion.
     if (__systemName == null && FTPReply.isPositiveCompletion(syst())) {
       __systemName = (getReplyStrings()[0]).substring(4);
     } else {
       throw new FtpExceptionBadSystResponse("Bad response of SYST: "
           + getReplyString());
     }

     return __systemName;
   }

   /***
    * Sends a NOOP command to the FTP server. This is useful for preventing
    * server timeouts.
    * <p>
    *
    * @return True if successfully completed, false if not.
    * @exception FTPConnectionClosedException
    *              If the FTP server prematurely closes the connection as a
    *              result of the client being idle or some other reason causing
    *              the server to send FTP reply code 421. This exception may be
    *              caught either as an IOException or independently as itself.
    * @exception IOException
    *              If an I/O error occurs while either sending a command to the
    *              server or receiving a reply from the server.
    ***/
   public boolean sendNoOp() throws IOException {
     return FTPReply.isPositiveCompletion(noop());
   }

   // client.stat(path);
   // client.sendCommand("STAT");
   // client.sendCommand("STAT",path);
   // client.sendCommand("MDTM",path);
   // client.sendCommand("SIZE",path);
   // client.sendCommand("HELP","SITE");
   // client.sendCommand("SYST");
   // client.setRestartOffset(120);

 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.nutch.protocol.ftp;

	import java.io.BufferedReader;
	import java.io.IOException;
	import java.io.InputStream;
	import java.io.InputStreamReader;
	import java.io.OutputStream;

	import java.net.InetAddress;
	import java.net.Socket;

	import java.util.List;

	import org.apache.commons.net.MalformedServerReplyException;

	import org.apache.commons.net.ftp.FTP;
	import org.apache.commons.net.ftp.FTPCommand;
	import org.apache.commons.net.ftp.FTPFile;
	import org.apache.commons.net.ftp.FTPFileEntryParser;
	import org.apache.commons.net.ftp.FTPReply;

	import org.apache.commons.net.ftp.FTPConnectionClosedException;

	/***********************************************
	* Client.java encapsulates functionalities necessary for nutch to get dir list
	* and retrieve file from an FTP server. This class takes care of all low level
	* details of interacting with an FTP server and provides a convenient higher
	* level interface.
	*
	* Modified from FtpClient.java in apache commons-net.
	*
	* Notes by John Xing: ftp server implementations are hardly uniform and none
	* seems to follow RFCs whole-heartedly. We have no choice, but assume common
	* denominator as following: (1) Use stream mode for data transfer. Block mode
	* will be better for multiple file downloading and partial file downloading.
	* However not every ftpd has block mode support. (2) Use passive mode for data
	* connection. So Nutch will work if we run behind firewall. (3) Data connection
	* is opened/closed per ftp command for the reasons listed in (1). There are ftp
	* servers out there, when partial downloading is enforced by closing data
	* channel socket on our client side, the server side immediately closes control
	* channel (socket). Our codes deal with such a bad behavior. (4) LIST is used
	* to obtain remote file attributes if possible. MDTM and SIZE would be nice, but
	* not as ubiquitously implemented as LIST. (5) Avoid using ABOR in single
	* thread? Do not use it at all.
	*
	* About exceptions: Some specific exceptions are re-thrown as one of
	* FtpException.java In fact, each function throws FtpException.java or pass
	* IOException.
	*
	* @author John Xing
	***********************************************/

	public class Client extends FTP {
	private int __dataTimeout;
	private int __passivePort;
	private String __passiveHost;
	// private int __fileType, __fileFormat;
	private boolean __remoteVerificationEnabled;
	// private FTPFileEntryParser __entryParser;
	private String __systemName;

	/** Public default constructor */
	public Client() {
	__initDefaults();
	__dataTimeout = -1;
	__remoteVerificationEnabled = true;
	}

	// defaults when initialize
	private void __initDefaults() {
	__passiveHost = null;
	__passivePort = -1;
	__systemName = null;
	// __fileType = FTP.ASCII_FILE_TYPE;
	// __fileFormat = FTP.NON_PRINT_TEXT_FORMAT;
	// __entryParser = null;
	}

	// parse reply for pass()
	private void __parsePassiveModeReply(String reply)
	throws MalformedServerReplyException {
	int i, index, lastIndex;
	String octet1, octet2;
	StringBuffer host;

	reply = reply.substring(reply.indexOf('(') + 1, reply.indexOf(')')).trim();

	host = new StringBuffer(24);
	lastIndex = 0;
	index = reply.indexOf(',');
	host.append(reply.substring(lastIndex, index));

	for (i = 0; i < 3; i++) {
	host.append('.');
	lastIndex = index + 1;
	index = reply.indexOf(',', lastIndex);
	host.append(reply.substring(lastIndex, index));
	}

	lastIndex = index + 1;
	index = reply.indexOf(',', lastIndex);

	octet1 = reply.substring(lastIndex, index);
	octet2 = reply.substring(index + 1);

	// index and lastIndex now used as temporaries
	try {
	index = Integer.parseInt(octet1);
	lastIndex = Integer.parseInt(octet2);
	} catch (NumberFormatException e) {
	throw new MalformedServerReplyException(
	"Could not parse passive host information.\nServer Reply: " + reply);
	}

	index <<= 8;
	index \|= lastIndex;

	__passiveHost = host.toString();
	__passivePort = index;
	}

	/**
	* open a passive data connection socket
	*
	* @param command
	* @param arg
	* @return
	* @throws IOException
	* @throws FtpExceptionCanNotHaveDataConnection
	*/
	protected Socket __openPassiveDataConnection(int command, String arg)
	throws IOException, FtpExceptionCanNotHaveDataConnection {
	Socket socket;

	// // 20040317, xing, accommodate ill-behaved servers, see below
	// int port_previous = __passivePort;

	if (pasv() != FTPReply.ENTERING_PASSIVE_MODE)
	throw new FtpExceptionCanNotHaveDataConnection("pasv() failed. "
	+ getReplyString());

	try {
	__parsePassiveModeReply(getReplyStrings()[0]);
	} catch (MalformedServerReplyException e) {
	throw new FtpExceptionCanNotHaveDataConnection(e.getMessage());
	}

	// // 20040317, xing, accommodate ill-behaved servers, see above
	// int count = 0;
	// System.err.println("__passivePort "+__passivePort);
	// System.err.println("port_previous "+port_previous);
	// while (__passivePort == port_previous) {
	// // just quit if too many tries. make it an exception here?
	// if (count++ > 10)
	// return null;
	// // slow down further for each new try
	// Thread.sleep(500*count);
	// if (pasv() != FTPReply.ENTERING_PASSIVE_MODE)
	// throw new FtpExceptionCanNotHaveDataConnection(
	// "pasv() failed. " + getReplyString());
	// //return null;
	// try {
	// __parsePassiveModeReply(getReplyStrings()[0]);
	// } catch (MalformedServerReplyException e) {
	// throw new FtpExceptionCanNotHaveDataConnection(e.getMessage());
	// }
	// }

	socket = _socketFactory_.createSocket(__passiveHost, __passivePort);

	if (!FTPReply.isPositivePreliminary(sendCommand(command, arg))) {
	socket.close();
	return null;
	}

	if (__remoteVerificationEnabled && !verifyRemote(socket)) {
	InetAddress host1, host2;

	host1 = socket.getInetAddress();
	host2 = getRemoteAddress();

	socket.close();

	// our precaution
	throw new FtpExceptionCanNotHaveDataConnection(
	"Host attempting data connection " + host1.getHostAddress()
	+ " is not same as server " + host2.getHostAddress()
	+ " So we intentionally close it for security precaution.");
	}

	if (__dataTimeout >= 0)
	socket.setSoTimeout(__dataTimeout);

	return socket;
	}

	/***
	* Sets the timeout in milliseconds to use for data connection. set
	* immediately after opening the data connection.
	***/
	public void setDataTimeout(int timeout) {
	__dataTimeout = timeout;
	}

	/***
	* Closes the connection to the FTP server and restores connection parameters
	* to the default values.
	* <p>
	*
	* @exception IOException
	* If an error occurs while disconnecting.
	***/
	public void disconnect() throws IOException {
	__initDefaults();
	super.disconnect();
	// no worry for data connection, since we always close it
	// in every ftp command that invloves data connection
	}

	/***
	* Enable or disable verification that the remote host taking part of a data
	* connection is the same as the host to which the control connection is
	* attached. The default is for verification to be enabled. You may set this
	* value at any time, whether the FTPClient is currently connected or not.
	* <p>
	*
	* @param enable
	* True to enable verification, false to disable verification.
	***/
	public void setRemoteVerificationEnabled(boolean enable) {
	__remoteVerificationEnabled = enable;
	}

	/***
	* Return whether or not verification of the remote host participating in data
	* connections is enabled. The default behavior is for verification to be
	* enabled.
	* <p>
	*
	* @return True if verification is enabled, false if not.
	***/
	public boolean isRemoteVerificationEnabled() {
	return __remoteVerificationEnabled;
	}

	/***
	* Login to the FTP server using the provided username and password.
	* <p>
	*
	* @param username
	* The username to login under.
	* @param password
	* The password to use.
	* @return True if successfully completed, false if not.
	* @exception FTPConnectionClosedException
	* If the FTP server prematurely closes the connection as a
	* result of the client being idle or some other reason causing
	* the server to send FTP reply code 421. This exception may be
	* caught either as an IOException or independently as itself.
	* @exception IOException
	* If an I/O error occurs while either sending a command to the
	* server or receiving a reply from the server.
	***/
	public boolean login(String username, String password) throws IOException {
	user(username);

	if (FTPReply.isPositiveCompletion(getReplyCode()))
	return true;

	// If we get here, we either have an error code, or an intermmediate
	// reply requesting password.
	if (!FTPReply.isPositiveIntermediate(getReplyCode()))
	return false;

	return FTPReply.isPositiveCompletion(pass(password));
	}

	/***
	* Logout of the FTP server by sending the QUIT command.
	* <p>
	*
	* @return True if successfully completed, false if not.
	* @exception FTPConnectionClosedException
	* If the FTP server prematurely closes the connection as a
	* result of the client being idle or some other reason causing
	* the server to send FTP reply code 421. This exception may be
	* caught either as an IOException or independently as itself.
	* @exception IOException
	* If an I/O error occurs while either sending a command to the
	* server or receiving a reply from the server.
	***/
	public boolean logout() throws IOException {
	return FTPReply.isPositiveCompletion(quit());
	}

	/**
	* retrieve list reply for path
	*
	* @param path
	* @param entries
	* @param limit
	* @param parser
	* @throws IOException
	* @throws FtpExceptionCanNotHaveDataConnection
	* @throws FtpExceptionUnknownForcedDataClose
	* @throws FtpExceptionControlClosedByForcedDataClose
	*/
	public void retrieveList(String path, List<FTPFile> entries, int limit,
	FTPFileEntryParser parser) throws IOException,
	FtpExceptionCanNotHaveDataConnection, FtpExceptionUnknownForcedDataClose,
	FtpExceptionControlClosedByForcedDataClose {
	Socket socket = __openPassiveDataConnection(FTPCommand.LIST, path);

	if (socket == null)
	throw new FtpExceptionCanNotHaveDataConnection("LIST "
	+ ((path == null) ? "" : path));

	BufferedReader reader = new BufferedReader(new InputStreamReader(
	socket.getInputStream()));

	// force-close data channel socket, when download limit is reached
	// boolean mandatory_close = false;

	// List entries = new LinkedList();
	int count = 0;
	String line = parser.readNextEntry(reader);
	while (line != null) {
	FTPFile ftpFile = parser.parseFTPEntry(line);
	// skip non-formatted lines
	if (ftpFile == null) {
	line = parser.readNextEntry(reader);
	continue;
	}
	entries.add(ftpFile);
	count += line.length();
	// impose download limit if limit >= 0, otherwise no limit
	// here, cut off is up to the line when total bytes is just over limit
	if (limit >= 0 && count > limit) {
	// mandatory_close = true;
	break;
	}
	line = parser.readNextEntry(reader);
	}

	// if (mandatory_close)
	// you always close here, no matter mandatory_close or not.
	// however different ftp servers respond differently, see below.
	socket.close();

	// scenarios:
	// (1) mandatory_close is false, download limit not reached
	// no special care here
	// (2) mandatory_close is true, download limit is reached
	// different servers have different reply codes:

	try {
	int reply = getReply();
	if (!_notBadReply(reply))
	throw new FtpExceptionUnknownForcedDataClose(getReplyString());
	} catch (FTPConnectionClosedException e) {
	// some ftp servers will close control channel if data channel socket
	// is closed by our end before all data has been read out. Check:
	// tux414.q-tam.hp.com FTP server (hp.com version whp02)
	// so must catch FTPConnectionClosedException thrown by getReply() above
	// disconnect();
	throw new FtpExceptionControlClosedByForcedDataClose(e.getMessage());
	}

	}

	/**
	* retrieve file for path
	*
	* @param path
	* @param os
	* @param limit
	* @throws IOException
	* @throws FtpExceptionCanNotHaveDataConnection
	* @throws FtpExceptionUnknownForcedDataClose
	* @throws FtpExceptionControlClosedByForcedDataClose
	*/
	public void retrieveFile(String path, OutputStream os, int limit)
	throws IOException, FtpExceptionCanNotHaveDataConnection,
	FtpExceptionUnknownForcedDataClose,
	FtpExceptionControlClosedByForcedDataClose {

	Socket socket = __openPassiveDataConnection(FTPCommand.RETR, path);

	if (socket == null)
	throw new FtpExceptionCanNotHaveDataConnection("RETR "
	+ ((path == null) ? "" : path));

	InputStream input = socket.getInputStream();

	// 20040318, xing, treat everything as BINARY_FILE_TYPE for now
	// do we ever need ASCII_FILE_TYPE?
	// if (__fileType == ASCII_FILE_TYPE)
	// input = new FromNetASCIIInputStream(input);

	// fixme, should we instruct server here for binary file type?

	// force-close data channel socket
	// boolean mandatory_close = false;

	int len;
	int count = 0;
	byte[] buf = new byte[org.apache.commons.net.io.Util.DEFAULT_COPY_BUFFER_SIZE];
	while ((len = input.read(buf, 0, buf.length)) != -1) {
	count += len;
	// impose download limit if limit >= 0, otherwise no limit
	// here, cut off is exactly of limit bytes
	if (limit >= 0 && count > limit) {
	os.write(buf, 0, len - (count - limit));
	// mandatory_close = true;
	break;
	}
	os.write(buf, 0, len);
	os.flush();
	}

	// if (mandatory_close)
	// you always close here, no matter mandatory_close or not.
	// however different ftp servers respond differently, see below.
	socket.close();

	// scenarios:
	// (1) mandatory_close is false, download limit not reached
	// no special care here
	// (2) mandatory_close is true, download limit is reached
	// different servers have different reply codes:

	// do not need this
	// sendCommand("ABOR");

	try {
	int reply = getReply();
	if (!_notBadReply(reply))
	throw new FtpExceptionUnknownForcedDataClose(getReplyString());
	} catch (FTPConnectionClosedException e) {
	// some ftp servers will close control channel if data channel socket
	// is closed by our end before all data has been read out. Check:
	// tux414.q-tam.hp.com FTP server (hp.com version whp02)
	// so must catch FTPConnectionClosedException thrown by getReply() above
	// disconnect();
	throw new FtpExceptionControlClosedByForcedDataClose(e.getMessage());
	}

	}

	/**
	* reply check after closing data connection
	*
	* @param reply
	* @return
	*/
	private boolean _notBadReply(int reply) {

	if (FTPReply.isPositiveCompletion(reply)) {
	// do nothing
	} else if (reply == 426) { // FTPReply.TRANSFER_ABORTED
	// some ftp servers reply 426, e.g.,
	// foggy FTP server (Version wu-2.6.2(2)
	// there is second reply witing? no!
	// getReply();
	} else if (reply == 450) { // FTPReply.FILE_ACTION_NOT_TAKEN
	// some ftp servers reply 450, e.g.,
	// ProFTPD [ftp.kernel.org]
	// there is second reply witing? no!
	// getReply();
	} else if (reply == 451) { // FTPReply.ACTION_ABORTED
	// some ftp servers reply 451, e.g.,
	// ProFTPD [ftp.kernel.org]
	// there is second reply witing? no!
	// getReply();
	} else {
	// what other kind of ftp server out there?
	return false;
	}

	return true;
	}

	/***
	* Sets the file type to be transferred. This should be one of
	* <code> FTP.ASCII_FILE_TYPE </code>, <code> FTP.IMAGE_FILE_TYPE </code>,
	* etc. The file type only needs to be set when you want to change the type.
	* After changing it, the new type stays in effect until you change it again.
	* The default file type is <code> FTP.ASCII_FILE_TYPE </code> if this method
	* is never called.
	* <p>
	*
	* @param fileType
	* The <code> _FILE_TYPE </code> constant indcating the type of file.
	* @return True if successfully completed, false if not.
	* @exception FTPConnectionClosedException
	* If the FTP server prematurely closes the connection as a
	* result of the client being idle or some other reason causing
	* the server to send FTP reply code 421. This exception may be
	* caught either as an IOException or independently as itself.
	* @exception IOException
	* If an I/O error occurs while either sending a command to the
	* server or receiving a reply from the server.
	***/
	public boolean setFileType(int fileType) throws IOException {
	if (FTPReply.isPositiveCompletion(type(fileType))) {
	/*
	* __fileType = fileType; __fileFormat = FTP.NON_PRINT_TEXT_FORMAT;
	*/
	return true;
	}
	return false;
	}

	/***
	* Fetches the system type name from the server and returns the string. This
	* value is cached for the duration of the connection after the first call to
	* this method. In other words, only the first time that you invoke this
	* method will it issue a SYST command to the FTP server. FTPClient will
	* remember the value and return the cached value until a call to disconnect.
	* <p>
	*
	* @return The system type name obtained from the server. null if the
	* information could not be obtained.
	* @exception FTPConnectionClosedException
	* If the FTP server prematurely closes the connection as a
	* result of the client being idle or some other reason causing
	* the server to send FTP reply code 421. This exception may be
	* caught either as an IOException or independently as itself.
	* @exception IOException
	* If an I/O error occurs while either sending a command to the
	* server or receiving a reply from the server.
	***/
	public String getSystemName() throws IOException, FtpExceptionBadSystResponse {
	// if (syst() == FTPReply.NAME_SYSTEM_TYPE)
	// Technically, we should expect a NAME_SYSTEM_TYPE response, but
	// in practice FTP servers deviate, so we soften the condition to
	// a positive completion.
	if (__systemName == null && FTPReply.isPositiveCompletion(syst())) {
	__systemName = (getReplyStrings()[0]).substring(4);
	} else {
	throw new FtpExceptionBadSystResponse("Bad response of SYST: "
	+ getReplyString());
	}

	return __systemName;
	}

	/***
	* Sends a NOOP command to the FTP server. This is useful for preventing
	* server timeouts.
	* <p>
	*
	* @return True if successfully completed, false if not.
	* @exception FTPConnectionClosedException
	* If the FTP server prematurely closes the connection as a
	* result of the client being idle or some other reason causing
	* the server to send FTP reply code 421. This exception may be
	* caught either as an IOException or independently as itself.
	* @exception IOException
	* If an I/O error occurs while either sending a command to the
	* server or receiving a reply from the server.
	***/
	public boolean sendNoOp() throws IOException {
	return FTPReply.isPositiveCompletion(noop());
	}

	// client.stat(path);
	// client.sendCommand("STAT");
	// client.sendCommand("STAT",path);
	// client.sendCommand("MDTM",path);
	// client.sendCommand("SIZE",path);
	// client.sendCommand("HELP","SITE");
	// client.sendCommand("SYST");
	// client.setRestartOffset(120);

	}