Blanquier added the comment:
Hi,
I send you the file as attached document.
I use the test_sftp_upload() as entry point.
Le 30/09/2015 16:43, R. David Murray a écrit :
> R. David Murray added the comment:
>
> As Eric said, we really can't diagnose this unless you show is your python
> code (as code, not as a partial screenshot).
>
> ----------
>
> _______________________________________
> Python tracker <rep...@bugs.python.org>
> <http://bugs.python.org/issue25278>
> _______________________________________
>
----------
Added file: http://bugs.python.org/file40630/gui_ftp.py
_______________________________________
Python tracker <rep...@bugs.python.org>
<http://bugs.python.org/issue25278>
_______________________________________
# -*- coding: iso-8859-1 -*-
# -----------------------------------------------------------------------
__file__ = "gui_ftp.py"
__doc__= "FTP functions for ul/downloading one or several server in parallele
with multiple connections."
__author__ = "Philippe Blanquier"
__version__ = "v0.1"
__date__ = "24/06/2011"
__copyright__ = "Alcatel-Lucent"
__contact__ = "philippe.blanqu...@alcatel-lucent.com"
# -----------------------------------------------------------------------
# Python include
import os,sys,ftplib,glob,time,threading,socket,copy
# Private include
import gui_common,gui_scenario_manager,gui_user
### ------------ ###
### Private data ###
### ------------ ###
_ftps_certificate_file_name =
"C:"+os.sep+"Nemo_Pyc"+os.sep+"sftp_certificate.crt" #: FTPS certificate for
labo servers
_ftp_error_tag = "FTP"
_ftp_block_size = 64*gui_common.kilo_unit #: Maximum chunk size to read/write
on the low-level socket object created to do the actual transfer
_ftp_repeat_nb = 10 #: Maximal command repeatition number before saying 'KO'...
_ftp_repeat_waiting_delay = 30.0 #: Delay between 2 repetition when a failure
is detected
_upload_cmd = "Upload"
_download_cmd = "Download"
_file_size_tag = "File size"
_ftp_posix_separator = '/' #: Normalized FTP directory separator ( UNIX POSIX !)
_ftp_dir_size = -1 #: Dummy FTP directory size
_ftp_create_dir_mutex = gui_common.nice_mutex( this_name = "FTP_Dir_Mutex") #:
Internal directory creation protection
# see: http://en.wikipedia.org/wiki/List_of_FTP_server_return_codes
#
# FTP server return codes always have three digits, and each digit has a
special meaning.
#
# The first digit denotes whether the response is good, bad or incomplete:
# 1xx Positive Preliminary reply
# 2xx Positive Completion reply
# 3xx Positive Intermediate reply
# 4xx Transient Negative Completion reply
# 5xx Permanent Negative Completion reply
# 6xx Protected reply
# The second digit is a grouping digit and encodes the following information:
# x0x Syntax
# x1x Information
# x2x Connections
# x3x Authentication and accounting
# x4x Unspecified as of RFC 959.
# x5x File system
### The requested action is being taken. Expect a reply before proceeding with
a new command.
# 100 Series: The requested action is being initiated, expect another reply
before proceeding with a new command.
# 110 Restart marker replay . In this case, the text is exact and not left to
the particular implementation;
# it must read: MARK yyyy = mmmm where yyyy is User-process data stream
marker, and mmmm server's equivalent marker (note the spaces between markers
and "=").
# 120 Service ready in nnn minutes.
# 125 Data connection already open; transfer starting.
# 150 File status okay; about to open data connection.
### The requested action has been successfully completed.
# 200 Command okay.
# 202 Command not implemented, superfluous at this site.
# 211 System status, or system help reply.
# 212 Directory status.
# 213 File status.
# 214 Help message.On how to use the server or the meaning of a particular
non-standard command. This reply is useful only to the human user.
# 215 NAME system type. Where NAME is an official system name from the
registry kept by IANA.
# 220 Service ready for new user.
# 221 Service closing control connection.
# 225 Data connection open; no transfer in progress.
# 226 Closing data connection. Requested file action successful (for example,
file transfer or file abort).
# 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2).
# 228 Entering Long Passive Mode (long address, port).
# 229 Entering Extended Passive Mode (|||port|).
# 230 User logged in, proceed. Logged out if appropriate.
# 231 User logged out; service terminated.
# 232 Logout command noted, will complete when transfer done.
# 250 Requested file action okay, completed.
# 257 "PATHNAME" created.
### The command has been accepted, but the requested action is being held
pending receipt of further information.
# 331 User name okay, need password.
# 332 Need account for login.
# 350 Requested file action pending further information
### <<<exception ftplib.error_temp>>> 400...499
### The command was not accepted and the requested action did not take place.
### The error condition is temporary, however, and the action may be requested
again.
# 421 Service not available, closing control connection.
# 421 Connection timed out
# 425 Can't open data connection.
# 426 Connection closed; transfer aborted.
# 430 Invalid username or password
# 434 Requested host unavailable.
# 450 Requested file action not taken.
# 451 Requested action aborted. Local error in processing.
# 452 Requested action not taken. Insufficient storage space in system.File
unavailable (e.g., file busy).
### <<<exception ftplib.error_perm>>> 500...599
### The command was not accepted and the requested action did not take place.
# 500 Syntax error, command unrecognized. This may include errors such as
command line too long.
# 501 Syntax error in parameters or arguments.
# 502 Command not implemented.
# 503 Bad sequence of commands.
# 504 Command not implemented for that parameter.
# 530 Not logged in.
# 532 Need account for storing files.
# 550 Requested action not taken. File unavailable (e.g., file not found, no
access).
# 551 Requested action aborted. Page type unknown.
# 552 Requested file action aborted. Exceeded storage allocation (for current
directory or dataset).
# 553 Requested action not taken. File name not allowed.
# 631 Integrity protected reply.
# 632 Confidentiality and integrity protected reply.
# 633 Confidentiality protected reply.
### Common Winsock Error Codes (full error:
http://kb.globalscape.com/KnowledgebaseArticle10140.aspx)
#10054 Connection reset by peer. The connection was forcibly closed by the
remote host. (Informational)
#10060 Cannot connect to remote server. Generally a time-out error. Try
switching from PASV to PORT mode, or try increasing the time-out value.
#10061 Cannot connect to remote server. The connection is actively refused by
the server. Try switching the connection port.
#10066 Directory not empty. The server will not delete this directory while
there are files/folders in it. If you want to remove the directory, first
archive or delete the files in it.
#10068 Too many users, server is full. Try logging in at another time.
# The following are the FTP commands (see RFC 959):
# ABOR <CRLF>
# ACCT <SP> <account-information> <CRLF>
# ALLO <SP> <decimal-integer>
# [<SP> R <SP> <decimal-integer>] <CRLF>
# APPE <SP> <pathname> <CRLF>
# CDUP <CRLF>
# CWD <SP> <pathname> <CRLF>
# DELE <SP> <pathname> <CRLF>
# HELP [<SP> <string>] <CRLF>
# LIST [<SP> <pathname>] <CRLF>
# MODE <SP> <mode-code> <CRLF>
# MKD <SP> <pathname> <CRLF>
# NLST [<SP> <pathname>] <CRLF>
# NOOP <CRLF>
# PASS <SP> <password> <CRLF>
# PASV <CRLF>
# PORT <SP> <host-port> <CRLF>
# PWD <CRLF>
# QUIT <CRLF>
# REIN <CRLF>
# REST <SP> <marker> <CRLF>
# RETR <SP> <pathname> <CRLF>
# RMD <SP> <pathname> <CRLF>
# RNFR <SP> <pathname> <CRLF>
# RNTO <SP> <pathname> <CRLF>
# SITE <SP> <string> <CRLF>
# SMNT <SP> <pathname> <CRLF>
# STAT [<SP> <pathname>] <CRLF>
# STOR <SP> <pathname> <CRLF>
# STOU <CRLF>
# STRU <SP> <structure-code> <CRLF>
# SYST <CRLF>
# TYPE <SP> <type-code> <CRLF>
# USER <SP> <username> <CRLF>
### ----------------- ###
### Private functions ###
### ----------------- ###
# ------------------------------------------
# Private Function : _ftp_close_connection()
# ------------------------------------------
def _ftp_close_connection( this_ftp_id = None):
"""
Close the FTP connection.
@param this_ftp_id: FTP connection ident
@type this_ftp_id: ftplib object
@note:
- use 'quit()'command first: this is the polite way to close
a connection
- if 'quit()' fails, use 'close()' command
@return: None
@see: https://docs.python.org/release/2.6.9/library/ftplib.html
"""
if this_ftp_id is not None:
# Stop current transfert (if any)
#try:
# this_ftp_id.abort()
#except:
# pass
# Be polite with the remote host to quit and close the
connection.
try:
this_ftp_id.quit()
except:
# Oops ! Close the connection unilaterally !
try:
this_ftp_id.close()
except:
# Ignore the error
pass
return
# --------------------------------------------
# Private Function : _ftp_get_max_connection()
# --------------------------------------------
def _ftp_get_max_connection( this_address = "0.0.0.0"):
"""
Determine the maximal connection in regard of the remote server adress
IP.
- 4 for intranet or VPN to Labo server: speed up for end user
- 1 for other situation
@param this_address: server IP address
@type this_address: ascii string
@note: the values MUST be compatible with the Filezilla server
preferences running on the remote servers !
"""
if gui_common.valid_labo_ip_addr( this_ip_addr = this_address
, this_all_server = True
) is True:
answer = 4
else:
# Unknown
answer = 1
return answer
# ---------------------
# _ftp_repeat_command()
# ---------------------
def _ftp_repeat_command( this_err_msg = None):
"""
Try to detect FTP errors whose commands could be repeated (i.e:
'beyond help').
Some errors are only taken into account at present for
repetitions:
- 421 User limit reached.
- 421 Max connections reached.
- 421 Too many users are connected, please try again
later.
- 421 Connection timed out
- 425 Can't open data connection (*)
- 426 Connection closed; transfer aborted (*)
- 450 Requested file action not taken.
- 451 Requested action aborted. Local error in
processing.
- 550 can't access file. (on SFTP 'STOR' command) (*)
- SSL: DECRYPTION_FAILED_OR_BAD_RECORD_MAC
- EOF occurred in violation of protocol
@param this_err_msg: full FTP error
@type this_err_msg: ascii string
@return: True = repeat allowed, False = no repeatition
@rtype: boolean
@note: The errors (*) 425 & 426 could occure during upload...
"""
if this_err_msg is None:
return False
beyond_help_error_list = [ "User limit reached"
, "Max connections reached"
, "Too many users are connected"
, "Connection timed out"
, _file_size_tag
# <<<Special tags>>>
# [Errno 10013] An attempt was made to
access a socket in a way forbidden by its access permissions.
, "Errno 10013"
# [Errno 10060] A connection attempt
failed because the connected party did not properly respond after a period of
time, or established connection failed because connected host has failed to
respond
, "Errno 10060"
# "426 Connection closed; transfer
aborted. May occure during transfert. Try to work around using activ/passif
mode... see:
http://support.isotools.fr/ticket/Customer/KBArticle.aspx?articleid=217
, "transfer aborted"
# 425 Can't open data connection
, "Can't open data connection"
# 450 Requested file action not taken.
, "Requested file action not taken"
# 450 TLS session of data connection
has not resumed or the session does not match the control connection
, "TLS session of data connection has
not resumed"
# 451 Requested action aborted. Local
error in processing.
, "Local error in processing"
# 550 can't access file.
, "can't access file"
# SSL errors
, "DECRYPTION_FAILED_OR_BAD_RECORD_MAC"
, "EOF occurred in violation of
protocol"
]
answer = False
for code_idx in beyond_help_error_list:
if code_idx in this_err_msg:
# Found !
answer = True
break
# Next code_idx
continue
return answer
# --------------------------
# Private Class : _ftp_mutex
# --------------------------
class _ftp_mutex(gui_common.Singleton):
"""
Manage only one acces to each remote host when a remote directory has
to be created
"""
ip_mutex_dico = {}
# ---------------------
# _ftp_mutex.__init__()
# ---------------------
def __init__( self
, this_ip_addr
):
if _ftp_mutex.ip_mutex_dico.get(this_ip_addr, None) is None:
_ftp_mutex.ip_mutex_dico[this_ip_addr] =
gui_common.nice_mutex( this_name = "FTP_Dir_Mutex_%s"%(this_ip_addr)) #:
Internal directory creation protection
return
# ------------------
# _ftp_mutex.error()
# ------------------
def error( self
, this_msg
):
"""
Shortcut access to display an error
@param this_msg: input message describing the error
@type this_msg: ascii string
@return: None
"""
gui_common.save_error( this_msg = this_msg
, this_object_class = self
, this_file_name = __file__
, this_mail_subject = _ftp_error_tag+"
Mutex"
)
# ----------------
# _ftp_mutex.get()
# ----------------
def get( self
, this_ip_addr
):
local_mutex = _ftp_mutex.ip_mutex_dico.get(this_ip_addr, None)
if local_mutex is None:
# Oops !
self.error( this_msg = "No mutex for %s"%(this_ip_addr))
else:
local_mutex.get()
return
# --------------------
# _ftp_mutex.release()
# --------------------
def release( self
, this_ip_addr
):
local_mutex = _ftp_mutex.ip_mutex_dico.get(this_ip_addr, None)
if local_mutex is None:
# Oops !
self.error( this_msg = "No mutex for %s"%(this_ip_addr))
else:
local_mutex.release()
return
# ----------------------------------
# Private Class : _ftp_slave_thread
# ---------------------------------
class _ftp_slave_thread( threading.Thread):
"""
Independant FPT thread to upload or download in parallele on a
dedicated server.
@note: A file size check is done to prevent a 'null file size' on the
tranfered file (ftplib bug ?)
@note: see: http://support.microsoft.com/kb/466868/fr
@sort:
_*,a*,b*,c*,d*,e*,f*,g*,h*,i*,j*,k*,l*,m*,n*,o*,p*,q*,r*,s*,t*,u*,v*,w*,x*,y*,z*
"""
# ----------------------------
# _ftp_slave_thread.__init__()
# ----------------------------
def __init__( self
, this_ftp_id_id = None
, this_server_ip_address = None
, this_file_manager_id = None
, this_upload = True
, this_user_remote_dir = _ftp_posix_separator
, this_user_local_dir = "."
, this_debug_mode = False
, this_start_time = 0
, this_check_size = True
):
"""
Elementary thread to manage one FTP connection.
@param this_ftp_id_id: FTP connection ident
@type this_ftp_id_id: FTP object
@param this_server_ip_address: remote IP address (for error)
@type this_server_ip_address: ascii string
@param this_file_manager_id: file manager ident returned by
_ftp_open_many_connection()
@type this_file_manager_id: gui_scenario_manager object
@param this_upload: True = connection for an upload, False =
connection for an download
@type this_upload: boolean
@param this_user_remote_dir: user remote directory
@type this_user_remote_dir: ascii string
@param this_debug_mode: True = display debug information, False
= run silently
@type this_debug_mode: boolean
@param this_start_time: start time of the FTP transfert
(seconds since the epoch)
@type this_start_time: float
@param this_check_size: True = Check sent and downloaded file
size, False = No check
@type this_check_size: boolean
@return: None
"""
threading.Thread.__init__(self)
self.ftp_id = this_ftp_id_id #: FTP connection to be used
self.server_ip_address = this_server_ip_address #: Server IP
address
self.file_manager_id = this_file_manager_id #: File manager
self.debug_mode = this_debug_mode #: Flag for debug purpose
self.result = True #: private result/error
self.dbg_check_size = this_check_size #: for debug purpose
self.start_time = this_start_time #: Keep in mind start time
self.last_remote_dir = ""
self.ftp_dir_mutex = _ftp_mutex(this_ip_addr =
self.server_ip_address)
self.user_local_dir = this_user_local_dir #: Current host
directory name
if (this_user_remote_dir is None) or (this_user_remote_dir ==
_ftp_posix_separator):
self.user_remote_dir = self.ftp_id.pwd() # Use the
default user directory
else:
# Use the POSIX separator for remote directory !
self.user_remote_dir =
this_user_remote_dir.replace(os.sep,_ftp_posix_separator) #: User directory on
the server
self.do_upload = this_upload
return
# ---------------------------
# _ftp_slave_thread.__del__()
# ---------------------------
def __del__(self):
"""
For debug purpose
@return: None
"""
self.close_connection()
return
# -------------------------
# _ftp_slave_thread.error()
# -------------------------
def error( self
, this_msg
):
"""
Shortcut access to display an error
@param this_msg: input message describing the error
@type this_msg: ascii string
@return: None
"""
# Add elapsed running time (for server timer delay purpose)
this_msg += "\nRunning time %s"%(gui_common.elapsed_time_ascii(
this_start_time = self.start_time))
if self.debug_mode is False:
gui_common.save_error( this_msg = this_msg
, this_object_class = self
, this_file_name = __file__
, this_mail_subject =
_ftp_error_tag
)
else:
gui_common.display_error( this_msg = this_msg
, this_object_class = self
, this_file_name = __file__
)
self.result = False
return
# ---------------------------
# _ftp_slave_thread.warning()
# ---------------------------
def warning( self
, this_msg
):
"""
Shortcut access to display a warning
@param this_msg: input message describing the error
@type this_msg: ascii string
@return: None
"""
# Add elapsed running time (for server timer delay purpose)
this_msg += "\nRunning time %s"%(gui_common.elapsed_time_ascii(
this_start_time = self.start_time))
if self.debug_mode is False:
gui_common.save_warning( this_msg = this_msg
, this_object_class = self
, this_file_name = __file__
, this_caller_name =
gui_common.get_caller_name()
)
else:
gui_common.display_warning( this_msg = this_msg
, this_object_class = self
, this_file_name = __file__
, this_caller_name =
gui_common.get_caller_name()
)
return
# ------------------------------------
# _ftp_slave_thread.close_connection()
# ------------------------------------
def close_connection( self ):
"""
Close the FTP connection.
@return: None
"""
_ftp_close_connection( this_ftp_id = self.ftp_id)
self.ftp_id = None
return
# -------------------------------------
# _ftp_slave_thread.change_remote_dir()
# -------------------------------------
def change_remote_dir( self
, this_new_dir = os.curdir
):
"""
Change the remote directory.
@param this_new_dir: new remote POSIX directory
@type this_new_dir: ascii string
@return: True = done, False = error detected
@rtype: boolean
@note:
- the directory MUST have been created previously.
- the remote directory has the POSIX standard format
"""
# Force the absolute remote path
if this_new_dir == "":
# default remote directory
this_new_dir = _ftp_posix_separator
else:
if this_new_dir[0] != _ftp_posix_separator:
this_new_dir = _ftp_posix_separator+this_new_dir
if self.last_remote_dir == this_new_dir:
# Nothing to do !
return True
# Be carefull when multiple clients are connected on the same
remote host and want to create a new directory...
self.ftp_dir_mutex.get(this_ip_addr = self.server_ip_address)
err_msg = None
try:
self.ftp_id.cwd( this_new_dir )
except ftplib.error_temp, e:
if "500 can't access directory" in e.args[0]:
err_msg = e.args[0]
except ftplib.error_reply, e:
err_msg = e.args[0]
except (ftplib.error_perm,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.timeout,msg:
err_msg = gui_common.get_socket_error_msg( this_error =
msg)
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg( this_error =
msg)
except gui_common.runing_errors,why:
err_msg = str(why)
except:
err_msg = "ERROR"
if err_msg is None:
# Yeap !
self.last_remote_dir = this_new_dir
else:
# Oops !
if ("550 CWD failed" in err_msg) and ("directory not
found" in err_msg):
# The directory must be created
try:
self.ftp_id.mkd( dirname = this_new_dir)
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.error, msg:
err_msg =
gui_common.get_socket_error_msg( this_error = msg)
except gui_common.runing_errors,why:
err_msg = str(why)
except:
err_msg = "ERROR"
else:
err_msg = None
if err_msg is not None:
err_msg = "mkd('%s') on %s: %s"%(
this_new_dir
,
self.server_ip_address
,
err_msg
)
self.error( this_msg = err_msg)
else:
# Same player shoots again !
try:
self.ftp_id.cwd( this_new_dir )
except ftplib.error_temp, e:
if "500 can't access file" in
e.args[0]:
err_msg = e.args[0]
except ftplib.error_reply, e:
err_msg = e.args[0]
except
(ftplib.error_perm,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.timeout,msg:
err_msg =
gui_common.get_socket_error_msg( this_error = msg)
except socket.error, msg:
err_msg =
gui_common.get_socket_error_msg( this_error = msg)
except gui_common.runing_errors,why:
err_msg = str(why)
except:
err_msg = "ERROR"
if err_msg is None:
# Yeap !
self.last_remote_dir =
this_new_dir
else:
err_msg = "cwd('%s') on %s:
%s"%( this_new_dir
, self.server_ip_address
, err_msg
)
self.error( this_msg = err_msg)
else:
err_msg = "cwd('%s') on %s: %s"%( this_new_dir
,
self.server_ip_address
, err_msg
)
self.error( this_msg = err_msg)
# Remote host directory creation should be successful
self.ftp_dir_mutex.release(this_ip_addr =
self.server_ip_address)
return err_msg is None
# --------------------------
# _ftp_slave_thread.upload()
# --------------------------
def upload( self
, this_full_src_file
, this_full_dest_file
, this_size
):
"""
FTP upload one file.
@param this_full_src_file: full file name on the local host
@type this_full_src_file: ascii string
@param this_full_dest_file: full file name on the server
@type this_full_dest_file: ascii string
@param this_size: file size (in bytes)
@type this_size: integer
@return: True = Done, False = Error detected
@rtype: boolean
@warning: The ftplib.storbinary() should use socket.MSG_WAITALL
when data are sent over the socket.
Unfortunatly, this parameter does not exist on
Windows.
So, sent file may have erroneous size on the
destination host...
@note: If an error occured during the data transfert, the
waiting delay is doubled for each new try.
"""
if self.debug_mode is True:
msg = "%s %s --> %s"%( gui_common.ftp_upload_marker
, this_full_src_file
, this_full_dest_file
)
gui_common.display_screen( this_msg = msg
, this_object_class = self
, this_file_name = __file__
, this_caller_name = None
, this_user_name = None
)
# Extract remote parameters (UNIX path format...)
path_list = this_full_dest_file.split(_ftp_posix_separator)
remote_dir = _ftp_posix_separator.join(path_list[:-1])
remote_file = path_list[-1]
# And then, go on to the remote directory (if not already done)
if self.change_remote_dir( this_new_dir = remote_dir) is not
True:
return False
# Sometimes,the FTPlib could miss the answer, or the answer
could be lost in the network...
attempt_number = _ftp_repeat_nb
while attempt_number > 0:
attempt_number -= 1
err_msg = None
# Open the local file
try:
src_file_id = open( this_full_src_file, 'rb',
gui_common.open_buffer_option)
except gui_common.runing_errors,why:
err_msg = "%s\n%s"%( str(why)
,
gui_common.which_process_uses_it( this_file = this_full_src_file)
)
self.error( this_msg = err_msg)
return False
except:
err_msg = "open(%s,'rb')\n%s"%(
this_full_src_file
,
gui_common.which_process_uses_it( this_file = this_full_src_file)
)
self.error( this_msg = err_msg)
return False
# Upload first
try:
# Use binary transfert for ASCII and binary
files ;-)
self.ftp_id.storbinary( cmd = 'STOR
%s'%(remote_file)
, fp = src_file_id
, blocksize =
_ftp_block_size
)
except socket.timeout,msg:
err_msg = gui_common.get_socket_error_msg(
this_error = msg)
except socket.error,msg:
if msg.errno == 0:
# <<<SFTP: a desynchronisation may
occure>>>
# b'[Errno 0] Error' occures
sometime...
# so the "226 Successfully transferred
"/nemo_tmp/dst_dir/fzs-2015-09-22.log" is ignored !!
# and will be treated later whence an
error is forseen (incorrect response treatement) ...
try:
missed_answer =
self.ftp_id.getresp() # pop the answer server !
except:
# Nothing on the line...
err_msg = None
else:
if missed_answer[0] == '2':
err_msg = None
else:
# It's an error
err_msg = missed_answer
#print "*** %s *** %s
STOR %s --> socket.error
'%s'"%(self.name,self.server_ip_address,remote_file,str(msg))
else:
# Real error
err_msg =
gui_common.get_socket_error_msg( this_error = msg)
except (ftplib.error_temp, ftplib.error_reply,
ftplib.error_perm, ftplib.error_proto),e:
# <<<Python SFTP lib: bug>>>
#--- on server
# (000346)30/09/2015 11:27:18 - nemo_pyc
(135.238.179.15)> TYPE I
# (000346)30/09/2015 11:27:18 - nemo_pyc
(135.238.179.15)> 200 Type set to I
# (000346)30/09/2015 11:27:48 - nemo_pyc
(135.238.179.15)> TYPE I
# (000346)30/09/2015 11:27:48 - nemo_pyc
(135.238.179.15)> 200 Type set to I
# (000346)30/09/2015 11:27:48 - nemo_pyc
(135.238.179.15)> PASV
# (000346)30/09/2015 11:27:48 - nemo_pyc
(135.238.179.15)> 227 Entering Passive Mode (135,120,162,6,202,181)
# (000346)30/09/2015 11:27:48 - nemo_pyc
(135.238.179.15)> SIZE Handle.exe
# (000346)30/09/2015 11:27:48 - nemo_pyc
(135.238.179.15)> 213 536256
# (000346)30/09/2015 11:27:48 - nemo_pyc
(135.238.179.15)> QUIT
# (000346)30/09/2015 11:27:48 - nemo_pyc
(135.238.179.15)> 221 Goodbye
# (000346)30/09/2015 11:27:48 - nemo_pyc
(135.238.179.15)> disconnected.
#--- on client:
# *** FTP_135.120.162.6_Upload_1 ***
135.120.162.6 STOR Handle.exe --> ftplib.error '550 can't access file.'
# *** FTP_135.120.162.6_Upload_1 ***
135.120.162.6 STOR Handle.exe --> ftplib.error '200 Type set to I'
# *** FTP_135.120.162.6_Upload_1 ***
135.120.162.6 STOR Handle.exe OK
missed_answer = e.args[0] # or e.message
for tag_idx in [ "Type set to I"
, "Opening data channel"
, "SSL connection for data
connection established"
]:
if tag_idx in missed_answer:
# Ignore the current message
and take the next pending one (if any)
try:
missed_answer =
self.ftp_id.getresp() # pop the answer server !
except:
# Nothing on the line...
missed_answer = '2xx'
break
# Next tag_idx
continue
if missed_answer[0] == '2':
err_msg = None
#print "*** %s *** %s STOR %s -->
ftplib.error '%s'"%(self.name,self.server_ip_address,remote_file,str(e))
else:
# It's an error
err_msg = missed_answer
except gui_common.runing_errors,why:
err_msg = str(why)
except:
err_msg = "storbinary()"
else:
err_msg = None
# Done with this file
try:
src_file_id.close()
except:
pass
if _ftp_repeat_command( this_err_msg = err_msg) is
False:
break
# while attempt_number
time.sleep(_ftp_repeat_waiting_delay)
continue
if err_msg is not None:
# Oops !
full_err_msg = "%s %s: STOR '%s' --> '%s': %s"%(
gui_common.ftp_upload_marker
,
self.server_ip_address
,
this_full_src_file
,
remote_file
, err_msg
)
self.error( this_msg = full_err_msg)
else:
# The FTP upload has been done
if self.dbg_check_size is True:
# Get the remote file size
remote_file_size = 0
# Because the Windows OS may take a while to
update its internal buffers, we must be carrefull...
attempt_number = _ftp_repeat_nb
while attempt_number > 0:
attempt_number -= 1
try:
remote_file_size =
self.ftp_id.size(remote_file)
except
(socket.timeout,socket.error),msg:
err_msg =
gui_common.get_socket_error_msg( this_error = msg)
if err_msg == "":
err_msg = None
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto),e:
err_msg = e.args[0]
if err_msg == "":
err_msg = None
except gui_common.runing_errors,why:
err_msg = str(why)
except:
err_msg =
"ftp.size(%s)"%(remote_file)
else:
# We have read a remote size...
if this_size !=
remote_file_size:
# Different file
sizes...
err_msg = "%s: local
'%s' = %d, remote '%s' = %d"%( _file_size_tag
, this_full_src_file, this_size
, remote_file, remote_file_size
)
else:
# Yeap !
err_msg = None
break
if _ftp_repeat_command( this_err_msg =
err_msg) is False:
break
# while attempt_number
time.sleep(_ftp_repeat_waiting_delay)
continue
# The result is...
if err_msg is not None:
# Oops !
full_err_msg = "%s %s: SIZE '%s': %s"%(
gui_common.ftp_upload_marker
,
self.server_ip_address
,
remote_file
,
err_msg
)
self.error( this_msg = full_err_msg)
return err_msg is None
# ----------------------------
# _ftp_slave_thread.download()
# ----------------------------
def download( self
, this_full_src_file
, this_full_dest_file
, this_size
):
"""
FTP download one file.
@param this_full_src_file: full file name on the server
@type this_full_src_file: ascii string
@param this_full_dest_file: full file name on the local host
@type this_full_dest_file: ascii string
@param this_size: file size (in bytes)
@type this_size: integer
@return: True = Done, False = Error detected
@rtype: boolean
@note: If an error occured during the data transfert, the
waiting delay is doubled for each new try.
"""
if self.debug_mode is True:
msg = "%s %s --> %s"%( gui_common.ftp_download_marker
, this_full_src_file
, this_full_dest_file
)
gui_common.display_screen( this_msg = msg
, this_object_class = self
, this_file_name = __file__
, this_caller_name = None
, this_user_name = None
)
if this_size == _ftp_dir_size:
# Local directory has been already created
return True
if this_size == 0:
# Ignore empty remote file for downloading
return True
# Extract remote parameters (UNIX path format...)
path_list = this_full_src_file.split(_ftp_posix_separator)
remote_dir = _ftp_posix_separator.join(path_list[:-1])
remote_file = path_list[-1]
if remote_dir != "":
# And then, go on to the remote directory (if not
already done)
if self.change_remote_dir( this_new_dir = remote_dir)
is not True:
return False
# Get remote file
err_msg = None
dst_file_id = None
dst_dir = os.path.dirname(this_full_dest_file)
if os.path.isdir(dst_dir) is False:
# Create the local directory
if gui_common.build_directories( this_directory_list =
[dst_dir,]) is False:
err_msg = "Can't create '%s'
directory"%(dst_dir)
self.error( this_msg = err_msg)
return False
# Sometimes,the FTPlib could miss the answer, or the answer
could be lost in the network...
attempt_number = 0
while attempt_number < _ftp_repeat_nb:
attempt_number += 1
err_msg = None
# Open the destination file to store its content
try:
dst_file_id = open( this_full_dest_file, 'wb',
gui_common.open_buffer_option)
except gui_common.runing_errors,why:
err_msg = "%s\n%s"%( str(why)
,
gui_common.which_process_uses_it( this_file = this_full_dest_file)
)
if "[Errno 2]" in err_msg:
# Should never occur
err_msg += gui_common.test_path(
this_path = this_full_dest_file
,
this_directory_only = True
)
except:
err_msg = "open(%s, 'wb')\n%s"%(
this_full_dest_file
,
gui_common.which_process_uses_it( this_file = this_full_dest_file)
)
if err_msg is not None:
# Oops ! Can't open a file for writting on my
host...
full_err_msg = "%s from %s: %s"%(
gui_common.ftp_download_marker
,
self.server_ip_address
, err_msg
)
self.error( this_msg = full_err_msg)
break
# Retreive back the remote file content
timeout_occures = False
try:
# Use binary transfert for ASCII and binary
files ;-)
self.ftp_id.retrbinary( cmd = 'RETR
%s'%(remote_file)
, callback =
dst_file_id.write
, blocksize =
_ftp_block_size
)
except socket.timeout,msg:
err_msg = gui_common.get_socket_error_msg(
this_error = msg)
timeout_occures = True
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg(
this_error = msg)
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e:
# The error "550 can't access file" can occure
when we try to download a file which is already opened for writing by an
another process
err_msg = e.args[0]
if "550" in err_msg:
err_msg += ". The file is maybe opened
for writing by an another process..."
except gui_common.runing_errors,why:
err_msg = str(why)
except:
err_msg = "ftp.retrbinary()"
else:
err_msg = None
# Close the file before threating the error ;-)
gui_common.force_file_synchro( this_file_id =
dst_file_id)
try :
dst_file_id.close()
except:
pass
# The result is...
if timeout_occures is not True:
if err_msg is None:
break
if _ftp_repeat_command( this_err_msg = err_msg) is
False:
break
# while attempt_number
time.sleep(_ftp_repeat_waiting_delay)
continue
if err_msg is not None:
# Oops !
full_err_msg = "%s %s: '%s' --> RETR '%s': %s"%(
gui_common.ftp_download_marker
,
self.server_ip_address
,
this_full_src_file
,
this_full_dest_file
, err_msg
)
self.error( this_msg = full_err_msg)
else:
# Download done, no error !
if self.dbg_check_size is True:
# Get host file size
try:
file_host_size =
os.path.getsize(this_full_dest_file)
except gui_common.runing_errors,why:
# Should never occure
err_msg = str(why)
except:
# Idem !
err_msg =
"os.path.getsize(%s)"%(this_full_dest_file)
else:
if file_host_size != this_size:
err_msg = "%s: host %s = %d,
remote %s = %d"%( _file_size_tag
, this_full_dest_file, file_host_size
, this_full_src_file, this_size
)
if err_msg is not None:
full_err_msg = "%s %s: SIZE: %s"%(
gui_common.ftp_download_marker
,
self.server_ip_address
,
err_msg
)
self.error( this_msg = full_err_msg)
return err_msg is None
# -----------------------
# _ftp_slave_thread.run()
# -----------------------
def run(self):
"""
Main body of the thread dedicated to a FTP elementary transfert.
"""
file_to_transfert_info = self.file_manager_id.get_waiting()
while file_to_transfert_info is not None:
# I have a file to transfert
(full_src_file, full_dest_file, file_size) =
file_to_transfert_info
# Go, go, go !
if self.do_upload is True:
# Upload wanted
tranfert_result = self.upload(
this_full_src_file = full_src_file
,
this_full_dest_file = full_dest_file
, this_size =
file_size
)
else:
# Download wanted
tranfert_result = self.download(
this_full_src_file = full_src_file
,
this_full_dest_file = full_dest_file
, this_size =
file_size
)
if tranfert_result is False:
# Can't transfert. Mark it as 'running' again.
It's craps... I know :-(
self.file_manager_id.add_running(
this_scenario_name = file_to_transfert_info)
# Pray for a succes by an another thread
break
else:
# Forever loop until all files have been
transfered
file_to_transfert_info =
self.file_manager_id.get_waiting()
# while file_to_transfert_info
continue
# Close the FTP connection
self.close_connection()
return
# ----------------------------------------------
# Private Function : _ftp_open_many_connection()
# ----------------------------------------------
def _ftp_open_many_connection( this_address
, this_login
, this_password
, this_timeout = gui_common.ftp_timeout_s
, this_upload = True
, this_user_remote_dir = _ftp_posix_separator
, this_max_connection = 2
, this_debug_mode = False
, this_nickname = None
, this_check_size = True
, this_use_sftp = False
):
"""
Open at most some connections on a dedicated serveur and associated a
thread to manage each.
@param this_address: remote IP address
@type this_address: ascii string
@param this_login: user login
@type this_login: ascii string
@param this_password: user password
@type this_password: ascii string
@param this_timeout: maximal timeout in seconds for blocking operations
like the connection attempt
@type this_timeout: int
@param this_upload: True = connection for an upload, False = connection
for an download
@type this_upload: boolean
@param this_user_remote_dir: remote user directory
@type this_user_remote_dir: ascii string
@param this_max_connection: number of connections to speed up the
transfert
@type this_max_connection: integer
@param this_debug_mode: True = display debug information, False = run
silently
@type this_debug_mode: boolean
@param this_nickname: connection nickname
@type this_nickname: ascii string
@param this_check_size: True = Check sent and downloaded file size,
False = No check
@type this_check_size: boolean
@param this_use_sftp: True = use FTPS, False = use normal FTP
@type this_use_sftp: boolean
@return: (file manager ident, ftp thread ident) tuple
@rtype: list of tuple
@note: the number of connection may be less than the one expected on
detected remote error.
"""
answer = []
file_manager_id = gui_scenario_manager.scenario_manager(
this_debug_mode = this_debug_mode)
# Be sure to have an integer
this_timeout = int(this_timeout)
# Increase timeout if neccessary
if gui_common.vpn_ip_addr() is True:
# When running with VPN
this_timeout *= 2
start_time = time.time()
cnx_idx = 0
if this_upload is True:
ud = "Upload"
else:
ud = "Download"
while this_max_connection > 0:
this_max_connection -= 1
err_msg = None
attempt_number = _ftp_repeat_nb
# Be carefull: the server may be temporarily full
while attempt_number > 0:
attempt_number -= 1
# Open a connection on the server
try:
if this_use_sftp is True:
# FTPS
ftp_id = ftplib.FTP_TLS( host =
this_address
, user =
this_login
, passwd =
this_password
, keyfile = None
, certfile =
_ftps_certificate_file_name
, timeout =
this_timeout
)
else:
# Default FTP
ftp_id = ftplib.FTP( host =
this_address
, user = this_login
, passwd =
this_password
, timeout =
this_timeout
)
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg(
this_error = msg)
except gui_common.runing_errors,why:
err_msg = str(why)
except:
err_msg = "ftplib.FTP()"
else:
# Yes !
err_msg = None
break
if _ftp_repeat_command( this_err_msg = err_msg) is
False:
break
# while attempt_number
time.sleep(_ftp_repeat_waiting_delay)
continue
if err_msg is not None:
# Oops ! No FTP connection...
msg = "Unable to open FTP connection (addr: %s, login:
%s, password: %s, timeout: %ds)\n| %s"%( this_address
, this_login
, this_password
, this_timeout
, err_msg
)
gui_common.save_error( this_msg = msg
, this_object_class = None
, this_file_name = __file__
, this_mail_subject = "%s %s:
%s"%(_ftp_error_tag,this_address,err_msg)
)
# We can not open as many connection as we would
like... Run whith the openned ones as 'fail-soft' mode ;-)
return answer
else:
if this_use_sftp is True:
# Switch ON secure data connection
ftp_id.prot_p()
# Set FTP socket options
if gui_common.set_socket_options( this_socket_id =
ftp_id.sock
, this_timeout_s =
this_timeout
, this_device =
gui_common.io_device_ethernet0
, this_server_purpose =
False
, this_debug =
this_debug_mode
, this_mtu =
gui_common.ip_mtu
) is False:
_ftp_close_connection( this_ftp_id = ftp_id)
return answer
# Set FTP debug trace level
# 0: no debugging output (default)
# 1: print commands and responses but not body text
etc.
# 2: also print raw lines read and sent before
stripping CR/LF
if this_debug_mode is True:
ftp_id.set_debuglevel( level = 1 )
else:
ftp_id.set_debuglevel( level = 0 )
# Force 'passive mode'
ftp_id.set_pasv( True )
# Create a dedicated thread which manage the transfert.
It will be launched later by the caller.
ftp_thread_id = _ftp_slave_thread( this_ftp_id_id =
ftp_id
,
this_server_ip_address = this_address
, this_file_manager_id
= file_manager_id
, this_upload =
this_upload
, this_user_remote_dir
= this_user_remote_dir
, this_start_time =
start_time
, this_debug_mode =
this_debug_mode
, this_check_size =
this_check_size
)
cnx_idx += 1
if this_nickname is None:
thread_name = "FTP_%s_%s_%d"%( this_address
, ud
, cnx_idx
)
else:
thread_name = "FTP_%s_%s_%s_%d"%( this_address
, this_nickname
, ud
, cnx_idx
)
ftp_thread_id.setName(thread_name)
info = (file_manager_id, ftp_id, ftp_thread_id)
answer.append(info)
# While this_max_connection
continue
return answer
# -----------------------------------------------
# Private Function : _ftp_close_connection_list()
# -----------------------------------------------
def _ftp_close_connection_list( this_list ):
"""
Close opened FTP connection thanks to _ftp_open_many_connection()
@param this_list: (file manager ident, ftp thread ident) tuple
@type this_list: list of tuples comming from _ftp_open_many_connection()
"""
for ftp_cnx_info in this_list :
(file_manager_id, ftp_id, ftp_thread_id) = ftp_cnx_info
_ftp_close_connection( this_ftp_id = ftp_id)
# Next ftp_cnx_info
continue
return
# --------------------------------------
# Function : _ftp_get_directory_detail()
# --------------------------------------
def _ftp_get_directory_detail( this_ftp_id = None
, this_dir = None
):
"""
Get detailed remote directory info (Filezilla Server)
@param this_ftp_id: open FTP connection
@type this_ftp_id: ftplib object
@param this_ftp_id: remote directory. if None, the current one is used
@type this_ftp_id: ascii string
@return
"""
# Go to the given directory
if this_dir is not None:
this_ftp_id.cwd( this_dir )
# Ask information
file_list = []
this_ftp_id.dir(file_list.append)
# The Filezilla Server answer format looks like:
# -rw-r--r-- 1 ftp ftp 1608780 Aug 29 2012 FileZilla Server.log
# -rw-r--r-- 1 ftp ftp 2634337 Mar 10 17:17 fzs-2015-03-10.log
# -rw-r--r-- 1 ftp ftp 7296369 Mar 11 17:39 fzs-2015-03-11.log
current_year = int(time.strftime("%Y", time.gmtime()))
answer = []
for info_idx in file_list:
splitted_line = info_idx.split()
user_group_other_rights = splitted_line[0]
directory_flag = user_group_other_rights[0] # 'd' or '-'
user_group_other_rights = user_group_other_rights[1:]
inode_number = splitted_line[1]
user_name = splitted_line[2]
group_name = splitted_line[3]
size = int(splitted_line[4])
month = splitted_line[5]
date = int(splitted_line[6])
if ':' in splitted_line[7]:
# Hour only
hour_minute = splitted_line[7]
year = current_year
else:
# Year only
hour_minute = "00:00"
year = int(splitted_line[7])
file = " ".join(splitted_line[8:])
element = (directory_flag, size, year, month, date,
hour_minute, file)
answer.append(element)
# Next info_idx
continue
return answer
# ---------------------------------
# Function : _ftp_get_server_info()
# ---------------------------------
def _ftp_get_server_info( this_ftp_id ):
"""
Return the FTP server information like name and features
@param this_ftp_id: open FTP connection
@type this_ftp_id: ftplib object
@return: remote FTP server name and features
@rtype: ascii string
"""
err_msg = None
try:
system_info = this_ftp_id.sendcmd( cmd = 'SYST')
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg( this_error = msg)
else:
expected_answer = "215 "
if system_info.startswith(expected_answer) is True:
# Yeap ! We have the offical system name like '215 UNIX
emulated by FileZilla', ignore the number...
system_info = system_info[len(expected_answer):]
if err_msg is not None:
gui_common.save_error( this_msg = "sendcmd('SYST') :
%s"%(err_msg)
, this_object_class = this_ftp_id
, this_file_name = __file__
, this_mail_subject = "%s:
%s"%(_ftp_error_tag,err_msg)
)
system_info = "Unknown system"
# Feature Negotiation... [RFC2389]
feature_list = []
try:
response = this_ftp_id.sendcmd('FEAT')
except:
# Command not supported ! Snif
pass
else:
# The Filezilla server answer looks like: b'211-Features:\n
MDTM\n REST STREAM\n SIZE\n MLST type*;size*;modify*;\n MLSD\n UTF8\n CLNT\n
MFMT\n211 End'
expected_answer = "211"
if response.startswith(expected_answer) is True:
# Yeap ! ignore the number and the 'End'...
feature_list = [idx for idx in
response.split(':')[1].split('\n')[:-1] if idx != ""]
# ---------------------------------------------
# <<<Options (OPTS)>>>
# Synopsis
# The OPTS command is a required command if the FEAT
command is also implemented.
# The OPTS command is used to provide additional
information for extended features supported by the FTP server.
# Description
# The OPTS command will be followed by the name of
the command requiring additional information.
# Following the command being configured will be
additional parameters that have meaning only in the context of the command
being configured.
# FTP Voyager currently issues the OPTS command
under two circumstances:
# MODE - FTP Voyager will issue an OPTS MODE
command to servers to configure extended options for a given transfer mode.
Most commonly MODE is implemented in the "MODE Z LEVEL X" command for
supporting on-the-fly compression to configure the level of compression to use
for the session where 'X' is a number between 1 (fastest, least compression)
and 10 (slowest, most compression). FTP Voyager uses a default value of 6.
# MLST - FTP Voyager may issue an OPTS MLST
command to servers supporting the MLST command in order to configure the amount
of information sent by the server in response to the MLST command. It is issued
in the format OPTS MLST Info1;Info2;Info3 etc, with each type of information
separated by a semicolon. Information types can include Type (filetype), Size
(filesize), Modify (last modification date), Create (date of file creation),
and more. The information types the server supports are sent to the client as
part of the server's response to the FEAT command, and used to determine the
contents of the OPTS MLST command issued to the server.
# UTF8 - Configures the server to enable (ON) or
disable (OFF) UTF-8 encoding which is useful for non-ANSI character sets. This
is very useful for Asian file names and paths.
#
# COMMAND:> OPTS MLST
Type;Size;Modify;UNIX.mode;UNIX.owner;UNIX.group;
# 200 MLST OPTS
Type;Size;Modify;UNIX.mode;UNIX.owner;UNIX.group;
# ---------------------------------------------
# <<<Modify Fact: Modification Time (MFMT)>>>
# Synopsis
# The MFMT command is used to modify a file or
folder's last modified date and time information. MFMT duplicates similar
functionality implemented by the MDTM command.
# Description
# Traditionally, when a file or folder is uploaded
to an FTP server, the last modified date and time of the file or folder is set
to the transfer date and time.
# Using MFMT, FTP clients can inform supporting FTP
servers of the proper last modified date and time to use for the file or folder.
# The format of the command is MFMT YYYYMMDDHHMMSS
path, where:
# YYYY - the 4-digit year
# MM - the 2-digit month
# DD - the 2-digit day of the month
# HH - the hour in 24-hour format
# MM - the minute
# SS - the seconds
# All times are represented in GMT/UTC.
# ---------------------------------------------
# <<<Modification Time (MDTM)>>>
# Synopsis
# The MDTM command implemented by FTP is used to
preserve a file's original date and time information after file transfer.
# Description
# Traditionally, when a file is uploaded to an FTP
server, the date/time of the file is set to the transfer date/time.
# Using MDTM, FTP Voyager can inform supporting FTP
servers of the proper date/time to use for the file.
# The format of the command is MDTM YYYYMMDDHHMMSS,
where:
# YYYY - the 4-digit year
# MM - the 2-digit month
# DD - the 2-digit day of the month
# HH - the hour in 24-hour format
# MM - the minute
# SS - the seconds
# ---------------------------------------------
answer = (system_info, feature_list)
return answer
# -----------------------------------------
# Private Function : _ftp_get_remote_info()
# -----------------------------------------
def _ftp_get_remote_info( this_ftp_id = None
, this_ftp_tag = None
, this_tag_selector = []
, this_remote_directory_flag = None
):
"""
Return the listing of a remote file / directory
@param this_ftp_id: open FTP connection
@type this_ftp_id: ftplib object
@param this_ftp_tag: target directory or file to list
@type this_ftp_tag: ascii string
@param this_tag_selector: optional tag selector
@type this_tag_selector: list of ascii string
@param this_remote_directory_flag: False = remote file, True = remote
directory, None = to be determined
@type this_remote_directory_flag: boolean
@return: list of (full remote path name, is_directory, file_size)
@rtype: list of tuple (ascii string, boolean, integer)
@note:
- use the standard MLSD command instead of LIST (result os
dependant)
- NLST returns the file names without any type :-(
- The MLST command only provides detailed information about a
single file or folder
while the MLSD can provide information about multiple files
and folders.
"""
callback_answer_list = []
# ---------------------------------
# Private: ftp_mlsd_mlst_callback()
# ---------------------------------
def ftp_mlsd_mlst_callback( this_input ):
"""
Analyse the standard MLSD & MLST answer (local function for
recursivity)
Update 'answer' in relation with the incoming data
The FTP MLSD answer should looks like:
- type=dir;modify=20100114090849; Lib
- type=file;modify=20100114090849;size=1190652; swig.exe
the MLST answer should looks like:
- type=file;size=386456;modify=20150317130322;
/AGERMANE.3.zip
"""
# Be carefull: if the command has been sent with retbinary we
have all result in one answer with '\n\r' as separator !
for input_idx in this_input.split(gui_common.char_crlf):
if input_idx == "":
# Ignore it !
continue
# For each record element
full_splitted = [element_idx.strip() for element_idx in
input_idx.split(';')]
# Get name and apply a filter
name = full_splitted[-1].strip()
if name[0] == _ftp_posix_separator:
name = name[1:]
match = True
for tag_idx in this_tag_selector:
if tag_idx not in name:
# Discard it !
match = False
break
# Next tag_idx
continue
if match is True:
# Get 'type' info: file or directory
splitted_record = full_splitted[0].split('=')
if splitted_record[0] != "type":
gui_common.save_error( this_msg =
"unexpected format: %s"%(this_input)
,
this_object_class = this_ftp_id
, this_file_name =
__file__
,
this_mail_subject = _ftp_error_tag
)
return
is_a_directory = (splitted_record[1] == "dir")
# Get file size (not available for directory)
file_size = 0
if is_a_directory is False:
# !!! 'size' & 'modify' parameters are
inverted between MLST & MLSD results !!!
for idx in
range(1,len(full_splitted)-1):
splitted_record =
full_splitted[idx].split('=')
if splitted_record[0] == "size":
# Yeap !
file_size =
int(splitted_record[1])
break
# Next idx
continue
# The result is...
result = (name, is_a_directory, file_size)
callback_answer_list.append(result)
# Next input_idx
continue
return
# -------------------------------
# Private: sort_callback_result()
# -------------------------------
def sort_callback_result(tag1,tag2):
(name1, is_a_directory1, file_size1) = tag1
(name2, is_a_directory2, file_size2) = tag2
if is_a_directory1 is True:
if is_a_directory2 is True:
# 2 directories --> sort by directory name
if name1 < name2:
return -1
else:
return 1
else:
# Dir 1, file 2
return 1
else:
if is_a_directory2 is True:
# File 1, dir 2
return -1
else:
# 2 files --> sort by by file name
if name1 < name2:
return -1
else:
return 1
return 0
#
# Body: _ftp_get_remote_info()
#
answer = []
err_msg = None
# Do some checks
if this_ftp_tag is None:
# All default directory content
this_ftp_tag = _ftp_posix_separator
remote_directory = _ftp_posix_separator
# FTP or SFTP ?
try:
sftp_in_use = this_ftp_id._prot_p
except:
sftp_in_use = False
# What is the remote target ?
if this_remote_directory_flag is None:
# Verify if it's a remote file or a remote directory
try:
remote_file_size = this_ftp_id.size( this_ftp_tag)
except ftplib.error_perm, e:
# It's a directory
this_remote_directory_flag = True
else:
# It's a file. Ask remote file information
this_remote_directory_flag = False
# Get remote information
if this_remote_directory_flag is True:
remote_directory = this_ftp_tag+_ftp_posix_separator
# Go to the selected directory
try:
this_ftp_id.cwd( dirname = this_ftp_tag)
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg( this_error =
msg)
else:
# Ask remote directory information
try:
if sftp_in_use is True:
result = this_ftp_id.retrbinary( cmd =
'MLSD'
,
callback = ftp_mlsd_mlst_callback
)
else:
result = this_ftp_id.retrlines( cmd =
'MLSD'
,
callback = ftp_mlsd_mlst_callback
)
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg(
this_error = msg)
if err_msg is not None:
gui_common.save_error( this_msg =
"retrlines('MLSD'): %s"%(err_msg)
, this_object_class =
this_ftp_id
, this_file_name = __file__
, this_mail_subject = "%s:
%s"%(_ftp_error_tag,err_msg)
)
return answer
else:
remote_directory = ""
# Obtain remote file information
try:
result = this_ftp_id.sendcmd( cmd = 'MLST
'+this_ftp_tag)
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg( this_error =
msg)
else:
# The MLST answer looks like: b'250-Listing
/AGERMANE.3.zip\n type=file;size=386456;modify=20150317130322;
/AGERMANE.3.zip\n250 End'
splitted_answer = result.split('\n')
if '250' in splitted_answer[0]:
# Yeap !
this_remote_directory_flag = False
ftp_mlsd_mlst_callback ( this_input =
splitted_answer[1])
else:
err_msg = "Unexpected answer: '%s'"%(result)
if err_msg is not None:
gui_common.save_error( this_msg = "sendcmd('MLST'):
%s"%(err_msg)
, this_object_class = ftp_id
, this_file_name = __file__
, this_mail_subject = "%s:
%s"%(_ftp_error_tag,err_msg)
)
return answer
# Analyse the result of the file or of the current directory...
for remote_info_idx in callback_answer_list:
try:
(name, is_directory, file_size) = remote_info_idx
except gui_common.runing_errors,why:
err_msg = "Unexpected 'local_info_idx' format: %s -
'%s'"%(str(why), str(remote_info_idx))
gui_common.save_error( this_msg = err_msg
, this_object_class = this_ftp_id
, this_file_name = __file__
, this_mail_subject = "%s:
%s"%(_ftp_error_tag,err_msg)
)
return answer
if is_directory is True:
# It's a directory, go on recursively
if this_ftp_tag is None:
new_dir = _ftp_posix_separator+name
else:
new_dir = name
answer2 = _ftp_get_remote_info( this_ftp_id =
this_ftp_id
, this_ftp_tag = new_dir
, this_tag_selector =
this_tag_selector
,
this_remote_directory_flag = is_directory
)
# Build a full path answer
for remote_info_jdx in answer2:
try:
(name2, is_directory2, file_size2) =
remote_info_jdx
except gui_common.runing_errors,why:
err_msg = "Unexpected 'remote_info_jdx'
format: %s - '%s'"%(str(why), str(remote_info_jdx))
gui_common.save_error( this_msg =
err_msg
,
this_object_class = this_ftp_id
, this_file_name =
__file__
,
this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg)
)
return answer
remote_info_jdx =
(this_ftp_tag+_ftp_posix_separator+name2, is_directory2, file_size2)
answer.append(remote_info_jdx)
# Next remote_info_jdx
continue
# Return to the parent directory
try:
result = this_ftp_id.sendcmd( cmd = 'CDUP')
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg(
this_error = msg)
if err_msg is not None:
gui_common.save_error( this_msg =
"sendcmd(CDUP): %s"%(err_msg)
, this_object_class =
this_ftp_id
, this_file_name = __file__
, this_mail_subject = "%s:
%s"%(_ftp_error_tag,err_msg)
)
return answer
# Do a difference between directory size and empty file
size
file_size = _ftp_dir_size
# Answer with full path name
remote_info_idx = (remote_directory+name, is_directory,
file_size)
answer.append(remote_info_idx)
# Next remote_info_idx
continue
return answer
# ---------------------------
# Private Class : _ftp_thread
# ---------------------------
class _ftp_thread( threading.Thread):
"""
Independant FPT thread to upload or download several host in parallele
@sort:
_*,a*,b*,c*,d*,e*,f*,g*,h*,i*,j*,k*,l*,m*,n*,o*,p*,q*,r*,s*,t*,u*,v*,w*,x*,y*,z*
"""
# ----------------------
# _ftp_thread.__init__()
# ----------------------
def __init__( self
, this_address = "1.2.3.4"
, this_login = "dummy_login"
, this_password = "dumthr_password"
, this_timeout = gui_common.ftp_timeout_s
, this_src_path = None
, this_dst_path = None
, this_direction= _upload_cmd
, this_cleanup_flag = False
, this_debug_mode = False
):
"""
FTP thread initialisation
@param this_address: address of the server
@type this_address: ascii
@param this_login: remote user login
@type this_login: ascii
@param this_password: remote user password
@type this_password: ascii
@param this_timeout: timeout in seconds for blocking operations
like the connection attempt
@type this_timeout: real
@param this_src_path: source path name (file or directory). A
wildcard '*' is allowed.
@type this_src_path: ascii string
@param this_dst_path: server destination path name
@type this_dst_path: ascii string
@param this_direction: 'Upload' or 'Download' transfert
direction
@type this_direction: ascii string
@param this_cleanup_flag: force to clean up the destination
directory (True) or not (False)
@type this_cleanup_flag: boolean
@param this_debug_mode: True = with debug mode, False = silent
@type this_debug_mode: boolean
@return: None
"""
# Initialise my thread
self.thread_id = threading.Thread.__init__(self)
self.result = True
self.cleanup_flag = this_cleanup_flag
# Save input parameters
self.address = this_address
self.login = this_login
self.timeout = this_timeout
self.src_path = this_src_path
self.dst_path = this_dst_path
self.direction = this_direction
self.debug_mode = this_debug_mode
# Force the Lab server password if possible
if this_login == gui_common.main_login:
self.password = gui_common.get_root_password(
this_ip_address = self.address)
if self.password is None:
# It's an another host, use the given password
self.password = this_password
else:
# Use the given password
self.password = this_password
return
# -------------------
# _ftp_thread.error()
# -------------------
def error( self
, this_msg
):
"""
Shortcut access to display an error
@param this_msg: input message describing the error
@type this_msg: ascii string
@return: None
"""
gui_common.save_error( this_msg = this_msg
, this_object_class = self
, this_file_name = __file__
, this_mail_subject = _ftp_error_tag
)
self.result = False
return
# ------------------------
# _ftp_thread.save_trace()
# ------------------------
def save_trace( self
, this_msg
, this_caller_name
):
"""
Shortcut access to display an error
@param this_msg: input message describing the error
@type this_msg: ascii string
@param this_caller_name: caller function name
@type this_caller_name: ascii string
@return: None
"""
gui_common.save_trace( this_msg = this_msg
, this_object_class = self
, this_file_name = __file__
, this_caller_name = this_caller_name
)
return
# -----------------
# _ftp_thread.run()
# -----------------
def run(self):
"""
FTP thread body
@return: None
"""
if self.direction == _upload_cmd:
# Upload wanted
if self.cleanup_flag is True:
# Clean Up wanted on the remote host
if self.debug_mode is True:
self.save_trace( this_msg = "Try to
remove server directory %s : %s"%(self.address,self.dst_path)
, this_caller_name =
"run()"
)
self.result = ftp_remove( this_address =
self.address
, this_login =
self.login
, this_password =
self.password
, this_timeout =
self.timeout
, this_dst_path =
self.dst_path
)
if self.result is not True:
self.error( this_msg = "Remove server
directory %s : %s"%(self.address,self.dst_path))
else:
if self.debug_mode is True:
self.save_trace( this_msg =
"Remove server directory %s done : %s"%(self.address,self.dst_path)
,
this_caller_name = "run()"
)
# And then upload !
self.result = ftp_upload( this_address = self.address
, this_login = self.login
, this_password =
self.password
, this_timeout = self.timeout
, this_src_path =
self.src_path
, this_dst_path =
self.dst_path
)
if self.result is True:
if self.debug_mode is True:
self.save_trace( this_msg = "Upload
server %s done"%(self.address)
, this_caller_name =
"run()"
)
return
if self.direction == _download_cmd:
if self.cleanup_flag is True:
# Clean Up wanted on the running host
if os.path.isdir(self.dst_path) is True:
# And the directory already exists
self.result =
gui_common.remove_directory( this_target = self.dst_path+os.sep+"*")
if self.result is not True:
# Oops !
return
# Retreive back all files and directories
self.result = ftp_download( this_address =
self.address
, this_login = self.login
, this_password =
self.password
, this_timeout =
self.timeout
, this_src_path =
self.src_path
, this_dst_path =
self.dst_path
)
if self.result is not True:
self.error( this_msg = "Download server %s with
%s"%(self.address,self.src_path))
else:
if self.debug_mode is True:
self.save_trace( this_msg = "Download
from server %s done"%(self.address)
, this_caller_name =
"run()"
)
return
# Should never occure
self.error( this_msg = "Unknown command: %s"%(self.direction))
return
### ---------------- ###
### Public functions ###
### ---------------- ###
# -----------------------
# Function : ftp_remove()
# -----------------------
def ftp_remove( this_address = "1.2.3.4"
, this_login = "dummy_login"
, this_password = "dummy_password"
, this_timeout = gui_common.ftp_timeout_s
, this_dst_path = None
, this_debug_mode = False
, this_use_sftp = False
):
"""
FTP Remove recursivly file(s) and directory(ies). Use recursivity when
a directory is found.
@param this_address: address of the server
@type this_address: ascii
@param this_login: remote user login
@type this_login: ascii
@param this_password: remote user password
@type this_password: ascii
@param this_timeout: timeout in seconds for blocking operations like
the connection attempt
@type this_timeout: real
@param this_dst_path: local destination directory
@type this_dst_path: ascii string
@param this_debug_mode: current debug mode
@param this_use_sftp: True = use FTPS, False = use normal FTP
@type this_use_sftp: boolean
@type this_debug_mode: boolean
@return: True : transfert done / False : error
@rtype: boolean
"""
#-------------------------------
# local function for recursivity
#-------------------------------
def recursive_ftp_remove( this_dst_path
, this_is_a_directory = False
):
"""
local function for recursivity calls
@param this_src_path: server source directory
@type this_src_path: ascii string
@param this_is_a_directory: private parameter: True =
directory, False = file (could not exist !)
@type this_is_a_directory: boolean
"""
answer = True
# Be careful with the first call... ;-)*
dst_dir_list = []
err_msg = None
if this_is_a_directory is False:
# Determine if it's a file or a directory
try:
# Is 'file' file or directory ?
remote_file_size = ftp_id.size(this_dst_path)
# If here : it's a file and it exists
dst_dir_list.append(this_dst_path)
except ftplib.error_perm, e:
# It's a directory or the file does not exist !
recursive_ftp_remove( this_dst_path =
this_dst_path
, this_is_a_directory = True
)
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg(
this_error = msg)
except gui_common.runing_errors,why:
err_msg = str(why)
if err_msg is not None:
gui_common.save_error( this_msg = "delete(%s)
on %s : %s"%(this_dst_path,this_address,err_msg)
, this_object_class =
ftp_id
, this_file_name = __file__
, this_mail_subject = "%s:
%s"%(_ftp_error_tag,err_msg)
)
return False
else:
# It's a directory or the file does not exist !
try:
dst_dir_list = ftp_id.nlst(this_dst_path)
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e:
# Be careful with '550 Directory not found'
err_msg = str(e.args[0])
if '550' in err_msg:
# The file does not exist ! It wasn't a
directory...
return True
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg(
this_error = msg)
except gui_common.runing_errors,why:
err_msg = str(why)
if err_msg is not None:
gui_common.save_error( this_msg = "nlst(%s) on
%s : %s"%(this_dst_path,this_address,err_msg)
, this_object_class =
ftp_id
, this_file_name = __file__
, this_mail_subject = "%s:
%s"%(_ftp_error_tag,err_msg)
)
return False
# Exam the result list
for dst_file in dst_dir_list:
# Display debug information (if wanted)
if this_debug_mode is True:
gui_common.save_trace( this_msg =
"recursive_ftp_remove(%s) : %s"%(this_address,dst_file)
, this_object_class = None
, this_file_name = None
, this_caller_name =
"recursive_ftp_remove()"
)
# Is it a file or a directory ?
try:
remote_file_size = ftp_id.size(dst_file)
# If here : it's a file. Remove it.
attempt_number = _ftp_repeat_nb
while attempt_number > 0:
attempt_number -= 1
try:
ftp_id.delete(dst_file)
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.error, msg:
err_msg =
gui_common.get_socket_error_msg( this_error = msg)
except gui_common.runing_errors,why:
err_msg = str(why)
else:
err_msg = None
break
if _ftp_repeat_command( this_err_msg =
err_msg) is False:
break
# while attempt_number
time.sleep(_ftp_repeat_waiting_delay)
continue
if err_msg is not None:
gui_common.save_error( this_msg =
"recursive_ftp_remove(%s) -> delete(%s) %d bytes:
%s"%(this_address,this_dst_path,remote_file_size,err_msg)
,
this_object_class = ftp_id
, this_file_name =
__file__
,
this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg)
)
return False
except ftplib.error_perm, e:
# 5xx errors, It's may be a directory or it
does not exist...
if recursive_ftp_remove( dst_file
, this_is_a_directory =
True
) is not True:
return False
# Remove the directory
attempt_number = _ftp_repeat_nb
while attempt_number > 0:
attempt_number -= 1
try:
ftp_id.rmd(dst_file)
except ftplib.all_errors, e:
err_msg = e.args[0]
except socket.error, msg:
err_msg =
gui_common.get_socket_error_msg( this_error = msg)
except gui_common.runing_errors,why:
err_msg = str(why)
else:
err_msg = None
break
# Do a last control... in case of '550
Directory not found'
if "Directory not found" in err_msg:
# The error is normal in this
case ! It should occure during the first call ;-)
err_msg = None
break
if _ftp_repeat_command( this_err_msg =
err_msg) is False:
break
# while attempt_number
time.sleep(_ftp_repeat_waiting_delay)
continue
if err_msg is not None:
gui_common.save_error( this_msg =
"recursive_ftp_remove(%s) -> rmd(%s) : %s"%(this_address,this_dst_path,err_msg)
,
this_object_class = ftp_id
, this_file_name =
__file__
,
this_mail_subject = "%s: %s"%(_ftp_error_tag,err_msg)
)
return False
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg(
this_error = msg)
except gui_common.runing_errors,why:
err_msg = str(why)
except :
err_msg = "???"
if err_msg is not None:
gui_common.save_error( this_msg =
"recursive_ftp_remove(%s) -> delete(%s) :
%s"%(this_address,this_dst_path,err_msg)
, this_object_class =
ftp_id
, this_file_name = __file__
, this_mail_subject = "%s:
%s"%(_ftp_error_tag,err_msg)
)
return False
# Next dst_file
continue
return True
#
# Main body of ftp_remove()
#
if this_debug_mode is True:
gui_common.save_trace( this_msg = "ftp_remove(%s) destination
path %s, login: %s, pass:
%s"%(this_address,this_dst_path,this_login,this_password)
, this_object_class = None
, this_file_name = __file__
, this_caller_name = "ftp_remove()"
)
err_msg = None
# Open the FTP connection
try:
if this_use_sftp is True:
# FTPS
ftp_id = ftplib.FTP_TLS( host = this_address
, user = this_login
, passwd = this_password
, keyfile = None
, certfile =
_ftps_certificate_file_name
, timeout = this_timeout
)
else:
# Default FTP
ftp_id = ftplib.FTP( host = this_address
, user = this_login
, passwd = this_password
, timeout = this_timeout
)
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg( this_error = msg)
except gui_common.runing_errors,why:
err_msg = str(why)
if err_msg is not None:
gui_common.save_error( this_msg = "ftp_remove(%s) : Unable to
open a FTP connection (login: %s, password: %s) :
%s"%(this_address,this_login,this_password,err_msg)
, this_file_name = __file__
, this_mail_subject = "%s:
%s"%(_ftp_error_tag,err_msg)
)
return False
if this_debug_mode is True:
# Set FTP debug trace level
# 0: no debugging output (default)
# 1: print commands and responses but not body text etc.
# 2: also print raw lines read and sent before stripping CR/LF
ftp_id.set_debuglevel( level = 1 )
else:
# The default value
ftp_id.set_debuglevel( level = 0 )
# Go on
if this_dst_path is None :
# Take the current remote directory as default one
try:
this_dst_path = ftp_id.pwd()
except:
gui_common.save_error( this_msg = "ftp_remove(%s) :
Unable to obtain the current FTP remote directory"%(this_address)
, this_object_class = ftp_id
, this_file_name = __file__
, this_mail_subject =
_ftp_error_tag
)
return False
# Go
result = recursive_ftp_remove(this_dst_path)
# Close the FTP connection
_ftp_close_connection( this_ftp_id = ftp_id)
# Done
return result
# -------------------------
# Function : ftp_download()
# -------------------------
def ftp_download( this_address = "1.2.3.4"
, this_login = "dummy_login"
, this_password = "dummy_down_password"
, this_timeout = gui_common.ftp_timeout_s
, this_src_path = None
, this_dst_path = None
, this_nickname = None
, this_check_size = False
, this_max_connection = None
, this_remote_directory_flag = None
, this_debug_mode = False
):
"""
Download FTP a file or a directory and its content. Use recursivity
when a directory is found.
@param this_address: address of the server
@type this_address: ascii
@param this_login: remote user login
@type this_login: ascii
@param this_password: remote user password
@type this_password: ascii
@param this_timeout: timeout in seconds for blocking operations like
the connection attempt
@type this_timeout: real
@param this_src_path: server source directory or file path. Wildcard
'*' is allowed on the basename.
@type this_src_path: ascii string
@param this_dst_path: local destination directory
@type this_dst_path: ascii string
@param this_user_conf: user configuration
@type this_user_conf: gui_user.configuration ident
@param this_check_size: True = Check sent and downloaded file size,
False = No check
@type this_check_size: boolean
@param this_max_connection: [optional] max connection number
@type this_max_connection: integer
@param this_remote_directory_flag: False = remote file, True = remote
directory, None = to be determined
@type this_remote_directory_flag: boolean
@return: True : transfert done / False : error
@rtype: boolean
"""
if this_debug_mode is True:
msg = "ftp_download(%s) : from %s to %s, login: %s, pass:
%s"%(this_address,this_src_path,this_dst_path,this_login,this_password)
gui_common.save_trace( this_msg = msg
, this_object_class = None
, this_file_name = __file__
)
err_msg = None
dummy = os.path.basename(this_src_path)
if "*" in dummy:
# It's a file or a directory with a wildcard
tag_list = [tag_idx for tag_idx in dummy.split("*") if tag_idx
!= ""]
src_path = os.path.dirname(this_src_path)
if src_path == "":
# No directory path, use the current one !
src_path = os.curdir
else:
# It should be a directory name or a file name
tag_list = []
src_path = this_src_path
# FTP or SFTP ?
sftp_in_use = gui_common.get_sftp_usage( this_ip_address = this_address)
if this_max_connection is None:
# Get the maximal allowed connection number
max_expected_connections_number = _ftp_get_max_connection(
this_address = this_address)
else:
# Use the wanted user connection number
max_expected_connections_number = this_max_connection
ftp_cnx_info_list = _ftp_open_many_connection( this_address =
this_address
, this_login = this_login
, this_password =
this_password
, this_timeout =
this_timeout
, this_upload = False
, this_max_connection =
max_expected_connections_number
, this_user_remote_dir =
src_path
, this_nickname =
this_nickname
, this_check_size =
this_check_size
, this_use_sftp =
gui_common.get_sftp_usage( this_ip_address = this_address)
, this_debug_mode =
this_debug_mode
)
if ftp_cnx_info_list == []:
# No opened connection on this server
return False
# Use the first connection for me. It will be closed by the slave
thread.
(file_manager_id, ftp_id, _ftp_thread_id) = ftp_cnx_info_list[0]
# Go down to work !
if this_dst_path is None :
# Take the current source directory as default one
this_dst_path = os.getcwd()
if this_src_path is None :
# Take the current remote directory as default one ;-)
try:
this_src_path = ftp_id.pwd()
except
(ftplib.error_temp,ftplib.error_reply,ftplib.error_perm,ftplib.error_proto), e:
err_msg = e.args[0]
except socket.error, msg:
err_msg = gui_common.get_socket_error_msg( this_error =
msg)
except:
err_msg = "Unable to obtain the current FTP remote
directory"
if err_msg is not None:
gui_common.save_error( this_msg = err_msg
, this_object_class = ftp_id
, this_file_name = __file__
, this_mail_subject =
_ftp_error_tag
)
# Close the FTP connections
_ftp_close_connection_list( this_list =
ftp_cnx_info_list)
del file_manager_idx
return False
else:
# Use POSIX name format
this_src_path =
this_src_path.replace(os.sep,_ftp_posix_separator)
# Get remote files & directories
remote_file_list = _ftp_get_remote_info( this_ftp_id = ftp_id
, this_ftp_tag = this_src_path
, this_tag_selector = tag_list
)
if remote_file_list == []:
# Nothing to transfert
_ftp_close_connection_list( this_list = ftp_cnx_info_list)
del ftp_cnx_info_list
gui_common.garbage_collector_start()
return True
# Determine the src&dest files & directories
local_dir_list = []
for file_info_idx in remote_file_list:
try:
(remote_file_name, is_directory, file_size) =
file_info_idx
except gui_common.runing_errors,why:
gui_common.save_error( this_msg = "unexpected
file_info_idx format: %s, %s"%(str(file_info_idx),str(why))
, this_file_name = __file__
, this_mail_subject =
_ftp_error_tag
)
_ftp_close_connection_list( this_list =
ftp_cnx_info_list)
gui_common.garbage_collector_start()
return False
real_dst_path = this_dst_path+os.sep+remote_file_name
# FTP format: Linux --> Windows
real_dst_path = real_dst_path.replace('/',os.sep)
if is_directory is True:
# Remote directory
local_dir_list.append(real_dst_path)
else:
# Remote File
file_info = (remote_file_name, real_dst_path, file_size)
file_manager_id.add_waiting( this_scenario_name =
file_info
,
this_ignore_duplication_warning = True
)
# Next file_info_idx
continue
if local_dir_list != []:
# Build local directories
gui_common.build_directories( this_directory_list =
local_dir_list)
# Retreive back the files now !
for ftp_cnx_info in ftp_cnx_info_list:
(file_manager_id, ftp_id, ftp_thread_id) = ftp_cnx_info
ftp_thread_id.start()
# Next ftp_cnx_info
continue
# Wait end of transfert
result = True
for ftp_cnx_info in ftp_cnx_info_list:
(file_manager_id, ftp_id, ftp_thread_id) = ftp_cnx_info
ftp_thread_id.join()
result = result and ftp_thread_id.result
# Next ftp_cnx_info
continue
_ftp_close_connection_list( this_list = ftp_cnx_info_list)
del ftp_cnx_info_list
# Check if the transfert has been done correctly
remaining_file_list = file_manager_id.get_running_list(this_name_only =
True)
if remaining_file_list != []:
# Some files have not been transfered
subject = "%s Some files have not been downloaded to %s from
%s"%( gui_common.ftp_download_marker
, gui_common.get_ip_address()
, this_address
)
file_list = [this_src_path,]
file_list += [os.path.basename(full_src_file) for full_src_file
in remaining_file_list if isinstance(full_src_file,(str,unicode)) is True]
gui_common.send_mail( this_subject = subject
, this_content =
gui_common.char_line_feed.join([src_file for (src_file, dest_file, file_size)
in remaining_file_list])
, this_receiver_email_addr =
gui_common.administrator_email
)
# Done
gui_common.garbage_collector_start()
return result
# -----------------------
# Function : ftp_upload()
# -----------------------
def ftp_upload( this_address = "1.2.3.4"
, this_login = "dummy_up_login"
, this_password = "dummy_up_password"
, this_timeout = gui_common.ftp_timeout_s
, this_src_path = None
, this_dst_path = None
, this_unwanted_tag_list = [gui_common.svn_dir_name,]
, this_nickname = None
, this_check_size = True
, this_max_connection = None
, this_debug_mode = False
):
"""
Upload FTP a file or a directory and its content. Use recursivity when
a directory is found.
@param this_address: address of the server
@type this_address: ascii
@param this_login: remote user login
@type this_login: ascii
@param this_password: remote user password
@type this_password: ascii
@param this_timeout: timeout in seconds for blocking operations like
the connection attempt
@type this_timeout: real
@param this_src_path: source path name (file or directory). A wildcard
'*' is allowed inside a file name (not the directory one).
@type this_src_path: ascii string
@param this_dst_path: server destination path name
@type this_dst_path: ascii string
@param this_unwanted_tag_list: do not upload files/directory whose name
is present
@type this_unwanted_tag_list: list of ascii string
@param this_debug_mode: True = with debug mode, False = silent
@type this_debug_mode: boolean
@return: True : transfert done / False : error
@rtype: boolean
@note:
- It is not possible to create a new remote directory whose
name is different from the local one.
"""
# --------------------------------------
# Private: recursive_build_upload_list()
# --------------------------------------
def recursive_build_upload_list( this_src_path
, this_dst_path
, this_unwanted_tag_list
, this_recursiv_call = False
):
"""
local function for recursivity upload calls
@param this_src_path: source path name (file or directory).
@type this_src_path: ascii string
@param this_dst_path: server destination directory
@type this_dst_path: ascii string
@param this_recursiv_call: True = recursiv call, the directory
is already taken into account, False = first call
@type this_recursiv_call: boolean
@return: True : transfert done / False : error
@rtype: boolean
Note: Use binary transfert for ASCII and binary files tranfert
;-)
"""
dst_path = copy.copy(this_dst_path)
err_msg = None
# Be careful with the first call... ;-)
if os.path.isdir(this_src_path) is True:
if this_recursiv_call is False:
base_name = os.path.basename(this_src_path)
# Use the POSIX separator for remote files !
dst_path += os.sep+base_name
dst_path =
dst_path.replace(os.sep,_ftp_posix_separator)
# Get the directory list
src_dir_list = os.listdir(this_src_path)
else:
if os.path.isfile(this_src_path) is True:
# Only one file to upload
src_dir_list =
[os.path.basename(this_src_path),]
this_src_path = os.path.dirname(this_src_path)
else:
# Do not follow links, ignore it
return True
# Exam the result list
for src_file in src_dir_list:
keep_file = True
for unwanted_tag_idx in this_unwanted_tag_list:
if unwanted_tag_idx in src_file:
# Not wanted !
keep_file = False
break
# Next unwanted_tag_idx
continue
if keep_file is False:
# Next !
continue
full_src_file = this_src_path+os.sep+src_file
# Use the POSIX separator for remote files !
full_dest_file = dst_path+os.sep+src_file
full_dest_file =
full_dest_file.replace(os.sep,_ftp_posix_separator)
if os.path.isfile(full_src_file) is True:
# Add the file for transfert
try:
file_host_size =
os.path.getsize(full_src_file)
except gui_common.runing_errors,why:
# Should never occure
err_msg = str(why)
except:
# Idem !
err_msg =
"os.path.getsize(%s)"%(full_src_file)
else:
info = (full_src_file, full_dest_file,
file_host_size)
file_manager_id.add_waiting(
this_scenario_name = info)
if err_msg is not None:
gui_common.save_warning( this_msg =
"os.path.getsize(%s): %s"%(full_src_file,err_msg)
,
this_object_class = ftp_id
, this_file_name
= __file__
,
this_caller_name = "recursive_build_upload_list()"
)
return False
else:
if os.path.isdir(full_src_file) is True:
new_dst_path =
dst_path+_ftp_posix_separator+os.path.basename(full_src_file)
# Do a recursive call
if recursive_build_upload_list(
this_src_path = full_src_file
,
this_dst_path = new_dst_path
,
this_unwanted_tag_list = this_unwanted_tag_list
,
this_recursiv_call = True
) is
False:
return False
# Next src file
continue
return True
#
# Main body of ftp_upload()
#
if isinstance(this_dst_path,(str,unicode)) is False:
subject = "%s Dest path is not an ASCII string '%s'"%(
gui_common.ftp_upload_marker
,
str(this_dst_path)
)
current_thread_id = threading.currentThread()
content =
"%s\n%s"%(str(current_thread_id),gui_common.get_thread_stack())
gui_common.send_mail( this_subject = subject
, this_content = content
, this_receiver_email_addr =
gui_common.administrator_email
)
return False
if isinstance(this_src_path,(str,unicode)) is False:
subject = "%s Src path is not an ASCII string '%s'"%(
gui_common.ftp_upload_marker
,
str(this_src_path)
)
current_thread_id = threading.currentThread()
content =
"%s\n%s"%(str(current_thread_id),gui_common.get_thread_stack())
gui_common.send_mail( this_subject = subject
, this_content = content
, this_receiver_email_addr =
gui_common.administrator_email
)
return False
if this_debug_mode is True:
gui_common.save_trace( this_msg = "ftp_upload(%s) from %s to
%s, login: %s, pass:
%s"%(this_address,this_src_path,this_dst_path,this_login,this_password)
, this_object_class = None
, this_file_name = __file__
, this_caller_name = "ftp_upload()"
)
# FTP or SFTP ?
sftp_in_use = gui_common.get_sftp_usage( this_ip_address = this_address)
# Be carreful when transmiting one file
if os.path.isfile(this_src_path) is True:
# Extract remote parameters (UNIX path format...)
path_list =
this_dst_path.replace(os.sep,_ftp_posix_separator).split(_ftp_posix_separator)
user_remote_dir = _ftp_posix_separator.join(path_list[:-1])
# One file , one upload needed
max_expected_connections_number = 1
else:
# The dst path is a POSIX directory
user_remote_dir =
this_dst_path.replace(os.sep,_ftp_posix_separator)
if this_max_connection is None:
max_expected_connections_number =
_ftp_get_max_connection( this_address = this_address)
else:
max_expected_connections_number = this_max_connection
# Open FTP connections
ftp_cnx_info_list = _ftp_open_many_connection( this_address =
this_address
, this_login = this_login
, this_password =
this_password
, this_timeout =
this_timeout
, this_upload = True
, this_user_remote_dir =
this_dst_path
, this_max_connection =
max_expected_connections_number
, this_debug_mode =
this_debug_mode
, this_nickname =
this_nickname
, this_check_size =
this_check_size
, this_use_sftp =
sftp_in_use
)
if ftp_cnx_info_list == []:
# No open connection on this server
return False
# Use the first connection to obtain file names. It will be closed by
the slave thread.
(file_manager_id, ftp_id, _ftp_thread_id) = ftp_cnx_info_list[0]
err_msg = None
# Go on
src_directory_with_wildcard = '*' in this_src_path
if this_dst_path is None:
# Use the default one
this_dst_path = _ftp_thread_id.user_remote_dir
# This_dst_path may be as well as a file or a directory name.
if src_directory_with_wildcard is True:
# A wildcard has been given, Take the remote path as is !
# Multiple targets
target_list = glob.glob(this_src_path)
else:
# No Wildcard: only one target
target_list = [this_src_path,]
# Avoid to have // directory inside recursive_build_upload_list()
function
if user_remote_dir == _ftp_posix_separator:
user_remote_dir = ""
# Check if the remote base directory exists or not
if src_directory_with_wildcard is True:
src_initial_path = this_src_path.split("*")[0]
else:
src_initial_path = this_src_path
# Run my hen !
for target_file in target_list:
if recursive_build_upload_list( this_src_path = target_file
, this_dst_path = user_remote_dir
, this_unwanted_tag_list =
this_unwanted_tag_list
) is not True:
_ftp_close_connection_list( this_list =
ftp_cnx_info_list)
del ftp_cnx_info_list
gui_common.garbage_collector_start()
return False
# Next target_file
continue
# Start the file transferts now !
for ftp_cnx_info in ftp_cnx_info_list:
(file_manager_idx, ftp_idx, ftp_thread_idx) = ftp_cnx_info
ftp_thread_idx.start()
# Next ftp_cnx_info
continue
# Wait end of transfert
for ftp_cnx_info in ftp_cnx_info_list:
(file_manager_idx, ftp_idx, ftp_thread_idx) = ftp_cnx_info
ftp_thread_idx.join()
# Next ftp_cnx_info
continue
# Check if the transfert has been done correctly
remaining_file_list = file_manager_id.get_running_list(this_name_only =
True)
result = (remaining_file_list == [])
if result is False:
# Some files have not been transfered
subject = "%s Some files have not been uploaded to %s"%(
gui_common.ftp_upload_marker
,
this_address
)
#
file_list = ["Files not uploaded:\n",]
for file_to_transfert_info in remaining_file_list:
try:
(full_src_file, full_dest_file, file_size) =
file_to_transfert_info
except:
dummy = "<<<Format ?>>>
"+str(file_to_transfert_info)
else:
dummy = "full_src_file: %s\nfull_dest_file:
%s\nfile_size: %d\n"%file_to_transfert_info
file_list.append(dummy)
continue
gui_common.send_mail( this_subject = subject
, this_content =
gui_common.char_line_feed.join(file_list)
, this_receiver_email_addr =
gui_common.administrator_email
)
# Done
_ftp_close_connection_list( this_list = ftp_cnx_info_list)
del ftp_cnx_info_list
gui_common.garbage_collector_start()
return result
# --------------------------------
# Function : ftp_parallel_upload()
# --------------------------------
def ftp_parallel_upload( this_address_list = []
, this_login = "dummy_login"
, this_password = "dummy_par_up_password"
, this_timeout = gui_common.ftp_timeout_s
, this_src_path = None
, this_dst_path = None
, this_cleanup_flag = False
, this_debug_mode = False
):
"""
Upload FTP simultaneously on miscellaneous remote servers.
@param this_address_list: list of remote address
@type this_address_list: list
@param this_login: remote user login
@type this_login: ascii
@param this_password: remote user password
@type this_password: ascii
@param this_timeout: timeout in seconds for blocking operations like
the connection attempt
@type this_timeout: real
@param this_src_path: local source directory
@type this_src_path: ascii string
@param this_dst_path: server destination directory
@type this_dst_path: ascii string
@param this_cleanup_flag: force to clean up the destination directory
(True) or not (False)
@type this_cleanup_flag: boolean
@param this_debug_mode: current debug mode
@type this_debug_mode: boolean
@return: updated server list without any error
@rtype: list
"""
if isinstance(this_dst_path,(str,unicode)) is False:
subject = "%s Dest path is not an ASCII string '%s'"%(
gui_common.ftp_upload_marker
,
str(this_dst_path)
)
current_thread_id = threading.currentThread()
content =
"%s\n%s"%(str(current_thread_id),gui_common.get_thread_stack())
gui_common.send_mail( this_subject = subject
, this_content = content
, this_receiver_email_addr =
gui_common.administrator_email
)
return False
if isinstance(this_src_path,(str,unicode)) is False:
subject = "%s Src path is not an ASCII string '%s'"%(
gui_common.ftp_upload_marker
,
str(this_src_path)
)
current_thread_id = threading.currentThread()
content =
"%s\n%s"%(str(current_thread_id),gui_common.get_thread_stack())
gui_common.send_mail( this_subject = subject
, this_content = content
, this_receiver_email_addr =
gui_common.administrator_email
)
return False
updated_server_list = []
son_id_list = []
ip_address = gui_common.get_ip_address()
for host_address in this_address_list:
if host_address != ip_address:
# Update this server
son_name = "FTP_Upload_"+host_address
son_id = _ftp_thread( this_address = host_address
, this_login = this_login
, this_password = this_password
, this_timeout = this_timeout
, this_src_path = this_src_path
, this_dst_path = this_dst_path
, this_direction= _upload_cmd
, this_cleanup_flag =
this_cleanup_flag
, this_debug_mode = this_debug_mode
)
son_id_list.append(son_id)
son_id.setName(son_name)
son_id.start()
# Next host_address
continue
# Wait the end of my sons
for son_id in son_id_list:
son_id.join()
if son_id.result is True:
updated_server_list.append(son_id.address)
# Next son_id
continue
# Done
del son_id_list
gui_common.garbage_collector_start()
return updated_server_list
# ----------------------------------
# Function : ftp_parallel_download()
# ----------------------------------
def ftp_parallel_download( this_address_list = []
, this_login = "dummy_login"
, this_password = "dummy_par_down_password"
, this_timeout = gui_common.ftp_timeout_s
, this_src_path = None
, this_dst_path = None
, this_debug_mode = False
):
"""
Download FTP simultaneously from miscellaneous remote servers.
@param this_address_list: list of remote address
@type this_address_list: list
@param this_login: remote user login
@type this_login: ascii
@param this_password: remote user password
@type this_password: ascii
@param this_timeout: timeout in seconds for blocking operations like
the connection attempt
@type this_timeout: real
@param this_src_path: local source directory
@type this_src_path: ascii string
@param this_dst_path: server destination directory
@type this_dst_path: ascii string
@param this_debug_mode: current debug mode
@type this_debug_mode: boolean
@return: downloaded server list without any error
@rtype: list
"""
downloaded_server_list,son_id_list = [],[]
ip_address = gui_common.get_ip_address()
for host_address in this_address_list:
if host_address != ip_address:
# Download from this server
son_name = "FTP_Download_"+host_address
son_id = _ftp_thread( this_address = host_address
, this_login = this_login
, this_password = this_password
, this_timeout = this_timeout
, this_src_path = this_src_path
, this_dst_path = this_dst_path
, this_direction= _download_cmd
, this_cleanup_flag = False
, this_debug_mode = this_debug_mode
)
son_id_list.append(son_id)
son_id.setName(son_name)
son_id.start()
# Next host_address
continue
# Wait the end of my sons
for son_id in son_id_list:
son_id.join()
if son_id.result is True:
downloaded_server_list.append(son_id.address)
# Next son_id
continue
del son_id_list
gui_common.garbage_collector_start()
return downloaded_server_list
# ----------------------------------------------------
# If the module is run alone (i.e. for debug) : Main()
# ----------------------------------------------------
if __name__ == '__main__':
#
# Unitary Test
#
# --------------------------------
# SFTP upload error ON server side
# previous upload Ok
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> TYPE I
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 200
Type set to I
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> PASV
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 227
Entering Passive Mode (135,120,162,6,213,254)
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> STOR
Introduction_PUCch_F2_Model.tex
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 150
Opening data channel for file upload to server of
"/Introduction_PUCch_F2_Model.tex"
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> SSL
connection for data connection established
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 226
Successfully transferred "/Introduction_PUCch_F2_Model.tex"
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> SIZE
Introduction_PUCch_F2_Model.tex
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 213
3155
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> TYPE I
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 200
Type set to I
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> PASV
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 227
Entering Passive Mode (135,120,162,6,236,58)
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> STOR
Introduction_PUCch_F1_Model.tex
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 150
Opening data channel for file upload to server of
"/Introduction_PUCch_F1_Model.tex"
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> SSL
connection for data connection established
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 226
Successfully transferred "/Introduction_PUCch_F1_Model.tex"
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> SIZE
Introduction_PUCch_F1_Model.tex
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 213
3148
# until here, it's ok ! Same player shoots again...
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> TYPE I
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 200
Type set to I
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> PASV
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 227
Entering Passive Mode (135,120,162,6,200,16)
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> STOR
Handle.exe
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 150
Opening data channel for file upload to server of "/Handle.exe"
# the problem is comming...
# (000336)30/09/2015 08:56:59 - nemo_pyc (135.238.179.15)> 550
can't access file. <<<<<< Instead of: "SSL connection for data connection
established"
# (000336)30/09/2015 08:57:29 - nemo_pyc (135.238.179.15)> TYPE I
# (000336)30/09/2015 08:57:29 - nemo_pyc (135.238.179.15)> 200
Type set to I
# (000336)30/09/2015 08:57:59 - nemo_pyc (135.238.179.15)>
×3DG}
# (000336)30/09/2015 08:58:04 - nemo_pyc (135.238.179.15)>
'Y'jQÌ~.
# ¾0åà ×3DG~
# (000336)30/09/2015 08:58:04 - nemo_pyc (135.238.179.15)> 500
Syntax error, command unrecognized.
# (000336)30/09/2015 08:58:09 - nemo_pyc (135.238.179.15)>
ÍÏ!xdïæA:\ ×3DG
# (000336)30/09/2015 08:58:09 - nemo_pyc (135.238.179.15)> 500
Syntax error, command unrecognized.
# (000336)30/09/2015 08:58:09 - nemo_pyc (135.238.179.15)> ÓL.³_8;
# (000336)30/09/2015 08:58:09 - nemo_pyc (135.238.179.15)> 500
Syntax error, command unrecognized.
# (000336)30/09/2015 08:58:09 - nemo_pyc (135.238.179.15)> PzRã¿
# (000336)30/09/2015 08:58:09 - nemo_pyc (135.238.179.15)> 500
Syntax error, command unrecognized.
# (000336)30/09/2015 08:58:14 - nemo_pyc (135.238.179.15)>
×3DG
# (000336)30/09/2015 08:58:18 - nemo_pyc (135.238.179.15)>
ËÊî÷_¹S² ÃYS ×3DG
# (000336)30/09/2015 08:58:18 - nemo_pyc (135.238.179.15)> 500
Syntax error, command unrecognized.
# (000336)30/09/2015 08:58:23 - nemo_pyc (135.238.179.15)> l
8]Ô©kjü¾¥ µ ×3DG
# (000336)30/09/2015 08:58:23 - nemo_pyc (135.238.179.15)> 500
Syntax error, command unrecognized.
# (000336)30/09/2015 08:58:27 - nemo_pyc (135.238.179.15)>
òÄX#¤¯·Vb]$¥» ×3DG
# (000336)30/09/2015 08:58:27 - nemo_pyc (135.238.179.15)> 500
Syntax error, command unrecognized.
# (000336)30/09/2015 08:58:32 - nemo_pyc (135.238.179.15)>
ç^bóV YáÓÍÎ"·7 ×3DG
# (000336)30/09/2015 08:58:32 - nemo_pyc (135.238.179.15)> 500
Syntax error, command unrecognized.
# (000336)30/09/2015 08:58:37 - nemo_pyc (135.238.179.15)>
nBgIÑÐ ' Im&FP ×3DG
# (000336)30/09/2015 08:58:37 - nemo_pyc (135.238.179.15)> 500
Syntax error, command unrecognized.
# (000336)30/09/2015 08:58:41 - nemo_pyc (135.238.179.15)> õO
ÉrmåºZÅQ} ×3DG
# (000336)30/09/2015 08:58:41 - nemo_pyc (135.238.179.15)> 500
Syntax error, command unrecognized.
# (000336)30/09/2015 08:58:41 - nemo_pyc (135.238.179.15)>
+õY|ÁÒÕ°AIw
# (000336)30/09/2015 08:58:41 - nemo_pyc (135.238.179.15)> 500
Syntax error, command unrecognized.
# --------------------------------
# SFTP upload OK ON Client side
# +--- 135.238.179.15 - Wed Sep 30 09:07:27 2015 - FRVILN0H300423
# | Process : <_MainProcess(MainProcess, started)>
# | Thread : <_ftp_slave_thread(FTP_135.120.162.9_Upload_1, started
3752)>
# | File : gui_ftp.py
# | Comment : <<<FTP Upload>>> C:\Nemo_Pyc\Handle.exe -->
nemo_tmp/dst_dir/Nemo_Pyc/Handle.exe
# +---
# *cmd* 'TYPE I'
# *resp* '200 Type set to I'
# *cmd* 'PASV'
# *resp* '227 Entering Passive Mode (135,120,162,9,226,12)'
# *cmd* 'STOR Handle.exe'
# *resp* '150 Opening data channel for file upload to server of
"/nemo_tmp/dst_dir/Nemo_Pyc/Handle.exe"'
# *resp* '226 Successfully transferred
"/nemo_tmp/dst_dir/Nemo_Pyc/Handle.exe"'
# *cmd* 'SIZE Handle.exe'
# *resp* '213 536256'
# ----------------------------------
def test_ftp_upload():
user_conf = gui_user.configuration()
user_conf.debug_mode = False
test_dst_dir = "Nemo_Tmp/up"
if gui_common.get_ip_address() ==
user_conf.get_selected_dispatcher():
# Test dispatcher - labo
remote_ip = "135.120.162.1"
src_dir = "C:\\Nemo_Tmp\\up"
# Send one dir
print "=== ftp_upload: to %s ==="%(remote_ip)
ftp_upload( this_address = remote_ip
, this_login = gui_common.main_login
, this_password =
gui_common.main_passwd_default
, this_timeout = gui_common.ftp_timeout_s
, this_src_path = src_dir
, this_dst_path = test_dst_dir
, this_debug_mode = user_conf.debug_mode
)
print "=== ftp_upload: one dir/* to %s ==="%(remote_ip)
dir_name = src_dir+os.sep+"*"
ftp_upload( this_address = remote_ip
, this_login = gui_common.main_login
, this_password =
gui_common.main_passwd_default
, this_timeout = gui_common.ftp_timeout_s
, this_src_path = dir_name
, this_dst_path = test_dst_dir
, this_debug_mode = user_conf.debug_mode
)
else:
# Test laptop - labo
server_list = [ "135.120.162.1"
, "135.120.162.2"
, "135.120.162.3"
, "135.120.162.4"
, "135.120.162.5"
, "135.120.162.6"
, "135.120.162.7"
, "135.120.162.8"
, "135.120.162.9"
]
remote_ip = user_conf.get_selected_dispatcher()
src_dir =
"C:\\Nemo_Tests\\BLANQUI1.3\\Nemo2\\python\\sce"
# Send in parallel
print "=== ftp_parallel_upload: %s to servers
==="%(src_dir)
ftp_parallel_upload( this_address_list = server_list
, this_login =
gui_common.main_login
, this_password =
gui_common.main_passwd_new
, this_timeout =
gui_common.ftp_timeout_s
, this_src_path = src_dir
, this_dst_path = test_dst_dir
, this_cleanup_flag = True
, this_debug_mode = False
)
# Send some directories
print "=== ftp_upload: %s/* to %s ==="%(src_dir,
remote_ip)
ftp_upload( this_address = remote_ip
, this_login = gui_common.main_login
, this_password =
gui_common.main_passwd_default
, this_timeout = gui_common.ftp_timeout_s
, this_src_path = src_dir+os.sep+"*"
, this_dst_path = test_dst_dir
, this_debug_mode = user_conf.debug_mode
)
# Send one file
file_name =
user_conf.ftp_pyc_directory+os.sep+"gui_ftp.pyc"
print "=== ftp_upload: %s to %s
==="%(file_name,remote_ip)
ftp_upload( this_address = remote_ip
, this_login = gui_common.main_login
, this_password =
gui_common.main_passwd_default
, this_timeout = gui_common.ftp_timeout_s
, this_src_path = file_name
, this_dst_path = test_dst_dir
, this_debug_mode = user_conf.debug_mode
)
# Send one dir
dir_name = user_conf.ftp_pyc_directory+os.sep+"*"
print "=== ftp_upload: one dir to %s ==="%(remote_ip)
ftp_upload( this_address = remote_ip
, this_login = gui_common.main_login
, this_password =
gui_common.main_passwd_default
, this_timeout = gui_common.ftp_timeout_s
, this_src_path = dir_name
, this_dst_path = test_dst_dir
, this_debug_mode = user_conf.debug_mode
)
return
# ----------------------------------
def test_ftp_download():
# Download test
user_conf = gui_user.configuration()
user_conf.debug_mode = False
user_remote_dir = "BLANQUI1.3"
if gui_common.get_ip_address() ==
user_conf.get_selected_dispatcher():
# Test dispatcher - labo
remote_ip = "135.120.162.1"
src_dir = "Nemo_Tmp\\up"
dest_dir = "c:\\Nemo_Tmp\\up1"
try:
os.mkdir(dest_dir)
except:
pass
print "=== ftp_download ==="
ftp_download( this_address = dest_ip_addr
, this_login = user_conf.ftp_login
, this_password = user_conf.ftp_password
, this_timeout = gui_common.ftp_timeout_s
, this_src_path = user_remote_dir
, this_dst_path = dest_dir
, this_debug_mode = user_conf.debug_mode
)
else:
# Test laptop - labo
dest_dir = "Z:"+os.sep+"ftp_down"
dest_ip_addr = user_conf.get_selected_dispatcher()
if os.path.exists(dest_dir) is True:
if gui_common.remove_directory( this_target =
dest_dir) is not True:
print "ERROR"
print "=== ftp_parallel_download ==="
server_list = [ "135.120.162.1"
, "135.120.162.2"
, "135.120.162.3"
, "135.120.162.4"
, "135.120.162.5"
, "135.120.162.6"
, "135.120.162.7"
, "135.120.162.8"
, "135.120.162.9"
]
ftp_parallel_download( this_address_list = server_list
, this_login =
gui_common.main_login
, this_password =
gui_common.main_passwd_new
, this_timeout =
gui_common.ftp_timeout_s
, this_src_path = user_remote_dir
, this_dst_path = dest_dir
, this_debug_mode = False
)
print "=== ftp_download ==="
ftp_download( this_address = dest_ip_addr
, this_login = user_conf.ftp_login
, this_password = user_conf.ftp_password
, this_timeout = gui_common.ftp_timeout_s
, this_src_path = user_remote_dir
, this_dst_path = dest_dir
, this_debug_mode = user_conf.debug_mode
)
return
# ----------------------------------
def test_system_info():
dest_dir= "D:\\dummy\\Logs"
user_conf = gui_user.configuration()
user_conf.debug_mode = False
ip_address = user_conf.get_selected_dispatcher()
ftp_cnx_info_list = _ftp_open_many_connection( this_address =
ip_address
, this_login =
gui_common.main_login
, this_password =
gui_common.main_passwd_default
, this_timeout =
10.0
, this_upload =
False
,
this_user_remote_dir = None
,
this_max_connection = 1
, this_check_size
= False
, this_use_sftp =
gui_common.get_sftp_usage( this_ip_address = ip_address)
)
(file_manager_id, ftp_id, _ftp_thread_id) = ftp_cnx_info_list[0]
# Get remote server info (Filezilla Server)
answer = _ftp_get_server_info( this_ftp_id = ftp_id)
(system_info, feature_list) = answer
print "System :" + system_info
print "Feature:"
for feature_idx in feature_list:
print " "+feature_idx
continue
# Get detailed remote directory info (Filezilla Server)
answer = _ftp_get_directory_detail( this_ftp_id = ftp_id
, this_dir = "Nemo_Tests"
)
for idx in answer:
(directory_flag, size, year, month, date, hour_minute,
file) = idx
msg = "%s %8d %d.%s.%02d.%s %s"%idx
print msg
continue
_ftp_close_connection( this_ftp_id = ftp_id)
return
# ----------------------------------
def test_sftp_download( this_address = "135.120.162.9"
, this_login = gui_common.main_login
, this_password = gui_common.main_passwd_default
, this_timeout = 6000.0 # 100 mn for debugging !
):
ftp_download( this_address = this_address
, this_login = gui_common.main_login
, this_password = gui_common.main_passwd_default
, this_timeout = this_timeout
, this_src_path = "nemo_pyc"
, this_dst_path =
gui_common.nemo_all_result_directory
, this_check_size = True
, this_max_connection = None
, this_debug_mode = False # True
)
return
# ----------------------------------
def test_sftp_upload( this_address = "135.120.162.9"
, this_login = gui_common.main_login
, this_password = gui_common.main_passwd_default
, this_timeout = 6000.0 # 100 mn for debugging !
):
src_dir =
"D:\\z_Code\\NEMO2_2015_03_24_EXTENDEDCP_UPDATES_3062\\main\\Automatic_Launcher"
src_dir = "C:\\Nemo_Pyc"
ftp_upload( this_address = this_address
, this_login = gui_common.main_login
, this_password = gui_common.main_passwd_default
, this_timeout = this_timeout
, this_src_path = src_dir
, this_dst_path = "nemo_tmp\\dst_dir"
, this_check_size = True
, this_max_connection = None
, this_debug_mode = False # True
)
return
# ----------------------------------
start_time = time.time()
test_sftp_upload()
#test_sftp_download()
#test_ftp_upload()
#test_ftp_download()
#test_system_info()
msg = "<Normal> elapsed time: %s"%(gui_common.elapsed_time_ascii(
this_start_time = start_time))
gui_common.wait_CR_pressed( this_message = msg)
#improve_software_pref()
_______________________________________________
Python-bugs-list mailing list
Unsubscribe:
https://mail.python.org/mailman/options/python-bugs-list/archive%40mail-archive.com
