Hi Everyone, Attached are three scripts that I intend to use for transporting a file via IBM's WebSphere MQSeries middle-ware.
The sender script uses a regular expression to replace characters not
[a-z][A-Z][0-9] or white space with their hex value equivalents.
The purpose of this is to make the contents of a transported file
viewable inside MQSeries queues.
The attached code works just fine for transporting ASCII text files but
it bombs with the following trace when run against binary files.
Does anyone know why this might be occurring?
Traceback (most recent call last):
File "M:\MQ\MQ\Scripts\receiver.py", line 267, in ?
receiver.run()
File "M:\MQ\MQ\Scripts\receiver.py", line 110, in run
self.get()
File "M:\MQ\MQ\Scripts\receiver.py", line 139, in get
tree = ElementTree(file=buff)
File "C:\Python24\Lib\site-packages\elementtree\ElementTree.py", line
543, in __init__
self.parse(file)
File "C:\Python24\Lib\site-packages\elementtree\ElementTree.py", line
583, in parse
parser.feed(data)
File "C:\Python24\Lib\site-packages\elementtree\ElementTree.py", line
1242, in feed
self._parser.Parse(data, 0)
xml.parsers.expat.ExpatError: not well-formed (invalid token): line 3,
column 39
I think that a particular character is perhaps not being translated to
ascii as required but I don't know which one it mught be.
Part of the reason for this encoding mechanism is so that it plays nice
with existing perl code already in place.
Not my choice, but what can you do.
Any help you can provide on this would be greatly appreciated.
--
Thank you,
Andrew Robert
#!c:\python24\python # Program: sender.py # Authors: Andrew Robert and Dariusz Suchojad # # Function: To send files to an MQSeries Queue via a server connection # # # Module requirements not included with Python are ElementTree and Pymqi. # # Element tree can be found at http://effbot.org/downloads/#elementtree # # Pymqi can be found athttp://pymqi.sourceforge.net/ # # Logging with tie-ins to the NT event viewer require the Python Windows # extensions located at http://sourceforge.net/project/showfikes.php?group_id=78018 # # Conventions used in this program are that all queue managers and queues are upper case. # # # Ver Date Programmer Modification # v1.0 - 05/09/06 AAR Initial release # v1.1 - 05/10/06 AAR Change how file is opened to rb # v1.2 - 05/11/06 AAR Modified to connect via server connections only # v1.3 - 05/11/06 AAR Added requirement to specify destination file name # v1.4 - 05/11/06 AAR Change value test logic to prevent usage of # empty string values # Removed test for exception on qmgr connect # Converted certain comment lines to doc strings # v1.5 - 05/13/06 AAR Added checksum generation of transmitted file # v1.6 - 05/14/06 AAR Checksum generation enhanced to better handle # large files # v1.7 - 05/15/06 AAR Added event logging to NT event application viewer. # This leverages built in date/time stamping as well # as presenting a central repository for all events # v1.8 - 05/15/06 AAR update logging logic, removing if/elif/else conditions # Added POSIX plaform support for logging events # v1.9 - 05/16/06 AAR Add ability to take command options from input file # v1.10 - 05/19/06 AAR Append python module library path to include ts-mqadm-d share # Farm out event logging, checksum, mailer to # separate/reusable modules stored on ts-mqadm-d share # temporarily removed ability to send mail or validate addressing # v1.11 - 05/19/06 AAR Move signal handling to external module and update code # references # v1.12 - 05/21/06 AAR Change file encoding from base64 to quopri methodology. # Quopri is an encoding strategy based on RFC 1521 for mime # v1.13 - 05/22/06 AAR Revert encoding structure back to base64. quopri structure # does not encode in XML well # v1.14 - 05/22/06 AAR Determine if input file is ascii or binary. adjust encoding # on file type # v1.15 - 05/26/06 AAR Add import of custom file encoder function # remove import of base64 module # remove call to base64 encoder # modify content element creation so that it is done in # place instead of as separate steps # import os, sys, string, exceptions, signal import pymqi as mq from pymqi import CMQC from optparse import OptionParser from cStringIO import StringIO from elementtree.ElementTree import Element, ElementTree, dump sys.path.append(r'\\netapp1\ts-mqadm-d\python_common_module_library') import mqevlog, csums, sigh, file_encoder def test_options(qmanager='', queue='',file_name='',dest='' ): """ Minimal command line option testing """ if not qmanager : print "\n\nNo queue manager specified\n" mqevlog.event(sys.argv[0],"error","No queue manager specified") sys.exit(1) if not queue : print "\n\nNo queue specified\n" mqevlog.event(sys.argv[0],"error","No queue specified") sys.exit(1) if not file_name : print "\n\nNo file name specified\n" mqevlog.event(sys.argv[0],"error","No file name specified") sys.exit(1) if not dest : print "\n\nNo destination file name specified\n" mqevlog.event(sys.argv[0],"error","No destination file name specified") sys.exit(1) class Sender(object): def __init__(self,qm_name,queue_name,file_name,dest): self.qm_name = qm_name self.queue_name = queue_name self.file_name = file_name self.dest_file_name = dest # Will be set later self.qm = None self.message = None def run(self): self.connect() self.prepare_message() self.put() def connect(self): """ Connect to queue manager. """ try: self.qm = mq.QueueManager(options.qmanager.upper() ) except mq.MQMIError, e: # Log an error perhaps mqevlog.event(sys.argv[0],"error",e) sys.exit(1) def prepare_message(self): """ Make an XML message from a file """ # Let's say we're interested in size of file and its creation time stat = os.stat(self.file_name) file_size, creation_time = stat[6],stat[9] cs = csums.getsum(self.file_name) # Prepare necessary elements root = Element("file") # Populate elements elem = Element("file_size") elem.text = str(file_size) root.append(elem) elem = Element("dest") elem.text = self.dest_file_name root.append(elem) elem = Element("creation_time") elem.text = str(creation_time) root.append(elem) elem = Element("checksum") elem.text = cs root.append(elem) elem = Element("contents") elem.text = file_encoder.code(open(self.file_name,"rb").read()) root.append(elem) # Create an actual XML tree tree = ElementTree(root) # Temporary buffer buff = StringIO() buff.write('<?xml version="1.0" encoding="UTF-8"?>') tree.write(buff) # OK, create the message self.message = buff.getvalue() def put(self): """ Put a message onto queue. """ queue = mq.Queue(self.qm, self.queue_name) try: queue.put(self.message) except : msg = "Unable to put message on queue %s :: %s" % ( options.qmanager.upper(), options.queue.upper() ) mqevlog.event(sys.argv[0],"critical",msg) sys.exit(1) msg = "%s via %s::%s as %s" % (options.filename, \ options.qmanager.upper(), \ options.queue.upper(), \ options.dest) mqevlog.event(sys.argv[0],"info",msg) queue.close() if __name__ == "__main__": # # Setup signal handlers to detect if program is # aborted, terminated, or Ctrl-C'ed out of # signal.signal(signal.SIGABRT, sigh.handler) signal.signal(signal.SIGINT, sigh.handler) signal.signal(signal.SIGTERM, sigh.handler) # # Parse command line options and automatically build help/usage display # parser = OptionParser() if len(sys.argv) == 2: lines = open(sys.argv[1],"rb").readlines() for line in lines: line=line.strip() if not line: continue short, long, dest, help, default = line.split(";") help = "\t\t" + help # Prepend tabs to help message parser.add_option(short, long, dest=dest, help=help, default=default) else: parser.add_option("-m","--qmanager", dest="qmanager", help="\t\tQueue Manager to inquire against"), parser.add_option("-q","--queue", dest="queue", help="\t\tQueue the message will be sent to"), parser.add_option("-d","--dest", dest="dest", help="\t\tDestination File Name"), parser.add_option("-f", "--file", action="store", type="string", dest="filename", help="File to be transmitted", metavar="FILE") (options, args) = parser.parse_args() test_options(options.qmanager, options.queue, options.filename, options.dest) # Create receiver object with required parameters sender = Sender(options.qmanager.upper(), \ options.queue.upper(), \ options.filename , \ options.dest) sender.run()
#!/usr/bin/python
#
# Author: Andrew Robert
# program: file_encoder.py
#
# Function: Analyze a file, locate any characters in the file that are not
# a-zA-z,0-9, white space or tabs, and replace the special characters
# with their hex value
#
#
# v1 05/25/06 AAR Initial Creation
# v1.1 05/26/06 AAR Change name of called coder function
# Merge decoder file function to this file
import re,base64
# Evaluate captured character as hex
def ret_hex(value):
return '%'+base64.b16encode(value)
# Evaluate the value of whatever was matched
def enc_hex_match(match):
return ret_hex(match.group(0))
# convert the character back to ascii
def ret_ascii(value):
return base64.b16decode(value)
# Evaluate the value of whatever was matched
def enc_ascii_match(match):
arg=match.group()
#print "I got ",arg
#remove the artifically inserted % sign
arg=arg[1:]
#print "sliced it is",arg
# decode the result
return ret_ascii(arg)
def code(data):
"""
Read data block. Locate any characters that meet the regular
expression match. Pass the located characters to a function
and get back the hex-value equivalent of the original character
"""
converted=''
new=''
for line in data:
new= (re.sub('[^\w\s]',enc_hex_match, line))
converted=converted+new
return converted
def decode(data):
"""
Read data block. Locate any characters converted to hex and
translate them back to their original ascii values
"""
converted = ''
new = ''
for line in data:
new=re.sub('%[0-9A-F][0-9A-F]',enc_ascii_match, line)
converted=converted+new
return converted
#!/usr/bin/python # Program: receiver.py # Authors: Andrew Robert and Dariusz Suchojad # # # Function: To extract files from an MQSeries Queue # This program is intened to be run as a trigger process on # a queue manager # # Module requirements not included with Python are ElementTree and Pymqi. # # Element tree can be found at http://effbot.org/downloads/#elementtree # Pymqi can be found athttp://pymqi.sourceforge.net/ # # # Conventions used in this program are that all queue managers and queues # are upper case. # # # Ver Date Programmer Modification # v1.0 - 05/09/06 AAR Initial release # v1.1 - 05/10/06 AAR changed write file statement to use # wb option # v1.2 - 05/10/06 AAR Add code to verify that file can be written # v1.3 - 05/11/06 AAR Convert to use server connection only # Recode so that dest file name is used # v1.4 - 05/11/06 AAR Change value test logic to prevent usage of # empty string values # Removed test for exception on qmgr connect # Added test to check if no messages on queue. # Converted certain comment lines to doc strings # v1.5 - 05/13/06 AAR Added md5 checksum check against received file # v1.6 - 05/14/06 AAR Checksum generation enhanced to better handle # large files # v1.7 - 05/15/06 AAR Added logging to NT application event viewer # v1.8 - 05/15/06 AAR update logging logic, removing if/elif/else conditions # Added POSIX plaform support for logging events # v1.9 - 05/16/06 AAR Added code to prevent output file overwrites if output file # already exists # v1.10 - 05/16/06 AAR Add ability to get command options from configuration file # v1.11 - 05/17/06 AAR Download all files from specified queue until none exist # v1.12 - 05/19/06 AAR Modify so input can accept binary data from trigger monitor process # Move command-line/input file/trigger data handling from main to functions # Append python module library path to include ts-mqadm-d share # Farm out event logging, checksum, mailer and trig data unpacking to # separate/reusable modules stored on ts-mqadm-d share # temporarily removed ability to send mail or validate addressing # v1.13 - 05/19/06 AAR Move signal handling to external module and update code references # Removed some debugging print statements # v1.14 - 05/22/06 AAR Fix bug in input file read block # v1.15 - 05/22/06 AAR Remove some debugging print statements # v1.16 - 05/22/06 AAR reads file type xml element and decodes from base64 if binary # v1.17 - 05/22/06 AAR trigger monitor struct unpack now being done from external file # as a class object # # # Format for input file if used # #-m;--qmanager;qmanager; Queue Manager to inquire against;ATEST #-q;--queue;queue;Queue the message will be sent to;RECEIVER.Q #-t;--to;mto;Address any mail messages will be sent to;[EMAIL PROTECTED] import os, sys, string, exceptions, signal import pymqi as mq from pymqi import CMQC from cStringIO import StringIO from elementtree.ElementTree import Element, ElementTree, dump from optparse import OptionParser sys.path.append(r'\\netapp1\ts-mqadm-d\python_common_module_library') import csums, mqevlog, mailer, sigh, file_encoder from mqtrig_class import TriggerMessage def commandline(): parser.add_option("-m","--qmanager", dest="qmanager", help="\t\tQueue Manager to inquire against"), parser.add_option("-q","--queue", dest="queue", help="\t\tQueue the message will be sent to"), parser.add_option("-t","--to", dest="mto", help="\t\taddress any mail messages will be sent to") def input_file(lines): for line in lines: line=line.strip() if not line: continue short, long, dest, help, default = line.split(";") help = "\t\t" + help # Prepend tabs to help message parser.add_option(short, long, dest=dest, help=help, default=default) class Receiver(object): def __init__(self,qm_name,queue_name): self.qm_name = qm_name self.queue_name = queue_name # Will be set later self.qm = None self.message = None def run(self): self.connect() self.get() def connect(self): """ Connect to queue manager """ try: self.qm = mq.QueueManager(options.qmanager.upper() ) except mq.PYIFError, err: mqevlog.event("error",err) sys.exit(1) def get(self): """ Get a message from queue. """ queue = mq.Queue(self.qm, self.queue_name) while True: try: var = queue.get(self.message) except mq.MQMIError,e: if e.reason != CMQC.MQRC_NO_MSG_AVAILABLE: mqevlog.event("error",e) sys.exit(1) break else: buff = StringIO(var) tree = ElementTree(file=buff) # Extract required elements and assign to local variables key = "this should be a well-kept secret" file_name = tree.find("dest").text creation_time = tree.find("creation_time").text contents = tree.find("contents").text check = tree.find("checksum").text #Dump to temporary file open(r'temp.file',"wb").write(contents) #Open temp file get_temp= open(r'temp.file','r').readlines() #Decode temp file original = file_encoder.decode(get_temp) os.unlink(r'temp.file') # Drop file to disk if os.path.exists(file_name) is False: open(file_name,"wb").write(original) else: mqevlog.event(sys.argv[0],"error","Output file path/name already exists") sys.exit(1) # Get checksum of newly created file sum=csums.getsum(file_name) # Compare checksum of created file with value transmitted csums.checksum_compare(sys.argv[0],sum,check,file_name) def test_options(qmanager='', mto='', queue=''): """ Verify that required values are specified on commmand line and that specified port is within range """ if not qmanager : print "\n\nNo queue manager specified\n" mqevlog.event(sys.argv[0],"error","No queue manager specified") sys.exit(1) if not mto : print "\n\nNo e-mail address specified\n" mqevlog.event(sys.argv[0],"error","No e-mail address specified") sys.exit(1) if not queue : print "\n\nNo queue specified\n" mqevlog.event(sys.argv[0],"error","No queue specified") sys.exit(1) if __name__ == "__main__": # # Setup signal handlers to detect if program is # aborted, terminated, or Ctrl-C'ed out of # signal.signal(signal.SIGABRT, sigh.handler) signal.signal(signal.SIGINT, sigh.handler) signal.signal(signal.SIGTERM, sigh.handler) # Mail variables rfc822_specials = '()<>@,;:\\"[]' # # Parse command line options and automatically build help/usage display # parser = OptionParser() if len(sys.argv) == 2: data=sys.argv[1] if data[0:3] == "TMC": value=TriggerMessage(data) short='-m' long='--qmanager' help="Queue Manager to inquire against" dest='qmanager' parser.add_option(short,long,dest=dest,help=help,default=value.qmgr) short='-q' long='--queue' help='Queue the message will be pulled taken' dest='queue' parser.add_option(short,long,dest=dest,help=help,default=value.qname) short='-t' long='--to' help='address any mail messages will be sent to' dest='mto' parser.add_option(short,long,dest=dest,help=help,default=value.triggerdata) else: lines = open(sys.argv[1],"rb").readlines() input_file(lines) else: commandline() (options, args) = parser.parse_args() # Test to ensure command line options are passed test_options(options.qmanager, options.mto, options.queue) # Create receiver object with required parameters receiver = Receiver(options.qmanager.upper(), \ options.queue.upper()) # # Get the file # receiver.run()
signature.asc
Description: OpenPGP digital signature
_______________________________________________ Tutor maillist - Tutor@python.org http://mail.python.org/mailman/listinfo/tutor
