On Dec 11, 2009, at 11:49 AM, David Mashburn wrote: > Hello Cython Developers, > > I wanted to announce "Cpyx", a module I've been working on off and > on since 2006 that I use to automatically compile and also inline > Cython code in my work (mostly because I like to do everything in > one step). > > This is more-or-less a prototype, but it works for me on Windows, > Mac, and Ubuntu, so I thought I'd share! > > I know it has similar goals to pyx_import, but I think the two are > quite compilementary... (and I couldn't figure out how to get numpy > support in pyx_import when it came out...) > > My main hope for this is that it can give people a starting point > for using manual compilation/distutils > on their system (it is very verbose by default) and that it can > automatically inline code with numpy support! > > If you find it useful, I think it is almost mature enough to be > included in cython, and if not I certainly enjoy using it! > > In any case, I'd love some feedback.
Thanks for posting this. This reminds me a bit of what we do with Cython for the notebook in sage. One comment I have is that a lot of paths seem to be hard coded, and may not always be accurate depending on how/where Python is installed or what version of the OS you have. There is the handy sys.prefix that you can use to determine the running Python's directory and include paths. - Robert > Thanks for all the hard work you all are doing with Greg's brainchild! > -David Mashburn > # Author: David Mashburn > # Created July 2006 > # Last Modified December 11, 2009 > # License: ??? (Apache 2) -- whatever is easiest for cython folks... > > # This module is for the automatic compilation (and also inlining) of > # Pyrex / Cython code... > # It can use distutils or manual compilation with gcc (or another > compiler) > # It can work with a single existing C source and automatically > compile it as well > # It has been tested on Windows, Mac, and Ubuntu Linux > > # That said, I make no guarantees that it will work as expected! > # Numpy support is automatically enabled for the non-distutils > version... > > # Unless the printCmds option is set to False, the script will > output every action taken > # and command run > > # My main goal for this is to aid people in learning how to compile > cython code > # on their system, and give them a starting point so they can tweak > what they want... > > # My other goal is to automate the Cython compile process so I can > do everything in > # one step after getting it set up :) > > # I really like the inline feature a lot for testing! > # And try it with PySlices, the latest incarnation of the wxPython > shell, PyCrust! (Shameless plug...) > > import os > import sys > import glob > import random > import numpy > import SetEnvironVars > > # Making this work in Vista... > # Download the latest mingw (5.x.x): > # add C:\MinGW\bin to the PATH environment variable > > # Should work with latest MingW on Windows 7... > > # Making this work on Mac... > # Download Xcode from the apple developer site (create a login) and > install it: > # http://connect.apple.com > > # Sample output for Cpyx on Windows: > # Pieces: > # gcc -c -IC:/Python25/include PyrexExample.c -o PyrexExample.o > # gcc -shared PyrexExample.o -LC:/Python25/libs -lpython25 -o > PyrexExample.pyd > # All-in-one: > # gcc -shared PyrexExample.c -IC:/Python25/include -LC:/Python25/ > libs -lpython25 -o PyrexExample.pyd > # All-in-one with linking dll... > # gcc -shared numpyTest.c -IC:/Python25/include -LC:/Python25/libs - > LC:/Users/mashbudn/Programming/Python/Pyx -lpython25 -lnumpyTestC -o > numpyTest.pyd > > myPythonDir=os.environ['MYPYTHON'] > myPyrexDir=os.environ['MYPYREX'] > globalUseCython=True > > if sys.platform=='win32': > pyrexcName='"' + 'C:\\Python25\\Scripts\\pyrexc.py' + '"' # Full > path to the Pyrex compiler script > cythonName='"' + 'C:\\Python25\\Scripts\\cython.py' + '"' # Full > path to the Cython compiler script > pythonName='C:\\Python25\\python.exe' # Full path to python.exe > sitePackages='C:\\Python25\\Lib\\site-packages' > pythonInclude='C:/Python25/include' > pythonLibs='C:/Python25/libs' > elif sys.platform=='darwin': > pyrexcName='"' + '/Library/Frameworks/Python.framework/Versions/ > 5.1.1/bin/pyrexc' + '"' # Full path to the Pyrex compiler script > cythonName='"' + '/Library/Frameworks/Python.framework/Versions/ > 5.1.1/bin/cython' + '"' # Full path to the Cython compiler script > pythonName='/Library/Frameworks/Python.framework/Versions/5.1.1/ > bin/python' # Full path to python.exe > sitePackages='/Library/Frameworks/Python.framework/Versions/5.1.1/ > lib/python2.5/site-packages' > pythonInclude='/Library/Frameworks/Python.framework/Versions/ > 5.1.1/include' > pythonLibs='/Library/Frameworks/Python.framework/Versions/5.1.1/ > lib/python2.5/config/' # contains libpython2.5.so > elif sys.platform=='linux2': > pyrexcName='"' + '/usr/bin/pyrexc' + '"' # Full path to the Pyrex > compiler script > cythonName='"' + '/usr/bin/cython' + '"' # Full path to the > Cython compiler script > pythonName='/usr/bin/python2.5' # Full path to python.exe > sitePackages='/usr/lib/python2.5/site-packages' > pythonInclude='/usr/include/python2.5' > pythonLibs='/usr/lib' # contains libpython2.5.so > else: > print 'Platform "' + sys.platform + '" not supported yet' > > # New way to find numpy's arrayobject.h to include > arrayobjecthPath = > os.path.join(numpy.get_include(),'numpy','arrayobject.h') > arrayObjectDir = numpy.get_include() > > def Cdll(cNameIn='',printCmds=True,gccOptions=''): > cwd=os.getcwd() > > (cPath,cName)=os.path.split(cNameIn) # input path and input file > name > if cPath=='': > cPath=myPyrexDir # directory used for all Pyrex stuff > dllPath=cPath > > stripName=(os.path.splitext(cName))[0] # input file name without > extension > > if sys.platform=='win32': dllName='"' + > os.path.join(dllPath,stripName+'.dll') + '"' > elif sys.platform=='darwin': dllName='"' + > os.path.join(dllPath,'lib'+stripName+'.so') + '"' > elif sys.platform=='linux2': dllName='"' + > os.path.join(dllPath,'lib'+stripName+'.so') + '"' > else: print 'Platform "' + sys.platform > + '" not supported yet' > > cName='"' + os.path.join(cPath,stripName+'.c') + '"' # redefine > cName > hName='"' + os.path.join(cPath,stripName+'.h') + '"' > oName='"' + os.path.join(dllPath,stripName+'.o') + '"' > > os.chdir(cPath) > > cmd=' '.join(['gcc',gccOptions,'-fPIC','-c',cName,'-o',stripName > +'.o']) > if printCmds: > print '\n', cmd > os.system(cmd) > > cmd=' '.join(['gcc','-shared','-o',dllName,oName]) > if printCmds: > print '\n', cmd > os.system(cmd) > > os.chdir(cwd) > > def > Cpyx > (pyxNameIn > = > 'PyrexExample > .pyx > ',useDistutils > =False,useCython=globalUseCython,gccOptions='',printCmds=True): > cwd=os.getcwd() > > (pyxPath,pyxName)=os.path.split(pyxNameIn) # input path and input > file name > > if pyxPath=='': > pydPath=mainDir=myPyrexDir # directory used for all Pyrex stuff > else: > pydPath=mainDir=pyxPath > > pyxStrip=(os.path.splitext(pyxName))[0] # input file name without > extension > > extName='"' + pyxStrip + '"' > pyxName='"' + os.path.join(mainDir,pyxStrip+'.pyx') + '"' # Full > path to the PYX file (must be in Python/Pyx folder) redefine pyxName > pyx2cName='"' + os.path.join(pydPath,pyxStrip+'.c') + '"' # Full > path to the C file to be created > pydName='"' + os.path.join(pydPath,pyxStrip+'.pyd') + '"' # Full > path to the PYD file to be created > soName='"' + os.path.join(pydPath,pyxStrip+'.so') + '"' # Full > path to the lib*.so file to be created > setupName='"' + os.path.join(pydPath,'setup.py') + '"' # Full > path of the Setup File to be created > > # run the main pyrex command to make the C file > pyxCompiler = cythonName if useCython else pyrexcName > cmd=' '.join([pythonName,pyxCompiler,pyxName,'-o',pyx2cName]) > if printCmds: > print '\n', cmd > os.system(cmd) > > if useDistutils: > > #write setup.py which will make a PYD file that can be imported > setupText="""### This file is setup.py ### > from distutils.core import setup > from distutils.extension import Extension > from Pyrex.Distutils import build_ext > > setup( > name = 'Lock module', > ext_modules=[ > Extension(""" + extName + ', [' + pyxName.replace('\\','\\\\') + > ']' + """), > ], > cmdclass = {'build_ext': build_ext} > )""" > > if printCmds: > print 'Write Stuff to ', setupName[1:-1] > fid = open(setupName[1:-1],'w') # [1:-1] removes quotes > fid.write(setupText) > fid.close() > > # run setup.py > > os.chdir(mainDir) > > if sys.platform=='win32': cmd=' > '.join([pythonName,setupName,'build_ext','--compiler=mingw32','-- > inplace']) > elif sys.platform=='darwin': cmd=' > '.join([pythonName,setupName,'build_ext','--inplace']) > elif sys.platform=='linux2': cmd=' > '.join([pythonName,setupName,'build_ext','--inplace']) > else: print 'Platform "' + > sys.platform + '" not supported yet' > > else: > if sys.platform=='win32': cmd=' > '.join(['gcc',gccOptions,'-fPIC','-shared',pyx2cName,'- > I'+pythonInclude,'-L'+pythonLibs,'-lpython25','-o',pydName]) > elif sys.platform=='darwin': cmd=' > '.join(['gcc',gccOptions,'-fno-strict-aliasing','-Wno-long-double','- > no-cpp-precomp','-mno-fused-madd','-fno-common', > '-dynamic','- > DNDEBUG','-g','-O3','-bundle','-undefined dynamic_lookup','- > I'+pythonInclude, > '- > I'+pythonInclude+'/python2.5','-I'+arrayObjectDir,'-L'+pythonLibs,'- > L/usr/local/lib',pyx2cName,'-o',soName]) > elif sys.platform=='linux2': cmd=' > '.join(['gcc',gccOptions,'-fPIC','-shared',pyx2cName,'- > I'+pythonInclude,'-L'+pythonLibs,'-lpython2.5','-o',soName]) > else: print 'Platform "' + > sys.platform + '" not supported yet' > > if printCmds: > print '\n', cmd > os.system(cmd) > > os.chdir(cwd) > > def > CpyxLib > (pyxNameIn > = > 'PyrexExample > .pyx > ',cNameIn > = > 'CTestC > .c > ',recompile > = > True > ,useDistutils > =False,useCython=globalUseCython,gccOptions='',printCmds=True): > cwd=os.getcwd() > > (pyxPath,pyxName)=os.path.split(pyxNameIn) # input path and input > file name > (cPath,cName)=os.path.split(cNameIn) # input path and input file > name > > if cPath=='': > dllPath=cPath=myPyrexDir # directory used for all Pyrex stuff > > if pyxPath=='': > pydPath=mainDir=myPyrexDir # directory used for all Pyrex stuff > else: > pydPath=mainDir=pyxPath > > pyxStrip=(os.path.splitext(pyxName))[0] # input file name without > extension > cStrip=(os.path.splitext(cName))[0] # input file name without > extension > > extName='"' + pyxStrip + '"' > pyxName='"' + os.path.join(mainDir,pyxStrip+'.pyx') + '"' # Full > path to the PYX file (must be in Python\\Pyrex folder) > > if sys.platform=='win32': > dllName='"' + os.path.join(dllPath,cStrip+'.dll') + '"' > libName='"' + os.path.join(dllPath,cStrip) + '"' > library_dirs_txt='' > elif sys.platform=='darwin': > dllName='"' + os.path.join(dllPath,'lib'+cStrip+'.so') + '"' > libName='"' + cStrip + '"' > library_dirs_txt=""" > library_dirs=[""" + '"' + pydPath.replace('\\','\\\\') + '"' + > """], > runtime_library_dirs=[""" + '"' + pydPath.replace('\\','\\\\') + > '"' + """],""" > elif sys.platform=='linux2': > dllName='"' + os.path.join(dllPath,'lib'+cStrip+'.so') + '"' > libName='"' + cStrip + '"' > library_dirs_txt=""" > library_dirs=[""" + '"' + pydPath.replace('\\','\\\\') + '"' + > """], > runtime_library_dirs=[""" + '"' + pydPath.replace('\\','\\\\') + > '"' + """],""" > else: > print 'Platform "' + sys.platform + '" not supported yet' > > cName='"' + os.path.join(cPath,cStrip+'.c') + '"' # redefine cName > hName='"' + os.path.join(cPath,cStrip+'.h') + '"' > oName='"' + os.path.join(dllPath,cStrip+'.o') + '"' > > os.chdir(cPath) > > pyx2cName='"' + os.path.join(pydPath,pyxStrip+'.c') + '"' # Full > path to the C file to be created > setupName='"' + os.path.join(pydPath,'setup.py') + '"' # Full > path of the Setup File to be created > pydName='"' + os.path.join(pydPath,pyxStrip+'.pyd') + '"' # Full > path to the PYD file to be created > soName='"' + os.path.join(pydPath,pyxStrip+'.so') + '"' # Full > path to the lib*.so file to be created > > # compile the DLL needed for the link to the C file > if recompile: > Cdll(cName[1:-1],printCmds=printCmds,gccOptions=gccOptions) # > [1:-1] to remove the quotes > > # run the main pyrex command to make the C file > pyxCompiler = cythonName if useCython else pyrexcName > cmd=' '.join([pythonName,pyxCompiler,pyxName,'-o',pyx2cName]) > if printCmds: > print '\n', cmd > os.system(cmd) > > if useDistutils: > #write setup.py which will make a PYD file that can be imported > setupText="""### This file is setup.py ### > from distutils.core import setup > from distutils.extension import Extension > from Pyrex.Distutils import build_ext > > setup( > name = 'Lock module', > ext_modules=[ > Extension(""" + extName + ', [' + pyxName.replace('\\','\\\\') + > """],"""+library_dirs_txt+""" > libraries=[""" + libName.replace('\\','\\\\') + ']' + """), > ], > cmdclass = {'build_ext': build_ext} > )""" > if printCmds: > print 'Write Stuff to ', setupName[1:-1] > fid = open(setupName[1:-1],'w') # [1:-1] removes quotes > fid.write(setupText) > fid.close() > > # run setup.py > os.chdir(mainDir) > > if sys.platform=='win32': cmd=' > '.join([pythonName,setupName,'build_ext','--compiler=mingw32','-- > inplace']) > elif sys.platform=='darwin': cmd=' > '.join([pythonName,setupName,'build_ext','--inplace']) > elif sys.platform=='linux2': cmd=' > '.join([pythonName,setupName,'build_ext','--inplace']) > else: print 'Platform "' + > sys.platform + '" not supported yet' > else: > if sys.platform=='win32': cmd=' > '.join(['gcc',gccOptions,'-fPIC','-shared',pyx2cName,'- > I'+pythonInclude,'-L'+pythonLibs,'-L'+cPath,'-Wl,-R'+cPath,'- > lpython25','-l'+cStrip,'-o',pydName]) > elif sys.platform=='darwin': cmd=' > '.join(['gcc',gccOptions,'-fno-strict-aliasing','-Wno-long-double','- > no-cpp-precomp','-mno-fused-madd','-fno-common', > '-dynamic','- > DNDEBUG','-g','-O3','-bundle','-undefined dynamic_lookup','- > I'+pythonInclude, > '- > I'+pythonInclude+'/python2.5','-I'+arrayObjectDir,'-L'+pythonLibs,'- > L/usr/local/lib','-L'+cPath,'-Wl,-R'+cPath, > '- > l'+cStrip,pyx2cName,'-o',soName]) > elif sys.platform=='linux2': cmd=' > '.join(['gcc',gccOptions,'-fPIC','-shared',pyx2cName,'- > I'+pythonInclude,'-L'+pythonLibs,'-L'+cPath,'-Wl,-R'+cPath,'- > lpython2.5','-l'+cStrip,'-o',soName]) > else: print 'Platform "' + > sys.platform + '" not supported yet' > > if printCmds: > print '\n', cmd > os.system(cmd) > > os.chdir(cwd) > > #Shamelessly steal the idea used by scipy.weave.inline but for Pyrex/ > Cython instead... > # In order to be able to import *, have to use exec in the calling > module... > def > PyrexInline > (code > ,cleanUp > = > False > ,useDistutils=False,useCython=False,gccOptions='',printCmds=True): > '''PyrexInline returns a string that is an import statement to > the temporary cython module'''+ \ > '''Use this like: exec(PyrexInline(r"""<somecode>""",<options>))''' > > testCode=r""" > cdef extern from "stdio.h": > ctypedef struct FILE > > FILE * stdout > int printf(char *format,...) > int fflush( FILE *stream ) > > def PyrexPrint(mystring): > printf(mystring) > fflush(stdout) > > PyrexPrint('HelloWorld!') > """ > tmpPath=os.path.expanduser('~/.Cpyx_tmp') > if not os.path.isdir(tmpPath): > os.mkdir(tmpPath) > if tmpPath not in sys.path: > sys.path.append(tmpPath) > if cleanUp: > CleanTmp() > > # Ensure you always get a new module! > # This means there is no reason to "reload" > # Also means memory gets majorly eaten up! > # Can't have everything! > moduleName='Pyrex'+str(random.randint(0,1e18)) > file=os.path.join(tmpPath,moduleName+'.pyx') > > fid=open(file,'w') > fid.write(code) > fid.close() > > > Cpyx > (file > ,useDistutils > = > useDistutils > ,useCython=useCython,gccOptions=gccOptions,printCmds=printCmds) > > #cmd="""import """+moduleName+""" as LoadPyrexInline""" > cmd="""from """+moduleName+""" import *""" > if printCmds: > print cmd > return cmd > > # Create a dummy function that defaults to using Cython instead for > clarity... > def > CythonInline > (code > ,cleanUp > = > False,useDistutils=False,useCython=True,gccOptions='',printCmds=True): > return > PyrexInline > (code > ,cleanUp > = > cleanUp > ,useDistutils > = > useDistutils > ,useCython=useCython,gccOptions=gccOptions,printCmds=printCmds) > > def CleanTmp(): > tmpPath=os.path.expanduser('~/.Cpyx_tmp') > for i in glob.glob(os.path.join(tmpPath,'*')): > os.remove(i) > _______________________________________________ > Cython-dev mailing list > [email protected] > http://codespeak.net/mailman/listinfo/cython-dev _______________________________________________ Cython-dev mailing list [email protected] http://codespeak.net/mailman/listinfo/cython-dev
