On Dec 13, 2009, at 11:15 PM, David Mashburn wrote: > Thanks Robert! > > I had been hoping to upgrade this because I knew the hard-coding was > going to be problematic... Thanks for the tip-off about where to look > for the paths! > > Could you point me to the code where this is done in Sage? I'd love > to > look at it for ideas!
http://hg.sagemath.org/sage-main/file/5db805d3bdaf/sage/misc/cython.py > Thanks, > -David > > Robert Bradshaw wrote: >> On Dec 11, 2009, at 11:49 AM, David Mashburn wrote: >> >> >>> Hello Cython Developers, >>> >>> I wanted to announce "Cpyx", a module I've been working on off and >>> on since 2006 that I use to automatically compile and also inline >>> Cython code in my work (mostly because I like to do everything in >>> one step). >>> >>> This is more-or-less a prototype, but it works for me on Windows, >>> Mac, and Ubuntu, so I thought I'd share! >>> >>> I know it has similar goals to pyx_import, but I think the two are >>> quite compilementary... (and I couldn't figure out how to get numpy >>> support in pyx_import when it came out...) >>> >>> My main hope for this is that it can give people a starting point >>> for using manual compilation/distutils >>> on their system (it is very verbose by default) and that it can >>> automatically inline code with numpy support! >>> >>> If you find it useful, I think it is almost mature enough to be >>> included in cython, and if not I certainly enjoy using it! >>> >>> In any case, I'd love some feedback. >>> >> >> Thanks for posting this. This reminds me a bit of what we do with >> Cython for the notebook in sage. One comment I have is that a lot of >> paths seem to be hard coded, and may not always be accurate depending >> on how/where Python is installed or what version of the OS you have. >> There is the handy sys.prefix that you can use to determine the >> running Python's directory and include paths. >> >> - Robert >> >> >>> Thanks for all the hard work you all are doing with Greg's >>> brainchild! >>> -David Mashburn >>> # Author: David Mashburn >>> # Created July 2006 >>> # Last Modified December 11, 2009 >>> # License: ??? (Apache 2) -- whatever is easiest for cython folks... >>> >>> # This module is for the automatic compilation (and also inlining) >>> of >>> # Pyrex / Cython code... >>> # It can use distutils or manual compilation with gcc (or another >>> compiler) >>> # It can work with a single existing C source and automatically >>> compile it as well >>> # It has been tested on Windows, Mac, and Ubuntu Linux >>> >>> # That said, I make no guarantees that it will work as expected! >>> # Numpy support is automatically enabled for the non-distutils >>> version... >>> >>> # Unless the printCmds option is set to False, the script will >>> output every action taken >>> # and command run >>> >>> # My main goal for this is to aid people in learning how to compile >>> cython code >>> # on their system, and give them a starting point so they can tweak >>> what they want... >>> >>> # My other goal is to automate the Cython compile process so I can >>> do everything in >>> # one step after getting it set up :) >>> >>> # I really like the inline feature a lot for testing! >>> # And try it with PySlices, the latest incarnation of the wxPython >>> shell, PyCrust! (Shameless plug...) >>> >>> import os >>> import sys >>> import glob >>> import random >>> import numpy >>> import SetEnvironVars >>> >>> # Making this work in Vista... >>> # Download the latest mingw (5.x.x): >>> # add C:\MinGW\bin to the PATH environment variable >>> >>> # Should work with latest MingW on Windows 7... >>> >>> # Making this work on Mac... >>> # Download Xcode from the apple developer site (create a login) and >>> install it: >>> # http://connect.apple.com >>> >>> # Sample output for Cpyx on Windows: >>> # Pieces: >>> # gcc -c -IC:/Python25/include PyrexExample.c -o PyrexExample.o >>> # gcc -shared PyrexExample.o -LC:/Python25/libs -lpython25 -o >>> PyrexExample.pyd >>> # All-in-one: >>> # gcc -shared PyrexExample.c -IC:/Python25/include -LC:/Python25/ >>> libs -lpython25 -o PyrexExample.pyd >>> # All-in-one with linking dll... >>> # gcc -shared numpyTest.c -IC:/Python25/include -LC:/Python25/libs - >>> LC:/Users/mashbudn/Programming/Python/Pyx -lpython25 -lnumpyTestC -o >>> numpyTest.pyd >>> >>> myPythonDir=os.environ['MYPYTHON'] >>> myPyrexDir=os.environ['MYPYREX'] >>> globalUseCython=True >>> >>> if sys.platform=='win32': >>> pyrexcName='"' + 'C:\\Python25\\Scripts\\pyrexc.py' + '"' # Full >>> path to the Pyrex compiler script >>> cythonName='"' + 'C:\\Python25\\Scripts\\cython.py' + '"' # Full >>> path to the Cython compiler script >>> pythonName='C:\\Python25\\python.exe' # Full path to python.exe >>> sitePackages='C:\\Python25\\Lib\\site-packages' >>> pythonInclude='C:/Python25/include' >>> pythonLibs='C:/Python25/libs' >>> elif sys.platform=='darwin': >>> pyrexcName='"' + '/Library/Frameworks/Python.framework/Versions/ >>> 5.1.1/bin/pyrexc' + '"' # Full path to the Pyrex compiler script >>> cythonName='"' + '/Library/Frameworks/Python.framework/Versions/ >>> 5.1.1/bin/cython' + '"' # Full path to the Cython compiler script >>> pythonName='/Library/Frameworks/Python.framework/Versions/5.1.1/ >>> bin/python' # Full path to python.exe >>> sitePackages='/Library/Frameworks/Python.framework/Versions/5.1.1/ >>> lib/python2.5/site-packages' >>> pythonInclude='/Library/Frameworks/Python.framework/Versions/ >>> 5.1.1/include' >>> pythonLibs='/Library/Frameworks/Python.framework/Versions/5.1.1/ >>> lib/python2.5/config/' # contains libpython2.5.so >>> elif sys.platform=='linux2': >>> pyrexcName='"' + '/usr/bin/pyrexc' + '"' # Full path to the Pyrex >>> compiler script >>> cythonName='"' + '/usr/bin/cython' + '"' # Full path to the >>> Cython compiler script >>> pythonName='/usr/bin/python2.5' # Full path to python.exe >>> sitePackages='/usr/lib/python2.5/site-packages' >>> pythonInclude='/usr/include/python2.5' >>> pythonLibs='/usr/lib' # contains libpython2.5.so >>> else: >>> print 'Platform "' + sys.platform + '" not supported yet' >>> >>> # New way to find numpy's arrayobject.h to include >>> arrayobjecthPath = >>> os.path.join(numpy.get_include(),'numpy','arrayobject.h') >>> arrayObjectDir = numpy.get_include() >>> >>> def Cdll(cNameIn='',printCmds=True,gccOptions=''): >>> cwd=os.getcwd() >>> >>> (cPath,cName)=os.path.split(cNameIn) # input path and input file >>> name >>> if cPath=='': >>> cPath=myPyrexDir # directory used for all Pyrex stuff >>> dllPath=cPath >>> >>> stripName=(os.path.splitext(cName))[0] # input file name without >>> extension >>> >>> if sys.platform=='win32': dllName='"' + >>> os.path.join(dllPath,stripName+'.dll') + '"' >>> elif sys.platform=='darwin': dllName='"' + >>> os.path.join(dllPath,'lib'+stripName+'.so') + '"' >>> elif sys.platform=='linux2': dllName='"' + >>> os.path.join(dllPath,'lib'+stripName+'.so') + '"' >>> else: print 'Platform "' + sys.platform >>> + '" not supported yet' >>> >>> cName='"' + os.path.join(cPath,stripName+'.c') + '"' # redefine >>> cName >>> hName='"' + os.path.join(cPath,stripName+'.h') + '"' >>> oName='"' + os.path.join(dllPath,stripName+'.o') + '"' >>> >>> os.chdir(cPath) >>> >>> cmd=' '.join(['gcc',gccOptions,'-fPIC','-c',cName,'-o',stripName >>> +'.o']) >>> if printCmds: >>> print '\n', cmd >>> os.system(cmd) >>> >>> cmd=' '.join(['gcc','-shared','-o',dllName,oName]) >>> if printCmds: >>> print '\n', cmd >>> os.system(cmd) >>> >>> os.chdir(cwd) >>> >>> def >>> Cpyx >>> (pyxNameIn >>> = >>> 'PyrexExample >>> .pyx >>> ',useDistutils >>> =False,useCython=globalUseCython,gccOptions='',printCmds=True): >>> cwd=os.getcwd() >>> >>> (pyxPath,pyxName)=os.path.split(pyxNameIn) # input path and input >>> file name >>> >>> if pyxPath=='': >>> pydPath=mainDir=myPyrexDir # directory used for all Pyrex >>> stuff >>> else: >>> pydPath=mainDir=pyxPath >>> >>> pyxStrip=(os.path.splitext(pyxName))[0] # input file name without >>> extension >>> >>> extName='"' + pyxStrip + '"' >>> pyxName='"' + os.path.join(mainDir,pyxStrip+'.pyx') + '"' # Full >>> path to the PYX file (must be in Python/Pyx folder) redefine pyxName >>> pyx2cName='"' + os.path.join(pydPath,pyxStrip+'.c') + '"' # Full >>> path to the C file to be created >>> pydName='"' + os.path.join(pydPath,pyxStrip+'.pyd') + '"' # Full >>> path to the PYD file to be created >>> soName='"' + os.path.join(pydPath,pyxStrip+'.so') + '"' # Full >>> path to the lib*.so file to be created >>> setupName='"' + os.path.join(pydPath,'setup.py') + '"' # Full >>> path of the Setup File to be created >>> >>> # run the main pyrex command to make the C file >>> pyxCompiler = cythonName if useCython else pyrexcName >>> cmd=' '.join([pythonName,pyxCompiler,pyxName,'-o',pyx2cName]) >>> if printCmds: >>> print '\n', cmd >>> os.system(cmd) >>> >>> if useDistutils: >>> >>> #write setup.py which will make a PYD file that can be >>> imported >>> setupText="""### This file is setup.py ### >>> from distutils.core import setup >>> from distutils.extension import Extension >>> from Pyrex.Distutils import build_ext >>> >>> setup( >>> name = 'Lock module', >>> ext_modules=[ >>> Extension(""" + extName + ', [' + pyxName.replace('\\','\\\\') + >>> ']' + """), >>> ], >>> cmdclass = {'build_ext': build_ext} >>> )""" >>> >>> if printCmds: >>> print 'Write Stuff to ', setupName[1:-1] >>> fid = open(setupName[1:-1],'w') # [1:-1] removes quotes >>> fid.write(setupText) >>> fid.close() >>> >>> # run setup.py >>> >>> os.chdir(mainDir) >>> >>> if sys.platform=='win32': cmd=' >>> '.join([pythonName,setupName,'build_ext','--compiler=mingw32','-- >>> inplace']) >>> elif sys.platform=='darwin': cmd=' >>> '.join([pythonName,setupName,'build_ext','--inplace']) >>> elif sys.platform=='linux2': cmd=' >>> '.join([pythonName,setupName,'build_ext','--inplace']) >>> else: print 'Platform "' + >>> sys.platform + '" not supported yet' >>> >>> else: >>> if sys.platform=='win32': cmd=' >>> '.join(['gcc',gccOptions,'-fPIC','-shared',pyx2cName,'- >>> I'+pythonInclude,'-L'+pythonLibs,'-lpython25','-o',pydName]) >>> elif sys.platform=='darwin': cmd=' >>> '.join(['gcc',gccOptions,'-fno-strict-aliasing','-Wno-long- >>> double','- >>> no-cpp-precomp','-mno-fused-madd','-fno-common', >>> '-dynamic','- >>> DNDEBUG','-g','-O3','-bundle','-undefined dynamic_lookup','- >>> I'+pythonInclude, >>> '- >>> I'+pythonInclude+'/python2.5','-I'+arrayObjectDir,'-L'+pythonLibs,'- >>> L/usr/local/lib',pyx2cName,'-o',soName]) >>> elif sys.platform=='linux2': cmd=' >>> '.join(['gcc',gccOptions,'-fPIC','-shared',pyx2cName,'- >>> I'+pythonInclude,'-L'+pythonLibs,'-lpython2.5','-o',soName]) >>> else: print 'Platform "' + >>> sys.platform + '" not supported yet' >>> >>> if printCmds: >>> print '\n', cmd >>> os.system(cmd) >>> >>> os.chdir(cwd) >>> >>> def >>> CpyxLib >>> (pyxNameIn >>> = >>> 'PyrexExample >>> .pyx >>> ',cNameIn >>> = >>> 'CTestC >>> .c >>> ',recompile >>> = >>> True >>> ,useDistutils >>> =False,useCython=globalUseCython,gccOptions='',printCmds=True): >>> cwd=os.getcwd() >>> >>> (pyxPath,pyxName)=os.path.split(pyxNameIn) # input path and input >>> file name >>> (cPath,cName)=os.path.split(cNameIn) # input path and input file >>> name >>> >>> if cPath=='': >>> dllPath=cPath=myPyrexDir # directory used for all Pyrex stuff >>> >>> if pyxPath=='': >>> pydPath=mainDir=myPyrexDir # directory used for all Pyrex >>> stuff >>> else: >>> pydPath=mainDir=pyxPath >>> >>> pyxStrip=(os.path.splitext(pyxName))[0] # input file name without >>> extension >>> cStrip=(os.path.splitext(cName))[0] # input file name without >>> extension >>> >>> extName='"' + pyxStrip + '"' >>> pyxName='"' + os.path.join(mainDir,pyxStrip+'.pyx') + '"' # Full >>> path to the PYX file (must be in Python\\Pyrex folder) >>> >>> if sys.platform=='win32': >>> dllName='"' + os.path.join(dllPath,cStrip+'.dll') + '"' >>> libName='"' + os.path.join(dllPath,cStrip) + '"' >>> library_dirs_txt='' >>> elif sys.platform=='darwin': >>> dllName='"' + os.path.join(dllPath,'lib'+cStrip+'.so') + '"' >>> libName='"' + cStrip + '"' >>> library_dirs_txt=""" >>> library_dirs=[""" + '"' + pydPath.replace('\\','\\\\') + '"' + >>> """], >>> runtime_library_dirs=[""" + '"' + pydPath.replace('\\','\\\\') + >>> '"' + """],""" >>> elif sys.platform=='linux2': >>> dllName='"' + os.path.join(dllPath,'lib'+cStrip+'.so') + '"' >>> libName='"' + cStrip + '"' >>> library_dirs_txt=""" >>> library_dirs=[""" + '"' + pydPath.replace('\\','\\\\') + '"' + >>> """], >>> runtime_library_dirs=[""" + '"' + pydPath.replace('\\','\\\\') + >>> '"' + """],""" >>> else: >>> print 'Platform "' + sys.platform + '" not supported yet' >>> >>> cName='"' + os.path.join(cPath,cStrip+'.c') + '"' # redefine cName >>> hName='"' + os.path.join(cPath,cStrip+'.h') + '"' >>> oName='"' + os.path.join(dllPath,cStrip+'.o') + '"' >>> >>> os.chdir(cPath) >>> >>> pyx2cName='"' + os.path.join(pydPath,pyxStrip+'.c') + '"' # Full >>> path to the C file to be created >>> setupName='"' + os.path.join(pydPath,'setup.py') + '"' # Full >>> path of the Setup File to be created >>> pydName='"' + os.path.join(pydPath,pyxStrip+'.pyd') + '"' # Full >>> path to the PYD file to be created >>> soName='"' + os.path.join(pydPath,pyxStrip+'.so') + '"' # Full >>> path to the lib*.so file to be created >>> >>> # compile the DLL needed for the link to the C file >>> if recompile: >>> Cdll(cName[1:-1],printCmds=printCmds,gccOptions=gccOptions) # >>> [1:-1] to remove the quotes >>> >>> # run the main pyrex command to make the C file >>> pyxCompiler = cythonName if useCython else pyrexcName >>> cmd=' '.join([pythonName,pyxCompiler,pyxName,'-o',pyx2cName]) >>> if printCmds: >>> print '\n', cmd >>> os.system(cmd) >>> >>> if useDistutils: >>> #write setup.py which will make a PYD file that can be >>> imported >>> setupText="""### This file is setup.py ### >>> from distutils.core import setup >>> from distutils.extension import Extension >>> from Pyrex.Distutils import build_ext >>> >>> setup( >>> name = 'Lock module', >>> ext_modules=[ >>> Extension(""" + extName + ', [' + pyxName.replace('\\','\\\\') + >>> """],"""+library_dirs_txt+""" >>> libraries=[""" + libName.replace('\\','\\\\') + ']' + """), >>> ], >>> cmdclass = {'build_ext': build_ext} >>> )""" >>> if printCmds: >>> print 'Write Stuff to ', setupName[1:-1] >>> fid = open(setupName[1:-1],'w') # [1:-1] removes quotes >>> fid.write(setupText) >>> fid.close() >>> >>> # run setup.py >>> os.chdir(mainDir) >>> >>> if sys.platform=='win32': cmd=' >>> '.join([pythonName,setupName,'build_ext','--compiler=mingw32','-- >>> inplace']) >>> elif sys.platform=='darwin': cmd=' >>> '.join([pythonName,setupName,'build_ext','--inplace']) >>> elif sys.platform=='linux2': cmd=' >>> '.join([pythonName,setupName,'build_ext','--inplace']) >>> else: print 'Platform "' + >>> sys.platform + '" not supported yet' >>> else: >>> if sys.platform=='win32': cmd=' >>> '.join(['gcc',gccOptions,'-fPIC','-shared',pyx2cName,'- >>> I'+pythonInclude,'-L'+pythonLibs,'-L'+cPath,'-Wl,-R'+cPath,'- >>> lpython25','-l'+cStrip,'-o',pydName]) >>> elif sys.platform=='darwin': cmd=' >>> '.join(['gcc',gccOptions,'-fno-strict-aliasing','-Wno-long- >>> double','- >>> no-cpp-precomp','-mno-fused-madd','-fno-common', >>> '-dynamic','- >>> DNDEBUG','-g','-O3','-bundle','-undefined dynamic_lookup','- >>> I'+pythonInclude, >>> '- >>> I'+pythonInclude+'/python2.5','-I'+arrayObjectDir,'-L'+pythonLibs,'- >>> L/usr/local/lib','-L'+cPath,'-Wl,-R'+cPath, >>> '- >>> l'+cStrip,pyx2cName,'-o',soName]) >>> elif sys.platform=='linux2': cmd=' >>> '.join(['gcc',gccOptions,'-fPIC','-shared',pyx2cName,'- >>> I'+pythonInclude,'-L'+pythonLibs,'-L'+cPath,'-Wl,-R'+cPath,'- >>> lpython2.5','-l'+cStrip,'-o',soName]) >>> else: print 'Platform "' + >>> sys.platform + '" not supported yet' >>> >>> if printCmds: >>> print '\n', cmd >>> os.system(cmd) >>> >>> os.chdir(cwd) >>> >>> #Shamelessly steal the idea used by scipy.weave.inline but for >>> Pyrex/ >>> Cython instead... >>> # In order to be able to import *, have to use exec in the calling >>> module... >>> def >>> PyrexInline >>> (code >>> ,cleanUp >>> = >>> False >>> ,useDistutils=False,useCython=False,gccOptions='',printCmds=True): >>> '''PyrexInline returns a string that is an import statement to >>> the temporary cython module'''+ \ >>> '''Use this like: >>> exec(PyrexInline(r"""<somecode>""",<options>))''' >>> >>> testCode=r""" >>> cdef extern from "stdio.h": >>> ctypedef struct FILE >>> >>> FILE * stdout >>> int printf(char *format,...) >>> int fflush( FILE *stream ) >>> >>> def PyrexPrint(mystring): >>> printf(mystring) >>> fflush(stdout) >>> >>> PyrexPrint('HelloWorld!') >>> """ >>> tmpPath=os.path.expanduser('~/.Cpyx_tmp') >>> if not os.path.isdir(tmpPath): >>> os.mkdir(tmpPath) >>> if tmpPath not in sys.path: >>> sys.path.append(tmpPath) >>> if cleanUp: >>> CleanTmp() >>> >>> # Ensure you always get a new module! >>> # This means there is no reason to "reload" >>> # Also means memory gets majorly eaten up! >>> # Can't have everything! >>> moduleName='Pyrex'+str(random.randint(0,1e18)) >>> file=os.path.join(tmpPath,moduleName+'.pyx') >>> >>> fid=open(file,'w') >>> fid.write(code) >>> fid.close() >>> >>> >>> Cpyx >>> (file >>> ,useDistutils >>> = >>> useDistutils >>> ,useCython=useCython,gccOptions=gccOptions,printCmds=printCmds) >>> >>> #cmd="""import """+moduleName+""" as LoadPyrexInline""" >>> cmd="""from """+moduleName+""" import *""" >>> if printCmds: >>> print cmd >>> return cmd >>> >>> # Create a dummy function that defaults to using Cython instead for >>> clarity... >>> def >>> CythonInline >>> (code >>> ,cleanUp >>> = >>> False >>> ,useDistutils=False,useCython=True,gccOptions='',printCmds=True): >>> return >>> PyrexInline >>> (code >>> ,cleanUp >>> = >>> cleanUp >>> ,useDistutils >>> = >>> useDistutils >>> ,useCython=useCython,gccOptions=gccOptions,printCmds=printCmds) >>> >>> def CleanTmp(): >>> tmpPath=os.path.expanduser('~/.Cpyx_tmp') >>> for i in glob.glob(os.path.join(tmpPath,'*')): >>> os.remove(i) >>> _______________________________________________ >>> Cython-dev mailing list >>> [email protected] >>> http://codespeak.net/mailman/listinfo/cython-dev >>> >> >> _______________________________________________ >> Cython-dev mailing list >> [email protected] >> http://codespeak.net/mailman/listinfo/cython-dev >> > > _______________________________________________ > Cython-dev mailing list > [email protected] > http://codespeak.net/mailman/listinfo/cython-dev _______________________________________________ Cython-dev mailing list [email protected] http://codespeak.net/mailman/listinfo/cython-dev
