Thanks Robert! I had been hoping to upgrade this because I knew the hard-coding was going to be problematic... Thanks for the tip-off about where to look for the paths!
Could you point me to the code where this is done in Sage? I'd love to look at it for ideas! Thanks, -David Robert Bradshaw wrote: > On Dec 11, 2009, at 11:49 AM, David Mashburn wrote: > > >> Hello Cython Developers, >> >> I wanted to announce "Cpyx", a module I've been working on off and >> on since 2006 that I use to automatically compile and also inline >> Cython code in my work (mostly because I like to do everything in >> one step). >> >> This is more-or-less a prototype, but it works for me on Windows, >> Mac, and Ubuntu, so I thought I'd share! >> >> I know it has similar goals to pyx_import, but I think the two are >> quite compilementary... (and I couldn't figure out how to get numpy >> support in pyx_import when it came out...) >> >> My main hope for this is that it can give people a starting point >> for using manual compilation/distutils >> on their system (it is very verbose by default) and that it can >> automatically inline code with numpy support! >> >> If you find it useful, I think it is almost mature enough to be >> included in cython, and if not I certainly enjoy using it! >> >> In any case, I'd love some feedback. >> > > Thanks for posting this. This reminds me a bit of what we do with > Cython for the notebook in sage. One comment I have is that a lot of > paths seem to be hard coded, and may not always be accurate depending > on how/where Python is installed or what version of the OS you have. > There is the handy sys.prefix that you can use to determine the > running Python's directory and include paths. > > - Robert > > >> Thanks for all the hard work you all are doing with Greg's brainchild! >> -David Mashburn >> # Author: David Mashburn >> # Created July 2006 >> # Last Modified December 11, 2009 >> # License: ??? (Apache 2) -- whatever is easiest for cython folks... >> >> # This module is for the automatic compilation (and also inlining) of >> # Pyrex / Cython code... >> # It can use distutils or manual compilation with gcc (or another >> compiler) >> # It can work with a single existing C source and automatically >> compile it as well >> # It has been tested on Windows, Mac, and Ubuntu Linux >> >> # That said, I make no guarantees that it will work as expected! >> # Numpy support is automatically enabled for the non-distutils >> version... >> >> # Unless the printCmds option is set to False, the script will >> output every action taken >> # and command run >> >> # My main goal for this is to aid people in learning how to compile >> cython code >> # on their system, and give them a starting point so they can tweak >> what they want... >> >> # My other goal is to automate the Cython compile process so I can >> do everything in >> # one step after getting it set up :) >> >> # I really like the inline feature a lot for testing! >> # And try it with PySlices, the latest incarnation of the wxPython >> shell, PyCrust! (Shameless plug...) >> >> import os >> import sys >> import glob >> import random >> import numpy >> import SetEnvironVars >> >> # Making this work in Vista... >> # Download the latest mingw (5.x.x): >> # add C:\MinGW\bin to the PATH environment variable >> >> # Should work with latest MingW on Windows 7... >> >> # Making this work on Mac... >> # Download Xcode from the apple developer site (create a login) and >> install it: >> # http://connect.apple.com >> >> # Sample output for Cpyx on Windows: >> # Pieces: >> # gcc -c -IC:/Python25/include PyrexExample.c -o PyrexExample.o >> # gcc -shared PyrexExample.o -LC:/Python25/libs -lpython25 -o >> PyrexExample.pyd >> # All-in-one: >> # gcc -shared PyrexExample.c -IC:/Python25/include -LC:/Python25/ >> libs -lpython25 -o PyrexExample.pyd >> # All-in-one with linking dll... >> # gcc -shared numpyTest.c -IC:/Python25/include -LC:/Python25/libs - >> LC:/Users/mashbudn/Programming/Python/Pyx -lpython25 -lnumpyTestC -o >> numpyTest.pyd >> >> myPythonDir=os.environ['MYPYTHON'] >> myPyrexDir=os.environ['MYPYREX'] >> globalUseCython=True >> >> if sys.platform=='win32': >> pyrexcName='"' + 'C:\\Python25\\Scripts\\pyrexc.py' + '"' # Full >> path to the Pyrex compiler script >> cythonName='"' + 'C:\\Python25\\Scripts\\cython.py' + '"' # Full >> path to the Cython compiler script >> pythonName='C:\\Python25\\python.exe' # Full path to python.exe >> sitePackages='C:\\Python25\\Lib\\site-packages' >> pythonInclude='C:/Python25/include' >> pythonLibs='C:/Python25/libs' >> elif sys.platform=='darwin': >> pyrexcName='"' + '/Library/Frameworks/Python.framework/Versions/ >> 5.1.1/bin/pyrexc' + '"' # Full path to the Pyrex compiler script >> cythonName='"' + '/Library/Frameworks/Python.framework/Versions/ >> 5.1.1/bin/cython' + '"' # Full path to the Cython compiler script >> pythonName='/Library/Frameworks/Python.framework/Versions/5.1.1/ >> bin/python' # Full path to python.exe >> sitePackages='/Library/Frameworks/Python.framework/Versions/5.1.1/ >> lib/python2.5/site-packages' >> pythonInclude='/Library/Frameworks/Python.framework/Versions/ >> 5.1.1/include' >> pythonLibs='/Library/Frameworks/Python.framework/Versions/5.1.1/ >> lib/python2.5/config/' # contains libpython2.5.so >> elif sys.platform=='linux2': >> pyrexcName='"' + '/usr/bin/pyrexc' + '"' # Full path to the Pyrex >> compiler script >> cythonName='"' + '/usr/bin/cython' + '"' # Full path to the >> Cython compiler script >> pythonName='/usr/bin/python2.5' # Full path to python.exe >> sitePackages='/usr/lib/python2.5/site-packages' >> pythonInclude='/usr/include/python2.5' >> pythonLibs='/usr/lib' # contains libpython2.5.so >> else: >> print 'Platform "' + sys.platform + '" not supported yet' >> >> # New way to find numpy's arrayobject.h to include >> arrayobjecthPath = >> os.path.join(numpy.get_include(),'numpy','arrayobject.h') >> arrayObjectDir = numpy.get_include() >> >> def Cdll(cNameIn='',printCmds=True,gccOptions=''): >> cwd=os.getcwd() >> >> (cPath,cName)=os.path.split(cNameIn) # input path and input file >> name >> if cPath=='': >> cPath=myPyrexDir # directory used for all Pyrex stuff >> dllPath=cPath >> >> stripName=(os.path.splitext(cName))[0] # input file name without >> extension >> >> if sys.platform=='win32': dllName='"' + >> os.path.join(dllPath,stripName+'.dll') + '"' >> elif sys.platform=='darwin': dllName='"' + >> os.path.join(dllPath,'lib'+stripName+'.so') + '"' >> elif sys.platform=='linux2': dllName='"' + >> os.path.join(dllPath,'lib'+stripName+'.so') + '"' >> else: print 'Platform "' + sys.platform >> + '" not supported yet' >> >> cName='"' + os.path.join(cPath,stripName+'.c') + '"' # redefine >> cName >> hName='"' + os.path.join(cPath,stripName+'.h') + '"' >> oName='"' + os.path.join(dllPath,stripName+'.o') + '"' >> >> os.chdir(cPath) >> >> cmd=' '.join(['gcc',gccOptions,'-fPIC','-c',cName,'-o',stripName >> +'.o']) >> if printCmds: >> print '\n', cmd >> os.system(cmd) >> >> cmd=' '.join(['gcc','-shared','-o',dllName,oName]) >> if printCmds: >> print '\n', cmd >> os.system(cmd) >> >> os.chdir(cwd) >> >> def >> Cpyx >> (pyxNameIn >> = >> 'PyrexExample >> .pyx >> ',useDistutils >> =False,useCython=globalUseCython,gccOptions='',printCmds=True): >> cwd=os.getcwd() >> >> (pyxPath,pyxName)=os.path.split(pyxNameIn) # input path and input >> file name >> >> if pyxPath=='': >> pydPath=mainDir=myPyrexDir # directory used for all Pyrex stuff >> else: >> pydPath=mainDir=pyxPath >> >> pyxStrip=(os.path.splitext(pyxName))[0] # input file name without >> extension >> >> extName='"' + pyxStrip + '"' >> pyxName='"' + os.path.join(mainDir,pyxStrip+'.pyx') + '"' # Full >> path to the PYX file (must be in Python/Pyx folder) redefine pyxName >> pyx2cName='"' + os.path.join(pydPath,pyxStrip+'.c') + '"' # Full >> path to the C file to be created >> pydName='"' + os.path.join(pydPath,pyxStrip+'.pyd') + '"' # Full >> path to the PYD file to be created >> soName='"' + os.path.join(pydPath,pyxStrip+'.so') + '"' # Full >> path to the lib*.so file to be created >> setupName='"' + os.path.join(pydPath,'setup.py') + '"' # Full >> path of the Setup File to be created >> >> # run the main pyrex command to make the C file >> pyxCompiler = cythonName if useCython else pyrexcName >> cmd=' '.join([pythonName,pyxCompiler,pyxName,'-o',pyx2cName]) >> if printCmds: >> print '\n', cmd >> os.system(cmd) >> >> if useDistutils: >> >> #write setup.py which will make a PYD file that can be imported >> setupText="""### This file is setup.py ### >> from distutils.core import setup >> from distutils.extension import Extension >> from Pyrex.Distutils import build_ext >> >> setup( >> name = 'Lock module', >> ext_modules=[ >> Extension(""" + extName + ', [' + pyxName.replace('\\','\\\\') + >> ']' + """), >> ], >> cmdclass = {'build_ext': build_ext} >> )""" >> >> if printCmds: >> print 'Write Stuff to ', setupName[1:-1] >> fid = open(setupName[1:-1],'w') # [1:-1] removes quotes >> fid.write(setupText) >> fid.close() >> >> # run setup.py >> >> os.chdir(mainDir) >> >> if sys.platform=='win32': cmd=' >> '.join([pythonName,setupName,'build_ext','--compiler=mingw32','-- >> inplace']) >> elif sys.platform=='darwin': cmd=' >> '.join([pythonName,setupName,'build_ext','--inplace']) >> elif sys.platform=='linux2': cmd=' >> '.join([pythonName,setupName,'build_ext','--inplace']) >> else: print 'Platform "' + >> sys.platform + '" not supported yet' >> >> else: >> if sys.platform=='win32': cmd=' >> '.join(['gcc',gccOptions,'-fPIC','-shared',pyx2cName,'- >> I'+pythonInclude,'-L'+pythonLibs,'-lpython25','-o',pydName]) >> elif sys.platform=='darwin': cmd=' >> '.join(['gcc',gccOptions,'-fno-strict-aliasing','-Wno-long-double','- >> no-cpp-precomp','-mno-fused-madd','-fno-common', >> '-dynamic','- >> DNDEBUG','-g','-O3','-bundle','-undefined dynamic_lookup','- >> I'+pythonInclude, >> '- >> I'+pythonInclude+'/python2.5','-I'+arrayObjectDir,'-L'+pythonLibs,'- >> L/usr/local/lib',pyx2cName,'-o',soName]) >> elif sys.platform=='linux2': cmd=' >> '.join(['gcc',gccOptions,'-fPIC','-shared',pyx2cName,'- >> I'+pythonInclude,'-L'+pythonLibs,'-lpython2.5','-o',soName]) >> else: print 'Platform "' + >> sys.platform + '" not supported yet' >> >> if printCmds: >> print '\n', cmd >> os.system(cmd) >> >> os.chdir(cwd) >> >> def >> CpyxLib >> (pyxNameIn >> = >> 'PyrexExample >> .pyx >> ',cNameIn >> = >> 'CTestC >> .c >> ',recompile >> = >> True >> ,useDistutils >> =False,useCython=globalUseCython,gccOptions='',printCmds=True): >> cwd=os.getcwd() >> >> (pyxPath,pyxName)=os.path.split(pyxNameIn) # input path and input >> file name >> (cPath,cName)=os.path.split(cNameIn) # input path and input file >> name >> >> if cPath=='': >> dllPath=cPath=myPyrexDir # directory used for all Pyrex stuff >> >> if pyxPath=='': >> pydPath=mainDir=myPyrexDir # directory used for all Pyrex stuff >> else: >> pydPath=mainDir=pyxPath >> >> pyxStrip=(os.path.splitext(pyxName))[0] # input file name without >> extension >> cStrip=(os.path.splitext(cName))[0] # input file name without >> extension >> >> extName='"' + pyxStrip + '"' >> pyxName='"' + os.path.join(mainDir,pyxStrip+'.pyx') + '"' # Full >> path to the PYX file (must be in Python\\Pyrex folder) >> >> if sys.platform=='win32': >> dllName='"' + os.path.join(dllPath,cStrip+'.dll') + '"' >> libName='"' + os.path.join(dllPath,cStrip) + '"' >> library_dirs_txt='' >> elif sys.platform=='darwin': >> dllName='"' + os.path.join(dllPath,'lib'+cStrip+'.so') + '"' >> libName='"' + cStrip + '"' >> library_dirs_txt=""" >> library_dirs=[""" + '"' + pydPath.replace('\\','\\\\') + '"' + >> """], >> runtime_library_dirs=[""" + '"' + pydPath.replace('\\','\\\\') + >> '"' + """],""" >> elif sys.platform=='linux2': >> dllName='"' + os.path.join(dllPath,'lib'+cStrip+'.so') + '"' >> libName='"' + cStrip + '"' >> library_dirs_txt=""" >> library_dirs=[""" + '"' + pydPath.replace('\\','\\\\') + '"' + >> """], >> runtime_library_dirs=[""" + '"' + pydPath.replace('\\','\\\\') + >> '"' + """],""" >> else: >> print 'Platform "' + sys.platform + '" not supported yet' >> >> cName='"' + os.path.join(cPath,cStrip+'.c') + '"' # redefine cName >> hName='"' + os.path.join(cPath,cStrip+'.h') + '"' >> oName='"' + os.path.join(dllPath,cStrip+'.o') + '"' >> >> os.chdir(cPath) >> >> pyx2cName='"' + os.path.join(pydPath,pyxStrip+'.c') + '"' # Full >> path to the C file to be created >> setupName='"' + os.path.join(pydPath,'setup.py') + '"' # Full >> path of the Setup File to be created >> pydName='"' + os.path.join(pydPath,pyxStrip+'.pyd') + '"' # Full >> path to the PYD file to be created >> soName='"' + os.path.join(pydPath,pyxStrip+'.so') + '"' # Full >> path to the lib*.so file to be created >> >> # compile the DLL needed for the link to the C file >> if recompile: >> Cdll(cName[1:-1],printCmds=printCmds,gccOptions=gccOptions) # >> [1:-1] to remove the quotes >> >> # run the main pyrex command to make the C file >> pyxCompiler = cythonName if useCython else pyrexcName >> cmd=' '.join([pythonName,pyxCompiler,pyxName,'-o',pyx2cName]) >> if printCmds: >> print '\n', cmd >> os.system(cmd) >> >> if useDistutils: >> #write setup.py which will make a PYD file that can be imported >> setupText="""### This file is setup.py ### >> from distutils.core import setup >> from distutils.extension import Extension >> from Pyrex.Distutils import build_ext >> >> setup( >> name = 'Lock module', >> ext_modules=[ >> Extension(""" + extName + ', [' + pyxName.replace('\\','\\\\') + >> """],"""+library_dirs_txt+""" >> libraries=[""" + libName.replace('\\','\\\\') + ']' + """), >> ], >> cmdclass = {'build_ext': build_ext} >> )""" >> if printCmds: >> print 'Write Stuff to ', setupName[1:-1] >> fid = open(setupName[1:-1],'w') # [1:-1] removes quotes >> fid.write(setupText) >> fid.close() >> >> # run setup.py >> os.chdir(mainDir) >> >> if sys.platform=='win32': cmd=' >> '.join([pythonName,setupName,'build_ext','--compiler=mingw32','-- >> inplace']) >> elif sys.platform=='darwin': cmd=' >> '.join([pythonName,setupName,'build_ext','--inplace']) >> elif sys.platform=='linux2': cmd=' >> '.join([pythonName,setupName,'build_ext','--inplace']) >> else: print 'Platform "' + >> sys.platform + '" not supported yet' >> else: >> if sys.platform=='win32': cmd=' >> '.join(['gcc',gccOptions,'-fPIC','-shared',pyx2cName,'- >> I'+pythonInclude,'-L'+pythonLibs,'-L'+cPath,'-Wl,-R'+cPath,'- >> lpython25','-l'+cStrip,'-o',pydName]) >> elif sys.platform=='darwin': cmd=' >> '.join(['gcc',gccOptions,'-fno-strict-aliasing','-Wno-long-double','- >> no-cpp-precomp','-mno-fused-madd','-fno-common', >> '-dynamic','- >> DNDEBUG','-g','-O3','-bundle','-undefined dynamic_lookup','- >> I'+pythonInclude, >> '- >> I'+pythonInclude+'/python2.5','-I'+arrayObjectDir,'-L'+pythonLibs,'- >> L/usr/local/lib','-L'+cPath,'-Wl,-R'+cPath, >> '- >> l'+cStrip,pyx2cName,'-o',soName]) >> elif sys.platform=='linux2': cmd=' >> '.join(['gcc',gccOptions,'-fPIC','-shared',pyx2cName,'- >> I'+pythonInclude,'-L'+pythonLibs,'-L'+cPath,'-Wl,-R'+cPath,'- >> lpython2.5','-l'+cStrip,'-o',soName]) >> else: print 'Platform "' + >> sys.platform + '" not supported yet' >> >> if printCmds: >> print '\n', cmd >> os.system(cmd) >> >> os.chdir(cwd) >> >> #Shamelessly steal the idea used by scipy.weave.inline but for Pyrex/ >> Cython instead... >> # In order to be able to import *, have to use exec in the calling >> module... >> def >> PyrexInline >> (code >> ,cleanUp >> = >> False >> ,useDistutils=False,useCython=False,gccOptions='',printCmds=True): >> '''PyrexInline returns a string that is an import statement to >> the temporary cython module'''+ \ >> '''Use this like: exec(PyrexInline(r"""<somecode>""",<options>))''' >> >> testCode=r""" >> cdef extern from "stdio.h": >> ctypedef struct FILE >> >> FILE * stdout >> int printf(char *format,...) >> int fflush( FILE *stream ) >> >> def PyrexPrint(mystring): >> printf(mystring) >> fflush(stdout) >> >> PyrexPrint('HelloWorld!') >> """ >> tmpPath=os.path.expanduser('~/.Cpyx_tmp') >> if not os.path.isdir(tmpPath): >> os.mkdir(tmpPath) >> if tmpPath not in sys.path: >> sys.path.append(tmpPath) >> if cleanUp: >> CleanTmp() >> >> # Ensure you always get a new module! >> # This means there is no reason to "reload" >> # Also means memory gets majorly eaten up! >> # Can't have everything! >> moduleName='Pyrex'+str(random.randint(0,1e18)) >> file=os.path.join(tmpPath,moduleName+'.pyx') >> >> fid=open(file,'w') >> fid.write(code) >> fid.close() >> >> >> Cpyx >> (file >> ,useDistutils >> = >> useDistutils >> ,useCython=useCython,gccOptions=gccOptions,printCmds=printCmds) >> >> #cmd="""import """+moduleName+""" as LoadPyrexInline""" >> cmd="""from """+moduleName+""" import *""" >> if printCmds: >> print cmd >> return cmd >> >> # Create a dummy function that defaults to using Cython instead for >> clarity... >> def >> CythonInline >> (code >> ,cleanUp >> = >> False,useDistutils=False,useCython=True,gccOptions='',printCmds=True): >> return >> PyrexInline >> (code >> ,cleanUp >> = >> cleanUp >> ,useDistutils >> = >> useDistutils >> ,useCython=useCython,gccOptions=gccOptions,printCmds=printCmds) >> >> def CleanTmp(): >> tmpPath=os.path.expanduser('~/.Cpyx_tmp') >> for i in glob.glob(os.path.join(tmpPath,'*')): >> os.remove(i) >> _______________________________________________ >> Cython-dev mailing list >> [email protected] >> http://codespeak.net/mailman/listinfo/cython-dev >> > > _______________________________________________ > Cython-dev mailing list > [email protected] > http://codespeak.net/mailman/listinfo/cython-dev > _______________________________________________ Cython-dev mailing list [email protected] http://codespeak.net/mailman/listinfo/cython-dev
