Author: Matti Picus <[email protected]>
Branch:
Changeset: r69094:506a567ec761
Date: 2014-02-07 09:14 +0200
http://bitbucket.org/pypy/pypy/changeset/506a567ec761/
Log: merge precompiled-headers which uses Makefile with precompiled
headers on MSVC
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -910,6 +910,8 @@
# implement function callbacks and generate function decls
functions = []
pypy_decls = []
+ pypy_decls.append("#ifndef _PYPY_PYPY_DECL_H\n")
+ pypy_decls.append("#define _PYPY_PYPY_DECL_H\n")
pypy_decls.append("#ifndef PYPY_STANDALONE\n")
pypy_decls.append("#ifdef __cplusplus")
pypy_decls.append("extern \"C\" {")
@@ -953,6 +955,7 @@
pypy_decls.append("}")
pypy_decls.append("#endif")
pypy_decls.append("#endif /*PYPY_STANDALONE*/\n")
+ pypy_decls.append("#endif /*_PYPY_PYPY_DECL_H*/\n")
pypy_decl_h = udir.join('pypy_decl.h')
pypy_decl_h.write('\n'.join(pypy_decls))
diff --git a/rpython/translator/c/dlltool.py b/rpython/translator/c/dlltool.py
--- a/rpython/translator/c/dlltool.py
+++ b/rpython/translator/c/dlltool.py
@@ -21,7 +21,8 @@
entrypoints.append(getfunctionptr(graph))
return entrypoints
- def gen_makefile(self, targetdir, exe_name=None):
+ def gen_makefile(self, targetdir, exe_name=None,
+ headers_to_precompile=[]):
pass # XXX finish
def compile(self):
@@ -30,6 +31,8 @@
extsymeci = ExternalCompilationInfo(export_symbols=export_symbols)
self.eci = self.eci.merge(extsymeci)
files = [self.c_source_filename] + self.extrafiles
+ files += self.eventually_copy(self.eci.separate_module_files)
+ self.eci.separate_module_files = ()
oname = self.name
self.so_name = self.translator.platform.compile(files, self.eci,
standalone=False,
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -260,12 +260,13 @@
defines['PYPY_MAIN_FUNCTION'] = "pypy_main_startup"
self.eci = self.eci.merge(ExternalCompilationInfo(
export_symbols=["pypy_main_startup", "pypy_debug_file"]))
- self.eci, cfile, extra = gen_source(db, modulename, targetdir,
- self.eci, defines=defines,
- split=self.split)
+ self.eci, cfile, extra, headers_to_precompile = \
+ gen_source(db, modulename, targetdir,
+ self.eci, defines=defines, split=self.split)
self.c_source_filename = py.path.local(cfile)
self.extrafiles = self.eventually_copy(extra)
- self.gen_makefile(targetdir, exe_name=exe_name)
+ self.gen_makefile(targetdir, exe_name=exe_name,
+ headers_to_precompile=headers_to_precompile)
return cfile
def eventually_copy(self, cfiles):
@@ -375,18 +376,22 @@
self._compiled = True
return self.executable_name
- def gen_makefile(self, targetdir, exe_name=None):
- cfiles = [self.c_source_filename] + self.extrafiles
+ def gen_makefile(self, targetdir, exe_name=None, headers_to_precompile=[]):
+ module_files = self.eventually_copy(self.eci.separate_module_files)
+ self.eci.separate_module_files = []
+ cfiles = [self.c_source_filename] + self.extrafiles +
list(module_files)
if exe_name is not None:
exe_name = targetdir.join(exe_name)
mk = self.translator.platform.gen_makefile(
cfiles, self.eci,
path=targetdir, exe_name=exe_name,
+ headers_to_precompile=headers_to_precompile,
+ no_precompile_cfiles = module_files,
shared=self.config.translation.shared)
if self.has_profopt():
profopt = self.config.translation.profopt
- mk.definition('ABS_TARGET', '$(shell python -c "import sys,os;
print os.path.abspath(sys.argv[1])" $(TARGET))')
+ mk.definition('ABS_TARGET', str(targetdir.join('$(TARGET)')))
mk.definition('DEFAULT_TARGET', 'profopt')
mk.definition('PROFOPT', profopt)
@@ -511,6 +516,7 @@
def __init__(self, database):
self.database = database
self.extrafiles = []
+ self.headers_to_precompile = []
self.path = None
self.namespace = NameManager()
@@ -539,6 +545,8 @@
filepath = self.path.join(name)
if name.endswith('.c'):
self.extrafiles.append(filepath)
+ if name.endswith('.h'):
+ self.headers_to_precompile.append(filepath)
return filepath.open('w')
def getextrafiles(self):
@@ -686,11 +694,11 @@
print >> fc,
'/***********************************************************/'
print >> fc, '/*** Implementations
***/'
print >> fc
- print >> fc, '#define PYPY_FILE_NAME "%s"' % name
print >> fc, '#include "common_header.h"'
print >> fc, '#include "structdef.h"'
print >> fc, '#include "forwarddecl.h"'
print >> fc, '#include "preimpl.h"'
+ print >> fc, '#define PYPY_FILE_NAME "%s"' % name
print >> fc, '#include "src/g_include.h"'
print >> fc
print >> fc, MARKER
@@ -732,12 +740,14 @@
print >> f, "#endif"
def gen_preimpl(f, database):
+ f.write('#ifndef _PY_PREIMPLE_H\n#define _PY_PREIMPL_H\n')
if database.translator is None or database.translator.rtyper is None:
return
preimplementationlines = pre_include_code_lines(
database, database.translator.rtyper)
for line in preimplementationlines:
print >> f, line
+ f.write('#endif /* _PY_PREIMPL_H */\n')
def gen_startupcode(f, database):
# generate the start-up code and put it into a function
@@ -799,6 +809,7 @@
f = filename.open('w')
incfilename = targetdir.join('common_header.h')
fi = incfilename.open('w')
+ fi.write('#ifndef _PY_COMMON_HEADER_H\n#define _PY_COMMON_HEADER_H\n')
#
# Header
@@ -811,6 +822,7 @@
eci.write_c_header(fi)
print >> fi, '#include "src/g_prerequisite.h"'
+ fi.write('#endif /* _PY_COMMON_HEADER_H*/\n')
fi.close()
@@ -822,6 +834,8 @@
sg.set_strategy(targetdir, split)
database.prepare_inline_helpers()
sg.gen_readable_parts_of_source(f)
+ headers_to_precompile = sg.headers_to_precompile[:]
+ headers_to_precompile.insert(0, incfilename)
gen_startupcode(f, database)
f.close()
@@ -834,5 +848,4 @@
eci = add_extra_files(eci)
eci = eci.convert_sources_to_files()
- files, eci = eci.get_module_files()
- return eci, filename, sg.getextrafiles() + list(files)
+ return eci, filename, sg.getextrafiles(), headers_to_precompile
diff --git a/rpython/translator/platform/__init__.py
b/rpython/translator/platform/__init__.py
--- a/rpython/translator/platform/__init__.py
+++ b/rpython/translator/platform/__init__.py
@@ -100,7 +100,8 @@
return ExecutionResult(returncode, stdout, stderr)
def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
- shared=False):
+ shared=False, headers_to_precompile=[],
+ no_precompile_cfiles = []):
raise NotImplementedError("Pure abstract baseclass")
def __repr__(self):
diff --git a/rpython/translator/platform/darwin.py
b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -50,14 +50,17 @@
return ["-Wl,-exported_symbols_list,%s" % (response_file,)]
def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
- shared=False):
+ shared=False, headers_to_precompile=[],
+ no_precompile_cfiles = []):
# ensure frameworks are passed in the Makefile
fs = self._frameworks(eci.frameworks)
if len(fs) > 0:
# concat (-framework, FrameworkName) pairs
self.extra_libs += tuple(map(" ".join, zip(fs[::2], fs[1::2])))
mk = super(Darwin, self).gen_makefile(cfiles, eci, exe_name, path,
- shared)
+ shared=shared,
+ headers_to_precompile=headers_to_precompile,
+ no_precompile_cfiles = no_precompile_cfiles)
return mk
diff --git a/rpython/translator/platform/posix.py
b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -83,7 +83,8 @@
return [entry[2:] for entry in out.split()]
def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
- shared=False):
+ shared=False, headers_to_precompile=[],
+ no_precompile_cfiles = []):
cfiles = self._all_cfiles(cfiles, eci)
if path is None:
diff --git a/rpython/translator/platform/test/test_distutils.py
b/rpython/translator/platform/test/test_distutils.py
--- a/rpython/translator/platform/test/test_distutils.py
+++ b/rpython/translator/platform/test/test_distutils.py
@@ -11,3 +11,7 @@
def test_900_files(self):
py.test.skip('Makefiles not suppoerted')
+
+ def test_precompiled_headers(self):
+ py.test.skip('Makefiles not suppoerted')
+
diff --git a/rpython/translator/platform/test/test_makefile.py
b/rpython/translator/platform/test/test_makefile.py
--- a/rpython/translator/platform/test/test_makefile.py
+++ b/rpython/translator/platform/test/test_makefile.py
@@ -1,7 +1,10 @@
from rpython.translator.platform.posix import GnuMakefile as Makefile
+from rpython.translator.platform import host
+from rpython.tool.udir import udir
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
from StringIO import StringIO
-import re
+import re, sys, py
def test_simple_makefile():
m = Makefile()
@@ -29,3 +32,112 @@
val = s.getvalue()
assert not re.search('CC += +xxx', val, re.M)
assert re.search('CC += +yyy', val, re.M)
+
+class TestMakefile(object):
+ platform = host
+ strict_on_stderr = True
+
+ def check_res(self, res, expected='42\n'):
+ assert res.out == expected
+ if self.strict_on_stderr:
+ assert res.err == ''
+ assert res.returncode == 0
+
+ def test_900_files(self):
+ txt = '#include <stdio.h>\n'
+ for i in range(900):
+ txt += 'int func%03d();\n' % i
+ txt += 'int main() {\n int j=0;'
+ for i in range(900):
+ txt += ' j += func%03d();\n' % i
+ txt += ' printf("%d\\n", j);\n'
+ txt += ' return 0;};\n'
+ cfile = udir.join('test_900_files.c')
+ cfile.write(txt)
+ cfiles = [cfile]
+ for i in range(900):
+ cfile2 = udir.join('implement%03d.c' %i)
+ cfile2.write('''
+ int func%03d()
+ {
+ return %d;
+ }
+ ''' % (i, i))
+ cfiles.append(cfile2)
+ mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(),
path=udir)
+ mk.write()
+ self.platform.execute_makefile(mk)
+ res = self.platform.execute(udir.join('test_900_files'))
+ self.check_res(res, '%d\n' %sum(range(900)))
+
+ def test_precompiled_headers(self):
+ if self.platform.cc != 'cl.exe':
+ py.test.skip("Only MSVC profits from precompiled headers")
+ import time
+ tmpdir = udir.join('precompiled_headers').ensure(dir=1)
+ # Create an eci that should not use precompiled headers
+ eci = ExternalCompilationInfo(include_dirs=[tmpdir])
+ main_c = tmpdir.join('main_no_pch.c')
+ eci.separate_module_files = [main_c]
+ ncfiles = 10
+ nprecompiled_headers = 20
+ txt = ''
+ for i in range(ncfiles):
+ txt += "int func%03d();\n" % i
+ txt += "\nint main(int argc, char * argv[])\n"
+ txt += "{\n int i=0;\n"
+ for i in range(ncfiles):
+ txt += " i += func%03d();\n" % i
+ txt += ' printf("%d\\n", i);\n'
+ txt += " return 0;\n};\n"
+ main_c.write(txt)
+ # Create some large headers with dummy functions to be precompiled
+ cfiles_precompiled_headers = []
+ for i in range(nprecompiled_headers):
+ pch_name =tmpdir.join('pcheader%03d.h' % i)
+ txt = '#ifndef PCHEADER%03d_H\n#define PCHEADER%03d_H\n' %(i, i)
+ for j in range(3000):
+ txt += "int pcfunc%03d_%03d();\n" %(i, j)
+ txt += '#endif'
+ pch_name.write(txt)
+ cfiles_precompiled_headers.append(pch_name)
+ # Create some cfiles with headers we want precompiled
+ cfiles = []
+ for i in range(ncfiles):
+ c_name =tmpdir.join('implement%03d.c' % i)
+ txt = ''
+ for pch_name in cfiles_precompiled_headers:
+ txt += '#include "%s"\n' % pch_name
+ txt += "int func%03d(){ return %d;};\n" % (i, i)
+ c_name.write(txt)
+ cfiles.append(c_name)
+ if sys.platform == 'win32':
+ clean = ('clean', '', 'for %f in ( $(OBJECTS) $(TARGET) ) do @if
exist %f del /f %f')
+ get_time = time.clock
+ else:
+ clean = ('clean', '', 'rm -f $(OBJECTS) $(TARGET) ')
+ get_time = time.time
+ #write a non-precompiled header makefile
+ mk = self.platform.gen_makefile(cfiles, eci, path=tmpdir)
+ mk.rule(*clean)
+ mk.write()
+ t0 = get_time()
+ self.platform.execute_makefile(mk)
+ t1 = get_time()
+ t_normal = t1 - t0
+ self.platform.execute_makefile(mk, extra_opts=['clean'])
+ # Write a super-duper makefile with precompiled headers
+ mk = self.platform.gen_makefile(cfiles, eci, path=tmpdir,
+ headers_to_precompile=cfiles_precompiled_headers,)
+ mk.rule(*clean)
+ mk.write()
+ t0 = get_time()
+ self.platform.execute_makefile(mk)
+ t1 = get_time()
+ t_precompiled = t1 - t0
+ res = self.platform.execute(mk.exe_name)
+ self.check_res(res, '%d\n' %sum(range(ncfiles)))
+ print "precompiled haeder 'make' time %.2f, non-precompiled header
time %.2f" %(t_precompiled, t_normal)
+ assert t_precompiled < t_normal * 0.5
+
+
diff --git a/rpython/translator/platform/test/test_platform.py
b/rpython/translator/platform/test/test_platform.py
--- a/rpython/translator/platform/test/test_platform.py
+++ b/rpython/translator/platform/test/test_platform.py
@@ -59,34 +59,6 @@
res = self.platform.execute(executable)
self.check_res(res)
- def test_900_files(self):
- txt = '#include <stdio.h>\n'
- for i in range(900):
- txt += 'int func%03d();\n' % i
- txt += 'int main() {\n int j=0;'
- for i in range(900):
- txt += ' j += func%03d();\n' % i
- txt += ' printf("%d\\n", j);\n'
- txt += ' return 0;};\n'
- cfile = udir.join('test_900_files.c')
- cfile.write(txt)
- cfiles = [cfile]
- for i in range(900):
- cfile2 = udir.join('implement%03d.c' %i)
- cfile2.write('''
- int func%03d()
- {
- return %d;
- }
- ''' % (i, i))
- cfiles.append(cfile2)
- mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(),
path=udir)
- mk.write()
- self.platform.execute_makefile(mk)
- res = self.platform.execute(udir.join('test_900_files'))
- self.check_res(res, '%d\n' %sum(range(900)))
-
-
def test_nice_errors(self):
cfile = udir.join('test_nice_errors.c')
cfile.write('')
diff --git a/rpython/translator/platform/windows.py
b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -249,7 +249,8 @@
def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
- shared=False):
+ shared=False, headers_to_precompile=[],
+ no_precompile_cfiles = []):
cfiles = self._all_cfiles(cfiles, eci)
if path is None:
@@ -318,15 +319,54 @@
if self.x64:
definitions.append(('_WIN64', '1'))
+ rules = [
+ ('all', '$(DEFAULT_TARGET)', []),
+ ('.asm.obj', '', '$(MASM) /nologo /Fo$@ /c $< $(INCLUDEDIRS)'),
+ ]
+
+ if len(headers_to_precompile)>0:
+ stdafx_h = path.join('stdafx.h')
+ txt = '#ifndef PYPY_STDAFX_H\n'
+ txt += '#define PYPY_STDAFX_H\n'
+ txt += '\n'.join(['#include "' + m.pathrel(c) + '"' for c in
headers_to_precompile])
+ txt += '\n#endif\n'
+ stdafx_h.write(txt)
+ stdafx_c = path.join('stdafx.c')
+ stdafx_c.write('#include "stdafx.h"\n')
+ definitions.append(('CREATE_PCH', '/Ycstdafx.h /Fpstdafx.pch
/FIstdafx.h'))
+ definitions.append(('USE_PCH', '/Yustdafx.h /Fpstdafx.pch
/FIstdafx.h'))
+ rules.append(('$(OBJECTS)', 'stdafx.pch', []))
+ rules.append(('stdafx.pch', 'stdafx.h',
+ '$(CC) stdafx.c /c /nologo $(CFLAGS) $(CFLAGSEXTRA) '
+ '$(CREATE_PCH) $(INCLUDEDIRS)'))
+ rules.append(('.c.obj', '',
+ '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) $(USE_PCH) '
+ '/Fo$@ /c $< $(INCLUDEDIRS)'))
+ #Do not use precompiled headers for some files
+ #rules.append((r'{..\module_cache}.c{..\module_cache}.obj', '',
+ # '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) /Fo$@ /c $<
$(INCLUDEDIRS)'))
+ # nmake cannot handle wildcard target specifications, so we must
+ # create a rule for compiling each file from eci since they cannot
use
+ # precompiled headers :(
+ no_precompile = []
+ for f in list(no_precompile_cfiles):
+ f = m.pathrel(py.path.local(f))
+ if f not in no_precompile and f.endswith('.c'):
+ no_precompile.append(f)
+ target = f[:-1] + 'obj'
+ rules.append((target, f,
+ '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) '
+ '/Fo%s /c %s $(INCLUDEDIRS)' %(target, f)))
+
+ else:
+ rules.append(('.c.obj', '',
+ '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) '
+ '/Fo$@ /c $< $(INCLUDEDIRS)'))
+
+
for args in definitions:
m.definition(*args)
- rules = [
- ('all', '$(DEFAULT_TARGET)', []),
- ('.c.obj', '', '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) /Fo$@ /c $<
$(INCLUDEDIRS)'),
- ('.asm.obj', '', '$(MASM) /nologo /Fo$@ /c $< $(INCLUDEDIRS)'),
- ]
-
for rule in rules:
m.rule(*rule)
@@ -392,6 +432,25 @@
self._handle_error(returncode, stdout, stderr, path.join('make'))
+class WinDefinition(posix.Definition):
+ def write(self, f):
+ def write_list(prefix, lst):
+ lst = lst or ['']
+ for i, fn in enumerate(lst):
+ print >> f, prefix, fn,
+ if i < len(lst)-1:
+ print >> f, '\\'
+ else:
+ print >> f
+ prefix = ' ' * len(prefix)
+ name, value = self.name, self.value
+ if isinstance(value, str):
+ f.write('%s = %s\n' % (name, value))
+ else:
+ write_list('%s =' % (name,), value)
+ f.write('\n')
+
+
class NMakefile(posix.GnuMakefile):
def write(self, out=None):
# nmake expands macros when it parses rules.
@@ -410,6 +469,14 @@
if out is None:
f.close()
+ def definition(self, name, value):
+ defs = self.defs
+ defn = WinDefinition(name, value)
+ if name in defs:
+ self.lines[defs[name]] = defn
+ else:
+ defs[name] = len(self.lines)
+ self.lines.append(defn)
class MingwPlatform(posix.BasePosix):
name = 'mingw32'
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
