Author: Philip Jenvey <pjen...@underboss.org> Branch: py3k Changeset: r70403:10edbabbaae9 Date: 2014-04-02 11:22 -0700 http://bitbucket.org/pypy/pypy/changeset/10edbabbaae9/
Log: merge default diff --git a/lib-python/2.7/test/test_argparse.py b/lib-python/2.7/test/test_argparse.py --- a/lib-python/2.7/test/test_argparse.py +++ b/lib-python/2.7/test/test_argparse.py @@ -10,6 +10,7 @@ import tempfile import unittest import argparse +import gc from StringIO import StringIO @@ -47,7 +48,11 @@ def tearDown(self): os.chdir(self.old_dir) - shutil.rmtree(self.temp_dir, True) + gc.collect() + for root, dirs, files in os.walk(self.temp_dir, topdown=False): + for name in files: + os.chmod(os.path.join(self.temp_dir, name), stat.S_IWRITE) + shutil.rmtree(self.temp_dir, True) def create_readonly_file(self, filename): file_path = os.path.join(self.temp_dir, filename) diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -429,12 +429,27 @@ Could we use LLVM? ------------------ -There is a (static) translation backend using LLVM in the branch -``llvm-translation-backend``. It can translate PyPy with or without the JIT on -Linux. +In theory yes. But we tried to use it 5 or 6 times already, as a +translation backend or as a JIT backend --- and failed each time. -Using LLVM as our JIT backend looks interesting as well -- we made an attempt, -but it failed: LLVM has no way to patch the generated machine code. +In more details: using LLVM as a (static) translation backend is +pointless nowadays because you can generate C code and compile it with +clang. (Note that compiling PyPy with clang gives a result that is not +faster than compiling it with gcc.) We might in theory get extra +benefits from LLVM's GC integration, but this requires more work on the +LLVM side before it would be remotely useful. Anyway, it could be +interfaced via a custom primitive in the C code. (The latest such +experimental backend is in the branch ``llvm-translation-backend``, +which can translate PyPy with or without the JIT on Linux.) + +On the other hand, using LLVM as our JIT backend looks interesting as +well --- but again we made an attempt, and it failed: LLVM has no way to +patch the generated machine code. + +So the position of the core PyPy developers is that if anyone wants to +make an N+1'th attempt with LLVM, they are welcome, and will be happy to +provide help in the IRC channel, but they are left with the burden of proof +that (a) it works and (b) it gives important benefits. ---------------------- How do I compile PyPy? diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/stm.rst @@ -0,0 +1,244 @@ +====================== +Transactional Memory +====================== + +.. contents:: + + +This page is about ``pypy-stm``, a special in-development version of +PyPy which can run multiple independent CPU-hungry threads in the same +process in parallel. It is side-stepping what is known in the Python +world as the "global interpreter lock (GIL)" problem. + +"STM" stands for Software Transactional Memory, the technique used +internally. This page describes ``pypy-stm`` from the perspective of a +user, describes work in progress, and finally gives references to more +implementation details. + +This work was done by Remi Meier and Armin Rigo. + + +Introduction and current status +=============================== + +``pypy-stm`` is a variant of the regular PyPy interpreter. With caveats +listed below, it should be in theory within 25%-50% of the speed of +PyPy, comparing the JITting version in both cases. It is called STM for +Software Transactional Memory, which is the internal technique used (see +`Reference to implementation details`_). + +**pypy-stm requires 64-bit Linux for now.** + +Development is done in the branch `stmgc-c7`_. If you are only +interested in trying it out, you can download a Ubuntu 12.04 binary +here__. The current version supports four "segments", which means that +it will run up to four threads in parallel (in other words, you get a +GIL effect again, but only if trying to execute more than 4 threads). + +To build a version from sources, you first need to compile a custom +version of clang; we recommend downloading `llvm and clang like +described here`__, but at revision 201645 (use ``svn co -r 201645 ...`` +for all checkouts). Then apply all the patches in `this directory`__: +they are fixes for the very extensive usage that pypy-stm does of a +clang-only feature (without them, you get crashes of clang). Then get +the branch `stmgc-c7`_ of PyPy and run:: + + rpython/bin/rpython -Ojit --stm pypy/goal/targetpypystandalone.py + +.. _`stmgc-c7`: https://bitbucket.org/pypy/pypy/src/stmgc-c7/ +.. __: http://buildbot.pypy.org/nightly/stmgc-c7/ +.. __: http://clang.llvm.org/get_started.html +.. __: https://bitbucket.org/pypy/stmgc/src/default/c7/llvmfix/ + + +Caveats: + +* It should generally work. Please do `report bugs`_ that manifest as a + crash or wrong behavior (markedly different from the behavior of a + regular PyPy). Performance bugs are likely to be known issues; we're + working on them. + +* The JIT warm-up time is abysmal (as opposed to the regular PyPy's, + which is "only" bad). Moreover, you should run it with a command like + ``pypy-stm --jit trace_limit=60000 args...``; the default value of + 6000 for ``trace_limit`` is currently too low (6000 should become + reasonable again as we improve). Also, in order to produce machine + code, the JIT needs to enter a special single-threaded mode for now. + This all means that you *will* get very bad performance results if + your program doesn't run for *many* seconds for now. + +* The GC is new; although clearly inspired by PyPy's regular GC, it + misses a number of optimizations for now. Programs allocating large + numbers of small objects that don't immediately die, as well as + programs that modify large lists or dicts, suffer from these missing + optimizations. + +* The GC has no support for destructors: the ``__del__`` method is + never called (including on file objects, which won't be closed for + you). This is of course temporary. + +* The STM system is based on very efficient read/write barriers, which + are mostly done (their placement could be improved a bit in + JIT-generated machine code). But the overall bookkeeping logic could + see more improvements (see Statistics_ below). + +* You can use `atomic sections`_, but the most visible missing thing is + that you don't get reports about the "conflicts" you get. This would + be the first thing that you need in order to start using atomic + sections more extensively. Also, for now: for better results, try to + explicitly force a transaction break just before (and possibly after) + each large atomic section, with ``time.sleep(0)``. + +* Forking the process is slow because the complete memory needs to be + copied manually right now. + +* Very long-running processes should eventually crash on an assertion + error because of a non-implemented overflow of an internal 29-bit + number, but this requires at the very least ten hours --- more + probably, several days or more. + +.. _`report bugs`: https://bugs.pypy.org/ + + + +Statistics +========== + +When a non-main thread finishes, you get statistics printed to stderr, +looking like that:: + + thread 0x7f73377fe600: + outside transaction 42182 0.506 s + run current 85466 0.000 s + run committed 34262 3.178 s + run aborted write write 6982 0.083 s + run aborted write read 550 0.005 s + run aborted inevitable 388 0.010 s + run aborted other 0 0.000 s + wait free segment 0 0.000 s + wait write read 78 0.027 s + wait inevitable 887 0.490 s + wait other 0 0.000 s + bookkeeping 51418 0.606 s + minor gc 162970 1.135 s + major gc 1 0.019 s + sync pause 59173 1.738 s + spin loop 129512 0.094 s + +The first number is a counter; the second number gives the associated +time (the amount of real time that the thread was in this state; the sum +of all the times should be equal to the total time between the thread's +start and the thread's end). The most important points are "run +committed", which gives the amount of useful work, and "outside +transaction", which should give the time spent e.g. in library calls +(right now it seems to be a bit larger than that; to investigate). +Everything else is overhead of various forms. (Short-, medium- and +long-term future work involves reducing this overhead :-) + +These statistics are not printed out for the main thread, for now. + + +Atomic sections +=============== + +While one of the goal of pypy-stm is to give a GIL-free but otherwise +unmodified Python, the other goal is to push for a better way to use +multithreading. For this, you (as the Python programmer) get an API +in the ``__pypy__.thread`` submodule: + +* ``__pypy__.thread.atomic``: a context manager (i.e. you use it in + a ``with __pypy__.thread.atomic:`` statement). It runs the whole + block of code without breaking the current transaction --- from + the point of view of a regular CPython/PyPy, this is equivalent to + saying that the GIL will not be released at all between the start and + the end of this block of code. + +The obvious usage is to use atomic blocks in the same way as one would +use locks: to protect changes to some shared data, you do them in a +``with atomic`` block, just like you would otherwise do them in a ``with +mylock`` block after ``mylock = thread.allocate_lock()``. This allows +you not to care about acquiring the correct locks in the correct order; +it is equivalent to having only one global lock. This is how +transactional memory is `generally described`__: as a way to efficiently +execute such atomic blocks, running them in parallel while giving the +illusion that they run in some serial order. + +.. __: http://en.wikipedia.org/wiki/Transactional_memory + +However, the less obvious intended usage of atomic sections is as a +wide-ranging replacement of explicit threads. You can turn a program +that is not multi-threaded at all into a program that uses threads +internally, together with large atomic sections to keep the behavior +unchanged. This capability can be hidden in a library or in the +framework you use; the end user's code does not need to be explicitly +aware of using threads. For a simple example of this, see +`lib_pypy/transaction.py`_. The idea is that if you have a program +where the function ``f(key, value)`` runs on every item of some big +dictionary, you can replace the loop with:: + + for key, value in bigdict.items(): + transaction.add(f, key, value) + transaction.run() + +This code runs the various calls to ``f(key, value)`` using a thread +pool, but every single call is done in an atomic section. The end +result is that the behavior should be exactly equivalent: you don't get +any extra multithreading issue. + +.. _`lib_pypy/transaction.py`: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/lib_pypy/transaction.py + +================== + +Other APIs in pypy-stm: + +* ``__pypy__.thread.getsegmentlimit()``: return the number of "segments" + in this pypy-stm. This is the limit above which more threads will not + be able to execute on more cores. (Right now it is limited to 4 due + to inter-segment overhead, but should be increased in the future. It + should also be settable, and the default value should depend on the + number of actual CPUs.) + +* ``__pypy__.thread.exclusive_atomic``: same as ``atomic``, but + raises an exception if you attempt to nest it inside another + ``atomic``. + +* ``__pypy__.thread.signals_enabled``: a context manager that runs + its block with signals enabled. By default, signals are only + enabled in the main thread; a non-main thread will not receive + signals (this is like CPython). Enabling signals in non-main threads + is useful for libraries where threads are hidden and the end user is + not expecting his code to run elsewhere than in the main thread. + +Note that all of this API is (or will be) implemented in a regular PyPy +too: for example, ``with atomic`` will simply mean "don't release the +GIL" and ``getsegmentlimit()`` will return 1. + +================== + + +Reference to implementation details +=================================== + +The core of the implementation is in a separate C library called stmgc_, +in the c7_ subdirectory. Please see the `README.txt`_ for more +information. + +.. _stmgc: https://bitbucket.org/pypy/stmgc/src/default/ +.. _c7: https://bitbucket.org/pypy/stmgc/src/default/c7/ +.. _`README.txt`: https://bitbucket.org/pypy/stmgc/raw/default/c7/README.txt + +PyPy itself adds on top of it the automatic placement of read__ and write__ +barriers and of `"becomes-inevitable-now" barriers`__, the logic to +`start/stop transactions as an RPython transformation`__ and as +`supporting`__ `C code`__, and the support in the JIT (mostly as a +`transformation step on the trace`__ and generation of custom assembler +in `assembler.py`__). + +.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/readbarrier.py +.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/memory/gctransform/stmframework.py +.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/inevitable.py +.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/jitdriver.py +.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stmgcintf.h +.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stmgcintf.c +.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/jit/backend/llsupport/stmrewrite.py +.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/jit/backend/x86/assembler.py diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -127,3 +127,10 @@ .. branch: win32-fixes4 fix more tests for win32 + +.. branch: latest-improve-doc +Fix broken links in documentation + +.. branch: ast-issue1673 +fix ast classes __dict__ are always empty problem and fix the ast deepcopy issue when +there is missing field \ No newline at end of file diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -49,13 +49,19 @@ w_type = space.type(self) w_fields = w_type.getdictvalue(space, "_fields") for w_name in space.fixedview(w_fields): - space.setitem(w_dict, w_name, + try: + space.setitem(w_dict, w_name, space.getattr(self, w_name)) + except OperationError: + pass w_attrs = space.findattr(w_type, space.wrap("_attributes")) if w_attrs: for w_name in space.fixedview(w_attrs): - space.setitem(w_dict, w_name, + try: + space.setitem(w_dict, w_name, space.getattr(self, w_name)) + except OperationError: + pass return space.newtuple([space.type(self), space.newtuple([]), w_dict]) @@ -3090,7 +3096,8 @@ w_self.setdictvalue(space, 'lineno', w_new_value) w_self.initialization_state &= ~1 return - w_self.deldictvalue(space, 'lineno') + # need to save the original object too + w_self.setdictvalue(space, 'lineno', w_new_value) w_self.initialization_state |= 1 def stmt_del_lineno(space, w_self): @@ -3117,7 +3124,8 @@ w_self.setdictvalue(space, 'col_offset', w_new_value) w_self.initialization_state &= ~2 return - w_self.deldictvalue(space, 'col_offset') + # need to save the original object too + w_self.setdictvalue(space, 'col_offset', w_new_value) w_self.initialization_state |= 2 def stmt_del_col_offset(space, w_self): @@ -3155,7 +3163,8 @@ w_self.setdictvalue(space, 'name', w_new_value) w_self.initialization_state &= ~4 return - w_self.deldictvalue(space, 'name') + # need to save the original object too + w_self.setdictvalue(space, 'name', w_new_value) w_self.initialization_state |= 4 def FunctionDef_del_name(space, w_self): @@ -3314,7 +3323,8 @@ w_self.setdictvalue(space, 'name', w_new_value) w_self.initialization_state &= ~4 return - w_self.deldictvalue(space, 'name') + # need to save the original object too + w_self.setdictvalue(space, 'name', w_new_value) w_self.initialization_state |= 4 def ClassDef_del_name(space, w_self): @@ -4635,7 +4645,8 @@ w_self.setdictvalue(space, 'module', w_new_value) w_self.initialization_state &= ~4 return - w_self.deldictvalue(space, 'module') + # need to save the original object too + w_self.setdictvalue(space, 'module', w_new_value) w_self.initialization_state |= 4 def ImportFrom_del_module(space, w_self): @@ -4684,7 +4695,8 @@ w_self.setdictvalue(space, 'level', w_new_value) w_self.initialization_state &= ~16 return - w_self.deldictvalue(space, 'level') + # need to save the original object too + w_self.setdictvalue(space, 'level', w_new_value) w_self.initialization_state |= 16 def ImportFrom_del_level(space, w_self): @@ -4936,7 +4948,8 @@ w_self.setdictvalue(space, 'lineno', w_new_value) w_self.initialization_state &= ~1 return - w_self.deldictvalue(space, 'lineno') + # need to save the original object too + w_self.setdictvalue(space, 'lineno', w_new_value) w_self.initialization_state |= 1 def expr_del_lineno(space, w_self): @@ -4963,7 +4976,8 @@ w_self.setdictvalue(space, 'col_offset', w_new_value) w_self.initialization_state &= ~2 return - w_self.deldictvalue(space, 'col_offset') + # need to save the original object too + w_self.setdictvalue(space, 'col_offset', w_new_value) w_self.initialization_state |= 2 def expr_del_col_offset(space, w_self): @@ -6237,7 +6251,8 @@ w_self.setdictvalue(space, 'n', w_new_value) w_self.initialization_state &= ~4 return - w_self.deldictvalue(space, 'n') + # need to save the original object too + w_self.setdictvalue(space, 'n', w_new_value) w_self.initialization_state |= 4 def Num_del_n(space, w_self): @@ -6288,7 +6303,8 @@ w_self.setdictvalue(space, 's', w_new_value) w_self.initialization_state &= ~4 return - w_self.deldictvalue(space, 's') + # need to save the original object too + w_self.setdictvalue(space, 's', w_new_value) w_self.initialization_state |= 4 def Str_del_s(space, w_self): @@ -6339,7 +6355,8 @@ w_self.setdictvalue(space, 's', w_new_value) w_self.initialization_state &= ~4 return - w_self.deldictvalue(space, 's') + # need to save the original object too + w_self.setdictvalue(space, 's', w_new_value) w_self.initialization_state |= 4 def Bytes_del_s(space, w_self): @@ -6438,7 +6455,8 @@ w_self.setdictvalue(space, 'attr', w_new_value) w_self.initialization_state &= ~8 return - w_self.deldictvalue(space, 'attr') + # need to save the original object too + w_self.setdictvalue(space, 'attr', w_new_value) w_self.initialization_state |= 8 def Attribute_del_attr(space, w_self): @@ -6718,7 +6736,8 @@ w_self.setdictvalue(space, 'id', w_new_value) w_self.initialization_state &= ~4 return - w_self.deldictvalue(space, 'id') + # need to save the original object too + w_self.setdictvalue(space, 'id', w_new_value) w_self.initialization_state |= 4 def Name_del_id(space, w_self): @@ -6953,7 +6972,8 @@ w_self.setdictvalue(space, 'value', w_new_value) w_self.initialization_state &= ~4 return - w_self.deldictvalue(space, 'value') + # need to save the original object too + w_self.setdictvalue(space, 'value', w_new_value) w_self.initialization_state |= 4 def Const_del_value(space, w_self): @@ -7604,7 +7624,8 @@ w_self.setdictvalue(space, 'lineno', w_new_value) w_self.initialization_state &= ~1 return - w_self.deldictvalue(space, 'lineno') + # need to save the original object too + w_self.setdictvalue(space, 'lineno', w_new_value) w_self.initialization_state |= 1 def excepthandler_del_lineno(space, w_self): @@ -7631,7 +7652,8 @@ w_self.setdictvalue(space, 'col_offset', w_new_value) w_self.initialization_state &= ~2 return - w_self.deldictvalue(space, 'col_offset') + # need to save the original object too + w_self.setdictvalue(space, 'col_offset', w_new_value) w_self.initialization_state |= 2 def excepthandler_del_col_offset(space, w_self): @@ -7701,7 +7723,8 @@ w_self.setdictvalue(space, 'name', w_new_value) w_self.initialization_state &= ~8 return - w_self.deldictvalue(space, 'name') + # need to save the original object too + w_self.setdictvalue(space, 'name', w_new_value) w_self.initialization_state |= 8 def ExceptHandler_del_name(space, w_self): @@ -7804,7 +7827,8 @@ w_self.setdictvalue(space, 'vararg', w_new_value) w_self.initialization_state &= ~2 return - w_self.deldictvalue(space, 'vararg') + # need to save the original object too + w_self.setdictvalue(space, 'vararg', w_new_value) w_self.initialization_state |= 2 def arguments_del_vararg(space, w_self): @@ -7887,7 +7911,8 @@ w_self.setdictvalue(space, 'kwarg', w_new_value) w_self.initialization_state &= ~16 return - w_self.deldictvalue(space, 'kwarg') + # need to save the original object too + w_self.setdictvalue(space, 'kwarg', w_new_value) w_self.initialization_state |= 16 def arguments_del_kwarg(space, w_self): @@ -8024,7 +8049,8 @@ w_self.setdictvalue(space, 'arg', w_new_value) w_self.initialization_state &= ~1 return - w_self.deldictvalue(space, 'arg') + # need to save the original object too + w_self.setdictvalue(space, 'arg', w_new_value) w_self.initialization_state |= 1 def arg_del_arg(space, w_self): @@ -8107,7 +8133,8 @@ w_self.setdictvalue(space, 'arg', w_new_value) w_self.initialization_state &= ~1 return - w_self.deldictvalue(space, 'arg') + # need to save the original object too + w_self.setdictvalue(space, 'arg', w_new_value) w_self.initialization_state |= 1 def keyword_del_arg(space, w_self): @@ -8190,7 +8217,8 @@ w_self.setdictvalue(space, 'name', w_new_value) w_self.initialization_state &= ~1 return - w_self.deldictvalue(space, 'name') + # need to save the original object too + w_self.setdictvalue(space, 'name', w_new_value) w_self.initialization_state |= 1 def alias_del_name(space, w_self): @@ -8222,7 +8250,8 @@ w_self.setdictvalue(space, 'asname', w_new_value) w_self.initialization_state &= ~2 return - w_self.deldictvalue(space, 'asname') + # need to save the original object too + w_self.setdictvalue(space, 'asname', w_new_value) w_self.initialization_state |= 2 def alias_del_asname(space, w_self): diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -470,6 +470,7 @@ self.emit("raise OperationError(space.w_TypeError, " "space.w_None)", 3) else: + save_original_object = True level = 2 if field.opt and field.type.value != "int": self.emit("if space.is_w(w_new_value, space.w_None):", 2) @@ -607,13 +608,19 @@ w_type = space.type(self) w_fields = w_type.getdictvalue(space, "_fields") for w_name in space.fixedview(w_fields): - space.setitem(w_dict, w_name, + try: + space.setitem(w_dict, w_name, space.getattr(self, w_name)) + except OperationError: + pass w_attrs = space.findattr(w_type, space.wrap("_attributes")) if w_attrs: for w_name in space.fixedview(w_attrs): - space.setitem(w_dict, w_name, + try: + space.setitem(w_dict, w_name, space.getattr(self, w_name)) + except OperationError: + pass return space.newtuple([space.type(self), space.newtuple([]), w_dict]) diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py --- a/pypy/module/_ast/test/test_ast.py +++ b/pypy/module/_ast/test/test_ast.py @@ -403,3 +403,40 @@ mod.body[0].body[0].handlers[0].lineno = 7 mod.body[0].body[0].handlers[1].lineno = 6 code = compile(mod, "<test>", "exec") + + def test_dict_astNode(self): + import ast + num_node = ast.Num(n=2, lineno=2, col_offset=3) + dict_res = num_node.__dict__ + + assert dict_res == {'n':2, 'lineno':2, 'col_offset':3} + + def test_issue1673_Num_notfullinit(self): + import ast + import copy + num_node = ast.Num(n=2,lineno=2) + assert num_node.n == 2 + assert num_node.lineno == 2 + num_node2 = copy.deepcopy(num_node) + + def test_issue1673_Num_fullinit(self): + import ast + import copy + num_node = ast.Num(n=2,lineno=2,col_offset=3) + num_node2 = copy.deepcopy(num_node) + assert num_node.n == num_node2.n + assert num_node.lineno == num_node2.lineno + assert num_node.col_offset == num_node2.col_offset + dict_res = num_node2.__dict__ + assert dict_res == {'n':2, 'lineno':2, 'col_offset':3} + + def test_issue1673_Str(self): + import ast + import copy + str_node = ast.Str(n=2,lineno=2) + assert str_node.n == 2 + assert str_node.lineno == 2 + str_node2 = copy.deepcopy(str_node) + dict_res = str_node2.__dict__ + assert dict_res == {'n':2, 'lineno':2} + \ No newline at end of file diff --git a/rpython/tool/runsubprocess.py b/rpython/tool/runsubprocess.py --- a/rpython/tool/runsubprocess.py +++ b/rpython/tool/runsubprocess.py @@ -11,6 +11,9 @@ def run_subprocess(executable, args, env=None, cwd=None): return _run(executable, args, env, cwd) +shell_default = False +if sys.platform == 'win32': + shell_default = True def _run(executable, args, env, cwd): # unless overridden below if isinstance(args, str): @@ -21,7 +24,9 @@ args = [str(executable)] else: args = [str(executable)] + args - shell = False + # shell=True on unix-like is a known security vulnerability, but + # on windows shell=True does not properly propogate the env dict + shell = shell_default # Just before spawning the subprocess, do a gc.collect(). This # should help if we are running on top of PyPy, if the subprocess diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -414,7 +414,8 @@ try: returncode, stdout, stderr = _run_subprocess( 'nmake', - ['/nologo', '/f', str(path.join('Makefile'))] + extra_opts) + ['/nologo', '/f', str(path.join('Makefile'))] + extra_opts, + env = self.c_environ) finally: oldcwd.chdir() _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit