https://github.com/python/cpython/commit/2c8f329dc634290fb88636f85c05e473bc0104d5
commit: 2c8f329dc634290fb88636f85c05e473bc0104d5
branch: main
author: Irit Katriel <[email protected]>
committer: iritkatriel <[email protected]>
date: 2025-03-28T10:35:20Z
summary:
gh-131738: optimize builtin any/all/tuple calls with a generator expression arg
(#131737)
files:
A
Misc/NEWS.d/next/Core_and_Builtins/2025-03-25-20-38-06.gh-issue-131738.eCb0OQ.rst
M Include/internal/pycore_global_objects_fini_generated.h
M Include/internal/pycore_global_strings.h
M Include/internal/pycore_interp_structs.h
M Include/internal/pycore_magic_number.h
M Include/internal/pycore_opcode_utils.h
M Include/internal/pycore_runtime_init_generated.h
M Include/internal/pycore_unicodeobject_generated.h
M Lib/opcode.py
M Lib/test/test_builtin.py
M Objects/genobject.c
M Python/bytecodes.c
M Python/codegen.c
M Python/executor_cases.c.h
M Python/generated_cases.c.h
M Python/pylifecycle.c
diff --git a/Include/internal/pycore_global_objects_fini_generated.h
b/Include/internal/pycore_global_objects_fini_generated.h
index 1a2e387f894865..605c9e1c480c6e 100644
--- a/Include/internal/pycore_global_objects_fini_generated.h
+++ b/Include/internal/pycore_global_objects_fini_generated.h
@@ -792,7 +792,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(any));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arg));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs));
diff --git a/Include/internal/pycore_global_strings.h
b/Include/internal/pycore_global_strings.h
index 2177f8dc1a6a06..76bc00cfaecf9b 100644
--- a/Include/internal/pycore_global_strings.h
+++ b/Include/internal/pycore_global_strings.h
@@ -283,7 +283,9 @@ struct _Py_global_strings {
STRUCT_FOR_ID(aggregate_class)
STRUCT_FOR_ID(alias)
STRUCT_FOR_ID(align)
+ STRUCT_FOR_ID(all)
STRUCT_FOR_ID(allow_code)
+ STRUCT_FOR_ID(any)
STRUCT_FOR_ID(append)
STRUCT_FOR_ID(arg)
STRUCT_FOR_ID(argdefs)
diff --git a/Include/internal/pycore_interp_structs.h
b/Include/internal/pycore_interp_structs.h
index ee92a1e6f2dec9..44a4231fbc5e30 100644
--- a/Include/internal/pycore_interp_structs.h
+++ b/Include/internal/pycore_interp_structs.h
@@ -9,6 +9,7 @@ extern "C" {
#include "pycore_ast_state.h" // struct ast_state
#include "pycore_llist.h" // struct llist_node
+#include "pycore_opcode_utils.h" // NUM_COMMON_CONSTANTS
#include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR
#include "pycore_structs.h" // PyHamtObject
#include "pycore_tstate.h" // _PyThreadStateImpl
@@ -912,6 +913,7 @@ struct _is {
struct ast_state ast;
struct types_state types;
struct callable_cache callable_cache;
+ PyObject *common_consts[NUM_COMMON_CONSTANTS];
bool jit;
struct _PyExecutorObject *executor_list_head;
size_t trace_run_counter;
diff --git a/Include/internal/pycore_magic_number.h
b/Include/internal/pycore_magic_number.h
index 839a86e5830e78..edcc52b337364f 100644
--- a/Include/internal/pycore_magic_number.h
+++ b/Include/internal/pycore_magic_number.h
@@ -272,6 +272,7 @@ Known values:
Python 3.14a6 3617 (Branch monitoring for async for loops)
Python 3.14a6 3618 (Add oparg to END_ASYNC_FOR)
Python 3.14a6 3619 (Renumber RESUME opcode from 149 to 128)
+ Python 3.14a6 3620 (Optimize bytecode for all/any/tuple called on a genexp)
Python 3.15 will start with 3650
@@ -284,7 +285,7 @@ PC/launcher.c must also be updated.
*/
-#define PYC_MAGIC_NUMBER 3619
+#define PYC_MAGIC_NUMBER 3620
/* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
(little-endian) and then appending b'\r\n'. */
#define PYC_MAGIC_NUMBER_TOKEN \
diff --git a/Include/internal/pycore_opcode_utils.h
b/Include/internal/pycore_opcode_utils.h
index 0872231d1f2d11..b3056e7bb84c69 100644
--- a/Include/internal/pycore_opcode_utils.h
+++ b/Include/internal/pycore_opcode_utils.h
@@ -8,8 +8,6 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
-#include "opcode_ids.h"
-
#define MAX_REAL_OPCODE 254
#define IS_WITHIN_OPCODE_RANGE(opcode) \
@@ -67,7 +65,10 @@ extern "C" {
/* Values used as the oparg for LOAD_COMMON_CONSTANT */
#define CONSTANT_ASSERTIONERROR 0
#define CONSTANT_NOTIMPLEMENTEDERROR 1
-#define NUM_COMMON_CONSTANTS 2
+#define CONSTANT_BUILTIN_TUPLE 2
+#define CONSTANT_BUILTIN_ALL 3
+#define CONSTANT_BUILTIN_ANY 4
+#define NUM_COMMON_CONSTANTS 5
/* Values used in the oparg for RESUME */
#define RESUME_AT_FUNC_START 0
diff --git a/Include/internal/pycore_runtime_init_generated.h
b/Include/internal/pycore_runtime_init_generated.h
index 2582d32b88372a..3927cb8adf14c5 100644
--- a/Include/internal/pycore_runtime_init_generated.h
+++ b/Include/internal/pycore_runtime_init_generated.h
@@ -790,7 +790,9 @@ extern "C" {
INIT_ID(aggregate_class), \
INIT_ID(alias), \
INIT_ID(align), \
+ INIT_ID(all), \
INIT_ID(allow_code), \
+ INIT_ID(any), \
INIT_ID(append), \
INIT_ID(arg), \
INIT_ID(argdefs), \
diff --git a/Include/internal/pycore_unicodeobject_generated.h
b/Include/internal/pycore_unicodeobject_generated.h
index 2d26ca6949551e..67c71986410eef 100644
--- a/Include/internal/pycore_unicodeobject_generated.h
+++ b/Include/internal/pycore_unicodeobject_generated.h
@@ -920,10 +920,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
+ string = &_Py_ID(all);
+ _PyUnicode_InternStatic(interp, &string);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(allow_code);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
+ string = &_Py_ID(any);
+ _PyUnicode_InternStatic(interp, &string);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(append);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
diff --git a/Lib/opcode.py b/Lib/opcode.py
index ea7c27698dd338..0e9520b6832499 100644
--- a/Lib/opcode.py
+++ b/Lib/opcode.py
@@ -9,6 +9,7 @@
"HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname",
"hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"]
+import builtins
import _opcode
from _opcode import stack_effect
@@ -38,7 +39,8 @@
_intrinsic_1_descs = _opcode.get_intrinsic1_descs()
_intrinsic_2_descs = _opcode.get_intrinsic2_descs()
_special_method_names = _opcode.get_special_method_names()
-_common_constants = [AssertionError, NotImplementedError]
+_common_constants = [builtins.AssertionError, builtins.NotImplementedError,
+ builtins.tuple, builtins.all, builtins.any]
_nb_ops = _opcode.get_nb_ops()
hascompare = [opmap["COMPARE_OP"]]
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 5546e6345e9208..27b14c36b51b50 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -225,6 +225,8 @@ def test_all(self):
self.assertEqual(all(x > 42 for x in S), True)
S = [50, 40, 60]
self.assertEqual(all(x > 42 for x in S), False)
+ S = [50, 40, 60, TestFailingBool()]
+ self.assertEqual(all(x > 42 for x in S), False)
def test_any(self):
self.assertEqual(any([None, None, None]), False)
@@ -238,9 +240,59 @@ def test_any(self):
self.assertEqual(any([1, TestFailingBool()]), True) # Short-circuit
S = [40, 60, 30]
self.assertEqual(any(x > 42 for x in S), True)
+ S = [40, 60, 30, TestFailingBool()]
+ self.assertEqual(any(x > 42 for x in S), True)
S = [10, 20, 30]
self.assertEqual(any(x > 42 for x in S), False)
+ def test_all_any_tuple_optimization(self):
+ def f_all():
+ return all(x-2 for x in [1,2,3])
+
+ def f_any():
+ return any(x-1 for x in [1,2,3])
+
+ def f_tuple():
+ return tuple(2*x for x in [1,2,3])
+
+ funcs = [f_all, f_any, f_tuple]
+
+ for f in funcs:
+ # check that generator code object is not duplicated
+ code_objs = [c for c in f.__code__.co_consts if isinstance(c,
type(f.__code__))]
+ self.assertEqual(len(code_objs), 1)
+
+
+ # check the overriding the builtins works
+
+ global all, any, tuple
+ saved = all, any, tuple
+ try:
+ all = lambda x : "all"
+ any = lambda x : "any"
+ tuple = lambda x : "tuple"
+
+ overridden_outputs = [f() for f in funcs]
+ finally:
+ all, any, tuple = saved
+
+ self.assertEqual(overridden_outputs, ['all', 'any', 'tuple'])
+
+ # Now repeat, overriding the builtins module as well
+ saved = all, any, tuple
+ try:
+ builtins.all = all = lambda x : "all"
+ builtins.any = any = lambda x : "any"
+ builtins.tuple = tuple = lambda x : "tuple"
+
+ overridden_outputs = [f() for f in funcs]
+ finally:
+ all, any, tuple = saved
+ builtins.all, builtins.any, builtins.tuple = saved
+
+ self.assertEqual(overridden_outputs, ['all', 'any', 'tuple'])
+
+
def test_ascii(self):
self.assertEqual(ascii(''), '\'\'')
self.assertEqual(ascii(0), '0')
diff --git
a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-25-20-38-06.gh-issue-131738.eCb0OQ.rst
b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-25-20-38-06.gh-issue-131738.eCb0OQ.rst
new file mode 100644
index 00000000000000..ac99b2afa81838
--- /dev/null
+++
b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-25-20-38-06.gh-issue-131738.eCb0OQ.rst
@@ -0,0 +1 @@
+Compiler emits optimized code for builtin any/all/tuple calls over a generator
expression.
diff --git a/Objects/genobject.c b/Objects/genobject.c
index 0238613c2a19df..98b2c5004df8ac 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -19,6 +19,8 @@
#include "pycore_warnings.h" // _PyErr_WarnUnawaitedCoroutine()
+#include "opcode_ids.h" // RESUME, etc
+
// Forward declarations
static PyObject* gen_close(PyObject *, PyObject *);
static PyObject* async_gen_asend_new(PyAsyncGenObject *, PyObject *);
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index b6a482183b1d0b..b3b7441c31569a 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -1402,16 +1402,8 @@ dummy_func(
inst(LOAD_COMMON_CONSTANT, ( -- value)) {
// Keep in sync with _common_constants in opcode.py
- // If we ever have more than two constants, use a lookup table
- PyObject *val;
- if (oparg == CONSTANT_ASSERTIONERROR) {
- val = PyExc_AssertionError;
- }
- else {
- assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR);
- val = PyExc_NotImplementedError;
- }
- value = PyStackRef_FromPyObjectImmortal(val);
+ assert(oparg < NUM_COMMON_CONSTANTS);
+ value =
PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]);
}
inst(LOAD_BUILD_CLASS, ( -- bc)) {
diff --git a/Python/codegen.c b/Python/codegen.c
index 8cc484e98d64af..dc50737840f002 100644
--- a/Python/codegen.c
+++ b/Python/codegen.c
@@ -3820,6 +3820,92 @@ update_start_location_to_match_attr(compiler *c,
location loc,
return loc;
}
+static int
+maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end)
+{
+ asdl_expr_seq *args = e->v.Call.args;
+ asdl_keyword_seq *kwds = e->v.Call.keywords;
+ expr_ty func = e->v.Call.func;
+
+ if (! (func->kind == Name_kind &&
+ asdl_seq_LEN(args) == 1 &&
+ asdl_seq_LEN(kwds) == 0 &&
+ asdl_seq_GET(args, 0)->kind == GeneratorExp_kind))
+ {
+ return 0;
+ }
+
+ location loc = LOC(func);
+
+ int optimized = 0;
+ NEW_JUMP_TARGET_LABEL(c, skip_optimization);
+
+ int const_oparg = -1;
+ PyObject *initial_res = NULL;
+ int continue_jump_opcode = -1;
+ if (_PyUnicode_EqualToASCIIString(func->v.Name.id, "all")) {
+ const_oparg = CONSTANT_BUILTIN_ALL;
+ initial_res = Py_True;
+ continue_jump_opcode = POP_JUMP_IF_TRUE;
+ }
+ else if (_PyUnicode_EqualToASCIIString(func->v.Name.id, "any")) {
+ const_oparg = CONSTANT_BUILTIN_ANY;
+ initial_res = Py_False;
+ continue_jump_opcode = POP_JUMP_IF_FALSE;
+ }
+ else if (_PyUnicode_EqualToASCIIString(func->v.Name.id, "tuple")) {
+ const_oparg = CONSTANT_BUILTIN_TUPLE;
+ }
+ if (const_oparg != -1) {
+ ADDOP_I(c, loc, COPY, 1); // the function
+ ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, const_oparg);
+ ADDOP_COMPARE(c, loc, Is);
+ ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, skip_optimization);
+ ADDOP(c, loc, POP_TOP);
+
+ if (const_oparg == CONSTANT_BUILTIN_TUPLE) {
+ ADDOP_I(c, loc, BUILD_LIST, 0);
+ }
+ expr_ty generator_exp = asdl_seq_GET(args, 0);
+ VISIT(c, expr, generator_exp);
+
+ NEW_JUMP_TARGET_LABEL(c, loop);
+ NEW_JUMP_TARGET_LABEL(c, cleanup);
+
+ USE_LABEL(c, loop);
+ ADDOP_JUMP(c, loc, FOR_ITER, cleanup);
+ if (const_oparg == CONSTANT_BUILTIN_TUPLE) {
+ ADDOP_I(c, loc, LIST_APPEND, 2);
+ ADDOP_JUMP(c, loc, JUMP, loop);
+ }
+ else {
+ ADDOP(c, loc, TO_BOOL);
+ ADDOP_JUMP(c, loc, continue_jump_opcode, loop);
+ }
+
+ ADDOP(c, NO_LOCATION, POP_ITER);
+ if (const_oparg != CONSTANT_BUILTIN_TUPLE) {
+ ADDOP_LOAD_CONST(c, loc, initial_res == Py_True ? Py_False :
Py_True);
+ }
+ ADDOP_JUMP(c, loc, JUMP, end);
+
+ USE_LABEL(c, cleanup);
+ ADDOP(c, NO_LOCATION, END_FOR);
+ ADDOP(c, NO_LOCATION, POP_ITER);
+ if (const_oparg == CONSTANT_BUILTIN_TUPLE) {
+ ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_LIST_TO_TUPLE);
+ }
+ else {
+ ADDOP_LOAD_CONST(c, loc, initial_res);
+ }
+
+ optimized = 1;
+ ADDOP_JUMP(c, loc, JUMP, end);
+ }
+ USE_LABEL(c, skip_optimization);
+ return optimized;
+}
+
// Return 1 if the method call was optimized, 0 if not, and -1 on error.
static int
maybe_optimize_method_call(compiler *c, expr_ty e)
@@ -3926,14 +4012,18 @@ codegen_call(compiler *c, expr_ty e)
if (ret == 1) {
return SUCCESS;
}
+ NEW_JUMP_TARGET_LABEL(c, skip_normal_call);
RETURN_IF_ERROR(check_caller(c, e->v.Call.func));
VISIT(c, expr, e->v.Call.func);
+ RETURN_IF_ERROR(maybe_optimize_function_call(c, e, skip_normal_call));
location loc = LOC(e->v.Call.func);
ADDOP(c, loc, PUSH_NULL);
loc = LOC(e);
- return codegen_call_helper(c, loc, 0,
- e->v.Call.args,
- e->v.Call.keywords);
+ ret = codegen_call_helper(c, loc, 0,
+ e->v.Call.args,
+ e->v.Call.keywords);
+ USE_LABEL(c, skip_normal_call);
+ return ret;
}
static int
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index b3c32c5c5d029f..9306a6aea35435 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -1899,16 +1899,8 @@
_PyStackRef value;
oparg = CURRENT_OPARG();
// Keep in sync with _common_constants in opcode.py
- // If we ever have more than two constants, use a lookup table
- PyObject *val;
- if (oparg == CONSTANT_ASSERTIONERROR) {
- val = PyExc_AssertionError;
- }
- else {
- assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR);
- val = PyExc_NotImplementedError;
- }
- value = PyStackRef_FromPyObjectImmortal(val);
+ assert(oparg < NUM_COMMON_CONSTANTS);
+ value =
PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 0314f983506b0d..f1e22f6d3dd700 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -8723,16 +8723,8 @@
INSTRUCTION_STATS(LOAD_COMMON_CONSTANT);
_PyStackRef value;
// Keep in sync with _common_constants in opcode.py
- // If we ever have more than two constants, use a lookup table
- PyObject *val;
- if (oparg == CONSTANT_ASSERTIONERROR) {
- val = PyExc_AssertionError;
- }
- else {
- assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR);
- val = PyExc_NotImplementedError;
- }
- value = PyStackRef_FromPyObjectImmortal(val);
+ assert(oparg < NUM_COMMON_CONSTANTS);
+ value =
PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 484583e1c79579..934614e73b56f9 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -790,6 +790,26 @@ pycore_init_builtins(PyThreadState *tstate)
}
interp->callable_cache.len = len;
+ PyObject *all = PyDict_GetItemWithError(builtins_dict, &_Py_ID(all));
+ if (!all) {
+ goto error;
+ }
+
+ PyObject *any = PyDict_GetItemWithError(builtins_dict, &_Py_ID(any));
+ if (!any) {
+ goto error;
+ }
+
+ interp->common_consts[CONSTANT_ASSERTIONERROR] = PyExc_AssertionError;
+ interp->common_consts[CONSTANT_NOTIMPLEMENTEDERROR] =
PyExc_NotImplementedError;
+ interp->common_consts[CONSTANT_BUILTIN_TUPLE] = (PyObject*)&PyTuple_Type;
+ interp->common_consts[CONSTANT_BUILTIN_ALL] = all;
+ interp->common_consts[CONSTANT_BUILTIN_ANY] = any;
+
+ for (int i=0; i < NUM_COMMON_CONSTANTS; i++) {
+ assert(interp->common_consts[i] != NULL);
+ }
+
PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append));
if (list_append == NULL) {
goto error;
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
