https://github.com/medismailben updated https://github.com/llvm/llvm-project/pull/209647
>From 820a90b897961d946a648f122a5b0e54b4f26c04 Mon Sep 17 00:00:00 2001 From: Med Ismail Bennani <[email protected]> Date: Tue, 14 Jul 2026 15:43:29 -0700 Subject: [PATCH] [lldb/script] Add scripting extension template generator This patch adds a `scripting extension generate <ExtensionType>...` command that introspects a Python extension base class and emits a skeleton subclass with `# TODO: Implement` stubs for its abstract methods (or, with `-a`, every method), then opens the result in an editor. Generated imports use `from <module> import <class>`, and the generated `__init__` forwards its arguments to `super().__init__(...)` since every base class relies on its constructor to set up attributes (`self.target`, `self.process`, ...) that inherited, non-overridden methods depend on. When the host can't open an external editor (e.g. non-macOS), the command reports that as a message rather than an error, since the file is already written. To catch regressions in the generator itself instead of just checking that a file was produced, this patch adds `TestScriptingExtensionGenerate.py`, which generates a template for every extension kind on the fly and drives it through a live lldb session (`process launch -C`, `breakpoint set -P`, `target stop-hook add -P`, etc.). Signed-off-by: Med Ismail Bennani <[email protected]> --- .../examples/python/cpython_frame_provider.py | 296 ++++++++++++++++++ .../lldb/Interpreter/ScriptInterpreter.h | 12 + .../Commands/CommandObjectScripting.cpp | 161 ++++++++++ lldb/source/Commands/Options.td | 23 ++ lldb/source/Interpreter/ScriptInterpreter.cpp | 9 + .../Interpreter/embedded_interpreter.py | 32 ++ .../Python/ScriptInterpreterPython.cpp | 258 +++++++++++++++ .../Python/ScriptInterpreterPython.h | 15 + .../scripting_extension_generate/Makefile | 3 + .../TestScriptingExtensionGenerate.py | 164 ++++++++++ .../scripting_extension_generate/main.c | 12 + 11 files changed, 985 insertions(+) create mode 100644 lldb/examples/python/cpython_frame_provider.py create mode 100644 lldb/test/API/functionalities/scripting_extension_generate/Makefile create mode 100644 lldb/test/API/functionalities/scripting_extension_generate/TestScriptingExtensionGenerate.py create mode 100644 lldb/test/API/functionalities/scripting_extension_generate/main.c diff --git a/lldb/examples/python/cpython_frame_provider.py b/lldb/examples/python/cpython_frame_provider.py new file mode 100644 index 0000000000000..bde0eecbe7755 --- /dev/null +++ b/lldb/examples/python/cpython_frame_provider.py @@ -0,0 +1,296 @@ +""" +Scripted frame provider that replaces CPython's C interpreter frames with the +Python-level frames they are actually executing. + +When you unwind a stopped CPython process, every Python-level call shows up as an +opaque ``Python`_PyEval_EvalFrameDefault`` C frame, interleaved with interpreter +machinery (``_PyFunction_Vectorcall``, ``PyObject_Call``, ...). This provider +walks the interpreter's ``PyFrameObject`` chain out of target memory and swaps +each ``_PyEval_EvalFrameDefault`` frame for the corresponding Python function, +e.g.:: + + frame #30: 0x... Python`_PyEval_EvalFrameDefault + 17640 + +becomes:: + + frame #30: run_suite(...) at dotest.py:1101 + +The surrounding CPython machinery frames are collapsed (hidden) so the backtrace +reads as a clean Python call stack. Non-Python frames (liblldb, dyld, ...) are +passed through untouched. + +Usage: + + (lldb) command script import /path/to/cpython_frame_provider.py + (lldb) target frame-provider register -C cpython_frame_provider.CPythonFrameProvider + (lldb) bt + +To go back to the raw C frames, unregister the provider or use `bt --provider none`. + +IMPORTANT: The struct offsets below are specific to CPython 3.9 on a 64-bit, +little-endian, release build (no `--with-trace-refs`). They will not match other +Python versions; 3.11+ in particular replaced `PyFrameObject` with an in-thread +`_PyInterpreterFrame` stack and would need a different walker. +""" + +import lldb +from lldb.plugins.scripted_process import ScriptedFrame +from lldb.plugins.scripted_frame_provider import ScriptedFrameProvider + +# The C symbol every executing Python frame runs inside. +EVAL_FRAME_SYMBOL = "_PyEval_EvalFrameDefault" + +# CPython 3.9 struct field offsets (64-bit). See Include/cpython/*.h in the +# matching CPython source. Kept as named constants so a version bump is a +# localized edit rather than a scatter of magic numbers. +# +# struct _ts (PyThreadState): +TSTATE_FRAME = 24 # PyFrameObject *frame +# +# PyFrameObject (PyObject_VAR_HEAD is 24 bytes): +FRAME_F_BACK = 24 # struct _frame *f_back +FRAME_F_CODE = 32 # PyCodeObject *f_code +FRAME_F_LASTI = 104 # int f_lasti (byte offset of last executed instruction) +# +# PyCodeObject (PyObject_HEAD is 16 bytes): +CODE_CO_FIRSTLINENO = 40 # int co_firstlineno +CODE_CO_FILENAME = 104 # PyObject *co_filename +CODE_CO_NAME = 112 # PyObject *co_name +CODE_CO_LNOTAB = 120 # PyObject *co_lnotab (bytes) +# +# PyBytesObject (for co_lnotab): PyObject_VAR_HEAD (24) then Py_hash_t. +BYTES_OB_SIZE = 16 # Py_ssize_t ob_size +BYTES_OB_SVAL = 32 # char ob_sval[] +# +# Compact-ASCII PyUnicodeObject payload starts right after the PyASCIIObject +# header. Python identifiers and most source paths are ASCII, so reading a C +# string here recovers the text without decoding PEP 393 kinds. +UNICODE_ASCII_DATA = 48 + + +class PythonFrame(ScriptedFrame): + """A synthetic frame standing in for one CPython interpreter frame.""" + + def __init__(self, thread, idx, name, hidden, filename=None, line=0): + super().__init__(thread, lldb.SBStructuredData()) + self.idx = idx + self.name = name + self.hidden = hidden + self.filename = filename + self.line = line + + def get_id(self): + return self.idx + + def get_pc(self): + # No PC: a real address makes ScriptedFrame::Create resolve a symbol + # context from it (ScriptedFrame.cpp), and the `bt` formatter renders + # that symbol ("_PyEval_EvalFrameDefault + N") instead of our name. + # LLDB_INVALID_ADDRESS skips symbolication so get_function_name() wins. + return lldb.LLDB_INVALID_ADDRESS + + def get_function_name(self): + return self.name + + def get_symbol_context(self): + # With no PC, a synthetic line entry is the only way LLDB learns this + # frame's source location, so `frame select`/`source list` can show the + # actual .py source. The file:line in `bt` also comes from here, not the + # function name. + if not self.filename: + return None + line_entry = lldb.SBLineEntry() + line_entry.SetFileSpec(lldb.SBFileSpec(self.filename, True)) + line_entry.SetLine(self.line) + sym_ctx = lldb.SBSymbolContext() + sym_ctx.SetLineEntry(line_entry) + return sym_ctx + + def is_artificial(self): + return False + + def is_hidden(self): + return self.hidden + + def get_register_context(self): + return None + + +class CPythonFrameProvider(ScriptedFrameProvider): + """Replace `_PyEval_EvalFrameDefault` C frames with Python-level frames.""" + + def __init__(self, input_frames, args): + super().__init__(input_frames, args) + # Output plan, one entry per input frame: either an int (pass the input + # frame through) or a PythonFrame. Built lazily on first access. + self._plan = None + + @staticmethod + def get_description(): + return "Replace CPython interpreter C frames with Python-level frames" + + @staticmethod + def applies_to_thread(thread): + # Only touch threads that are actually running the Python interpreter. + for frame in thread: + if frame.GetFunctionName() == EVAL_FRAME_SYMBOL: + return True + return False + + def get_frame_at_index(self, index): + if self._plan is None: + self._plan = self._build_plan() + if index < 0 or index >= len(self._plan): + return None + return self._plan[index] + + # --- plan construction --------------------------------------------------- + + def _build_plan(self): + # Python frames, youngest first: [(filename, funcname, line), ...]. + py_frames = self._walk_python_frames() + + plan = [] + pyi = 0 + for i in range(len(self.input_frames)): + frame = self.input_frames[i] + name = frame.GetFunctionName() + + if name == EVAL_FRAME_SYMBOL: + if pyi < len(py_frames): + filename, funcname, line = py_frames[pyi] + pyi += 1 + # The name is just the function; the file:line comes from the + # synthetic line entry in PythonFrame.get_symbol_context(). + plan.append( + PythonFrame(self.thread, i, funcname, False, filename, line) + ) + else: + # More eval frames than decoded Python frames: the walk fell + # short. Show the raw C frame rather than hiding it, so a + # mismatch is visible instead of silently dropping frames. + plan.append(i) + elif self._is_cpython_internal(frame): + # Collapse the interpreter machinery between Python frames so the + # backtrace reads as a pure Python call stack. + plan.append(PythonFrame(self.thread, i, name or "<cpython>", True)) + else: + # Non-Python frame (liblldb, dyld, libsystem, ...): pass through. + plan.append(i) + return plan + + @staticmethod + def _is_cpython_internal(frame): + module = frame.GetModule() + if not module: + return False + filename = module.GetFileSpec().GetFilename() + return filename == "Python" + + # --- CPython memory decoding --------------------------------------------- + + def _walk_python_frames(self): + frames = [] + frame_ptr = self._current_frame_object() + while frame_ptr: + info = self._decode_frame(frame_ptr) + if info is None: + break + frames.append(info) + frame_ptr = self._read_ptr(frame_ptr + FRAME_F_BACK) + return frames + + def _current_frame_object(self): + """Return the address of the topmost PyFrameObject, or 0 on failure. + + Discovering the current thread state without debug info is the one step + that needs the interpreter's help, so we call PyGILState_GetThisThreadState + (a pure lookup that neither acquires the GIL nor allocates) and read its + `frame` field. Everything after this is a passive memory read. + """ + frame0 = self.thread.GetFrameAtIndex(0) + if not frame0.IsValid(): + return 0 + value = frame0.EvaluateExpression("(void *)PyGILState_GetThisThreadState()") + if not value.IsValid() or value.GetError().Fail(): + return 0 + tstate = value.GetValueAsUnsigned(0) + if not tstate: + return 0 + return self._read_ptr(tstate + TSTATE_FRAME) + + def _decode_frame(self, frame_ptr): + code_ptr = self._read_ptr(frame_ptr + FRAME_F_CODE) + if not code_ptr: + return None + filename = self._read_pystr(code_ptr + CODE_CO_FILENAME) or "<unknown>" + funcname = self._read_pystr(code_ptr + CODE_CO_NAME) or "<unknown>" + lasti = self._read_int(frame_ptr + FRAME_F_LASTI) + line = self._addr2line(code_ptr, lasti) + return (filename, funcname, line) + + def _addr2line(self, code_ptr, lasti): + """Reimplements PyCode_Addr2Line: map a bytecode offset to a source line.""" + first_line = self._read_uint(code_ptr + CODE_CO_FIRSTLINENO, 4) + if lasti < 0: + return first_line + lnotab_ptr = self._read_ptr(code_ptr + CODE_CO_LNOTAB) + if not lnotab_ptr: + return first_line + size = self._read_uint(lnotab_ptr + BYTES_OB_SIZE, 8) + if size <= 0: + return first_line + data = self._read_mem(lnotab_ptr + BYTES_OB_SVAL, size) + if data is None: + return first_line + + line = first_line + addr = 0 + for i in range(0, len(data) - 1, 2): + addr += data[i] + if addr > lasti: + break + line_incr = data[i + 1] + if line_incr >= 0x80: # signed char + line_incr -= 0x100 + line += line_incr + return line + + # --- raw memory helpers -------------------------------------------------- + + def _read_ptr(self, addr): + error = lldb.SBError() + val = self.process.ReadPointerFromMemory(addr, error) + return val if error.Success() else 0 + + def _read_uint(self, addr, size): + error = lldb.SBError() + val = self.process.ReadUnsignedFromMemory(addr, size, error) + return val if error.Success() else 0 + + def _read_int(self, addr): + val = self._read_uint(addr, 4) + return val - 0x100000000 if val >= 0x80000000 else val + + def _read_mem(self, addr, size): + error = lldb.SBError() + data = self.process.ReadMemory(addr, size, error) + return data if error.Success() else None + + def _read_pystr(self, ptr_field): + obj = self._read_ptr(ptr_field) + if not obj: + return None + error = lldb.SBError() + text = self.process.ReadCStringFromMemory(obj + UNICODE_ASCII_DATA, 4096, error) + return text if error.Success() and text else None + + +def __lldb_init_module(debugger, internal_dict): + debugger.HandleCommand( + "target frame-provider register -C %s.CPythonFrameProvider" % __name__ + ) + print( + "Registered CPythonFrameProvider. Run 'bt' to see Python-level frames " + "(or 'bt --provider none' for the raw C frames)." + ) diff --git a/lldb/include/lldb/Interpreter/ScriptInterpreter.h b/lldb/include/lldb/Interpreter/ScriptInterpreter.h index 2b9cb8f7bb866..12a915636ef92 100644 --- a/lldb/include/lldb/Interpreter/ScriptInterpreter.h +++ b/lldb/include/lldb/Interpreter/ScriptInterpreter.h @@ -39,6 +39,7 @@ #include "lldb/Utility/Broadcaster.h" #include "lldb/Utility/Status.h" #include "lldb/Utility/StructuredData.h" +#include "lldb/Utility/UnimplementedError.h" #include "lldb/lldb-private.h" #include <optional> @@ -170,6 +171,12 @@ class ScriptInterpreter : public PluginInterface { virtual StructuredData::DictionarySP GetInterpreterInfo(); + virtual llvm::Expected<FileSpec> GenerateExtensionTemplate( + const std::string name, + std::vector<std::pair<llvm::StringRef, + llvm::SmallVector<llvm::StringRef>>> &extensions, + bool m_generate_non_abstract_methods, std::string output_file); + ~ScriptInterpreter() override = default; virtual bool Interrupt() { return false; } @@ -526,6 +533,11 @@ class ScriptInterpreter : public PluginInterface { static lldb::ScriptLanguage StringToLanguage(const llvm::StringRef &string); + virtual llvm::Expected<std::string> + ExtensionToImportPath(lldb::ScriptedExtension extension) { + return llvm::make_error<UnimplementedError>(); + } + static llvm::StringLiteral ExtensionToString(lldb::ScriptedExtension extension); diff --git a/lldb/source/Commands/CommandObjectScripting.cpp b/lldb/source/Commands/CommandObjectScripting.cpp index ff02169df1172..e7c07a8e2da41 100644 --- a/lldb/source/Commands/CommandObjectScripting.cpp +++ b/lldb/source/Commands/CommandObjectScripting.cpp @@ -371,6 +371,164 @@ class CommandObjectScriptingExtensionList : public CommandObjectParsed { CommandOptions m_options; }; +#define LLDB_OPTIONS_scripting_extension_generate +#include "CommandOptions.inc" + +class CommandObjectScriptingExtensionGenerate : public CommandObjectParsed { +public: + CommandObjectScriptingExtensionGenerate(CommandInterpreter &interpreter) + : CommandObjectParsed(interpreter, "scripting extension generate", + "Generate a scripting extension template. ", + "scripting extension generate") { + AddSimpleArgumentList(eArgTypeScriptedExtension, eArgRepeatPlus); + } + + ~CommandObjectScriptingExtensionGenerate() override = default; + + Options *GetOptions() override { return &m_options; } + + class CommandOptions : public Options { + public: + CommandOptions() = default; + ~CommandOptions() override = default; + Status SetOptionValue(uint32_t option_idx, llvm::StringRef option_arg, + ExecutionContext *execution_context) override { + Status error; + const char short_option = + g_scripting_extension_generate_options[option_idx].short_option; + const char *long_option = + g_scripting_extension_generate_options[option_idx].long_option; + + switch (short_option) { + case 'a': + bool success; + if (OptionArgParser::ToBoolean(option_arg, true, &success)) + m_generate_non_abstract_methods = eLazyBoolYes; + else + m_generate_non_abstract_methods = eLazyBoolNo; + + if (!success) + error = Status::FromError( + CreateOptionParsingError(option_arg, short_option, long_option, + g_bool_parsing_error_message)); + break; + case 'l': + m_language = (lldb::ScriptLanguage)OptionArgParser::ToOptionEnum( + option_arg, GetDefinitions()[option_idx].enum_values, + eScriptLanguageNone, error); + if (!error.Success()) + error = Status::FromErrorStringWithFormatv( + "unrecognized value for language '{0}'", option_arg); + break; + case 'n': + m_generated_class_prefix = option_arg.str(); + break; + case 'o': + m_output_filepath = option_arg.str(); + break; + default: + llvm_unreachable("Unimplemented option"); + } + + return error; + } + + void OptionParsingStarting(ExecutionContext *execution_context) override { + m_generate_non_abstract_methods = eLazyBoolCalculate; + m_language = lldb::eScriptLanguageDefault; + m_generated_class_prefix.clear(); + m_output_filepath.clear(); + } + + llvm::ArrayRef<OptionDefinition> GetDefinitions() override { + return llvm::ArrayRef(g_scripting_extension_generate_options); + } + + LazyBool m_generate_non_abstract_methods; + lldb::ScriptLanguage m_language = lldb::eScriptLanguageDefault; + std::string m_generated_class_prefix; + std::string m_output_filepath; + }; + + void + HandleArgumentCompletion(CompletionRequest &request, + OptionElementVector &opt_element_vector) override { + uint32_t completion_mask = + lldb::eScriptedExtensionCompletion | lldb::eDiskFileCompletion; + lldb_private::CommandCompletions::InvokeCommonCompletionCallbacks( + GetCommandInterpreter(), completion_mask, request, nullptr); + } + +protected: + void DoExecute(Args &command, CommandReturnObject &result) override { + if (command.GetArgumentCount() == 0) { + result.SetError( + Status::FromErrorString("specify extension name to generate")); + return; + } + + std::vector<std::pair<llvm::StringRef, llvm::SmallVector<llvm::StringRef>>> + name_import_pair; + + for (size_t i = 0; i < command.GetArgumentCount(); i++) { + llvm::StringRef extension_name = command.GetArgumentAtIndex(i); + llvm::SmallVector<llvm::StringRef> extension_components; + extension_name.split(extension_components, "."); + lldb::ScriptedExtension extension = + ScriptInterpreter::StringToExtension(extension_components.back()); + if (extension == eScriptedExtensionInvalid) { + result.SetError(Status::FromErrorString("invalid extension name")); + return; + } + name_import_pair.push_back({extension_name, extension_components}); + } + + lldb::ScriptLanguage language = + (m_options.m_language == lldb::eScriptLanguageNone) + ? m_interpreter.GetDebugger().GetScriptLanguage() + : m_options.m_language; + + if (language == lldb::eScriptLanguageNone) { + result.AppendError( + "the script-lang setting is set to none - scripting not available"); + return; + } + + ScriptInterpreter *script_interpreter = + GetDebugger().GetScriptInterpreter(true, language); + + if (script_interpreter == nullptr) { + result.AppendError("no script interpreter"); + return; + } + + auto generated_file_or_err = script_interpreter->GenerateExtensionTemplate( + m_options.m_generated_class_prefix, name_import_pair, + m_options.m_generate_non_abstract_methods == eLazyBoolYes, + m_options.m_output_filepath); + if (!generated_file_or_err) { + result.SetError(generated_file_or_err.takeError()); + return; + } + + if (llvm::Error err = Host::OpenFileInExternalEditor( + "", *generated_file_or_err, 1, true)) { + // Opening the file in an editor is a convenience, not a requirement: + // the template was already written to disk successfully, so don't + // fail the whole command over it (e.g. no external editor available + // on this platform). + llvm::consumeError(std::move(err)); + result.AppendMessageWithFormatv( + "Generated scripting extension template: {0}", + generated_file_or_err->GetPath()); + } + result.SetStatus(eReturnStatusSuccessFinishNoResult); + } + +private: + CommandOptions m_options; +}; + class CommandObjectMultiwordScriptingExtension : public CommandObjectMultiword { public: CommandObjectMultiwordScriptingExtension(CommandInterpreter &interpreter) @@ -381,6 +539,9 @@ class CommandObjectMultiwordScriptingExtension : public CommandObjectMultiword { LoadSubCommand( "list", CommandObjectSP(new CommandObjectScriptingExtensionList(interpreter))); + LoadSubCommand("generate", + CommandObjectSP(new CommandObjectScriptingExtensionGenerate( + interpreter))); } ~CommandObjectMultiwordScriptingExtension() override = default; diff --git a/lldb/source/Commands/Options.td b/lldb/source/Commands/Options.td index bd6fa8efad0a5..86c8b86c8f1de 100644 --- a/lldb/source/Commands/Options.td +++ b/lldb/source/Commands/Options.td @@ -1587,6 +1587,29 @@ let Command = "scripting extension list" in { Desc<"Output the scripted extension list in json format.">; } +let Command = "scripting extension generate" in { + def scripting_extension_generate_all_methods + : Option<"all", "a">, + Arg<"Boolean">, + Desc<"Generate all the extension methods, not only the abstract ones.">; + def scripting_extension_generate_language + : Option<"language", "l">, + EnumArg<"ScriptLang">, + Desc<"Specify the scripting " + " language. If none is specified the default scripting language " + "is used.">; + def scripting_extension_generate_class_suffix + : Option<"name", "n">, + Arg<"ClassName">, + Desc<"Specify the class prefix name to include in the generated class " + "definitions.">; + def scripting_extension_generate_output_file + : Option<"output", "o">, + Group<1>, + Arg<"Filename">, + Desc<"File location where the extension is generated.">; +} + let Command = "source info" in { def source_info_count : Option<"count", "c">, Arg<"Count">, diff --git a/lldb/source/Interpreter/ScriptInterpreter.cpp b/lldb/source/Interpreter/ScriptInterpreter.cpp index 8420782b02814..c73fbd8204f03 100644 --- a/lldb/source/Interpreter/ScriptInterpreter.cpp +++ b/lldb/source/Interpreter/ScriptInterpreter.cpp @@ -15,6 +15,7 @@ #include "lldb/Utility/Status.h" #include "lldb/Utility/Stream.h" #include "lldb/Utility/StringList.h" +#include "lldb/Utility/UnimplementedError.h" #include "lldb/ValueObject/ValueObject.h" #include "llvm/ADT/StringSwitch.h" #if defined(_WIN32) @@ -50,6 +51,14 @@ StructuredData::DictionarySP ScriptInterpreter::GetInterpreterInfo() { return nullptr; } +llvm::Expected<FileSpec> ScriptInterpreter::GenerateExtensionTemplate( + const std::string name, + std::vector<std::pair<llvm::StringRef, llvm::SmallVector<llvm::StringRef>>> + &extensions, + bool generate_non_abstract_methods, std::string output_file) { + return llvm::make_error<UnimplementedError>(); +} + bool ScriptInterpreter::LoadScriptingModule( const char *filename, const LoadScriptOptions &options, lldb_private::Status &error, StructuredData::ObjectSP *module_sp, diff --git a/lldb/source/Interpreter/embedded_interpreter.py b/lldb/source/Interpreter/embedded_interpreter.py index 12c47bd712816..dfd8ff6fcfbc0 100644 --- a/lldb/source/Interpreter/embedded_interpreter.py +++ b/lldb/source/Interpreter/embedded_interpreter.py @@ -77,6 +77,38 @@ def run_python_interpreter(local_dict): if e.code: print("Script exited with code %s" % e.code) + +def generate_extension_schema(cls): + import inspect, json, typing + + def _get_function_metadata(func): + try: + hints = typing.get_type_hints(func) + type_hints = {k: str(v) for k, v in hints.items()} + except Exception: + type_hints = {} + return { + "signature": str(inspect.signature(func)), + "type_hints": type_hints, + "is_abstract": getattr(func, "__isabstractmethod__", False), + "doc": inspect.getdoc(func), + } + + members = [] + for name, member in inspect.getmembers(cls): + if inspect.isfunction(member): + members.append({"name": name, **_get_function_metadata(member)}) + return json.dumps( + { + "class": cls.__name__, + "module": cls.__module__, + "doc": inspect.getdoc(cls), + "members": members, + }, + separators=(",", ":"), + ) + + def run_one_line(local_dict, input_string): global g_run_one_line_str try: diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp index f3f2b801d43e2..d24bfb2c2afa2 100644 --- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp +++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp @@ -36,11 +36,13 @@ #include "lldb/Target/ThreadPlan.h" #include "lldb/Utility/Instrumentation.h" #include "lldb/Utility/LLDBLog.h" +#include "lldb/Utility/StructuredData.h" #include "lldb/Utility/Timer.h" #include "lldb/ValueObject/ValueObject.h" #include "lldb/lldb-enumerations.h" #include "lldb/lldb-forward.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorExtras.h" @@ -265,6 +267,262 @@ StructuredData::DictionarySP ScriptInterpreterPython::GetInterpreterInfo() { return info_json.CreateStructuredDictionary(); } +llvm::Expected<std::string> ScriptInterpreterPython::ExtensionToImportPath( + lldb::ScriptedExtension extension) { + switch (extension) { + case eScriptedExtensionOperatingSystem: + return "lldb.plugins.operating_system"; + case eScriptedExtensionScriptedPlatform: + return "lldb.plugins.scripted_platform"; + case eScriptedExtensionScriptedProcess: + return "lldb.plugins.scripted_process"; + case eScriptedExtensionScriptedHook: + return "lldb.plugins.scripted_hook"; + case eScriptedExtensionScriptedBreakpointResolver: + return "lldb.plugins.scripted_breakpoint"; + case eScriptedExtensionScriptedThreadPlan: + return "lldb.plugins.scripted_thread_plan"; + case eScriptedExtensionScriptedFrameProvider: + return "lldb.plugins.scripted_frame_provider"; + case eScriptedExtensionScriptedThread: + case eScriptedExtensionScriptedFrame: + return "lldb.plugins.scripted_process"; + case eScriptedExtensionInvalid: + return llvm::createStringError("invalid extension name"); + } + return llvm::createStringError("invalid extension name"); +} + +llvm::Expected<StructuredData::ObjectSP> +ScriptInterpreterPython::GetExtensionSchema( + llvm::SmallVector<llvm::StringRef> &extension_path) { + lldb::ScriptedExtension extension = + ScriptInterpreter::StringToExtension(extension_path.back()); + auto import_path_or_err = ExtensionToImportPath(extension); + if (!import_path_or_err) + return import_path_or_err.takeError(); + + StreamString command_stream; + // __import__(path, fromlist=['']) imports the submodule and returns it + // directly (rather than the top-level package), as a single expression -- + // this keeps the whole call eval-able in one line while guaranteeing the + // module is imported first; referencing "<import_path>.<ClassName>" + // directly would only work if something else had already imported + // <import_path> as a side effect. + command_stream.Printf("lldb.embedded_interpreter.generate_extension_schema(" + "__import__('%s', fromlist=['']).%s)", + import_path_or_err->c_str(), + ScriptInterpreter::ExtensionToString(extension).data()); + + // eScriptReturnTypeCharStrOrNone hands back a pointer into a temporary + // Python object's buffer that gets destroyed (and, for a freshly created + // string like this one, deallocated) as soon as ExecuteOneLineWithReturn + // returns -- reading it afterwards is a use-after-free. Use + // eScriptReturnTypeOpaqueObject instead, which transfers a real owned + // reference we can safely extract the string from. + void *result_obj = nullptr; + if (!ExecuteOneLineWithReturn( + command_stream.GetData(), + ScriptInterpreter::eScriptReturnTypeOpaqueObject, &result_obj, + ExecuteScriptOptions().SetEnableIO(false))) + return llvm::createStringError("invalid extension schema format"); + + // ExecuteOneLineWithReturn releases the GIL before returning, so touching + // the returned object (Str() below can execute arbitrary Python code) must + // re-acquire it first. py_result is scoped so its destructor (a DECREF) + // also runs before the GIL is released below, not after. + std::string schema_str; + { + PyGILState_STATE gil_state = PyGILState_Ensure(); + { + PythonObject py_result(PyRefType::Owned, + static_cast<PyObject *>(result_obj)); + if (py_result.IsAllocated() && py_result.get() != Py_None) + schema_str = py_result.Str().GetString().str(); + } + PyGILState_Release(gil_state); + } + + if (schema_str.empty()) + return llvm::createStringError("empty extension schema"); + return StructuredData::ParseJSON(schema_str); +} + +llvm::Error ScriptInterpreterPython::ParseExtensionSchema( + Stream &s, llvm::StringRef output_script_prefix, + llvm::SmallVector<llvm::StringRef> &extension_path, + bool generate_non_abstract_methods) { + auto schema_or_err = GetExtensionSchema(extension_path); + if (!schema_or_err) + return schema_or_err.takeError(); + + StructuredData::ObjectSP schema = *schema_or_err; + StructuredData::Dictionary *dict = schema->GetAsDictionary(); + + llvm::StringRef base_class, import_path; + if (!dict->GetValueForKeyAsString("class", base_class)) + return llvm::createStringError( + llvm::formatv("extension schema dictionary is missing 'class' key") + .str()); + if (!dict->GetValueForKeyAsString("module", import_path)) + return llvm::createStringError( + llvm::formatv("extension schema dictionary is missing 'module' key") + .str()); + + // imports + s.Printf("from %s import %s\n", import_path.data(), base_class.data()); + s.EOL(); + + // class definition + s.Printf("class %s%s(%s):\n", output_script_prefix.data(), base_class.data(), + base_class.data()); + s.IndentMore(); + // members + StructuredData::Array *members; + if (!dict->GetValueForKeyAsArray("members", members)) + return llvm::createStringError("missing 'members' key in extension schema"); + + size_t num_emitted = 0; + for (size_t i = 0; i < members->GetSize(); i++) { + auto maybe_dict = members->GetItemAtIndexAsDictionary(i); + if (!maybe_dict) + return llvm::createStringError( + llvm::formatv( + "member at index {0} in extension schema isn't a dictionary") + .str()); + + StructuredData::Dictionary *member_dict = *maybe_dict; + llvm::StringRef symbol, args; + if (!member_dict->GetValueForKeyAsString("name", symbol)) + return llvm::createStringError( + llvm::formatv( + "member at index {0} in extension schema is missing 'name' key") + .str()); + if (!member_dict->GetValueForKeyAsString("signature", args)) + return llvm::createStringError( + llvm::formatv("member at index {0} in extension schema is missing " + "'signature' key") + .str()); + + // only generate abstract method + bool is_abstract = false; + bool has_is_abstract = + member_dict->GetValueForKeyAsBoolean("is_abstract", is_abstract); + if (!generate_non_abstract_methods) + if (!has_is_abstract || !is_abstract) + continue; + + s.Indent(); + s.Printf("def %s%s:\n", symbol.data(), args.data()); + + s.IndentMore(); + llvm::StringRef documentation; + if (member_dict->GetValueForKeyAsString("doc", documentation)) { + s.Indent(); + s.PutCString("\"\"\"\n"); + + llvm::SmallVector<llvm::StringRef> lines; + documentation.split(lines, "\n"); + + for (llvm::StringRef line : lines) { + s.Indent(); + s.PutCString(line); + s.EOL(); + } + + s.Indent(); + s.PutCString("\"\"\""); + s.EOL(); + } + + if (symbol == "__init__") { + // The base class' constructor sets up attributes (e.g. self.target, + // self.process) that the inherited, non-overridden methods rely on. + // Forward the same arguments so that state is still initialized. + llvm::StringRef params = args.trim("()"); + llvm::SmallVector<llvm::StringRef> param_names; + params.split(param_names, ','); + std::vector<std::string> forwarded_args; + for (llvm::StringRef param : param_names) { + param = param.split(':').first.split('=').first.trim(); + if (param.empty() || param == "self") + continue; + forwarded_args.push_back(param.str()); + } + s.Indent(); + s.Printf("super().__init__(%s)\n", + llvm::join(forwarded_args, ", ").c_str()); + } + + s.Indent(); + s.PutCString("# TODO: Implement\n"); + s.Indent(); + s.PutCString("pass\n\n"); + s.IndentLess(); + ++num_emitted; + } + + // A class with no body is a Python syntax error, so emit `pass` when + // the base class has nothing to stub out (all methods concrete and + // `generate_non_abstract_methods` is false). + if (num_emitted == 0) { + s.Indent(); + s.PutCString("pass\n"); + } + + return llvm::Error::success(); +} + +llvm::Expected<FileSpec> ScriptInterpreterPython::GenerateExtensionTemplate( + const std::string name, + std::vector<std::pair<llvm::StringRef, llvm::SmallVector<llvm::StringRef>>> + &extensions, + bool generate_non_abstract_methods, std::string output_file) { + StreamString generated_file_stream; + generated_file_stream.PutCString("import lldb\n\n"); + + for (auto extension_pair : extensions) { + if (llvm::Error err = ParseExtensionSchema(generated_file_stream, name, + extension_pair.second, + generate_non_abstract_methods)) + return std::move(err); + + generated_file_stream.PutCString("\n\n"); + } + + FileSpec save_location; + if (output_file.empty()) { + const std::string file_name = + "lldb_" + llvm::StringRef(name).lower() + "_extension.py"; + save_location = HostInfo::GetGlobalTempDir(); + FileSystem::Instance().Resolve(save_location); + save_location.AppendPathComponent(file_name); + } else { + save_location = FileSpec(output_file); + FileSystem::Instance().Resolve(save_location); + } + + File::OpenOptions flags = File::eOpenOptionWriteOnly | + File::eOpenOptionCanCreate | + File::eOpenOptionTruncate; + + auto opened_file = FileSystem::Instance().Open(save_location, flags); + + if (!opened_file) + return opened_file.takeError(); + + FileUP file = std::move(opened_file.get()); + + size_t byte_size = generated_file_stream.GetSize(); + + Status error = file->Write(generated_file_stream.GetData(), byte_size); + + if (error.Fail() || byte_size != generated_file_stream.GetSize()) + return llvm::createStringError("Unable to write to destination file. Bytes " + "written do not match generated file size."); + return save_location; +} + void ScriptInterpreterPython::SharedLibraryDirectoryHelper( FileSpec &this_file) { // When we're loaded from python, this_file will point to the file inside the diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h index bade88614f3d9..c9836be0adb39 100644 --- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h +++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h @@ -41,7 +41,15 @@ class ScriptInterpreterPython : public ScriptInterpreter, : ScriptInterpreter(debugger, lldb::eScriptLanguagePython), IOHandlerDelegateMultiline("DONE") {} + llvm::Expected<std::string> + ExtensionToImportPath(lldb::ScriptedExtension extension) override; StructuredData::DictionarySP GetInterpreterInfo() override; + llvm::Expected<FileSpec> GenerateExtensionTemplate( + const std::string name, + std::vector<std::pair<llvm::StringRef, + llvm::SmallVector<llvm::StringRef>>> &extensions, + bool generate_non_abstract_methods, std::string output_file) override; + static void Initialize(); static void Terminate(); static llvm::StringRef GetPluginNameStatic() { return "script-python"; } @@ -50,6 +58,13 @@ class ScriptInterpreterPython : public ScriptInterpreter, static void SharedLibraryDirectoryHelper(FileSpec &this_file); protected: + llvm::Error + ParseExtensionSchema(Stream &s, llvm::StringRef output_script_prefix, + llvm::SmallVector<llvm::StringRef> &extension_path, + bool generate_non_abstract_methods); + llvm::Expected<StructuredData::ObjectSP> + GetExtensionSchema(llvm::SmallVector<llvm::StringRef> &extension_path); + static void ComputePythonDirForApple(llvm::SmallVectorImpl<char> &path); static void ComputePythonDir(llvm::SmallVectorImpl<char> &path); }; diff --git a/lldb/test/API/functionalities/scripting_extension_generate/Makefile b/lldb/test/API/functionalities/scripting_extension_generate/Makefile new file mode 100644 index 0000000000000..10495940055b6 --- /dev/null +++ b/lldb/test/API/functionalities/scripting_extension_generate/Makefile @@ -0,0 +1,3 @@ +C_SOURCES := main.c + +include Makefile.rules diff --git a/lldb/test/API/functionalities/scripting_extension_generate/TestScriptingExtensionGenerate.py b/lldb/test/API/functionalities/scripting_extension_generate/TestScriptingExtensionGenerate.py new file mode 100644 index 0000000000000..41bc43286329f --- /dev/null +++ b/lldb/test/API/functionalities/scripting_extension_generate/TestScriptingExtensionGenerate.py @@ -0,0 +1,164 @@ +""" +Meta-test: generate every kind of scripting extension template on the fly via +`scripting extension generate`, then actually register and drive each one +through a live lldb debug session, to catch regressions in the generator +itself (e.g. broken imports, wrong base-class name, missing __init__ +chaining) rather than just checking that a Python file was produced. + +The stub methods lldb calls into may legitimately report failure (e.g. an +`is_alive` stub that returns None can make `process launch` report an +error), so success/failure of the driving command is not asserted. What is +asserted is that lldb never surfaces an unhandled Python traceback while +running the generated code. +""" + +import os + +import lldb +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil + + +class TestScriptingExtensionGenerate(TestBase): + NO_DEBUG_INFO_TESTCASE = True + + def generate_extension(self, kind, generate_all=False): + """Run `scripting extension generate` for `kind`, returning the path + to the generated file. Fails the test if generation errors out.""" + output_path = self.getBuildArtifact(f"generated_{kind}.py") + if os.path.exists(output_path): + os.remove(output_path) + + all_flag = " -a true" if generate_all else "" + self.runCmd(f"scripting extension generate{all_flag} -o {output_path} {kind}") + + self.assertTrue( + os.path.exists(output_path), + f"scripting extension generate did not create {output_path}", + ) + return output_path + + def run_and_assert_no_traceback(self, cmd): + """Run `cmd` and fail the test if lldb surfaced an unhandled Python + traceback while executing it. The command itself may still report + failure (e.g. a stub method returning None) without that being a + generator bug.""" + self.runCmd(cmd, check=False) + output = self.res.GetOutput() + self.res.GetError() + self.assertNotIn( + "Traceback (most recent call last)", + output, + f"command {cmd!r} raised a Python exception:\n{output}", + ) + + def test_generate_operating_system(self): + """OperatingSystem: generate, import, and activate via the + target.process.python-os-plugin-path setting.""" + self.build() + path = self.generate_extension("OperatingSystem") + self.run_and_assert_no_traceback(f"command script import {path}") + + lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c")) + + self.run_and_assert_no_traceback( + f"settings set target.process.python-os-plugin-path {path}" + ) + self.run_and_assert_no_traceback("thread list") + self.runCmd("settings clear target.process.python-os-plugin-path", check=False) + + def test_generate_scripted_platform(self): + """ScriptedPlatform: generate, import, and select via `platform + select`.""" + path = self.generate_extension("ScriptedPlatform") + self.run_and_assert_no_traceback(f"command script import {path}") + + self.run_and_assert_no_traceback( + "platform select scripted-platform -C " + "generated_ScriptedPlatform.ScriptedPlatform" + ) + + def test_generate_scripted_process(self): + """ScriptedProcess: generate, import, and launch via `process launch + -C`.""" + self.build() + path = self.generate_extension("ScriptedProcess") + self.run_and_assert_no_traceback(f"command script import {path}") + + target = self.dbg.CreateTarget(self.getBuildArtifact("a.out")) + self.assertTrue(target, VALID_TARGET) + + self.run_and_assert_no_traceback( + "process launch -C generated_ScriptedProcess.ScriptedProcess" + ) + + def test_generate_scripted_thread_plan(self): + """ScriptedThreadPlan: generate, import, and drive via `thread + step-scripted -C`.""" + self.build() + path = self.generate_extension("ScriptedThreadPlan") + self.run_and_assert_no_traceback(f"command script import {path}") + + lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c")) + + self.run_and_assert_no_traceback( + "thread step-scripted -C " "generated_ScriptedThreadPlan.ScriptedThreadPlan" + ) + + def test_generate_scripted_breakpoint_resolver(self): + """ScriptedBreakpointResolver: generate, import, and set via + `breakpoint set -P`.""" + self.build() + path = self.generate_extension("ScriptedBreakpointResolver") + self.run_and_assert_no_traceback(f"command script import {path}") + + target = self.dbg.CreateTarget(self.getBuildArtifact("a.out")) + self.assertTrue(target, VALID_TARGET) + + self.run_and_assert_no_traceback( + "breakpoint set -P " + "generated_ScriptedBreakpointResolver.ScriptedBreakpointResolver" + ) + + def test_generate_scripted_hook_for_stop_hook(self): + """ScriptedHook drives `target stop-hook add -P` (the stop-hook is a + subset of the general scripted hook interface).""" + self.build() + path = self.generate_extension("ScriptedHook") + self.run_and_assert_no_traceback(f"command script import {path}") + + target = self.dbg.CreateTarget(self.getBuildArtifact("a.out")) + self.assertTrue(target, VALID_TARGET) + + self.run_and_assert_no_traceback( + "target stop-hook add -P generated_ScriptedHook.ScriptedHook" + ) + + def test_generate_scripted_frame_provider(self): + """ScriptedFrameProvider: generate, import, and register via `target + frame-provider register -C`. Uses -a to generate all methods since + `get_description` is a required static abstract method.""" + self.build() + path = self.generate_extension("ScriptedFrameProvider", generate_all=True) + self.run_and_assert_no_traceback(f"command script import {path}") + + lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c")) + + self.run_and_assert_no_traceback( + "target frame-provider register -C " + "generated_ScriptedFrameProvider.ScriptedFrameProvider" + ) + + def test_generate_scripted_hook(self): + """ScriptedHook: generate, import, and register via `target hook add + -P`.""" + self.build() + path = self.generate_extension("ScriptedHook") + self.run_and_assert_no_traceback(f"command script import {path}") + + target = self.dbg.CreateTarget(self.getBuildArtifact("a.out")) + self.assertTrue(target, VALID_TARGET) + + self.run_and_assert_no_traceback( + "target hook add -P generated_ScriptedHook.ScriptedHook" + ) diff --git a/lldb/test/API/functionalities/scripting_extension_generate/main.c b/lldb/test/API/functionalities/scripting_extension_generate/main.c new file mode 100644 index 0000000000000..e194b3bf9ec3f --- /dev/null +++ b/lldb/test/API/functionalities/scripting_extension_generate/main.c @@ -0,0 +1,12 @@ +#include <stdio.h> + +int foo(int x) { + int y = x + 1; // break here + return y; +} + +int main() { + int result = foo(41); + printf("%d\n", result); + return 0; +} _______________________________________________ lldb-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
