https://github.com/ldionne updated https://github.com/llvm/llvm-project/pull/90803
>From ecab1e5689f9f7ea6f87d9cc8c51910b733f6859 Mon Sep 17 00:00:00 2001 From: Louis Dionne <ldionn...@gmail.com> Date: Wed, 1 May 2024 18:10:07 -0600 Subject: [PATCH] [libc++][WIP] Move the libc++ test format to Lit This allows the generally-useful parts of the test format to be shipped alongside Lit, which would make it usable for third-party developers as well. This came up at C++Now in the context of the Beman project where we'd like to set up a non-LLVM project that can use the same testing framework as libc++. With the test format moved to Lit, the format would be available after installing Lit with pip, which is really convenient. --- libcxx/test/configs/cmake-bridge.cfg.in | 2 +- libcxx/utils/libcxx/test/dsl.py | 6 +- libcxx/utils/libcxx/test/format.py | 419 +++--------------- libcxxabi/test/configs/cmake-bridge.cfg.in | 2 +- libunwind/test/configs/cmake-bridge.cfg.in | 2 +- llvm/utils/lit/lit/formats/__init__.py | 1 + .../lit/lit/formats/standardlibrarytest.py | 355 +++++++++++++++ 7 files changed, 421 insertions(+), 366 deletions(-) create mode 100644 llvm/utils/lit/lit/formats/standardlibrarytest.py diff --git a/libcxx/test/configs/cmake-bridge.cfg.in b/libcxx/test/configs/cmake-bridge.cfg.in index 84b3270a8940ac..b220e5ebcafb17 100644 --- a/libcxx/test/configs/cmake-bridge.cfg.in +++ b/libcxx/test/configs/cmake-bridge.cfg.in @@ -18,7 +18,7 @@ import libcxx.test.format # Basic configuration of the test suite config.name = os.path.basename('@LIBCXX_TEST_CONFIG@') config.test_source_root = os.path.join('@LIBCXX_SOURCE_DIR@', 'test') -config.test_format = libcxx.test.format.CxxStandardLibraryTest() +config.test_format = libcxx.test.format.LibcxxTest() config.recursiveExpansionLimit = 10 config.test_exec_root = os.path.join('@CMAKE_BINARY_DIR@', 'test') diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py index 387862ae6f496d..6177dc9dccd327 100644 --- a/libcxx/utils/libcxx/test/dsl.py +++ b/libcxx/utils/libcxx/test/dsl.py @@ -99,7 +99,7 @@ def _executeWithFakeConfig(test, commands): order="smart", params={}, ) - return libcxx.test.format._executeScriptInternal(test, litConfig, commands) + return lit.formats.standardlibrarytest._executeScriptInternal(test, litConfig, commands) def _makeConfigTest(config): @@ -121,12 +121,12 @@ def _makeConfigTest(config): class TestWrapper(lit.Test.Test): def __enter__(self): - testDir, _ = libcxx.test.format._getTempPaths(self) + testDir, _ = lit.formats.standardlibrarytest._getTempPaths(self) os.makedirs(testDir) return self def __exit__(self, *args): - testDir, _ = libcxx.test.format._getTempPaths(self) + testDir, _ = lit.formats.standardlibrarytest._getTempPaths(self) shutil.rmtree(testDir) os.remove(tmp.name) diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py index 7e5281c0b74064..a670f57f67965d 100644 --- a/libcxx/utils/libcxx/test/format.py +++ b/libcxx/utils/libcxx/test/format.py @@ -7,50 +7,8 @@ # ===----------------------------------------------------------------------===## import lit -import libcxx.test.config as config import lit.formats import os -import re - - -def _getTempPaths(test): - """ - Return the values to use for the %T and %t substitutions, respectively. - - The difference between this and Lit's default behavior is that we guarantee - that %T is a path unique to the test being run. - """ - tmpDir, _ = lit.TestRunner.getTempPaths(test) - _, testName = os.path.split(test.getExecPath()) - tmpDir = os.path.join(tmpDir, testName + ".dir") - tmpBase = os.path.join(tmpDir, "t") - return tmpDir, tmpBase - - -def _checkBaseSubstitutions(substitutions): - substitutions = [s for (s, _) in substitutions] - for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]: - assert s in substitutions, "Required substitution {} was not provided".format(s) - -def _executeScriptInternal(test, litConfig, commands): - """ - Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands) - - TODO: This really should be easier to access from Lit itself - """ - parsedCommands = parseScript(test, preamble=commands) - - _, tmpBase = _getTempPaths(test) - execDir = os.path.dirname(test.getExecPath()) - try: - res = lit.TestRunner.executeScriptInternal( - test, litConfig, tmpBase, parsedCommands, execDir, debug=False - ) - except lit.TestRunner.ScriptFatal as e: - res = ("", str(e), 127, None) - (out, err, exitCode, timeoutInfo) = res - - return (out, err, exitCode, timeoutInfo, parsedCommands) def _validateModuleDependencies(modules): @@ -61,334 +19,75 @@ def _validateModuleDependencies(modules): ) -def parseScript(test, preamble): - """ - Extract the script from a test, with substitutions applied. - - Returns a list of commands ready to be executed. - - - test - The lit.Test to parse. - - - preamble - A list of commands to perform before any command in the test. - These commands can contain unexpanded substitutions, but they - must not be of the form 'RUN:' -- they must be proper commands - once substituted. - """ - # Get the default substitutions - tmpDir, tmpBase = _getTempPaths(test) +def _buildModule(test, litConfig, commands): + tmpDir, tmpBase = lit.formats.standardlibrarytest._getTempPaths(test) + execDir = os.path.dirname(test.getExecPath()) substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase) - # Check base substitutions and add the %{build}, %{verify} and %{run} convenience substitutions - # - # Note: We use -Wno-error with %{verify} to make sure that we don't treat all diagnostics as - # errors, which doesn't make sense for clang-verify tests because we may want to check - # for specific warning diagnostics. - _checkBaseSubstitutions(substitutions) - substitutions.append( - ("%{build}", "%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe") - ) - substitutions.append( - ( - "%{verify}", - "%{cxx} %s %{flags} %{compile_flags} -fsyntax-only -Wno-error -Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0", - ) - ) - substitutions.append(("%{run}", "%{exec} %t.exe")) - - # Parse the test file, including custom directives - additionalCompileFlags = [] - fileDependencies = [] - modules = [] # The enabled modules - moduleCompileFlags = [] # The compilation flags to use modules - parsers = [ - lit.TestRunner.IntegratedTestKeywordParser( - "FILE_DEPENDENCIES:", - lit.TestRunner.ParserKind.LIST, - initial_value=fileDependencies, - ), - lit.TestRunner.IntegratedTestKeywordParser( - "ADDITIONAL_COMPILE_FLAGS:", - lit.TestRunner.ParserKind.SPACE_LIST, - initial_value=additionalCompileFlags, - ), - lit.TestRunner.IntegratedTestKeywordParser( - "MODULE_DEPENDENCIES:", - lit.TestRunner.ParserKind.SPACE_LIST, - initial_value=modules, - ), - ] - - # Add conditional parsers for ADDITIONAL_COMPILE_FLAGS. This should be replaced by first - # class support for conditional keywords in Lit, which would allow evaluating arbitrary - # Lit boolean expressions instead. - for feature in test.config.available_features: - parser = lit.TestRunner.IntegratedTestKeywordParser( - "ADDITIONAL_COMPILE_FLAGS({}):".format(feature), - lit.TestRunner.ParserKind.SPACE_LIST, - initial_value=additionalCompileFlags, - ) - parsers.append(parser) - - scriptInTest = lit.TestRunner.parseIntegratedTestScript( - test, additional_parsers=parsers, require_script=not preamble - ) - if isinstance(scriptInTest, lit.Test.Result): - return scriptInTest - - script = [] - - # For each file dependency in FILE_DEPENDENCIES, inject a command to copy - # that file to the execution directory. Execute the copy from %S to allow - # relative paths from the test directory. - for dep in fileDependencies: - script += ["%dbg(SETUP) cd %S && cp {} %T".format(dep)] - script += preamble - script += scriptInTest - - # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS. - # Modules need to be built with the same compilation flags as the - # test. So add these flags before adding the modules. - substitutions = config._appendToSubstitution( - substitutions, "%{compile_flags}", " ".join(additionalCompileFlags) + substituted = lit.TestRunner.applySubstitutions( + commands, substitutions, recursion_limit=test.config.recursiveExpansionLimit ) - - if modules: - _validateModuleDependencies(modules) - - # The moduleCompileFlags are added to the %{compile_flags}, but - # the modules need to be built without these flags. So expand the - # %{compile_flags} eagerly and hardcode them in the build script. - compileFlags = config._getSubstitution("%{compile_flags}", test.config) - - # Building the modules needs to happen before the other script - # commands are executed. Therefore the commands are added to the - # front of the list. - if "std.compat" in modules: - script.insert( - 0, - "%dbg(MODULE std.compat) %{cxx} %{flags} " - f"{compileFlags} " - "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal " - "-fmodule-file=std=%T/std.pcm " # The std.compat module imports std. - "--precompile -o %T/std.compat.pcm -c %{module-dir}/std.compat.cppm", - ) - moduleCompileFlags.extend( - ["-fmodule-file=std.compat=%T/std.compat.pcm", "%T/std.compat.pcm"] - ) - - # Make sure the std module is built before std.compat. Libc++'s - # std.compat module depends on the std module. It is not - # known whether the compiler expects the modules in the order of - # their dependencies. However it's trivial to provide them in - # that order. - script.insert( - 0, - "%dbg(MODULE std) %{cxx} %{flags} " - f"{compileFlags} " - "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal " - "--precompile -o %T/std.pcm -c %{module-dir}/std.cppm", + (out, err, exitCode, _) = lit.TestRunner.executeScriptInternal(test, litConfig, tmpBase, substituted, execDir) + if exitCode != 0: + return lit.Test.Result( + lit.Test.UNRESOLVED, "Failed to build module std for '{}':\n{}\n{}".format(test.getFilePath(), out, err) ) - moduleCompileFlags.extend(["-fmodule-file=std=%T/std.pcm", "%T/std.pcm"]) - # Add compile flags required for the modules. - substitutions = config._appendToSubstitution( - substitutions, "%{compile_flags}", " ".join(moduleCompileFlags) - ) - - # Perform substitutions in the script itself. - script = lit.TestRunner.applySubstitutions( - script, substitutions, recursion_limit=test.config.recursiveExpansionLimit - ) - - return script - - -class CxxStandardLibraryTest(lit.formats.FileBasedTest): - """ - Lit test format for the C++ Standard Library conformance test suite. - - Lit tests are contained in files that follow a certain pattern, which determines the semantics of the test. - Under the hood, we basically generate a builtin Lit shell test that follows the ShTest format, and perform - the appropriate operations (compile/link/run). See - https://libcxx.llvm.org/TestingLibcxx.html#test-names - for a complete description of those semantics. - - Substitution requirements - =============================== - The test format operates by assuming that each test's configuration provides - the following substitutions, which it will reuse in the shell scripts it - constructs: - %{cxx} - A command that can be used to invoke the compiler - %{compile_flags} - Flags to use when compiling a test case - %{link_flags} - Flags to use when linking a test case - %{flags} - Flags to use either when compiling or linking a test case - %{exec} - A command to prefix the execution of executables - - Note that when building an executable (as opposed to only compiling a source - file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used - in the same command line. In other words, the test format doesn't perform - separate compilation and linking steps in this case. - - Additional provided substitutions and features - ============================================== - The test format will define the following substitutions for use inside tests: - - %{build} - Expands to a command-line that builds the current source - file with the %{flags}, %{compile_flags} and %{link_flags} - substitutions, and that produces an executable named %t.exe. - - %{verify} - Expands to a command-line that builds the current source - file with the %{flags} and %{compile_flags} substitutions - and enables clang-verify. This can be used to write .sh.cpp - tests that use clang-verify. Note that this substitution can - only be used when the 'verify-support' feature is available. - - %{run} - Equivalent to `%{exec} %t.exe`. This is intended to be used - in conjunction with the %{build} substitution. - """ - - def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig): - SUPPORTED_SUFFIXES = [ - "[.]pass[.]cpp$", - "[.]pass[.]mm$", - "[.]compile[.]pass[.]cpp$", - "[.]compile[.]pass[.]mm$", - "[.]compile[.]fail[.]cpp$", - "[.]link[.]pass[.]cpp$", - "[.]link[.]pass[.]mm$", - "[.]link[.]fail[.]cpp$", - "[.]sh[.][^.]+$", - "[.]gen[.][^.]+$", - "[.]verify[.]cpp$", - ] - - sourcePath = testSuite.getSourcePath(pathInSuite) - filename = os.path.basename(sourcePath) - - # Ignore dot files, excluded tests and tests with an unsupported suffix - hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES]) - if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename): - return - - # If this is a generated test, run the generation step and add - # as many Lit tests as necessary. - if re.search('[.]gen[.][^.]+$', filename): - for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig): - yield test - else: - yield lit.Test.Test(testSuite, pathInSuite, localConfig) +class LibcxxTest(lit.formats.StandardLibraryTest): def execute(self, test, litConfig): - supportsVerify = "verify-support" in test.config.available_features - filename = test.path_in_suite[-1] - - if re.search("[.]sh[.][^.]+$", filename): - steps = [] # The steps are already in the script - return self._executeShTest(test, litConfig, steps) - elif filename.endswith(".compile.pass.cpp") or filename.endswith( - ".compile.pass.mm" - ): - steps = [ - "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" - ] - return self._executeShTest(test, litConfig, steps) - elif filename.endswith(".compile.fail.cpp"): - steps = [ - "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" - ] - return self._executeShTest(test, litConfig, steps) - elif filename.endswith(".link.pass.cpp") or filename.endswith(".link.pass.mm"): - steps = [ - "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe" - ] - return self._executeShTest(test, litConfig, steps) - elif filename.endswith(".link.fail.cpp"): - steps = [ - "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o", - "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe", - ] - return self._executeShTest(test, litConfig, steps) - elif filename.endswith(".verify.cpp"): - if not supportsVerify: - return lit.Test.Result( - lit.Test.UNSUPPORTED, - "Test {} requires support for Clang-verify, which isn't supported by the compiler".format( - test.getFullName() - ), - ) - steps = ["%dbg(COMPILED WITH) %{verify}"] - return self._executeShTest(test, litConfig, steps) - # Make sure to check these ones last, since they will match other - # suffixes above too. - elif filename.endswith(".pass.cpp") or filename.endswith(".pass.mm"): - steps = [ - "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe", - "%dbg(EXECUTED AS) %{exec} %t.exe", - ] - return self._executeShTest(test, litConfig, steps) - else: - return lit.Test.Result( - lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename) - ) - - def _executeShTest(self, test, litConfig, steps): if test.config.unsupported: return lit.Test.Result(lit.Test.UNSUPPORTED, "Test is unsupported") - script = parseScript(test, steps) - if isinstance(script, lit.Test.Result): - return script - - if litConfig.noExecute: - return lit.Test.Result( - lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS + # Parse any MODULE_DEPENDENCIES in the test file. + modules = [] + parsers = [ + lit.TestRunner.IntegratedTestKeywordParser( + "MODULE_DEPENDENCIES:", + lit.TestRunner.ParserKind.SPACE_LIST, + initial_value=modules, ) - else: - _, tmpBase = _getTempPaths(test) - useExternalSh = False - return lit.TestRunner._runShTest( - test, litConfig, useExternalSh, script, tmpBase + ] + lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers, require_script=False) + + # Build the modules if needed and tweak the compiler flags of the rest of the test so + # it knows about the just-built modules. + moduleCompileFlags = [] + if modules: + _validateModuleDependencies(modules) + + # Make sure the std module is built before std.compat. Libc++'s + # std.compat module depends on the std module. It is not + # known whether the compiler expects the modules in the order of + # their dependencies. However it's trivial to provide them in + # that order. + commands = [ + "mkdir -p %T", + "%dbg(MODULE std) %{cxx} %{flags} %{compile_flags} " + "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal " + "--precompile -o %T/std.pcm -c %{module-dir}/std.cppm", + ] + res = _buildModule(test, litConfig, commands) + if isinstance(res, lit.Test.Result): + return res + moduleCompileFlags.extend(["-fmodule-file=std=%T/std.pcm", "%T/std.pcm"]) + + if "std.compat" in modules: + commands = [ + "mkdir -p %T", + "%dbg(MODULE std.compat) %{cxx} %{flags} %{compile_flags} " + "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal " + "-fmodule-file=std=%T/std.pcm " # The std.compat module imports std. + "--precompile -o %T/std.compat.pcm -c %{module-dir}/std.compat.cppm", + ] + res = _buildModule(test, litConfig, commands) + if isinstance(res, lit.Test.Result): + return res + moduleCompileFlags.extend(["-fmodule-file=std.compat=%T/std.compat.pcm", "%T/std.compat.pcm"]) + + # Add compile flags required for the test to use the just-built modules + test.config.substitutions = lit.formats.standardlibrarytest._appendToSubstitution( + test.config.substitutions, "%{compile_flags}", " ".join(moduleCompileFlags) ) - def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig): - generator = lit.Test.Test(testSuite, pathInSuite, localConfig) - - # Make sure we have a directory to execute the generator test in - generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite)) - os.makedirs(generatorExecDir, exist_ok=True) - - # Run the generator test - steps = [] # Steps must already be in the script - (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps) - if exitCode != 0: - raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}") - - # Split the generated output into multiple files and generate one test for each file - for subfile, content in self._splitFile(out): - generatedFile = testSuite.getExecPath(pathInSuite + (subfile,)) - os.makedirs(os.path.dirname(generatedFile), exist_ok=True) - with open(generatedFile, 'w') as f: - f.write(content) - yield lit.Test.Test(testSuite, (generatedFile,), localConfig) - - def _splitFile(self, input): - DELIM = r'^(//|#)---(.+)' - lines = input.splitlines() - currentFile = None - thisFileContent = [] - for line in lines: - match = re.match(DELIM, line) - if match: - if currentFile is not None: - yield (currentFile, '\n'.join(thisFileContent)) - currentFile = match.group(2).strip() - thisFileContent = [] - assert currentFile is not None, f"Some input to split-file doesn't belong to any file, input was:\n{input}" - thisFileContent.append(line) - if currentFile is not None: - yield (currentFile, '\n'.join(thisFileContent)) + return super().execute(test, litConfig) diff --git a/libcxxabi/test/configs/cmake-bridge.cfg.in b/libcxxabi/test/configs/cmake-bridge.cfg.in index 1d0f51d37437bd..634fd26a54ea6d 100644 --- a/libcxxabi/test/configs/cmake-bridge.cfg.in +++ b/libcxxabi/test/configs/cmake-bridge.cfg.in @@ -19,7 +19,7 @@ import libcxx.test.format # Basic configuration of the test suite config.name = os.path.basename('@LIBCXXABI_TEST_CONFIG@') config.test_source_root = os.path.join('@LIBCXXABI_SOURCE_DIR@', 'test') -config.test_format = libcxx.test.format.CxxStandardLibraryTest() +config.test_format = libcxx.test.format.LibcxxTest() config.recursiveExpansionLimit = 10 config.test_exec_root = os.path.join('@CMAKE_BINARY_DIR@', 'test') diff --git a/libunwind/test/configs/cmake-bridge.cfg.in b/libunwind/test/configs/cmake-bridge.cfg.in index c5f34c87abb92a..f74bc04e74c72b 100644 --- a/libunwind/test/configs/cmake-bridge.cfg.in +++ b/libunwind/test/configs/cmake-bridge.cfg.in @@ -18,7 +18,7 @@ import libcxx.test.format # Basic configuration of the test suite config.name = os.path.basename('@LIBUNWIND_TEST_CONFIG@') config.test_source_root = os.path.join('@LIBUNWIND_SOURCE_DIR@', 'test') -config.test_format = libcxx.test.format.CxxStandardLibraryTest() +config.test_format = libcxx.test.format.LibcxxTest() config.recursiveExpansionLimit = 10 config.test_exec_root = os.path.join('@CMAKE_BINARY_DIR@', 'test') diff --git a/llvm/utils/lit/lit/formats/__init__.py b/llvm/utils/lit/lit/formats/__init__.py index 6f3dd38eab9990..ccce2c9bb40d73 100644 --- a/llvm/utils/lit/lit/formats/__init__.py +++ b/llvm/utils/lit/lit/formats/__init__.py @@ -6,3 +6,4 @@ from lit.formats.googletest import GoogleTest # noqa: F401 from lit.formats.shtest import ShTest # noqa: F401 +from lit.formats.standardlibrarytest import StandardLibraryTest # noqa: F401 diff --git a/llvm/utils/lit/lit/formats/standardlibrarytest.py b/llvm/utils/lit/lit/formats/standardlibrarytest.py new file mode 100644 index 00000000000000..cb54daa72fdfc5 --- /dev/null +++ b/llvm/utils/lit/lit/formats/standardlibrarytest.py @@ -0,0 +1,355 @@ +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +import lit +import os +import re +from .base import FileBasedTest + + +def _getSubstitution(substitution, config): + """ + Helper function to get a specific substitution from a config object. + """ + for (orig, replacement) in config.substitutions: + if orig == substitution: + return replacement + raise ValueError("Substitution {} is not in the config.".format(substitution)) + + +def _appendToSubstitution(substitutions, key, value): + """ + Helper function to append a value to a specific substitution. + """ + return [(k, v + " " + value) if k == key else (k, v) for (k, v) in substitutions] + + +def _getTempPaths(test): + """ + Return the values to use for the %T and %t substitutions, respectively. + + The difference between this and Lit's default behavior is that we guarantee + that %T is a path unique to the test being run. + """ + tmpDir, _ = lit.TestRunner.getTempPaths(test) + _, testName = os.path.split(test.getExecPath()) + tmpDir = os.path.join(tmpDir, testName + ".dir") + tmpBase = os.path.join(tmpDir, "t") + return tmpDir, tmpBase + + +def _checkBaseSubstitutions(substitutions): + """ + Helper function to ensure that all the base substitutions required by this + format are provided. + """ + substitutions = [s for (s, _) in substitutions] + for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]: + assert s in substitutions, "Required substitution {} was not provided".format(s) + + +def _convenienceSubstitutions(): + """ + Return the convenience substitutions provided by this test format. + """ + return [ + ("%{build}", "%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe"), + ( + "%{verify}", + # Note: We use -Wno-error with %{verify} to make sure that we don't treat all diagnostics as + # errors, which doesn't make sense for clang-verify tests because we may want to check + # for specific warning diagnostics. + "%{cxx} %s %{flags} %{compile_flags} -fsyntax-only -Wno-error -Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0", + ), + ("%{run}", "%{exec} %t.exe") + ] + + +def _executeScriptInternal(test, litConfig, commands): + """ + Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands) + + TODO: This really should be easier to access from Lit itself + """ + parsedCommands = _parseScript(test, preamble=commands) + + _, tmpBase = _getTempPaths(test) + execDir = os.path.dirname(test.getExecPath()) + try: + res = lit.TestRunner.executeScriptInternal( + test, litConfig, tmpBase, parsedCommands, execDir, debug=False + ) + except lit.TestRunner.ScriptFatal as e: + res = ("", str(e), 127, None) + (out, err, exitCode, timeoutInfo) = res + + return (out, err, exitCode, timeoutInfo, parsedCommands) + + +def _parseScript(test, preamble): + """ + Extract the script from a test, with substitutions applied. + + Returns a list of commands ready to be executed. + + - test + The lit.Test to parse. + + - preamble + A list of commands to perform before any command in the test. + These commands can contain unexpanded substitutions, but they + must not be of the form 'RUN:' -- they must be proper commands + once substituted. + """ + # Get the default substitutions, ensure we have all the required substitutions + # and add a few convenience ones provided by this test format. + tmpDir, tmpBase = _getTempPaths(test) + substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase) + _checkBaseSubstitutions(substitutions) + substitutions += _convenienceSubstitutions() + + # Parse the test file, including custom directives + additionalCompileFlags = [] + fileDependencies = [] + parsers = [ + lit.TestRunner.IntegratedTestKeywordParser( + "FILE_DEPENDENCIES:", + lit.TestRunner.ParserKind.LIST, + initial_value=fileDependencies, + ), + lit.TestRunner.IntegratedTestKeywordParser( + "ADDITIONAL_COMPILE_FLAGS:", + lit.TestRunner.ParserKind.SPACE_LIST, + initial_value=additionalCompileFlags, + ) + ] + + # Add conditional parsers for ADDITIONAL_COMPILE_FLAGS. This should be replaced by first + # class support for conditional keywords in Lit, which would allow evaluating arbitrary + # Lit boolean expressions instead. + for feature in test.config.available_features: + parser = lit.TestRunner.IntegratedTestKeywordParser( + "ADDITIONAL_COMPILE_FLAGS({}):".format(feature), + lit.TestRunner.ParserKind.SPACE_LIST, + initial_value=additionalCompileFlags, + ) + parsers.append(parser) + + scriptInTest = lit.TestRunner.parseIntegratedTestScript( + test, additional_parsers=parsers, require_script=not preamble + ) + if isinstance(scriptInTest, lit.Test.Result): + return scriptInTest + + script = [] + + # For each file dependency in FILE_DEPENDENCIES, inject a command to copy + # that file to the execution directory. Execute the copy from %S to allow + # relative paths from the test directory. + for dep in fileDependencies: + script += ["%dbg(SETUP) cd %S && cp {} %T".format(dep)] + script += preamble + script += scriptInTest + + # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS. + substitutions = _appendToSubstitution( + substitutions, "%{compile_flags}", " ".join(additionalCompileFlags) + ) + + # Perform substitutions in the script itself. + script = lit.TestRunner.applySubstitutions( + script, substitutions, recursion_limit=test.config.recursiveExpansionLimit + ) + + return script + + +class StandardLibraryTest(FileBasedTest): + """ + Lit test format designed for testing Standard Libraries. + + Lit tests are contained in files that follow a certain pattern, which determines the semantics of the test. + Under the hood, we basically generate a builtin Lit shell test that follows the ShTest format, and perform + the appropriate operations (compile/link/run). See https://libcxx.llvm.org/TestingLibcxx.html#test-names + for a complete description of those semantics. + + Substitution requirements + ========================= + The test format operates by assuming that each test's configuration provides the following substitutions, + which it will reuse in the shell scripts it constructs: + %{cxx} - A command that can be used to invoke the compiler + %{compile_flags} - Flags to use when compiling a test case + %{link_flags} - Flags to use when linking a test case + %{flags} - Flags to use either when compiling or linking a test case + %{exec} - A command to prefix the execution of executables + + Note that when building an executable (as opposed to only compiling a source file), all three of %{flags}, + %{compile_flags} and %{link_flags} will be used in the same command line. In other words, the test format + doesn't perform separate compilation and linking steps in this case. + + Additional provided substitutions and features + ============================================== + The test format will define the following substitutions for use inside tests: + + %{build} + Expands to a command-line that builds the current source + file with the %{flags}, %{compile_flags} and %{link_flags} + substitutions, and that produces an executable named %t.exe. + + %{verify} + Expands to a command-line that builds the current source + file with the %{flags} and %{compile_flags} substitutions + and enables clang-verify. This can be used to write .sh.cpp + tests that use clang-verify. Note that this substitution can + only be used when the 'verify-support' feature is available. + + %{run} + Equivalent to `%{exec} %t.exe`. This is intended to be used + in conjunction with the %{build} substitution. + """ + + def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig): + SUPPORTED_SUFFIXES = [ + "[.]pass[.]cpp$", + "[.]pass[.]mm$", + "[.]compile[.]pass[.]cpp$", + "[.]compile[.]pass[.]mm$", + "[.]compile[.]fail[.]cpp$", + "[.]link[.]pass[.]cpp$", + "[.]link[.]pass[.]mm$", + "[.]link[.]fail[.]cpp$", + "[.]sh[.][^.]+$", + "[.]gen[.][^.]+$", + "[.]verify[.]cpp$", + ] + + sourcePath = testSuite.getSourcePath(pathInSuite) + filename = os.path.basename(sourcePath) + + # Ignore dot files, excluded tests and tests with an unsupported suffix + hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES]) + if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename): + return + + # If this is a generated test, run the generation step and add + # as many Lit tests as necessary. + if re.search('[.]gen[.][^.]+$', filename): + for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig): + yield test + else: + yield lit.Test.Test(testSuite, pathInSuite, localConfig) + + def execute(self, test, litConfig): + supportsVerify = "verify-support" in test.config.available_features + filename = test.path_in_suite[-1] + + if re.search("[.]sh[.][^.]+$", filename): + steps = [] # The steps are already in the script + return self._executeShTest(test, litConfig, steps) + elif filename.endswith(".compile.pass.cpp") or filename.endswith( + ".compile.pass.mm" + ): + steps = [ + "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" + ] + return self._executeShTest(test, litConfig, steps) + elif filename.endswith(".compile.fail.cpp"): + steps = [ + "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" + ] + return self._executeShTest(test, litConfig, steps) + elif filename.endswith(".link.pass.cpp") or filename.endswith(".link.pass.mm"): + steps = [ + "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe" + ] + return self._executeShTest(test, litConfig, steps) + elif filename.endswith(".link.fail.cpp"): + steps = [ + "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o", + "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe", + ] + return self._executeShTest(test, litConfig, steps) + elif filename.endswith(".verify.cpp"): + if not supportsVerify: + return lit.Test.Result( + lit.Test.UNSUPPORTED, + "Test {} requires support for Clang-verify, which isn't supported by the compiler".format( + test.getFullName() + ), + ) + steps = ["%dbg(COMPILED WITH) %{verify}"] + return self._executeShTest(test, litConfig, steps) + # Make sure to check these ones last, since they will match other + # suffixes above too. + elif filename.endswith(".pass.cpp") or filename.endswith(".pass.mm"): + steps = [ + "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe", + "%dbg(EXECUTED AS) %{exec} %t.exe", + ] + return self._executeShTest(test, litConfig, steps) + else: + return lit.Test.Result( + lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename) + ) + + def _executeShTest(self, test, litConfig, steps): + if test.config.unsupported: + return lit.Test.Result(lit.Test.UNSUPPORTED, "Test is unsupported") + + script = _parseScript(test, steps) + if isinstance(script, lit.Test.Result): + return script + + if litConfig.noExecute: + return lit.Test.Result( + lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS + ) + else: + _, tmpBase = _getTempPaths(test) + useExternalSh = False + return lit.TestRunner._runShTest( + test, litConfig, useExternalSh, script, tmpBase + ) + + def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig): + generator = lit.Test.Test(testSuite, pathInSuite, localConfig) + + # Make sure we have a directory to execute the generator test in + generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite)) + os.makedirs(generatorExecDir, exist_ok=True) + + # Run the generator test + steps = [] # Steps must already be in the script + (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps) + if exitCode != 0: + raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}") + + # Split the generated output into multiple files and generate one test for each file + for subfile, content in self._splitFile(out): + generatedFile = testSuite.getExecPath(pathInSuite + (subfile,)) + os.makedirs(os.path.dirname(generatedFile), exist_ok=True) + with open(generatedFile, 'w') as f: + f.write(content) + yield lit.Test.Test(testSuite, (generatedFile,), localConfig) + + def _splitFile(self, input): + DELIM = r'^(//|#)---(.+)' + lines = input.splitlines() + currentFile = None + thisFileContent = [] + for line in lines: + match = re.match(DELIM, line) + if match: + if currentFile is not None: + yield (currentFile, '\n'.join(thisFileContent)) + currentFile = match.group(2).strip() + thisFileContent = [] + assert currentFile is not None, f"Some input to split-file doesn't belong to any file, input was:\n{input}" + thisFileContent.append(line) + if currentFile is not None: + yield (currentFile, '\n'.join(thisFileContent)) _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits