This revision was landed with ongoing or failed builds.
This revision was automatically updated to reflect the committed changes.
Closed by commit rG05d613ea931b: [lit][clang] Avoid realpath on Windows due to
MAX_PATH limitations (authored by compnerd).
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D154130/new/
https://reviews.llvm.org/D154130
Files:
clang/include/clang/Basic/FileManager.h
clang/lib/Basic/FileManager.cpp
clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
clang/test/Lexer/case-insensitive-include-win.c
llvm/cmake/modules/AddLLVM.cmake
llvm/docs/CommandGuide/lit.rst
llvm/docs/TestingGuide.rst
llvm/utils/lit/lit/TestRunner.py
llvm/utils/lit/lit/builtin_commands/diff.py
llvm/utils/lit/lit/discovery.py
llvm/utils/lit/lit/util.py
llvm/utils/lit/tests/Inputs/config-map-discovery/driver.py
llvm/utils/lit/tests/Inputs/config-map-discovery/lit.alt.cfg
llvm/utils/lit/tests/Inputs/use-llvm-tool-required/lit.cfg
llvm/utils/lit/tests/Inputs/use-llvm-tool/lit.cfg
llvm/utils/llvm-lit/llvm-lit.in
Index: llvm/utils/llvm-lit/llvm-lit.in
===================================================================
--- llvm/utils/llvm-lit/llvm-lit.in
+++ llvm/utils/llvm-lit/llvm-lit.in
@@ -8,7 +8,7 @@
def map_config(source_dir, site_config):
global config_map
- source_dir = os.path.realpath(source_dir)
+ source_dir = os.path.abspath(source_dir)
source_dir = os.path.normcase(source_dir)
site_config = os.path.normpath(site_config)
config_map[source_dir] = site_config
Index: llvm/utils/lit/tests/Inputs/use-llvm-tool/lit.cfg
===================================================================
--- llvm/utils/lit/tests/Inputs/use-llvm-tool/lit.cfg
+++ llvm/utils/lit/tests/Inputs/use-llvm-tool/lit.cfg
@@ -1,4 +1,5 @@
import lit.formats
+import lit.util
config.name = "use-llvm-tool"
config.suffixes = [".txt"]
@@ -7,7 +8,7 @@
config.test_exec_root = None
import os.path
-this_dir = os.path.realpath(os.path.dirname(__file__))
+this_dir = os.path.dirname(lit.util.abs_path_preserve_drive(__file__))
config.llvm_tools_dir = os.path.join(this_dir, "build")
import lit.llvm
Index: llvm/utils/lit/tests/Inputs/use-llvm-tool-required/lit.cfg
===================================================================
--- llvm/utils/lit/tests/Inputs/use-llvm-tool-required/lit.cfg
+++ llvm/utils/lit/tests/Inputs/use-llvm-tool-required/lit.cfg
@@ -1,4 +1,5 @@
import lit.formats
+import lit.util
config.name = "use-llvm-tool-required"
config.suffixes = [".txt"]
@@ -7,7 +8,7 @@
config.test_exec_root = None
import os.path
-config.llvm_tools_dir = os.path.realpath(os.path.dirname(__file__))
+config.llvm_tools_dir = os.path.dirname(lit.util.abs_path_preserve_drive(__file__))
import lit.llvm
lit.llvm.initialize(lit_config, config)
Index: llvm/utils/lit/tests/Inputs/config-map-discovery/lit.alt.cfg
===================================================================
--- llvm/utils/lit/tests/Inputs/config-map-discovery/lit.alt.cfg
+++ llvm/utils/lit/tests/Inputs/config-map-discovery/lit.alt.cfg
@@ -5,5 +5,5 @@
config.test_format = lit.formats.ShTest()
import os
-config.test_exec_root = os.path.realpath(os.path.dirname(__file__))
+config.test_exec_root = os.path.dirname(lit.util.abs_path_preserve_drive(__file__))
config.test_source_root = os.path.join(config.test_exec_root, "tests")
Index: llvm/utils/lit/tests/Inputs/config-map-discovery/driver.py
===================================================================
--- llvm/utils/lit/tests/Inputs/config-map-discovery/driver.py
+++ llvm/utils/lit/tests/Inputs/config-map-discovery/driver.py
@@ -2,9 +2,7 @@
import os
import sys
-main_config = sys.argv[1]
-main_config = os.path.realpath(main_config)
-main_config = os.path.normcase(main_config)
+main_config = lit.util.abs_path_preserve_drive(sys.argv[1])
config_map = {main_config: sys.argv[2]}
builtin_parameters = {"config_map": config_map}
Index: llvm/utils/lit/lit/util.py
===================================================================
--- llvm/utils/lit/lit/util.py
+++ llvm/utils/lit/lit/util.py
@@ -127,6 +127,23 @@
return n
+def abs_path_preserve_drive(path):
+ """Return the absolute path without resolving drive mappings on Windows.
+
+ """
+ if platform.system() == "Windows":
+ # Windows has limitations on path length (MAX_PATH) that
+ # can be worked around using substitute drives, which map
+ # a drive letter to a longer path on another drive.
+ # Since Python 3.8, os.path.realpath resolves sustitute drives,
+ # so we should not use it. In Python 3.7, os.path.realpath
+ # was implemented as os.path.abspath.
+ return os.path.normpath(os.path.abspath(path))
+ else:
+ # On UNIX, the current directory always has symbolic links resolved,
+ # so any program accepting relative paths cannot preserve symbolic
+ # links in paths and we should always use os.path.realpath.
+ return os.path.normpath(os.path.realpath(path))
def mkdir(path):
try:
Index: llvm/utils/lit/lit/discovery.py
===================================================================
--- llvm/utils/lit/lit/discovery.py
+++ llvm/utils/lit/lit/discovery.py
@@ -7,7 +7,7 @@
import sys
from lit.TestingConfig import TestingConfig
-from lit import LitConfig, Test
+from lit import LitConfig, Test, util
def chooseConfigFileFromDir(dir, config_names):
@@ -56,8 +56,8 @@
# configuration to load instead.
config_map = litConfig.params.get("config_map")
if config_map:
- cfgpath = os.path.realpath(cfgpath)
- target = config_map.get(os.path.normcase(cfgpath))
+ cfgpath = util.abs_path_preserve_drive(cfgpath)
+ target = config_map.get(cfgpath)
if target:
cfgpath = target
@@ -67,13 +67,13 @@
cfg = TestingConfig.fromdefaults(litConfig)
cfg.load_from_path(cfgpath, litConfig)
- source_root = os.path.realpath(cfg.test_source_root or path)
- exec_root = os.path.realpath(cfg.test_exec_root or path)
+ source_root = util.abs_path_preserve_drive(cfg.test_source_root or path)
+ exec_root = util.abs_path_preserve_drive(cfg.test_exec_root or path)
return Test.TestSuite(cfg.name, source_root, exec_root, cfg), ()
def search(path):
# Check for an already instantiated test suite.
- real_path = os.path.realpath(path)
+ real_path = util.abs_path_preserve_drive(path)
res = cache.get(real_path)
if res is None:
cache[real_path] = res = search1(path)
Index: llvm/utils/lit/lit/builtin_commands/diff.py
===================================================================
--- llvm/utils/lit/lit/builtin_commands/diff.py
+++ llvm/utils/lit/lit/builtin_commands/diff.py
@@ -281,7 +281,7 @@
try:
for file in args:
if file != "-" and not os.path.isabs(file):
- file = os.path.realpath(os.path.join(os.getcwd(), file))
+ file = util.abs_path_preserve_drive(file)
if flags.recursive_diff:
if file == "-":
Index: llvm/utils/lit/lit/TestRunner.py
===================================================================
--- llvm/utils/lit/lit/TestRunner.py
+++ llvm/utils/lit/lit/TestRunner.py
@@ -74,7 +74,7 @@
if os.path.isabs(newdir):
self.cwd = newdir
else:
- self.cwd = os.path.realpath(os.path.join(self.cwd, newdir))
+ self.cwd = lit.util.abs_path_preserve_drive(os.path.join(self.cwd, newdir))
class TimeoutHelper(object):
@@ -427,7 +427,7 @@
dir = to_unicode(dir) if kIsWindows else to_bytes(dir)
cwd = to_unicode(cwd) if kIsWindows else to_bytes(cwd)
if not os.path.isabs(dir):
- dir = os.path.realpath(os.path.join(cwd, dir))
+ dir = lit.util.abs_path_preserve_drive(os.path.join(cwd, dir))
if parent:
lit.util.mkdir_p(dir)
else:
@@ -473,7 +473,7 @@
path = to_unicode(path) if kIsWindows else to_bytes(path)
cwd = to_unicode(cwd) if kIsWindows else to_bytes(cwd)
if not os.path.isabs(path):
- path = os.path.realpath(os.path.join(cwd, path))
+ path = lit.util.abs_path_preserve_drive(os.path.join(cwd, path))
if force and not os.path.exists(path):
continue
try:
@@ -1243,18 +1243,10 @@
substitutions.extend(test.config.substitutions)
tmpName = tmpBase + ".tmp"
baseName = os.path.basename(tmpBase)
- substitutions.extend(
- [
- ("%s", sourcepath),
- ("%S", sourcedir),
- ("%p", sourcedir),
- ("%{pathsep}", os.pathsep),
- ("%t", tmpName),
- ("%basename_t", baseName),
- ("%T", tmpDir),
- ]
- )
+ substitutions.append(("%{pathsep}", os.pathsep))
+ substitutions.append(("%basename_t", baseName))
+
substitutions.extend(
[
("%{fs-src-root}", pathlib.Path(sourcedir).anchor),
@@ -1263,49 +1255,47 @@
]
)
- # "%/[STpst]" should be normalized.
- substitutions.extend(
- [
- ("%/s", sourcepath.replace("\\", "/")),
- ("%/S", sourcedir.replace("\\", "/")),
- ("%/p", sourcedir.replace("\\", "/")),
- ("%/t", tmpBase.replace("\\", "/") + ".tmp"),
- ("%/T", tmpDir.replace("\\", "/")),
- ("%/et", tmpName.replace("\\", "\\\\\\\\\\\\\\\\")),
- ]
- )
+ substitutions.append(("%/et", tmpName.replace("\\", "\\\\\\\\\\\\\\\\")))
- # "%{/[STpst]:regex_replacement}" should be normalized like "%/[STpst]" but we're
- # also in a regex replacement context of a s@@@ regex.
def regex_escape(s):
s = s.replace("@", r"\@")
s = s.replace("&", r"\&")
return s
- substitutions.extend(
- [
- ("%{/s:regex_replacement}", regex_escape(sourcepath.replace("\\", "/"))),
- ("%{/S:regex_replacement}", regex_escape(sourcedir.replace("\\", "/"))),
- ("%{/p:regex_replacement}", regex_escape(sourcedir.replace("\\", "/"))),
- (
- "%{/t:regex_replacement}",
- regex_escape(tmpBase.replace("\\", "/")) + ".tmp",
- ),
- ("%{/T:regex_replacement}", regex_escape(tmpDir.replace("\\", "/"))),
- ]
- )
+ path_substitutions = [
+ ("s", sourcepath), ("S", sourcedir), ("p", sourcedir),
+ ("t", tmpName), ("T", tmpDir)
+ ]
+ for path_substitution in path_substitutions:
+ letter = path_substitution[0]
+ path = path_substitution[1]
+
+ # Original path variant
+ substitutions.append(("%" + letter, path))
+
+ # Normalized path separator variant
+ substitutions.append(("%/" + letter, path.replace("\\", "/")))
+
+ # realpath variants
+ # Windows paths with substitute drives are not expanded by default
+ # as they are used to avoid MAX_PATH issues, but sometimes we do
+ # need the fully expanded path.
+ real_path = os.path.realpath(path)
+ substitutions.append(("%{" + letter + ":real}", real_path))
+ substitutions.append(("%{/" + letter + ":real}",
+ real_path.replace("\\", "/")))
+
+ # "%{/[STpst]:regex_replacement}" should be normalized like
+ # "%/[STpst]" but we're also in a regex replacement context
+ # of a s@@@ regex.
+ substitutions.append(
+ ("%{/" + letter + ":regex_replacement}",
+ regex_escape(path.replace("\\", "/"))))
+
+ # "%:[STpst]" are normalized paths without colons and without
+ # a leading slash.
+ substitutions.append(("%:" + letter, colonNormalizePath(path)))
- # "%:[STpst]" are normalized paths without colons and without a leading
- # slash.
- substitutions.extend(
- [
- ("%:s", colonNormalizePath(sourcepath)),
- ("%:S", colonNormalizePath(sourcedir)),
- ("%:p", colonNormalizePath(sourcedir)),
- ("%:t", colonNormalizePath(tmpBase + ".tmp")),
- ("%:T", colonNormalizePath(tmpDir)),
- ]
- )
return substitutions
Index: llvm/docs/TestingGuide.rst
===================================================================
--- llvm/docs/TestingGuide.rst
+++ llvm/docs/TestingGuide.rst
@@ -671,7 +671,7 @@
``${fs-sep}``
Expands to the file system separator, i.e. ``/`` or ``\`` on Windows.
-``%/s, %/S, %/t, %/T:``
+``%/s, %/S, %/t, %/T``
Act like the corresponding substitution above but replace any ``\``
character with a ``/``. This is useful to normalize path separators.
@@ -680,7 +680,17 @@
Example: ``%/s: C:/Desktop Files/foo_test.s.tmp``
-``%:s, %:S, %:t, %:T:``
+``%{s:real}, %{S:real}, %{t:real}, %{T:real}``
+``%{/s:real}, %{/S:real}, %{/t:real}, %{/T:real}``
+
+ Act like the corresponding substitution, including with ``/``, but use
+ the real path by expanding all symbolic links and substitute drives.
+
+ Example: ``%s: S:\foo_test.s.tmp``
+
+ Example: ``%{/s:real}: C:/SDrive/foo_test.s.tmp``
+
+``%:s, %:S, %:t, %:T``
Act like the corresponding substitution above but remove colons at
the beginning of Windows paths. This is useful to allow concatenation
Index: llvm/docs/CommandGuide/lit.rst
===================================================================
--- llvm/docs/CommandGuide/lit.rst
+++ llvm/docs/CommandGuide/lit.rst
@@ -541,6 +541,16 @@
%/p %p but ``\`` is replaced by ``/``
%/t %t but ``\`` is replaced by ``/``
%/T %T but ``\`` is replaced by ``/``
+ %{s:real} %s after expanding all symbolic links and substitute drives
+ %{S:real} %S after expanding all symbolic links and substitute drives
+ %{p:real} %p after expanding all symbolic links and substitute drives
+ %{t:real} %t after expanding all symbolic links and substitute drives
+ %{T:real} %T after expanding all symbolic links and substitute drives
+ %{/s:real} %/s after expanding all symbolic links and substitute drives
+ %{/S:real} %/S after expanding all symbolic links and substitute drives
+ %{/p:real} %/p after expanding all symbolic links and substitute drives
+ %{/t:real} %/t after expanding all symbolic links and substitute drives
+ %{/T:real} %/T after expanding all symbolic links and substitute drives
%{/s:regex_replacement} %/s but escaped for use in the replacement of a ``s@@@`` command in sed
%{/S:regex_replacement} %/S but escaped for use in the replacement of a ``s@@@`` command in sed
%{/p:regex_replacement} %/p but escaped for use in the replacement of a ``s@@@`` command in sed
Index: llvm/cmake/modules/AddLLVM.cmake
===================================================================
--- llvm/cmake/modules/AddLLVM.cmake
+++ llvm/cmake/modules/AddLLVM.cmake
@@ -1711,10 +1711,15 @@
# use it and can't be in a lit module. Use with make_paths_relative().
string(CONCAT LLVM_LIT_PATH_FUNCTION
"# Allow generated file to be relocatable.\n"
- "from pathlib import Path\n"
+ "import os\n"
+ "import platform\n"
"def path(p):\n"
" if not p: return ''\n"
- " return str((Path(__file__).parent / p).resolve())\n"
+ " # Follows lit.util.abs_path_preserve_drive, which cannot be imported here.\n"
+ " if platform.system() == 'Windows':\n"
+ " return os.path.abspath(os.path.join(os.path.dirname(__file__), p))\n"
+ " else:\n"
+ " return os.path.realpath(os.path.join(os.path.dirname(__file__), p))\n"
)
# This function provides an automatic way to 'configure'-like generate a file
Index: clang/test/Lexer/case-insensitive-include-win.c
===================================================================
--- clang/test/Lexer/case-insensitive-include-win.c
+++ clang/test/Lexer/case-insensitive-include-win.c
@@ -2,9 +2,15 @@
// This file should only include code that really needs a Windows host OS to
// run.
+// Note: We must use the real path here, because the logic to detect case
+// mismatches must resolve the real path to figure out the original casing.
+// If we use %t and we are on a substitute drive S: mapping to C:\subst,
+// then we will compare "S:\test.dir\FOO.h" to "C:\subst\test.dir\foo.h"
+// and avoid emitting the diagnostic because the structure is different.
+
// REQUIRES: system-windows
// RUN: mkdir -p %t.dir
// RUN: touch %t.dir/foo.h
-// RUN: not %clang_cl /FI\\?\%t.dir\FOO.h /WX -fsyntax-only %s 2>&1 | FileCheck %s
+// RUN: not %clang_cl /FI \\?\%{t:real}.dir\FOO.h /WX -fsyntax-only %s 2>&1 | FileCheck %s
// CHECK: non-portable path to file '"\\?\
Index: clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
===================================================================
--- clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
+++ clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
@@ -187,9 +187,7 @@
if (ExternalFileEntries.count(File))
return false;
- StringRef FileName = File->tryGetRealPathName().empty()
- ? File->getName()
- : File->tryGetRealPathName();
+ StringRef FileName = SM.getFileManager().getCanonicalName(File);
// Try to reduce the include name the same way we tried to include it.
bool IsQuoted = false;
Index: clang/lib/Basic/FileManager.cpp
===================================================================
--- clang/lib/Basic/FileManager.cpp
+++ clang/lib/Basic/FileManager.cpp
@@ -632,33 +632,48 @@
}
StringRef FileManager::getCanonicalName(DirectoryEntryRef Dir) {
- auto Known = CanonicalNames.find(Dir);
- if (Known != CanonicalNames.end())
- return Known->second;
-
- StringRef CanonicalName(Dir.getName());
-
- SmallString<4096> CanonicalNameBuf;
- if (!FS->getRealPath(Dir.getName(), CanonicalNameBuf))
- CanonicalName = CanonicalNameBuf.str().copy(CanonicalNameStorage);
-
- CanonicalNames.insert({Dir, CanonicalName});
- return CanonicalName;
+ return getCanonicalName(Dir, Dir.getName());
}
StringRef FileManager::getCanonicalName(const FileEntry *File) {
- llvm::DenseMap<const void *, llvm::StringRef>::iterator Known
- = CanonicalNames.find(File);
+ return getCanonicalName(File, File->getName());
+}
+
+StringRef FileManager::getCanonicalName(const void *Entry, StringRef Name) {
+ llvm::DenseMap<const void *, llvm::StringRef>::iterator Known =
+ CanonicalNames.find(Entry);
if (Known != CanonicalNames.end())
return Known->second;
- StringRef CanonicalName(File->getName());
-
- SmallString<4096> CanonicalNameBuf;
- if (!FS->getRealPath(File->getName(), CanonicalNameBuf))
- CanonicalName = CanonicalNameBuf.str().copy(CanonicalNameStorage);
+ // Name comes from FileEntry/DirectoryEntry::getName(), so it is safe to
+ // store it in the DenseMap below.
+ StringRef CanonicalName(Name);
+
+ SmallString<256> AbsPathBuf;
+ SmallString<256> RealPathBuf;
+ if (!FS->getRealPath(Name, RealPathBuf)) {
+ if (is_style_windows(llvm::sys::path::Style::native)) {
+ // For Windows paths, only use the real path if it doesn't resolve
+ // a substitute drive, as those are used to avoid MAX_PATH issues.
+ AbsPathBuf = Name;
+ if (!FS->makeAbsolute(AbsPathBuf)) {
+ if (llvm::sys::path::root_name(RealPathBuf) ==
+ llvm::sys::path::root_name(AbsPathBuf)) {
+ CanonicalName = RealPathBuf.str().copy(CanonicalNameStorage);
+ } else {
+ // Fallback to using the absolute path.
+ // Simplifying /../ is semantically valid on Windows even in the
+ // presence of symbolic links.
+ llvm::sys::path::remove_dots(AbsPathBuf, /*remove_dot_dot=*/true);
+ CanonicalName = AbsPathBuf.str().copy(CanonicalNameStorage);
+ }
+ }
+ } else {
+ CanonicalName = RealPathBuf.str().copy(CanonicalNameStorage);
+ }
+ }
- CanonicalNames.insert({File, CanonicalName});
+ CanonicalNames.insert({Entry, CanonicalName});
return CanonicalName;
}
Index: clang/include/clang/Basic/FileManager.h
===================================================================
--- clang/include/clang/Basic/FileManager.h
+++ clang/include/clang/Basic/FileManager.h
@@ -329,6 +329,13 @@
/// required, which is (almost) never.
StringRef getCanonicalName(const FileEntry *File);
+private:
+ /// Retrieve the canonical name for a given file or directory.
+ ///
+ /// The first param is a key in the CanonicalNames array.
+ StringRef getCanonicalName(const void *Entry, StringRef Name);
+
+public:
void PrintStats() const;
};
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
