Author: Andy Kaylor Date: 2026-06-22T10:52:29-07:00 New Revision: 1d8a54f7cf233be64ffce08a3c5697ea8f1cccb5
URL: https://github.com/llvm/llvm-project/commit/1d8a54f7cf233be64ffce08a3c5697ea8f1cccb5 DIFF: https://github.com/llvm/llvm-project/commit/1d8a54f7cf233be64ffce08a3c5697ea8f1cccb5.diff LOG: [CIR] Lower const arrays as a single llvm.mlir.constant (#203590) When compiling the blender benchmark for SPEC CPU2017, we hit a case where a very large array (more than 400k elements) is initialized with constant values. However, because it contains trailing zeros, CIR generates a constant record initializer (an array of elements, plus a zero-initialized trailing array). We were lowering this to the LLVM dialect using a global initializer function with a huge number of calls to insertelement. The subsequent lowering to LLVM IR constant folded back to a constant initializer, but it took about 40 minutes to compile. The recent fix to avoid calling insertelement for the array initialization didn't fix this case because it handled only arrays, not records. This change updates the lowering to the LLVM dialect to lower constant array attributes to a single llvm.mlir.const value rather than attempting to build a chain of insertvalue ops whenever possible. Added: Modified: clang/include/clang/CIR/LoweringHelpers.h clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp clang/lib/CIR/Lowering/LoweringHelpers.cpp clang/test/CIR/Lowering/const-array-bulk-lowering-fallbacks.cir clang/test/CIR/Lowering/const-array-of-pointers.cir Removed: ################################################################################ diff --git a/clang/include/clang/CIR/LoweringHelpers.h b/clang/include/clang/CIR/LoweringHelpers.h index 3f3e939621c37..b4567828392c9 100644 --- a/clang/include/clang/CIR/LoweringHelpers.h +++ b/clang/include/clang/CIR/LoweringHelpers.h @@ -38,6 +38,11 @@ lowerConstArrayAttr(cir::ConstArrayAttr constArr, const mlir::TypeConverter *converter, mlir::ModuleOp moduleOp = {}); +std::optional<mlir::Attribute> +lowerConstRecordAttr(cir::ConstRecordAttr constRecord, + const mlir::TypeConverter *converter, + mlir::ModuleOp moduleOp = {}); + mlir::Value getConstAPInt(mlir::OpBuilder &bld, mlir::Location loc, mlir::Type typ, const llvm::APInt &val); diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index c844375a000e0..25fa6d1625301 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -481,6 +481,12 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) { mlir::Location loc = parentOp->getLoc(); mlir::Value result; + // When the array can be represented as a single dense constant, emit one + // llvm.mlir.constant instead of a chain of llvm.insertvalue ops. + if (std::optional<mlir::Attribute> denseAttr = + lowerConstArrayAttr(attr, converter)) + return mlir::LLVM::ConstantOp::create(rewriter, loc, llvmTy, *denseAttr); + if (attr.hasTrailingZeros()) { mlir::Type arrayTy = attr.getType(); result = mlir::LLVM::ZeroOp::create(rewriter, loc, @@ -2503,11 +2509,27 @@ mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite( } } return matchAndRewriteRegionInitializedGlobal(op, init.value(), rewriter); - } else if (mlir::isa<cir::ConstVectorAttr, cir::ConstRecordAttr, - cir::ConstPtrAttr, cir::ConstComplexAttr, - cir::GlobalViewAttr, cir::TypeInfoAttr, cir::UndefAttr, - cir::PoisonAttr, cir::VTableAttr, cir::ZeroAttr>( - init.value())) { + } else if (auto constRecord = + mlir::dyn_cast<cir::ConstRecordAttr>(init.value())) { + // Bulk-emit llvm.mlir.global when every member of the record can be + // lowered to a constant attribute. The LLVM dialect global translation + // turns an ArrayAttr (one element per struct field) into an + // llvm::ConstantStruct, so the whole initializer becomes a single + // attribute on the global instead of an insertvalue region. + mlir::ModuleOp modOp = op->getParentOfType<mlir::ModuleOp>(); + if (std::optional<mlir::Attribute> bulkInit = + lowerConstRecordAttr(constRecord, typeConverter, modOp)) { + mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter); + rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>( + op, llvmType, isConst, linkage, symbol, bulkInit.value(), alignment, + addrSpace, isDsoLocal, isThreadLocal, comdatAttr, attributes); + return mlir::success(); + } + return matchAndRewriteRegionInitializedGlobal(op, init.value(), rewriter); + } else if (mlir::isa<cir::ConstVectorAttr, cir::ConstPtrAttr, + cir::ConstComplexAttr, cir::GlobalViewAttr, + cir::TypeInfoAttr, cir::UndefAttr, cir::PoisonAttr, + cir::VTableAttr, cir::ZeroAttr>(init.value())) { // TODO(cir): once LLVM's dialect has proper equivalent attributes this // should be updated. For now, we use a custom op to initialize globals // to the appropriate value. diff --git a/clang/lib/CIR/Lowering/LoweringHelpers.cpp b/clang/lib/CIR/Lowering/LoweringHelpers.cpp index 92e64e188633e..f298095e56824 100644 --- a/clang/lib/CIR/Lowering/LoweringHelpers.cpp +++ b/clang/lib/CIR/Lowering/LoweringHelpers.cpp @@ -272,6 +272,69 @@ lowerConstArrayAttr(cir::ConstArrayAttr constArr, return std::nullopt; } +/// Lower a constant attribute that initializes a single member of a record (or +/// a leaf of a nested aggregate) to an LLVM-dialect attribute that can be +/// attached directly to an \c llvm.mlir.global, avoiding an insertvalue +/// initializer region. Returns \c std::nullopt when the attribute cannot be +/// represented as a single constant attribute (e.g. an indexed +/// \c GlobalViewAttr), in which case the caller falls back to the region-based +/// lowering. +static std::optional<mlir::Attribute> +lowerConstRecordMemberAttr(mlir::Attribute attr, + const mlir::TypeConverter *converter, + mlir::ModuleOp moduleOp) { + mlir::MLIRContext *ctx = attr.getContext(); + + if (auto arrayAttr = mlir::dyn_cast<cir::ConstArrayAttr>(attr)) + return lowerConstArrayAttr(arrayAttr, converter, moduleOp); + + if (auto recordAttr = mlir::dyn_cast<cir::ConstRecordAttr>(attr)) + return lowerConstRecordAttr(recordAttr, converter, moduleOp); + + if (mlir::isa<cir::ZeroAttr>(attr)) + return mlir::LLVM::ZeroAttr::get(ctx); + + if (mlir::isa<cir::UndefAttr>(attr)) + return mlir::LLVM::UndefAttr::get(ctx); + + if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(attr)) + return mlir::IntegerAttr::get(converter->convertType(intAttr.getType()), + intAttr.getValue()); + + if (auto boolAttr = mlir::dyn_cast<cir::BoolAttr>(attr)) + return mlir::IntegerAttr::get(converter->convertType(boolAttr.getType()), + boolAttr.getValue() ? 1 : 0); + + if (auto fpAttr = mlir::dyn_cast<cir::FPAttr>(attr)) + return mlir::FloatAttr::get(converter->convertType(fpAttr.getType()), + fpAttr.getValue()); + + // Null pointers and simple address-of-global references can be represented + // as constant attributes; anything more complex uses the region fallback. + return lowerPointerElementAttr(attr, ctx, moduleOp, converter); +} + +std::optional<mlir::Attribute> +lowerConstRecordAttr(cir::ConstRecordAttr constRecord, + const mlir::TypeConverter *converter, + mlir::ModuleOp moduleOp) { + // Build one constant attribute per record member. The LLVM dialect global + // translation accepts an ArrayAttr (one element per struct field) and emits + // an llvm::ConstantStruct, so the whole initializer can be a single + // attribute on the global instead of an insertvalue region. + mlir::ArrayAttr memberAttrs = constRecord.getMembers(); + llvm::SmallVector<mlir::Attribute> loweredMembers; + loweredMembers.reserve(memberAttrs.size()); + for (mlir::Attribute member : memberAttrs) { + std::optional<mlir::Attribute> lowered = + lowerConstRecordMemberAttr(member, converter, moduleOp); + if (!lowered) + return std::nullopt; + loweredMembers.push_back(*lowered); + } + return mlir::ArrayAttr::get(constRecord.getContext(), loweredMembers); +} + mlir::Value getConstAPInt(mlir::OpBuilder &bld, mlir::Location loc, mlir::Type typ, const llvm::APInt &val) { return mlir::LLVM::ConstantOp::create(bld, loc, typ, val); diff --git a/clang/test/CIR/Lowering/const-array-bulk-lowering-fallbacks.cir b/clang/test/CIR/Lowering/const-array-bulk-lowering-fallbacks.cir index 56d80c655cb72..2c894a87a4cb6 100644 --- a/clang/test/CIR/Lowering/const-array-bulk-lowering-fallbacks.cir +++ b/clang/test/CIR/Lowering/const-array-bulk-lowering-fallbacks.cir @@ -2,6 +2,8 @@ !s32i = !cir.int<s, 32> !s8i = !cir.int<s, 8> +!rec_data = !cir.struct<{!cir.array<!s8i x 4>, !cir.array<!s8i x 2>}> +!rec_mixed = !cir.struct<{!s32i, !cir.ptr<!s32i>}> module attributes {cir.triple = "x86_64-unknown-linux-gnu"} { cir.global external @tz_arr = @@ -34,6 +36,11 @@ module attributes {cir.triple = "x86_64-unknown-linux-gnu"} { #cir.global_view<@s1> : !cir.ptr<!s8i> ]> : !cir.array<!cir.ptr<!s8i> x 2> ]> : !cir.array<!cir.array<!cir.ptr<!s8i> x 2> x 2> + + cir.global constant external dso_local @blob = #cir.const_record<{ + #cir.const_array<[#cir.int<66> : !s8i, #cir.int<76> : !s8i, #cir.int<69> : !s8i, #cir.int<78> : !s8i]> : !cir.array<!s8i x 4>, + #cir.zero : !cir.array<!s8i x 2> + }> : !rec_data } // CHECK-LABEL: llvm.mlir.global external @tz_arr( @@ -53,3 +60,9 @@ module attributes {cir.triple = "x86_64-unknown-linux-gnu"} { // CHECK-LABEL: llvm.mlir.global external @ptr2d // CHECK: llvm.insertvalue + +// CHECK-LABEL: llvm.mlir.global external constant @blob( +// CHECK-SAME: [dense<[66, 76, 69, 78]> : tensor<4xi8>, #llvm.zero] +// CHECK-SAME: : !llvm.struct<(array<4 x i8>, array<2 x i8>)> +// CHECK-NOT: llvm.insertvalue +// CHECK-NOT: llvm.return diff --git a/clang/test/CIR/Lowering/const-array-of-pointers.cir b/clang/test/CIR/Lowering/const-array-of-pointers.cir index 47797c0cf3271..5d590e086fe8b 100644 --- a/clang/test/CIR/Lowering/const-array-of-pointers.cir +++ b/clang/test/CIR/Lowering/const-array-of-pointers.cir @@ -1,6 +1,7 @@ // RUN: cir-opt %s --cir-to-llvm -o - | FileCheck %s !s32i = !cir.int<s, 32> +!rec_mixed = !cir.struct<{!s32i, !cir.ptr<!s32i>}> module attributes {cir.triple = "x86_64-unknown-linux-gnu"} { cir.global "private" constant cir_private dso_local @g0 = #cir.int<0> : !s32i @@ -12,7 +13,18 @@ module attributes {cir.triple = "x86_64-unknown-linux-gnu"} { #cir.global_view<@g1> : !cir.ptr<!s32i>, #cir.global_view<@g2> : !cir.ptr<!s32i> ]> : !cir.array<!cir.ptr<!s32i> x 3> + + cir.global constant external dso_local @mixed = #cir.const_record<{ + #cir.int<7> : !s32i, + #cir.global_view<@g0> : !cir.ptr<!s32i> + }> : !rec_mixed } // CHECK-LABEL: llvm.mlir.global external constant @ptrs( // CHECK-NOT: llvm.insertvalue + +// CHECK-LABEL: llvm.mlir.global external constant @mixed( +// CHECK-SAME: [7 : i32, @g0] +// CHECK-SAME: : !llvm.struct<(i32, ptr)> +// CHECK-NOT: llvm.insertvalue +// CHECK-NOT: llvm.return _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
