summaryrefslogtreecommitdiff
path: root/clang/lib/CodeGen
diff options
context:
space:
mode:
authorDimitry Andric <dim@FreeBSD.org>2022-07-03 14:10:23 +0000
committerDimitry Andric <dim@FreeBSD.org>2022-07-03 14:10:23 +0000
commit145449b1e420787bb99721a429341fa6be3adfb6 (patch)
tree1d56ae694a6de602e348dd80165cf881a36600ed /clang/lib/CodeGen
parentecbca9f5fb7d7613d2b94982c4825eb0d33d6842 (diff)
Vendor import of llvm-project main llvmorg-15-init-15358-g53dc0f107877.vendor/llvm-project/llvmorg-15-init-15358-g53dc0f107877
Diffstat (limited to 'clang/lib/CodeGen')
-rw-r--r--clang/lib/CodeGen/ABIInfo.h1
-rw-r--r--clang/lib/CodeGen/Address.h5
-rw-r--r--clang/lib/CodeGen/BackendUtil.cpp631
-rw-r--r--clang/lib/CodeGen/CGAtomic.cpp27
-rw-r--r--clang/lib/CodeGen/CGBlocks.cpp80
-rw-r--r--clang/lib/CodeGen/CGBuilder.h75
-rw-r--r--clang/lib/CodeGen/CGBuiltin.cpp2069
-rw-r--r--clang/lib/CodeGen/CGCUDANV.cpp86
-rw-r--r--clang/lib/CodeGen/CGCUDARuntime.h13
-rw-r--r--clang/lib/CodeGen/CGCXXABI.cpp3
-rw-r--r--clang/lib/CodeGen/CGCXXABI.h5
-rw-r--r--clang/lib/CodeGen/CGCall.cpp238
-rw-r--r--clang/lib/CodeGen/CGClass.cpp242
-rw-r--r--clang/lib/CodeGen/CGCleanup.cpp30
-rw-r--r--clang/lib/CodeGen/CGCoroutine.cpp107
-rw-r--r--clang/lib/CodeGen/CGDebugInfo.cpp501
-rw-r--r--clang/lib/CodeGen/CGDebugInfo.h44
-rw-r--r--clang/lib/CodeGen/CGDecl.cpp73
-rw-r--r--clang/lib/CodeGen/CGDeclCXX.cpp14
-rw-r--r--clang/lib/CodeGen/CGException.cpp5
-rw-r--r--clang/lib/CodeGen/CGExpr.cpp407
-rw-r--r--clang/lib/CodeGen/CGExprCXX.cpp24
-rw-r--r--clang/lib/CodeGen/CGExprConstant.cpp47
-rw-r--r--clang/lib/CodeGen/CGExprScalar.cpp104
-rw-r--r--clang/lib/CodeGen/CGHLSLRuntime.cpp52
-rw-r--r--clang/lib/CodeGen/CGHLSLRuntime.h38
-rw-r--r--clang/lib/CodeGen/CGNonTrivialStruct.cpp14
-rw-r--r--clang/lib/CodeGen/CGObjC.cpp64
-rw-r--r--clang/lib/CodeGen/CGObjCGNU.cpp84
-rw-r--r--clang/lib/CodeGen/CGObjCMac.cpp69
-rw-r--r--clang/lib/CodeGen/CGObjCRuntime.cpp2
-rw-r--r--clang/lib/CodeGen/CGOpenCLRuntime.cpp51
-rw-r--r--clang/lib/CodeGen/CGOpenCLRuntime.h6
-rw-r--r--clang/lib/CodeGen/CGOpenMPRuntime.cpp1030
-rw-r--r--clang/lib/CodeGen/CGOpenMPRuntime.h29
-rw-r--r--clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp173
-rw-r--r--clang/lib/CodeGen/CGRecordLayout.h6
-rw-r--r--clang/lib/CodeGen/CGStmt.cpp70
-rw-r--r--clang/lib/CodeGen/CGStmtOpenMP.cpp995
-rw-r--r--clang/lib/CodeGen/CGVTT.cpp4
-rw-r--r--clang/lib/CodeGen/CGVTables.cpp26
-rw-r--r--clang/lib/CodeGen/CodeGenAction.cpp38
-rw-r--r--clang/lib/CodeGen/CodeGenFunction.cpp198
-rw-r--r--clang/lib/CodeGen/CodeGenFunction.h176
-rw-r--r--clang/lib/CodeGen/CodeGenModule.cpp689
-rw-r--r--clang/lib/CodeGen/CodeGenModule.h120
-rw-r--r--clang/lib/CodeGen/CodeGenPGO.cpp2
-rw-r--r--clang/lib/CodeGen/CodeGenTypes.cpp59
-rw-r--r--clang/lib/CodeGen/CodeGenTypes.h5
-rw-r--r--clang/lib/CodeGen/ConstantInitBuilder.cpp2
-rw-r--r--clang/lib/CodeGen/CoverageMappingGen.cpp52
-rw-r--r--clang/lib/CodeGen/CoverageMappingGen.h20
-rw-r--r--clang/lib/CodeGen/ItaniumCXXABI.cpp34
-rw-r--r--clang/lib/CodeGen/MacroPPCallbacks.cpp2
-rw-r--r--clang/lib/CodeGen/MacroPPCallbacks.h6
-rw-r--r--clang/lib/CodeGen/MicrosoftCXXABI.cpp25
-rw-r--r--clang/lib/CodeGen/ModuleBuilder.cpp12
-rw-r--r--clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp4
-rw-r--r--clang/lib/CodeGen/SanitizerMetadata.cpp119
-rw-r--r--clang/lib/CodeGen/SanitizerMetadata.h21
-rw-r--r--clang/lib/CodeGen/TargetInfo.cpp595
-rw-r--r--clang/lib/CodeGen/TargetInfo.h2
62 files changed, 5710 insertions, 4015 deletions
diff --git a/clang/lib/CodeGen/ABIInfo.h b/clang/lib/CodeGen/ABIInfo.h
index 0d12183055e1..6214148adab9 100644
--- a/clang/lib/CodeGen/ABIInfo.h
+++ b/clang/lib/CodeGen/ABIInfo.h
@@ -100,6 +100,7 @@ namespace swiftcall {
virtual bool isHomogeneousAggregateSmallEnough(const Type *Base,
uint64_t Members) const;
+ virtual bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const;
bool isHomogeneousAggregate(QualType Ty, const Type *&Base,
uint64_t &Members) const;
diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h
index 3ac0f4f0d7e5..bddeac1d6dcb 100644
--- a/clang/lib/CodeGen/Address.h
+++ b/clang/lib/CodeGen/Address.h
@@ -87,11 +87,6 @@ public:
"Incorrect pointer element type");
}
- // Deprecated: Use constructor with explicit element type instead.
- Address(llvm::Value *Pointer, CharUnits Alignment)
- : Address(Pointer, Pointer->getType()->getPointerElementType(),
- Alignment) {}
-
static Address invalid() { return Address(nullptr); }
bool isValid() const { return A.getPointer() != nullptr; }
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index a4d330c0ba93..eb40e446057f 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -39,6 +39,7 @@
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/SubtargetFeature.h"
#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Object/OffloadBinary.h"
#include "llvm/Passes/PassBuilder.h"
#include "llvm/Passes/PassPlugin.h"
#include "llvm/Passes/StandardInstrumentations.h"
@@ -52,7 +53,6 @@
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
-#include "llvm/Transforms/Coroutines.h"
#include "llvm/Transforms/Coroutines/CoroCleanup.h"
#include "llvm/Transforms/Coroutines/CoroEarly.h"
#include "llvm/Transforms/Coroutines/CoroElide.h"
@@ -60,7 +60,6 @@
#include "llvm/Transforms/IPO.h"
#include "llvm/Transforms/IPO/AlwaysInliner.h"
#include "llvm/Transforms/IPO/LowerTypeTests.h"
-#include "llvm/Transforms/IPO/PassManagerBuilder.h"
#include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
#include "llvm/Transforms/InstCombine/InstCombine.h"
#include "llvm/Transforms/Instrumentation.h"
@@ -118,6 +117,8 @@ class EmitAssemblyHelper {
std::unique_ptr<raw_pwrite_stream> OS;
+ Triple TargetTriple;
+
TargetIRAnalysis getTargetIRAnalysis() const {
if (TM)
return TM->getTargetIRAnalysis();
@@ -125,8 +126,6 @@ class EmitAssemblyHelper {
return TargetIRAnalysis();
}
- void CreatePasses(legacy::PassManager &MPM, legacy::FunctionPassManager &FPM);
-
/// Generates the TargetMachine.
/// Leaves TM unchanged if it is unable to create the target machine.
/// Some of our clang tests specify triples which are not built
@@ -162,6 +161,16 @@ class EmitAssemblyHelper {
std::unique_ptr<raw_pwrite_stream> &OS,
std::unique_ptr<llvm::ToolOutputFile> &DwoOS);
+ /// Check whether we should emit a module summary for regular LTO.
+ /// The module summary should be emitted by default for regular LTO
+ /// except for ld64 targets.
+ ///
+ /// \return True if the module summary should be emitted.
+ bool shouldEmitRegularLTOSummary() const {
+ return CodeGenOpts.PrepareForLTO && !CodeGenOpts.DisableLLVMPasses &&
+ TargetTriple.getVendor() != llvm::Triple::Apple;
+ }
+
public:
EmitAssemblyHelper(DiagnosticsEngine &_Diags,
const HeaderSearchOptions &HeaderSearchOpts,
@@ -170,7 +179,8 @@ public:
const LangOptions &LOpts, Module *M)
: Diags(_Diags), HSOpts(HeaderSearchOpts), CodeGenOpts(CGOpts),
TargetOpts(TOpts), LangOpts(LOpts), TheModule(M),
- CodeGenerationTime("codegen", "Code Generation Time") {}
+ CodeGenerationTime("codegen", "Code Generation Time"),
+ TargetTriple(TheModule->getTargetTriple()) {}
~EmitAssemblyHelper() {
if (CodeGenOpts.DisableFree)
@@ -179,60 +189,10 @@ public:
std::unique_ptr<TargetMachine> TM;
- // Emit output using the legacy pass manager for the optimization pipeline.
- // This will be removed soon when using the legacy pass manager for the
- // optimization pipeline is no longer supported.
- void EmitAssemblyWithLegacyPassManager(BackendAction Action,
- std::unique_ptr<raw_pwrite_stream> OS);
-
- // Emit output using the new pass manager for the optimization pipeline. This
- // is the default.
+ // Emit output using the new pass manager for the optimization pipeline.
void EmitAssembly(BackendAction Action,
std::unique_ptr<raw_pwrite_stream> OS);
};
-
-// We need this wrapper to access LangOpts and CGOpts from extension functions
-// that we add to the PassManagerBuilder.
-class PassManagerBuilderWrapper : public PassManagerBuilder {
-public:
- PassManagerBuilderWrapper(const Triple &TargetTriple,
- const CodeGenOptions &CGOpts,
- const LangOptions &LangOpts)
- : TargetTriple(TargetTriple), CGOpts(CGOpts), LangOpts(LangOpts) {}
- const Triple &getTargetTriple() const { return TargetTriple; }
- const CodeGenOptions &getCGOpts() const { return CGOpts; }
- const LangOptions &getLangOpts() const { return LangOpts; }
-
-private:
- const Triple &TargetTriple;
- const CodeGenOptions &CGOpts;
- const LangOptions &LangOpts;
-};
-}
-
-static void addObjCARCAPElimPass(const PassManagerBuilder &Builder, PassManagerBase &PM) {
- if (Builder.OptLevel > 0)
- PM.add(createObjCARCAPElimPass());
-}
-
-static void addObjCARCExpandPass(const PassManagerBuilder &Builder, PassManagerBase &PM) {
- if (Builder.OptLevel > 0)
- PM.add(createObjCARCExpandPass());
-}
-
-static void addObjCARCOptPass(const PassManagerBuilder &Builder, PassManagerBase &PM) {
- if (Builder.OptLevel > 0)
- PM.add(createObjCARCOptPass());
-}
-
-static void addAddDiscriminatorsPass(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- PM.add(createAddDiscriminatorsPass());
-}
-
-static void addBoundsCheckingPass(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- PM.add(createBoundsCheckingLegacyPass());
}
static SanitizerCoverageOptions
@@ -258,17 +218,6 @@ getSancovOptsFromCGOpts(const CodeGenOptions &CGOpts) {
return Opts;
}
-static void addSanitizerCoveragePass(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- const PassManagerBuilderWrapper &BuilderWrapper =
- static_cast<const PassManagerBuilderWrapper &>(Builder);
- const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts();
- auto Opts = getSancovOptsFromCGOpts(CGOpts);
- PM.add(createModuleSanitizerCoverageLegacyPassPass(
- Opts, CGOpts.SanitizeCoverageAllowlistFiles,
- CGOpts.SanitizeCoverageIgnorelistFiles));
-}
-
// Check if ASan should use GC-friendly instrumentation for globals.
// First of all, there is no point if -fdata-sections is off (expect for MachO,
// where this is not a factor). Also, on ELF this feature requires an assembler
@@ -281,134 +230,20 @@ static bool asanUseGlobalsGC(const Triple &T, const CodeGenOptions &CGOpts) {
case Triple::COFF:
return true;
case Triple::ELF:
- return CGOpts.DataSections && !CGOpts.DisableIntegratedAS;
+ return !CGOpts.DisableIntegratedAS;
case Triple::GOFF:
llvm::report_fatal_error("ASan not implemented for GOFF");
case Triple::XCOFF:
llvm::report_fatal_error("ASan not implemented for XCOFF.");
case Triple::Wasm:
+ case Triple::DXContainer:
+ case Triple::SPIRV:
case Triple::UnknownObjectFormat:
break;
}
return false;
}
-static void addMemProfilerPasses(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- PM.add(createMemProfilerFunctionPass());
- PM.add(createModuleMemProfilerLegacyPassPass());
-}
-
-static void addAddressSanitizerPasses(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- const PassManagerBuilderWrapper &BuilderWrapper =
- static_cast<const PassManagerBuilderWrapper&>(Builder);
- const Triple &T = BuilderWrapper.getTargetTriple();
- const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts();
- bool Recover = CGOpts.SanitizeRecover.has(SanitizerKind::Address);
- bool UseAfterScope = CGOpts.SanitizeAddressUseAfterScope;
- bool UseOdrIndicator = CGOpts.SanitizeAddressUseOdrIndicator;
- bool UseGlobalsGC = asanUseGlobalsGC(T, CGOpts);
- llvm::AsanDtorKind DestructorKind = CGOpts.getSanitizeAddressDtor();
- llvm::AsanDetectStackUseAfterReturnMode UseAfterReturn =
- CGOpts.getSanitizeAddressUseAfterReturn();
- PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/ false, Recover,
- UseAfterScope, UseAfterReturn));
- PM.add(createModuleAddressSanitizerLegacyPassPass(
- /*CompileKernel*/ false, Recover, UseGlobalsGC, UseOdrIndicator,
- DestructorKind));
-}
-
-static void addKernelAddressSanitizerPasses(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- PM.add(createAddressSanitizerFunctionPass(
- /*CompileKernel*/ true, /*Recover*/ true, /*UseAfterScope*/ false,
- /*UseAfterReturn*/ llvm::AsanDetectStackUseAfterReturnMode::Never));
- PM.add(createModuleAddressSanitizerLegacyPassPass(
- /*CompileKernel*/ true, /*Recover*/ true, /*UseGlobalsGC*/ true,
- /*UseOdrIndicator*/ false));
-}
-
-static void addHWAddressSanitizerPasses(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- const PassManagerBuilderWrapper &BuilderWrapper =
- static_cast<const PassManagerBuilderWrapper &>(Builder);
- const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts();
- bool Recover = CGOpts.SanitizeRecover.has(SanitizerKind::HWAddress);
- PM.add(createHWAddressSanitizerLegacyPassPass(
- /*CompileKernel*/ false, Recover,
- /*DisableOptimization*/ CGOpts.OptimizationLevel == 0));
-}
-
-static void addKernelHWAddressSanitizerPasses(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- const PassManagerBuilderWrapper &BuilderWrapper =
- static_cast<const PassManagerBuilderWrapper &>(Builder);
- const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts();
- PM.add(createHWAddressSanitizerLegacyPassPass(
- /*CompileKernel*/ true, /*Recover*/ true,
- /*DisableOptimization*/ CGOpts.OptimizationLevel == 0));
-}
-
-static void addGeneralOptsForMemorySanitizer(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM,
- bool CompileKernel) {
- const PassManagerBuilderWrapper &BuilderWrapper =
- static_cast<const PassManagerBuilderWrapper&>(Builder);
- const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts();
- int TrackOrigins = CGOpts.SanitizeMemoryTrackOrigins;
- bool Recover = CGOpts.SanitizeRecover.has(SanitizerKind::Memory);
- PM.add(createMemorySanitizerLegacyPassPass(
- MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel,
- CGOpts.SanitizeMemoryParamRetval != 0}));
-
- // MemorySanitizer inserts complex instrumentation that mostly follows
- // the logic of the original code, but operates on "shadow" values.
- // It can benefit from re-running some general purpose optimization passes.
- if (Builder.OptLevel > 0) {
- PM.add(createEarlyCSEPass());
- PM.add(createReassociatePass());
- PM.add(createLICMPass());
- PM.add(createGVNPass());
- PM.add(createInstructionCombiningPass());
- PM.add(createDeadStoreEliminationPass());
- }
-}
-
-static void addMemorySanitizerPass(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- addGeneralOptsForMemorySanitizer(Builder, PM, /*CompileKernel*/ false);
-}
-
-static void addKernelMemorySanitizerPass(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- addGeneralOptsForMemorySanitizer(Builder, PM, /*CompileKernel*/ true);
-}
-
-static void addThreadSanitizerPass(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- PM.add(createThreadSanitizerLegacyPassPass());
-}
-
-static void addDataFlowSanitizerPass(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- const PassManagerBuilderWrapper &BuilderWrapper =
- static_cast<const PassManagerBuilderWrapper&>(Builder);
- const LangOptions &LangOpts = BuilderWrapper.getLangOpts();
- PM.add(createDataFlowSanitizerLegacyPassPass(LangOpts.NoSanitizeFiles));
-}
-
-static void addEntryExitInstrumentationPass(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- PM.add(createEntryExitInstrumenterPass());
-}
-
-static void
-addPostInlineEntryExitInstrumentationPass(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- PM.add(createPostInlineEntryExitInstrumenterPass());
-}
-
static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple,
const CodeGenOptions &CodeGenOpts) {
TargetLibraryInfoImpl *TLII = new TargetLibraryInfoImpl(TargetTriple);
@@ -443,17 +278,6 @@ static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple,
return TLII;
}
-static void addSymbolRewriterPass(const CodeGenOptions &Opts,
- legacy::PassManager *MPM) {
- llvm::SymbolRewriter::RewriteDescriptorList DL;
-
- llvm::SymbolRewriter::RewriteMapParser MapParser;
- for (const auto &MapFile : Opts.RewriteMapFiles)
- MapParser.parse(MapFile, &DL);
-
- MPM->add(createRewriteSymbolsPass(DL));
-}
-
static CodeGenOpt::Level getCGOptLevel(const CodeGenOptions &CodeGenOpts) {
switch (CodeGenOpts.OptimizationLevel) {
default:
@@ -546,6 +370,8 @@ static bool initTargetOptions(DiagnosticsEngine &Diags,
Options.BinutilsVersion =
llvm::TargetMachine::parseBinutilsVersion(CodeGenOpts.BinutilsVersion);
Options.UseInitArray = CodeGenOpts.UseInitArray;
+ Options.LowerGlobalDtorsViaCxaAtExit =
+ CodeGenOpts.RegisterGlobalDtorsWithAtExit;
Options.DisableIntegratedAS = CodeGenOpts.DisableIntegratedAS;
Options.CompressDebugSections = CodeGenOpts.getCompressDebugSections();
Options.RelaxELFRelocations = CodeGenOpts.RelaxELFRelocations;
@@ -606,6 +432,10 @@ static bool initTargetOptions(DiagnosticsEngine &Diags,
Options.EnableAIXExtendedAltivecABI = CodeGenOpts.EnableAIXExtendedAltivecABI;
Options.XRayOmitFunctionIndex = CodeGenOpts.XRayOmitFunctionIndex;
Options.LoopAlignment = CodeGenOpts.LoopAlignment;
+ Options.DebugStrictDwarf = CodeGenOpts.DebugStrictDwarf;
+ Options.ObjectFilenameForDebug = CodeGenOpts.ObjectFilenameForDebug;
+ Options.Hotpatch = CodeGenOpts.HotPatch;
+ Options.JMCInstrument = CodeGenOpts.JMCInstrument;
switch (CodeGenOpts.getSwiftAsyncFramePointer()) {
case CodeGenOptions::SwiftAsyncFramePointerKind::Auto:
@@ -623,9 +453,13 @@ static bool initTargetOptions(DiagnosticsEngine &Diags,
}
Options.MCOptions.SplitDwarfFile = CodeGenOpts.SplitDwarfFile;
+ Options.MCOptions.EmitDwarfUnwind = CodeGenOpts.getEmitDwarfUnwind();
Options.MCOptions.MCRelaxAll = CodeGenOpts.RelaxAll;
Options.MCOptions.MCSaveTempLabels = CodeGenOpts.SaveTempLabels;
- Options.MCOptions.MCUseDwarfDirectory = !CodeGenOpts.NoDwarfDirectoryAsm;
+ Options.MCOptions.MCUseDwarfDirectory =
+ CodeGenOpts.NoDwarfDirectoryAsm
+ ? llvm::MCTargetOptions::DisableDwarfDirectory
+ : llvm::MCTargetOptions::EnableDwarfDirectory;
Options.MCOptions.MCNoExecStack = CodeGenOpts.NoExecStack;
Options.MCOptions.MCIncrementalLinkerCompatible =
CodeGenOpts.IncrementalLinkerCompatible;
@@ -644,9 +478,7 @@ static bool initTargetOptions(DiagnosticsEngine &Diags,
Entry.IgnoreSysRoot ? Entry.Path : HSOpts.Sysroot + Entry.Path);
Options.MCOptions.Argv0 = CodeGenOpts.Argv0;
Options.MCOptions.CommandLineArgs = CodeGenOpts.CommandLineArgs;
- Options.DebugStrictDwarf = CodeGenOpts.DebugStrictDwarf;
- Options.ObjectFilenameForDebug = CodeGenOpts.ObjectFilenameForDebug;
- Options.Hotpatch = CodeGenOpts.HotPatch;
+ Options.MisExpect = CodeGenOpts.MisExpect;
return true;
}
@@ -680,233 +512,6 @@ getInstrProfOptions(const CodeGenOptions &CodeGenOpts,
return Options;
}
-void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM,
- legacy::FunctionPassManager &FPM) {
- // Handle disabling of all LLVM passes, where we want to preserve the
- // internal module before any optimization.
- if (CodeGenOpts.DisableLLVMPasses)
- return;
-
- // Figure out TargetLibraryInfo. This needs to be added to MPM and FPM
- // manually (and not via PMBuilder), since some passes (eg. InstrProfiling)
- // are inserted before PMBuilder ones - they'd get the default-constructed
- // TLI with an unknown target otherwise.
- Triple TargetTriple(TheModule->getTargetTriple());
- std::unique_ptr<TargetLibraryInfoImpl> TLII(
- createTLII(TargetTriple, CodeGenOpts));
-
- // If we reached here with a non-empty index file name, then the index file
- // was empty and we are not performing ThinLTO backend compilation (used in
- // testing in a distributed build environment). Drop any the type test
- // assume sequences inserted for whole program vtables so that codegen doesn't
- // complain.
- if (!CodeGenOpts.ThinLTOIndexFile.empty())
- MPM.add(createLowerTypeTestsPass(/*ExportSummary=*/nullptr,
- /*ImportSummary=*/nullptr,
- /*DropTypeTests=*/true));
-
- PassManagerBuilderWrapper PMBuilder(TargetTriple, CodeGenOpts, LangOpts);
-
- // At O0 and O1 we only run the always inliner which is more efficient. At
- // higher optimization levels we run the normal inliner.
- if (CodeGenOpts.OptimizationLevel <= 1) {
- bool InsertLifetimeIntrinsics = ((CodeGenOpts.OptimizationLevel != 0 &&
- !CodeGenOpts.DisableLifetimeMarkers) ||
- LangOpts.Coroutines);
- PMBuilder.Inliner = createAlwaysInlinerLegacyPass(InsertLifetimeIntrinsics);
- } else {
- // We do not want to inline hot callsites for SamplePGO module-summary build
- // because profile annotation will happen again in ThinLTO backend, and we
- // want the IR of the hot path to match the profile.
- PMBuilder.Inliner = createFunctionInliningPass(
- CodeGenOpts.OptimizationLevel, CodeGenOpts.OptimizeSize,
- (!CodeGenOpts.SampleProfileFile.empty() &&
- CodeGenOpts.PrepareForThinLTO));
- }
-
- PMBuilder.OptLevel = CodeGenOpts.OptimizationLevel;
- PMBuilder.SizeLevel = CodeGenOpts.OptimizeSize;
- PMBuilder.SLPVectorize = CodeGenOpts.VectorizeSLP;
- PMBuilder.LoopVectorize = CodeGenOpts.VectorizeLoop;
- // Only enable CGProfilePass when using integrated assembler, since
- // non-integrated assemblers don't recognize .cgprofile section.
- PMBuilder.CallGraphProfile = !CodeGenOpts.DisableIntegratedAS;
-
- PMBuilder.DisableUnrollLoops = !CodeGenOpts.UnrollLoops;
- // Loop interleaving in the loop vectorizer has historically been set to be
- // enabled when loop unrolling is enabled.
- PMBuilder.LoopsInterleaved = CodeGenOpts.UnrollLoops;
- PMBuilder.MergeFunctions = CodeGenOpts.MergeFunctions;
- PMBuilder.PrepareForThinLTO = CodeGenOpts.PrepareForThinLTO;
- PMBuilder.PrepareForLTO = CodeGenOpts.PrepareForLTO;
- PMBuilder.RerollLoops = CodeGenOpts.RerollLoops;
-
- MPM.add(new TargetLibraryInfoWrapperPass(*TLII));
-
- if (TM)
- TM->adjustPassManager(PMBuilder);
-
- if (CodeGenOpts.DebugInfoForProfiling ||
- !CodeGenOpts.SampleProfileFile.empty())
- PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible,
- addAddDiscriminatorsPass);
-
- // In ObjC ARC mode, add the main ARC optimization passes.
- if (LangOpts.ObjCAutoRefCount) {
- PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible,
- addObjCARCExpandPass);
- PMBuilder.addExtension(PassManagerBuilder::EP_ModuleOptimizerEarly,
- addObjCARCAPElimPass);
- PMBuilder.addExtension(PassManagerBuilder::EP_ScalarOptimizerLate,
- addObjCARCOptPass);
- }
-
- if (LangOpts.Coroutines)
- addCoroutinePassesToExtensionPoints(PMBuilder);
-
- if (!CodeGenOpts.MemoryProfileOutput.empty()) {
- PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
- addMemProfilerPasses);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addMemProfilerPasses);
- }
-
- if (LangOpts.Sanitize.has(SanitizerKind::LocalBounds)) {
- PMBuilder.addExtension(PassManagerBuilder::EP_ScalarOptimizerLate,
- addBoundsCheckingPass);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addBoundsCheckingPass);
- }
-
- if (CodeGenOpts.hasSanitizeCoverage()) {
- PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
- addSanitizerCoveragePass);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addSanitizerCoveragePass);
- }
-
- if (LangOpts.Sanitize.has(SanitizerKind::Address)) {
- PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
- addAddressSanitizerPasses);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addAddressSanitizerPasses);
- }
-
- if (LangOpts.Sanitize.has(SanitizerKind::KernelAddress)) {
- PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
- addKernelAddressSanitizerPasses);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addKernelAddressSanitizerPasses);
- }
-
- if (LangOpts.Sanitize.has(SanitizerKind::HWAddress)) {
- PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
- addHWAddressSanitizerPasses);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addHWAddressSanitizerPasses);
- }
-
- if (LangOpts.Sanitize.has(SanitizerKind::KernelHWAddress)) {
- PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
- addKernelHWAddressSanitizerPasses);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addKernelHWAddressSanitizerPasses);
- }
-
- if (LangOpts.Sanitize.has(SanitizerKind::Memory)) {
- PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
- addMemorySanitizerPass);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addMemorySanitizerPass);
- }
-
- if (LangOpts.Sanitize.has(SanitizerKind::KernelMemory)) {
- PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
- addKernelMemorySanitizerPass);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addKernelMemorySanitizerPass);
- }
-
- if (LangOpts.Sanitize.has(SanitizerKind::Thread)) {
- PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
- addThreadSanitizerPass);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addThreadSanitizerPass);
- }
-
- if (LangOpts.Sanitize.has(SanitizerKind::DataFlow)) {
- PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
- addDataFlowSanitizerPass);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addDataFlowSanitizerPass);
- }
-
- if (CodeGenOpts.InstrumentFunctions ||
- CodeGenOpts.InstrumentFunctionEntryBare ||
- CodeGenOpts.InstrumentFunctionsAfterInlining ||
- CodeGenOpts.InstrumentForProfiling) {
- PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible,
- addEntryExitInstrumentationPass);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addEntryExitInstrumentationPass);
- PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
- addPostInlineEntryExitInstrumentationPass);
- PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
- addPostInlineEntryExitInstrumentationPass);
- }
-
- // Set up the per-function pass manager.
- FPM.add(new TargetLibraryInfoWrapperPass(*TLII));
- if (CodeGenOpts.VerifyModule)
- FPM.add(createVerifierPass());
-
- // Set up the per-module pass manager.
- if (!CodeGenOpts.RewriteMapFiles.empty())
- addSymbolRewriterPass(CodeGenOpts, &MPM);
-
- if (Optional<GCOVOptions> Options = getGCOVOptions(CodeGenOpts, LangOpts)) {
- MPM.add(createGCOVProfilerPass(*Options));
- if (CodeGenOpts.getDebugInfo() == codegenoptions::NoDebugInfo)
- MPM.add(createStripSymbolsPass(true));
- }
-
- if (Optional<InstrProfOptions> Options =
- getInstrProfOptions(CodeGenOpts, LangOpts))
- MPM.add(createInstrProfilingLegacyPass(*Options, false));
-
- bool hasIRInstr = false;
- if (CodeGenOpts.hasProfileIRInstr()) {
- PMBuilder.EnablePGOInstrGen = true;
- hasIRInstr = true;
- }
- if (CodeGenOpts.hasProfileCSIRInstr()) {
- assert(!CodeGenOpts.hasProfileCSIRUse() &&
- "Cannot have both CSProfileUse pass and CSProfileGen pass at the "
- "same time");
- assert(!hasIRInstr &&
- "Cannot have both ProfileGen pass and CSProfileGen pass at the "
- "same time");
- PMBuilder.EnablePGOCSInstrGen = true;
- hasIRInstr = true;
- }
- if (hasIRInstr) {
- if (!CodeGenOpts.InstrProfileOutput.empty())
- PMBuilder.PGOInstrGen = CodeGenOpts.InstrProfileOutput;
- else
- PMBuilder.PGOInstrGen = getDefaultProfileGenName();
- }
- if (CodeGenOpts.hasProfileIRUse()) {
- PMBuilder.PGOInstrUse = CodeGenOpts.ProfileInstrumentUsePath;
- PMBuilder.EnablePGOCSInstrUse = CodeGenOpts.hasProfileCSIRUse();
- }
-
- if (!CodeGenOpts.SampleProfileFile.empty())
- PMBuilder.PGOSampleUse = CodeGenOpts.SampleProfileFile;
-
- PMBuilder.populateFunctionPassManager(FPM);
- PMBuilder.populateModulePassManager(MPM);
-}
-
static void setCommandLineOpts(const CodeGenOptions &CodeGenOpts) {
SmallVector<const char *, 16> BackendArgs;
BackendArgs.push_back("clang"); // Fake program name.
@@ -961,7 +566,6 @@ bool EmitAssemblyHelper::AddEmitPasses(legacy::PassManager &CodeGenPasses,
raw_pwrite_stream &OS,
raw_pwrite_stream *DwoOS) {
// Add LibraryInfo.
- llvm::Triple TargetTriple(TheModule->getTargetTriple());
std::unique_ptr<TargetLibraryInfoImpl> TLII(
createTLII(TargetTriple, CodeGenOpts));
CodeGenPasses.add(new TargetLibraryInfoWrapperPass(*TLII));
@@ -985,139 +589,6 @@ bool EmitAssemblyHelper::AddEmitPasses(legacy::PassManager &CodeGenPasses,
return true;
}
-void EmitAssemblyHelper::EmitAssemblyWithLegacyPassManager(
- BackendAction Action, std::unique_ptr<raw_pwrite_stream> OS) {
- TimeRegion Region(CodeGenOpts.TimePasses ? &CodeGenerationTime : nullptr);
-
- setCommandLineOpts(CodeGenOpts);
-
- bool UsesCodeGen = actionRequiresCodeGen(Action);
- CreateTargetMachine(UsesCodeGen);
-
- if (UsesCodeGen && !TM)
- return;
- if (TM)
- TheModule->setDataLayout(TM->createDataLayout());
-
- DebugifyCustomPassManager PerModulePasses;
- DebugInfoPerPassMap DIPreservationMap;
- if (CodeGenOpts.EnableDIPreservationVerify) {
- PerModulePasses.setDebugifyMode(DebugifyMode::OriginalDebugInfo);
- PerModulePasses.setDIPreservationMap(DIPreservationMap);
-
- if (!CodeGenOpts.DIBugsReportFilePath.empty())
- PerModulePasses.setOrigDIVerifyBugsReportFilePath(
- CodeGenOpts.DIBugsReportFilePath);
- }
- PerModulePasses.add(
- createTargetTransformInfoWrapperPass(getTargetIRAnalysis()));
-
- legacy::FunctionPassManager PerFunctionPasses(TheModule);
- PerFunctionPasses.add(
- createTargetTransformInfoWrapperPass(getTargetIRAnalysis()));
-
- CreatePasses(PerModulePasses, PerFunctionPasses);
-
- // Add a verifier pass if requested. We don't have to do this if the action
- // requires code generation because there will already be a verifier pass in
- // the code-generation pipeline.
- if (!UsesCodeGen && CodeGenOpts.VerifyModule)
- PerModulePasses.add(createVerifierPass());
-
- legacy::PassManager CodeGenPasses;
- CodeGenPasses.add(
- createTargetTransformInfoWrapperPass(getTargetIRAnalysis()));
-
- std::unique_ptr<llvm::ToolOutputFile> ThinLinkOS, DwoOS;
-
- switch (Action) {
- case Backend_EmitNothing:
- break;
-
- case Backend_EmitBC:
- if (CodeGenOpts.PrepareForThinLTO && !CodeGenOpts.DisableLLVMPasses) {
- if (!CodeGenOpts.ThinLinkBitcodeFile.empty()) {
- ThinLinkOS = openOutputFile(CodeGenOpts.ThinLinkBitcodeFile);
- if (!ThinLinkOS)
- return;
- }
- if (!TheModule->getModuleFlag("EnableSplitLTOUnit"))
- TheModule->addModuleFlag(Module::Error, "EnableSplitLTOUnit",
- CodeGenOpts.EnableSplitLTOUnit);
- PerModulePasses.add(createWriteThinLTOBitcodePass(
- *OS, ThinLinkOS ? &ThinLinkOS->os() : nullptr));
- } else {
- // Emit a module summary by default for Regular LTO except for ld64
- // targets
- bool EmitLTOSummary =
- (CodeGenOpts.PrepareForLTO &&
- !CodeGenOpts.DisableLLVMPasses &&
- llvm::Triple(TheModule->getTargetTriple()).getVendor() !=
- llvm::Triple::Apple);
- if (EmitLTOSummary) {
- if (!TheModule->getModuleFlag("ThinLTO"))
- TheModule->addModuleFlag(Module::Error, "ThinLTO", uint32_t(0));
- if (!TheModule->getModuleFlag("EnableSplitLTOUnit"))
- TheModule->addModuleFlag(Module::Error, "EnableSplitLTOUnit",
- uint32_t(1));
- }
-
- PerModulePasses.add(createBitcodeWriterPass(
- *OS, CodeGenOpts.EmitLLVMUseLists, EmitLTOSummary));
- }
- break;
-
- case Backend_EmitLL:
- PerModulePasses.add(
- createPrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists));
- break;
-
- default:
- if (!CodeGenOpts.SplitDwarfOutput.empty()) {
- DwoOS = openOutputFile(CodeGenOpts.SplitDwarfOutput);
- if (!DwoOS)
- return;
- }
- if (!AddEmitPasses(CodeGenPasses, Action, *OS,
- DwoOS ? &DwoOS->os() : nullptr))
- return;
- }
-
- // Before executing passes, print the final values of the LLVM options.
- cl::PrintOptionValues();
-
- // Run passes. For now we do all passes at once, but eventually we
- // would like to have the option of streaming code generation.
-
- {
- PrettyStackTraceString CrashInfo("Per-function optimization");
- llvm::TimeTraceScope TimeScope("PerFunctionPasses");
-
- PerFunctionPasses.doInitialization();
- for (Function &F : *TheModule)
- if (!F.isDeclaration())
- PerFunctionPasses.run(F);
- PerFunctionPasses.doFinalization();
- }
-
- {
- PrettyStackTraceString CrashInfo("Per-module optimization passes");
- llvm::TimeTraceScope TimeScope("PerModulePasses");
- PerModulePasses.run(*TheModule);
- }
-
- {
- PrettyStackTraceString CrashInfo("Code generation");
- llvm::TimeTraceScope TimeScope("CodeGenPasses");
- CodeGenPasses.run(*TheModule);
- }
-
- if (ThinLinkOS)
- ThinLinkOS->keep();
- if (DwoOS)
- DwoOS->keep();
-}
-
static OptimizationLevel mapToLevel(const CodeGenOptions &Opts) {
switch (Opts.OptimizationLevel) {
default:
@@ -1205,7 +676,6 @@ static void addSanitizers(const Triple &TargetTriple,
Opts.Recover = CodeGenOpts.SanitizeRecover.has(Mask);
Opts.UseAfterScope = CodeGenOpts.SanitizeAddressUseAfterScope;
Opts.UseAfterReturn = CodeGenOpts.getSanitizeAddressUseAfterReturn();
- MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
MPM.addPass(ModuleAddressSanitizerPass(
Opts, UseGlobalGC, UseOdrIndicator, DestructorKind));
}
@@ -1270,7 +740,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
assert(!CodeGenOpts.hasProfileCSIRUse() &&
"Cannot have both CSProfileUse pass and CSProfileGen pass at "
"the same time");
- if (PGOOpt.hasValue()) {
+ if (PGOOpt) {
assert(PGOOpt->Action != PGOOptions::IRInstr &&
PGOOpt->Action != PGOOptions::SampleUse &&
"Cannot run CSProfileGen pass with ProfileGen or SampleUse "
@@ -1334,7 +804,6 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
// Register the target library analysis directly and give it a customized
// preset TLI.
- Triple TargetTriple(TheModule->getTargetTriple());
std::unique_ptr<TargetLibraryInfoImpl> TLII(
createTLII(TargetTriple, CodeGenOpts));
FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
@@ -1468,10 +937,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
} else {
// Emit a module summary by default for Regular LTO except for ld64
// targets
- bool EmitLTOSummary =
- (CodeGenOpts.PrepareForLTO && !CodeGenOpts.DisableLLVMPasses &&
- llvm::Triple(TheModule->getTargetTriple()).getVendor() !=
- llvm::Triple::Apple);
+ bool EmitLTOSummary = shouldEmitRegularLTOSummary();
if (EmitLTOSummary) {
if (!TheModule->getModuleFlag("ThinLTO"))
TheModule->addModuleFlag(Module::Error, "ThinLTO", uint32_t(0));
@@ -1536,14 +1002,6 @@ void EmitAssemblyHelper::RunCodegenPipeline(
}
}
-/// A clean version of `EmitAssembly` that uses the new pass manager.
-///
-/// Not all features are currently supported in this system, but where
-/// necessary it falls back to the legacy pass manager to at least provide
-/// basic functionality.
-///
-/// This API is planned to have its functionality finished and then to replace
-/// `EmitAssembly` at some point in the future when the default switches.
void EmitAssemblyHelper::EmitAssembly(BackendAction Action,
std::unique_ptr<raw_pwrite_stream> OS) {
TimeRegion Region(CodeGenOpts.TimePasses ? &CodeGenerationTime : nullptr);
@@ -1631,7 +1089,6 @@ static void runThinLTOBackend(
}
Conf.ProfileRemapping = std::move(ProfileRemapping);
- Conf.UseNewPM = !CGOpts.LegacyPassManager;
Conf.DebugPassManager = CGOpts.DebugPassManager;
Conf.RemarksWithHotness = CGOpts.DiagnosticsWithHotness;
Conf.RemarksFilename = CGOpts.OptRecordFile;
@@ -1721,11 +1178,7 @@ void clang::EmitBackendOutput(DiagnosticsEngine &Diags,
}
EmitAssemblyHelper AsmHelper(Diags, HeaderOpts, CGOpts, TOpts, LOpts, M);
-
- if (CGOpts.LegacyPassManager)
- AsmHelper.EmitAssemblyWithLegacyPassManager(Action, std::move(OS));
- else
- AsmHelper.EmitAssembly(Action, std::move(OS));
+ AsmHelper.EmitAssembly(Action, std::move(OS));
// Verify clang's TargetInfo DataLayout against the LLVM TargetMachine's
// DataLayout.
@@ -1758,24 +1211,16 @@ void clang::EmbedObject(llvm::Module *M, const CodeGenOptions &CGOpts,
return;
for (StringRef OffloadObject : CGOpts.OffloadObjects) {
- if (OffloadObject.count(',') != 1) {
- Diags.Report(Diags.getCustomDiagID(
- DiagnosticsEngine::Error, "Invalid string pair for embedding '%0'"))
- << OffloadObject;
- return;
- }
- auto FilenameAndSection = OffloadObject.split(',');
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> ObjectOrErr =
- llvm::MemoryBuffer::getFileOrSTDIN(std::get<0>(FilenameAndSection));
+ llvm::MemoryBuffer::getFileOrSTDIN(OffloadObject);
if (std::error_code EC = ObjectOrErr.getError()) {
auto DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
"could not open '%0' for embedding");
- Diags.Report(DiagID) << std::get<0>(FilenameAndSection);
+ Diags.Report(DiagID) << OffloadObject;
return;
}
- SmallString<128> SectionName(
- {".llvm.offloading.", std::get<1>(FilenameAndSection)});
- llvm::embedBufferInModule(*M, **ObjectOrErr, SectionName);
+ llvm::embedBufferInModule(*M, **ObjectOrErr, ".llvm.offloading",
+ Align(object::OffloadBinary::getAlignment()));
}
}
diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index 10569ae2c3f9..dee0cb64be97 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -86,15 +86,14 @@ namespace {
lvalue.getAlignment();
VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
CGF.Int8Ty, VoidPtrAddr, OffsetInChars.getQuantity());
+ llvm::Type *IntTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- VoidPtrAddr,
- CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
- "atomic_bitfield_base");
+ VoidPtrAddr, IntTy->getPointerTo(), "atomic_bitfield_base");
BFI = OrigBFI;
BFI.Offset = Offset;
BFI.StorageSize = AtomicSizeInBits;
BFI.StorageOffset += OffsetInChars;
- LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
+ LVal = LValue::MakeBitfield(Address(Addr, IntTy, lvalue.getAlignment()),
BFI, lvalue.getType(), lvalue.getBaseInfo(),
lvalue.getTBAAInfo());
AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
@@ -149,7 +148,16 @@ namespace {
return LVal.getExtVectorPointer();
}
Address getAtomicAddress() const {
- return Address(getAtomicPointer(), getAtomicAlignment());
+ llvm::Type *ElTy;
+ if (LVal.isSimple())
+ ElTy = LVal.getAddress(CGF).getElementType();
+ else if (LVal.isBitField())
+ ElTy = LVal.getBitFieldAddress().getElementType();
+ else if (LVal.isVectorElt())
+ ElTy = LVal.getVectorAddress().getElementType();
+ else
+ ElTy = LVal.getExtVectorAddress().getElementType();
+ return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
}
Address getAtomicAddressAsAtomicIntPointer() const {
@@ -296,7 +304,8 @@ Address AtomicInfo::CreateTempAlloca() const {
// Cast to pointer to value type for bitfields.
if (LVal.isBitField())
return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- TempAlloca, getAtomicAddress().getType());
+ TempAlloca, getAtomicAddress().getType(),
+ getAtomicAddress().getElementType());
return TempAlloca;
}
@@ -779,9 +788,9 @@ AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
ValTy =
CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
- llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
- SizeInBits)->getPointerTo();
- Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
+ llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits);
+ Address Ptr = Address(CGF.Builder.CreateBitCast(Val, ITy->getPointerTo()),
+ ITy, Align);
Val = CGF.EmitLoadOfScalar(Ptr, false,
CGF.getContext().getPointerType(ValTy),
Loc);
diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp
index 1f1de3df857c..ff6ca0914e0d 100644
--- a/clang/lib/CodeGen/CGBlocks.cpp
+++ b/clang/lib/CodeGen/CGBlocks.cpp
@@ -216,8 +216,9 @@ static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
llvm::Constant *disposeHelper = buildDisposeHelper(CGM, blockInfo);
elements.add(disposeHelper);
- if (cast<llvm::Function>(copyHelper->getOperand(0))->hasInternalLinkage() ||
- cast<llvm::Function>(disposeHelper->getOperand(0))
+ if (cast<llvm::Function>(copyHelper->stripPointerCasts())
+ ->hasInternalLinkage() ||
+ cast<llvm::Function>(disposeHelper->stripPointerCasts())
->hasInternalLinkage())
hasInternalHelper = true;
}
@@ -1105,7 +1106,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
if (IsOpenCL) {
CGM.getOpenCLRuntime().recordBlockInfo(blockInfo.BlockExpression, InvokeFn,
- result);
+ result, blockInfo.StructureType);
}
return result;
@@ -1160,8 +1161,7 @@ llvm::Type *CodeGenModule::getGenericBlockLiteralType() {
SmallVector<llvm::Type *, 8> StructFields(
{IntTy, IntTy, getOpenCLRuntime().getGenericVoidPointerType()});
if (auto *Helper = getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
- for (auto I : Helper->getCustomFieldTypes())
- StructFields.push_back(I);
+ llvm::append_range(StructFields, Helper->getCustomFieldTypes());
}
GenericBlockLiteralType = llvm::StructType::create(
StructFields, "struct.__opencl_block_literal_generic");
@@ -1263,10 +1263,9 @@ Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable) {
// to byref*.
auto &byrefInfo = getBlockByrefInfo(variable);
- addr = Address(Builder.CreateLoad(addr), byrefInfo.ByrefAlignment);
+ addr = Address(Builder.CreateLoad(addr), Int8Ty, byrefInfo.ByrefAlignment);
- auto byrefPointerType = llvm::PointerType::get(byrefInfo.Type, 0);
- addr = Builder.CreateBitCast(addr, byrefPointerType, "byref.addr");
+ addr = Builder.CreateElementBitCast(addr, byrefInfo.Type, "byref.addr");
addr = emitBlockByrefAddress(addr, byrefInfo, /*follow*/ true,
variable->getName());
@@ -1402,7 +1401,8 @@ static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
if (CGM.getContext().getLangOpts().OpenCL)
CGM.getOpenCLRuntime().recordBlockInfo(
blockInfo.BlockExpression,
- cast<llvm::Function>(blockFn->stripPointerCasts()), Result);
+ cast<llvm::Function>(blockFn->stripPointerCasts()), Result,
+ literal->getValueType());
return Result;
}
@@ -1441,15 +1441,12 @@ void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D,
Address CodeGenFunction::LoadBlockStruct() {
assert(BlockInfo && "not in a block invocation function!");
assert(BlockPointer && "no block pointer set!");
- return Address(BlockPointer, BlockInfo->BlockAlign);
+ return Address(BlockPointer, BlockInfo->StructureType, BlockInfo->BlockAlign);
}
-llvm::Function *
-CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
- const CGBlockInfo &blockInfo,
- const DeclMapTy &ldm,
- bool IsLambdaConversionToBlock,
- bool BuildGlobalBlock) {
+llvm::Function *CodeGenFunction::GenerateBlockFunction(
+ GlobalDecl GD, const CGBlockInfo &blockInfo, const DeclMapTy &ldm,
+ bool IsLambdaConversionToBlock, bool BuildGlobalBlock) {
const BlockDecl *blockDecl = blockInfo.getBlockDecl();
CurGD = GD;
@@ -1940,15 +1937,15 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
StartFunction(GlobalDecl(), ReturnTy, Fn, FI, args);
auto AL = ApplyDebugLocation::CreateArtificial(*this);
- llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
-
Address src = GetAddrOfLocalVar(&SrcDecl);
- src = Address(Builder.CreateLoad(src), blockInfo.BlockAlign);
- src = Builder.CreateBitCast(src, structPtrTy, "block.source");
+ src = Address(Builder.CreateLoad(src), Int8Ty, blockInfo.BlockAlign);
+ src = Builder.CreateElementBitCast(src, blockInfo.StructureType,
+ "block.source");
Address dst = GetAddrOfLocalVar(&DstDecl);
- dst = Address(Builder.CreateLoad(dst), blockInfo.BlockAlign);
- dst = Builder.CreateBitCast(dst, structPtrTy, "block.dest");
+ dst = Address(Builder.CreateLoad(dst), Int8Ty, blockInfo.BlockAlign);
+ dst =
+ Builder.CreateElementBitCast(dst, blockInfo.StructureType, "block.dest");
for (auto &capture : blockInfo.SortedCaptures) {
if (capture.isConstantOrTrivial())
@@ -2130,11 +2127,9 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
auto AL = ApplyDebugLocation::CreateArtificial(*this);
- llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
-
Address src = GetAddrOfLocalVar(&SrcDecl);
- src = Address(Builder.CreateLoad(src), blockInfo.BlockAlign);
- src = Builder.CreateBitCast(src, structPtrTy, "block");
+ src = Address(Builder.CreateLoad(src), Int8Ty, blockInfo.BlockAlign);
+ src = Builder.CreateElementBitCast(src, blockInfo.StructureType, "block");
CodeGenFunction::RunCleanupsScope cleanups(*this);
@@ -2171,9 +2166,9 @@ public:
void emitCopy(CodeGenFunction &CGF, Address destField,
Address srcField) override {
- destField = CGF.Builder.CreateBitCast(destField, CGF.VoidPtrTy);
+ destField = CGF.Builder.CreateElementBitCast(destField, CGF.Int8Ty);
- srcField = CGF.Builder.CreateBitCast(srcField, CGF.VoidPtrPtrTy);
+ srcField = CGF.Builder.CreateElementBitCast(srcField, CGF.Int8PtrTy);
llvm::Value *srcValue = CGF.Builder.CreateLoad(srcField);
unsigned flags = (Flags | BLOCK_BYREF_CALLER).getBitMask();
@@ -2186,7 +2181,7 @@ public:
}
void emitDispose(CodeGenFunction &CGF, Address field) override {
- field = CGF.Builder.CreateBitCast(field, CGF.Int8PtrTy->getPointerTo(0));
+ field = CGF.Builder.CreateElementBitCast(field, CGF.Int8PtrTy);
llvm::Value *value = CGF.Builder.CreateLoad(field);
CGF.BuildBlockRelease(value, Flags | BLOCK_BYREF_CALLER, false);
@@ -2376,23 +2371,21 @@ generateByrefCopyHelper(CodeGenFunction &CGF, const BlockByrefInfo &byrefInfo,
auto AL = ApplyDebugLocation::CreateArtificial(CGF);
if (generator.needsCopy()) {
- llvm::Type *byrefPtrType = byrefInfo.Type->getPointerTo(0);
-
// dst->x
Address destField = CGF.GetAddrOfLocalVar(&Dst);
- destField = Address(CGF.Builder.CreateLoad(destField),
+ destField = Address(CGF.Builder.CreateLoad(destField), CGF.Int8Ty,
byrefInfo.ByrefAlignment);
- destField = CGF.Builder.CreateBitCast(destField, byrefPtrType);
- destField = CGF.emitBlockByrefAddress(destField, byrefInfo, false,
- "dest-object");
+ destField = CGF.Builder.CreateElementBitCast(destField, byrefInfo.Type);
+ destField =
+ CGF.emitBlockByrefAddress(destField, byrefInfo, false, "dest-object");
// src->x
Address srcField = CGF.GetAddrOfLocalVar(&Src);
- srcField = Address(CGF.Builder.CreateLoad(srcField),
+ srcField = Address(CGF.Builder.CreateLoad(srcField), CGF.Int8Ty,
byrefInfo.ByrefAlignment);
- srcField = CGF.Builder.CreateBitCast(srcField, byrefPtrType);
- srcField = CGF.emitBlockByrefAddress(srcField, byrefInfo, false,
- "src-object");
+ srcField = CGF.Builder.CreateElementBitCast(srcField, byrefInfo.Type);
+ srcField =
+ CGF.emitBlockByrefAddress(srcField, byrefInfo, false, "src-object");
generator.emitCopy(CGF, destField, srcField);
}
@@ -2446,9 +2439,9 @@ generateByrefDisposeHelper(CodeGenFunction &CGF,
if (generator.needsDispose()) {
Address addr = CGF.GetAddrOfLocalVar(&Src);
- addr = Address(CGF.Builder.CreateLoad(addr), byrefInfo.ByrefAlignment);
- auto byrefPtrType = byrefInfo.Type->getPointerTo(0);
- addr = CGF.Builder.CreateBitCast(addr, byrefPtrType);
+ addr = Address(CGF.Builder.CreateLoad(addr), CGF.Int8Ty,
+ byrefInfo.ByrefAlignment);
+ addr = CGF.Builder.CreateElementBitCast(addr, byrefInfo.Type);
addr = CGF.emitBlockByrefAddress(addr, byrefInfo, false, "object");
generator.emitDispose(CGF, addr);
@@ -2594,7 +2587,8 @@ Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
// Chase the forwarding address if requested.
if (followForward) {
Address forwardingAddr = Builder.CreateStructGEP(baseAddr, 1, "forwarding");
- baseAddr = Address(Builder.CreateLoad(forwardingAddr), info.ByrefAlignment);
+ baseAddr = Address(Builder.CreateLoad(forwardingAddr), info.Type,
+ info.ByrefAlignment);
}
return Builder.CreateStructGEP(baseAddr, info.FieldIndex, name);
diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h
index 7c9f41e84eaf..2fcfea64ede6 100644
--- a/clang/lib/CodeGen/CGBuilder.h
+++ b/clang/lib/CodeGen/CGBuilder.h
@@ -9,10 +9,11 @@
#ifndef LLVM_CLANG_LIB_CODEGEN_CGBUILDER_H
#define LLVM_CLANG_LIB_CODEGEN_CGBUILDER_H
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/IRBuilder.h"
#include "Address.h"
#include "CodeGenTypeCache.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Type.h"
namespace clang {
namespace CodeGen {
@@ -31,6 +32,7 @@ public:
void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name,
llvm::BasicBlock *BB,
llvm::BasicBlock::iterator InsertPt) const override;
+
private:
CodeGenFunction *CGF = nullptr;
};
@@ -44,17 +46,18 @@ class CGBuilderTy : public CGBuilderBaseTy {
/// Storing a reference to the type cache here makes it a lot easier
/// to build natural-feeling, target-specific IR.
const CodeGenTypeCache &TypeCache;
+
public:
CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::LLVMContext &C)
- : CGBuilderBaseTy(C), TypeCache(TypeCache) {}
- CGBuilderTy(const CodeGenTypeCache &TypeCache,
- llvm::LLVMContext &C, const llvm::ConstantFolder &F,
+ : CGBuilderBaseTy(C), TypeCache(TypeCache) {}
+ CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::LLVMContext &C,
+ const llvm::ConstantFolder &F,
const CGBuilderInserterTy &Inserter)
- : CGBuilderBaseTy(C, F, Inserter), TypeCache(TypeCache) {}
+ : CGBuilderBaseTy(C, F, Inserter), TypeCache(TypeCache) {}
CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::Instruction *I)
- : CGBuilderBaseTy(I), TypeCache(TypeCache) {}
+ : CGBuilderBaseTy(I), TypeCache(TypeCache) {}
CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::BasicBlock *BB)
- : CGBuilderBaseTy(BB), TypeCache(TypeCache) {}
+ : CGBuilderBaseTy(BB), TypeCache(TypeCache) {}
llvm::ConstantInt *getSize(CharUnits N) {
return llvm::ConstantInt::get(TypeCache.SizeTy, N.getQuantity());
@@ -101,7 +104,8 @@ public:
using CGBuilderBaseTy::CreateAlignedStore;
llvm::StoreInst *CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr,
- CharUnits Align, bool IsVolatile = false) {
+ CharUnits Align,
+ bool IsVolatile = false) {
return CreateAlignedStore(Val, Addr, Align.getAsAlign(), IsVolatile);
}
@@ -150,18 +154,13 @@ public:
Ordering, SSID);
}
- using CGBuilderBaseTy::CreateBitCast;
- Address CreateBitCast(Address Addr, llvm::Type *Ty,
- const llvm::Twine &Name = "") {
- return Address(CreateBitCast(Addr.getPointer(), Ty, Name),
- Addr.getAlignment());
- }
-
using CGBuilderBaseTy::CreateAddrSpaceCast;
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty,
const llvm::Twine &Name = "") {
- return Address(CreateAddrSpaceCast(Addr.getPointer(), Ty, Name),
- Addr.getAlignment());
+ assert(cast<llvm::PointerType>(Ty)->isOpaqueOrPointeeTypeMatches(
+ Addr.getElementType()) &&
+ "Should not change the element type");
+ return Addr.withPointer(CreateAddrSpaceCast(Addr.getPointer(), Ty, Name));
}
/// Cast the element type of the given address to a different type,
@@ -169,16 +168,17 @@ public:
Address CreateElementBitCast(Address Addr, llvm::Type *Ty,
const llvm::Twine &Name = "") {
auto *PtrTy = Ty->getPointerTo(Addr.getAddressSpace());
- return Address(CreateBitCast(Addr.getPointer(), PtrTy, Name),
- Ty, Addr.getAlignment());
+ return Address(CreateBitCast(Addr.getPointer(), PtrTy, Name), Ty,
+ Addr.getAlignment());
}
using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast;
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty,
+ llvm::Type *ElementTy,
const llvm::Twine &Name = "") {
llvm::Value *Ptr =
- CreatePointerBitCastOrAddrSpaceCast(Addr.getPointer(), Ty, Name);
- return Address(Ptr, Addr.getAlignment());
+ CreatePointerBitCastOrAddrSpaceCast(Addr.getPointer(), Ty, Name);
+ return Address(Ptr, ElementTy, Addr.getAlignment());
}
/// Given
@@ -196,10 +196,10 @@ public:
const llvm::StructLayout *Layout = DL.getStructLayout(ElTy);
auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index));
- return Address(CreateStructGEP(Addr.getElementType(),
- Addr.getPointer(), Index, Name),
- ElTy->getElementType(Index),
- Addr.getAlignment().alignmentAtOffset(Offset));
+ return Address(
+ CreateStructGEP(Addr.getElementType(), Addr.getPointer(), Index, Name),
+ ElTy->getElementType(Index),
+ Addr.getAlignment().alignmentAtOffset(Offset));
}
/// Given
@@ -267,10 +267,10 @@ public:
CharUnits EltSize =
CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType()));
- return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), Index,
- Name),
- Addr.getElementType(),
- Addr.getAlignment().alignmentOfArrayElement(EltSize));
+ return Address(
+ CreateGEP(Addr.getElementType(), Addr.getPointer(), Index, Name),
+ Addr.getElementType(),
+ Addr.getAlignment().alignmentOfArrayElement(EltSize));
}
/// Given a pointer to i8, adjust it by a given constant offset.
@@ -344,9 +344,16 @@ public:
Dest.getAlignment().getAsAlign(), IsVolatile);
}
+ using CGBuilderBaseTy::CreateMemSetInline;
+ llvm::CallInst *CreateMemSetInline(Address Dest, llvm::Value *Value,
+ uint64_t Size) {
+ return CreateMemSetInline(Dest.getPointer(),
+ Dest.getAlignment().getAsAlign(), Value,
+ getInt64(Size));
+ }
+
using CGBuilderBaseTy::CreatePreserveStructAccessIndex;
- Address CreatePreserveStructAccessIndex(Address Addr,
- unsigned Index,
+ Address CreatePreserveStructAccessIndex(Address Addr, unsigned Index,
unsigned FieldIndex,
llvm::MDNode *DbgInfo) {
llvm::StructType *ElTy = cast<llvm::StructType>(Addr.getElementType());
@@ -366,7 +373,7 @@ public:
}
};
-} // end namespace CodeGen
-} // end namespace clang
+} // end namespace CodeGen
+} // end namespace clang
#endif
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index d071c7a5b4a4..8c7ee6b078f2 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -45,6 +45,7 @@
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/IR/IntrinsicsS390.h"
+#include "llvm/IR/IntrinsicsVE.h"
#include "llvm/IR/IntrinsicsWebAssembly.h"
#include "llvm/IR/IntrinsicsX86.h"
#include "llvm/IR/MDBuilder.h"
@@ -373,7 +374,7 @@ static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,
llvm::Type *Int128PtrTy = Int128Ty->getPointerTo();
Destination = CGF.Builder.CreateBitCast(Destination, Int128PtrTy);
Address ComparandResult(CGF.Builder.CreateBitCast(ComparandPtr, Int128PtrTy),
- CGF.getContext().toCharUnitsFromBits(128));
+ Int128Ty, CGF.getContext().toCharUnitsFromBits(128));
// (((i128)hi) << 64) | ((i128)lo)
ExchangeHigh = CGF.Builder.CreateZExt(ExchangeHigh, Int128Ty);
@@ -961,7 +962,7 @@ static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
Value *BitBaseI8 = CGF.Builder.CreatePointerCast(BitBase, CGF.Int8PtrTy);
Address ByteAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, BitBaseI8,
ByteIndex, "bittest.byteaddr"),
- CharUnits::One());
+ CGF.Int8Ty, CharUnits::One());
Value *PosLow =
CGF.Builder.CreateAnd(CGF.Builder.CreateTrunc(BitPos, CGF.Int8Ty),
llvm::ConstantInt::get(CGF.Int8Ty, 0x7));
@@ -1168,141 +1169,141 @@ translateArmToMsvcIntrin(unsigned BuiltinID) {
switch (BuiltinID) {
default:
return None;
- case ARM::BI_BitScanForward:
- case ARM::BI_BitScanForward64:
+ case clang::ARM::BI_BitScanForward:
+ case clang::ARM::BI_BitScanForward64:
return MSVCIntrin::_BitScanForward;
- case ARM::BI_BitScanReverse:
- case ARM::BI_BitScanReverse64:
+ case clang::ARM::BI_BitScanReverse:
+ case clang::ARM::BI_BitScanReverse64:
return MSVCIntrin::_BitScanReverse;
- case ARM::BI_InterlockedAnd64:
+ case clang::ARM::BI_InterlockedAnd64:
return MSVCIntrin::_InterlockedAnd;
- case ARM::BI_InterlockedExchange64:
+ case clang::ARM::BI_InterlockedExchange64:
return MSVCIntrin::_InterlockedExchange;
- case ARM::BI_InterlockedExchangeAdd64:
+ case clang::ARM::BI_InterlockedExchangeAdd64:
return MSVCIntrin::_InterlockedExchangeAdd;
- case ARM::BI_InterlockedExchangeSub64:
+ case clang::ARM::BI_InterlockedExchangeSub64:
return MSVCIntrin::_InterlockedExchangeSub;
- case ARM::BI_InterlockedOr64:
+ case clang::ARM::BI_InterlockedOr64:
return MSVCIntrin::_InterlockedOr;
- case ARM::BI_InterlockedXor64:
+ case clang::ARM::BI_InterlockedXor64:
return MSVCIntrin::_InterlockedXor;
- case ARM::BI_InterlockedDecrement64:
+ case clang::ARM::BI_InterlockedDecrement64:
return MSVCIntrin::_InterlockedDecrement;
- case ARM::BI_InterlockedIncrement64:
+ case clang::ARM::BI_InterlockedIncrement64:
return MSVCIntrin::_InterlockedIncrement;
- case ARM::BI_InterlockedExchangeAdd8_acq:
- case ARM::BI_InterlockedExchangeAdd16_acq:
- case ARM::BI_InterlockedExchangeAdd_acq:
- case ARM::BI_InterlockedExchangeAdd64_acq:
+ case clang::ARM::BI_InterlockedExchangeAdd8_acq:
+ case clang::ARM::BI_InterlockedExchangeAdd16_acq:
+ case clang::ARM::BI_InterlockedExchangeAdd_acq:
+ case clang::ARM::BI_InterlockedExchangeAdd64_acq:
return MSVCIntrin::_InterlockedExchangeAdd_acq;
- case ARM::BI_InterlockedExchangeAdd8_rel:
- case ARM::BI_InterlockedExchangeAdd16_rel:
- case ARM::BI_InterlockedExchangeAdd_rel:
- case ARM::BI_InterlockedExchangeAdd64_rel:
+ case clang::ARM::BI_InterlockedExchangeAdd8_rel:
+ case clang::ARM::BI_InterlockedExchangeAdd16_rel:
+ case clang::ARM::BI_InterlockedExchangeAdd_rel:
+ case clang::ARM::BI_InterlockedExchangeAdd64_rel:
return MSVCIntrin::_InterlockedExchangeAdd_rel;
- case ARM::BI_InterlockedExchangeAdd8_nf:
- case ARM::BI_InterlockedExchangeAdd16_nf:
- case ARM::BI_InterlockedExchangeAdd_nf:
- case ARM::BI_InterlockedExchangeAdd64_nf:
+ case clang::ARM::BI_InterlockedExchangeAdd8_nf:
+ case clang::ARM::BI_InterlockedExchangeAdd16_nf:
+ case clang::ARM::BI_InterlockedExchangeAdd_nf:
+ case clang::ARM::BI_InterlockedExchangeAdd64_nf:
return MSVCIntrin::_InterlockedExchangeAdd_nf;
- case ARM::BI_InterlockedExchange8_acq:
- case ARM::BI_InterlockedExchange16_acq:
- case ARM::BI_InterlockedExchange_acq:
- case ARM::BI_InterlockedExchange64_acq:
+ case clang::ARM::BI_InterlockedExchange8_acq:
+ case clang::ARM::BI_InterlockedExchange16_acq:
+ case clang::ARM::BI_InterlockedExchange_acq:
+ case clang::ARM::BI_InterlockedExchange64_acq:
return MSVCIntrin::_InterlockedExchange_acq;
- case ARM::BI_InterlockedExchange8_rel:
- case ARM::BI_InterlockedExchange16_rel:
- case ARM::BI_InterlockedExchange_rel:
- case ARM::BI_InterlockedExchange64_rel:
+ case clang::ARM::BI_InterlockedExchange8_rel:
+ case clang::ARM::BI_InterlockedExchange16_rel:
+ case clang::ARM::BI_InterlockedExchange_rel:
+ case clang::ARM::BI_InterlockedExchange64_rel:
return MSVCIntrin::_InterlockedExchange_rel;
- case ARM::BI_InterlockedExchange8_nf:
- case ARM::BI_InterlockedExchange16_nf:
- case ARM::BI_InterlockedExchange_nf:
- case ARM::BI_InterlockedExchange64_nf:
+ case clang::ARM::BI_InterlockedExchange8_nf:
+ case clang::ARM::BI_InterlockedExchange16_nf:
+ case clang::ARM::BI_InterlockedExchange_nf:
+ case clang::ARM::BI_InterlockedExchange64_nf:
return MSVCIntrin::_InterlockedExchange_nf;
- case ARM::BI_InterlockedCompareExchange8_acq:
- case ARM::BI_InterlockedCompareExchange16_acq:
- case ARM::BI_InterlockedCompareExchange_acq:
- case ARM::BI_InterlockedCompareExchange64_acq:
+ case clang::ARM::BI_InterlockedCompareExchange8_acq:
+ case clang::ARM::BI_InterlockedCompareExchange16_acq:
+ case clang::ARM::BI_InterlockedCompareExchange_acq:
+ case clang::ARM::BI_InterlockedCompareExchange64_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
- case ARM::BI_InterlockedCompareExchange8_rel:
- case ARM::BI_InterlockedCompareExchange16_rel:
- case ARM::BI_InterlockedCompareExchange_rel:
- case ARM::BI_InterlockedCompareExchange64_rel:
+ case clang::ARM::BI_InterlockedCompareExchange8_rel:
+ case clang::ARM::BI_InterlockedCompareExchange16_rel:
+ case clang::ARM::BI_InterlockedCompareExchange_rel:
+ case clang::ARM::BI_InterlockedCompareExchange64_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
- case ARM::BI_InterlockedCompareExchange8_nf:
- case ARM::BI_InterlockedCompareExchange16_nf:
- case ARM::BI_InterlockedCompareExchange_nf:
- case ARM::BI_InterlockedCompareExchange64_nf:
+ case clang::ARM::BI_InterlockedCompareExchange8_nf:
+ case clang::ARM::BI_InterlockedCompareExchange16_nf:
+ case clang::ARM::BI_InterlockedCompareExchange_nf:
+ case clang::ARM::BI_InterlockedCompareExchange64_nf:
return MSVCIntrin::_InterlockedCompareExchange_nf;
- case ARM::BI_InterlockedOr8_acq:
- case ARM::BI_InterlockedOr16_acq:
- case ARM::BI_InterlockedOr_acq:
- case ARM::BI_InterlockedOr64_acq:
+ case clang::ARM::BI_InterlockedOr8_acq:
+ case clang::ARM::BI_InterlockedOr16_acq:
+ case clang::ARM::BI_InterlockedOr_acq:
+ case clang::ARM::BI_InterlockedOr64_acq:
return MSVCIntrin::_InterlockedOr_acq;
- case ARM::BI_InterlockedOr8_rel:
- case ARM::BI_InterlockedOr16_rel:
- case ARM::BI_InterlockedOr_rel:
- case ARM::BI_InterlockedOr64_rel:
+ case clang::ARM::BI_InterlockedOr8_rel:
+ case clang::ARM::BI_InterlockedOr16_rel:
+ case clang::ARM::BI_InterlockedOr_rel:
+ case clang::ARM::BI_InterlockedOr64_rel:
return MSVCIntrin::_InterlockedOr_rel;
- case ARM::BI_InterlockedOr8_nf:
- case ARM::BI_InterlockedOr16_nf:
- case ARM::BI_InterlockedOr_nf:
- case ARM::BI_InterlockedOr64_nf:
+ case clang::ARM::BI_InterlockedOr8_nf:
+ case clang::ARM::BI_InterlockedOr16_nf:
+ case clang::ARM::BI_InterlockedOr_nf:
+ case clang::ARM::BI_InterlockedOr64_nf:
return MSVCIntrin::_InterlockedOr_nf;
- case ARM::BI_InterlockedXor8_acq:
- case ARM::BI_InterlockedXor16_acq:
- case ARM::BI_InterlockedXor_acq:
- case ARM::BI_InterlockedXor64_acq:
+ case clang::ARM::BI_InterlockedXor8_acq:
+ case clang::ARM::BI_InterlockedXor16_acq:
+ case clang::ARM::BI_InterlockedXor_acq:
+ case clang::ARM::BI_InterlockedXor64_acq:
return MSVCIntrin::_InterlockedXor_acq;
- case ARM::BI_InterlockedXor8_rel:
- case ARM::BI_InterlockedXor16_rel:
- case ARM::BI_InterlockedXor_rel:
- case ARM::BI_InterlockedXor64_rel:
+ case clang::ARM::BI_InterlockedXor8_rel:
+ case clang::ARM::BI_InterlockedXor16_rel:
+ case clang::ARM::BI_InterlockedXor_rel:
+ case clang::ARM::BI_InterlockedXor64_rel:
return MSVCIntrin::_InterlockedXor_rel;
- case ARM::BI_InterlockedXor8_nf:
- case ARM::BI_InterlockedXor16_nf:
- case ARM::BI_InterlockedXor_nf:
- case ARM::BI_InterlockedXor64_nf:
+ case clang::ARM::BI_InterlockedXor8_nf:
+ case clang::ARM::BI_InterlockedXor16_nf:
+ case clang::ARM::BI_InterlockedXor_nf:
+ case clang::ARM::BI_InterlockedXor64_nf:
return MSVCIntrin::_InterlockedXor_nf;
- case ARM::BI_InterlockedAnd8_acq:
- case ARM::BI_InterlockedAnd16_acq:
- case ARM::BI_InterlockedAnd_acq:
- case ARM::BI_InterlockedAnd64_acq:
+ case clang::ARM::BI_InterlockedAnd8_acq:
+ case clang::ARM::BI_InterlockedAnd16_acq:
+ case clang::ARM::BI_InterlockedAnd_acq:
+ case clang::ARM::BI_InterlockedAnd64_acq:
return MSVCIntrin::_InterlockedAnd_acq;
- case ARM::BI_InterlockedAnd8_rel:
- case ARM::BI_InterlockedAnd16_rel:
- case ARM::BI_InterlockedAnd_rel:
- case ARM::BI_InterlockedAnd64_rel:
+ case clang::ARM::BI_InterlockedAnd8_rel:
+ case clang::ARM::BI_InterlockedAnd16_rel:
+ case clang::ARM::BI_InterlockedAnd_rel:
+ case clang::ARM::BI_InterlockedAnd64_rel:
return MSVCIntrin::_InterlockedAnd_rel;
- case ARM::BI_InterlockedAnd8_nf:
- case ARM::BI_InterlockedAnd16_nf:
- case ARM::BI_InterlockedAnd_nf:
- case ARM::BI_InterlockedAnd64_nf:
+ case clang::ARM::BI_InterlockedAnd8_nf:
+ case clang::ARM::BI_InterlockedAnd16_nf:
+ case clang::ARM::BI_InterlockedAnd_nf:
+ case clang::ARM::BI_InterlockedAnd64_nf:
return MSVCIntrin::_InterlockedAnd_nf;
- case ARM::BI_InterlockedIncrement16_acq:
- case ARM::BI_InterlockedIncrement_acq:
- case ARM::BI_InterlockedIncrement64_acq:
+ case clang::ARM::BI_InterlockedIncrement16_acq:
+ case clang::ARM::BI_InterlockedIncrement_acq:
+ case clang::ARM::BI_InterlockedIncrement64_acq:
return MSVCIntrin::_InterlockedIncrement_acq;
- case ARM::BI_InterlockedIncrement16_rel:
- case ARM::BI_InterlockedIncrement_rel:
- case ARM::BI_InterlockedIncrement64_rel:
+ case clang::ARM::BI_InterlockedIncrement16_rel:
+ case clang::ARM::BI_InterlockedIncrement_rel:
+ case clang::ARM::BI_InterlockedIncrement64_rel:
return MSVCIntrin::_InterlockedIncrement_rel;
- case ARM::BI_InterlockedIncrement16_nf:
- case ARM::BI_InterlockedIncrement_nf:
- case ARM::BI_InterlockedIncrement64_nf:
+ case clang::ARM::BI_InterlockedIncrement16_nf:
+ case clang::ARM::BI_InterlockedIncrement_nf:
+ case clang::ARM::BI_InterlockedIncrement64_nf:
return MSVCIntrin::_InterlockedIncrement_nf;
- case ARM::BI_InterlockedDecrement16_acq:
- case ARM::BI_InterlockedDecrement_acq:
- case ARM::BI_InterlockedDecrement64_acq:
+ case clang::ARM::BI_InterlockedDecrement16_acq:
+ case clang::ARM::BI_InterlockedDecrement_acq:
+ case clang::ARM::BI_InterlockedDecrement64_acq:
return MSVCIntrin::_InterlockedDecrement_acq;
- case ARM::BI_InterlockedDecrement16_rel:
- case ARM::BI_InterlockedDecrement_rel:
- case ARM::BI_InterlockedDecrement64_rel:
+ case clang::ARM::BI_InterlockedDecrement16_rel:
+ case clang::ARM::BI_InterlockedDecrement_rel:
+ case clang::ARM::BI_InterlockedDecrement64_rel:
return MSVCIntrin::_InterlockedDecrement_rel;
- case ARM::BI_InterlockedDecrement16_nf:
- case ARM::BI_InterlockedDecrement_nf:
- case ARM::BI_InterlockedDecrement64_nf:
+ case clang::ARM::BI_InterlockedDecrement16_nf:
+ case clang::ARM::BI_InterlockedDecrement_nf:
+ case clang::ARM::BI_InterlockedDecrement64_nf:
return MSVCIntrin::_InterlockedDecrement_nf;
}
llvm_unreachable("must return from switch");
@@ -1314,149 +1315,149 @@ translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
switch (BuiltinID) {
default:
return None;
- case AArch64::BI_BitScanForward:
- case AArch64::BI_BitScanForward64:
+ case clang::AArch64::BI_BitScanForward:
+ case clang::AArch64::BI_BitScanForward64:
return MSVCIntrin::_BitScanForward;
- case AArch64::BI_BitScanReverse:
- case AArch64::BI_BitScanReverse64:
+ case clang::AArch64::BI_BitScanReverse:
+ case clang::AArch64::BI_BitScanReverse64:
return MSVCIntrin::_BitScanReverse;
- case AArch64::BI_InterlockedAnd64:
+ case clang::AArch64::BI_InterlockedAnd64:
return MSVCIntrin::_InterlockedAnd;
- case AArch64::BI_InterlockedExchange64:
+ case clang::AArch64::BI_InterlockedExchange64:
return MSVCIntrin::_InterlockedExchange;
- case AArch64::BI_InterlockedExchangeAdd64:
+ case clang::AArch64::BI_InterlockedExchangeAdd64:
return MSVCIntrin::_InterlockedExchangeAdd;
- case AArch64::BI_InterlockedExchangeSub64:
+ case clang::AArch64::BI_InterlockedExchangeSub64:
return MSVCIntrin::_InterlockedExchangeSub;
- case AArch64::BI_InterlockedOr64:
+ case clang::AArch64::BI_InterlockedOr64:
return MSVCIntrin::_InterlockedOr;
- case AArch64::BI_InterlockedXor64:
+ case clang::AArch64::BI_InterlockedXor64:
return MSVCIntrin::_InterlockedXor;
- case AArch64::BI_InterlockedDecrement64:
+ case clang::AArch64::BI_InterlockedDecrement64:
return MSVCIntrin::_InterlockedDecrement;
- case AArch64::BI_InterlockedIncrement64:
+ case clang::AArch64::BI_InterlockedIncrement64:
return MSVCIntrin::_InterlockedIncrement;
- case AArch64::BI_InterlockedExchangeAdd8_acq:
- case AArch64::BI_InterlockedExchangeAdd16_acq:
- case AArch64::BI_InterlockedExchangeAdd_acq:
- case AArch64::BI_InterlockedExchangeAdd64_acq:
+ case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
+ case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
+ case clang::AArch64::BI_InterlockedExchangeAdd_acq:
+ case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
return MSVCIntrin::_InterlockedExchangeAdd_acq;
- case AArch64::BI_InterlockedExchangeAdd8_rel:
- case AArch64::BI_InterlockedExchangeAdd16_rel:
- case AArch64::BI_InterlockedExchangeAdd_rel:
- case AArch64::BI_InterlockedExchangeAdd64_rel:
+ case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
+ case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
+ case clang::AArch64::BI_InterlockedExchangeAdd_rel:
+ case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
return MSVCIntrin::_InterlockedExchangeAdd_rel;
- case AArch64::BI_InterlockedExchangeAdd8_nf:
- case AArch64::BI_InterlockedExchangeAdd16_nf:
- case AArch64::BI_InterlockedExchangeAdd_nf:
- case AArch64::BI_InterlockedExchangeAdd64_nf:
+ case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
+ case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
+ case clang::AArch64::BI_InterlockedExchangeAdd_nf:
+ case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
return MSVCIntrin::_InterlockedExchangeAdd_nf;
- case AArch64::BI_InterlockedExchange8_acq:
- case AArch64::BI_InterlockedExchange16_acq:
- case AArch64::BI_InterlockedExchange_acq:
- case AArch64::BI_InterlockedExchange64_acq:
+ case clang::AArch64::BI_InterlockedExchange8_acq:
+ case clang::AArch64::BI_InterlockedExchange16_acq:
+ case clang::AArch64::BI_InterlockedExchange_acq:
+ case clang::AArch64::BI_InterlockedExchange64_acq:
return MSVCIntrin::_InterlockedExchange_acq;
- case AArch64::BI_InterlockedExchange8_rel:
- case AArch64::BI_InterlockedExchange16_rel:
- case AArch64::BI_InterlockedExchange_rel:
- case AArch64::BI_InterlockedExchange64_rel:
+ case clang::AArch64::BI_InterlockedExchange8_rel:
+ case clang::AArch64::BI_InterlockedExchange16_rel:
+ case clang::AArch64::BI_InterlockedExchange_rel:
+ case clang::AArch64::BI_InterlockedExchange64_rel:
return MSVCIntrin::_InterlockedExchange_rel;
- case AArch64::BI_InterlockedExchange8_nf:
- case AArch64::BI_InterlockedExchange16_nf:
- case AArch64::BI_InterlockedExchange_nf:
- case AArch64::BI_InterlockedExchange64_nf:
+ case clang::AArch64::BI_InterlockedExchange8_nf:
+ case clang::AArch64::BI_InterlockedExchange16_nf:
+ case clang::AArch64::BI_InterlockedExchange_nf:
+ case clang::AArch64::BI_InterlockedExchange64_nf:
return MSVCIntrin::_InterlockedExchange_nf;
- case AArch64::BI_InterlockedCompareExchange8_acq:
- case AArch64::BI_InterlockedCompareExchange16_acq:
- case AArch64::BI_InterlockedCompareExchange_acq:
- case AArch64::BI_InterlockedCompareExchange64_acq:
+ case clang::AArch64::BI_InterlockedCompareExchange8_acq:
+ case clang::AArch64::BI_InterlockedCompareExchange16_acq:
+ case clang::AArch64::BI_InterlockedCompareExchange_acq:
+ case clang::AArch64::BI_InterlockedCompareExchange64_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
- case AArch64::BI_InterlockedCompareExchange8_rel:
- case AArch64::BI_InterlockedCompareExchange16_rel:
- case AArch64::BI_InterlockedCompareExchange_rel:
- case AArch64::BI_InterlockedCompareExchange64_rel:
+ case clang::AArch64::BI_InterlockedCompareExchange8_rel:
+ case clang::AArch64::BI_InterlockedCompareExchange16_rel:
+ case clang::AArch64::BI_InterlockedCompareExchange_rel:
+ case clang::AArch64::BI_InterlockedCompareExchange64_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
- case AArch64::BI_InterlockedCompareExchange8_nf:
- case AArch64::BI_InterlockedCompareExchange16_nf:
- case AArch64::BI_InterlockedCompareExchange_nf:
- case AArch64::BI_InterlockedCompareExchange64_nf:
+ case clang::AArch64::BI_InterlockedCompareExchange8_nf:
+ case clang::AArch64::BI_InterlockedCompareExchange16_nf:
+ case clang::AArch64::BI_InterlockedCompareExchange_nf:
+ case clang::AArch64::BI_InterlockedCompareExchange64_nf:
return MSVCIntrin::_InterlockedCompareExchange_nf;
- case AArch64::BI_InterlockedCompareExchange128:
+ case clang::AArch64::BI_InterlockedCompareExchange128:
return MSVCIntrin::_InterlockedCompareExchange128;
- case AArch64::BI_InterlockedCompareExchange128_acq:
+ case clang::AArch64::BI_InterlockedCompareExchange128_acq:
return MSVCIntrin::_InterlockedCompareExchange128_acq;
- case AArch64::BI_InterlockedCompareExchange128_nf:
+ case clang::AArch64::BI_InterlockedCompareExchange128_nf:
return MSVCIntrin::_InterlockedCompareExchange128_nf;
- case AArch64::BI_InterlockedCompareExchange128_rel:
+ case clang::AArch64::BI_InterlockedCompareExchange128_rel:
return MSVCIntrin::_InterlockedCompareExchange128_rel;
- case AArch64::BI_InterlockedOr8_acq:
- case AArch64::BI_InterlockedOr16_acq:
- case AArch64::BI_InterlockedOr_acq:
- case AArch64::BI_InterlockedOr64_acq:
+ case clang::AArch64::BI_InterlockedOr8_acq:
+ case clang::AArch64::BI_InterlockedOr16_acq:
+ case clang::AArch64::BI_InterlockedOr_acq:
+ case clang::AArch64::BI_InterlockedOr64_acq:
return MSVCIntrin::_InterlockedOr_acq;
- case AArch64::BI_InterlockedOr8_rel:
- case AArch64::BI_InterlockedOr16_rel:
- case AArch64::BI_InterlockedOr_rel:
- case AArch64::BI_InterlockedOr64_rel:
+ case clang::AArch64::BI_InterlockedOr8_rel:
+ case clang::AArch64::BI_InterlockedOr16_rel:
+ case clang::AArch64::BI_InterlockedOr_rel:
+ case clang::AArch64::BI_InterlockedOr64_rel:
return MSVCIntrin::_InterlockedOr_rel;
- case AArch64::BI_InterlockedOr8_nf:
- case AArch64::BI_InterlockedOr16_nf:
- case AArch64::BI_InterlockedOr_nf:
- case AArch64::BI_InterlockedOr64_nf:
+ case clang::AArch64::BI_InterlockedOr8_nf:
+ case clang::AArch64::BI_InterlockedOr16_nf:
+ case clang::AArch64::BI_InterlockedOr_nf:
+ case clang::AArch64::BI_InterlockedOr64_nf:
return MSVCIntrin::_InterlockedOr_nf;
- case AArch64::BI_InterlockedXor8_acq:
- case AArch64::BI_InterlockedXor16_acq:
- case AArch64::BI_InterlockedXor_acq:
- case AArch64::BI_InterlockedXor64_acq:
+ case clang::AArch64::BI_InterlockedXor8_acq:
+ case clang::AArch64::BI_InterlockedXor16_acq:
+ case clang::AArch64::BI_InterlockedXor_acq:
+ case clang::AArch64::BI_InterlockedXor64_acq:
return MSVCIntrin::_InterlockedXor_acq;
- case AArch64::BI_InterlockedXor8_rel:
- case AArch64::BI_InterlockedXor16_rel:
- case AArch64::BI_InterlockedXor_rel:
- case AArch64::BI_InterlockedXor64_rel:
+ case clang::AArch64::BI_InterlockedXor8_rel:
+ case clang::AArch64::BI_InterlockedXor16_rel:
+ case clang::AArch64::BI_InterlockedXor_rel:
+ case clang::AArch64::BI_InterlockedXor64_rel:
return MSVCIntrin::_InterlockedXor_rel;
- case AArch64::BI_InterlockedXor8_nf:
- case AArch64::BI_InterlockedXor16_nf:
- case AArch64::BI_InterlockedXor_nf:
- case AArch64::BI_InterlockedXor64_nf:
+ case clang::AArch64::BI_InterlockedXor8_nf:
+ case clang::AArch64::BI_InterlockedXor16_nf:
+ case clang::AArch64::BI_InterlockedXor_nf:
+ case clang::AArch64::BI_InterlockedXor64_nf:
return MSVCIntrin::_InterlockedXor_nf;
- case AArch64::BI_InterlockedAnd8_acq:
- case AArch64::BI_InterlockedAnd16_acq:
- case AArch64::BI_InterlockedAnd_acq:
- case AArch64::BI_InterlockedAnd64_acq:
+ case clang::AArch64::BI_InterlockedAnd8_acq:
+ case clang::AArch64::BI_InterlockedAnd16_acq:
+ case clang::AArch64::BI_InterlockedAnd_acq:
+ case clang::AArch64::BI_InterlockedAnd64_acq:
return MSVCIntrin::_InterlockedAnd_acq;
- case AArch64::BI_InterlockedAnd8_rel:
- case AArch64::BI_InterlockedAnd16_rel:
- case AArch64::BI_InterlockedAnd_rel:
- case AArch64::BI_InterlockedAnd64_rel:
+ case clang::AArch64::BI_InterlockedAnd8_rel:
+ case clang::AArch64::BI_InterlockedAnd16_rel:
+ case clang::AArch64::BI_InterlockedAnd_rel:
+ case clang::AArch64::BI_InterlockedAnd64_rel:
return MSVCIntrin::_InterlockedAnd_rel;
- case AArch64::BI_InterlockedAnd8_nf:
- case AArch64::BI_InterlockedAnd16_nf:
- case AArch64::BI_InterlockedAnd_nf:
- case AArch64::BI_InterlockedAnd64_nf:
+ case clang::AArch64::BI_InterlockedAnd8_nf:
+ case clang::AArch64::BI_InterlockedAnd16_nf:
+ case clang::AArch64::BI_InterlockedAnd_nf:
+ case clang::AArch64::BI_InterlockedAnd64_nf:
return MSVCIntrin::_InterlockedAnd_nf;
- case AArch64::BI_InterlockedIncrement16_acq:
- case AArch64::BI_InterlockedIncrement_acq:
- case AArch64::BI_InterlockedIncrement64_acq:
+ case clang::AArch64::BI_InterlockedIncrement16_acq:
+ case clang::AArch64::BI_InterlockedIncrement_acq:
+ case clang::AArch64::BI_InterlockedIncrement64_acq:
return MSVCIntrin::_InterlockedIncrement_acq;
- case AArch64::BI_InterlockedIncrement16_rel:
- case AArch64::BI_InterlockedIncrement_rel:
- case AArch64::BI_InterlockedIncrement64_rel:
+ case clang::AArch64::BI_InterlockedIncrement16_rel:
+ case clang::AArch64::BI_InterlockedIncrement_rel:
+ case clang::AArch64::BI_InterlockedIncrement64_rel:
return MSVCIntrin::_InterlockedIncrement_rel;
- case AArch64::BI_InterlockedIncrement16_nf:
- case AArch64::BI_InterlockedIncrement_nf:
- case AArch64::BI_InterlockedIncrement64_nf:
+ case clang::AArch64::BI_InterlockedIncrement16_nf:
+ case clang::AArch64::BI_InterlockedIncrement_nf:
+ case clang::AArch64::BI_InterlockedIncrement64_nf:
return MSVCIntrin::_InterlockedIncrement_nf;
- case AArch64::BI_InterlockedDecrement16_acq:
- case AArch64::BI_InterlockedDecrement_acq:
- case AArch64::BI_InterlockedDecrement64_acq:
+ case clang::AArch64::BI_InterlockedDecrement16_acq:
+ case clang::AArch64::BI_InterlockedDecrement_acq:
+ case clang::AArch64::BI_InterlockedDecrement64_acq:
return MSVCIntrin::_InterlockedDecrement_acq;
- case AArch64::BI_InterlockedDecrement16_rel:
- case AArch64::BI_InterlockedDecrement_rel:
- case AArch64::BI_InterlockedDecrement64_rel:
+ case clang::AArch64::BI_InterlockedDecrement16_rel:
+ case clang::AArch64::BI_InterlockedDecrement_rel:
+ case clang::AArch64::BI_InterlockedDecrement64_rel:
return MSVCIntrin::_InterlockedDecrement_rel;
- case AArch64::BI_InterlockedDecrement16_nf:
- case AArch64::BI_InterlockedDecrement_nf:
- case AArch64::BI_InterlockedDecrement64_nf:
+ case clang::AArch64::BI_InterlockedDecrement16_nf:
+ case clang::AArch64::BI_InterlockedDecrement_nf:
+ case clang::AArch64::BI_InterlockedDecrement64_nf:
return MSVCIntrin::_InterlockedDecrement_nf;
}
llvm_unreachable("must return from switch");
@@ -1778,8 +1779,9 @@ llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
auto AL = ApplyDebugLocation::CreateArtificial(*this);
CharUnits Offset;
- Address BufAddr(Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"),
- BufferAlignment);
+ Address BufAddr =
+ Address(Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Int8Ty,
+ BufferAlignment);
Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()),
Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()),
@@ -1800,8 +1802,8 @@ llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
Address Arg = GetAddrOfLocalVar(Args[I]);
Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset, "argData");
- Addr = Builder.CreateBitCast(Addr, Arg.getPointer()->getType(),
- "argDataCast");
+ Addr =
+ Builder.CreateElementBitCast(Addr, Arg.getElementType(), "argDataCast");
Builder.CreateStore(Builder.CreateLoad(Arg), Addr);
Offset += Size;
++I;
@@ -2000,7 +2002,7 @@ EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
// Signed overflow occurs if the result is greater than INT_MAX or lesser
// than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
auto IntMax =
- llvm::APInt::getSignedMaxValue(ResultInfo.Width).zextOrSelf(OpWidth);
+ llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
llvm::Value *MaxResult =
CGF.Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
CGF.Builder.CreateZExt(IsNegative, OpTy));
@@ -2041,89 +2043,6 @@ EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
return RValue::get(Overflow);
}
-static llvm::Value *dumpRecord(CodeGenFunction &CGF, QualType RType,
- Value *&RecordPtr, CharUnits Align,
- llvm::FunctionCallee Func, int Lvl) {
- ASTContext &Context = CGF.getContext();
- RecordDecl *RD = RType->castAs<RecordType>()->getDecl()->getDefinition();
- std::string Pad = std::string(Lvl * 4, ' ');
-
- Value *GString =
- CGF.Builder.CreateGlobalStringPtr(RType.getAsString() + " {\n");
- Value *Res = CGF.Builder.CreateCall(Func, {GString});
-
- static llvm::DenseMap<QualType, const char *> Types;
- if (Types.empty()) {
- Types[Context.CharTy] = "%c";
- Types[Context.BoolTy] = "%d";
- Types[Context.SignedCharTy] = "%hhd";
- Types[Context.UnsignedCharTy] = "%hhu";
- Types[Context.IntTy] = "%d";
- Types[Context.UnsignedIntTy] = "%u";
- Types[Context.LongTy] = "%ld";
- Types[Context.UnsignedLongTy] = "%lu";
- Types[Context.LongLongTy] = "%lld";
- Types[Context.UnsignedLongLongTy] = "%llu";
- Types[Context.ShortTy] = "%hd";
- Types[Context.UnsignedShortTy] = "%hu";
- Types[Context.VoidPtrTy] = "%p";
- Types[Context.FloatTy] = "%f";
- Types[Context.DoubleTy] = "%f";
- Types[Context.LongDoubleTy] = "%Lf";
- Types[Context.getPointerType(Context.CharTy)] = "%s";
- Types[Context.getPointerType(Context.getConstType(Context.CharTy))] = "%s";
- }
-
- for (const auto *FD : RD->fields()) {
- Value *FieldPtr = RecordPtr;
- if (RD->isUnion())
- FieldPtr = CGF.Builder.CreatePointerCast(
- FieldPtr, CGF.ConvertType(Context.getPointerType(FD->getType())));
- else
- FieldPtr = CGF.Builder.CreateStructGEP(CGF.ConvertType(RType), FieldPtr,
- FD->getFieldIndex());
-
- GString = CGF.Builder.CreateGlobalStringPtr(
- llvm::Twine(Pad)
- .concat(FD->getType().getAsString())
- .concat(llvm::Twine(' '))
- .concat(FD->getNameAsString())
- .concat(" : ")
- .str());
- Value *TmpRes = CGF.Builder.CreateCall(Func, {GString});
- Res = CGF.Builder.CreateAdd(Res, TmpRes);
-
- QualType CanonicalType =
- FD->getType().getUnqualifiedType().getCanonicalType();
-
- // We check whether we are in a recursive type
- if (CanonicalType->isRecordType()) {
- TmpRes = dumpRecord(CGF, CanonicalType, FieldPtr, Align, Func, Lvl + 1);
- Res = CGF.Builder.CreateAdd(TmpRes, Res);
- continue;
- }
-
- // We try to determine the best format to print the current field
- llvm::Twine Format = Types.find(CanonicalType) == Types.end()
- ? Types[Context.VoidPtrTy]
- : Types[CanonicalType];
-
- Address FieldAddress = Address(FieldPtr, Align);
- FieldPtr = CGF.Builder.CreateLoad(FieldAddress);
-
- // FIXME Need to handle bitfield here
- GString = CGF.Builder.CreateGlobalStringPtr(
- Format.concat(llvm::Twine('\n')).str());
- TmpRes = CGF.Builder.CreateCall(Func, {GString, FieldPtr});
- Res = CGF.Builder.CreateAdd(Res, TmpRes);
- }
-
- GString = CGF.Builder.CreateGlobalStringPtr(Pad + "}\n");
- Value *TmpRes = CGF.Builder.CreateCall(Func, {GString});
- Res = CGF.Builder.CreateAdd(Res, TmpRes);
- return Res;
-}
-
static bool
TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty,
llvm::SmallPtrSetImpl<const Decl *> &Seen) {
@@ -2252,8 +2171,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
ReturnValueSlot ReturnValue) {
const FunctionDecl *FD = GD.getDecl()->getAsFunction();
// See if we can constant fold this builtin. If so, don't emit it at all.
+ // TODO: Extend this handling to all builtin calls that we can constant-fold.
Expr::EvalResult Result;
- if (E->EvaluateAsRValue(Result, CGM.getContext()) &&
+ if (E->isPRValue() && E->EvaluateAsRValue(Result, CGM.getContext()) &&
!Result.hasSideEffects()) {
if (Result.Val.isInt())
return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
@@ -2649,23 +2569,6 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(ComplexVal.first);
}
- case Builtin::BI__builtin_dump_struct: {
- llvm::Type *LLVMIntTy = getTypes().ConvertType(getContext().IntTy);
- llvm::FunctionType *LLVMFuncType = llvm::FunctionType::get(
- LLVMIntTy, {llvm::Type::getInt8PtrTy(getLLVMContext())}, true);
-
- Value *Func = EmitScalarExpr(E->getArg(1)->IgnoreImpCasts());
- CharUnits Arg0Align = EmitPointerWithAlignment(E->getArg(0)).getAlignment();
-
- const Expr *Arg0 = E->getArg(0)->IgnoreImpCasts();
- QualType Arg0Type = Arg0->getType()->getPointeeType();
-
- Value *RecordPtr = EmitScalarExpr(Arg0);
- Value *Res = dumpRecord(*this, Arg0Type, RecordPtr, Arg0Align,
- {LLVMFuncType, Func}, 0);
- return RValue::get(Res);
- }
-
case Builtin::BI__builtin_preserve_access_index: {
// Only enabled preserved access index region when debuginfo
// is available as debuginfo is needed to preserve user-level
@@ -2929,7 +2832,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
}
case Builtin::BI__builtin_bswap16:
case Builtin::BI__builtin_bswap32:
- case Builtin::BI__builtin_bswap64: {
+ case Builtin::BI__builtin_bswap64:
+ case Builtin::BI_byteswap_ushort:
+ case Builtin::BI_byteswap_ulong:
+ case Builtin::BI_byteswap_uint64: {
return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap));
}
case Builtin::BI__builtin_bitreverse8:
@@ -3154,6 +3060,25 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(
emitUnaryBuiltin(*this, E, llvm::Intrinsic::trunc, "elt.trunc"));
+ case Builtin::BI__builtin_elementwise_add_sat:
+ case Builtin::BI__builtin_elementwise_sub_sat: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Result;
+ assert(Op0->getType()->isIntOrIntVectorTy() && "integer type expected");
+ QualType Ty = E->getArg(0)->getType();
+ if (auto *VecTy = Ty->getAs<VectorType>())
+ Ty = VecTy->getElementType();
+ bool IsSigned = Ty->isSignedIntegerType();
+ unsigned Opc;
+ if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
+ Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
+ else
+ Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
+ Result = Builder.CreateBinaryIntrinsic(Opc, Op0, Op1, nullptr, "elt.sat");
+ return RValue::get(Result);
+ }
+
case Builtin::BI__builtin_elementwise_max: {
Value *Op0 = EmitScalarExpr(E->getArg(0));
Value *Op1 = EmitScalarExpr(E->getArg(1));
@@ -3218,6 +3143,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
*this, E, GetIntrinsicID(E->getArg(0)->getType()), "rdx.min"));
}
+ case Builtin::BI__builtin_reduce_add:
+ return RValue::get(emitUnaryBuiltin(
+ *this, E, llvm::Intrinsic::vector_reduce_add, "rdx.add"));
+ case Builtin::BI__builtin_reduce_mul:
+ return RValue::get(emitUnaryBuiltin(
+ *this, E, llvm::Intrinsic::vector_reduce_mul, "rdx.mul"));
case Builtin::BI__builtin_reduce_xor:
return RValue::get(emitUnaryBuiltin(
*this, E, llvm::Intrinsic::vector_reduce_xor, "rdx.xor"));
@@ -3231,14 +3162,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BI__builtin_matrix_transpose: {
auto *MatrixTy = E->getArg(0)->getType()->castAs<ConstantMatrixType>();
Value *MatValue = EmitScalarExpr(E->getArg(0));
- MatrixBuilder<CGBuilderTy> MB(Builder);
+ MatrixBuilder MB(Builder);
Value *Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
MatrixTy->getNumColumns());
return RValue::get(Result);
}
case Builtin::BI__builtin_matrix_column_major_load: {
- MatrixBuilder<CGBuilderTy> MB(Builder);
+ MatrixBuilder MB(Builder);
// Emit everything that isn't dependent on the first parameter type
Value *Stride = EmitScalarExpr(E->getArg(3));
const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
@@ -3250,14 +3181,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(),
E->getArg(0)->getExprLoc(), FD, 0);
Value *Result = MB.CreateColumnMajorLoad(
- Src.getPointer(), Align(Src.getAlignment().getQuantity()), Stride,
- IsVolatile, ResultTy->getNumRows(), ResultTy->getNumColumns(),
+ Src.getElementType(), Src.getPointer(),
+ Align(Src.getAlignment().getQuantity()), Stride, IsVolatile,
+ ResultTy->getNumRows(), ResultTy->getNumColumns(),
"matrix");
return RValue::get(Result);
}
case Builtin::BI__builtin_matrix_column_major_store: {
- MatrixBuilder<CGBuilderTy> MB(Builder);
+ MatrixBuilder MB(Builder);
Value *Matrix = EmitScalarExpr(E->getArg(0));
Address Dst = EmitPointerWithAlignment(E->getArg(1));
Value *Stride = EmitScalarExpr(E->getArg(2));
@@ -3576,6 +3508,17 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
return RValue::get(Dest.getPointer());
}
+ case Builtin::BI__builtin_memset_inline: {
+ Address Dest = EmitPointerWithAlignment(E->getArg(0));
+ Value *ByteVal =
+ Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty());
+ uint64_t Size =
+ E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
+ EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
+ E->getArg(0)->getExprLoc(), FD, 0);
+ Builder.CreateMemSetInline(Dest, ByteVal, Size);
+ return RValue::get(nullptr);
+ }
case Builtin::BI__builtin___memset_chk: {
// fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
Expr::EvalResult SizeResult, DstSizeResult;
@@ -3818,7 +3761,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
// Call LLVM's EH setjmp, which is lightweight.
Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
- Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
+ Buf = Builder.CreateElementBitCast(Buf, Int8Ty);
return RValue::get(Builder.CreateCall(F, Buf.getPointer()));
}
case Builtin::BI__builtin_longjmp: {
@@ -4129,8 +4072,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
Address Ptr = EmitPointerWithAlignment(E->getArg(0));
- unsigned AddrSpace = Ptr.getPointer()->getType()->getPointerAddressSpace();
- Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
+ Ptr = Builder.CreateElementBitCast(Ptr, Int8Ty);
Value *NewVal = Builder.getInt8(0);
Value *Order = EmitScalarExpr(E->getArg(1));
if (isa<llvm::ConstantInt>(Order)) {
@@ -4524,6 +4466,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(Carry);
}
+ case Builtin::BIaddressof:
+ case Builtin::BI__addressof:
case Builtin::BI__builtin_addressof:
return RValue::get(EmitLValue(E->getArg(0)).getPointer(*this));
case Builtin::BI__builtin_function_start:
@@ -4683,6 +4627,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
}
break;
+ // C++ std:: builtins.
+ case Builtin::BImove:
+ case Builtin::BImove_if_noexcept:
+ case Builtin::BIforward:
+ case Builtin::BIas_const:
+ return RValue::get(EmitLValue(E->getArg(0)).getPointer(*this));
case Builtin::BI__GetExceptionInfo: {
if (llvm::GlobalVariable *GV =
CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType()))
@@ -5176,7 +5126,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BI__builtin_thread_pointer: {
if (!getContext().getTargetInfo().isTLSSupported())
CGM.ErrorUnsupported(E, "__builtin_thread_pointer");
- // Fall through - it's already mapped to the intrinsic by GCCBuiltin.
+ // Fall through - it's already mapped to the intrinsic by ClangBuiltin.
break;
}
case Builtin::BI__builtin_os_log_format:
@@ -5319,7 +5269,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
StringRef Prefix =
llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
if (!Prefix.empty()) {
- IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix.data(), Name);
+ IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
// NOTE we don't need to perform a compatibility flag check here since the
// intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
// MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
@@ -5369,7 +5319,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
"Must be able to losslessly bit cast to param");
- ArgValue = Builder.CreateBitCast(ArgValue, PTy);
+ // Cast vector type (e.g., v256i32) to x86_amx, this only happen
+ // in amx intrinsics.
+ if (PTy->isX86_AMXTy())
+ ArgValue = Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
+ {ArgValue->getType()}, {ArgValue});
+ else
+ ArgValue = Builder.CreateBitCast(ArgValue, PTy);
}
Args.push_back(ArgValue);
@@ -5393,7 +5349,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&
"Must be able to losslessly bit cast result type");
- V = Builder.CreateBitCast(V, RetTy);
+ // Cast x86_amx to vector type (e.g., v256i32), this only happen
+ // in amx intrinsics.
+ if (V->getType()->isX86_AMXTy())
+ V = Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
+ {V});
+ else
+ V = Builder.CreateBitCast(V, RetTy);
}
return RValue::get(V);
@@ -6899,8 +6861,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
case NEON::BI__builtin_neon_vld1_dup_v:
case NEON::BI__builtin_neon_vld1q_dup_v: {
Value *V = UndefValue::get(Ty);
- Ty = llvm::PointerType::getUnqual(VTy->getElementType());
- PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty);
+ PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType());
LoadInst *Ld = Builder.CreateLoad(PtrOp0);
llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
@@ -7294,7 +7255,10 @@ Value *CodeGenFunction::EmitAArch64CompareBuiltinExpr(
Op = Builder.CreateBitCast(Op, OTy);
if (OTy->getScalarType()->isFloatingPointTy()) {
- Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
+ if (Fp == CmpInst::FCMP_OEQ)
+ Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
+ else
+ Op = Builder.CreateFCmpS(Fp, Op, Constant::getNullValue(OTy));
} else {
Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
}
@@ -7345,27 +7309,27 @@ Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
switch (BuiltinID) {
default:
return nullptr;
- case ARM::BI__builtin_arm_nop:
+ case clang::ARM::BI__builtin_arm_nop:
Value = 0;
break;
- case ARM::BI__builtin_arm_yield:
- case ARM::BI__yield:
+ case clang::ARM::BI__builtin_arm_yield:
+ case clang::ARM::BI__yield:
Value = 1;
break;
- case ARM::BI__builtin_arm_wfe:
- case ARM::BI__wfe:
+ case clang::ARM::BI__builtin_arm_wfe:
+ case clang::ARM::BI__wfe:
Value = 2;
break;
- case ARM::BI__builtin_arm_wfi:
- case ARM::BI__wfi:
+ case clang::ARM::BI__builtin_arm_wfi:
+ case clang::ARM::BI__wfi:
Value = 3;
break;
- case ARM::BI__builtin_arm_sev:
- case ARM::BI__sev:
+ case clang::ARM::BI__builtin_arm_sev:
+ case clang::ARM::BI__sev:
Value = 4;
break;
- case ARM::BI__builtin_arm_sevl:
- case ARM::BI__sevl:
+ case clang::ARM::BI__builtin_arm_sevl:
+ case clang::ARM::BI__sevl:
Value = 5;
break;
}
@@ -7498,7 +7462,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
if (auto Hint = GetValueForARMHint(BuiltinID))
return Hint;
- if (BuiltinID == ARM::BI__emit) {
+ if (BuiltinID == clang::ARM::BI__emit) {
bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
llvm::FunctionType *FTy =
llvm::FunctionType::get(VoidTy, /*Variadic=*/false);
@@ -7519,12 +7483,12 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
return Builder.CreateCall(Emit);
}
- if (BuiltinID == ARM::BI__builtin_arm_dbg) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
Value *Option = EmitScalarExpr(E->getArg(0));
return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
}
- if (BuiltinID == ARM::BI__builtin_arm_prefetch) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
Value *Address = EmitScalarExpr(E->getArg(0));
Value *RW = EmitScalarExpr(E->getArg(1));
Value *IsData = EmitScalarExpr(E->getArg(2));
@@ -7536,23 +7500,23 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
return Builder.CreateCall(F, {Address, RW, Locality, IsData});
}
- if (BuiltinID == ARM::BI__builtin_arm_rbit) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
return Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
}
- if (BuiltinID == ARM::BI__builtin_arm_cls) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls), Arg, "cls");
}
- if (BuiltinID == ARM::BI__builtin_arm_cls64) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls64), Arg,
"cls");
}
- if (BuiltinID == ARM::BI__clear_cache) {
+ if (BuiltinID == clang::ARM::BI__clear_cache) {
assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
const FunctionDecl *FD = E->getDirectCallee();
Value *Ops[2];
@@ -7564,16 +7528,16 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
}
- if (BuiltinID == ARM::BI__builtin_arm_mcrr ||
- BuiltinID == ARM::BI__builtin_arm_mcrr2) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
+ BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
Function *F;
switch (BuiltinID) {
default: llvm_unreachable("unexpected builtin");
- case ARM::BI__builtin_arm_mcrr:
+ case clang::ARM::BI__builtin_arm_mcrr:
F = CGM.getIntrinsic(Intrinsic::arm_mcrr);
break;
- case ARM::BI__builtin_arm_mcrr2:
+ case clang::ARM::BI__builtin_arm_mcrr2:
F = CGM.getIntrinsic(Intrinsic::arm_mcrr2);
break;
}
@@ -7598,16 +7562,16 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
}
- if (BuiltinID == ARM::BI__builtin_arm_mrrc ||
- BuiltinID == ARM::BI__builtin_arm_mrrc2) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
+ BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
Function *F;
switch (BuiltinID) {
default: llvm_unreachable("unexpected builtin");
- case ARM::BI__builtin_arm_mrrc:
+ case clang::ARM::BI__builtin_arm_mrrc:
F = CGM.getIntrinsic(Intrinsic::arm_mrrc);
break;
- case ARM::BI__builtin_arm_mrrc2:
+ case clang::ARM::BI__builtin_arm_mrrc2:
F = CGM.getIntrinsic(Intrinsic::arm_mrrc2);
break;
}
@@ -7632,21 +7596,21 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
return Builder.CreateBitCast(RtAndRt2, ConvertType(E->getType()));
}
- if (BuiltinID == ARM::BI__builtin_arm_ldrexd ||
- ((BuiltinID == ARM::BI__builtin_arm_ldrex ||
- BuiltinID == ARM::BI__builtin_arm_ldaex) &&
+ if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
+ ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
+ BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
getContext().getTypeSize(E->getType()) == 64) ||
- BuiltinID == ARM::BI__ldrexd) {
+ BuiltinID == clang::ARM::BI__ldrexd) {
Function *F;
switch (BuiltinID) {
default: llvm_unreachable("unexpected builtin");
- case ARM::BI__builtin_arm_ldaex:
+ case clang::ARM::BI__builtin_arm_ldaex:
F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
break;
- case ARM::BI__builtin_arm_ldrexd:
- case ARM::BI__builtin_arm_ldrex:
- case ARM::BI__ldrexd:
+ case clang::ARM::BI__builtin_arm_ldrexd:
+ case clang::ARM::BI__builtin_arm_ldrex:
+ case clang::ARM::BI__ldrexd:
F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
break;
}
@@ -7666,46 +7630,49 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
return Builder.CreateBitCast(Val, ConvertType(E->getType()));
}
- if (BuiltinID == ARM::BI__builtin_arm_ldrex ||
- BuiltinID == ARM::BI__builtin_arm_ldaex) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
+ BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
Value *LoadAddr = EmitScalarExpr(E->getArg(0));
QualType Ty = E->getType();
llvm::Type *RealResTy = ConvertType(Ty);
- llvm::Type *PtrTy = llvm::IntegerType::get(
- getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
+ llvm::Type *IntTy =
+ llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
+ llvm::Type *PtrTy = IntTy->getPointerTo();
LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
- Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex
- ? Intrinsic::arm_ldaex
- : Intrinsic::arm_ldrex,
- PtrTy);
- Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex");
+ Function *F = CGM.getIntrinsic(
+ BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
+ : Intrinsic::arm_ldrex,
+ PtrTy);
+ CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldrex");
+ Val->addParamAttr(
+ 0, Attribute::get(getLLVMContext(), Attribute::ElementType, IntTy));
if (RealResTy->isPointerTy())
return Builder.CreateIntToPtr(Val, RealResTy);
else {
llvm::Type *IntResTy = llvm::IntegerType::get(
getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
- Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
- return Builder.CreateBitCast(Val, RealResTy);
+ return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
+ RealResTy);
}
}
- if (BuiltinID == ARM::BI__builtin_arm_strexd ||
- ((BuiltinID == ARM::BI__builtin_arm_stlex ||
- BuiltinID == ARM::BI__builtin_arm_strex) &&
+ if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
+ ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
+ BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
getContext().getTypeSize(E->getArg(0)->getType()) == 64)) {
- Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
- ? Intrinsic::arm_stlexd
- : Intrinsic::arm_strexd);
+ Function *F = CGM.getIntrinsic(
+ BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
+ : Intrinsic::arm_strexd);
llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty);
Address Tmp = CreateMemTemp(E->getArg(0)->getType());
Value *Val = EmitScalarExpr(E->getArg(0));
Builder.CreateStore(Val, Tmp);
- Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
+ Address LdPtr = Builder.CreateElementBitCast(Tmp, STy);
Val = Builder.CreateLoad(LdPtr);
Value *Arg0 = Builder.CreateExtractValue(Val, 0);
@@ -7714,8 +7681,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd");
}
- if (BuiltinID == ARM::BI__builtin_arm_strex ||
- BuiltinID == ARM::BI__builtin_arm_stlex) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
+ BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
Value *StoreVal = EmitScalarExpr(E->getArg(0));
Value *StoreAddr = EmitScalarExpr(E->getArg(1));
@@ -7734,14 +7701,18 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
}
- Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
- ? Intrinsic::arm_stlex
- : Intrinsic::arm_strex,
- StoreAddr->getType());
- return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex");
+ Function *F = CGM.getIntrinsic(
+ BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
+ : Intrinsic::arm_strex,
+ StoreAddr->getType());
+
+ CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex");
+ CI->addParamAttr(
+ 1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
+ return CI;
}
- if (BuiltinID == ARM::BI__builtin_arm_clrex) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
return Builder.CreateCall(F);
}
@@ -7749,19 +7720,19 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
// CRC32
Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
switch (BuiltinID) {
- case ARM::BI__builtin_arm_crc32b:
+ case clang::ARM::BI__builtin_arm_crc32b:
CRCIntrinsicID = Intrinsic::arm_crc32b; break;
- case ARM::BI__builtin_arm_crc32cb:
+ case clang::ARM::BI__builtin_arm_crc32cb:
CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
- case ARM::BI__builtin_arm_crc32h:
+ case clang::ARM::BI__builtin_arm_crc32h:
CRCIntrinsicID = Intrinsic::arm_crc32h; break;
- case ARM::BI__builtin_arm_crc32ch:
+ case clang::ARM::BI__builtin_arm_crc32ch:
CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
- case ARM::BI__builtin_arm_crc32w:
- case ARM::BI__builtin_arm_crc32d:
+ case clang::ARM::BI__builtin_arm_crc32w:
+ case clang::ARM::BI__builtin_arm_crc32d:
CRCIntrinsicID = Intrinsic::arm_crc32w; break;
- case ARM::BI__builtin_arm_crc32cw:
- case ARM::BI__builtin_arm_crc32cd:
+ case clang::ARM::BI__builtin_arm_crc32cw:
+ case clang::ARM::BI__builtin_arm_crc32cd:
CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
}
@@ -7771,8 +7742,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
// crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w
// intrinsics, hence we need different codegen for these cases.
- if (BuiltinID == ARM::BI__builtin_arm_crc32d ||
- BuiltinID == ARM::BI__builtin_arm_crc32cd) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
+ BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
Value *Arg1b = Builder.CreateLShr(Arg1, C1);
@@ -7789,24 +7760,24 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
}
}
- if (BuiltinID == ARM::BI__builtin_arm_rsr ||
- BuiltinID == ARM::BI__builtin_arm_rsr64 ||
- BuiltinID == ARM::BI__builtin_arm_rsrp ||
- BuiltinID == ARM::BI__builtin_arm_wsr ||
- BuiltinID == ARM::BI__builtin_arm_wsr64 ||
- BuiltinID == ARM::BI__builtin_arm_wsrp) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
+ BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
+ BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
+ BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
+ BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
+ BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
SpecialRegisterAccessKind AccessKind = Write;
- if (BuiltinID == ARM::BI__builtin_arm_rsr ||
- BuiltinID == ARM::BI__builtin_arm_rsr64 ||
- BuiltinID == ARM::BI__builtin_arm_rsrp)
+ if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
+ BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
+ BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
AccessKind = VolatileRead;
- bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp ||
- BuiltinID == ARM::BI__builtin_arm_wsrp;
+ bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
+ BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
- bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 ||
- BuiltinID == ARM::BI__builtin_arm_wsr64;
+ bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
+ BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
llvm::Type *ValueType;
llvm::Type *RegisterType;
@@ -7823,6 +7794,11 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
AccessKind);
}
+ if (BuiltinID == ARM::BI__builtin_sponentry) {
+ llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
+ return Builder.CreateCall(F);
+ }
+
// Handle MSVC intrinsics before argument evaluation to prevent double
// evaluation.
if (Optional<MSVCIntrin> MsvcIntId = translateArmToMsvcIntrin(BuiltinID))
@@ -7981,10 +7957,11 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
// The ARM _MoveToCoprocessor builtins put the input register value as
// the first argument, but the LLVM intrinsic expects it as the third one.
- case ARM::BI_MoveToCoprocessor:
- case ARM::BI_MoveToCoprocessor2: {
- Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ?
- Intrinsic::arm_mcr : Intrinsic::arm_mcr2);
+ case clang::ARM::BI_MoveToCoprocessor:
+ case clang::ARM::BI_MoveToCoprocessor2: {
+ Function *F = CGM.getIntrinsic(BuiltinID == clang::ARM::BI_MoveToCoprocessor
+ ? Intrinsic::arm_mcr
+ : Intrinsic::arm_mcr2);
return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
Ops[3], Ops[4], Ops[5]});
}
@@ -7997,11 +7974,11 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
if (!Result)
return nullptr;
- if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
- BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
+ if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
+ BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
// Determine the overloaded type of this builtin.
llvm::Type *Ty;
- if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
+ if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
Ty = FloatTy;
else
Ty = DoubleTy;
@@ -8126,8 +8103,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
case NEON::BI__builtin_neon_vst1_lane_v: {
Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
- Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
- auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty));
+ auto St = Builder.CreateStore(
+ Ops[1], Builder.CreateElementBitCast(PtrOp0, Ops[1]->getType()));
return St;
}
case NEON::BI__builtin_neon_vtbl1_v:
@@ -9005,7 +8982,10 @@ Value *CodeGenFunction::EmitSVEMaskedLoad(const CallExpr *E,
BasePtr = Builder.CreateBitCast(BasePtr, MemEltTy->getPointerTo());
Function *F = CGM.getIntrinsic(BuiltinID, MemoryTy);
- Value *Load = Builder.CreateCall(F, {Predicate, BasePtr});
+ auto *Load =
+ cast<llvm::Instruction>(Builder.CreateCall(F, {Predicate, BasePtr}));
+ auto TBAAInfo = CGM.getTBAAAccessInfo(LangPTy->getPointeeType());
+ CGM.DecorateInstructionWithTBAA(Load, TBAAInfo);
return IsZExtReturn ? Builder.CreateZExt(Load, VectorTy)
: Builder.CreateSExt(Load, VectorTy);
@@ -9033,7 +9013,11 @@ Value *CodeGenFunction::EmitSVEMaskedStore(const CallExpr *E,
BasePtr = Builder.CreateBitCast(BasePtr, MemEltTy->getPointerTo());
Function *F = CGM.getIntrinsic(BuiltinID, MemoryTy);
- return Builder.CreateCall(F, {Val, Predicate, BasePtr});
+ auto *Store =
+ cast<llvm::Instruction>(Builder.CreateCall(F, {Val, Predicate, BasePtr}));
+ auto TBAAInfo = CGM.getTBAAAccessInfo(LangPTy->getPointeeType());
+ CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
+ return Store;
}
// Limit the usage of scalable llvm IR generated by the ACLE by using the
@@ -9427,34 +9411,34 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
const CallExpr *E,
llvm::Triple::ArchType Arch) {
- if (BuiltinID >= AArch64::FirstSVEBuiltin &&
- BuiltinID <= AArch64::LastSVEBuiltin)
+ if (BuiltinID >= clang::AArch64::FirstSVEBuiltin &&
+ BuiltinID <= clang::AArch64::LastSVEBuiltin)
return EmitAArch64SVEBuiltinExpr(BuiltinID, E);
unsigned HintID = static_cast<unsigned>(-1);
switch (BuiltinID) {
default: break;
- case AArch64::BI__builtin_arm_nop:
+ case clang::AArch64::BI__builtin_arm_nop:
HintID = 0;
break;
- case AArch64::BI__builtin_arm_yield:
- case AArch64::BI__yield:
+ case clang::AArch64::BI__builtin_arm_yield:
+ case clang::AArch64::BI__yield:
HintID = 1;
break;
- case AArch64::BI__builtin_arm_wfe:
- case AArch64::BI__wfe:
+ case clang::AArch64::BI__builtin_arm_wfe:
+ case clang::AArch64::BI__wfe:
HintID = 2;
break;
- case AArch64::BI__builtin_arm_wfi:
- case AArch64::BI__wfi:
+ case clang::AArch64::BI__builtin_arm_wfi:
+ case clang::AArch64::BI__wfi:
HintID = 3;
break;
- case AArch64::BI__builtin_arm_sev:
- case AArch64::BI__sev:
+ case clang::AArch64::BI__builtin_arm_sev:
+ case clang::AArch64::BI__sev:
HintID = 4;
break;
- case AArch64::BI__builtin_arm_sevl:
- case AArch64::BI__sevl:
+ case clang::AArch64::BI__builtin_arm_sevl:
+ case clang::AArch64::BI__sevl:
HintID = 5;
break;
}
@@ -9464,7 +9448,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
}
- if (BuiltinID == AArch64::BI__builtin_arm_prefetch) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_prefetch) {
Value *Address = EmitScalarExpr(E->getArg(0));
Value *RW = EmitScalarExpr(E->getArg(1));
Value *CacheLevel = EmitScalarExpr(E->getArg(2));
@@ -9487,14 +9471,14 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Builder.CreateCall(F, {Address, RW, Locality, IsData});
}
- if (BuiltinID == AArch64::BI__builtin_arm_rbit) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
assert((getContext().getTypeSize(E->getType()) == 32) &&
"rbit of unusual size!");
llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
return Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
}
- if (BuiltinID == AArch64::BI__builtin_arm_rbit64) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
assert((getContext().getTypeSize(E->getType()) == 64) &&
"rbit of unusual size!");
llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
@@ -9502,50 +9486,50 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
}
- if (BuiltinID == AArch64::BI__builtin_arm_cls) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls), Arg,
"cls");
}
- if (BuiltinID == AArch64::BI__builtin_arm_cls64) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls64), Arg,
"cls");
}
- if (BuiltinID == AArch64::BI__builtin_arm_frint32zf ||
- BuiltinID == AArch64::BI__builtin_arm_frint32z) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_frint32zf ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_frint32z) {
llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
llvm::Type *Ty = Arg->getType();
return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32z, Ty),
Arg, "frint32z");
}
- if (BuiltinID == AArch64::BI__builtin_arm_frint64zf ||
- BuiltinID == AArch64::BI__builtin_arm_frint64z) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_frint64zf ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_frint64z) {
llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
llvm::Type *Ty = Arg->getType();
return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64z, Ty),
Arg, "frint64z");
}
- if (BuiltinID == AArch64::BI__builtin_arm_frint32xf ||
- BuiltinID == AArch64::BI__builtin_arm_frint32x) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_frint32xf ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_frint32x) {
llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
llvm::Type *Ty = Arg->getType();
return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32x, Ty),
Arg, "frint32x");
}
- if (BuiltinID == AArch64::BI__builtin_arm_frint64xf ||
- BuiltinID == AArch64::BI__builtin_arm_frint64x) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_frint64xf ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_frint64x) {
llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
llvm::Type *Ty = Arg->getType();
return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64x, Ty),
Arg, "frint64x");
}
- if (BuiltinID == AArch64::BI__builtin_arm_jcvt) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
assert((getContext().getTypeSize(E->getType()) == 32) &&
"__jcvt of unusual size!");
llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
@@ -9553,14 +9537,14 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
CGM.getIntrinsic(Intrinsic::aarch64_fjcvtzs), Arg);
}
- if (BuiltinID == AArch64::BI__builtin_arm_ld64b ||
- BuiltinID == AArch64::BI__builtin_arm_st64b ||
- BuiltinID == AArch64::BI__builtin_arm_st64bv ||
- BuiltinID == AArch64::BI__builtin_arm_st64bv0) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
llvm::Value *MemAddr = EmitScalarExpr(E->getArg(0));
llvm::Value *ValPtr = EmitScalarExpr(E->getArg(1));
- if (BuiltinID == AArch64::BI__builtin_arm_ld64b) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
// Load from the address via an LLVM intrinsic, receiving a
// tuple of 8 i64 words, and store each one to ValPtr.
Function *F = CGM.getIntrinsic(Intrinsic::aarch64_ld64b);
@@ -9569,7 +9553,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
for (size_t i = 0; i < 8; i++) {
llvm::Value *ValOffsetPtr =
Builder.CreateGEP(Int64Ty, ValPtr, Builder.getInt32(i));
- Address Addr(ValOffsetPtr, CharUnits::fromQuantity(8));
+ Address Addr =
+ Address(ValOffsetPtr, Int64Ty, CharUnits::fromQuantity(8));
ToRet = Builder.CreateStore(Builder.CreateExtractValue(Val, i), Addr);
}
return ToRet;
@@ -9581,24 +9566,25 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
for (size_t i = 0; i < 8; i++) {
llvm::Value *ValOffsetPtr =
Builder.CreateGEP(Int64Ty, ValPtr, Builder.getInt32(i));
- Address Addr(ValOffsetPtr, CharUnits::fromQuantity(8));
+ Address Addr =
+ Address(ValOffsetPtr, Int64Ty, CharUnits::fromQuantity(8));
Args.push_back(Builder.CreateLoad(Addr));
}
- auto Intr = (BuiltinID == AArch64::BI__builtin_arm_st64b
+ auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
? Intrinsic::aarch64_st64b
- : BuiltinID == AArch64::BI__builtin_arm_st64bv
- ? Intrinsic::aarch64_st64bv
- : Intrinsic::aarch64_st64bv0);
+ : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
+ ? Intrinsic::aarch64_st64bv
+ : Intrinsic::aarch64_st64bv0);
Function *F = CGM.getIntrinsic(Intr);
return Builder.CreateCall(F, Args);
}
}
- if (BuiltinID == AArch64::BI__builtin_arm_rndr ||
- BuiltinID == AArch64::BI__builtin_arm_rndrrs) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
- auto Intr = (BuiltinID == AArch64::BI__builtin_arm_rndr
+ auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
? Intrinsic::aarch64_rndr
: Intrinsic::aarch64_rndrrs);
Function *F = CGM.getIntrinsic(Intr);
@@ -9612,7 +9598,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Status;
}
- if (BuiltinID == AArch64::BI__clear_cache) {
+ if (BuiltinID == clang::AArch64::BI__clear_cache) {
assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
const FunctionDecl *FD = E->getDirectCallee();
Value *Ops[2];
@@ -9624,12 +9610,13 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
}
- if ((BuiltinID == AArch64::BI__builtin_arm_ldrex ||
- BuiltinID == AArch64::BI__builtin_arm_ldaex) &&
+ if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
getContext().getTypeSize(E->getType()) == 128) {
- Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
- ? Intrinsic::aarch64_ldaxp
- : Intrinsic::aarch64_ldxp);
+ Function *F =
+ CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
+ ? Intrinsic::aarch64_ldaxp
+ : Intrinsic::aarch64_ldxp);
Value *LdPtr = EmitScalarExpr(E->getArg(0));
Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
@@ -9645,43 +9632,48 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
Val = Builder.CreateOr(Val, Val1);
return Builder.CreateBitCast(Val, ConvertType(E->getType()));
- } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex ||
- BuiltinID == AArch64::BI__builtin_arm_ldaex) {
+ } else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
Value *LoadAddr = EmitScalarExpr(E->getArg(0));
QualType Ty = E->getType();
llvm::Type *RealResTy = ConvertType(Ty);
- llvm::Type *PtrTy = llvm::IntegerType::get(
- getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
+ llvm::Type *IntTy =
+ llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
+ llvm::Type *PtrTy = IntTy->getPointerTo();
LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
- Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
- ? Intrinsic::aarch64_ldaxr
- : Intrinsic::aarch64_ldxr,
- PtrTy);
- Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
+ Function *F =
+ CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
+ ? Intrinsic::aarch64_ldaxr
+ : Intrinsic::aarch64_ldxr,
+ PtrTy);
+ CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
+ Val->addParamAttr(
+ 0, Attribute::get(getLLVMContext(), Attribute::ElementType, IntTy));
if (RealResTy->isPointerTy())
return Builder.CreateIntToPtr(Val, RealResTy);
llvm::Type *IntResTy = llvm::IntegerType::get(
getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
- Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
- return Builder.CreateBitCast(Val, RealResTy);
+ return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
+ RealResTy);
}
- if ((BuiltinID == AArch64::BI__builtin_arm_strex ||
- BuiltinID == AArch64::BI__builtin_arm_stlex) &&
+ if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
getContext().getTypeSize(E->getArg(0)->getType()) == 128) {
- Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
- ? Intrinsic::aarch64_stlxp
- : Intrinsic::aarch64_stxp);
+ Function *F =
+ CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_stlex
+ ? Intrinsic::aarch64_stlxp
+ : Intrinsic::aarch64_stxp);
llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty);
Address Tmp = CreateMemTemp(E->getArg(0)->getType());
EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true);
- Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy));
+ Tmp = Builder.CreateElementBitCast(Tmp, STy);
llvm::Value *Val = Builder.CreateLoad(Tmp);
Value *Arg0 = Builder.CreateExtractValue(Val, 0);
@@ -9691,8 +9683,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp");
}
- if (BuiltinID == AArch64::BI__builtin_arm_strex ||
- BuiltinID == AArch64::BI__builtin_arm_stlex) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
Value *StoreVal = EmitScalarExpr(E->getArg(0));
Value *StoreAddr = EmitScalarExpr(E->getArg(1));
@@ -9711,14 +9703,18 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
}
- Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
- ? Intrinsic::aarch64_stlxr
- : Intrinsic::aarch64_stxr,
- StoreAddr->getType());
- return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
+ Function *F =
+ CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_stlex
+ ? Intrinsic::aarch64_stlxr
+ : Intrinsic::aarch64_stxr,
+ StoreAddr->getType());
+ CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
+ CI->addParamAttr(
+ 1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
+ return CI;
}
- if (BuiltinID == AArch64::BI__getReg) {
+ if (BuiltinID == clang::AArch64::BI__getReg) {
Expr::EvalResult Result;
if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
llvm_unreachable("Sema will ensure that the parameter is constant");
@@ -9736,33 +9732,42 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Builder.CreateCall(F, Metadata);
}
- if (BuiltinID == AArch64::BI__builtin_arm_clrex) {
+ if (BuiltinID == clang::AArch64::BI__break) {
+ Expr::EvalResult Result;
+ if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
+ llvm_unreachable("Sema will ensure that the parameter is constant");
+
+ llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::aarch64_break);
+ return Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0))});
+ }
+
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
return Builder.CreateCall(F);
}
- if (BuiltinID == AArch64::BI_ReadWriteBarrier)
+ if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
llvm::SyncScope::SingleThread);
// CRC32
Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
switch (BuiltinID) {
- case AArch64::BI__builtin_arm_crc32b:
+ case clang::AArch64::BI__builtin_arm_crc32b:
CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
- case AArch64::BI__builtin_arm_crc32cb:
+ case clang::AArch64::BI__builtin_arm_crc32cb:
CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
- case AArch64::BI__builtin_arm_crc32h:
+ case clang::AArch64::BI__builtin_arm_crc32h:
CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
- case AArch64::BI__builtin_arm_crc32ch:
+ case clang::AArch64::BI__builtin_arm_crc32ch:
CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
- case AArch64::BI__builtin_arm_crc32w:
+ case clang::AArch64::BI__builtin_arm_crc32w:
CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
- case AArch64::BI__builtin_arm_crc32cw:
+ case clang::AArch64::BI__builtin_arm_crc32cw:
CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
- case AArch64::BI__builtin_arm_crc32d:
+ case clang::AArch64::BI__builtin_arm_crc32d:
CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
- case AArch64::BI__builtin_arm_crc32cd:
+ case clang::AArch64::BI__builtin_arm_crc32cd:
CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
}
@@ -9792,17 +9797,17 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
// Memory Tagging Extensions (MTE) Intrinsics
Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
switch (BuiltinID) {
- case AArch64::BI__builtin_arm_irg:
+ case clang::AArch64::BI__builtin_arm_irg:
MTEIntrinsicID = Intrinsic::aarch64_irg; break;
- case AArch64::BI__builtin_arm_addg:
+ case clang::AArch64::BI__builtin_arm_addg:
MTEIntrinsicID = Intrinsic::aarch64_addg; break;
- case AArch64::BI__builtin_arm_gmi:
+ case clang::AArch64::BI__builtin_arm_gmi:
MTEIntrinsicID = Intrinsic::aarch64_gmi; break;
- case AArch64::BI__builtin_arm_ldg:
+ case clang::AArch64::BI__builtin_arm_ldg:
MTEIntrinsicID = Intrinsic::aarch64_ldg; break;
- case AArch64::BI__builtin_arm_stg:
+ case clang::AArch64::BI__builtin_arm_stg:
MTEIntrinsicID = Intrinsic::aarch64_stg; break;
- case AArch64::BI__builtin_arm_subp:
+ case clang::AArch64::BI__builtin_arm_subp:
MTEIntrinsicID = Intrinsic::aarch64_subp; break;
}
@@ -9867,24 +9872,24 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
}
}
- if (BuiltinID == AArch64::BI__builtin_arm_rsr ||
- BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
- BuiltinID == AArch64::BI__builtin_arm_rsrp ||
- BuiltinID == AArch64::BI__builtin_arm_wsr ||
- BuiltinID == AArch64::BI__builtin_arm_wsr64 ||
- BuiltinID == AArch64::BI__builtin_arm_wsrp) {
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
SpecialRegisterAccessKind AccessKind = Write;
- if (BuiltinID == AArch64::BI__builtin_arm_rsr ||
- BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
- BuiltinID == AArch64::BI__builtin_arm_rsrp)
+ if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
AccessKind = VolatileRead;
- bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp ||
- BuiltinID == AArch64::BI__builtin_arm_wsrp;
+ bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
- bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr &&
- BuiltinID != AArch64::BI__builtin_arm_wsr;
+ bool Is64Bit = BuiltinID != clang::AArch64::BI__builtin_arm_rsr &&
+ BuiltinID != clang::AArch64::BI__builtin_arm_wsr;
llvm::Type *ValueType;
llvm::Type *RegisterType = Int64Ty;
@@ -9900,8 +9905,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
AccessKind);
}
- if (BuiltinID == AArch64::BI_ReadStatusReg ||
- BuiltinID == AArch64::BI_WriteStatusReg) {
+ if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
+ BuiltinID == clang::AArch64::BI_WriteStatusReg) {
LLVMContext &Context = CGM.getLLVMContext();
unsigned SysReg =
@@ -9922,7 +9927,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
llvm::Type *RegisterType = Int64Ty;
llvm::Type *Types[] = { RegisterType };
- if (BuiltinID == AArch64::BI_ReadStatusReg) {
+ if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
return Builder.CreateCall(F, Metadata);
@@ -9934,22 +9939,23 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Builder.CreateCall(F, { Metadata, ArgValue });
}
- if (BuiltinID == AArch64::BI_AddressOfReturnAddress) {
+ if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
llvm::Function *F =
CGM.getIntrinsic(Intrinsic::addressofreturnaddress, AllocaInt8PtrTy);
return Builder.CreateCall(F);
}
- if (BuiltinID == AArch64::BI__builtin_sponentry) {
+ if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
return Builder.CreateCall(F);
}
- if (BuiltinID == AArch64::BI__mulh || BuiltinID == AArch64::BI__umulh) {
+ if (BuiltinID == clang::AArch64::BI__mulh ||
+ BuiltinID == clang::AArch64::BI__umulh) {
llvm::Type *ResType = ConvertType(E->getType());
llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
- bool IsSigned = BuiltinID == AArch64::BI__mulh;
+ bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
Value *LHS =
Builder.CreateIntCast(EmitScalarExpr(E->getArg(0)), Int128Ty, IsSigned);
Value *RHS =
@@ -9968,6 +9974,55 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return HigherBits;
}
+ if (BuiltinID == AArch64::BI__writex18byte ||
+ BuiltinID == AArch64::BI__writex18word ||
+ BuiltinID == AArch64::BI__writex18dword ||
+ BuiltinID == AArch64::BI__writex18qword) {
+ llvm::Type *IntTy = ConvertType(E->getArg(1)->getType());
+
+ // Read x18 as i8*
+ LLVMContext &Context = CGM.getLLVMContext();
+ llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")};
+ llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
+ llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
+ llvm::Function *F =
+ CGM.getIntrinsic(llvm::Intrinsic::read_register, {Int64Ty});
+ llvm::Value *X18 = Builder.CreateCall(F, Metadata);
+ X18 = Builder.CreateIntToPtr(X18, llvm::PointerType::get(Int8Ty, 0));
+
+ // Store val at x18 + offset
+ Value *Offset = Builder.CreateZExt(EmitScalarExpr(E->getArg(0)), Int64Ty);
+ Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
+ Ptr = Builder.CreatePointerCast(Ptr, llvm::PointerType::get(IntTy, 0));
+ Value *Val = EmitScalarExpr(E->getArg(1));
+ StoreInst *Store = Builder.CreateAlignedStore(Val, Ptr, CharUnits::One());
+ return Store;
+ }
+
+ if (BuiltinID == AArch64::BI__readx18byte ||
+ BuiltinID == AArch64::BI__readx18word ||
+ BuiltinID == AArch64::BI__readx18dword ||
+ BuiltinID == AArch64::BI__readx18qword) {
+ llvm::Type *IntTy = ConvertType(E->getType());
+
+ // Read x18 as i8*
+ LLVMContext &Context = CGM.getLLVMContext();
+ llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")};
+ llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
+ llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
+ llvm::Function *F =
+ CGM.getIntrinsic(llvm::Intrinsic::read_register, {Int64Ty});
+ llvm::Value *X18 = Builder.CreateCall(F, Metadata);
+ X18 = Builder.CreateIntToPtr(X18, llvm::PointerType::get(Int8Ty, 0));
+
+ // Load x18 + offset
+ Value *Offset = Builder.CreateZExt(EmitScalarExpr(E->getArg(0)), Int64Ty);
+ Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
+ Ptr = Builder.CreatePointerCast(Ptr, llvm::PointerType::get(IntTy, 0));
+ LoadInst *Load = Builder.CreateAlignedLoad(IntTy, Ptr, CharUnits::One());
+ return Load;
+ }
+
// Handle MSVC intrinsics before argument evaluation to prevent double
// evaluation.
if (Optional<MSVCIntrin> MsvcIntId = translateAarch64ToMsvcIntrin(BuiltinID))
@@ -10299,7 +10354,10 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
Ops.push_back(EmitScalarExpr(E->getArg(1)));
Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
- Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
+ if (P == llvm::FCmpInst::FCMP_OEQ)
+ Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
+ else
+ Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd");
}
case NEON::BI__builtin_neon_vceqs_f32:
@@ -10319,7 +10377,10 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
Ops.push_back(EmitScalarExpr(E->getArg(1)));
Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
- Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
+ if (P == llvm::FCmpInst::FCMP_OEQ)
+ Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
+ else
+ Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd");
}
case NEON::BI__builtin_neon_vceqh_f16:
@@ -10339,7 +10400,10 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
Ops.push_back(EmitScalarExpr(E->getArg(1)));
Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy);
- Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
+ if (P == llvm::FCmpInst::FCMP_OEQ)
+ Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
+ else
+ Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd");
}
case NEON::BI__builtin_neon_vceqd_s64:
@@ -10684,7 +10748,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
"vgetq_lane");
}
- case AArch64::BI_InterlockedAdd: {
+ case clang::AArch64::BI_InterlockedAdd: {
Value *Arg0 = EmitScalarExpr(E->getArg(0));
Value *Arg1 = EmitScalarExpr(E->getArg(1));
AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
@@ -12520,13 +12584,6 @@ static Value *EmitX86SExtMask(CodeGenFunction &CGF, Value *Op,
return CGF.Builder.CreateSExt(Mask, DstTy, "vpmovm2");
}
-// Emit binary intrinsic with the same type used in result/args.
-static Value *EmitX86BinaryIntrinsic(CodeGenFunction &CGF,
- ArrayRef<Value *> Ops, Intrinsic::ID IID) {
- llvm::Function *F = CGF.CGM.getIntrinsic(IID, Ops[0]->getType());
- return CGF.Builder.CreateCall(F, {Ops[0], Ops[1]});
-}
-
Value *CodeGenFunction::EmitX86CpuIs(const CallExpr *E) {
const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts();
StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
@@ -14383,12 +14440,6 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true);
// Reductions
- case X86::BI__builtin_ia32_reduce_add_d512:
- case X86::BI__builtin_ia32_reduce_add_q512: {
- Function *F =
- CGM.getIntrinsic(Intrinsic::vector_reduce_add, Ops[0]->getType());
- return Builder.CreateCall(F, {Ops[0]});
- }
case X86::BI__builtin_ia32_reduce_fadd_pd512:
case X86::BI__builtin_ia32_reduce_fadd_ps512:
case X86::BI__builtin_ia32_reduce_fadd_ph512:
@@ -14429,12 +14480,6 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
Builder.getFastMathFlags().setNoNaNs();
return Builder.CreateCall(F, {Ops[0]});
}
- case X86::BI__builtin_ia32_reduce_mul_d512:
- case X86::BI__builtin_ia32_reduce_mul_q512: {
- Function *F =
- CGM.getIntrinsic(Intrinsic::vector_reduce_mul, Ops[0]->getType());
- return Builder.CreateCall(F, {Ops[0]});
- }
// 3DNow!
case X86::BI__builtin_ia32_pswapdsf:
@@ -14871,6 +14916,46 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
return EmitX86Select(*this, Ops[2], Res, Ops[1]);
}
+ case X86::BI__cpuid:
+ case X86::BI__cpuidex: {
+ Value *FuncId = EmitScalarExpr(E->getArg(1));
+ Value *SubFuncId = BuiltinID == X86::BI__cpuidex
+ ? EmitScalarExpr(E->getArg(2))
+ : llvm::ConstantInt::get(Int32Ty, 0);
+
+ llvm::StructType *CpuidRetTy =
+ llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty, Int32Ty);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(CpuidRetTy, {Int32Ty, Int32Ty}, false);
+
+ StringRef Asm, Constraints;
+ if (getTarget().getTriple().getArch() == llvm::Triple::x86) {
+ Asm = "cpuid";
+ Constraints = "={ax},={bx},={cx},={dx},{ax},{cx}";
+ } else {
+ // x86-64 uses %rbx as the base register, so preserve it.
+ Asm = "xchgq %rbx, ${1:q}\n"
+ "cpuid\n"
+ "xchgq %rbx, ${1:q}";
+ Constraints = "={ax},=r,={cx},={dx},0,2";
+ }
+
+ llvm::InlineAsm *IA = llvm::InlineAsm::get(FTy, Asm, Constraints,
+ /*hasSideEffects=*/false);
+ Value *IACall = Builder.CreateCall(IA, {FuncId, SubFuncId});
+ Value *BasePtr = EmitScalarExpr(E->getArg(0));
+ Value *Store = nullptr;
+ for (unsigned i = 0; i < 4; i++) {
+ Value *Extracted = Builder.CreateExtractValue(IACall, i);
+ Value *StorePtr = Builder.CreateConstInBoundsGEP1_32(Int32Ty, BasePtr, i);
+ Store = Builder.CreateAlignedStore(Extracted, StorePtr, getIntAlign());
+ }
+
+ // Return the last store instruction to signal that we have emitted the
+ // the intrinsic.
+ return Store;
+ }
+
case X86::BI__emul:
case X86::BI__emulu: {
llvm::Type *Int64Ty = llvm::IntegerType::get(getLLVMContext(), 64);
@@ -14980,34 +15065,6 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
Load->setVolatile(true);
return Load;
}
- case X86::BI__builtin_ia32_paddsb512:
- case X86::BI__builtin_ia32_paddsw512:
- case X86::BI__builtin_ia32_paddsb256:
- case X86::BI__builtin_ia32_paddsw256:
- case X86::BI__builtin_ia32_paddsb128:
- case X86::BI__builtin_ia32_paddsw128:
- return EmitX86BinaryIntrinsic(*this, Ops, Intrinsic::sadd_sat);
- case X86::BI__builtin_ia32_paddusb512:
- case X86::BI__builtin_ia32_paddusw512:
- case X86::BI__builtin_ia32_paddusb256:
- case X86::BI__builtin_ia32_paddusw256:
- case X86::BI__builtin_ia32_paddusb128:
- case X86::BI__builtin_ia32_paddusw128:
- return EmitX86BinaryIntrinsic(*this, Ops, Intrinsic::uadd_sat);
- case X86::BI__builtin_ia32_psubsb512:
- case X86::BI__builtin_ia32_psubsw512:
- case X86::BI__builtin_ia32_psubsb256:
- case X86::BI__builtin_ia32_psubsw256:
- case X86::BI__builtin_ia32_psubsb128:
- case X86::BI__builtin_ia32_psubsw128:
- return EmitX86BinaryIntrinsic(*this, Ops, Intrinsic::ssub_sat);
- case X86::BI__builtin_ia32_psubusb512:
- case X86::BI__builtin_ia32_psubusw512:
- case X86::BI__builtin_ia32_psubusb256:
- case X86::BI__builtin_ia32_psubusw256:
- case X86::BI__builtin_ia32_psubusb128:
- case X86::BI__builtin_ia32_psubusw128:
- return EmitX86BinaryIntrinsic(*this, Ops, Intrinsic::usub_sat);
case X86::BI__builtin_ia32_encodekey128_u32: {
Intrinsic::ID IID = Intrinsic::x86_encodekey128;
@@ -15189,14 +15246,17 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
const CallExpr *E) {
- SmallVector<Value*, 4> Ops;
-
- for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
- if (E->getArg(i)->getType()->isArrayType())
- Ops.push_back(EmitArrayToPointerDecay(E->getArg(i)).getPointer());
- else
- Ops.push_back(EmitScalarExpr(E->getArg(i)));
- }
+ // Do not emit the builtin arguments in the arguments of a function call,
+ // because the evaluation order of function arguments is not specified in C++.
+ // This is important when testing to ensure the arguments are emitted in the
+ // same order every time. Eg:
+ // Instead of:
+ // return Builder.CreateFDiv(EmitScalarExpr(E->getArg(0)),
+ // EmitScalarExpr(E->getArg(1)), "swdiv");
+ // Use:
+ // Value *Op0 = EmitScalarExpr(E->getArg(0));
+ // Value *Op1 = EmitScalarExpr(E->getArg(1));
+ // return Builder.CreateFDiv(Op0, Op1, "swdiv")
Intrinsic::ID ID = Intrinsic::not_intrinsic;
@@ -15223,6 +15283,9 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
case PPC::BI__builtin_vsx_lxvl:
case PPC::BI__builtin_vsx_lxvll:
{
+ SmallVector<Value *, 2> Ops;
+ Ops.push_back(EmitScalarExpr(E->getArg(0)));
+ Ops.push_back(EmitScalarExpr(E->getArg(1)));
if(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
BuiltinID == PPC::BI__builtin_vsx_lxvll){
Ops[0] = Builder.CreateBitCast(Ops[0], Int8PtrTy);
@@ -15291,6 +15354,10 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
case PPC::BI__builtin_vsx_stxvl:
case PPC::BI__builtin_vsx_stxvll:
{
+ SmallVector<Value *, 3> Ops;
+ Ops.push_back(EmitScalarExpr(E->getArg(0)));
+ Ops.push_back(EmitScalarExpr(E->getArg(1)));
+ Ops.push_back(EmitScalarExpr(E->getArg(2)));
if(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
BuiltinID == PPC::BI__builtin_vsx_stxvll ){
Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
@@ -15343,13 +15410,15 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
// Essentially boils down to performing an unaligned VMX load sequence so
// as to avoid crossing a page boundary and then shuffling the elements
// into the right side of the vector register.
- int64_t NumBytes = cast<ConstantInt>(Ops[1])->getZExtValue();
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
llvm::Type *ResTy = ConvertType(E->getType());
bool IsLE = getTarget().isLittleEndian();
// If the user wants the entire vector, just load the entire vector.
if (NumBytes == 16) {
- Value *BC = Builder.CreateBitCast(Ops[0], ResTy->getPointerTo());
+ Value *BC = Builder.CreateBitCast(Op0, ResTy->getPointerTo());
Value *LD =
Builder.CreateLoad(Address(BC, ResTy, CharUnits::fromQuantity(1)));
if (!IsLE)
@@ -15367,16 +15436,14 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
: Intrinsic::ppc_altivec_lvsl);
llvm::Function *Vperm = CGM.getIntrinsic(Intrinsic::ppc_altivec_vperm);
Value *HiMem = Builder.CreateGEP(
- Int8Ty, Ops[0], ConstantInt::get(Ops[1]->getType(), NumBytes - 1));
- Value *LoLd = Builder.CreateCall(Lvx, Ops[0], "ld.lo");
+ Int8Ty, Op0, ConstantInt::get(Op1->getType(), NumBytes - 1));
+ Value *LoLd = Builder.CreateCall(Lvx, Op0, "ld.lo");
Value *HiLd = Builder.CreateCall(Lvx, HiMem, "ld.hi");
- Value *Mask1 = Builder.CreateCall(Lvs, Ops[0], "mask1");
+ Value *Mask1 = Builder.CreateCall(Lvs, Op0, "mask1");
- Ops.clear();
- Ops.push_back(IsLE ? HiLd : LoLd);
- Ops.push_back(IsLE ? LoLd : HiLd);
- Ops.push_back(Mask1);
- Value *AllElts = Builder.CreateCall(Vperm, Ops, "shuffle1");
+ Op0 = IsLE ? HiLd : LoLd;
+ Op1 = IsLE ? LoLd : HiLd;
+ Value *AllElts = Builder.CreateCall(Vperm, {Op0, Op1, Mask1}, "shuffle1");
Constant *Zero = llvm::Constant::getNullValue(IsLE ? ResTy : AllElts->getType());
if (IsLE) {
@@ -15397,23 +15464,25 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
Builder.CreateCall(Vperm, {Zero, AllElts, Mask2}, "shuffle2"), ResTy);
}
case PPC::BI__builtin_vsx_strmb: {
- int64_t NumBytes = cast<ConstantInt>(Ops[1])->getZExtValue();
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
bool IsLE = getTarget().isLittleEndian();
auto StoreSubVec = [&](unsigned Width, unsigned Offset, unsigned EltNo) {
// Storing the whole vector, simply store it on BE and reverse bytes and
// store on LE.
if (Width == 16) {
- Value *BC =
- Builder.CreateBitCast(Ops[0], Ops[2]->getType()->getPointerTo());
- Value *StVec = Ops[2];
+ Value *BC = Builder.CreateBitCast(Op0, Op2->getType()->getPointerTo());
+ Value *StVec = Op2;
if (IsLE) {
SmallVector<int, 16> RevMask;
for (int Idx = 0; Idx < 16; Idx++)
RevMask.push_back(15 - Idx);
- StVec = Builder.CreateShuffleVector(Ops[2], Ops[2], RevMask);
+ StVec = Builder.CreateShuffleVector(Op2, Op2, RevMask);
}
return Builder.CreateStore(
- StVec, Address(BC, Ops[2]->getType(), CharUnits::fromQuantity(1)));
+ StVec, Address(BC, Op2->getType(), CharUnits::fromQuantity(1)));
}
auto *ConvTy = Int64Ty;
unsigned NumElts = 0;
@@ -15438,9 +15507,9 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
break;
}
Value *Vec = Builder.CreateBitCast(
- Ops[2], llvm::FixedVectorType::get(ConvTy, NumElts));
- Value *Ptr = Builder.CreateGEP(Int8Ty, Ops[0],
- ConstantInt::get(Int64Ty, Offset));
+ Op2, llvm::FixedVectorType::get(ConvTy, NumElts));
+ Value *Ptr =
+ Builder.CreateGEP(Int8Ty, Op0, ConstantInt::get(Int64Ty, Offset));
Value *PtrBC = Builder.CreateBitCast(Ptr, ConvTy->getPointerTo());
Value *Elt = Builder.CreateExtractElement(Vec, EltNo);
if (IsLE && Width > 1) {
@@ -15512,62 +15581,65 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
return Builder.CreateCall(F, {X, Undef});
}
- case PPC::BI__builtin_altivec_vec_replace_elt:
- case PPC::BI__builtin_altivec_vec_replace_unaligned: {
- // The third argument of vec_replace_elt and vec_replace_unaligned must
- // be a compile time constant and will be emitted either to the vinsw
- // or vinsd instruction.
- ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
+ case PPC::BI__builtin_altivec_vinsd:
+ case PPC::BI__builtin_altivec_vinsw:
+ case PPC::BI__builtin_altivec_vinsd_elt:
+ case PPC::BI__builtin_altivec_vinsw_elt: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+
+ bool IsUnaligned = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
+ BuiltinID == PPC::BI__builtin_altivec_vinsd);
+
+ bool Is32bit = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
+ BuiltinID == PPC::BI__builtin_altivec_vinsw_elt);
+
+ // The third argument must be a compile time constant.
+ ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
assert(ArgCI &&
"Third Arg to vinsw/vinsd intrinsic must be a constant integer!");
- llvm::Type *ResultType = ConvertType(E->getType());
- llvm::Function *F = nullptr;
- Value *Call = nullptr;
+
+ // Valid value for the third argument is dependent on the input type and
+ // builtin called.
+ int ValidMaxValue = 0;
+ if (IsUnaligned)
+ ValidMaxValue = (Is32bit) ? 12 : 8;
+ else
+ ValidMaxValue = (Is32bit) ? 3 : 1;
+
+ // Get value of third argument.
int64_t ConstArg = ArgCI->getSExtValue();
- unsigned ArgWidth = Ops[1]->getType()->getPrimitiveSizeInBits();
- bool Is32Bit = false;
- assert((ArgWidth == 32 || ArgWidth == 64) && "Invalid argument width");
- // The input to vec_replace_elt is an element index, not a byte index.
- if (BuiltinID == PPC::BI__builtin_altivec_vec_replace_elt)
- ConstArg *= ArgWidth / 8;
- if (ArgWidth == 32) {
- Is32Bit = true;
- // When the second argument is 32 bits, it can either be an integer or
- // a float. The vinsw intrinsic is used in this case.
- F = CGM.getIntrinsic(Intrinsic::ppc_altivec_vinsw);
+
+ // Compose range checking error message.
+ std::string RangeErrMsg = IsUnaligned ? "byte" : "element";
+ RangeErrMsg += " number " + llvm::to_string(ConstArg);
+ RangeErrMsg += " is outside of the valid range [0, ";
+ RangeErrMsg += llvm::to_string(ValidMaxValue) + "]";
+
+ // Issue error if third argument is not within the valid range.
+ if (ConstArg < 0 || ConstArg > ValidMaxValue)
+ CGM.Error(E->getExprLoc(), RangeErrMsg);
+
+ // Input to vec_replace_elt is an element index, convert to byte index.
+ if (!IsUnaligned) {
+ ConstArg *= Is32bit ? 4 : 8;
// Fix the constant according to endianess.
if (getTarget().isLittleEndian())
- ConstArg = 12 - ConstArg;
- } else {
- // When the second argument is 64 bits, it can either be a long long or
- // a double. The vinsd intrinsic is used in this case.
- F = CGM.getIntrinsic(Intrinsic::ppc_altivec_vinsd);
- // Fix the constant for little endian.
- if (getTarget().isLittleEndian())
- ConstArg = 8 - ConstArg;
- }
- Ops[2] = ConstantInt::getSigned(Int32Ty, ConstArg);
- // Depending on ArgWidth, the input vector could be a float or a double.
- // If the input vector is a float type, bitcast the inputs to integers. Or,
- // if the input vector is a double, bitcast the inputs to 64-bit integers.
- if (!Ops[1]->getType()->isIntegerTy(ArgWidth)) {
- Ops[0] = Builder.CreateBitCast(
- Ops[0], Is32Bit ? llvm::FixedVectorType::get(Int32Ty, 4)
- : llvm::FixedVectorType::get(Int64Ty, 2));
- Ops[1] = Builder.CreateBitCast(Ops[1], Is32Bit ? Int32Ty : Int64Ty);
+ ConstArg = (Is32bit ? 12 : 8) - ConstArg;
}
- // Emit the call to vinsw or vinsd.
- Call = Builder.CreateCall(F, Ops);
- // Depending on the builtin, bitcast to the approriate result type.
- if (BuiltinID == PPC::BI__builtin_altivec_vec_replace_elt &&
- !Ops[1]->getType()->isIntegerTy())
- return Builder.CreateBitCast(Call, ResultType);
- else if (BuiltinID == PPC::BI__builtin_altivec_vec_replace_elt &&
- Ops[1]->getType()->isIntegerTy())
- return Call;
- else
- return Builder.CreateBitCast(Call,
- llvm::FixedVectorType::get(Int8Ty, 16));
+
+ ID = Is32bit ? Intrinsic::ppc_altivec_vinsw : Intrinsic::ppc_altivec_vinsd;
+ Op2 = ConstantInt::getSigned(Int32Ty, ConstArg);
+ // Casting input to vector int as per intrinsic definition.
+ Op0 =
+ Is32bit
+ ? Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int32Ty, 4))
+ : Builder.CreateBitCast(Op0,
+ llvm::FixedVectorType::get(Int64Ty, 2));
+ return Builder.CreateBitCast(
+ Builder.CreateCall(CGM.getIntrinsic(ID), {Op0, Op1, Op2}), ResultType);
}
case PPC::BI__builtin_altivec_vpopcntb:
case PPC::BI__builtin_altivec_vpopcnth:
@@ -15580,15 +15652,60 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
}
case PPC::BI__builtin_altivec_vadduqm:
case PPC::BI__builtin_altivec_vsubuqm: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
- Ops[0] =
- Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int128Ty, 1));
- Ops[1] =
- Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(Int128Ty, 1));
+ Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int128Ty, 1));
+ Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int128Ty, 1));
if (BuiltinID == PPC::BI__builtin_altivec_vadduqm)
- return Builder.CreateAdd(Ops[0], Ops[1], "vadduqm");
+ return Builder.CreateAdd(Op0, Op1, "vadduqm");
else
- return Builder.CreateSub(Ops[0], Ops[1], "vsubuqm");
+ return Builder.CreateSub(Op0, Op1, "vsubuqm");
+ }
+ case PPC::BI__builtin_altivec_vaddcuq_c:
+ case PPC::BI__builtin_altivec_vsubcuq_c: {
+ SmallVector<Value *, 2> Ops;
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
+ llvm::IntegerType::get(getLLVMContext(), 128), 1);
+ Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty));
+ Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty));
+ ID = (BuiltinID == PPC::BI__builtin_altivec_vaddcuq_c)
+ ? Intrinsic::ppc_altivec_vaddcuq
+ : Intrinsic::ppc_altivec_vsubcuq;
+ return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, "");
+ }
+ case PPC::BI__builtin_altivec_vaddeuqm_c:
+ case PPC::BI__builtin_altivec_vaddecuq_c:
+ case PPC::BI__builtin_altivec_vsubeuqm_c:
+ case PPC::BI__builtin_altivec_vsubecuq_c: {
+ SmallVector<Value *, 3> Ops;
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
+ llvm::IntegerType::get(getLLVMContext(), 128), 1);
+ Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty));
+ Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty));
+ Ops.push_back(Builder.CreateBitCast(Op2, V1I128Ty));
+ switch (BuiltinID) {
+ default:
+ llvm_unreachable("Unsupported intrinsic!");
+ case PPC::BI__builtin_altivec_vaddeuqm_c:
+ ID = Intrinsic::ppc_altivec_vaddeuqm;
+ break;
+ case PPC::BI__builtin_altivec_vaddecuq_c:
+ ID = Intrinsic::ppc_altivec_vaddecuq;
+ break;
+ case PPC::BI__builtin_altivec_vsubeuqm_c:
+ ID = Intrinsic::ppc_altivec_vsubeuqm;
+ break;
+ case PPC::BI__builtin_altivec_vsubecuq_c:
+ ID = Intrinsic::ppc_altivec_vsubecuq;
+ break;
+ }
+ return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, "");
}
// Rotate and insert under mask operation.
// __rldimi(rs, is, shift, mask)
@@ -15597,29 +15714,37 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
// (rotl(rs, shift) & mask) | (is & ~mask)
case PPC::BI__builtin_ppc_rldimi:
case PPC::BI__builtin_ppc_rlwimi: {
- llvm::Type *Ty = Ops[0]->getType();
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ Value *Op3 = EmitScalarExpr(E->getArg(3));
+ llvm::Type *Ty = Op0->getType();
Function *F = CGM.getIntrinsic(Intrinsic::fshl, Ty);
if (BuiltinID == PPC::BI__builtin_ppc_rldimi)
- Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
- Value *Shift = Builder.CreateCall(F, {Ops[0], Ops[0], Ops[2]});
- Value *X = Builder.CreateAnd(Shift, Ops[3]);
- Value *Y = Builder.CreateAnd(Ops[1], Builder.CreateNot(Ops[3]));
+ Op2 = Builder.CreateZExt(Op2, Int64Ty);
+ Value *Shift = Builder.CreateCall(F, {Op0, Op0, Op2});
+ Value *X = Builder.CreateAnd(Shift, Op3);
+ Value *Y = Builder.CreateAnd(Op1, Builder.CreateNot(Op3));
return Builder.CreateOr(X, Y);
}
// Rotate and insert under mask operation.
// __rlwnm(rs, shift, mask)
// rotl(rs, shift) & mask
case PPC::BI__builtin_ppc_rlwnm: {
- llvm::Type *Ty = Ops[0]->getType();
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ llvm::Type *Ty = Op0->getType();
Function *F = CGM.getIntrinsic(Intrinsic::fshl, Ty);
- Value *Shift = Builder.CreateCall(F, {Ops[0], Ops[0], Ops[1]});
- return Builder.CreateAnd(Shift, Ops[2]);
+ Value *Shift = Builder.CreateCall(F, {Op0, Op0, Op1});
+ return Builder.CreateAnd(Shift, Op2);
}
case PPC::BI__builtin_ppc_poppar4:
case PPC::BI__builtin_ppc_poppar8: {
- llvm::Type *ArgType = Ops[0]->getType();
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ llvm::Type *ArgType = Op0->getType();
Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
- Value *Tmp = Builder.CreateCall(F, Ops[0]);
+ Value *Tmp = Builder.CreateCall(F, Op0);
llvm::Type *ResultType = ConvertType(E->getType());
Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
@@ -15629,10 +15754,12 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
return Result;
}
case PPC::BI__builtin_ppc_cmpb: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
if (getTarget().getTriple().isPPC64()) {
Function *F =
CGM.getIntrinsic(Intrinsic::ppc_cmpb, {Int64Ty, Int64Ty, Int64Ty});
- return Builder.CreateCall(F, Ops, "cmpb");
+ return Builder.CreateCall(F, {Op0, Op1}, "cmpb");
}
// For 32 bit, emit the code as below:
// %conv = trunc i64 %a to i32
@@ -15650,13 +15777,13 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
// ret i64 %or
Function *F =
CGM.getIntrinsic(Intrinsic::ppc_cmpb, {Int32Ty, Int32Ty, Int32Ty});
- Value *ArgOneLo = Builder.CreateTrunc(Ops[0], Int32Ty);
- Value *ArgTwoLo = Builder.CreateTrunc(Ops[1], Int32Ty);
+ Value *ArgOneLo = Builder.CreateTrunc(Op0, Int32Ty);
+ Value *ArgTwoLo = Builder.CreateTrunc(Op1, Int32Ty);
Constant *ShiftAmt = ConstantInt::get(Int64Ty, 32);
Value *ArgOneHi =
- Builder.CreateTrunc(Builder.CreateLShr(Ops[0], ShiftAmt), Int32Ty);
+ Builder.CreateTrunc(Builder.CreateLShr(Op0, ShiftAmt), Int32Ty);
Value *ArgTwoHi =
- Builder.CreateTrunc(Builder.CreateLShr(Ops[1], ShiftAmt), Int32Ty);
+ Builder.CreateTrunc(Builder.CreateLShr(Op1, ShiftAmt), Int32Ty);
Value *ResLo = Builder.CreateZExt(
Builder.CreateCall(F, {ArgOneLo, ArgTwoLo}, "cmpb"), Int64Ty);
Value *ResHiShift = Builder.CreateZExt(
@@ -15750,30 +15877,37 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
return FDiv;
}
case PPC::BI__builtin_ppc_alignx: {
- ConstantInt *AlignmentCI = cast<ConstantInt>(Ops[0]);
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ ConstantInt *AlignmentCI = cast<ConstantInt>(Op0);
if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
AlignmentCI = ConstantInt::get(AlignmentCI->getType(),
llvm::Value::MaximumAlignment);
- emitAlignmentAssumption(Ops[1], E->getArg(1),
+ emitAlignmentAssumption(Op1, E->getArg(1),
/*The expr loc is sufficient.*/ SourceLocation(),
AlignmentCI, nullptr);
- return Ops[1];
+ return Op1;
}
case PPC::BI__builtin_ppc_rdlam: {
- llvm::Type *Ty = Ops[0]->getType();
- Value *ShiftAmt = Builder.CreateIntCast(Ops[1], Ty, false);
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ llvm::Type *Ty = Op0->getType();
+ Value *ShiftAmt = Builder.CreateIntCast(Op1, Ty, false);
Function *F = CGM.getIntrinsic(Intrinsic::fshl, Ty);
- Value *Rotate = Builder.CreateCall(F, {Ops[0], Ops[0], ShiftAmt});
- return Builder.CreateAnd(Rotate, Ops[2]);
+ Value *Rotate = Builder.CreateCall(F, {Op0, Op0, ShiftAmt});
+ return Builder.CreateAnd(Rotate, Op2);
}
case PPC::BI__builtin_ppc_load2r: {
Function *F = CGM.getIntrinsic(Intrinsic::ppc_load2r);
- Ops[0] = Builder.CreateBitCast(Ops[0], Int8PtrTy);
- Value *LoadIntrinsic = Builder.CreateCall(F, Ops);
+ Value *Op0 = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int8PtrTy);
+ Value *LoadIntrinsic = Builder.CreateCall(F, {Op0});
return Builder.CreateTrunc(LoadIntrinsic, Int16Ty);
}
// FMA variations
+ case PPC::BI__builtin_ppc_fnmsub:
+ case PPC::BI__builtin_ppc_fnmsubs:
case PPC::BI__builtin_vsx_xvmaddadp:
case PPC::BI__builtin_vsx_xvmaddasp:
case PPC::BI__builtin_vsx_xvnmaddadp:
@@ -15812,6 +15946,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
F, {X, Y, Builder.CreateFNeg(Z, "neg")});
else
return Builder.CreateCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")});
+ case PPC::BI__builtin_ppc_fnmsub:
+ case PPC::BI__builtin_ppc_fnmsubs:
case PPC::BI__builtin_vsx_xvnmsubadp:
case PPC::BI__builtin_vsx_xvnmsubasp:
if (Builder.getIsFPConstrained())
@@ -15820,20 +15956,22 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
F, {X, Y, Builder.CreateFNeg(Z, "neg")}),
"neg");
else
- return Builder.CreateFNeg(
- Builder.CreateCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")}),
- "neg");
- }
+ return Builder.CreateCall(
+ CGM.getIntrinsic(Intrinsic::ppc_fnmsub, ResultType), {X, Y, Z});
+ }
llvm_unreachable("Unknown FMA operation");
return nullptr; // Suppress no-return warning
}
case PPC::BI__builtin_vsx_insertword: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxinsertw);
// Third argument is a compile time constant int. It must be clamped to
// to the range [0, 12].
- ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
+ ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
assert(ArgCI &&
"Third arg to xxinsertw intrinsic must be constant integer");
const int64_t MaxIndex = 12;
@@ -15844,40 +15982,38 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
// word from the first argument, and inserts it in the second argument. The
// instruction extracts the word from its second input register and inserts
// it into its first input register, so swap the first and second arguments.
- std::swap(Ops[0], Ops[1]);
+ std::swap(Op0, Op1);
// Need to cast the second argument from a vector of unsigned int to a
// vector of long long.
- Ops[1] =
- Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(Int64Ty, 2));
+ Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int64Ty, 2));
if (getTarget().isLittleEndian()) {
// Reverse the double words in the vector we will extract from.
- Ops[0] =
- Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2));
- Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ArrayRef<int>{1, 0});
+ Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int64Ty, 2));
+ Op0 = Builder.CreateShuffleVector(Op0, Op0, ArrayRef<int>{1, 0});
// Reverse the index.
Index = MaxIndex - Index;
}
// Intrinsic expects the first arg to be a vector of int.
- Ops[0] =
- Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 4));
- Ops[2] = ConstantInt::getSigned(Int32Ty, Index);
- return Builder.CreateCall(F, Ops);
+ Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int32Ty, 4));
+ Op2 = ConstantInt::getSigned(Int32Ty, Index);
+ return Builder.CreateCall(F, {Op0, Op1, Op2});
}
case PPC::BI__builtin_vsx_extractuword: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
// Intrinsic expects the first argument to be a vector of doublewords.
- Ops[0] =
- Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2));
+ Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int64Ty, 2));
// The second argument is a compile time constant int that needs to
// be clamped to the range [0, 12].
- ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[1]);
+ ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op1);
assert(ArgCI &&
"Second Arg to xxextractuw intrinsic must be a constant integer!");
const int64_t MaxIndex = 12;
@@ -15886,29 +16022,30 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
if (getTarget().isLittleEndian()) {
// Reverse the index.
Index = MaxIndex - Index;
- Ops[1] = ConstantInt::getSigned(Int32Ty, Index);
+ Op1 = ConstantInt::getSigned(Int32Ty, Index);
// Emit the call, then reverse the double words of the results vector.
- Value *Call = Builder.CreateCall(F, Ops);
+ Value *Call = Builder.CreateCall(F, {Op0, Op1});
Value *ShuffleCall =
Builder.CreateShuffleVector(Call, Call, ArrayRef<int>{1, 0});
return ShuffleCall;
} else {
- Ops[1] = ConstantInt::getSigned(Int32Ty, Index);
- return Builder.CreateCall(F, Ops);
+ Op1 = ConstantInt::getSigned(Int32Ty, Index);
+ return Builder.CreateCall(F, {Op0, Op1});
}
}
case PPC::BI__builtin_vsx_xxpermdi: {
- ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
assert(ArgCI && "Third arg must be constant integer!");
unsigned Index = ArgCI->getZExtValue();
- Ops[0] =
- Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2));
- Ops[1] =
- Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(Int64Ty, 2));
+ Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int64Ty, 2));
+ Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int64Ty, 2));
// Account for endianness by treating this as just a shuffle. So we use the
// same indices for both LE and BE in order to produce expected results in
@@ -15917,21 +16054,21 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
int ElemIdx1 = 2 + (Index & 1);
int ShuffleElts[2] = {ElemIdx0, ElemIdx1};
- Value *ShuffleCall =
- Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleElts);
+ Value *ShuffleCall = Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
QualType BIRetType = E->getType();
auto RetTy = ConvertType(BIRetType);
return Builder.CreateBitCast(ShuffleCall, RetTy);
}
case PPC::BI__builtin_vsx_xxsldwi: {
- ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
assert(ArgCI && "Third argument must be a compile time constant");
unsigned Index = ArgCI->getZExtValue() & 0x3;
- Ops[0] =
- Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 4));
- Ops[1] =
- Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(Int32Ty, 4));
+ Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int32Ty, 4));
+ Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int32Ty, 4));
// Create a shuffle mask
int ElemIdx0;
@@ -15955,28 +16092,31 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
}
int ShuffleElts[4] = {ElemIdx0, ElemIdx1, ElemIdx2, ElemIdx3};
- Value *ShuffleCall =
- Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleElts);
+ Value *ShuffleCall = Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
QualType BIRetType = E->getType();
auto RetTy = ConvertType(BIRetType);
return Builder.CreateBitCast(ShuffleCall, RetTy);
}
case PPC::BI__builtin_pack_vector_int128: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
bool isLittleEndian = getTarget().isLittleEndian();
Value *UndefValue =
- llvm::UndefValue::get(llvm::FixedVectorType::get(Ops[0]->getType(), 2));
+ llvm::UndefValue::get(llvm::FixedVectorType::get(Op0->getType(), 2));
Value *Res = Builder.CreateInsertElement(
- UndefValue, Ops[0], (uint64_t)(isLittleEndian ? 1 : 0));
- Res = Builder.CreateInsertElement(Res, Ops[1],
+ UndefValue, Op0, (uint64_t)(isLittleEndian ? 1 : 0));
+ Res = Builder.CreateInsertElement(Res, Op1,
(uint64_t)(isLittleEndian ? 0 : 1));
return Builder.CreateBitCast(Res, ConvertType(E->getType()));
}
case PPC::BI__builtin_unpack_vector_int128: {
- ConstantInt *Index = cast<ConstantInt>(Ops[1]);
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ ConstantInt *Index = cast<ConstantInt>(Op1);
Value *Unpacked = Builder.CreateBitCast(
- Ops[0], llvm::FixedVectorType::get(ConvertType(E->getType()), 2));
+ Op0, llvm::FixedVectorType::get(ConvertType(E->getType()), 2));
if (getTarget().isLittleEndian())
Index = ConstantInt::get(Index->getType(), 1 - Index->getZExtValue());
@@ -15986,9 +16126,9 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
case PPC::BI__builtin_ppc_sthcx: {
llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_sthcx);
- Ops[0] = Builder.CreateBitCast(Ops[0], Int8PtrTy);
- Ops[1] = Builder.CreateSExt(Ops[1], Int32Ty);
- return Builder.CreateCall(F, Ops);
+ Value *Op0 = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int8PtrTy);
+ Value *Op1 = Builder.CreateSExt(EmitScalarExpr(E->getArg(1)), Int32Ty);
+ return Builder.CreateCall(F, {Op0, Op1});
}
// The PPC MMA builtins take a pointer to a __vector_quad as an argument.
@@ -16001,6 +16141,12 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
case PPC::BI__builtin_##Name:
#include "clang/Basic/BuiltinsPPC.def"
{
+ SmallVector<Value *, 4> Ops;
+ for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
+ if (E->getArg(i)->getType()->isArrayType())
+ Ops.push_back(EmitArrayToPointerDecay(E->getArg(i)).getPointer());
+ else
+ Ops.push_back(EmitScalarExpr(E->getArg(i)));
// The first argument of these two builtins is a pointer used to store their
// result. However, the llvm intrinsics return their result in multiple
// return values. So, here we emit code extracting these values from the
@@ -16084,8 +16230,9 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
Value *OldVal = Builder.CreateLoad(OldValAddr);
QualType AtomicTy = E->getArg(0)->getType()->getPointeeType();
LValue LV = MakeAddrLValue(Addr, AtomicTy);
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
auto Pair = EmitAtomicCompareExchange(
- LV, RValue::get(OldVal), RValue::get(Ops[2]), E->getExprLoc(),
+ LV, RValue::get(OldVal), RValue::get(Op2), E->getExprLoc(),
llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Monotonic, true);
// Unlike c11's atomic_compare_exchange, accroding to
// https://www.ibm.com/docs/en/xl-c-and-cpp-aix/16.1?topic=functions-compare-swap-compare-swaplp
@@ -16125,38 +16272,45 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
case PPC::BI__builtin_ppc_lbarx:
return emitPPCLoadReserveIntrinsic(*this, BuiltinID, E);
case PPC::BI__builtin_ppc_mfspr: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
llvm::Type *RetType = CGM.getDataLayout().getTypeSizeInBits(VoidPtrTy) == 32
? Int32Ty
: Int64Ty;
Function *F = CGM.getIntrinsic(Intrinsic::ppc_mfspr, RetType);
- return Builder.CreateCall(F, Ops);
+ return Builder.CreateCall(F, {Op0});
}
case PPC::BI__builtin_ppc_mtspr: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
llvm::Type *RetType = CGM.getDataLayout().getTypeSizeInBits(VoidPtrTy) == 32
? Int32Ty
: Int64Ty;
Function *F = CGM.getIntrinsic(Intrinsic::ppc_mtspr, RetType);
- return Builder.CreateCall(F, Ops);
+ return Builder.CreateCall(F, {Op0, Op1});
}
case PPC::BI__builtin_ppc_popcntb: {
Value *ArgValue = EmitScalarExpr(E->getArg(0));
llvm::Type *ArgType = ArgValue->getType();
Function *F = CGM.getIntrinsic(Intrinsic::ppc_popcntb, {ArgType, ArgType});
- return Builder.CreateCall(F, Ops, "popcntb");
+ return Builder.CreateCall(F, {ArgValue}, "popcntb");
}
case PPC::BI__builtin_ppc_mtfsf: {
// The builtin takes a uint32 that needs to be cast to an
// f64 to be passed to the intrinsic.
- Value *Cast = Builder.CreateUIToFP(Ops[1], DoubleTy);
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Cast = Builder.CreateUIToFP(Op1, DoubleTy);
llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_mtfsf);
- return Builder.CreateCall(F, {Ops[0], Cast}, "");
+ return Builder.CreateCall(F, {Op0, Cast}, "");
}
case PPC::BI__builtin_ppc_swdiv_nochk:
case PPC::BI__builtin_ppc_swdivs_nochk: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
FastMathFlags FMF = Builder.getFastMathFlags();
Builder.getFastMathFlags().setFast();
- Value *FDiv = Builder.CreateFDiv(Ops[0], Ops[1], "swdiv_nochk");
+ Value *FDiv = Builder.CreateFDiv(Op0, Op1, "swdiv_nochk");
Builder.getFastMathFlags() &= (FMF);
return FDiv;
}
@@ -16196,7 +16350,9 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
Intrinsic::experimental_constrained_sqrt))
.getScalarVal();
case PPC::BI__builtin_ppc_test_data_class: {
- llvm::Type *ArgType = EmitScalarExpr(E->getArg(0))->getType();
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ llvm::Type *ArgType = Op0->getType();
unsigned IntrinsicID;
if (ArgType->isDoubleTy())
IntrinsicID = Intrinsic::ppc_test_data_class_d;
@@ -16204,12 +16360,63 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
IntrinsicID = Intrinsic::ppc_test_data_class_f;
else
llvm_unreachable("Invalid Argument Type");
- return Builder.CreateCall(CGM.getIntrinsic(IntrinsicID), Ops,
+ return Builder.CreateCall(CGM.getIntrinsic(IntrinsicID), {Op0, Op1},
"test_data_class");
}
+ case PPC::BI__builtin_ppc_maxfe: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ Value *Op3 = EmitScalarExpr(E->getArg(3));
+ return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_maxfe),
+ {Op0, Op1, Op2, Op3});
+ }
+ case PPC::BI__builtin_ppc_maxfl: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ Value *Op3 = EmitScalarExpr(E->getArg(3));
+ return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_maxfl),
+ {Op0, Op1, Op2, Op3});
+ }
+ case PPC::BI__builtin_ppc_maxfs: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ Value *Op3 = EmitScalarExpr(E->getArg(3));
+ return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_maxfs),
+ {Op0, Op1, Op2, Op3});
+ }
+ case PPC::BI__builtin_ppc_minfe: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ Value *Op3 = EmitScalarExpr(E->getArg(3));
+ return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_minfe),
+ {Op0, Op1, Op2, Op3});
+ }
+ case PPC::BI__builtin_ppc_minfl: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ Value *Op3 = EmitScalarExpr(E->getArg(3));
+ return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_minfl),
+ {Op0, Op1, Op2, Op3});
+ }
+ case PPC::BI__builtin_ppc_minfs: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ Value *Op3 = EmitScalarExpr(E->getArg(3));
+ return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_minfs),
+ {Op0, Op1, Op2, Op3});
+ }
case PPC::BI__builtin_ppc_swdiv:
- case PPC::BI__builtin_ppc_swdivs:
- return Builder.CreateFDiv(Ops[0], Ops[1], "swdiv");
+ case PPC::BI__builtin_ppc_swdivs: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ return Builder.CreateFDiv(Op0, Op1, "swdiv");
+ }
}
}
@@ -16232,12 +16439,31 @@ Value *EmitAMDGPUDispatchPtr(CodeGenFunction &CGF,
return CGF.Builder.CreateAddrSpaceCast(Call, RetTy);
}
+Value *EmitAMDGPUImplicitArgPtr(CodeGenFunction &CGF) {
+ auto *F = CGF.CGM.getIntrinsic(Intrinsic::amdgcn_implicitarg_ptr);
+ auto *Call = CGF.Builder.CreateCall(F);
+ Call->addRetAttr(
+ Attribute::getWithDereferenceableBytes(Call->getContext(), 256));
+ Call->addRetAttr(Attribute::getWithAlignment(Call->getContext(), Align(8)));
+ return Call;
+}
+
// \p Index is 0, 1, and 2 for x, y, and z dimension, respectively.
Value *EmitAMDGPUWorkGroupSize(CodeGenFunction &CGF, unsigned Index) {
- const unsigned XOffset = 4;
- auto *DP = EmitAMDGPUDispatchPtr(CGF);
- // Indexing the HSA kernel_dispatch_packet struct.
- auto *Offset = llvm::ConstantInt::get(CGF.Int32Ty, XOffset + Index * 2);
+ bool IsCOV_5 = CGF.getTarget().getTargetOpts().CodeObjectVersion ==
+ clang::TargetOptions::COV_5;
+ Constant *Offset;
+ Value *DP;
+ if (IsCOV_5) {
+ // Indexing the implicit kernarg segment.
+ Offset = llvm::ConstantInt::get(CGF.Int32Ty, 12 + Index * 2);
+ DP = EmitAMDGPUImplicitArgPtr(CGF);
+ } else {
+ // Indexing the HSA kernel_dispatch_packet struct.
+ Offset = llvm::ConstantInt::get(CGF.Int32Ty, 4 + Index * 2);
+ DP = EmitAMDGPUDispatchPtr(CGF);
+ }
+
auto *GEP = CGF.Builder.CreateGEP(CGF.Int8Ty, DP, Offset);
auto *DstTy =
CGF.Int16Ty->getPointerTo(GEP->getType()->getPointerAddressSpace());
@@ -16506,7 +16732,9 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
- case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64: {
+ case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
+ case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
+ case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16: {
Intrinsic::ID IID;
llvm::Type *ArgTy = llvm::Type::getDoubleTy(getLLVMContext());
switch (BuiltinID) {
@@ -16537,6 +16765,15 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
IID = Intrinsic::amdgcn_flat_atomic_fmax;
break;
+ case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
+ ArgTy = llvm::Type::getFloatTy(getLLVMContext());
+ IID = Intrinsic::amdgcn_flat_atomic_fadd;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
+ ArgTy = llvm::FixedVectorType::get(
+ llvm::Type::getHalfTy(getLLVMContext()), 2);
+ IID = Intrinsic::amdgcn_flat_atomic_fadd;
+ break;
}
llvm::Value *Addr = EmitScalarExpr(E->getArg(0));
llvm::Value *Val = EmitScalarExpr(E->getArg(1));
@@ -16544,6 +16781,22 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
CGM.getIntrinsic(IID, {ArgTy, Addr->getType(), Val->getType()});
return Builder.CreateCall(F, {Addr, Val});
}
+ case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
+ case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16: {
+ Intrinsic::ID IID;
+ switch (BuiltinID) {
+ case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
+ IID = Intrinsic::amdgcn_global_atomic_fadd_v2bf16;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
+ IID = Intrinsic::amdgcn_flat_atomic_fadd_v2bf16;
+ break;
+ }
+ llvm::Value *Addr = EmitScalarExpr(E->getArg(0));
+ llvm::Value *Val = EmitScalarExpr(E->getArg(1));
+ llvm::Function *F = CGM.getIntrinsic(IID, {Addr->getType()});
+ return Builder.CreateCall(F, {Addr, Val});
+ }
case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32: {
Intrinsic::ID IID;
@@ -16608,6 +16861,69 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
RayInverseDir, TextureDescr});
}
+ case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
+ case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
+ case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
+ case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
+ case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
+ case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
+ case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
+ case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
+ case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
+ case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
+ case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
+ case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64: {
+
+ // These operations perform a matrix multiplication and accumulation of
+ // the form:
+ // D = A * B + C
+ // The return type always matches the type of matrix C.
+ unsigned ArgForMatchingRetType;
+ unsigned BuiltinWMMAOp;
+
+ switch (BuiltinID) {
+ case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
+ case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
+ ArgForMatchingRetType = 2;
+ BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
+ case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
+ ArgForMatchingRetType = 2;
+ BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
+ case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
+ ArgForMatchingRetType = 2;
+ BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
+ case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
+ ArgForMatchingRetType = 2;
+ BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
+ case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
+ ArgForMatchingRetType = 4;
+ BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
+ break;
+ case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
+ case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
+ ArgForMatchingRetType = 4;
+ BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
+ break;
+ }
+
+ SmallVector<Value *, 6> Args;
+ for (int i = 0, e = E->getNumArgs(); i != e; ++i)
+ Args.push_back(EmitScalarExpr(E->getArg(i)));
+
+ Function *F = CGM.getIntrinsic(BuiltinWMMAOp,
+ {Args[ArgForMatchingRetType]->getType()});
+
+ return Builder.CreateCall(F, Args);
+ }
+
// amdgcn workitem
case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_x, 0, 1024);
@@ -17411,18 +17727,19 @@ Value *
CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E) {
auto MakeLdg = [&](unsigned IntrinsicID) {
Value *Ptr = EmitScalarExpr(E->getArg(0));
- clang::CharUnits Align =
- CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
+ QualType ArgType = E->getArg(0)->getType();
+ clang::CharUnits Align = CGM.getNaturalPointeeTypeAlignment(ArgType);
+ llvm::Type *ElemTy = ConvertTypeForMem(ArgType->getPointeeType());
return Builder.CreateCall(
- CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
- Ptr->getType()}),
+ CGM.getIntrinsic(IntrinsicID, {ElemTy, Ptr->getType()}),
{Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())});
};
auto MakeScopedAtomic = [&](unsigned IntrinsicID) {
Value *Ptr = EmitScalarExpr(E->getArg(0));
+ llvm::Type *ElemTy =
+ ConvertTypeForMem(E->getArg(0)->getType()->getPointeeType());
return Builder.CreateCall(
- CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
- Ptr->getType()}),
+ CGM.getIntrinsic(IntrinsicID, {ElemTy, Ptr->getType()}),
{Ptr, EmitScalarExpr(E->getArg(1))});
};
switch (BuiltinID) {
@@ -17628,20 +17945,22 @@ CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E) {
case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
Value *Ptr = EmitScalarExpr(E->getArg(0));
+ llvm::Type *ElemTy =
+ ConvertTypeForMem(E->getArg(0)->getType()->getPointeeType());
return Builder.CreateCall(
CGM.getIntrinsic(
- Intrinsic::nvvm_atomic_cas_gen_i_cta,
- {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
+ Intrinsic::nvvm_atomic_cas_gen_i_cta, {ElemTy, Ptr->getType()}),
{Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
}
case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
Value *Ptr = EmitScalarExpr(E->getArg(0));
+ llvm::Type *ElemTy =
+ ConvertTypeForMem(E->getArg(0)->getType()->getPointeeType());
return Builder.CreateCall(
CGM.getIntrinsic(
- Intrinsic::nvvm_atomic_cas_gen_i_sys,
- {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
+ Intrinsic::nvvm_atomic_cas_gen_i_sys, {ElemTy, Ptr->getType()}),
{Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
}
case NVPTX::BI__nvvm_match_all_sync_i32p:
@@ -17936,7 +18255,7 @@ RValue CodeGenFunction::EmitBuiltinIsAligned(const CallExpr *E) {
/// Generate (x & ~(y-1)) to align down or ((x+(y-1)) & ~(y-1)) to align up.
/// Note: For pointer types we can avoid ptrtoint/inttoptr pairs by using the
-/// llvm.ptrmask instrinsic (with a GEP before in the align_up case).
+/// llvm.ptrmask intrinsic (with a GEP before in the align_up case).
/// TODO: actually use ptrmask once most optimization passes know about it.
RValue CodeGenFunction::EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp) {
BuiltinAlignArgs Args(E, *this);
@@ -18368,15 +18687,15 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
CGM.getIntrinsic(IntNo, {ConvertType(E->getType()), Low->getType()});
return Builder.CreateCall(Callee, {Low, High});
}
- case WebAssembly::BI__builtin_wasm_trunc_sat_zero_s_f64x2_i32x4:
- case WebAssembly::BI__builtin_wasm_trunc_sat_zero_u_f64x2_i32x4: {
+ case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
+ case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4: {
Value *Vec = EmitScalarExpr(E->getArg(0));
unsigned IntNo;
switch (BuiltinID) {
- case WebAssembly::BI__builtin_wasm_trunc_sat_zero_s_f64x2_i32x4:
+ case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
IntNo = Intrinsic::fptosi_sat;
break;
- case WebAssembly::BI__builtin_wasm_trunc_sat_zero_u_f64x2_i32x4:
+ case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4:
IntNo = Intrinsic::fptoui_sat;
break;
default:
@@ -18467,8 +18786,8 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
}
case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
- case WebAssembly::BI__builtin_wasm_relaxed_trunc_zero_s_i32x4_f64x2:
- case WebAssembly::BI__builtin_wasm_relaxed_trunc_zero_u_i32x4_f64x2: {
+ case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
+ case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2: {
Value *Vec = EmitScalarExpr(E->getArg(0));
unsigned IntNo;
switch (BuiltinID) {
@@ -18478,11 +18797,11 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
IntNo = Intrinsic::wasm_relaxed_trunc_unsigned;
break;
- case WebAssembly::BI__builtin_wasm_relaxed_trunc_zero_s_i32x4_f64x2:
- IntNo = Intrinsic::wasm_relaxed_trunc_zero_signed;
+ case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
+ IntNo = Intrinsic::wasm_relaxed_trunc_signed_zero;
break;
- case WebAssembly::BI__builtin_wasm_relaxed_trunc_zero_u_i32x4_f64x2:
- IntNo = Intrinsic::wasm_relaxed_trunc_zero_unsigned;
+ case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2:
+ IntNo = Intrinsic::wasm_relaxed_trunc_unsigned_zero;
break;
default:
llvm_unreachable("unexpected builtin ID");
@@ -18490,13 +18809,33 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
Function *Callee = CGM.getIntrinsic(IntNo);
return Builder.CreateCall(Callee, {Vec});
}
+ case WebAssembly::BI__builtin_wasm_relaxed_q15mulr_s_i16x8: {
+ Value *LHS = EmitScalarExpr(E->getArg(0));
+ Value *RHS = EmitScalarExpr(E->getArg(1));
+ Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_relaxed_q15mulr_signed);
+ return Builder.CreateCall(Callee, {LHS, RHS});
+ }
+ case WebAssembly::BI__builtin_wasm_dot_i8x16_i7x16_s_i16x8: {
+ Value *LHS = EmitScalarExpr(E->getArg(0));
+ Value *RHS = EmitScalarExpr(E->getArg(1));
+ Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_dot_i8x16_i7x16_signed);
+ return Builder.CreateCall(Callee, {LHS, RHS});
+ }
+ case WebAssembly::BI__builtin_wasm_dot_i8x16_i7x16_add_s_i32x4: {
+ Value *LHS = EmitScalarExpr(E->getArg(0));
+ Value *RHS = EmitScalarExpr(E->getArg(1));
+ Value *Acc = EmitScalarExpr(E->getArg(2));
+ Function *Callee =
+ CGM.getIntrinsic(Intrinsic::wasm_dot_i8x16_i7x16_add_signed);
+ return Builder.CreateCall(Callee, {LHS, RHS, Acc});
+ }
default:
return nullptr;
}
}
static std::pair<Intrinsic::ID, unsigned>
-getIntrinsicForHexagonNonGCCBuiltin(unsigned BuiltinID) {
+getIntrinsicForHexagonNonClangBuiltin(unsigned BuiltinID) {
struct Info {
unsigned BuiltinID;
Intrinsic::ID IntrinsicID;
@@ -18556,7 +18895,7 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
const CallExpr *E) {
Intrinsic::ID ID;
unsigned VecLen;
- std::tie(ID, VecLen) = getIntrinsicForHexagonNonGCCBuiltin(BuiltinID);
+ std::tie(ID, VecLen) = getIntrinsicForHexagonNonClangBuiltin(BuiltinID);
auto MakeCircOp = [this, E](unsigned IntID, bool IsLoad) {
// The base pointer is passed by address, so it needs to be loaded.
@@ -18733,8 +19072,34 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
SmallVector<Value *, 4> Ops;
llvm::Type *ResultType = ConvertType(E->getType());
- for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
- Ops.push_back(EmitScalarExpr(E->getArg(i)));
+ // Find out if any arguments are required to be integer constant expressions.
+ unsigned ICEArguments = 0;
+ ASTContext::GetBuiltinTypeError Error;
+ getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
+ if (Error == ASTContext::GE_Missing_type) {
+ // Vector intrinsics don't have a type string.
+ assert(BuiltinID >= clang::RISCV::FirstRVVBuiltin &&
+ BuiltinID <= clang::RISCV::LastRVVBuiltin);
+ ICEArguments = 0;
+ if (BuiltinID == RISCVVector::BI__builtin_rvv_vget_v ||
+ BuiltinID == RISCVVector::BI__builtin_rvv_vset_v)
+ ICEArguments = 1 << 1;
+ } else {
+ assert(Error == ASTContext::GE_None && "Unexpected error");
+ }
+
+ for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
+ // If this is a normal argument, just emit it as a scalar.
+ if ((ICEArguments & (1 << i)) == 0) {
+ Ops.push_back(EmitScalarExpr(E->getArg(i)));
+ continue;
+ }
+
+ // If this is required to be a constant, constant fold it so that we know
+ // that the generated intrinsic gets a ConstantInt.
+ Ops.push_back(llvm::ConstantInt::get(
+ getLLVMContext(), *E->getArg(i)->getIntegerConstantExpr(getContext())));
+ }
Intrinsic::ID ID = Intrinsic::not_intrinsic;
unsigned NF = 1;
@@ -18746,6 +19111,10 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
default: llvm_unreachable("unexpected builtin ID");
case RISCV::BI__builtin_riscv_orc_b_32:
case RISCV::BI__builtin_riscv_orc_b_64:
+ case RISCV::BI__builtin_riscv_clz_32:
+ case RISCV::BI__builtin_riscv_clz_64:
+ case RISCV::BI__builtin_riscv_ctz_32:
+ case RISCV::BI__builtin_riscv_ctz_64:
case RISCV::BI__builtin_riscv_clmul:
case RISCV::BI__builtin_riscv_clmulh:
case RISCV::BI__builtin_riscv_clmulr:
@@ -18763,6 +19132,8 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
case RISCV::BI__builtin_riscv_shfl_64:
case RISCV::BI__builtin_riscv_unshfl_32:
case RISCV::BI__builtin_riscv_unshfl_64:
+ case RISCV::BI__builtin_riscv_xperm4:
+ case RISCV::BI__builtin_riscv_xperm8:
case RISCV::BI__builtin_riscv_xperm_n:
case RISCV::BI__builtin_riscv_xperm_b:
case RISCV::BI__builtin_riscv_xperm_h:
@@ -18778,7 +19149,10 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
case RISCV::BI__builtin_riscv_fsl_32:
case RISCV::BI__builtin_riscv_fsr_32:
case RISCV::BI__builtin_riscv_fsl_64:
- case RISCV::BI__builtin_riscv_fsr_64: {
+ case RISCV::BI__builtin_riscv_fsr_64:
+ case RISCV::BI__builtin_riscv_brev8:
+ case RISCV::BI__builtin_riscv_zip_32:
+ case RISCV::BI__builtin_riscv_unzip_32: {
switch (BuiltinID) {
default: llvm_unreachable("unexpected builtin ID");
// Zbb
@@ -18786,6 +19160,16 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
case RISCV::BI__builtin_riscv_orc_b_64:
ID = Intrinsic::riscv_orc_b;
break;
+ case RISCV::BI__builtin_riscv_clz_32:
+ case RISCV::BI__builtin_riscv_clz_64: {
+ Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
+ return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)});
+ }
+ case RISCV::BI__builtin_riscv_ctz_32:
+ case RISCV::BI__builtin_riscv_ctz_64: {
+ Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType());
+ return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)});
+ }
// Zbc
case RISCV::BI__builtin_riscv_clmul:
@@ -18879,11 +19263,140 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
case RISCV::BI__builtin_riscv_fsr_64:
ID = Intrinsic::riscv_fsr;
break;
+
+ // Zbkx
+ case RISCV::BI__builtin_riscv_xperm8:
+ ID = Intrinsic::riscv_xperm8;
+ break;
+ case RISCV::BI__builtin_riscv_xperm4:
+ ID = Intrinsic::riscv_xperm4;
+ break;
+
+ // Zbkb
+ case RISCV::BI__builtin_riscv_brev8:
+ ID = Intrinsic::riscv_brev8;
+ break;
+ case RISCV::BI__builtin_riscv_zip_32:
+ ID = Intrinsic::riscv_zip;
+ break;
+ case RISCV::BI__builtin_riscv_unzip_32:
+ ID = Intrinsic::riscv_unzip;
+ break;
}
IntrinsicTypes = {ResultType};
break;
}
+
+ // Zk builtins
+
+ // Zknd
+ case RISCV::BI__builtin_riscv_aes32dsi_32:
+ ID = Intrinsic::riscv_aes32dsi;
+ break;
+ case RISCV::BI__builtin_riscv_aes32dsmi_32:
+ ID = Intrinsic::riscv_aes32dsmi;
+ break;
+ case RISCV::BI__builtin_riscv_aes64ds_64:
+ ID = Intrinsic::riscv_aes64ds;
+ break;
+ case RISCV::BI__builtin_riscv_aes64dsm_64:
+ ID = Intrinsic::riscv_aes64dsm;
+ break;
+ case RISCV::BI__builtin_riscv_aes64im_64:
+ ID = Intrinsic::riscv_aes64im;
+ break;
+
+ // Zkne
+ case RISCV::BI__builtin_riscv_aes32esi_32:
+ ID = Intrinsic::riscv_aes32esi;
+ break;
+ case RISCV::BI__builtin_riscv_aes32esmi_32:
+ ID = Intrinsic::riscv_aes32esmi;
+ break;
+ case RISCV::BI__builtin_riscv_aes64es_64:
+ ID = Intrinsic::riscv_aes64es;
+ break;
+ case RISCV::BI__builtin_riscv_aes64esm_64:
+ ID = Intrinsic::riscv_aes64esm;
+ break;
+
+ // Zknd & Zkne
+ case RISCV::BI__builtin_riscv_aes64ks1i_64:
+ ID = Intrinsic::riscv_aes64ks1i;
+ break;
+ case RISCV::BI__builtin_riscv_aes64ks2_64:
+ ID = Intrinsic::riscv_aes64ks2;
+ break;
+
+ // Zknh
+ case RISCV::BI__builtin_riscv_sha256sig0:
+ ID = Intrinsic::riscv_sha256sig0;
+ IntrinsicTypes = {ResultType};
+ break;
+ case RISCV::BI__builtin_riscv_sha256sig1:
+ ID = Intrinsic::riscv_sha256sig1;
+ IntrinsicTypes = {ResultType};
+ break;
+ case RISCV::BI__builtin_riscv_sha256sum0:
+ ID = Intrinsic::riscv_sha256sum0;
+ IntrinsicTypes = {ResultType};
+ break;
+ case RISCV::BI__builtin_riscv_sha256sum1:
+ ID = Intrinsic::riscv_sha256sum1;
+ IntrinsicTypes = {ResultType};
+ break;
+ case RISCV::BI__builtin_riscv_sha512sig0_64:
+ ID = Intrinsic::riscv_sha512sig0;
+ break;
+ case RISCV::BI__builtin_riscv_sha512sig0h_32:
+ ID = Intrinsic::riscv_sha512sig0h;
+ break;
+ case RISCV::BI__builtin_riscv_sha512sig0l_32:
+ ID = Intrinsic::riscv_sha512sig0l;
+ break;
+ case RISCV::BI__builtin_riscv_sha512sig1_64:
+ ID = Intrinsic::riscv_sha512sig1;
+ break;
+ case RISCV::BI__builtin_riscv_sha512sig1h_32:
+ ID = Intrinsic::riscv_sha512sig1h;
+ break;
+ case RISCV::BI__builtin_riscv_sha512sig1l_32:
+ ID = Intrinsic::riscv_sha512sig1l;
+ break;
+ case RISCV::BI__builtin_riscv_sha512sum0_64:
+ ID = Intrinsic::riscv_sha512sum0;
+ break;
+ case RISCV::BI__builtin_riscv_sha512sum0r_32:
+ ID = Intrinsic::riscv_sha512sum0r;
+ break;
+ case RISCV::BI__builtin_riscv_sha512sum1_64:
+ ID = Intrinsic::riscv_sha512sum1;
+ break;
+ case RISCV::BI__builtin_riscv_sha512sum1r_32:
+ ID = Intrinsic::riscv_sha512sum1r;
+ break;
+
+ // Zksed
+ case RISCV::BI__builtin_riscv_sm4ks:
+ ID = Intrinsic::riscv_sm4ks;
+ IntrinsicTypes = {ResultType};
+ break;
+ case RISCV::BI__builtin_riscv_sm4ed:
+ ID = Intrinsic::riscv_sm4ed;
+ IntrinsicTypes = {ResultType};
+ break;
+
+ // Zksh
+ case RISCV::BI__builtin_riscv_sm3p0:
+ ID = Intrinsic::riscv_sm3p0;
+ IntrinsicTypes = {ResultType};
+ break;
+ case RISCV::BI__builtin_riscv_sm3p1:
+ ID = Intrinsic::riscv_sm3p1;
+ IntrinsicTypes = {ResultType};
+ break;
+
// Vector builtins are handled from here.
#include "clang/Basic/riscv_vector_builtin_cg.inc"
}
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index c4e3f7f54f4f..6f2679cb15e4 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -157,6 +157,8 @@ private:
llvm::Function *makeModuleDtorFunction();
/// Transform managed variables for device compilation.
void transformManagedVars();
+ /// Create offloading entries to register globals in RDC mode.
+ void createOffloadingEntries();
public:
CGNVCUDARuntime(CodeGenModule &CGM);
@@ -210,7 +212,8 @@ static std::unique_ptr<MangleContext> InitDeviceMC(CodeGenModule &CGM) {
CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM)
: CGCUDARuntime(CGM), Context(CGM.getLLVMContext()),
TheModule(CGM.getModule()),
- RelocatableDeviceCode(CGM.getLangOpts().GPURelocatableDeviceCode),
+ RelocatableDeviceCode(CGM.getLangOpts().GPURelocatableDeviceCode ||
+ CGM.getLangOpts().OffloadingNewDriver),
DeviceMC(InitDeviceMC(CGM)) {
CodeGen::CodeGenTypes &Types = CGM.getTypes();
ASTContext &Ctx = CGM.getContext();
@@ -281,13 +284,12 @@ std::string CGNVCUDARuntime::getDeviceSideName(const NamedDecl *ND) {
DeviceSideName = std::string(ND->getIdentifier()->getName());
// Make unique name for device side static file-scope variable for HIP.
- if (CGM.getContext().shouldExternalizeStaticVar(ND) &&
- CGM.getLangOpts().GPURelocatableDeviceCode &&
- !CGM.getLangOpts().CUID.empty()) {
+ if (CGM.getContext().shouldExternalize(ND) &&
+ CGM.getLangOpts().GPURelocatableDeviceCode) {
SmallString<256> Buffer;
llvm::raw_svector_ostream Out(Buffer);
Out << DeviceSideName;
- CGM.printPostfixForExternalizedStaticVar(Out);
+ CGM.printPostfixForExternalizedDecl(Out, ND);
DeviceSideName = std::string(Out.str());
}
return DeviceSideName;
@@ -332,15 +334,22 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end");
// Lookup cudaLaunchKernel/hipLaunchKernel function.
+ // HIP kernel launching API name depends on -fgpu-default-stream option. For
+ // the default value 'legacy', it is hipLaunchKernel. For 'per-thread',
+ // it is hipLaunchKernel_spt.
// cudaError_t cudaLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
// void **args, size_t sharedMem,
// cudaStream_t stream);
- // hipError_t hipLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
- // void **args, size_t sharedMem,
- // hipStream_t stream);
+ // hipError_t hipLaunchKernel[_spt](const void *func, dim3 gridDim,
+ // dim3 blockDim, void **args,
+ // size_t sharedMem, hipStream_t stream);
TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl();
DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl);
- auto LaunchKernelName = addPrefixToName("LaunchKernel");
+ std::string KernelLaunchAPI = "LaunchKernel";
+ if (CGF.getLangOpts().HIP && CGF.getLangOpts().GPUDefaultStream ==
+ LangOptions::GPUDefaultStreamKind::PerThread)
+ KernelLaunchAPI = KernelLaunchAPI + "_spt";
+ auto LaunchKernelName = addPrefixToName(KernelLaunchAPI);
IdentifierInfo &cudaLaunchKernelII =
CGM.getContext().Idents.get(LaunchKernelName);
FunctionDecl *cudaLaunchKernelFD = nullptr;
@@ -652,7 +661,7 @@ llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() {
///
/// For CUDA:
/// \code
-/// void __cuda_module_ctor(void*) {
+/// void __cuda_module_ctor() {
/// Handle = __cudaRegisterFatBinary(GpuBinaryBlob);
/// __cuda_register_globals(Handle);
/// }
@@ -660,7 +669,7 @@ llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() {
///
/// For HIP:
/// \code
-/// void __hip_module_ctor(void*) {
+/// void __hip_module_ctor() {
/// if (__hip_gpubin_handle == 0) {
/// __hip_gpubin_handle = __hipRegisterFatBinary(GpuBinaryBlob);
/// __hip_register_globals(__hip_gpubin_handle);
@@ -710,7 +719,7 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() {
}
llvm::Function *ModuleCtorFunc = llvm::Function::Create(
- llvm::FunctionType::get(VoidTy, VoidPtrTy, false),
+ llvm::FunctionType::get(VoidTy, false),
llvm::GlobalValue::InternalLinkage,
addUnderscoredPrefixToName("_module_ctor"), &TheModule);
llvm::BasicBlock *CtorEntryBB =
@@ -822,7 +831,7 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() {
if (Linkage != llvm::GlobalValue::InternalLinkage)
GpuBinaryHandle->setVisibility(llvm::GlobalValue::HiddenVisibility);
Address GpuBinaryAddr(
- GpuBinaryHandle,
+ GpuBinaryHandle, VoidPtrPtrTy,
CharUnits::fromQuantity(GpuBinaryHandle->getAlignment()));
{
auto *HandleValue = CtorBuilder.CreateLoad(GpuBinaryAddr);
@@ -924,14 +933,14 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() {
///
/// For CUDA:
/// \code
-/// void __cuda_module_dtor(void*) {
+/// void __cuda_module_dtor() {
/// __cudaUnregisterFatBinary(Handle);
/// }
/// \endcode
///
/// For HIP:
/// \code
-/// void __hip_module_dtor(void*) {
+/// void __hip_module_dtor() {
/// if (__hip_gpubin_handle) {
/// __hipUnregisterFatBinary(__hip_gpubin_handle);
/// __hip_gpubin_handle = 0;
@@ -949,7 +958,7 @@ llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() {
addUnderscoredPrefixToName("UnregisterFatBinary"));
llvm::Function *ModuleDtorFunc = llvm::Function::Create(
- llvm::FunctionType::get(VoidTy, VoidPtrTy, false),
+ llvm::FunctionType::get(VoidTy, false),
llvm::GlobalValue::InternalLinkage,
addUnderscoredPrefixToName("_module_dtor"), &TheModule);
@@ -958,8 +967,9 @@ llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() {
CGBuilderTy DtorBuilder(CGM, Context);
DtorBuilder.SetInsertPoint(DtorEntryBB);
- Address GpuBinaryAddr(GpuBinaryHandle, CharUnits::fromQuantity(
- GpuBinaryHandle->getAlignment()));
+ Address GpuBinaryAddr(
+ GpuBinaryHandle, GpuBinaryHandle->getValueType(),
+ CharUnits::fromQuantity(GpuBinaryHandle->getAlignment()));
auto *HandleValue = DtorBuilder.CreateLoad(GpuBinaryAddr);
// There is only one HIP fat binary per linked module, however there are
// multiple destructor functions. Make sure the fat binary is unregistered
@@ -1099,6 +1109,40 @@ void CGNVCUDARuntime::transformManagedVars() {
}
}
+// Creates offloading entries for all the kernels and globals that must be
+// registered. The linker will provide a pointer to this section so we can
+// register the symbols with the linked device image.
+void CGNVCUDARuntime::createOffloadingEntries() {
+ llvm::OpenMPIRBuilder OMPBuilder(CGM.getModule());
+ OMPBuilder.initialize();
+
+ StringRef Section = "cuda_offloading_entries";
+ for (KernelInfo &I : EmittedKernels)
+ OMPBuilder.emitOffloadingEntry(KernelHandles[I.Kernel],
+ getDeviceSideName(cast<NamedDecl>(I.D)), 0,
+ DeviceVarFlags::OffloadGlobalEntry, Section);
+
+ for (VarInfo &I : DeviceVars) {
+ uint64_t VarSize =
+ CGM.getDataLayout().getTypeAllocSize(I.Var->getValueType());
+ if (I.Flags.getKind() == DeviceVarFlags::Variable) {
+ OMPBuilder.emitOffloadingEntry(
+ I.Var, getDeviceSideName(I.D), VarSize,
+ I.Flags.isManaged() ? DeviceVarFlags::OffloadGlobalManagedEntry
+ : DeviceVarFlags::OffloadGlobalEntry,
+ Section);
+ } else if (I.Flags.getKind() == DeviceVarFlags::Surface) {
+ OMPBuilder.emitOffloadingEntry(I.Var, getDeviceSideName(I.D), VarSize,
+ DeviceVarFlags::OffloadGlobalSurfaceEntry,
+ Section);
+ } else if (I.Flags.getKind() == DeviceVarFlags::Texture) {
+ OMPBuilder.emitOffloadingEntry(I.Var, getDeviceSideName(I.D), VarSize,
+ DeviceVarFlags::OffloadGlobalTextureEntry,
+ Section);
+ }
+ }
+}
+
// Returns module constructor to be added.
llvm::Function *CGNVCUDARuntime::finalizeModule() {
if (CGM.getLangOpts().CUDAIsDevice) {
@@ -1127,7 +1171,11 @@ llvm::Function *CGNVCUDARuntime::finalizeModule() {
}
return nullptr;
}
- return makeModuleCtorFunction();
+ if (!(CGM.getLangOpts().OffloadingNewDriver && RelocatableDeviceCode))
+ return makeModuleCtorFunction();
+
+ createOffloadingEntries();
+ return nullptr;
}
llvm::GlobalValue *CGNVCUDARuntime::getKernelHandle(llvm::Function *F,
diff --git a/clang/lib/CodeGen/CGCUDARuntime.h b/clang/lib/CodeGen/CGCUDARuntime.h
index 1c119dc77fd4..73c7ca7bc15f 100644
--- a/clang/lib/CodeGen/CGCUDARuntime.h
+++ b/clang/lib/CodeGen/CGCUDARuntime.h
@@ -52,6 +52,19 @@ public:
Texture, // Builtin texture
};
+ /// The kind flag for an offloading entry.
+ enum OffloadEntryKindFlag : uint32_t {
+ /// Mark the entry as a global entry. This indicates the presense of a
+ /// kernel if the size size field is zero and a variable otherwise.
+ OffloadGlobalEntry = 0x0,
+ /// Mark the entry as a managed global variable.
+ OffloadGlobalManagedEntry = 0x1,
+ /// Mark the entry as a surface variable.
+ OffloadGlobalSurfaceEntry = 0x2,
+ /// Mark the entry as a texture variable.
+ OffloadGlobalTextureEntry = 0x3,
+ };
+
private:
unsigned Kind : 2;
unsigned Extern : 1;
diff --git a/clang/lib/CodeGen/CGCXXABI.cpp b/clang/lib/CodeGen/CGCXXABI.cpp
index 0b441e382f11..42e6c916bed0 100644
--- a/clang/lib/CodeGen/CGCXXABI.cpp
+++ b/clang/lib/CodeGen/CGCXXABI.cpp
@@ -45,8 +45,7 @@ CGCallee CGCXXABI::EmitLoadOfMemberFunctionPointer(
ErrorUnsupportedABI(CGF, "calls through member pointers");
ThisPtrForCall = This.getPointer();
- const FunctionProtoType *FPT =
- MPT->getPointeeType()->getAs<FunctionProtoType>();
+ const auto *FPT = MPT->getPointeeType()->castAs<FunctionProtoType>();
const auto *RD =
cast<CXXRecordDecl>(MPT->getClass()->castAs<RecordType>()->getDecl());
llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(
diff --git a/clang/lib/CodeGen/CGCXXABI.h b/clang/lib/CodeGen/CGCXXABI.h
index b96222b3ce28..a46f7f37141f 100644
--- a/clang/lib/CodeGen/CGCXXABI.h
+++ b/clang/lib/CodeGen/CGCXXABI.h
@@ -56,7 +56,10 @@ protected:
return CGF.CXXABIThisValue;
}
Address getThisAddress(CodeGenFunction &CGF) {
- return Address(CGF.CXXABIThisValue, CGF.CXXABIThisAlignment);
+ return Address(
+ CGF.CXXABIThisValue,
+ CGF.ConvertTypeForMem(CGF.CXXABIThisDecl->getType()->getPointeeType()),
+ CGF.CXXABIThisAlignment);
}
/// Issue a diagnostic about unsupported features in the ABI.
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index a37ff8844e88..4e26c35c6342 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -38,6 +38,7 @@
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Type.h"
#include "llvm/Transforms/Utils/Local.h"
using namespace clang;
using namespace CodeGen;
@@ -61,6 +62,8 @@ unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
// TODO: Add support for __vectorcall to LLVM.
case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall;
+ case CC_AArch64SVEPCS: return llvm::CallingConv::AArch64_SVE_VectorCall;
+ case CC_AMDGPUKernelCall: return llvm::CallingConv::AMDGPU_KERNEL;
case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
case CC_PreserveMost: return llvm::CallingConv::PreserveMost;
@@ -227,6 +230,12 @@ static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
if (D->hasAttr<AArch64VectorPcsAttr>())
return CC_AArch64VectorCall;
+ if (D->hasAttr<AArch64SVEPcsAttr>())
+ return CC_AArch64SVEPCS;
+
+ if (D->hasAttr<AMDGPUKernelCallAttr>())
+ return CC_AMDGPUKernelCall;
+
if (D->hasAttr<IntelOclBiccAttr>())
return CC_IntelOclBicc;
@@ -832,6 +841,7 @@ CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
FI->NumArgs = argTypes.size();
FI->HasExtParameterInfos = !paramInfos.empty();
FI->getArgsBuffer()[0].type = resultType;
+ FI->MaxVectorWidth = 0;
for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
FI->getArgsBuffer()[i + 1].type = argTypes[i];
for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
@@ -941,8 +951,7 @@ getTypeExpansion(QualType Ty, const ASTContext &Context) {
if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
assert(!CXXRD->isDynamicClass() &&
"cannot expand vtable pointers in dynamic classes");
- for (const CXXBaseSpecifier &BS : CXXRD->bases())
- Bases.push_back(&BS);
+ llvm::append_range(Bases, llvm::make_pointer_range(CXXRD->bases()));
}
for (const auto *FD : RD->fields()) {
@@ -1011,11 +1020,12 @@ static void forConstantArrayExpansion(CodeGenFunction &CGF,
CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
CharUnits EltAlign =
BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
+ llvm::Type *EltTy = CGF.ConvertTypeForMem(CAE->EltTy);
for (int i = 0, n = CAE->NumElts; i < n; i++) {
llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32(
BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i);
- Fn(Address(EltAddr, EltAlign));
+ Fn(Address(EltAddr, EltTy, EltAlign));
}
}
@@ -1056,10 +1066,19 @@ void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
// Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
// primitive store.
assert(isa<NoExpansion>(Exp.get()));
- if (LV.isBitField())
- EmitStoreThroughLValue(RValue::get(&*AI++), LV);
- else
- EmitStoreOfScalar(&*AI++, LV);
+ llvm::Value *Arg = &*AI++;
+ if (LV.isBitField()) {
+ EmitStoreThroughLValue(RValue::get(Arg), LV);
+ } else {
+ // TODO: currently there are some places are inconsistent in what LLVM
+ // pointer type they use (see D118744). Once clang uses opaque pointers
+ // all LLVM pointer types will be the same and we can remove this check.
+ if (Arg->getType()->isPointerTy()) {
+ Address Addr = LV.getAddress(*this);
+ Arg = Builder.CreateBitCast(Arg, Addr.getElementType());
+ }
+ EmitStoreOfScalar(Arg, LV);
+ }
}
}
@@ -1266,8 +1285,8 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
}
// If coercing a fixed vector to a scalable vector for ABI compatibility, and
- // the types match, use the llvm.experimental.vector.insert intrinsic to
- // perform the conversion.
+ // the types match, use the llvm.vector.insert intrinsic to perform the
+ // conversion.
if (auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(Ty)) {
if (auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) {
// If we are casting a fixed i8 vector to a scalable 16 x i1 predicate
@@ -1794,6 +1813,8 @@ void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
if (AttrOnCallSite) {
// Attributes that should go on the call site only.
+ // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
+ // the -fno-builtin-foo list.
if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
if (!CodeGenOpts.TrapFuncName.empty())
@@ -1828,7 +1849,7 @@ void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
CodeGenOpts.FP32DenormalMode.str());
}
- if (LangOpts.getFPExceptionMode() == LangOptions::FPE_Ignore)
+ if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
FuncAttrs.addAttribute("no-trapping-math", "true");
// TODO: Are these all needed?
@@ -1868,6 +1889,37 @@ void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
if (CodeGenOpts.SpeculativeLoadHardening)
FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
+
+ // Add zero-call-used-regs attribute.
+ switch (CodeGenOpts.getZeroCallUsedRegs()) {
+ case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
+ FuncAttrs.removeAttribute("zero-call-used-regs");
+ break;
+ case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
+ FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr-arg");
+ break;
+ case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
+ FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr");
+ break;
+ case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
+ FuncAttrs.addAttribute("zero-call-used-regs", "used-arg");
+ break;
+ case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
+ FuncAttrs.addAttribute("zero-call-used-regs", "used");
+ break;
+ case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
+ FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr-arg");
+ break;
+ case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
+ FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr");
+ break;
+ case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
+ FuncAttrs.addAttribute("zero-call-used-regs", "all-arg");
+ break;
+ case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
+ FuncAttrs.addAttribute("zero-call-used-regs", "all");
+ break;
+ }
}
if (getLangOpts().assumeFunctionsAreConvergent()) {
@@ -2156,6 +2208,15 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
if (TargetDecl->hasAttr<NoSplitStackAttr>())
FuncAttrs.removeAttribute("split-stack");
+ if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
+ // A function "__attribute__((...))" overrides the command-line flag.
+ auto Kind =
+ TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
+ FuncAttrs.removeAttribute("zero-call-used-regs");
+ FuncAttrs.addAttribute(
+ "zero-call-used-regs",
+ ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Kind));
+ }
// Add NonLazyBind attribute to function declarations when -fno-plt
// is used.
@@ -2243,7 +2304,7 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
getLangOpts().Sanitize.has(SanitizerKind::Return);
// Determine if the return type could be partially undef
- if (!CodeGenOpts.DisableNoundefAttrs && HasStrictReturn) {
+ if (CodeGenOpts.EnableNoundefAttrs && HasStrictReturn) {
if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
DetermineNoUndef(RetTy, getTypes(), DL, RetAI))
RetAttrs.addAttribute(llvm::Attribute::NoUndef);
@@ -2377,7 +2438,7 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
}
// Decide whether the argument we're handling could be partially undef
- if (!CodeGenOpts.DisableNoundefAttrs &&
+ if (CodeGenOpts.EnableNoundefAttrs &&
DetermineNoUndef(ParamType, getTypes(), DL, AI)) {
Attrs.addAttribute(llvm::Attribute::NoUndef);
}
@@ -2475,6 +2536,20 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
}
}
+ // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
+ // > For arguments to a __kernel function declared to be a pointer to a
+ // > data type, the OpenCL compiler can assume that the pointee is always
+ // > appropriately aligned as required by the data type.
+ if (TargetDecl && TargetDecl->hasAttr<OpenCLKernelAttr>() &&
+ ParamType->isPointerType()) {
+ QualType PTy = ParamType->getPointeeType();
+ if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
+ llvm::Align Alignment =
+ getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
+ Attrs.addAlignmentAttr(Alignment);
+ }
+ }
+
switch (FI.getExtParameterInfo(ArgNo).getABI()) {
case ParameterABI::Ordinary:
break;
@@ -2622,7 +2697,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
Address ArgStruct = Address::invalid();
if (IRFunctionArgs.hasInallocaArg()) {
ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
- FI.getArgStructAlignment());
+ FI.getArgStruct(), FI.getArgStructAlignment());
assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
}
@@ -2672,7 +2747,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
Address V =
Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
if (ArgI.getInAllocaIndirect())
- V = Address(Builder.CreateLoad(V),
+ V = Address(Builder.CreateLoad(V), ConvertTypeForMem(Ty),
getContext().getTypeAlignInChars(Ty));
ArgVals.push_back(ParamValue::forIndirect(V));
break;
@@ -2821,7 +2896,8 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
assert(pointeeTy->isPointerType());
Address temp =
CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
- Address arg = Address(V, getContext().getTypeAlignInChars(pointeeTy));
+ Address arg(V, ConvertTypeForMem(pointeeTy),
+ getContext().getTypeAlignInChars(pointeeTy));
llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
Builder.CreateStore(incomingErrorValue, temp);
V = temp.getPointer();
@@ -2854,8 +2930,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
// VLST arguments are coerced to VLATs at the function boundary for
// ABI consistency. If this is a VLST that was coerced to
// a VLAT at the function boundary and the types match up, use
- // llvm.experimental.vector.extract to convert back to the original
- // VLST.
+ // llvm.vector.extract to convert back to the original VLST.
if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) {
llvm::Value *Coerced = Fn->getArg(FirstIRArg);
if (auto *VecTyFrom =
@@ -3162,7 +3237,8 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
// ReturnValue to some other location.
auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
auto *SI = dyn_cast<llvm::StoreInst>(U);
- if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer())
+ if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer() ||
+ SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
return nullptr;
// These aren't actually possible for non-coerced returns, and we
// only care about non-coerced returns on this code path.
@@ -3176,28 +3252,19 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
if (IP->empty()) return nullptr;
- llvm::Instruction *I = &IP->back();
- // Skip lifetime markers
- for (llvm::BasicBlock::reverse_iterator II = IP->rbegin(),
- IE = IP->rend();
- II != IE; ++II) {
- if (llvm::IntrinsicInst *Intrinsic =
- dyn_cast<llvm::IntrinsicInst>(&*II)) {
- if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) {
- const llvm::Value *CastAddr = Intrinsic->getArgOperand(1);
- ++II;
- if (II == IE)
- break;
- if (isa<llvm::BitCastInst>(&*II) && (CastAddr == &*II))
- continue;
- }
- }
- I = &*II;
- break;
- }
+ // Look at directly preceding instruction, skipping bitcasts and lifetime
+ // markers.
+ for (llvm::Instruction &I : make_range(IP->rbegin(), IP->rend())) {
+ if (isa<llvm::BitCastInst>(&I))
+ continue;
+ if (auto *II = dyn_cast<llvm::IntrinsicInst>(&I))
+ if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
+ continue;
- return GetStoreIfValid(I);
+ return GetStoreIfValid(&I);
+ }
+ return nullptr;
}
llvm::StoreInst *store =
@@ -3398,7 +3465,7 @@ llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
int CharsPerElt =
ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
int MaskIndex = 0;
- llvm::Value *R = llvm::UndefValue::get(ATy);
+ llvm::Value *R = llvm::PoisonValue::get(ATy);
for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth,
DataLayout.isBigEndian());
@@ -3447,8 +3514,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
--EI;
llvm::Value *ArgStruct = &*EI;
llvm::Value *SRet = Builder.CreateStructGEP(
- EI->getType()->getPointerElementType(), ArgStruct,
- RetAI.getInAllocaFieldIndex());
+ FI.getArgStruct(), ArgStruct, RetAI.getInAllocaFieldIndex());
llvm::Type *Ty =
cast<llvm::GetElementPtrInst>(SRet)->getResultElementType();
RV = Builder.CreateAlignedLoad(Ty, SRet, getPointerAlign(), "sret");
@@ -3573,7 +3639,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
// Construct a return type that lacks padding elements.
llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
- RV = llvm::UndefValue::get(returnType);
+ RV = llvm::PoisonValue::get(returnType);
for (unsigned i = 0, e = results.size(); i != e; ++i) {
RV = Builder.CreateInsertValue(RV, results[i], i);
}
@@ -3680,14 +3746,14 @@ static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
// placeholders.
llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
llvm::Type *IRPtrTy = IRTy->getPointerTo();
- llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo());
+ llvm::Value *Placeholder = llvm::PoisonValue::get(IRPtrTy->getPointerTo());
// FIXME: When we generate this IR in one pass, we shouldn't need
// this win32-specific alignment hack.
CharUnits Align = CharUnits::fromQuantity(4);
Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
- return AggValueSlot::forAddr(Address(Placeholder, Align),
+ return AggValueSlot::forAddr(Address(Placeholder, IRTy, Align),
Ty.getQualifiers(),
AggValueSlot::IsNotDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
@@ -3870,7 +3936,9 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
// because of the crazy ObjC compatibility rules.
llvm::PointerType *destType =
- cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
+ cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
+ llvm::Type *destElemType =
+ CGF.ConvertTypeForMem(CRE->getType()->getPointeeType());
// If the address is a constant null, just pass the appropriate null.
if (isProvablyNull(srcAddr.getPointer())) {
@@ -3880,8 +3948,8 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
}
// Create the temporary.
- Address temp = CGF.CreateTempAlloca(destType->getPointerElementType(),
- CGF.getPointerAlign(), "icr.temp");
+ Address temp =
+ CGF.CreateTempAlloca(destElemType, CGF.getPointerAlign(), "icr.temp");
// Loading an l-value can introduce a cleanup if the l-value is __weak,
// and that cleanup will be conditional if we can't prove that the l-value
// isn't null, so we need to register a dominating point so that the cleanups
@@ -3891,8 +3959,8 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
// Zero-initialize it if we're not doing a copy-initialization.
bool shouldCopy = CRE->shouldCopy();
if (!shouldCopy) {
- llvm::Value *null = llvm::ConstantPointerNull::get(
- cast<llvm::PointerType>(destType->getPointerElementType()));
+ llvm::Value *null =
+ llvm::ConstantPointerNull::get(cast<llvm::PointerType>(destElemType));
CGF.Builder.CreateStore(null, temp);
}
@@ -3934,8 +4002,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
assert(srcRV.isScalar());
llvm::Value *src = srcRV.getScalarVal();
- src = CGF.Builder.CreateBitCast(src, destType->getPointerElementType(),
- "icr.cast");
+ src = CGF.Builder.CreateBitCast(src, destElemType, "icr.cast");
// Use an ordinary store, not a store-to-lvalue.
CGF.Builder.CreateStore(src, temp);
@@ -4331,7 +4398,7 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
type);
// This unreachable is a temporary marker which will be removed later.
llvm::Instruction *IsActive = Builder.CreateUnreachable();
- args.addArgCleanupDeactivation(EHStack.getInnermostEHScope(), IsActive);
+ args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive);
}
return;
}
@@ -4603,6 +4670,19 @@ public:
} // namespace
+static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
+ if (auto *VT = dyn_cast<llvm::VectorType>(Ty))
+ return VT->getPrimitiveSizeInBits().getKnownMinSize();
+ if (auto *AT = dyn_cast<llvm::ArrayType>(Ty))
+ return getMaxVectorWidth(AT->getElementType());
+
+ unsigned MaxVectorWidth = 0;
+ if (auto *ST = dyn_cast<llvm::StructType>(Ty))
+ for (auto *I : ST->elements())
+ MaxVectorWidth = std::max(MaxVectorWidth, getMaxVectorWidth(I));
+ return MaxVectorWidth;
+}
+
RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
const CGCallee &Callee,
ReturnValueSlot ReturnValue,
@@ -4677,7 +4757,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
AI->setAlignment(Align.getAsAlign());
AI->setUsedWithInAlloca(true);
assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
- ArgMemory = Address(AI, Align);
+ ArgMemory = Address(AI, ArgStruct, Align);
}
ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
@@ -4775,13 +4855,10 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
// Store the RValue into the argument struct.
Address Addr =
Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
- unsigned AS = Addr.getType()->getPointerAddressSpace();
- llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS);
// There are some cases where a trivial bitcast is not avoidable. The
// definition of a type later in a translation unit may change it's type
// from {}* to (%struct.foo*)*.
- if (Addr.getType() != MemType)
- Addr = Builder.CreateBitCast(Addr, MemType);
+ Addr = Builder.CreateElementBitCast(Addr, ConvertTypeForMem(I->Ty));
I->copyInto(*this, Addr);
}
break;
@@ -4904,8 +4981,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
QualType pointeeTy = I->Ty->getPointeeType();
- swiftErrorArg =
- Address(V, getContext().getTypeAlignInChars(pointeeTy));
+ swiftErrorArg = Address(V, ConvertTypeForMem(pointeeTy),
+ getContext().getTypeAlignInChars(pointeeTy));
swiftErrorTemp =
CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
@@ -5073,14 +5150,16 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
#ifndef NDEBUG
// Assert that these structs have equivalent element types.
llvm::StructType *FullTy = CallInfo.getArgStruct();
- llvm::StructType *DeclaredTy =
- cast<llvm::StructType>(LastParamTy->getPointerElementType());
- assert(DeclaredTy->getNumElements() == FullTy->getNumElements());
- for (llvm::StructType::element_iterator DI = DeclaredTy->element_begin(),
- DE = DeclaredTy->element_end(),
- FI = FullTy->element_begin();
- DI != DE; ++DI, ++FI)
- assert(*DI == *FI);
+ if (!LastParamTy->isOpaquePointerTy()) {
+ llvm::StructType *DeclaredTy = cast<llvm::StructType>(
+ LastParamTy->getNonOpaquePointerElementType());
+ assert(DeclaredTy->getNumElements() == FullTy->getNumElements());
+ for (auto DI = DeclaredTy->element_begin(),
+ DE = DeclaredTy->element_end(),
+ FI = FullTy->element_begin();
+ DI != DE; ++DI, ++FI)
+ assert(*DI == *FI);
+ }
#endif
Arg = Builder.CreateBitCast(Arg, LastParamTy);
}
@@ -5157,12 +5236,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
#endif
// Update the largest vector width if any arguments have vector types.
- for (unsigned i = 0; i < IRCallArgs.size(); ++i) {
- if (auto *VT = dyn_cast<llvm::VectorType>(IRCallArgs[i]->getType()))
- LargestVectorWidth =
- std::max((uint64_t)LargestVectorWidth,
- VT->getPrimitiveSizeInBits().getKnownMinSize());
- }
+ for (unsigned i = 0; i < IRCallArgs.size(); ++i)
+ LargestVectorWidth = std::max(LargestVectorWidth,
+ getMaxVectorWidth(IRCallArgs[i]->getType()));
// Compute the calling convention and attributes.
unsigned CallingConv;
@@ -5181,12 +5257,22 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
if (InNoMergeAttributedStmt)
Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoMerge);
+ // Add call-site noinline attribute if exists.
+ if (InNoInlineAttributedStmt)
+ Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);
+
+ // Add call-site always_inline attribute if exists.
+ if (InAlwaysInlineAttributedStmt)
+ Attrs =
+ Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);
+
// Apply some call-site-specific attributes.
// TODO: work this into building the attribute set.
// Apply always_inline to all calls within flatten functions.
// FIXME: should this really take priority over __try, below?
if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
+ !InNoInlineAttributedStmt &&
!(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>())) {
Attrs =
Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);
@@ -5274,10 +5360,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
CI->setName("call");
// Update largest vector width from the return type.
- if (auto *VT = dyn_cast<llvm::VectorType>(CI->getType()))
- LargestVectorWidth =
- std::max((uint64_t)LargestVectorWidth,
- VT->getPrimitiveSizeInBits().getKnownMinSize());
+ LargestVectorWidth =
+ std::max(LargestVectorWidth, getMaxVectorWidth(CI->getType()));
// Insert instrumentation or attach profile metadata at indirect call sites.
// For more details, see the comment before the definition of
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index 76b90924750c..153f299a1c4b 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -1649,92 +1649,73 @@ namespace {
}
};
- static void EmitSanitizerDtorCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
- CharUnits::QuantityType PoisonSize) {
- CodeGenFunction::SanitizerScope SanScope(&CGF);
- // Pass in void pointer and size of region as arguments to runtime
- // function
- llvm::Value *Args[] = {CGF.Builder.CreateBitCast(Ptr, CGF.VoidPtrTy),
- llvm::ConstantInt::get(CGF.SizeTy, PoisonSize)};
+ static void EmitSanitizerDtorCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
+ CharUnits::QuantityType PoisonSize) {
+ CodeGenFunction::SanitizerScope SanScope(&CGF);
+ // Pass in void pointer and size of region as arguments to runtime
+ // function
+ llvm::Value *Args[] = {CGF.Builder.CreateBitCast(Ptr, CGF.VoidPtrTy),
+ llvm::ConstantInt::get(CGF.SizeTy, PoisonSize)};
- llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy};
+ llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy};
- llvm::FunctionType *FnType =
- llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false);
- llvm::FunctionCallee Fn =
- CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback");
- CGF.EmitNounwindRuntimeCall(Fn, Args);
- }
-
- class SanitizeDtorMembers final : public EHScopeStack::Cleanup {
- const CXXDestructorDecl *Dtor;
+ llvm::FunctionType *FnType =
+ llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false);
+ llvm::FunctionCallee Fn =
+ CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback");
+ CGF.EmitNounwindRuntimeCall(Fn, Args);
+ }
- public:
- SanitizeDtorMembers(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
+ /// Poison base class with a trivial destructor.
+ struct SanitizeDtorTrivialBase final : EHScopeStack::Cleanup {
+ const CXXRecordDecl *BaseClass;
+ bool BaseIsVirtual;
+ SanitizeDtorTrivialBase(const CXXRecordDecl *Base, bool BaseIsVirtual)
+ : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
- // Generate function call for handling object poisoning.
- // Disables tail call elimination, to prevent the current stack frame
- // from disappearing from the stack trace.
void Emit(CodeGenFunction &CGF, Flags flags) override {
- const ASTRecordLayout &Layout =
- CGF.getContext().getASTRecordLayout(Dtor->getParent());
+ const CXXRecordDecl *DerivedClass =
+ cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
- // Nothing to poison.
- if (Layout.getFieldCount() == 0)
- return;
+ Address Addr = CGF.GetAddressOfDirectBaseInCompleteClass(
+ CGF.LoadCXXThisAddress(), DerivedClass, BaseClass, BaseIsVirtual);
- // Prevent the current stack frame from disappearing from the stack trace.
- CGF.CurFn->addFnAttr("disable-tail-calls", "true");
+ const ASTRecordLayout &BaseLayout =
+ CGF.getContext().getASTRecordLayout(BaseClass);
+ CharUnits BaseSize = BaseLayout.getSize();
- // Construct pointer to region to begin poisoning, and calculate poison
- // size, so that only members declared in this class are poisoned.
- ASTContext &Context = CGF.getContext();
+ if (!BaseSize.isPositive())
+ return;
- const RecordDecl *Decl = Dtor->getParent();
- auto Fields = Decl->fields();
- auto IsTrivial = [&](const FieldDecl *F) {
- return FieldHasTrivialDestructorBody(Context, F);
- };
+ EmitSanitizerDtorCallback(CGF, Addr.getPointer(), BaseSize.getQuantity());
- auto IsZeroSize = [&](const FieldDecl *F) {
- return F->isZeroSize(Context);
- };
+ // Prevent the current stack frame from disappearing from the stack trace.
+ CGF.CurFn->addFnAttr("disable-tail-calls", "true");
+ }
+ };
- // Poison blocks of fields with trivial destructors making sure that block
- // begin and end do not point to zero-sized fields. They don't have
- // correct offsets so can't be used to calculate poisoning range.
- for (auto It = Fields.begin(); It != Fields.end();) {
- It = std::find_if(It, Fields.end(), [&](const FieldDecl *F) {
- return IsTrivial(F) && !IsZeroSize(F);
- });
- if (It == Fields.end())
- break;
- auto Start = It++;
- It = std::find_if(It, Fields.end(), [&](const FieldDecl *F) {
- return !IsTrivial(F) && !IsZeroSize(F);
- });
+ class SanitizeDtorFieldRange final : public EHScopeStack::Cleanup {
+ const CXXDestructorDecl *Dtor;
+ unsigned StartIndex;
+ unsigned EndIndex;
- PoisonMembers(CGF, (*Start)->getFieldIndex(),
- It == Fields.end() ? -1 : (*It)->getFieldIndex());
- }
- }
+ public:
+ SanitizeDtorFieldRange(const CXXDestructorDecl *Dtor, unsigned StartIndex,
+ unsigned EndIndex)
+ : Dtor(Dtor), StartIndex(StartIndex), EndIndex(EndIndex) {}
- private:
- /// \param layoutStartOffset index of the ASTRecordLayout field to
- /// start poisoning (inclusive)
- /// \param layoutEndOffset index of the ASTRecordLayout field to
- /// end poisoning (exclusive)
- void PoisonMembers(CodeGenFunction &CGF, unsigned layoutStartOffset,
- unsigned layoutEndOffset) {
- ASTContext &Context = CGF.getContext();
+ // Generate function call for handling object poisoning.
+ // Disables tail call elimination, to prevent the current stack frame
+ // from disappearing from the stack trace.
+ void Emit(CodeGenFunction &CGF, Flags flags) override {
+ const ASTContext &Context = CGF.getContext();
const ASTRecordLayout &Layout =
Context.getASTRecordLayout(Dtor->getParent());
- // It's a first trivia field so it should be at the begining of char,
+ // It's a first trivial field so it should be at the begining of a char,
// still round up start offset just in case.
- CharUnits PoisonStart =
- Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutStartOffset) +
- Context.getCharWidth() - 1);
+ CharUnits PoisonStart = Context.toCharUnitsFromBits(
+ Layout.getFieldOffset(StartIndex) + Context.getCharWidth() - 1);
llvm::ConstantInt *OffsetSizePtr =
llvm::ConstantInt::get(CGF.SizeTy, PoisonStart.getQuantity());
@@ -1744,17 +1725,20 @@ namespace {
OffsetSizePtr);
CharUnits PoisonEnd;
- if (layoutEndOffset >= Layout.getFieldCount()) {
+ if (EndIndex >= Layout.getFieldCount()) {
PoisonEnd = Layout.getNonVirtualSize();
} else {
PoisonEnd =
- Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutEndOffset));
+ Context.toCharUnitsFromBits(Layout.getFieldOffset(EndIndex));
}
CharUnits PoisonSize = PoisonEnd - PoisonStart;
if (!PoisonSize.isPositive())
return;
EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize.getQuantity());
+
+ // Prevent the current stack frame from disappearing from the stack trace.
+ CGF.CurFn->addFnAttr("disable-tail-calls", "true");
}
};
@@ -1779,6 +1763,36 @@ namespace {
EmitSanitizerDtorCallback(CGF, VTablePtr, PoisonSize);
}
};
+
+ class SanitizeDtorCleanupBuilder {
+ ASTContext &Context;
+ EHScopeStack &EHStack;
+ const CXXDestructorDecl *DD;
+ llvm::Optional<unsigned> StartIndex;
+
+ public:
+ SanitizeDtorCleanupBuilder(ASTContext &Context, EHScopeStack &EHStack,
+ const CXXDestructorDecl *DD)
+ : Context(Context), EHStack(EHStack), DD(DD), StartIndex(llvm::None) {}
+ void PushCleanupForField(const FieldDecl *Field) {
+ if (Field->isZeroSize(Context))
+ return;
+ unsigned FieldIndex = Field->getFieldIndex();
+ if (FieldHasTrivialDestructorBody(Context, Field)) {
+ if (!StartIndex)
+ StartIndex = FieldIndex;
+ } else if (StartIndex) {
+ EHStack.pushCleanup<SanitizeDtorFieldRange>(
+ NormalAndEHCleanup, DD, StartIndex.getValue(), FieldIndex);
+ StartIndex = None;
+ }
+ }
+ void End() {
+ if (StartIndex)
+ EHStack.pushCleanup<SanitizeDtorFieldRange>(NormalAndEHCleanup, DD,
+ StartIndex.getValue(), -1);
+ }
+ };
} // end anonymous namespace
/// Emit all code that comes at the end of class's
@@ -1841,13 +1855,19 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
auto *BaseClassDecl =
cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
- // Ignore trivial destructors.
- if (BaseClassDecl->hasTrivialDestructor())
- continue;
-
- EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
- BaseClassDecl,
- /*BaseIsVirtual*/ true);
+ if (BaseClassDecl->hasTrivialDestructor()) {
+ // Under SanitizeMemoryUseAfterDtor, poison the trivial base class
+ // memory. For non-trival base classes the same is done in the class
+ // destructor.
+ if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
+ SanOpts.has(SanitizerKind::Memory) && !BaseClassDecl->isEmpty())
+ EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup,
+ BaseClassDecl,
+ /*BaseIsVirtual*/ true);
+ } else {
+ EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl,
+ /*BaseIsVirtual*/ true);
+ }
}
return;
@@ -1869,36 +1889,46 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
- // Ignore trivial destructors.
- if (BaseClassDecl->hasTrivialDestructor())
- continue;
-
- EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
- BaseClassDecl,
- /*BaseIsVirtual*/ false);
+ if (BaseClassDecl->hasTrivialDestructor()) {
+ if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
+ SanOpts.has(SanitizerKind::Memory) && !BaseClassDecl->isEmpty())
+ EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup,
+ BaseClassDecl,
+ /*BaseIsVirtual*/ false);
+ } else {
+ EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl,
+ /*BaseIsVirtual*/ false);
+ }
}
// Poison fields such that access after their destructors are
// invoked, and before the base class destructor runs, is invalid.
- if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
- SanOpts.has(SanitizerKind::Memory))
- EHStack.pushCleanup<SanitizeDtorMembers>(NormalAndEHCleanup, DD);
+ bool SanitizeFields = CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
+ SanOpts.has(SanitizerKind::Memory);
+ SanitizeDtorCleanupBuilder SanitizeBuilder(getContext(), EHStack, DD);
// Destroy direct fields.
for (const auto *Field : ClassDecl->fields()) {
+ if (SanitizeFields)
+ SanitizeBuilder.PushCleanupForField(Field);
+
QualType type = Field->getType();
QualType::DestructionKind dtorKind = type.isDestructedType();
- if (!dtorKind) continue;
+ if (!dtorKind)
+ continue;
// Anonymous union members do not have their destructors called.
const RecordType *RT = type->getAsUnionType();
- if (RT && RT->getDecl()->isAnonymousStructOrUnion()) continue;
+ if (RT && RT->getDecl()->isAnonymousStructOrUnion())
+ continue;
CleanupKind cleanupKind = getCleanupKind(dtorKind);
- EHStack.pushCleanup<DestroyField>(cleanupKind, Field,
- getDestroyer(dtorKind),
- cleanupKind & EHCleanup);
+ EHStack.pushCleanup<DestroyField>(
+ cleanupKind, Field, getDestroyer(dtorKind), cleanupKind & EHCleanup);
}
+
+ if (SanitizeFields)
+ SanitizeBuilder.End();
}
/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
@@ -2148,8 +2178,8 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
- Address Src(Args[1].getRValue(*this).getScalarVal(),
- CGM.getNaturalTypeAlignment(SrcTy));
+ Address Src = Address(Args[1].getRValue(*this).getScalarVal(), ConvertTypeForMem(SrcTy),
+ CGM.getNaturalTypeAlignment(SrcTy));
LValue SrcLVal = MakeAddrLValue(Src, SrcTy);
QualType DestTy = getContext().getTypeDeclType(ClassDecl);
LValue DestLVal = MakeAddrLValue(This, DestTy);
@@ -2332,8 +2362,8 @@ CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
// Push the src ptr.
QualType QT = *(FPT->param_type_begin());
llvm::Type *t = CGM.getTypes().ConvertType(QT);
- Src = Builder.CreateBitCast(Src, t);
- Args.add(RValue::get(Src.getPointer()), QT);
+ llvm::Value *SrcVal = Builder.CreateBitCast(Src.getPointer(), t);
+ Args.add(RValue::get(SrcVal), QT);
// Skip over first argument (Src).
EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(),
@@ -2665,9 +2695,9 @@ void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
if (SanOpts.has(SanitizerKind::CFIVCall))
EmitVTablePtrCheckForCall(RD, VTable, CodeGenFunction::CFITCK_VCall, Loc);
else if (CGM.getCodeGenOpts().WholeProgramVTables &&
- // Don't insert type test assumes if we are forcing public std
+ // Don't insert type test assumes if we are forcing public
// visibility.
- !CGM.HasLTOVisibilityPublicStd(RD)) {
+ !CGM.AlwaysHasLTOVisibilityPublic(RD)) {
llvm::Metadata *MD =
CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
llvm::Value *TypeId =
@@ -2691,8 +2721,7 @@ void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
EmitVTablePtrCheck(RD, VTable, TCK, Loc);
}
-void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
- llvm::Value *Derived,
+void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived,
bool MayBeNull,
CFITypeCheckKind TCK,
SourceLocation Loc) {
@@ -2715,7 +2744,7 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
if (MayBeNull) {
llvm::Value *DerivedNotNull =
- Builder.CreateIsNotNull(Derived, "cast.nonnull");
+ Builder.CreateIsNotNull(Derived.getPointer(), "cast.nonnull");
llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
ContBlock = createBasicBlock("cast.cont");
@@ -2726,8 +2755,8 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
}
llvm::Value *VTable;
- std::tie(VTable, ClassDecl) = CGM.getCXXABI().LoadVTablePtr(
- *this, Address(Derived, getPointerAlign()), ClassDecl);
+ std::tie(VTable, ClassDecl) =
+ CGM.getCXXABI().LoadVTablePtr(*this, Derived, ClassDecl);
EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
@@ -2829,7 +2858,8 @@ bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
}
llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
- const CXXRecordDecl *RD, llvm::Value *VTable, uint64_t VTableByteOffset) {
+ const CXXRecordDecl *RD, llvm::Value *VTable, llvm::Type *VTableTy,
+ uint64_t VTableByteOffset) {
SanitizerScope SanScope(this);
EmitSanitizerStatReport(llvm::SanStat_CFI_VCall);
@@ -2854,7 +2884,7 @@ llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
}
return Builder.CreateBitCast(Builder.CreateExtractValue(CheckedLoad, 0),
- VTable->getType()->getPointerElementType());
+ VTableTy);
}
void CodeGenFunction::EmitForwardingCallToLambda(
diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp
index b9364fcd2231..5035ed34358d 100644
--- a/clang/lib/CodeGen/CGCleanup.cpp
+++ b/clang/lib/CodeGen/CGCleanup.cpp
@@ -38,13 +38,13 @@ DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) {
// These automatically dominate and don't need to be saved.
if (!DominatingLLVMValue::needsSaving(V))
- return saved_type(V, ScalarLiteral);
+ return saved_type(V, nullptr, ScalarLiteral);
// Everything else needs an alloca.
Address addr =
CGF.CreateDefaultAlignTempAlloca(V->getType(), "saved-rvalue");
CGF.Builder.CreateStore(V, addr);
- return saved_type(addr.getPointer(), ScalarAddress);
+ return saved_type(addr.getPointer(), nullptr, ScalarAddress);
}
if (rv.isComplex()) {
@@ -54,19 +54,19 @@ DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) {
Address addr = CGF.CreateDefaultAlignTempAlloca(ComplexTy, "saved-complex");
CGF.Builder.CreateStore(V.first, CGF.Builder.CreateStructGEP(addr, 0));
CGF.Builder.CreateStore(V.second, CGF.Builder.CreateStructGEP(addr, 1));
- return saved_type(addr.getPointer(), ComplexAddress);
+ return saved_type(addr.getPointer(), nullptr, ComplexAddress);
}
assert(rv.isAggregate());
Address V = rv.getAggregateAddress(); // TODO: volatile?
if (!DominatingLLVMValue::needsSaving(V.getPointer()))
- return saved_type(V.getPointer(), AggregateLiteral,
+ return saved_type(V.getPointer(), V.getElementType(), AggregateLiteral,
V.getAlignment().getQuantity());
Address addr =
CGF.CreateTempAlloca(V.getType(), CGF.getPointerAlign(), "saved-rvalue");
CGF.Builder.CreateStore(V.getPointer(), addr);
- return saved_type(addr.getPointer(), AggregateAddress,
+ return saved_type(addr.getPointer(), V.getElementType(), AggregateAddress,
V.getAlignment().getQuantity());
}
@@ -75,8 +75,9 @@ DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) {
/// point.
RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) {
auto getSavingAddress = [&](llvm::Value *value) {
- auto alignment = cast<llvm::AllocaInst>(value)->getAlignment();
- return Address(value, CharUnits::fromQuantity(alignment));
+ auto *AI = cast<llvm::AllocaInst>(value);
+ return Address(value, AI->getAllocatedType(),
+ CharUnits::fromQuantity(AI->getAlign().value()));
};
switch (K) {
case ScalarLiteral:
@@ -84,10 +85,12 @@ RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) {
case ScalarAddress:
return RValue::get(CGF.Builder.CreateLoad(getSavingAddress(Value)));
case AggregateLiteral:
- return RValue::getAggregate(Address(Value, CharUnits::fromQuantity(Align)));
+ return RValue::getAggregate(
+ Address(Value, ElementType, CharUnits::fromQuantity(Align)));
case AggregateAddress: {
auto addr = CGF.Builder.CreateLoad(getSavingAddress(Value));
- return RValue::getAggregate(Address(addr, CharUnits::fromQuantity(Align)));
+ return RValue::getAggregate(
+ Address(addr, ElementType, CharUnits::fromQuantity(Align)));
}
case ComplexAddress: {
Address address = getSavingAddress(Value);
@@ -180,6 +183,15 @@ void *EHScopeStack::pushCleanup(CleanupKind Kind, size_t Size) {
bool IsNormalCleanup = Kind & NormalCleanup;
bool IsEHCleanup = Kind & EHCleanup;
bool IsLifetimeMarker = Kind & LifetimeMarker;
+
+ // Per C++ [except.terminate], it is implementation-defined whether none,
+ // some, or all cleanups are called before std::terminate. Thus, when
+ // terminate is the current EH scope, we may skip adding any EH cleanup
+ // scopes.
+ if (InnermostEHScope != stable_end() &&
+ find(InnermostEHScope)->getKind() == EHScope::Terminate)
+ IsEHCleanup = false;
+
EHCleanupScope *Scope =
new (Buffer) EHCleanupScope(IsNormalCleanup,
IsEHCleanup,
diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp
index c1763cbbc5a0..594c7d49df3c 100644
--- a/clang/lib/CodeGen/CGCoroutine.cpp
+++ b/clang/lib/CodeGen/CGCoroutine.cpp
@@ -238,8 +238,8 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co
auto Loc = S.getResumeExpr()->getExprLoc();
auto *Catch = new (CGF.getContext())
CXXCatchStmt(Loc, /*exDecl=*/nullptr, Coro.ExceptionHandler);
- auto *TryBody =
- CompoundStmt::Create(CGF.getContext(), S.getResumeExpr(), Loc, Loc);
+ auto *TryBody = CompoundStmt::Create(CGF.getContext(), S.getResumeExpr(),
+ FPOptionsOverride(), Loc, Loc);
TryStmt = CXXTryStmt::Create(CGF.getContext(), Loc, TryBody, Catch);
CGF.EnterCXXTryStmt(*TryStmt);
}
@@ -465,72 +465,6 @@ struct CallCoroDelete final : public EHScopeStack::Cleanup {
};
}
-namespace {
-struct GetReturnObjectManager {
- CodeGenFunction &CGF;
- CGBuilderTy &Builder;
- const CoroutineBodyStmt &S;
-
- Address GroActiveFlag;
- CodeGenFunction::AutoVarEmission GroEmission;
-
- GetReturnObjectManager(CodeGenFunction &CGF, const CoroutineBodyStmt &S)
- : CGF(CGF), Builder(CGF.Builder), S(S), GroActiveFlag(Address::invalid()),
- GroEmission(CodeGenFunction::AutoVarEmission::invalid()) {}
-
- // The gro variable has to outlive coroutine frame and coroutine promise, but,
- // it can only be initialized after coroutine promise was created, thus, we
- // split its emission in two parts. EmitGroAlloca emits an alloca and sets up
- // cleanups. Later when coroutine promise is available we initialize the gro
- // and sets the flag that the cleanup is now active.
-
- void EmitGroAlloca() {
- auto *GroDeclStmt = dyn_cast<DeclStmt>(S.getResultDecl());
- if (!GroDeclStmt) {
- // If get_return_object returns void, no need to do an alloca.
- return;
- }
-
- auto *GroVarDecl = cast<VarDecl>(GroDeclStmt->getSingleDecl());
-
- // Set GRO flag that it is not initialized yet
- GroActiveFlag =
- CGF.CreateTempAlloca(Builder.getInt1Ty(), CharUnits::One(), "gro.active");
- Builder.CreateStore(Builder.getFalse(), GroActiveFlag);
-
- GroEmission = CGF.EmitAutoVarAlloca(*GroVarDecl);
-
- // Remember the top of EHStack before emitting the cleanup.
- auto old_top = CGF.EHStack.stable_begin();
- CGF.EmitAutoVarCleanups(GroEmission);
- auto top = CGF.EHStack.stable_begin();
-
- // Make the cleanup conditional on gro.active
- for (auto b = CGF.EHStack.find(top), e = CGF.EHStack.find(old_top);
- b != e; b++) {
- if (auto *Cleanup = dyn_cast<EHCleanupScope>(&*b)) {
- assert(!Cleanup->hasActiveFlag() && "cleanup already has active flag?");
- Cleanup->setActiveFlag(GroActiveFlag);
- Cleanup->setTestFlagInEHCleanup();
- Cleanup->setTestFlagInNormalCleanup();
- }
- }
- }
-
- void EmitGroInit() {
- if (!GroActiveFlag.isValid()) {
- // No Gro variable was allocated. Simply emit the call to
- // get_return_object.
- CGF.EmitStmt(S.getResultDecl());
- return;
- }
-
- CGF.EmitAutoVarInit(GroEmission);
- Builder.CreateStore(Builder.getTrue(), GroActiveFlag);
- }
-};
-}
-
static void emitBodyAndFallthrough(CodeGenFunction &CGF,
const CoroutineBodyStmt &S, Stmt *Body) {
CGF.EmitStmt(Body);
@@ -597,13 +531,6 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
CGM.getIntrinsic(llvm::Intrinsic::coro_begin), {CoroId, Phi});
CurCoro.Data->CoroBegin = CoroBegin;
- // We need to emit `get_­return_­object` first. According to:
- // [dcl.fct.def.coroutine]p7
- // The call to get_­return_­object is sequenced before the call to
- // initial_­suspend and is invoked at most once.
- GetReturnObjectManager GroManager(*this, S);
- GroManager.EmitGroAlloca();
-
CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(RetBB);
{
CGDebugInfo *DI = getDebugInfo();
@@ -641,8 +568,23 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
// promise local variable was not emitted yet.
CoroId->setArgOperand(1, PromiseAddrVoidPtr);
- // Now we have the promise, initialize the GRO
- GroManager.EmitGroInit();
+ // ReturnValue should be valid as long as the coroutine's return type
+ // is not void. The assertion could help us to reduce the check later.
+ assert(ReturnValue.isValid() == (bool)S.getReturnStmt());
+ // Now we have the promise, initialize the GRO.
+ // We need to emit `get_return_object` first. According to:
+ // [dcl.fct.def.coroutine]p7
+ // The call to get_return_­object is sequenced before the call to
+ // initial_suspend and is invoked at most once.
+ //
+ // So we couldn't emit return value when we emit return statment,
+ // otherwise the call to get_return_object wouldn't be in front
+ // of initial_suspend.
+ if (ReturnValue.isValid()) {
+ EmitAnyExprToMem(S.getReturnValue(), ReturnValue,
+ S.getReturnValue()->getType().getQualifiers(),
+ /*IsInit*/ true);
+ }
EHStack.pushCleanup<CallCoroEnd>(EHCleanup);
@@ -705,12 +647,15 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
llvm::Function *CoroEnd = CGM.getIntrinsic(llvm::Intrinsic::coro_end);
Builder.CreateCall(CoroEnd, {NullPtr, Builder.getFalse()});
- if (Stmt *Ret = S.getReturnStmt())
+ if (Stmt *Ret = S.getReturnStmt()) {
+ // Since we already emitted the return value above, so we shouldn't
+ // emit it again here.
+ cast<ReturnStmt>(Ret)->setRetValue(nullptr);
EmitStmt(Ret);
+ }
- // LLVM require the frontend to add the function attribute. See
- // Coroutines.rst.
- CurFn->addFnAttr("coroutine.presplit", "0");
+ // LLVM require the frontend to mark the coroutine.
+ CurFn->setPresplitCoroutine();
}
// Emit coroutine intrinsic and patch up arguments of the token type.
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 1a9080604a79..7e9e86763af9 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -248,6 +248,7 @@ PrintingPolicy CGDebugInfo::getPrintingPolicy() const {
PP.PrintCanonicalTypes = true;
PP.UsePreferredNames = false;
PP.AlwaysIncludeTypeForTemplateArgument = true;
+ PP.UseEnumerators = false;
// Apply -fdebug-prefix-map.
PP.Callbacks = &PrintCB;
@@ -424,16 +425,16 @@ CGDebugInfo::createFile(StringRef FileName,
SmallString<128> DirBuf;
SmallString<128> FileBuf;
if (llvm::sys::path::is_absolute(RemappedFile)) {
- // Strip the common prefix (if it is more than just "/") from current
- // directory and FileName for a more space-efficient encoding.
+ // Strip the common prefix (if it is more than just "/" or "C:\") from
+ // current directory and FileName for a more space-efficient encoding.
auto FileIt = llvm::sys::path::begin(RemappedFile);
auto FileE = llvm::sys::path::end(RemappedFile);
auto CurDirIt = llvm::sys::path::begin(CurDir);
auto CurDirE = llvm::sys::path::end(CurDir);
for (; CurDirIt != CurDirE && *CurDirIt == *FileIt; ++CurDirIt, ++FileIt)
llvm::sys::path::append(DirBuf, *CurDirIt);
- if (std::distance(llvm::sys::path::begin(CurDir), CurDirIt) == 1) {
- // Don't strip the common prefix if it is only the root "/"
+ if (llvm::sys::path::root_path(DirBuf) == DirBuf) {
+ // Don't strip the common prefix if it is only the root ("/" or "C:\")
// since that would make LLVM diagnostic locations confusing.
Dir = {};
File = RemappedFile;
@@ -444,7 +445,8 @@ CGDebugInfo::createFile(StringRef FileName,
File = FileBuf;
}
} else {
- Dir = CurDir;
+ if (!llvm::sys::path::is_absolute(FileName))
+ Dir = CurDir;
File = RemappedFile;
}
llvm::DIFile *F = DBuilder.createFile(File, Dir, CSInfo, Source);
@@ -517,8 +519,9 @@ void CGDebugInfo::CreateCompileUnit() {
// a relative path, so we look into the actual file entry for the main
// file to determine the real absolute path for the file.
std::string MainFileDir;
- if (const FileEntry *MainFile = SM.getFileEntryForID(SM.getMainFileID())) {
- MainFileDir = std::string(MainFile->getDir()->getName());
+ if (Optional<FileEntryRef> MainFile =
+ SM.getFileEntryRefForID(SM.getMainFileID())) {
+ MainFileDir = std::string(MainFile->getDir().getName());
if (!llvm::sys::path::is_absolute(MainFileName)) {
llvm::SmallString<1024> MainFileDirSS(MainFileDir);
llvm::sys::path::append(MainFileDirSS, MainFileName);
@@ -927,28 +930,8 @@ static llvm::dwarf::Tag getNextQualifier(Qualifiers &Q) {
return (llvm::dwarf::Tag)0;
}
-// Strip MacroQualifiedTypeLoc and AttributedTypeLoc
-// as their corresponding types will be ignored
-// during code generation. Stripping them allows
-// to maintain proper TypeLoc for a given type
-// during code generation.
-static TypeLoc StripMacroAttributed(TypeLoc TL) {
- if (!TL)
- return TL;
-
- while (true) {
- if (auto MTL = TL.getAs<MacroQualifiedTypeLoc>())
- TL = MTL.getInnerLoc();
- else if (auto ATL = TL.getAs<AttributedTypeLoc>())
- TL = ATL.getModifiedLoc();
- else
- break;
- }
- return TL;
-}
-
-llvm::DIType *CGDebugInfo::CreateQualifiedType(QualType Ty, llvm::DIFile *Unit,
- TypeLoc TL) {
+llvm::DIType *CGDebugInfo::CreateQualifiedType(QualType Ty,
+ llvm::DIFile *Unit) {
QualifierCollector Qc;
const Type *T = Qc.strip(Ty);
@@ -962,15 +945,7 @@ llvm::DIType *CGDebugInfo::CreateQualifiedType(QualType Ty, llvm::DIFile *Unit,
return getOrCreateType(QualType(T, 0), Unit);
}
- QualType NextTy = Qc.apply(CGM.getContext(), T);
- TypeLoc NextTL;
- if (NextTy.hasQualifiers())
- NextTL = TL;
- else if (TL) {
- if (auto QTL = TL.getAs<QualifiedTypeLoc>())
- NextTL = StripMacroAttributed(QTL.getNextTypeLoc());
- }
- auto *FromTy = getOrCreateType(NextTy, Unit, NextTL);
+ auto *FromTy = getOrCreateType(Qc.apply(CGM.getContext(), T), Unit);
// No need to fill in the Name, Line, Size, Alignment, Offset in case of
// CVR derived types.
@@ -1014,10 +989,10 @@ llvm::DIType *CGDebugInfo::CreateType(const ObjCObjectPointerType *Ty,
Ty->getPointeeType(), Unit);
}
-llvm::DIType *CGDebugInfo::CreateType(const PointerType *Ty, llvm::DIFile *Unit,
- TypeLoc TL) {
+llvm::DIType *CGDebugInfo::CreateType(const PointerType *Ty,
+ llvm::DIFile *Unit) {
return CreatePointerLikeType(llvm::dwarf::DW_TAG_pointer_type, Ty,
- Ty->getPointeeType(), Unit, TL);
+ Ty->getPointeeType(), Unit);
}
/// \return whether a C++ mangling exists for the type defined by TD.
@@ -1158,8 +1133,7 @@ CGDebugInfo::getOrCreateRecordFwdDecl(const RecordType *Ty,
llvm::DIType *CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag,
const Type *Ty,
QualType PointeeTy,
- llvm::DIFile *Unit,
- TypeLoc TL) {
+ llvm::DIFile *Unit) {
// Bit size, align and offset of the type.
// Size is always the size of a pointer. We can't use getTypeSize here
// because that does not return the correct value for references.
@@ -1169,52 +1143,32 @@ llvm::DIType *CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag,
Optional<unsigned> DWARFAddressSpace =
CGM.getTarget().getDWARFAddressSpace(AddressSpace);
- llvm::DINodeArray Annotations = nullptr;
- TypeLoc NextTL;
- if (TL) {
- SmallVector<llvm::Metadata *, 4> Annots;
- NextTL = TL.getNextTypeLoc();
- if (NextTL) {
- // Traverse all MacroQualifiedTypeLoc, QualifiedTypeLoc and
- // AttributedTypeLoc type locations so we can collect
- // BTFTypeTag attributes for this pointer.
- while (true) {
- if (auto MTL = NextTL.getAs<MacroQualifiedTypeLoc>()) {
- NextTL = MTL.getInnerLoc();
- } else if (auto QTL = NextTL.getAs<QualifiedTypeLoc>()) {
- NextTL = QTL.getNextTypeLoc();
- } else if (auto ATL = NextTL.getAs<AttributedTypeLoc>()) {
- if (const auto *A = ATL.getAttrAs<BTFTypeTagAttr>()) {
- StringRef BTFTypeTag = A->getBTFTypeTag();
- if (!BTFTypeTag.empty()) {
- llvm::Metadata *Ops[2] = {
- llvm::MDString::get(CGM.getLLVMContext(),
- StringRef("btf_type_tag")),
- llvm::MDString::get(CGM.getLLVMContext(), BTFTypeTag)};
- Annots.insert(Annots.begin(),
- llvm::MDNode::get(CGM.getLLVMContext(), Ops));
- }
- }
- NextTL = ATL.getModifiedLoc();
- } else {
- break;
- }
- }
+ SmallVector<llvm::Metadata *, 4> Annots;
+ auto *BTFAttrTy = dyn_cast<BTFTagAttributedType>(PointeeTy);
+ while (BTFAttrTy) {
+ StringRef Tag = BTFAttrTy->getAttr()->getBTFTypeTag();
+ if (!Tag.empty()) {
+ llvm::Metadata *Ops[2] = {
+ llvm::MDString::get(CGM.getLLVMContext(), StringRef("btf_type_tag")),
+ llvm::MDString::get(CGM.getLLVMContext(), Tag)};
+ Annots.insert(Annots.begin(),
+ llvm::MDNode::get(CGM.getLLVMContext(), Ops));
}
-
- NextTL = StripMacroAttributed(TL.getNextTypeLoc());
- if (Annots.size() > 0)
- Annotations = DBuilder.getOrCreateArray(Annots);
+ BTFAttrTy = dyn_cast<BTFTagAttributedType>(BTFAttrTy->getWrappedType());
}
+ llvm::DINodeArray Annotations = nullptr;
+ if (Annots.size() > 0)
+ Annotations = DBuilder.getOrCreateArray(Annots);
+
if (Tag == llvm::dwarf::DW_TAG_reference_type ||
Tag == llvm::dwarf::DW_TAG_rvalue_reference_type)
return DBuilder.createReferenceType(Tag, getOrCreateType(PointeeTy, Unit),
Size, Align, DWARFAddressSpace);
else
- return DBuilder.createPointerType(getOrCreateType(PointeeTy, Unit, NextTL),
- Size, Align, DWARFAddressSpace,
- StringRef(), Annotations);
+ return DBuilder.createPointerType(getOrCreateType(PointeeTy, Unit), Size,
+ Align, DWARFAddressSpace, StringRef(),
+ Annotations);
}
llvm::DIType *CGDebugInfo::getOrCreateStructPtrType(StringRef Name,
@@ -1331,11 +1285,8 @@ llvm::DIType *CGDebugInfo::CreateType(const TemplateSpecializationType *Ty,
llvm::DIType *CGDebugInfo::CreateType(const TypedefType *Ty,
llvm::DIFile *Unit) {
- TypeLoc TL;
- if (const TypeSourceInfo *TSI = Ty->getDecl()->getTypeSourceInfo())
- TL = TSI->getTypeLoc();
llvm::DIType *Underlying =
- getOrCreateType(Ty->getDecl()->getUnderlyingType(), Unit, TL);
+ getOrCreateType(Ty->getDecl()->getUnderlyingType(), Unit);
if (Ty->getDecl()->hasAttr<NoDebugAttr>())
return Underlying;
@@ -1375,6 +1326,7 @@ static unsigned getDwarfCC(CallingConv CC) {
return llvm::dwarf::DW_CC_LLVM_X86_64SysV;
case CC_AAPCS:
case CC_AArch64VectorCall:
+ case CC_AArch64SVEPCS:
return llvm::dwarf::DW_CC_LLVM_AAPCS;
case CC_AAPCS_VFP:
return llvm::dwarf::DW_CC_LLVM_AAPCS_VFP;
@@ -1383,6 +1335,7 @@ static unsigned getDwarfCC(CallingConv CC) {
case CC_SpirFunction:
return llvm::dwarf::DW_CC_LLVM_SpirFunction;
case CC_OpenCLKernel:
+ case CC_AMDGPUKernelCall:
return llvm::dwarf::DW_CC_LLVM_OpenCLKernel;
case CC_Swift:
return llvm::dwarf::DW_CC_LLVM_Swift;
@@ -1409,7 +1362,7 @@ static llvm::DINode::DIFlags getRefFlags(const FunctionProtoType *Func) {
}
llvm::DIType *CGDebugInfo::CreateType(const FunctionType *Ty,
- llvm::DIFile *Unit, TypeLoc TL) {
+ llvm::DIFile *Unit) {
const auto *FPT = dyn_cast<FunctionProtoType>(Ty);
if (FPT) {
if (llvm::DIType *QTy = CreateQualifiedType(FPT, Unit))
@@ -1421,12 +1374,7 @@ llvm::DIType *CGDebugInfo::CreateType(const FunctionType *Ty,
SmallVector<llvm::Metadata *, 16> EltTys;
// Add the result type at least.
- TypeLoc RetTL;
- if (TL) {
- if (auto FTL = TL.getAs<FunctionTypeLoc>())
- RetTL = FTL.getReturnLoc();
- }
- EltTys.push_back(getOrCreateType(Ty->getReturnType(), Unit, RetTL));
+ EltTys.push_back(getOrCreateType(Ty->getReturnType(), Unit));
llvm::DINode::DIFlags Flags = llvm::DINode::FlagZero;
// Set up remainder of arguments if there is a prototype.
@@ -1435,30 +1383,8 @@ llvm::DIType *CGDebugInfo::CreateType(const FunctionType *Ty,
EltTys.push_back(DBuilder.createUnspecifiedParameter());
} else {
Flags = getRefFlags(FPT);
- bool DoneWithTL = false;
- if (TL) {
- if (auto FTL = TL.getAs<FunctionTypeLoc>()) {
- DoneWithTL = true;
- unsigned Idx = 0;
- unsigned FTL_NumParams = FTL.getNumParams();
- for (const QualType &ParamType : FPT->param_types()) {
- TypeLoc ParamTL;
- if (Idx < FTL_NumParams) {
- if (ParmVarDecl *Param = FTL.getParam(Idx)) {
- if (const TypeSourceInfo *TSI = Param->getTypeSourceInfo())
- ParamTL = TSI->getTypeLoc();
- }
- }
- EltTys.push_back(getOrCreateType(ParamType, Unit, ParamTL));
- Idx++;
- }
- }
- }
-
- if (!DoneWithTL) {
- for (const QualType &ParamType : FPT->param_types())
- EltTys.push_back(getOrCreateType(ParamType, Unit));
- }
+ for (const QualType &ParamType : FPT->param_types())
+ EltTys.push_back(getOrCreateType(ParamType, Unit));
if (FPT->isVariadic())
EltTys.push_back(DBuilder.createUnspecifiedParameter());
}
@@ -1529,13 +1455,11 @@ llvm::DIType *CGDebugInfo::createBitFieldType(const FieldDecl *BitFieldDecl,
Flags, DebugType, Annotations);
}
-llvm::DIType *
-CGDebugInfo::createFieldType(StringRef name, QualType type, SourceLocation loc,
- AccessSpecifier AS, uint64_t offsetInBits,
- uint32_t AlignInBits, llvm::DIFile *tunit,
- llvm::DIScope *scope, const RecordDecl *RD,
- llvm::DINodeArray Annotations, TypeLoc TL) {
- llvm::DIType *debugType = getOrCreateType(type, tunit, TL);
+llvm::DIType *CGDebugInfo::createFieldType(
+ StringRef name, QualType type, SourceLocation loc, AccessSpecifier AS,
+ uint64_t offsetInBits, uint32_t AlignInBits, llvm::DIFile *tunit,
+ llvm::DIScope *scope, const RecordDecl *RD, llvm::DINodeArray Annotations) {
+ llvm::DIType *debugType = getOrCreateType(type, tunit);
// Get the location for the field.
llvm::DIFile *file = getOrCreateFile(loc);
@@ -1643,12 +1567,9 @@ void CGDebugInfo::CollectRecordNormalField(
} else {
auto Align = getDeclAlignIfRequired(field, CGM.getContext());
llvm::DINodeArray Annotations = CollectBTFDeclTagAnnotations(field);
- TypeLoc TL;
- if (const TypeSourceInfo *TSI = field->getTypeSourceInfo())
- TL = TSI->getTypeLoc();
- FieldType = createFieldType(name, type, field->getLocation(),
- field->getAccess(), OffsetInBits, Align, tunit,
- RecordTy, RD, Annotations, TL);
+ FieldType =
+ createFieldType(name, type, field->getLocation(), field->getAccess(),
+ OffsetInBits, Align, tunit, RecordTy, RD, Annotations);
}
elements.push_back(FieldType);
@@ -1725,7 +1646,7 @@ void CGDebugInfo::CollectRecordFields(
llvm::DISubroutineType *
CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
llvm::DIFile *Unit, bool decl) {
- const FunctionProtoType *Func = Method->getType()->getAs<FunctionProtoType>();
+ const auto *Func = Method->getType()->castAs<FunctionProtoType>();
if (Method->isStatic())
return cast_or_null<llvm::DISubroutineType>(
getOrCreateType(QualType(Func, 0), Unit));
@@ -1760,9 +1681,17 @@ CGDebugInfo::getOrCreateInstanceMethodType(QualType ThisPtr,
SmallVector<llvm::Metadata *, 16> Elts;
// First element is always return type. For 'void' functions it is NULL.
QualType temp = Func->getReturnType();
- if (temp->getTypeClass() == Type::Auto && decl)
- Elts.push_back(CreateType(cast<AutoType>(temp)));
- else
+ if (temp->getTypeClass() == Type::Auto && decl) {
+ const AutoType *AT = cast<AutoType>(temp);
+
+ // It may be tricky in some cases to link the specification back the lambda
+ // call operator and so we skip emitting "auto" for lambdas. This is
+ // consistent with gcc as well.
+ if (AT->isDeduced() && ThisPtr->getPointeeCXXRecordDecl()->isLambda())
+ Elts.push_back(getOrCreateType(AT->getDeducedType(), Unit));
+ else
+ Elts.push_back(CreateType(AT));
+ } else
Elts.push_back(Args[0]);
// "this" pointer is always first argument.
@@ -3035,6 +2964,23 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty,
llvm::DIType *CGDebugInfo::CreateType(const VectorType *Ty,
llvm::DIFile *Unit) {
+ if (Ty->isExtVectorBoolType()) {
+ // Boolean ext_vector_type(N) are special because their real element type
+ // (bits of bit size) is not their Clang element type (_Bool of size byte).
+ // For now, we pretend the boolean vector were actually a vector of bytes
+ // (where each byte represents 8 bits of the actual vector).
+ // FIXME Debug info should actually represent this proper as a vector mask
+ // type.
+ auto &Ctx = CGM.getContext();
+ uint64_t Size = CGM.getContext().getTypeSize(Ty);
+ uint64_t NumVectorBytes = Size / Ctx.getCharWidth();
+
+ // Construct the vector of 'char' type.
+ QualType CharVecTy = Ctx.getVectorType(Ctx.CharTy, NumVectorBytes,
+ VectorType::GenericVector);
+ return CreateType(CharVecTy->getAs<VectorType>(), Unit);
+ }
+
llvm::DIType *ElementTy = getOrCreateType(Ty->getElementType(), Unit);
int64_t Count = Ty->getNumElements();
@@ -3348,6 +3294,9 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) {
case Type::Attributed:
T = cast<AttributedType>(T)->getEquivalentType();
break;
+ case Type::BTFTagAttributed:
+ T = cast<BTFTagAttributedType>(T)->getWrappedType();
+ break;
case Type::Elaborated:
T = cast<ElaboratedType>(T)->getNamedType();
break;
@@ -3409,8 +3358,7 @@ void CGDebugInfo::completeUnusedClass(const CXXRecordDecl &D) {
RetainedTypes.push_back(CGM.getContext().getRecordType(&D).getAsOpaquePtr());
}
-llvm::DIType *CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile *Unit,
- TypeLoc TL) {
+llvm::DIType *CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile *Unit) {
if (Ty.isNull())
return nullptr;
@@ -3427,7 +3375,7 @@ llvm::DIType *CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile *Unit,
if (auto *T = getTypeOrNull(Ty))
return T;
- llvm::DIType *Res = CreateTypeNode(Ty, Unit, TL);
+ llvm::DIType *Res = CreateTypeNode(Ty, Unit);
void *TyPtr = Ty.getAsOpaquePtr();
// And update the type cache.
@@ -3471,11 +3419,10 @@ llvm::DIModule *CGDebugInfo::getParentModuleOrNull(const Decl *D) {
return nullptr;
}
-llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit,
- TypeLoc TL) {
+llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) {
// Handle qualifiers, which recursively handles what they refer to.
if (Ty.hasLocalQualifiers())
- return CreateQualifiedType(Ty, Unit, TL);
+ return CreateQualifiedType(Ty, Unit);
// Work out details of type.
switch (Ty->getTypeClass()) {
@@ -3504,7 +3451,7 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit,
case Type::Complex:
return CreateType(cast<ComplexType>(Ty));
case Type::Pointer:
- return CreateType(cast<PointerType>(Ty), Unit, TL);
+ return CreateType(cast<PointerType>(Ty), Unit);
case Type::BlockPointer:
return CreateType(cast<BlockPointerType>(Ty), Unit);
case Type::Typedef:
@@ -3515,7 +3462,7 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit,
return CreateEnumType(cast<EnumType>(Ty));
case Type::FunctionProto:
case Type::FunctionNoProto:
- return CreateType(cast<FunctionType>(Ty), Unit, TL);
+ return CreateType(cast<FunctionType>(Ty), Unit);
case Type::ConstantArray:
case Type::VariableArray:
case Type::IncompleteArray:
@@ -3542,6 +3489,7 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit,
case Type::Auto:
case Type::Attributed:
+ case Type::BTFTagAttributed:
case Type::Adjusted:
case Type::Decayed:
case Type::DeducedTemplateSpecialization:
@@ -3615,7 +3563,11 @@ llvm::DICompositeType *CGDebugInfo::CreateLimitedType(const RecordType *Ty) {
return getOrCreateRecordFwdDecl(Ty, RDContext);
uint64_t Size = CGM.getContext().getTypeSize(Ty);
- auto Align = getDeclAlignIfRequired(D, CGM.getContext());
+ // __attribute__((aligned)) can increase or decrease alignment *except* on a
+ // struct or struct member, where it only increases alignment unless 'packed'
+ // is also specified. To handle this case, the `getTypeAlignIfRequired` needs
+ // to be used.
+ auto Align = getTypeAlignIfRequired(Ty, CGM.getContext());
SmallString<256> Identifier = getTypeIdentifier(Ty, CGM, TheCU);
@@ -3908,6 +3860,17 @@ llvm::DINode *CGDebugInfo::getDeclarationOrDefinition(const Decl *D) {
auto N = I->second;
if (auto *GVE = dyn_cast_or_null<llvm::DIGlobalVariableExpression>(N))
return GVE->getVariable();
+ return cast<llvm::DINode>(N);
+ }
+
+ // Search imported declaration cache if it is already defined
+ // as imported declaration.
+ auto IE = ImportedDeclCache.find(D->getCanonicalDecl());
+
+ if (IE != ImportedDeclCache.end()) {
+ auto N = IE->second;
+ if (auto *GVE = dyn_cast_or_null<llvm::DIImportedEntity>(N))
+ return cast<llvm::DINode>(GVE);
return dyn_cast_or_null<llvm::DINode>(N);
}
@@ -4064,12 +4027,7 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D,
getDwarfCC(CC));
}
- TypeLoc TL;
- if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
- if (const TypeSourceInfo *TSI = FD->getTypeSourceInfo())
- TL = TSI->getTypeLoc();
- }
- return cast<llvm::DISubroutineType>(getOrCreateType(FnType, F, TL));
+ return cast<llvm::DISubroutineType>(getOrCreateType(FnType, F));
}
QualType
@@ -4328,7 +4286,7 @@ void CGDebugInfo::AppendAddressSpaceXDeref(
return;
Expr.push_back(llvm::dwarf::DW_OP_constu);
- Expr.push_back(DWARFAddressSpace.getValue());
+ Expr.push_back(*DWARFAddressSpace);
Expr.push_back(llvm::dwarf::DW_OP_swap);
Expr.push_back(llvm::dwarf::DW_OP_xderef);
}
@@ -4471,12 +4429,8 @@ llvm::DILocalVariable *CGDebugInfo::EmitDeclare(const VarDecl *VD,
uint64_t XOffset = 0;
if (VD->hasAttr<BlocksAttr>())
Ty = EmitTypeForVarWithBlocksAttr(VD, &XOffset).WrappedType;
- else {
- TypeLoc TL;
- if (const TypeSourceInfo *TSI = VD->getTypeSourceInfo())
- TL = TSI->getTypeLoc();
- Ty = getOrCreateType(VD->getType(), Unit, TL);
- }
+ else
+ Ty = getOrCreateType(VD->getType(), Unit);
// If there is no debug info for this type then do not emit debug info
// for this variable.
@@ -4635,11 +4589,103 @@ llvm::DILocalVariable *CGDebugInfo::EmitDeclare(const VarDecl *VD,
return D;
}
+llvm::DILocalVariable *CGDebugInfo::EmitDeclare(const BindingDecl *BD,
+ llvm::Value *Storage,
+ llvm::Optional<unsigned> ArgNo,
+ CGBuilderTy &Builder,
+ const bool UsePointerValue) {
+ assert(CGM.getCodeGenOpts().hasReducedDebugInfo());
+ assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!");
+ if (BD->hasAttr<NoDebugAttr>())
+ return nullptr;
+
+ // Skip the tuple like case, we don't handle that here
+ if (isa<DeclRefExpr>(BD->getBinding()))
+ return nullptr;
+
+ llvm::DIFile *Unit = getOrCreateFile(BD->getLocation());
+ llvm::DIType *Ty = getOrCreateType(BD->getType(), Unit);
+
+ // If there is no debug info for this type then do not emit debug info
+ // for this variable.
+ if (!Ty)
+ return nullptr;
+
+ auto Align = getDeclAlignIfRequired(BD, CGM.getContext());
+ unsigned AddressSpace = CGM.getContext().getTargetAddressSpace(BD->getType());
+
+ SmallVector<uint64_t, 3> Expr;
+ AppendAddressSpaceXDeref(AddressSpace, Expr);
+
+ // Clang stores the sret pointer provided by the caller in a static alloca.
+ // Use DW_OP_deref to tell the debugger to load the pointer and treat it as
+ // the address of the variable.
+ if (UsePointerValue) {
+ assert(!llvm::is_contained(Expr, llvm::dwarf::DW_OP_deref) &&
+ "Debug info already contains DW_OP_deref.");
+ Expr.push_back(llvm::dwarf::DW_OP_deref);
+ }
+
+ unsigned Line = getLineNumber(BD->getLocation());
+ unsigned Column = getColumnNumber(BD->getLocation());
+ StringRef Name = BD->getName();
+ auto *Scope = cast<llvm::DIScope>(LexicalBlockStack.back());
+ // Create the descriptor for the variable.
+ llvm::DILocalVariable *D = DBuilder.createAutoVariable(
+ Scope, Name, Unit, Line, Ty, CGM.getLangOpts().Optimize,
+ llvm::DINode::FlagZero, Align);
+
+ if (const MemberExpr *ME = dyn_cast<MemberExpr>(BD->getBinding())) {
+ if (const FieldDecl *FD = dyn_cast<FieldDecl>(ME->getMemberDecl())) {
+ const unsigned fieldIndex = FD->getFieldIndex();
+ const clang::CXXRecordDecl *parent =
+ (const CXXRecordDecl *)FD->getParent();
+ const ASTRecordLayout &layout =
+ CGM.getContext().getASTRecordLayout(parent);
+ const uint64_t fieldOffset = layout.getFieldOffset(fieldIndex);
+
+ if (fieldOffset != 0) {
+ Expr.push_back(llvm::dwarf::DW_OP_plus_uconst);
+ Expr.push_back(
+ CGM.getContext().toCharUnitsFromBits(fieldOffset).getQuantity());
+ }
+ }
+ } else if (const ArraySubscriptExpr *ASE =
+ dyn_cast<ArraySubscriptExpr>(BD->getBinding())) {
+ if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(ASE->getIdx())) {
+ const uint64_t value = IL->getValue().getZExtValue();
+ const uint64_t typeSize = CGM.getContext().getTypeSize(BD->getType());
+
+ if (value != 0) {
+ Expr.push_back(llvm::dwarf::DW_OP_plus_uconst);
+ Expr.push_back(CGM.getContext()
+ .toCharUnitsFromBits(value * typeSize)
+ .getQuantity());
+ }
+ }
+ }
+
+ // Insert an llvm.dbg.declare into the current block.
+ DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr),
+ llvm::DILocation::get(CGM.getLLVMContext(), Line,
+ Column, Scope, CurInlinedAt),
+ Builder.GetInsertBlock());
+
+ return D;
+}
+
llvm::DILocalVariable *
CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD, llvm::Value *Storage,
CGBuilderTy &Builder,
const bool UsePointerValue) {
assert(CGM.getCodeGenOpts().hasReducedDebugInfo());
+
+ if (auto *DD = dyn_cast<DecompositionDecl>(VD))
+ for (auto *B : DD->bindings()) {
+ EmitDeclare(B, Storage, llvm::None, Builder,
+ VD->getType()->isReferenceType());
+ }
+
return EmitDeclare(VD, Storage, llvm::None, Builder, UsePointerValue);
}
@@ -4975,6 +5021,52 @@ llvm::DIGlobalVariableExpression *CGDebugInfo::CollectAnonRecordDecls(
return GVE;
}
+static bool ReferencesAnonymousEntity(ArrayRef<TemplateArgument> Args);
+static bool ReferencesAnonymousEntity(RecordType *RT) {
+ // Unnamed classes/lambdas can't be reconstituted due to a lack of column
+ // info we produce in the DWARF, so we can't get Clang's full name back.
+ // But so long as it's not one of those, it doesn't matter if some sub-type
+ // of the record (a template parameter) can't be reconstituted - because the
+ // un-reconstitutable type itself will carry its own name.
+ const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
+ if (!RD)
+ return false;
+ if (!RD->getIdentifier())
+ return true;
+ auto *TSpecial = dyn_cast<ClassTemplateSpecializationDecl>(RD);
+ if (!TSpecial)
+ return false;
+ return ReferencesAnonymousEntity(TSpecial->getTemplateArgs().asArray());
+}
+static bool ReferencesAnonymousEntity(ArrayRef<TemplateArgument> Args) {
+ return llvm::any_of(Args, [&](const TemplateArgument &TA) {
+ switch (TA.getKind()) {
+ case TemplateArgument::Pack:
+ return ReferencesAnonymousEntity(TA.getPackAsArray());
+ case TemplateArgument::Type: {
+ struct ReferencesAnonymous
+ : public RecursiveASTVisitor<ReferencesAnonymous> {
+ bool RefAnon = false;
+ bool VisitRecordType(RecordType *RT) {
+ if (ReferencesAnonymousEntity(RT)) {
+ RefAnon = true;
+ return false;
+ }
+ return true;
+ }
+ };
+ ReferencesAnonymous RT;
+ RT.TraverseType(TA.getAsType());
+ if (RT.RefAnon)
+ return true;
+ break;
+ }
+ default:
+ break;
+ }
+ return false;
+ });
+}
namespace {
struct ReconstitutableType : public RecursiveASTVisitor<ReconstitutableType> {
bool Reconstitutable = true;
@@ -4986,6 +5078,15 @@ struct ReconstitutableType : public RecursiveASTVisitor<ReconstitutableType> {
Reconstitutable = false;
return false;
}
+ bool VisitType(Type *T) {
+ // _BitInt(N) isn't reconstitutable because the bit width isn't encoded in
+ // the DWARF, only the byte width.
+ if (T->isBitIntType()) {
+ Reconstitutable = false;
+ return false;
+ }
+ return true;
+ }
bool TraverseEnumType(EnumType *ET) {
// Unnamed enums can't be reconstituted due to a lack of column info we
// produce in the DWARF, so we can't get Clang's full name back.
@@ -4994,24 +5095,21 @@ struct ReconstitutableType : public RecursiveASTVisitor<ReconstitutableType> {
Reconstitutable = false;
return false;
}
+ if (!ED->isExternallyVisible()) {
+ Reconstitutable = false;
+ return false;
+ }
}
return true;
}
bool VisitFunctionProtoType(FunctionProtoType *FT) {
// noexcept is not encoded in DWARF, so the reversi
Reconstitutable &= !isNoexceptExceptionSpec(FT->getExceptionSpecType());
+ Reconstitutable &= !FT->getNoReturnAttr();
return Reconstitutable;
}
- bool TraverseRecordType(RecordType *RT) {
- // Unnamed classes/lambdas can't be reconstituted due to a lack of column
- // info we produce in the DWARF, so we can't get Clang's full name back.
- // But so long as it's not one of those, it doesn't matter if some sub-type
- // of the record (a template parameter) can't be reconstituted - because the
- // un-reconstitutable type itself will carry its own name.
- const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
- if (!RD)
- return true;
- if (RD->isLambda() || !RD->getIdentifier()) {
+ bool VisitRecordType(RecordType *RT) {
+ if (ReferencesAnonymousEntity(RT)) {
Reconstitutable = false;
return false;
}
@@ -5035,6 +5133,10 @@ std::string CGDebugInfo::GetName(const Decl *D, bool Qualified) const {
return Name;
codegenoptions::DebugTemplateNamesKind TemplateNamesKind =
CGM.getCodeGenOpts().getDebugSimpleTemplateNames();
+
+ if (!CGM.getCodeGenOpts().hasReducedDebugInfo())
+ TemplateNamesKind = codegenoptions::DebugTemplateNamesKind::Full;
+
Optional<TemplateArgs> Args;
bool IsOperatorOverload = false; // isa<CXXConversionDecl>(ND);
@@ -5080,7 +5182,8 @@ std::string CGDebugInfo::GetName(const Decl *D, bool Qualified) const {
// harder to parse back into a large integer, etc - so punting on
// this for now. Re-parsing the integers back into APInt is probably
// feasible some day.
- return TA.getAsIntegral().getBitWidth() <= 64;
+ return TA.getAsIntegral().getBitWidth() <= 64 &&
+ IsReconstitutableType(TA.getIntegralType());
case TemplateArgument::Type:
return IsReconstitutableType(TA.getAsType());
default:
@@ -5123,7 +5226,7 @@ std::string CGDebugInfo::GetName(const Decl *D, bool Qualified) const {
TemplateNamesKind == codegenoptions::DebugTemplateNamesKind::Mangled;
// check if it's a template
if (Mangled)
- OS << "_STN";
+ OS << "_STN|";
OS << ND->getDeclName();
std::string EncodedOriginalName;
@@ -5200,14 +5303,10 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
}
AppendAddressSpaceXDeref(AddressSpace, Expr);
- TypeLoc TL;
- if (const TypeSourceInfo *TSI = D->getTypeSourceInfo())
- TL = TSI->getTypeLoc();
-
llvm::DINodeArray Annotations = CollectBTFDeclTagAnnotations(D);
GVE = DBuilder.createGlobalVariableExpression(
- DContext, DeclName, LinkageName, Unit, LineNo,
- getOrCreateType(T, Unit, TL), Var->hasLocalLinkage(), true,
+ DContext, DeclName, LinkageName, Unit, LineNo, getOrCreateType(T, Unit),
+ Var->hasLocalLinkage(), true,
Expr.empty() ? nullptr : DBuilder.createExpression(Expr),
getOrCreateStaticDataMemberDeclarationOrNull(D), TemplateParameters,
Align, Annotations);
@@ -5320,6 +5419,62 @@ void CGDebugInfo::EmitExternalVariable(llvm::GlobalVariable *Var,
Var->addDebugInfo(GVE);
}
+void CGDebugInfo::EmitGlobalAlias(const llvm::GlobalValue *GV,
+ const GlobalDecl GD) {
+
+ assert(GV);
+
+ if (!CGM.getCodeGenOpts().hasReducedDebugInfo())
+ return;
+
+ const auto *D = cast<ValueDecl>(GD.getDecl());
+ if (D->hasAttr<NoDebugAttr>())
+ return;
+
+ auto AliaseeDecl = CGM.getMangledNameDecl(GV->getName());
+ llvm::DINode *DI;
+
+ if (!AliaseeDecl)
+ // FIXME: Aliasee not declared yet - possibly declared later
+ // For example,
+ //
+ // 1 extern int newname __attribute__((alias("oldname")));
+ // 2 int oldname = 1;
+ //
+ // No debug info would be generated for 'newname' in this case.
+ //
+ // Fix compiler to generate "newname" as imported_declaration
+ // pointing to the DIE of "oldname".
+ return;
+ if (!(DI = getDeclarationOrDefinition(
+ AliaseeDecl.getCanonicalDecl().getDecl())))
+ return;
+
+ llvm::DIScope *DContext = getDeclContextDescriptor(D);
+ auto Loc = D->getLocation();
+
+ llvm::DIImportedEntity *ImportDI = DBuilder.createImportedDeclaration(
+ DContext, DI, getOrCreateFile(Loc), getLineNumber(Loc), D->getName());
+
+ // Record this DIE in the cache for nested declaration reference.
+ ImportedDeclCache[GD.getCanonicalDecl().getDecl()].reset(ImportDI);
+}
+
+void CGDebugInfo::AddStringLiteralDebugInfo(llvm::GlobalVariable *GV,
+ const StringLiteral *S) {
+ SourceLocation Loc = S->getStrTokenLoc(0);
+ PresumedLoc PLoc = CGM.getContext().getSourceManager().getPresumedLoc(Loc);
+ if (!PLoc.isValid())
+ return;
+
+ llvm::DIFile *File = getOrCreateFile(Loc);
+ llvm::DIGlobalVariableExpression *Debug =
+ DBuilder.createGlobalVariableExpression(
+ nullptr, StringRef(), StringRef(), getOrCreateFile(Loc),
+ getLineNumber(Loc), getOrCreateType(S->getType(), File), true);
+ GV->addDebugInfo(Debug);
+}
+
llvm::DIScope *CGDebugInfo::getCurrentContextDescriptor(const Decl *D) {
if (!LexicalBlockStack.empty())
return LexicalBlockStack.back();
diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h
index a76426e585c8..38e3fa5b2fa9 100644
--- a/clang/lib/CodeGen/CGDebugInfo.h
+++ b/clang/lib/CodeGen/CGDebugInfo.h
@@ -152,8 +152,10 @@ class CGDebugInfo {
llvm::DenseMap<const char *, llvm::TrackingMDRef> DIFileCache;
llvm::DenseMap<const FunctionDecl *, llvm::TrackingMDRef> SPCache;
/// Cache declarations relevant to DW_TAG_imported_declarations (C++
- /// using declarations) that aren't covered by other more specific caches.
+ /// using declarations and global alias variables) that aren't covered
+ /// by other more specific caches.
llvm::DenseMap<const Decl *, llvm::TrackingMDRef> DeclCache;
+ llvm::DenseMap<const Decl *, llvm::TrackingMDRef> ImportedDeclCache;
llvm::DenseMap<const NamespaceDecl *, llvm::TrackingMDRef> NamespaceCache;
llvm::DenseMap<const NamespaceAliasDecl *, llvm::TrackingMDRef>
NamespaceAliasCache;
@@ -177,19 +179,16 @@ class CGDebugInfo {
llvm::DIType *CreateType(const ComplexType *Ty);
llvm::DIType *CreateType(const AutoType *Ty);
llvm::DIType *CreateType(const BitIntType *Ty);
- llvm::DIType *CreateQualifiedType(QualType Ty, llvm::DIFile *Fg,
- TypeLoc TL = TypeLoc());
+ llvm::DIType *CreateQualifiedType(QualType Ty, llvm::DIFile *Fg);
llvm::DIType *CreateQualifiedType(const FunctionProtoType *Ty,
llvm::DIFile *Fg);
llvm::DIType *CreateType(const TypedefType *Ty, llvm::DIFile *Fg);
llvm::DIType *CreateType(const TemplateSpecializationType *Ty,
llvm::DIFile *Fg);
llvm::DIType *CreateType(const ObjCObjectPointerType *Ty, llvm::DIFile *F);
- llvm::DIType *CreateType(const PointerType *Ty, llvm::DIFile *F,
- TypeLoc TL = TypeLoc());
+ llvm::DIType *CreateType(const PointerType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const BlockPointerType *Ty, llvm::DIFile *F);
- llvm::DIType *CreateType(const FunctionType *Ty, llvm::DIFile *F,
- TypeLoc TL = TypeLoc());
+ llvm::DIType *CreateType(const FunctionType *Ty, llvm::DIFile *F);
/// Get structure or union type.
llvm::DIType *CreateType(const RecordType *Tyg);
llvm::DIType *CreateTypeDefinition(const RecordType *Ty);
@@ -244,8 +243,7 @@ class CGDebugInfo {
/// \return namespace descriptor for the given namespace decl.
llvm::DINamespace *getOrCreateNamespace(const NamespaceDecl *N);
llvm::DIType *CreatePointerLikeType(llvm::dwarf::Tag Tag, const Type *Ty,
- QualType PointeeTy, llvm::DIFile *F,
- TypeLoc TL = TypeLoc());
+ QualType PointeeTy, llvm::DIFile *F);
llvm::DIType *getOrCreateStructPtrType(StringRef Name, llvm::DIType *&Cache);
/// A helper function to create a subprogram for a single member
@@ -311,8 +309,7 @@ class CGDebugInfo {
uint64_t offsetInBits, uint32_t AlignInBits,
llvm::DIFile *tunit, llvm::DIScope *scope,
const RecordDecl *RD = nullptr,
- llvm::DINodeArray Annotations = nullptr,
- TypeLoc TL = TypeLoc());
+ llvm::DINodeArray Annotations = nullptr);
llvm::DIType *createFieldType(StringRef name, QualType type,
SourceLocation loc, AccessSpecifier AS,
@@ -512,6 +509,9 @@ public:
/// Emit information about an external variable.
void EmitExternalVariable(llvm::GlobalVariable *GV, const VarDecl *Decl);
+ /// Emit information about global variable alias.
+ void EmitGlobalAlias(const llvm::GlobalValue *GV, const GlobalDecl Decl);
+
/// Emit C++ using directive.
void EmitUsingDirective(const UsingDirectiveDecl &UD);
@@ -533,6 +533,14 @@ public:
/// Emit an @import declaration.
void EmitImportDecl(const ImportDecl &ID);
+ /// DebugInfo isn't attached to string literals by default. While certain
+ /// aspects of debuginfo aren't useful for string literals (like a name), it's
+ /// nice to be able to symbolize the line and column information. This is
+ /// especially useful for sanitizers, as it allows symbolization of
+ /// heap-buffer-overflows on constant strings.
+ void AddStringLiteralDebugInfo(llvm::GlobalVariable *GV,
+ const StringLiteral *S);
+
/// Emit C++ namespace alias.
llvm::DIImportedEntity *EmitNamespaceAlias(const NamespaceAliasDecl &NA);
@@ -583,6 +591,14 @@ private:
CGBuilderTy &Builder,
const bool UsePointerValue = false);
+ /// Emit call to llvm.dbg.declare for a binding declaration.
+ /// Returns a pointer to the DILocalVariable associated with the
+ /// llvm.dbg.declare, or nullptr otherwise.
+ llvm::DILocalVariable *EmitDeclare(const BindingDecl *decl, llvm::Value *AI,
+ llvm::Optional<unsigned> ArgNo,
+ CGBuilderTy &Builder,
+ const bool UsePointerValue = false);
+
struct BlockByRefType {
/// The wrapper struct used inside the __block_literal struct.
llvm::DIType *BlockByRefWrapper;
@@ -632,8 +648,7 @@ private:
Optional<StringRef> Source);
/// Get the type from the cache or create a new type if necessary.
- llvm::DIType *getOrCreateType(QualType Ty, llvm::DIFile *Fg,
- TypeLoc TL = TypeLoc());
+ llvm::DIType *getOrCreateType(QualType Ty, llvm::DIFile *Fg);
/// Get a reference to a clang module. If \p CreateSkeletonCU is true,
/// this also creates a split dwarf skeleton compile unit.
@@ -648,8 +663,7 @@ private:
llvm::DICompositeType *getOrCreateLimitedType(const RecordType *Ty);
/// Create type metadata for a source language type.
- llvm::DIType *CreateTypeNode(QualType Ty, llvm::DIFile *Fg,
- TypeLoc TL = TypeLoc());
+ llvm::DIType *CreateTypeNode(QualType Ty, llvm::DIFile *Fg);
/// Create new member and increase Offset by FType's size.
llvm::DIType *CreateMemberType(llvm::DIFile *Unit, QualType FType,
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 18d658436086..f04af0d2cdf8 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -118,6 +118,7 @@ void CodeGenFunction::EmitDecl(const Decl &D) {
case Decl::Label: // __label__ x;
case Decl::Import:
case Decl::MSGuid: // __declspec(uuid("..."))
+ case Decl::UnnamedGlobalConstant:
case Decl::TemplateParamObject:
case Decl::OMPThreadPrivate:
case Decl::OMPAllocate:
@@ -341,6 +342,8 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
if (!Init) {
if (!getLangOpts().CPlusPlus)
CGM.ErrorUnsupported(D.getInit(), "constant l-value expression");
+ else if (D.hasFlexibleArrayInit(getContext()))
+ CGM.ErrorUnsupported(D.getInit(), "flexible array initializer");
else if (HaveInsertPoint()) {
// Since we have a static initializer, this global variable can't
// be constant.
@@ -351,6 +354,14 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
return GV;
}
+#ifndef NDEBUG
+ CharUnits VarSize = CGM.getContext().getTypeSizeInChars(D.getType()) +
+ D.getFlexibleArrayInitChars(getContext());
+ CharUnits CstSize = CharUnits::fromQuantity(
+ CGM.getDataLayout().getTypeAllocSize(Init->getType()));
+ assert(VarSize == CstSize && "Emitted constant has unexpected size");
+#endif
+
// The initializer may differ in type from the global. Rewrite
// the global to match the initializer. (We have to do this
// because some types, like unions, can't be completely represented
@@ -462,7 +473,7 @@ void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
LocalDeclMap.find(&D)->second = Address(castedAddr, elemTy, alignment);
CGM.setStaticLocalDeclAddress(&D, castedAddr);
- CGM.getSanitizerMetadata()->reportGlobalToASan(var, D);
+ CGM.getSanitizerMetadata()->reportGlobal(var, D);
// Emit global variable debug descriptor for static vars.
CGDebugInfo *DI = getDebugInfo();
@@ -1155,11 +1166,7 @@ static Address createUnnamedGlobalForMemcpyFrom(CodeGenModule &CGM,
llvm::Constant *Constant,
CharUnits Align) {
Address SrcPtr = CGM.createUnnamedGlobalFrom(D, Constant, Align);
- llvm::Type *BP = llvm::PointerType::getInt8PtrTy(CGM.getLLVMContext(),
- SrcPtr.getAddressSpace());
- if (SrcPtr.getType() != BP)
- SrcPtr = Builder.CreateBitCast(SrcPtr, BP);
- return SrcPtr;
+ return Builder.CreateElementBitCast(SrcPtr, CGM.Int8Ty);
}
static void emitStoresForConstant(CodeGenModule &CGM, const VarDecl &D,
@@ -1786,7 +1793,7 @@ void CodeGenFunction::emitZeroOrPatternForAutoVarInit(QualType type,
Cur->addIncoming(Begin.getPointer(), OriginBB);
CharUnits CurAlign = Loc.getAlignment().alignmentOfArrayElement(EltSize);
auto *I =
- Builder.CreateMemCpy(Address(Cur, CurAlign),
+ Builder.CreateMemCpy(Address(Cur, Int8Ty, CurAlign),
createUnnamedGlobalForMemcpyFrom(
CGM, D, Builder, Constant, ConstantAlign),
BaseSizeInChars, isVolatile);
@@ -1908,10 +1915,9 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
return EmitStoreThroughLValue(RValue::get(constant), lv, true);
}
- llvm::Type *BP = CGM.Int8Ty->getPointerTo(Loc.getAddressSpace());
- emitStoresForConstant(
- CGM, D, (Loc.getType() == BP) ? Loc : Builder.CreateBitCast(Loc, BP),
- type.isVolatileQualified(), Builder, constant, /*IsAutoInit=*/false);
+ emitStoresForConstant(CGM, D, Builder.CreateElementBitCast(Loc, CGM.Int8Ty),
+ type.isVolatileQualified(), Builder, constant,
+ /*IsAutoInit=*/false);
}
/// Emit an expression as an initializer for an object (variable, field, etc.)
@@ -2282,6 +2288,8 @@ static void emitPartialArrayDestroy(CodeGenFunction &CGF,
llvm::Value *begin, llvm::Value *end,
QualType type, CharUnits elementAlign,
CodeGenFunction::Destroyer *destroyer) {
+ llvm::Type *elemTy = CGF.ConvertTypeForMem(type);
+
// If the element type is itself an array, drill down.
unsigned arrayDepth = 0;
while (const ArrayType *arrayType = CGF.getContext().getAsArrayType(type)) {
@@ -2295,7 +2303,6 @@ static void emitPartialArrayDestroy(CodeGenFunction &CGF,
llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
SmallVector<llvm::Value*,4> gepIndices(arrayDepth+1, zero);
- llvm::Type *elemTy = begin->getType()->getPointerElementType();
begin = CGF.Builder.CreateInBoundsGEP(
elemTy, begin, gepIndices, "pad.arraybegin");
end = CGF.Builder.CreateInBoundsGEP(
@@ -2435,6 +2442,7 @@ namespace {
/// for the specified parameter and set up LocalDeclMap.
void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
unsigned ArgNo) {
+ bool NoDebugInfo = false;
// FIXME: Why isn't ImplicitParamDecl a ParmVarDecl?
assert((isa<ParmVarDecl>(D) || isa<ImplicitParamDecl>(D)) &&
"Invalid argument to EmitParmDecl");
@@ -2454,6 +2462,10 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
setBlockContextParameter(IPD, ArgNo, V);
return;
}
+ // Suppressing debug info for ThreadPrivateVar parameters, else it hides
+ // debug info of TLS variables.
+ NoDebugInfo =
+ (IPD->getParameterKind() == ImplicitParamDecl::ThreadPrivateVar);
}
Address DeclPtr = Address::invalid();
@@ -2462,12 +2474,10 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
bool IsScalar = hasScalarEvaluationKind(Ty);
// If we already have a pointer to the argument, reuse the input pointer.
if (Arg.isIndirect()) {
- DeclPtr = Arg.getIndirectAddress();
// If we have a prettier pointer type at this point, bitcast to that.
- unsigned AS = DeclPtr.getType()->getAddressSpace();
- llvm::Type *IRTy = ConvertTypeForMem(Ty)->getPointerTo(AS);
- if (DeclPtr.getType() != IRTy)
- DeclPtr = Builder.CreateBitCast(DeclPtr, IRTy, D.getName());
+ DeclPtr = Arg.getIndirectAddress();
+ DeclPtr = Builder.CreateElementBitCast(DeclPtr, ConvertTypeForMem(Ty),
+ D.getName());
// Indirect argument is in alloca address space, which may be different
// from the default address space.
auto AllocaAS = CGM.getASTAllocaAddressSpace();
@@ -2480,10 +2490,9 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
assert(getContext().getTargetAddressSpace(SrcLangAS) ==
CGM.getDataLayout().getAllocaAddrSpace());
auto DestAS = getContext().getTargetAddressSpace(DestLangAS);
- auto *T = V->getType()->getPointerElementType()->getPointerTo(DestAS);
- DeclPtr = Address(getTargetHooks().performAddrSpaceCast(
- *this, V, SrcLangAS, DestLangAS, T, true),
- DeclPtr.getAlignment());
+ auto *T = DeclPtr.getElementType()->getPointerTo(DestAS);
+ DeclPtr = DeclPtr.withPointer(getTargetHooks().performAddrSpaceCast(
+ *this, V, SrcLangAS, DestLangAS, T, true));
}
// Push a destructor cleanup for this parameter if the ABI requires it.
@@ -2587,7 +2596,8 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
// Emit debug info for param declarations in non-thunk functions.
if (CGDebugInfo *DI = getDebugInfo()) {
- if (CGM.getCodeGenOpts().hasReducedDebugInfo() && !CurFuncIsThunk) {
+ if (CGM.getCodeGenOpts().hasReducedDebugInfo() && !CurFuncIsThunk &&
+ !NoDebugInfo) {
llvm::DILocalVariable *DILocalVar = DI->EmitDeclareOfArgVariable(
&D, AllocaPtr.getPointer(), ArgNo, Builder);
if (const auto *Var = dyn_cast_or_null<ParmVarDecl>(&D))
@@ -2684,3 +2694,22 @@ void CodeGenModule::EmitOMPAllocateDecl(const OMPAllocateDecl *D) {
DummyGV->eraseFromParent();
}
}
+
+llvm::Optional<CharUnits>
+CodeGenModule::getOMPAllocateAlignment(const VarDecl *VD) {
+ if (const auto *AA = VD->getAttr<OMPAllocateDeclAttr>()) {
+ if (Expr *Alignment = AA->getAlignment()) {
+ unsigned UserAlign =
+ Alignment->EvaluateKnownConstInt(getContext()).getExtValue();
+ CharUnits NaturalAlign =
+ getNaturalTypeAlignment(VD->getType().getNonReferenceType());
+
+ // OpenMP5.1 pg 185 lines 7-10
+ // Each item in the align modifier list must be aligned to the maximum
+ // of the specified alignment and the type's natural alignment.
+ return CharUnits::fromQuantity(
+ std::max<unsigned>(UserAlign, NaturalAlign.getQuantity()));
+ }
+ }
+ return llvm::None;
+}
diff --git a/clang/lib/CodeGen/CGDeclCXX.cpp b/clang/lib/CodeGen/CGDeclCXX.cpp
index 7b880c1354e1..de5cb913220a 100644
--- a/clang/lib/CodeGen/CGDeclCXX.cpp
+++ b/clang/lib/CodeGen/CGDeclCXX.cpp
@@ -425,9 +425,8 @@ void CodeGenFunction::EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
llvm::Function *CodeGenModule::CreateGlobalInitOrCleanUpFunction(
llvm::FunctionType *FTy, const Twine &Name, const CGFunctionInfo &FI,
- SourceLocation Loc, bool TLS) {
- llvm::Function *Fn = llvm::Function::Create(
- FTy, llvm::GlobalValue::InternalLinkage, Name, &getModule());
+ SourceLocation Loc, bool TLS, llvm::GlobalVariable::LinkageTypes Linkage) {
+ llvm::Function *Fn = llvm::Function::Create(FTy, Linkage, Name, &getModule());
if (!getLangOpts().AppleKext && !TLS) {
// Set the section if needed.
@@ -435,7 +434,8 @@ llvm::Function *CodeGenModule::CreateGlobalInitOrCleanUpFunction(
Fn->setSection(Section);
}
- SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);
+ if (Linkage == llvm::GlobalVariable::InternalLinkage)
+ SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);
Fn->setCallingConv(getRuntimeCC());
@@ -458,8 +458,8 @@ llvm::Function *CodeGenModule::CreateGlobalInitOrCleanUpFunction(
!isInNoSanitizeList(SanitizerKind::KernelHWAddress, Fn, Loc))
Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
- if (getLangOpts().Sanitize.has(SanitizerKind::MemTag) &&
- !isInNoSanitizeList(SanitizerKind::MemTag, Fn, Loc))
+ if (getLangOpts().Sanitize.has(SanitizerKind::MemtagStack) &&
+ !isInNoSanitizeList(SanitizerKind::MemtagStack, Fn, Loc))
Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
if (getLangOpts().Sanitize.has(SanitizerKind::Thread) &&
@@ -707,7 +707,7 @@ CodeGenModule::EmitCXXGlobalInitFunc() {
// dynamic resource allocation on the device and program scope variables are
// destroyed by the runtime when program is released.
if (getLangOpts().OpenCL) {
- GenOpenCLArgMetadata(Fn);
+ GenKernelArgMetadata(Fn);
Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
}
diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp
index 91ecbecc843f..76c6beb090a9 100644
--- a/clang/lib/CodeGen/CGException.cpp
+++ b/clang/lib/CodeGen/CGException.cpp
@@ -1845,7 +1845,7 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
llvm::Value *ChildVar =
Builder.CreateBitCast(RecoverCall, ParentVar.getType());
ChildVar->setName(ParentVar.getName());
- return Address(ChildVar, ParentVar.getAlignment());
+ return ParentVar.withPointer(ChildVar);
}
void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF,
@@ -1931,7 +1931,8 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF,
FrameRecoverFn, {ParentI8Fn, ParentFP,
llvm::ConstantInt::get(Int32Ty, FrameEscapeIdx)});
ParentFP = Builder.CreateBitCast(ParentFP, CGM.VoidPtrPtrTy);
- ParentFP = Builder.CreateLoad(Address(ParentFP, getPointerAlign()));
+ ParentFP = Builder.CreateLoad(
+ Address(ParentFP, CGM.VoidPtrTy, getPointerAlign()));
}
}
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index bb5d18b74894..cbeb6c938bee 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -189,7 +189,17 @@ llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
/// ignoring the result.
void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
if (E->isPRValue())
- return (void) EmitAnyExpr(E, AggValueSlot::ignored(), true);
+ return (void)EmitAnyExpr(E, AggValueSlot::ignored(), true);
+
+ // if this is a bitfield-resulting conditional operator, we can special case
+ // emit this. The normal 'EmitLValue' version of this is particularly
+ // difficult to codegen for, since creating a single "LValue" for two
+ // different sized arguments here is not particularly doable.
+ if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
+ E->IgnoreParenNoopCasts(getContext()))) {
+ if (CondOp->getObjectKind() == OK_BitField)
+ return EmitIgnoredConditionalOperator(CondOp);
+ }
// Just emit it as an l-value and drop the result.
EmitLValue(E);
@@ -407,7 +417,7 @@ static Address createReferenceTemporary(CodeGenFunction &CGF,
GV->getValueType()->getPointerTo(
CGF.getContext().getTargetAddressSpace(LangAS::Default)));
// FIXME: Should we put the new global into a COMDAT?
- return Address(C, alignment);
+ return Address(C, GV->getValueType(), alignment);
}
return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca);
}
@@ -441,10 +451,10 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
ownership != Qualifiers::OCL_ExplicitNone) {
Address Object = createReferenceTemporary(*this, M, E);
if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {
- Object = Address(llvm::ConstantExpr::getBitCast(Var,
- ConvertTypeForMem(E->getType())
- ->getPointerTo(Object.getAddressSpace())),
- Object.getAlignment());
+ llvm::Type *Ty = ConvertTypeForMem(E->getType());
+ Object = Address(llvm::ConstantExpr::getBitCast(
+ Var, Ty->getPointerTo(Object.getAddressSpace())),
+ Ty, Object.getAlignment());
// createReferenceTemporary will promote the temporary to a global with a
// constant initializer if it can. It can only do this to a value of
@@ -499,9 +509,11 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
Address Object = createReferenceTemporary(*this, M, E, &Alloca);
if (auto *Var = dyn_cast<llvm::GlobalVariable>(
Object.getPointer()->stripPointerCasts())) {
+ llvm::Type *TemporaryType = ConvertTypeForMem(E->getType());
Object = Address(llvm::ConstantExpr::getBitCast(
cast<llvm::Constant>(Object.getPointer()),
- ConvertTypeForMem(E->getType())->getPointerTo()),
+ TemporaryType->getPointerTo()),
+ TemporaryType,
Object.getAlignment());
// If the temporary is a global and has a constant initializer or is a
// constant temporary that we promoted to a global, we may have already
@@ -745,23 +757,23 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
}
}
- uint64_t AlignVal = 0;
+ llvm::MaybeAlign AlignVal;
llvm::Value *PtrAsInt = nullptr;
if (SanOpts.has(SanitizerKind::Alignment) &&
!SkippedChecks.has(SanitizerKind::Alignment)) {
- AlignVal = Alignment.getQuantity();
+ AlignVal = Alignment.getAsMaybeAlign();
if (!Ty->isIncompleteType() && !AlignVal)
AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr,
/*ForPointeeType=*/true)
- .getQuantity();
+ .getAsMaybeAlign();
// The glvalue must be suitably aligned.
- if (AlignVal > 1 &&
- (!PtrToAlloca || PtrToAlloca->getAlignment() < AlignVal)) {
+ if (AlignVal && *AlignVal > llvm::Align(1) &&
+ (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy);
llvm::Value *Align = Builder.CreateAnd(
- PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal - 1));
+ PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal->value() - 1));
llvm::Value *Aligned =
Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));
if (Aligned != True)
@@ -770,12 +782,9 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
}
if (Checks.size() > 0) {
- // Make sure we're not losing information. Alignment needs to be a power of
- // 2
- assert(!AlignVal || (uint64_t)1 << llvm::Log2_64(AlignVal) == AlignVal);
llvm::Constant *StaticData[] = {
EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty),
- llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2_64(AlignVal) : 1),
+ llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1),
llvm::ConstantInt::get(Int8Ty, TCK)};
EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData,
PtrAsInt ? PtrAsInt : Ptr);
@@ -821,7 +830,8 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
// Load the vptr, and compute hash_16_bytes(TypeHash, vptr).
llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash);
llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0);
- Address VPtrAddr(Builder.CreateBitCast(Ptr, VPtrTy), getPointerAlign());
+ Address VPtrAddr(Builder.CreateBitCast(Ptr, VPtrTy), IntPtrTy,
+ getPointerAlign());
llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr);
llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty);
@@ -1106,17 +1116,17 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E,
if (SanOpts.has(SanitizerKind::CFIUnrelatedCast) &&
CE->getCastKind() == CK_BitCast) {
if (auto PT = E->getType()->getAs<PointerType>())
- EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr.getPointer(),
+ EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr,
/*MayBeNull=*/true,
CodeGenFunction::CFITCK_UnrelatedCast,
CE->getBeginLoc());
}
- if (CE->getCastKind() == CK_AddressSpaceConversion)
- return Builder.CreateAddrSpaceCast(Addr, ConvertType(E->getType()));
-
llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType());
- return Builder.CreateElementBitCast(Addr, ElemTy);
+ Addr = Builder.CreateElementBitCast(Addr, ElemTy);
+ if (CE->getCastKind() == CK_AddressSpaceConversion)
+ Addr = Builder.CreateAddrSpaceCast(Addr, ConvertType(E->getType()));
+ return Addr;
}
break;
@@ -1157,6 +1167,22 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E,
}
}
+ // std::addressof and variants.
+ if (auto *Call = dyn_cast<CallExpr>(E)) {
+ switch (Call->getBuiltinCallee()) {
+ default:
+ break;
+ case Builtin::BIaddressof:
+ case Builtin::BI__addressof:
+ case Builtin::BI__builtin_addressof: {
+ LValue LV = EmitLValue(Call->getArg(0));
+ if (BaseInfo) *BaseInfo = LV.getBaseInfo();
+ if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
+ return LV.getAddress(*this);
+ }
+ }
+ }
+
// TODO: conditional operators, comma.
// Otherwise, use the alignment of the type.
@@ -1208,9 +1234,10 @@ RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
const char *Name) {
ErrorUnsupported(E, Name);
- llvm::Type *Ty = llvm::PointerType::getUnqual(ConvertType(E->getType()));
- return MakeAddrLValue(Address(llvm::UndefValue::get(Ty), CharUnits::One()),
- E->getType());
+ llvm::Type *ElTy = ConvertType(E->getType());
+ llvm::Type *Ty = llvm::PointerType::getUnqual(ElTy);
+ return MakeAddrLValue(
+ Address(llvm::UndefValue::get(Ty), ElTy, CharUnits::One()), E->getType());
}
bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
@@ -1703,27 +1730,42 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
LValueBaseInfo BaseInfo,
TBAAAccessInfo TBAAInfo,
bool isNontemporal) {
- if (!CGM.getCodeGenOpts().PreserveVec3Type) {
- // For better performance, handle vector loads differently.
- if (Ty->isVectorType()) {
- const llvm::Type *EltTy = Addr.getElementType();
+ if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
+ // Boolean vectors use `iN` as storage type.
+ if (ClangVecTy->isExtVectorBoolType()) {
+ llvm::Type *ValTy = ConvertType(Ty);
+ unsigned ValNumElems =
+ cast<llvm::FixedVectorType>(ValTy)->getNumElements();
+ // Load the `iP` storage object (P is the padded vector size).
+ auto *RawIntV = Builder.CreateLoad(Addr, Volatile, "load_bits");
+ const auto *RawIntTy = RawIntV->getType();
+ assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
+ // Bitcast iP --> <P x i1>.
+ auto *PaddedVecTy = llvm::FixedVectorType::get(
+ Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
+ llvm::Value *V = Builder.CreateBitCast(RawIntV, PaddedVecTy);
+ // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
+ V = emitBoolVecConversion(V, ValNumElems, "extractvec");
- const auto *VTy = cast<llvm::FixedVectorType>(EltTy);
+ return EmitFromMemory(V, Ty);
+ }
- // Handle vectors of size 3 like size 4 for better performance.
- if (VTy->getNumElements() == 3) {
+ // Handle vectors of size 3 like size 4 for better performance.
+ const llvm::Type *EltTy = Addr.getElementType();
+ const auto *VTy = cast<llvm::FixedVectorType>(EltTy);
- // Bitcast to vec4 type.
- auto *vec4Ty = llvm::FixedVectorType::get(VTy->getElementType(), 4);
- Address Cast = Builder.CreateElementBitCast(Addr, vec4Ty, "castToVec4");
- // Now load value.
- llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4");
+ if (!CGM.getCodeGenOpts().PreserveVec3Type && VTy->getNumElements() == 3) {
- // Shuffle vector to get vec3.
- V = Builder.CreateShuffleVector(V, ArrayRef<int>{0, 1, 2},
- "extractVec");
- return EmitFromMemory(V, Ty);
- }
+ // Bitcast to vec4 type.
+ llvm::VectorType *vec4Ty =
+ llvm::FixedVectorType::get(VTy->getElementType(), 4);
+ Address Cast = Builder.CreateElementBitCast(Addr, vec4Ty, "castToVec4");
+ // Now load value.
+ llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4");
+
+ // Shuffle vector to get vec3.
+ V = Builder.CreateShuffleVector(V, ArrayRef<int>{0, 1, 2}, "extractVec");
+ return EmitFromMemory(V, Ty);
}
}
@@ -1774,6 +1816,17 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
"wrong value rep of bool");
return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool");
}
+ if (Ty->isExtVectorBoolType()) {
+ const auto *RawIntTy = Value->getType();
+ // Bitcast iP --> <P x i1>.
+ auto *PaddedVecTy = llvm::FixedVectorType::get(
+ Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
+ auto *V = Builder.CreateBitCast(Value, PaddedVecTy);
+ // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
+ llvm::Type *ValTy = ConvertType(Ty);
+ unsigned ValNumElems = cast<llvm::FixedVectorType>(ValTy)->getNumElements();
+ return emitBoolVecConversion(V, ValNumElems, "extractvec");
+ }
return Value;
}
@@ -1818,11 +1871,18 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
LValueBaseInfo BaseInfo,
TBAAAccessInfo TBAAInfo,
bool isInit, bool isNontemporal) {
- if (!CGM.getCodeGenOpts().PreserveVec3Type) {
- // Handle vectors differently to get better performance.
- if (Ty->isVectorType()) {
- llvm::Type *SrcTy = Value->getType();
- auto *VecTy = dyn_cast<llvm::VectorType>(SrcTy);
+ llvm::Type *SrcTy = Value->getType();
+ if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
+ auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy);
+ if (VecTy && ClangVecTy->isExtVectorBoolType()) {
+ auto *MemIntTy = cast<llvm::IntegerType>(Addr.getElementType());
+ // Expand to the memory bit width.
+ unsigned MemNumElems = MemIntTy->getPrimitiveSizeInBits();
+ // <N x i1> --> <P x i1>.
+ Value = emitBoolVecConversion(Value, MemNumElems, "insertvec");
+ // <P x i1> --> iP.
+ Value = Builder.CreateBitCast(Value, MemIntTy);
+ } else if (!CGM.getCodeGenOpts().PreserveVec3Type) {
// Handle vec3 special.
if (VecTy && cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) {
// Our source is a vec3, do a shuffle vector to make it a vec4.
@@ -1932,7 +1992,7 @@ RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
llvm::Value *Idx = LV.getMatrixIdx();
if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
const auto *const MatTy = LV.getType()->castAs<ConstantMatrixType>();
- llvm::MatrixBuilder<CGBuilderTy> MB(Builder);
+ llvm::MatrixBuilder MB(Builder);
MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
}
llvm::LoadInst *Load =
@@ -2059,8 +2119,19 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
// Read/modify/write the vector, inserting the new element.
llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(),
Dst.isVolatileQualified());
+ auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType());
+ if (IRStoreTy) {
+ auto *IRVecTy = llvm::FixedVectorType::get(
+ Builder.getInt1Ty(), IRStoreTy->getPrimitiveSizeInBits());
+ Vec = Builder.CreateBitCast(Vec, IRVecTy);
+ // iN --> <N x i1>.
+ }
Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
Dst.getVectorIdx(), "vecins");
+ if (IRStoreTy) {
+ // <N x i1> --> <iN>.
+ Vec = Builder.CreateBitCast(Vec, IRStoreTy);
+ }
Builder.CreateStore(Vec, Dst.getVectorAddress(),
Dst.isVolatileQualified());
return;
@@ -2078,7 +2149,7 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
llvm::Value *Idx = Dst.getMatrixIdx();
if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
- llvm::MatrixBuilder<CGBuilderTy> MB(Builder);
+ llvm::MatrixBuilder MB(Builder);
MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
}
llvm::Instruction *Load = Builder.CreateLoad(Dst.getMatrixAddress());
@@ -2499,9 +2570,10 @@ Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
LValueBaseInfo *BaseInfo,
TBAAAccessInfo *TBAAInfo) {
llvm::Value *Addr = Builder.CreateLoad(Ptr);
- return Address(Addr, CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(),
- BaseInfo, TBAAInfo,
- /*forPointeeType=*/true));
+ return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()),
+ CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo,
+ TBAAInfo,
+ /*forPointeeType=*/true));
}
LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
@@ -2701,8 +2773,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
llvm::Type *VarTy = getTypes().ConvertTypeForMem(VD->getType());
auto *PTy = llvm::PointerType::get(
VarTy, getContext().getTargetAddressSpace(VD->getType()));
- if (PTy != Addr.getType())
- Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy);
+ Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy, VarTy);
} else {
// Should we be using the alignment of the constant pointer we emitted?
CharUnits Alignment =
@@ -2741,8 +2812,10 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
LValue CapLVal =
EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD),
CapturedStmtInfo->getContextValue());
+ Address LValueAddress = CapLVal.getAddress(*this);
CapLVal = MakeAddrLValue(
- Address(CapLVal.getPointer(*this), getContext().getDeclAlign(VD)),
+ Address(LValueAddress.getPointer(), LValueAddress.getElementType(),
+ getContext().getDeclAlign(VD)),
CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl),
CapLVal.getTBAAInfo());
// Mark lvalue as nontemporal if the variable is marked as nontemporal
@@ -3183,7 +3256,7 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF,
B.addAttribute(llvm::Attribute::NoReturn)
.addAttribute(llvm::Attribute::NoUnwind);
}
- B.addAttribute(llvm::Attribute::UWTable);
+ B.addUWTableAttr(llvm::UWTableKind::Default);
llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
FnType, FnName,
@@ -3431,7 +3504,8 @@ void CodeGenFunction::EmitCfiCheckFail() {
CfiCheckFailDataTy,
Builder.CreatePointerCast(Data, CfiCheckFailDataTy->getPointerTo(0)), 0,
0);
- Address CheckKindAddr(V, getIntAlign());
+
+ Address CheckKindAddr(V, Int8Ty, getIntAlign());
llvm::Value *CheckKind = Builder.CreateLoad(CheckKindAddr);
llvm::Value *AllVtables = llvm::MetadataAsValue::get(
@@ -3808,7 +3882,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
// interfaces, so we can't rely on GEP to do this scaling
// correctly, so we need to cast to i8*. FIXME: is this actually
// true? A lot of other things in the fragile ABI would break...
- llvm::Type *OrigBaseTy = Addr.getType();
+ llvm::Type *OrigBaseElemTy = Addr.getElementType();
Addr = Builder.CreateElementBitCast(Addr, Int8Ty);
// Do the GEP.
@@ -3817,10 +3891,10 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
llvm::Value *EltPtr =
emitArraySubscriptGEP(*this, Addr.getElementType(), Addr.getPointer(),
ScaledIdx, false, SignedIndices, E->getExprLoc());
- Addr = Address(EltPtr, EltAlign);
+ Addr = Address(EltPtr, Addr.getElementType(), EltAlign);
// Cast back.
- Addr = Builder.CreateBitCast(Addr, OrigBaseTy);
+ Addr = Builder.CreateElementBitCast(Addr, OrigBaseElemTy);
} else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
// If this is A[i] where A is an array, the frontend will have decayed the
// base to be a ArrayToPointerDecay implicit cast. While correct, it is
@@ -3917,7 +3991,8 @@ static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base,
CGF.CGM.getNaturalTypeAlignment(ElTy, &TypeBaseInfo, &TypeTBAAInfo);
BaseInfo.mergeForCast(TypeBaseInfo);
TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(TBAAInfo, TypeTBAAInfo);
- return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)), Align);
+ return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)),
+ CGF.ConvertTypeForMem(ElTy), Align);
}
return CGF.EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo);
}
@@ -4374,7 +4449,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base,
// fields may leak the real address of dynamic object, which could result
// in miscompilation when leaked pointer would be compared.
auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer());
- addr = Address(stripped, addr.getAlignment());
+ addr = Address(stripped, addr.getElementType(), addr.getAlignment());
}
}
@@ -4395,7 +4470,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base,
addr = Address(
Builder.CreatePreserveUnionAccessIndex(
addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo),
- addr.getAlignment());
+ addr.getElementType(), addr.getAlignment());
}
if (FieldType->isReferenceType())
@@ -4518,94 +4593,140 @@ static Optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF,
return CGF.EmitLValue(Operand);
}
-LValue CodeGenFunction::
-EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
- if (!expr->isGLValue()) {
- // ?: here should be an aggregate.
- assert(hasAggregateEvaluationKind(expr->getType()) &&
- "Unexpected conditional operator!");
- return EmitAggExprToLValue(expr);
- }
-
- OpaqueValueMapping binding(*this, expr);
-
- const Expr *condExpr = expr->getCond();
+namespace {
+// Handle the case where the condition is a constant evaluatable simple integer,
+// which means we don't have to separately handle the true/false blocks.
+llvm::Optional<LValue> HandleConditionalOperatorLValueSimpleCase(
+ CodeGenFunction &CGF, const AbstractConditionalOperator *E) {
+ const Expr *condExpr = E->getCond();
bool CondExprBool;
- if (ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {
- const Expr *live = expr->getTrueExpr(), *dead = expr->getFalseExpr();
- if (!CondExprBool) std::swap(live, dead);
+ if (CGF.ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {
+ const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr();
+ if (!CondExprBool)
+ std::swap(Live, Dead);
- if (!ContainsLabel(dead)) {
+ if (!CGF.ContainsLabel(Dead)) {
// If the true case is live, we need to track its region.
if (CondExprBool)
- incrementProfileCounter(expr);
+ CGF.incrementProfileCounter(E);
// If a throw expression we emit it and return an undefined lvalue
// because it can't be used.
- if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(live->IgnoreParens())) {
- EmitCXXThrowExpr(ThrowExpr);
- llvm::Type *ElemTy = ConvertType(dead->getType());
+ if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Live->IgnoreParens())) {
+ CGF.EmitCXXThrowExpr(ThrowExpr);
+ llvm::Type *ElemTy = CGF.ConvertType(Dead->getType());
llvm::Type *Ty = llvm::PointerType::getUnqual(ElemTy);
- return MakeAddrLValue(
+ return CGF.MakeAddrLValue(
Address(llvm::UndefValue::get(Ty), ElemTy, CharUnits::One()),
- dead->getType());
+ Dead->getType());
}
- return EmitLValue(live);
+ return CGF.EmitLValue(Live);
}
}
+ return llvm::None;
+}
+struct ConditionalInfo {
+ llvm::BasicBlock *lhsBlock, *rhsBlock;
+ Optional<LValue> LHS, RHS;
+};
- llvm::BasicBlock *lhsBlock = createBasicBlock("cond.true");
- llvm::BasicBlock *rhsBlock = createBasicBlock("cond.false");
- llvm::BasicBlock *contBlock = createBasicBlock("cond.end");
+// Create and generate the 3 blocks for a conditional operator.
+// Leaves the 'current block' in the continuation basic block.
+template<typename FuncTy>
+ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF,
+ const AbstractConditionalOperator *E,
+ const FuncTy &BranchGenFunc) {
+ ConditionalInfo Info{CGF.createBasicBlock("cond.true"),
+ CGF.createBasicBlock("cond.false"), llvm::None,
+ llvm::None};
+ llvm::BasicBlock *endBlock = CGF.createBasicBlock("cond.end");
- ConditionalEvaluation eval(*this);
- EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock, getProfileCount(expr));
+ CodeGenFunction::ConditionalEvaluation eval(CGF);
+ CGF.EmitBranchOnBoolExpr(E->getCond(), Info.lhsBlock, Info.rhsBlock,
+ CGF.getProfileCount(E));
// Any temporaries created here are conditional.
- EmitBlock(lhsBlock);
- incrementProfileCounter(expr);
- eval.begin(*this);
- Optional<LValue> lhs =
- EmitLValueOrThrowExpression(*this, expr->getTrueExpr());
- eval.end(*this);
-
- if (lhs && !lhs->isSimple())
- return EmitUnsupportedLValue(expr, "conditional operator");
+ CGF.EmitBlock(Info.lhsBlock);
+ CGF.incrementProfileCounter(E);
+ eval.begin(CGF);
+ Info.LHS = BranchGenFunc(CGF, E->getTrueExpr());
+ eval.end(CGF);
+ Info.lhsBlock = CGF.Builder.GetInsertBlock();
- lhsBlock = Builder.GetInsertBlock();
- if (lhs)
- Builder.CreateBr(contBlock);
+ if (Info.LHS)
+ CGF.Builder.CreateBr(endBlock);
// Any temporaries created here are conditional.
- EmitBlock(rhsBlock);
- eval.begin(*this);
- Optional<LValue> rhs =
- EmitLValueOrThrowExpression(*this, expr->getFalseExpr());
- eval.end(*this);
- if (rhs && !rhs->isSimple())
- return EmitUnsupportedLValue(expr, "conditional operator");
- rhsBlock = Builder.GetInsertBlock();
+ CGF.EmitBlock(Info.rhsBlock);
+ eval.begin(CGF);
+ Info.RHS = BranchGenFunc(CGF, E->getFalseExpr());
+ eval.end(CGF);
+ Info.rhsBlock = CGF.Builder.GetInsertBlock();
+ CGF.EmitBlock(endBlock);
+
+ return Info;
+}
+} // namespace
+
+void CodeGenFunction::EmitIgnoredConditionalOperator(
+ const AbstractConditionalOperator *E) {
+ if (!E->isGLValue()) {
+ // ?: here should be an aggregate.
+ assert(hasAggregateEvaluationKind(E->getType()) &&
+ "Unexpected conditional operator!");
+ return (void)EmitAggExprToLValue(E);
+ }
+
+ OpaqueValueMapping binding(*this, E);
+ if (HandleConditionalOperatorLValueSimpleCase(*this, E))
+ return;
+
+ EmitConditionalBlocks(*this, E, [](CodeGenFunction &CGF, const Expr *E) {
+ CGF.EmitIgnoredExpr(E);
+ return LValue{};
+ });
+}
+LValue CodeGenFunction::EmitConditionalOperatorLValue(
+ const AbstractConditionalOperator *expr) {
+ if (!expr->isGLValue()) {
+ // ?: here should be an aggregate.
+ assert(hasAggregateEvaluationKind(expr->getType()) &&
+ "Unexpected conditional operator!");
+ return EmitAggExprToLValue(expr);
+ }
+
+ OpaqueValueMapping binding(*this, expr);
+ if (llvm::Optional<LValue> Res =
+ HandleConditionalOperatorLValueSimpleCase(*this, expr))
+ return *Res;
+
+ ConditionalInfo Info = EmitConditionalBlocks(
+ *this, expr, [](CodeGenFunction &CGF, const Expr *E) {
+ return EmitLValueOrThrowExpression(CGF, E);
+ });
- EmitBlock(contBlock);
+ if ((Info.LHS && !Info.LHS->isSimple()) ||
+ (Info.RHS && !Info.RHS->isSimple()))
+ return EmitUnsupportedLValue(expr, "conditional operator");
- if (lhs && rhs) {
- Address lhsAddr = lhs->getAddress(*this);
- Address rhsAddr = rhs->getAddress(*this);
+ if (Info.LHS && Info.RHS) {
+ Address lhsAddr = Info.LHS->getAddress(*this);
+ Address rhsAddr = Info.RHS->getAddress(*this);
llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue");
- phi->addIncoming(lhsAddr.getPointer(), lhsBlock);
- phi->addIncoming(rhsAddr.getPointer(), rhsBlock);
+ phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock);
+ phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock);
Address result(phi, lhsAddr.getElementType(),
std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment()));
AlignmentSource alignSource =
- std::max(lhs->getBaseInfo().getAlignmentSource(),
- rhs->getBaseInfo().getAlignmentSource());
+ std::max(Info.LHS->getBaseInfo().getAlignmentSource(),
+ Info.RHS->getBaseInfo().getAlignmentSource());
TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
- lhs->getTBAAInfo(), rhs->getTBAAInfo());
+ Info.LHS->getTBAAInfo(), Info.RHS->getTBAAInfo());
return MakeAddrLValue(result, expr->getType(), LValueBaseInfo(alignSource),
TBAAInfo);
} else {
- assert((lhs || rhs) &&
+ assert((Info.LHS || Info.RHS) &&
"both operands of glvalue conditional are throw-expressions?");
- return lhs ? *lhs : *rhs;
+ return Info.LHS ? *Info.LHS : *Info.RHS;
}
}
@@ -4747,7 +4868,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
Derived.getPointer(), E->getType());
if (SanOpts.has(SanitizerKind::CFIDerivedCast))
- EmitVTablePtrCheckForCast(E->getType(), Derived.getPointer(),
+ EmitVTablePtrCheckForCast(E->getType(), Derived,
/*MayBeNull=*/false, CFITCK_DerivedCast,
E->getBeginLoc());
@@ -4765,7 +4886,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
ConvertTypeForMem(CE->getTypeAsWritten()->getPointeeType()));
if (SanOpts.has(SanitizerKind::CFIUnrelatedCast))
- EmitVTablePtrCheckForCast(E->getType(), V.getPointer(),
+ EmitVTablePtrCheckForCast(E->getType(), V,
/*MayBeNull=*/false, CFITCK_UnrelatedCast,
E->getBeginLoc());
@@ -4779,7 +4900,8 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
*this, LV.getPointer(*this),
E->getSubExpr()->getType().getAddressSpace(),
E->getType().getAddressSpace(), ConvertType(DestTy));
- return MakeAddrLValue(Address(V, LV.getAddress(*this).getAlignment()),
+ return MakeAddrLValue(Address(V, ConvertTypeForMem(E->getType()),
+ LV.getAddress(*this).getAlignment()),
E->getType(), LV.getBaseInfo(), LV.getTBAAInfo());
}
case CK_ObjCObjectLValueCast: {
@@ -4895,15 +5017,34 @@ RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,
return EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue);
}
+// Detect the unusual situation where an inline version is shadowed by a
+// non-inline version. In that case we should pick the external one
+// everywhere. That's GCC behavior too.
+static bool OnlyHasInlineBuiltinDeclaration(const FunctionDecl *FD) {
+ for (const FunctionDecl *PD = FD; PD; PD = PD->getPreviousDecl())
+ if (!PD->isInlineBuiltinDeclaration())
+ return false;
+ return true;
+}
+
static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
if (auto builtinID = FD->getBuiltinID()) {
+ std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str();
+ std::string NoBuiltins = "no-builtins";
std::string FDInlineName = (FD->getName() + ".inline").str();
+
+ bool IsPredefinedLibFunction =
+ CGF.getContext().BuiltinInfo.isPredefinedLibFunction(builtinID);
+ bool HasAttributeNoBuiltin =
+ CGF.CurFn->getAttributes().hasFnAttr(NoBuiltinFD) ||
+ CGF.CurFn->getAttributes().hasFnAttr(NoBuiltins);
+
// When directing calling an inline builtin, call it through it's mangled
// name to make it clear it's not the actual builtin.
- if (FD->isInlineBuiltinDeclaration() &&
- CGF.CurFn->getName() != FDInlineName) {
+ if (CGF.CurFn->getName() != FDInlineName &&
+ OnlyHasInlineBuiltinDeclaration(FD)) {
llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD);
llvm::Function *Fn = llvm::cast<llvm::Function>(CalleePtr);
llvm::Module *M = Fn->getParent();
@@ -4919,8 +5060,11 @@ static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
// Replaceable builtins provide their own implementation of a builtin. If we
// are in an inline builtin implementation, avoid trivial infinite
- // recursion.
- else
+ // recursion. Honor __attribute__((no_builtin("foo"))) or
+ // __attribute__((no_builtin)) on the current function unless foo is
+ // not a predefined library function which means we must generate the
+ // builtin no matter what.
+ else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin)
return CGCallee::forBuiltin(builtinID, FD);
}
@@ -5333,7 +5477,8 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee
llvm::Value *Handle = Callee.getFunctionPointer();
auto *Cast =
Builder.CreateBitCast(Handle, Handle->getType()->getPointerTo());
- auto *Stub = Builder.CreateLoad(Address(Cast, CGM.getPointerAlign()));
+ auto *Stub = Builder.CreateLoad(
+ Address(Cast, Handle->getType(), CGM.getPointerAlign()));
Callee.setFunctionPointer(Stub);
}
llvm::CallBase *CallOrInvoke = nullptr;
diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp
index f06d21861740..3cc144361542 100644
--- a/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/clang/lib/CodeGen/CGExprCXX.cpp
@@ -548,11 +548,12 @@ static void EmitNullBaseClassInitialization(CodeGenFunction &CGF,
/*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage,
NullConstantForBase, Twine());
- CharUnits Align = std::max(Layout.getNonVirtualAlignment(),
- DestPtr.getAlignment());
+ CharUnits Align =
+ std::max(Layout.getNonVirtualAlignment(), DestPtr.getAlignment());
NullVariable->setAlignment(Align.getAsAlign());
- Address SrcPtr = Address(CGF.EmitCastToVoidPtr(NullVariable), Align);
+ Address SrcPtr =
+ Address(CGF.EmitCastToVoidPtr(NullVariable), CGF.Int8Ty, Align);
// Get and call the appropriate llvm.memcpy overload.
for (std::pair<CharUnits, CharUnits> Store : Stores) {
@@ -1244,10 +1245,10 @@ void CodeGenFunction::EmitNewArrayInitializer(
// Set up the current-element phi.
llvm::PHINode *CurPtrPhi =
- Builder.CreatePHI(CurPtr.getType(), 2, "array.cur");
+ Builder.CreatePHI(CurPtr.getType(), 2, "array.cur");
CurPtrPhi->addIncoming(CurPtr.getPointer(), EntryBB);
- CurPtr = Address(CurPtrPhi, ElementAlign);
+ CurPtr = Address(CurPtrPhi, CurPtr.getElementType(), ElementAlign);
// Store the new Cleanup position for irregular Cleanups.
if (EndOfInit.isValid())
@@ -1572,7 +1573,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
llvm::Value *allocSize =
EmitCXXNewAllocSize(*this, E, minElements, numElements,
allocSizeWithoutCookie);
- CharUnits allocAlign = getContext().getPreferredTypeAlignInChars(allocType);
+ CharUnits allocAlign = getContext().getTypeAlignInChars(allocType);
// Emit the allocation call. If the allocator is a global placement
// operator, just "inline" it directly.
@@ -1736,13 +1737,14 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
EmitNewInitializer(*this, E, allocType, elementTy, result, numElements,
allocSizeWithoutCookie);
+ llvm::Value *resultPtr = result.getPointer();
if (E->isArray()) {
// NewPtr is a pointer to the base element type. If we're
// allocating an array of arrays, we'll need to cast back to the
// array pointer type.
llvm::Type *resultType = ConvertTypeForMem(E->getType());
- if (result.getType() != resultType)
- result = Builder.CreateBitCast(result, resultType);
+ if (resultPtr->getType() != resultType)
+ resultPtr = Builder.CreateBitCast(resultPtr, resultType);
}
// Deactivate the 'operator delete' cleanup if we finished
@@ -1752,7 +1754,6 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
cleanupDominator->eraseFromParent();
}
- llvm::Value *resultPtr = result.getPointer();
if (nullCheck) {
conditional.end(*this);
@@ -1796,7 +1797,8 @@ void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,
CharUnits Align = CGM.getNaturalTypeAlignment(DDTag);
DestroyingDeleteTag = CreateTempAlloca(Ty, "destroying.delete.tag");
DestroyingDeleteTag->setAlignment(Align.getAsAlign());
- DeleteArgs.add(RValue::getAggregate(Address(DestroyingDeleteTag, Align)), DDTag);
+ DeleteArgs.add(
+ RValue::getAggregate(Address(DestroyingDeleteTag, Ty, Align)), DDTag);
}
// Pass the size if the delete function has a size_t parameter.
@@ -2101,7 +2103,7 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
Ptr = Address(Builder.CreateInBoundsGEP(Ptr.getElementType(),
Ptr.getPointer(), GEP, "del.first"),
- Ptr.getAlignment());
+ ConvertTypeForMem(DeleteTy), Ptr.getAlignment());
}
assert(ConvertTypeForMem(DeleteTy) == Ptr.getElementType());
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index ac4b4d1308ab..b83a87443250 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -439,22 +439,33 @@ llvm::Constant *ConstantAggregateBuilder::buildFrom(
// Can't emit as an array, carry on to emit as a struct.
}
+ // The size of the constant we plan to generate. This is usually just
+ // the size of the initialized type, but in AllowOversized mode (i.e.
+ // flexible array init), it can be larger.
CharUnits DesiredSize = Utils.getSize(DesiredTy);
+ if (Size > DesiredSize) {
+ assert(AllowOversized && "Elems are oversized");
+ DesiredSize = Size;
+ }
+
+ // The natural alignment of an unpacked LLVM struct with the given elements.
CharUnits Align = CharUnits::One();
for (llvm::Constant *C : Elems)
Align = std::max(Align, Utils.getAlignment(C));
+
+ // The natural size of an unpacked LLVM struct with the given elements.
CharUnits AlignedSize = Size.alignTo(Align);
bool Packed = false;
ArrayRef<llvm::Constant*> UnpackedElems = Elems;
llvm::SmallVector<llvm::Constant*, 32> UnpackedElemStorage;
- if ((DesiredSize < AlignedSize && !AllowOversized) ||
- DesiredSize.alignTo(Align) != DesiredSize) {
- // The natural layout would be the wrong size; force use of a packed layout.
+ if (DesiredSize < AlignedSize || DesiredSize.alignTo(Align) != DesiredSize) {
+ // The natural layout would be too big; force use of a packed layout.
NaturalLayout = false;
Packed = true;
} else if (DesiredSize > AlignedSize) {
- // The constant would be too small. Add padding to fix it.
+ // The natural layout would be too small. Add padding to fix it. (This
+ // is ignored if we choose a packed layout.)
UnpackedElemStorage.assign(Elems.begin(), Elems.end());
UnpackedElemStorage.push_back(Utils.getPadding(DesiredSize - Size));
UnpackedElems = UnpackedElemStorage;
@@ -482,7 +493,7 @@ llvm::Constant *ConstantAggregateBuilder::buildFrom(
// If we're using the packed layout, pad it out to the desired size if
// necessary.
if (Packed) {
- assert((SizeSoFar <= DesiredSize || AllowOversized) &&
+ assert(SizeSoFar <= DesiredSize &&
"requested size is too small for contents");
if (SizeSoFar < DesiredSize)
PackedElems.push_back(Utils.getPadding(DesiredSize - SizeSoFar));
@@ -692,8 +703,8 @@ bool ConstStructBuilder::Build(InitListExpr *ILE, bool AllowOverwrite) {
!declaresSameEntity(ILE->getInitializedFieldInUnion(), Field))
continue;
- // Don't emit anonymous bitfields or zero-sized fields.
- if (Field->isUnnamedBitfield() || Field->isZeroSize(CGM.getContext()))
+ // Don't emit anonymous bitfields.
+ if (Field->isUnnamedBitfield())
continue;
// Get the initializer. A struct can include fields without initializers,
@@ -704,6 +715,14 @@ bool ConstStructBuilder::Build(InitListExpr *ILE, bool AllowOverwrite) {
if (Init && isa<NoInitExpr>(Init))
continue;
+ // Zero-sized fields are not emitted, but their initializers may still
+ // prevent emission of this struct as a constant.
+ if (Field->isZeroSize(CGM.getContext())) {
+ if (Init->HasSideEffects(CGM.getContext()))
+ return false;
+ continue;
+ }
+
// When emitting a DesignatedInitUpdateExpr, a nested InitListExpr
// represents additional overwriting of our current constant value, and not
// a new constant to emit independently.
@@ -1092,7 +1111,16 @@ public:
destAS, destTy);
}
- case CK_LValueToRValue:
+ case CK_LValueToRValue: {
+ // We don't really support doing lvalue-to-rvalue conversions here; any
+ // interesting conversions should be done in Evaluate(). But as a
+ // special case, allow compound literals to support the gcc extension
+ // allowing "struct x {int x;} x = (struct x) {};".
+ if (auto *E = dyn_cast<CompoundLiteralExpr>(subExpr->IgnoreParens()))
+ return Visit(E->getInitializer(), destType);
+ return nullptr;
+ }
+
case CK_AtomicToNonAtomic:
case CK_NonAtomicToAtomic:
case CK_NoOp:
@@ -1909,6 +1937,9 @@ ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) {
if (auto *GD = dyn_cast<MSGuidDecl>(D))
return CGM.GetAddrOfMSGuidDecl(GD);
+ if (auto *GCD = dyn_cast<UnnamedGlobalConstantDecl>(D))
+ return CGM.GetAddrOfUnnamedGlobalConstantDecl(GCD);
+
if (auto *TPO = dyn_cast<TemplateParamObjectDecl>(D))
return CGM.GetAddrOfTemplateParamObject(TPO);
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 4e8933fffe03..b150aaa376eb 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -32,6 +32,7 @@
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/FixedPointBuilder.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
@@ -40,6 +41,7 @@
#include "llvm/IR/IntrinsicsPowerPC.h"
#include "llvm/IR/MatrixBuilder.h"
#include "llvm/IR/Module.h"
+#include "llvm/Support/TypeSize.h"
#include <cstdarg>
using namespace clang;
@@ -65,20 +67,14 @@ bool mayHaveIntegerOverflow(llvm::ConstantInt *LHS, llvm::ConstantInt *RHS,
const auto &LHSAP = LHS->getValue();
const auto &RHSAP = RHS->getValue();
if (Opcode == BO_Add) {
- if (Signed)
- Result = LHSAP.sadd_ov(RHSAP, Overflow);
- else
- Result = LHSAP.uadd_ov(RHSAP, Overflow);
+ Result = Signed ? LHSAP.sadd_ov(RHSAP, Overflow)
+ : LHSAP.uadd_ov(RHSAP, Overflow);
} else if (Opcode == BO_Sub) {
- if (Signed)
- Result = LHSAP.ssub_ov(RHSAP, Overflow);
- else
- Result = LHSAP.usub_ov(RHSAP, Overflow);
+ Result = Signed ? LHSAP.ssub_ov(RHSAP, Overflow)
+ : LHSAP.usub_ov(RHSAP, Overflow);
} else if (Opcode == BO_Mul) {
- if (Signed)
- Result = LHSAP.smul_ov(RHSAP, Overflow);
- else
- Result = LHSAP.umul_ov(RHSAP, Overflow);
+ Result = Signed ? LHSAP.smul_ov(RHSAP, Overflow)
+ : LHSAP.umul_ov(RHSAP, Overflow);
} else if (Opcode == BO_Div || Opcode == BO_Rem) {
if (Signed && !RHS->isZero())
Result = LHSAP.sdiv_ov(RHSAP, Overflow);
@@ -172,7 +168,7 @@ static llvm::Optional<QualType> getUnwidenedIntegerType(const ASTContext &Ctx,
/// Check if \p E is a widened promoted integer.
static bool IsWidenedIntegerOp(const ASTContext &Ctx, const Expr *E) {
- return getUnwidenedIntegerType(Ctx, E).hasValue();
+ return getUnwidenedIntegerType(Ctx, E).has_value();
}
/// Check if we can skip the overflow check for \p Op.
@@ -427,7 +423,8 @@ public:
if (Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(E)) {
if (E->isGLValue())
return CGF.Builder.CreateLoad(Address(
- Result, CGF.getContext().getTypeAlignInChars(E->getType())));
+ Result, CGF.ConvertTypeForMem(E->getType()),
+ CGF.getContext().getTypeAlignInChars(E->getType())));
return Result;
}
return Visit(E->getSubExpr());
@@ -731,7 +728,7 @@ public:
}
if (Ops.Ty->isConstantMatrixType()) {
- llvm::MatrixBuilder<CGBuilderTy> MB(Builder);
+ llvm::MatrixBuilder MB(Builder);
// We need to check the types of the operands of the operator to get the
// correct matrix dimensions.
auto *BO = cast<BinaryOperator>(Ops.E);
@@ -1393,8 +1390,8 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
if (isa<llvm::VectorType>(SrcTy) || isa<llvm::VectorType>(DstTy)) {
// Allow bitcast from vector to integer/fp of the same size.
- unsigned SrcSize = SrcTy->getPrimitiveSizeInBits();
- unsigned DstSize = DstTy->getPrimitiveSizeInBits();
+ llvm::TypeSize SrcSize = SrcTy->getPrimitiveSizeInBits();
+ llvm::TypeSize DstSize = DstTy->getPrimitiveSizeInBits();
if (SrcSize == DstSize)
return Builder.CreateBitCast(Src, DstTy, "conv");
@@ -1606,7 +1603,7 @@ ScalarExprEmitter::VisitSYCLUniqueStableNameExpr(SYCLUniqueStableNameExpr *E) {
Context.getTargetInfo().getConstantAddressSpace();
llvm::Constant *GlobalConstStr = Builder.CreateGlobalStringPtr(
E->ComputeName(Context), "__usn_str",
- static_cast<unsigned>(GlobalAS.getValueOr(LangAS::Default)));
+ static_cast<unsigned>(GlobalAS.value_or(LangAS::Default)));
unsigned ExprAS = Context.getTargetAddressSpace(E->getType());
@@ -1770,7 +1767,8 @@ Value *ScalarExprEmitter::VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
// loads the lvalue formed by the subscript expr. However, we have to be
// careful, because the base of a vector subscript is occasionally an rvalue,
// so we can't get it as an lvalue.
- if (!E->getBase()->getType()->isVectorType())
+ if (!E->getBase()->getType()->isVectorType() &&
+ !E->getBase()->getType()->isVLSTBuiltinType())
return EmitLoadOfLValue(E);
// Handle the vector case. The base must be a vector, the index must be an
@@ -1795,7 +1793,7 @@ Value *ScalarExprEmitter::VisitMatrixSubscriptExpr(MatrixSubscriptExpr *E) {
const auto *MatrixTy = E->getBase()->getType()->castAs<ConstantMatrixType>();
unsigned NumRows = MatrixTy->getNumRows();
- llvm::MatrixBuilder<CGBuilderTy> MB(Builder);
+ llvm::MatrixBuilder MB(Builder);
Value *Idx = MB.CreateIndex(RowIdx, ColumnIdx, NumRows);
if (CGF.CGM.getCodeGenOpts().OptimizationLevel > 0)
MB.CreateIndexAssumption(Idx, MatrixTy->getNumElementsFlattened());
@@ -2045,11 +2043,16 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
}
if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast)) {
- if (auto PT = DestTy->getAs<PointerType>())
- CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Src,
- /*MayBeNull=*/true,
- CodeGenFunction::CFITCK_UnrelatedCast,
- CE->getBeginLoc());
+ if (auto *PT = DestTy->getAs<PointerType>()) {
+ CGF.EmitVTablePtrCheckForCast(
+ PT->getPointeeType(),
+ Address(Src,
+ CGF.ConvertTypeForMem(
+ E->getType()->castAs<PointerType>()->getPointeeType()),
+ CGF.getPointerAlign()),
+ /*MayBeNull=*/true, CodeGenFunction::CFITCK_UnrelatedCast,
+ CE->getBeginLoc());
+ }
}
if (CGF.CGM.getCodeGenOpts().StrictVTablePointers) {
@@ -2081,8 +2084,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
}
// If Src is a fixed vector and Dst is a scalable vector, and both have the
- // same element type, use the llvm.experimental.vector.insert intrinsic to
- // perform the bitcast.
+ // same element type, use the llvm.vector.insert intrinsic to perform the
+ // bitcast.
if (const auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) {
if (const auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(DstTy)) {
// If we are casting a fixed i8 vector to a scalable 16 x i1 predicate
@@ -2093,7 +2096,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
if (ScalableDst == PredType &&
FixedSrc->getElementType() == Builder.getInt8Ty()) {
DstTy = llvm::ScalableVectorType::get(Builder.getInt8Ty(), 2);
- ScalableDst = dyn_cast<llvm::ScalableVectorType>(DstTy);
+ ScalableDst = cast<llvm::ScalableVectorType>(DstTy);
NeedsBitCast = true;
}
if (FixedSrc->getElementType() == ScalableDst->getElementType()) {
@@ -2109,8 +2112,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
}
// If Src is a scalable vector and Dst is a fixed vector, and both have the
- // same element type, use the llvm.experimental.vector.extract intrinsic to
- // perform the bitcast.
+ // same element type, use the llvm.vector.extract intrinsic to perform the
+ // bitcast.
if (const auto *ScalableSrc = dyn_cast<llvm::ScalableVectorType>(SrcTy)) {
if (const auto *FixedDst = dyn_cast<llvm::FixedVectorType>(DstTy)) {
// If we are casting a scalable 16 x i1 predicate vector to a fixed i8
@@ -2119,7 +2122,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
if (ScalableSrc == PredType &&
FixedDst->getElementType() == Builder.getInt8Ty()) {
SrcTy = llvm::ScalableVectorType::get(Builder.getInt8Ty(), 2);
- ScalableSrc = dyn_cast<llvm::ScalableVectorType>(SrcTy);
+ ScalableSrc = cast<llvm::ScalableVectorType>(SrcTy);
Src = Builder.CreateBitCast(Src, SrcTy);
}
if (ScalableSrc->getElementType() == FixedDst->getElementType()) {
@@ -2148,7 +2151,6 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
DestLV.setTBAAInfo(TBAAAccessInfo::getMayAliasInfo());
return EmitLoadOfLValue(DestLV, CE->getExprLoc());
}
-
return Builder.CreateBitCast(Src, DstTy);
}
case CK_AddressSpaceConversion: {
@@ -2204,10 +2206,10 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
Derived.getPointer(), DestTy->getPointeeType());
if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast))
- CGF.EmitVTablePtrCheckForCast(
- DestTy->getPointeeType(), Derived.getPointer(),
- /*MayBeNull=*/true, CodeGenFunction::CFITCK_DerivedCast,
- CE->getBeginLoc());
+ CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived,
+ /*MayBeNull=*/true,
+ CodeGenFunction::CFITCK_DerivedCast,
+ CE->getBeginLoc());
return Derived.getPointer();
}
@@ -2331,9 +2333,10 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
}
case CK_VectorSplat: {
llvm::Type *DstTy = ConvertType(DestTy);
- Value *Elt = Visit(const_cast<Expr*>(E));
+ Value *Elt = Visit(const_cast<Expr *>(E));
// Splat the element across to all elements
- unsigned NumElements = cast<llvm::FixedVectorType>(DstTy)->getNumElements();
+ llvm::ElementCount NumElements =
+ cast<llvm::VectorType>(DstTy)->getElementCount();
return Builder.CreateVectorSplat(NumElements, Elt, "splat");
}
@@ -2643,7 +2646,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
= CGF.getContext().getAsVariableArrayType(type)) {
llvm::Value *numElts = CGF.getVLASize(vla).NumElts;
if (!isInc) numElts = Builder.CreateNSWNeg(numElts, "vla.negsize");
- llvm::Type *elemTy = value->getType()->getPointerElementType();
+ llvm::Type *elemTy = CGF.ConvertTypeForMem(vla->getElementType());
if (CGF.getLangOpts().isSignedOverflowDefined())
value = Builder.CreateGEP(elemTy, value, numElts, "vla.inc");
else
@@ -2949,8 +2952,8 @@ Value *ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *E) {
CurrentType = ON.getBase()->getType();
// Compute the offset to the base.
- const RecordType *BaseRT = CurrentType->getAs<RecordType>();
- CXXRecordDecl *BaseRD = cast<CXXRecordDecl>(BaseRT->getDecl());
+ auto *BaseRT = CurrentType->castAs<RecordType>();
+ auto *BaseRD = cast<CXXRecordDecl>(BaseRT->getDecl());
CharUnits OffsetInt = RL.getBaseClassOffset(BaseRD);
Offset = llvm::ConstantInt::get(ResultType, OffsetInt.getQuantity());
break;
@@ -3263,7 +3266,7 @@ Value *ScalarExprEmitter::EmitDiv(const BinOpInfo &Ops) {
}
if (Ops.Ty->isConstantMatrixType()) {
- llvm::MatrixBuilder<CGBuilderTy> MB(Builder);
+ llvm::MatrixBuilder MB(Builder);
// We need to check the types of the operands of the operator to get the
// correct matrix dimensions.
auto *BO = cast<BinaryOperator>(Ops.E);
@@ -3521,7 +3524,7 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF,
// GEP indexes are signed, and scaling an index isn't permitted to
// signed-overflow, so we use the same semantics for our explicit
// multiply. We suppress this if overflow is not undefined behavior.
- llvm::Type *elemTy = pointer->getType()->getPointerElementType();
+ llvm::Type *elemTy = CGF.ConvertTypeForMem(vla->getElementType());
if (CGF.getLangOpts().isSignedOverflowDefined()) {
index = CGF.Builder.CreateMul(index, numElements, "vla.index");
pointer = CGF.Builder.CreateGEP(elemTy, pointer, index, "add.ptr");
@@ -3657,7 +3660,7 @@ Value *ScalarExprEmitter::EmitAdd(const BinOpInfo &op) {
}
if (op.Ty->isConstantMatrixType()) {
- llvm::MatrixBuilder<CGBuilderTy> MB(Builder);
+ llvm::MatrixBuilder MB(Builder);
CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, op.FPFeatures);
return MB.CreateAdd(op.LHS, op.RHS);
}
@@ -3807,7 +3810,7 @@ Value *ScalarExprEmitter::EmitSub(const BinOpInfo &op) {
}
if (op.Ty->isConstantMatrixType()) {
- llvm::MatrixBuilder<CGBuilderTy> MB(Builder);
+ llvm::MatrixBuilder MB(Builder);
CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, op.FPFeatures);
return MB.CreateSub(op.LHS, op.RHS);
}
@@ -4639,7 +4642,8 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
return tmp5;
}
- if (condExpr->getType()->isVectorType()) {
+ if (condExpr->getType()->isVectorType() ||
+ condExpr->getType()->isVLSTBuiltinType()) {
CGF.incrementProfileCounter(E);
llvm::Value *CondV = CGF.EmitScalarExpr(condExpr);
@@ -4819,6 +4823,10 @@ Value *ScalarExprEmitter::VisitAsTypeExpr(AsTypeExpr *E) {
? cast<llvm::FixedVectorType>(DstTy)->getNumElements()
: 0;
+ // Use bit vector expansion for ext_vector_type boolean vectors.
+ if (E->getType()->isExtVectorBoolType())
+ return CGF.emitBoolVecConversion(Src, NumElementsDst, "astype");
+
// Going from vec3 to non-vec3 is a special case and requires a shuffle
// vector to get a vec4, then a bitcast if the target type is different.
if (NumElementsSrc == 3 && NumElementsDst != 3) {
@@ -4902,7 +4910,9 @@ LValue CodeGenFunction::EmitObjCIsaExpr(const ObjCIsaExpr *E) {
Expr *BaseExpr = E->getBase();
Address Addr = Address::invalid();
if (BaseExpr->isPRValue()) {
- Addr = Address(EmitScalarExpr(BaseExpr), getPointerAlign());
+ llvm::Type *BaseTy =
+ ConvertTypeForMem(BaseExpr->getType()->getPointeeType());
+ Addr = Address(EmitScalarExpr(BaseExpr), BaseTy, getPointerAlign());
} else {
Addr = EmitLValue(BaseExpr).getAddress(*this);
}
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
new file mode 100644
index 000000000000..7dfcc65969a8
--- /dev/null
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -0,0 +1,52 @@
+//===----- CGHLSLRuntime.cpp - Interface to HLSL Runtimes -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides an abstract class for HLSL code generation. Concrete
+// subclasses of this implement code generation for specific HLSL
+// runtime libraries.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CGHLSLRuntime.h"
+#include "CodeGenModule.h"
+#include "clang/Basic/TargetOptions.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
+
+using namespace clang;
+using namespace CodeGen;
+using namespace llvm;
+
+namespace {
+void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {
+ // The validation of ValVersionStr is done at HLSLToolChain::TranslateArgs.
+ // Assume ValVersionStr is legal here.
+ VersionTuple Version;
+ if (Version.tryParse(ValVersionStr) || Version.getBuild() ||
+ Version.getSubminor() || !Version.getMinor()) {
+ return;
+ }
+
+ uint64_t Major = Version.getMajor();
+ uint64_t Minor = *Version.getMinor();
+
+ auto &Ctx = M.getContext();
+ IRBuilder<> B(M.getContext());
+ MDNode *Val = MDNode::get(Ctx, {ConstantAsMetadata::get(B.getInt32(Major)),
+ ConstantAsMetadata::get(B.getInt32(Minor))});
+ StringRef DxilValKey = "dx.valver";
+ M.addModuleFlag(llvm::Module::ModFlagBehavior::AppendUnique, DxilValKey, Val);
+}
+} // namespace
+
+void CGHLSLRuntime::finishCodeGen() {
+ auto &TargetOpts = CGM.getTarget().getTargetOpts();
+
+ llvm::Module &M = CGM.getModule();
+ addDxilValVersion(TargetOpts.DxilValidatorVersion, M);
+}
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
new file mode 100644
index 000000000000..268810f2ec9e
--- /dev/null
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -0,0 +1,38 @@
+//===----- CGHLSLRuntime.h - Interface to HLSL Runtimes -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides an abstract class for HLSL code generation. Concrete
+// subclasses of this implement code generation for specific HLSL
+// runtime libraries.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_CODEGEN_CGHLSLRUNTIME_H
+#define LLVM_CLANG_LIB_CODEGEN_CGHLSLRUNTIME_H
+
+namespace clang {
+
+namespace CodeGen {
+
+class CodeGenModule;
+
+class CGHLSLRuntime {
+protected:
+ CodeGenModule &CGM;
+
+public:
+ CGHLSLRuntime(CodeGenModule &CGM) : CGM(CGM) {}
+ virtual ~CGHLSLRuntime() {}
+
+ void finishCodeGen();
+};
+
+} // namespace CodeGen
+} // namespace clang
+
+#endif
diff --git a/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
index e3b0e069b830..0abf39ad1f28 100644
--- a/clang/lib/CodeGen/CGNonTrivialStruct.cpp
+++ b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
@@ -315,8 +315,7 @@ static const CGFunctionInfo &getFunctionInfo(CodeGenModule &CGM,
Ctx, nullptr, SourceLocation(), &Ctx.Idents.get(ValNameStr[I]), ParamTy,
ImplicitParamDecl::Other));
- for (auto &P : Params)
- Args.push_back(P);
+ llvm::append_range(Args, Params);
return CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
}
@@ -326,9 +325,9 @@ static std::array<Address, N> getParamAddrs(std::index_sequence<Ints...> IntSeq,
std::array<CharUnits, N> Alignments,
FunctionArgList Args,
CodeGenFunction *CGF) {
- return std::array<Address, N>{{
- Address(CGF->Builder.CreateLoad(CGF->GetAddrOfLocalVar(Args[Ints])),
- Alignments[Ints])...}};
+ return std::array<Address, N>{
+ {Address(CGF->Builder.CreateLoad(CGF->GetAddrOfLocalVar(Args[Ints])),
+ CGF->VoidPtrTy, Alignments[Ints])...}};
}
// Template classes that are used as bases for classes that emit special
@@ -400,8 +399,9 @@ template <class Derived> struct GenFuncBase {
std::array<Address, N> NewAddrs = Addrs;
for (unsigned I = 0; I < N; ++I)
- NewAddrs[I] = Address(
- PHIs[I], StartAddrs[I].getAlignment().alignmentAtOffset(EltSize));
+ NewAddrs[I] =
+ Address(PHIs[I], CGF.Int8PtrTy,
+ StartAddrs[I].getAlignment().alignmentAtOffset(EltSize));
EltQT = IsVolatile ? EltQT.withVolatile() : EltQT;
this->asDerived().visitWithKind(FK, EltQT, nullptr, CharUnits::Zero(),
diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp
index 8cc609186f9e..1b6acb2b7212 100644
--- a/clang/lib/CodeGen/CGObjC.cpp
+++ b/clang/lib/CodeGen/CGObjC.cpp
@@ -92,8 +92,9 @@ CodeGenFunction::EmitObjCBoxedExpr(const ObjCBoxedExpr *E) {
// and cast value to correct type
Address Temporary = CreateMemTemp(SubExpr->getType());
EmitAnyExprToMem(SubExpr, Temporary, Qualifiers(), /*isInit*/ true);
- Address BitCast = Builder.CreateBitCast(Temporary, ConvertType(ArgQT));
- Args.add(RValue::get(BitCast.getPointer()), ArgQT);
+ llvm::Value *BitCast =
+ Builder.CreateBitCast(Temporary.getPointer(), ConvertType(ArgQT));
+ Args.add(RValue::get(BitCast), ArgQT);
// Create char array to store type encoding
std::string Str;
@@ -441,7 +442,7 @@ CodeGen::RValue CGObjCRuntime::GeneratePossiblySpecializedMessageSend(
if (Optional<llvm::Value *> SpecializedResult =
tryGenerateSpecializedMessageSend(CGF, ResultType, Receiver, Args,
Sel, Method, isClassMessage)) {
- return RValue::get(SpecializedResult.getValue());
+ return RValue::get(*SpecializedResult);
}
return GenerateMessageSend(CGF, Return, ResultType, Sel, Receiver, Args, OID,
Method);
@@ -816,19 +817,20 @@ static void emitStructGetterCall(CodeGenFunction &CGF, ObjCIvarDecl *ivar,
bool isAtomic, bool hasStrong) {
ASTContext &Context = CGF.getContext();
- Address src =
+ llvm::Value *src =
CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(), CGF.LoadObjCSelf(), ivar, 0)
- .getAddress(CGF);
+ .getPointer(CGF);
// objc_copyStruct (ReturnValue, &structIvar,
// sizeof (Type of Ivar), isAtomic, false);
CallArgList args;
- Address dest = CGF.Builder.CreateBitCast(CGF.ReturnValue, CGF.VoidPtrTy);
- args.add(RValue::get(dest.getPointer()), Context.VoidPtrTy);
+ llvm::Value *dest =
+ CGF.Builder.CreateBitCast(CGF.ReturnValue.getPointer(), CGF.VoidPtrTy);
+ args.add(RValue::get(dest), Context.VoidPtrTy);
src = CGF.Builder.CreateBitCast(src, CGF.VoidPtrTy);
- args.add(RValue::get(src.getPointer()), Context.VoidPtrTy);
+ args.add(RValue::get(src), Context.VoidPtrTy);
CharUnits size = CGF.getContext().getTypeSizeInChars(ivar->getType());
args.add(RValue::get(CGF.CGM.getSize(size)), Context.getSizeType());
@@ -1149,11 +1151,10 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
// types, so there's no point in trying to pick a prettier type.
uint64_t ivarSize = getContext().toBits(strategy.getIvarSize());
llvm::Type *bitcastType = llvm::Type::getIntNTy(getLLVMContext(), ivarSize);
- bitcastType = bitcastType->getPointerTo(); // addrspace 0 okay
// Perform an atomic load. This does not impose ordering constraints.
Address ivarAddr = LV.getAddress(*this);
- ivarAddr = Builder.CreateBitCast(ivarAddr, bitcastType);
+ ivarAddr = Builder.CreateElementBitCast(ivarAddr, bitcastType);
llvm::LoadInst *load = Builder.CreateLoad(ivarAddr, "load");
load->setAtomic(llvm::AtomicOrdering::Unordered);
@@ -1164,12 +1165,11 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
uint64_t retTySize = CGM.getDataLayout().getTypeSizeInBits(retTy);
llvm::Value *ivarVal = load;
if (ivarSize > retTySize) {
- llvm::Type *newTy = llvm::Type::getIntNTy(getLLVMContext(), retTySize);
- ivarVal = Builder.CreateTrunc(load, newTy);
- bitcastType = newTy->getPointerTo();
+ bitcastType = llvm::Type::getIntNTy(getLLVMContext(), retTySize);
+ ivarVal = Builder.CreateTrunc(load, bitcastType);
}
Builder.CreateStore(ivarVal,
- Builder.CreateBitCast(ReturnValue, bitcastType));
+ Builder.CreateElementBitCast(ReturnValue, bitcastType));
// Make sure we don't do an autorelease.
AutoreleaseResult = false;
@@ -1911,8 +1911,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
// Fetch the value at the current index from the buffer.
llvm::Value *CurrentItemPtr = Builder.CreateGEP(
- EnumStateItems->getType()->getPointerElementType(), EnumStateItems, index,
- "currentitem.ptr");
+ ObjCIdType, EnumStateItems, index, "currentitem.ptr");
llvm::Value *CurrentItem =
Builder.CreateAlignedLoad(ObjCIdType, CurrentItemPtr, getPointerAlign());
@@ -2156,7 +2155,7 @@ static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF, Address addr,
// Cast the argument to 'id*'.
llvm::Type *origType = addr.getElementType();
- addr = CGF.Builder.CreateBitCast(addr, CGF.Int8PtrPtrTy);
+ addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8PtrTy);
// Call the function.
llvm::Value *result = CGF.EmitNounwindRuntimeCall(fn, addr.getPointer());
@@ -2610,7 +2609,7 @@ void CodeGenFunction::EmitARCDestroyWeak(Address addr) {
fn = getARCIntrinsic(llvm::Intrinsic::objc_destroyWeak, CGM);
// Cast the argument to 'id*'.
- addr = Builder.CreateBitCast(addr, Int8PtrPtrTy);
+ addr = Builder.CreateElementBitCast(addr, Int8PtrTy);
EmitNounwindRuntimeCall(fn, addr.getPointer());
}
@@ -3845,15 +3844,14 @@ CodeGenFunction::GenerateObjCAtomicGetterCopyHelperFunction(
SourceLocation());
RValue DV = EmitAnyExpr(&DstExpr);
- CharUnits Alignment
- = getContext().getTypeAlignInChars(TheCXXConstructExpr->getType());
+ CharUnits Alignment =
+ getContext().getTypeAlignInChars(TheCXXConstructExpr->getType());
EmitAggExpr(TheCXXConstructExpr,
- AggValueSlot::forAddr(Address(DV.getScalarVal(), Alignment),
- Qualifiers(),
- AggValueSlot::IsDestructed,
- AggValueSlot::DoesNotNeedGCBarriers,
- AggValueSlot::IsNotAliased,
- AggValueSlot::DoesNotOverlap));
+ AggValueSlot::forAddr(
+ Address(DV.getScalarVal(), ConvertTypeForMem(Ty), Alignment),
+ Qualifiers(), AggValueSlot::IsDestructed,
+ AggValueSlot::DoesNotNeedGCBarriers,
+ AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap));
FinishFunction();
HelperFn = llvm::ConstantExpr::getBitCast(Fn, VoidPtrTy);
@@ -3897,6 +3895,8 @@ static unsigned getBaseMachOPlatformID(const llvm::Triple &TT) {
return llvm::MachO::PLATFORM_TVOS;
case llvm::Triple::WatchOS:
return llvm::MachO::PLATFORM_WATCHOS;
+ case llvm::Triple::DriverKit:
+ return llvm::MachO::PLATFORM_DRIVERKIT;
default:
return /*Unknown platform*/ 0;
}
@@ -3915,8 +3915,8 @@ static llvm::Value *emitIsPlatformVersionAtLeast(CodeGenFunction &CGF,
Args.push_back(
llvm::ConstantInt::get(CGM.Int32Ty, getBaseMachOPlatformID(TT)));
Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, Version.getMajor()));
- Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, Min.getValueOr(0)));
- Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, SMin.getValueOr(0)));
+ Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, Min.value_or(0)));
+ Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, SMin.value_or(0)));
};
assert(!Version.empty() && "unexpected empty version");
@@ -3952,9 +3952,8 @@ CodeGenFunction::EmitBuiltinAvailable(const VersionTuple &Version) {
Optional<unsigned> Min = Version.getMinor(), SMin = Version.getSubminor();
llvm::Value *Args[] = {
llvm::ConstantInt::get(CGM.Int32Ty, Version.getMajor()),
- llvm::ConstantInt::get(CGM.Int32Ty, Min.getValueOr(0)),
- llvm::ConstantInt::get(CGM.Int32Ty, SMin.getValueOr(0))
- };
+ llvm::ConstantInt::get(CGM.Int32Ty, Min.value_or(0)),
+ llvm::ConstantInt::get(CGM.Int32Ty, SMin.value_or(0))};
llvm::Value *CallRes =
EmitNounwindRuntimeCall(CGM.IsOSVersionAtLeastFn, Args);
@@ -3977,6 +3976,9 @@ static bool isFoundationNeededForDarwinAvailabilityCheck(
case llvm::Triple::MacOSX:
FoundationDroppedInVersion = VersionTuple(/*Major=*/10, /*Minor=*/15);
break;
+ case llvm::Triple::DriverKit:
+ // DriverKit doesn't need Foundation.
+ return false;
default:
llvm_unreachable("Unexpected OS");
}
diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp
index 52b449090868..ec459f07f307 100644
--- a/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -107,6 +107,8 @@ protected:
/// SEL is included in a header somewhere, in which case it will be whatever
/// type is declared in that header, most likely {i8*, i8*}.
llvm::PointerType *SelectorTy;
+ /// Element type of SelectorTy.
+ llvm::Type *SelectorElemTy;
/// LLVM i8 type. Cached here to avoid repeatedly getting it in all of the
/// places where it's used
llvm::IntegerType *Int8Ty;
@@ -128,6 +130,8 @@ protected:
/// but if the runtime header declaring it is included then it may be a
/// pointer to a structure.
llvm::PointerType *IdTy;
+ /// Element type of IdTy.
+ llvm::Type *IdElemTy;
/// Pointer to a pointer to an Objective-C object. Used in the new ABI
/// message lookup function and some GC-related functions.
llvm::PointerType *PtrToIdTy;
@@ -313,12 +317,9 @@ protected:
/// Ensures that the value has the required type, by inserting a bitcast if
/// required. This function lets us avoid inserting bitcasts that are
/// redundant.
- llvm::Value* EnforceType(CGBuilderTy &B, llvm::Value *V, llvm::Type *Ty) {
- if (V->getType() == Ty) return V;
- return B.CreateBitCast(V, Ty);
- }
- Address EnforceType(CGBuilderTy &B, Address V, llvm::Type *Ty) {
- if (V.getType() == Ty) return V;
+ llvm::Value *EnforceType(CGBuilderTy &B, llvm::Value *V, llvm::Type *Ty) {
+ if (V->getType() == Ty)
+ return V;
return B.CreateBitCast(V, Ty);
}
@@ -700,8 +701,8 @@ protected:
llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, Address ObjCSuper,
llvm::Value *cmd, MessageSendInfo &MSI) override {
CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *lookupArgs[] = {EnforceType(Builder, ObjCSuper,
- PtrToObjCSuperTy).getPointer(), cmd};
+ llvm::Value *lookupArgs[] = {
+ EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd};
return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs);
}
@@ -978,9 +979,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
// Look for an existing one
llvm::StringMap<llvm::Constant*>::iterator old = ObjCStrings.find(Str);
if (old != ObjCStrings.end())
- return ConstantAddress(
- old->getValue(), old->getValue()->getType()->getPointerElementType(),
- Align);
+ return ConstantAddress(old->getValue(), IdElemTy, Align);
bool isNonASCII = SL->containsNonAscii();
@@ -1002,7 +1001,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
auto *ObjCStr = llvm::ConstantExpr::getIntToPtr(
llvm::ConstantInt::get(Int64Ty, str), IdTy);
ObjCStrings[Str] = ObjCStr;
- return ConstantAddress(ObjCStr, IdTy->getPointerElementType(), Align);
+ return ConstantAddress(ObjCStr, IdElemTy, Align);
}
StringRef StringClass = CGM.getLangOpts().ObjCConstantStringClass;
@@ -1116,7 +1115,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
llvm::Constant *ObjCStr = llvm::ConstantExpr::getBitCast(ObjCStrGV, IdTy);
ObjCStrings[Str] = ObjCStr;
ConstantStrings.push_back(ObjCStr);
- return ConstantAddress(ObjCStr, IdTy->getPointerElementType(), Align);
+ return ConstantAddress(ObjCStr, IdElemTy, Align);
}
void PushProperty(ConstantArrayBuilder &PropertiesArray,
@@ -1208,8 +1207,10 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
llvm::Value *cmd, MessageSendInfo &MSI) override {
// Don't access the slot unless we're trying to cache the result.
CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *lookupArgs[] = {CGObjCGNU::EnforceType(Builder, ObjCSuper,
- PtrToObjCSuperTy).getPointer(), cmd};
+ llvm::Value *lookupArgs[] = {CGObjCGNU::EnforceType(Builder,
+ ObjCSuper.getPointer(),
+ PtrToObjCSuperTy),
+ cmd};
return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs);
}
@@ -1263,8 +1264,8 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
llvm::Value *GetClassNamed(CodeGenFunction &CGF,
const std::string &Name,
bool isWeak) override {
- return CGF.Builder.CreateLoad(Address(GetClassVar(Name, isWeak),
- CGM.getPointerAlign()));
+ return CGF.Builder.CreateLoad(
+ Address(GetClassVar(Name, isWeak), IdTy, CGM.getPointerAlign()));
}
int32_t FlagsForOwnership(Qualifiers::ObjCLifetime Ownership) {
// typedef enum {
@@ -2170,8 +2171,10 @@ CGObjCGNU::CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion,
QualType selTy = CGM.getContext().getObjCSelType();
if (QualType() == selTy) {
SelectorTy = PtrToInt8Ty;
+ SelectorElemTy = Int8Ty;
} else {
SelectorTy = cast<llvm::PointerType>(CGM.getTypes().ConvertType(selTy));
+ SelectorElemTy = CGM.getTypes().ConvertTypeForMem(selTy->getPointeeType());
}
PtrToIntTy = llvm::PointerType::getUnqual(IntTy);
@@ -2189,8 +2192,11 @@ CGObjCGNU::CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion,
if (UnqualIdTy != QualType()) {
ASTIdTy = CGM.getContext().getCanonicalType(UnqualIdTy);
IdTy = cast<llvm::PointerType>(CGM.getTypes().ConvertType(ASTIdTy));
+ IdElemTy = CGM.getTypes().ConvertTypeForMem(
+ ASTIdTy.getTypePtr()->getPointeeType());
} else {
IdTy = PtrToInt8Ty;
+ IdElemTy = Int8Ty;
}
PtrToIdTy = llvm::PointerType::getUnqual(IdTy);
ProtocolTy = llvm::StructType::get(IdTy,
@@ -2347,7 +2353,7 @@ llvm::Value *CGObjCGNU::GetTypedSelector(CodeGenFunction &CGF, Selector Sel,
}
}
if (!SelValue) {
- SelValue = llvm::GlobalAlias::create(SelectorTy->getPointerElementType(), 0,
+ SelValue = llvm::GlobalAlias::create(SelectorElemTy, 0,
llvm::GlobalValue::PrivateLinkage,
".objc_selector_" + Sel.getAsString(),
&TheModule);
@@ -2577,16 +2583,14 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF,
if (IsClassMessage) {
if (!MetaClassPtrAlias) {
MetaClassPtrAlias = llvm::GlobalAlias::create(
- IdTy->getPointerElementType(), 0,
- llvm::GlobalValue::InternalLinkage,
+ IdElemTy, 0, llvm::GlobalValue::InternalLinkage,
".objc_metaclass_ref" + Class->getNameAsString(), &TheModule);
}
ReceiverClass = MetaClassPtrAlias;
} else {
if (!ClassPtrAlias) {
ClassPtrAlias = llvm::GlobalAlias::create(
- IdTy->getPointerElementType(), 0,
- llvm::GlobalValue::InternalLinkage,
+ IdElemTy, 0, llvm::GlobalValue::InternalLinkage,
".objc_class_ref" + Class->getNameAsString(), &TheModule);
}
ReceiverClass = ClassPtrAlias;
@@ -2612,8 +2616,6 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF,
Builder.CreateStore(Receiver, Builder.CreateStructGEP(ObjCSuper, 0));
Builder.CreateStore(ReceiverClass, Builder.CreateStructGEP(ObjCSuper, 1));
- ObjCSuper = EnforceType(Builder, ObjCSuper, PtrToObjCSuperTy);
-
// Get the IMP
llvm::Value *imp = LookupIMPSuper(CGF, ObjCSuper, cmd, MSI);
imp = EnforceType(Builder, imp, MSI.MessengerType);
@@ -3708,8 +3710,7 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
// Add all referenced protocols to a category.
GenerateProtocolHolderCategory();
- llvm::StructType *selStructTy =
- dyn_cast<llvm::StructType>(SelectorTy->getPointerElementType());
+ llvm::StructType *selStructTy = dyn_cast<llvm::StructType>(SelectorElemTy);
llvm::Type *selStructPtrTy = SelectorTy;
if (!selStructTy) {
selStructTy = llvm::StructType::get(CGM.getLLVMContext(),
@@ -3861,9 +3862,10 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
// The path to the source file where this module was declared
SourceManager &SM = CGM.getContext().getSourceManager();
- const FileEntry *mainFile = SM.getFileEntryForID(SM.getMainFileID());
+ Optional<FileEntryRef> mainFile =
+ SM.getFileEntryRefForID(SM.getMainFileID());
std::string path =
- (Twine(mainFile->getDir()->getName()) + "/" + mainFile->getName()).str();
+ (mainFile->getDir().getName() + "/" + mainFile->getName()).str();
module.add(MakeConstantString(path, ".objc_source_file_name"));
module.add(symtab);
@@ -4064,16 +4066,16 @@ void CGObjCGNU::EmitThrowStmt(CodeGenFunction &CGF,
llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGenFunction &CGF,
Address AddrWeakObj) {
CGBuilderTy &B = CGF.Builder;
- AddrWeakObj = EnforceType(B, AddrWeakObj, PtrToIdTy);
- return B.CreateCall(WeakReadFn, AddrWeakObj.getPointer());
+ return B.CreateCall(WeakReadFn,
+ EnforceType(B, AddrWeakObj.getPointer(), PtrToIdTy));
}
void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF,
llvm::Value *src, Address dst) {
CGBuilderTy &B = CGF.Builder;
src = EnforceType(B, src, IdTy);
- dst = EnforceType(B, dst, PtrToIdTy);
- B.CreateCall(WeakAssignFn, {src, dst.getPointer()});
+ llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy);
+ B.CreateCall(WeakAssignFn, {src, dstVal});
}
void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF,
@@ -4081,10 +4083,10 @@ void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF,
bool threadlocal) {
CGBuilderTy &B = CGF.Builder;
src = EnforceType(B, src, IdTy);
- dst = EnforceType(B, dst, PtrToIdTy);
+ llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy);
// FIXME. Add threadloca assign API
assert(!threadlocal && "EmitObjCGlobalAssign - Threal Local API NYI");
- B.CreateCall(GlobalAssignFn, {src, dst.getPointer()});
+ B.CreateCall(GlobalAssignFn, {src, dstVal});
}
void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF,
@@ -4092,16 +4094,16 @@ void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF,
llvm::Value *ivarOffset) {
CGBuilderTy &B = CGF.Builder;
src = EnforceType(B, src, IdTy);
- dst = EnforceType(B, dst, IdTy);
- B.CreateCall(IvarAssignFn, {src, dst.getPointer(), ivarOffset});
+ llvm::Value *dstVal = EnforceType(B, dst.getPointer(), IdTy);
+ B.CreateCall(IvarAssignFn, {src, dstVal, ivarOffset});
}
void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF,
llvm::Value *src, Address dst) {
CGBuilderTy &B = CGF.Builder;
src = EnforceType(B, src, IdTy);
- dst = EnforceType(B, dst, PtrToIdTy);
- B.CreateCall(StrongCastAssignFn, {src, dst.getPointer()});
+ llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy);
+ B.CreateCall(StrongCastAssignFn, {src, dstVal});
}
void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF,
@@ -4109,10 +4111,10 @@ void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF,
Address SrcPtr,
llvm::Value *Size) {
CGBuilderTy &B = CGF.Builder;
- DestPtr = EnforceType(B, DestPtr, PtrTy);
- SrcPtr = EnforceType(B, SrcPtr, PtrTy);
+ llvm::Value *DestPtrVal = EnforceType(B, DestPtr.getPointer(), PtrTy);
+ llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.getPointer(), PtrTy);
- B.CreateCall(MemMoveFn, {DestPtr.getPointer(), SrcPtr.getPointer(), Size});
+ B.CreateCall(MemMoveFn, {DestPtrVal, SrcPtrVal, Size});
}
llvm::GlobalVariable *CGObjCGNU::ObjCIvarOffsetVariable(
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index e7dba4c8feab..46e65eb1ed43 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -4915,11 +4915,11 @@ void CGObjCMac::EmitThrowStmt(CodeGen::CodeGenFunction &CGF,
llvm::Value * CGObjCMac::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF,
Address AddrWeakObj) {
llvm::Type* DestTy = AddrWeakObj.getElementType();
- AddrWeakObj = CGF.Builder.CreateBitCast(AddrWeakObj,
- ObjCTypes.PtrObjectPtrTy);
+ llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast(
+ AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy);
llvm::Value *read_weak =
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(),
- AddrWeakObj.getPointer(), "weakread");
+ AddrWeakObjVal, "weakread");
read_weak = CGF.Builder.CreateBitCast(read_weak, DestTy);
return read_weak;
}
@@ -4938,8 +4938,9 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
- dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst.getPointer() };
+ llvm::Value *dstVal =
+ CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+ llvm::Value *args[] = { src, dstVal };
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(),
args, "weakassign");
}
@@ -4959,8 +4960,9 @@ void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
- dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst.getPointer() };
+ llvm::Value *dstVal =
+ CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+ llvm::Value *args[] = {src, dstVal};
if (!threadlocal)
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(),
args, "globalassign");
@@ -4985,8 +4987,9 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
- dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst.getPointer(), ivarOffset };
+ llvm::Value *dstVal =
+ CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+ llvm::Value *args[] = {src, dstVal, ivarOffset};
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args);
}
@@ -5004,8 +5007,9 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF,
src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
- dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst.getPointer() };
+ llvm::Value *dstVal =
+ CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+ llvm::Value *args[] = {src, dstVal};
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(),
args, "strongassign");
}
@@ -5014,8 +5018,8 @@ void CGObjCMac::EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF,
Address DestPtr,
Address SrcPtr,
llvm::Value *size) {
- SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, ObjCTypes.Int8PtrTy);
- DestPtr = CGF.Builder.CreateBitCast(DestPtr, ObjCTypes.Int8PtrTy);
+ SrcPtr = CGF.Builder.CreateElementBitCast(SrcPtr, CGF.Int8Ty);
+ DestPtr = CGF.Builder.CreateElementBitCast(DestPtr, CGF.Int8Ty);
llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), size };
CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args);
}
@@ -5064,7 +5068,9 @@ std::string CGObjCCommonMac::GetSectionName(StringRef Section,
return ("." + Section.substr(2) + "$B").str();
case llvm::Triple::Wasm:
case llvm::Triple::GOFF:
+ case llvm::Triple::SPIRV:
case llvm::Triple::XCOFF:
+ case llvm::Triple::DXContainer:
llvm::report_fatal_error(
"Objective-C support is unimplemented for object file format");
}
@@ -5268,7 +5274,7 @@ Address CGObjCMac::EmitSelectorAddr(Selector Sel) {
Entry->setExternallyInitialized(true);
}
- return Address(Entry, Align);
+ return Address(Entry, ObjCTypes.SelectorPtrTy, Align);
}
llvm::Constant *CGObjCCommonMac::GetClassName(StringRef RuntimeName) {
@@ -7345,7 +7351,7 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF,
Address mref =
Address(CGF.Builder.CreateBitCast(messageRef, ObjCTypes.MessageRefPtrTy),
- CGF.getPointerAlign());
+ ObjCTypes.MessageRefTy, CGF.getPointerAlign());
// Update the message ref argument.
args[1].setRValue(RValue::get(mref.getPointer()));
@@ -7404,7 +7410,7 @@ CGObjCNonFragileABIMac::GetClassGlobal(StringRef Name,
: llvm::GlobalValue::ExternalLinkage;
llvm::GlobalVariable *GV = CGM.getModule().getGlobalVariable(Name);
- if (!GV || GV->getType() != ObjCTypes.ClassnfABITy->getPointerTo()) {
+ if (!GV || GV->getValueType() != ObjCTypes.ClassnfABITy) {
auto *NewGV = new llvm::GlobalVariable(ObjCTypes.ClassnfABITy, false, L,
nullptr, Name);
@@ -7639,7 +7645,7 @@ Address CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) {
CGM.addCompilerUsedGlobal(Entry);
}
- return Address(Entry, Align);
+ return Address(Entry, ObjCTypes.SelectorPtrTy, Align);
}
/// EmitObjCIvarAssign - Code gen for assigning to a __strong object.
@@ -7658,8 +7664,9 @@ void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
- dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst.getPointer(), ivarOffset };
+ llvm::Value *dstVal =
+ CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+ llvm::Value *args[] = {src, dstVal, ivarOffset};
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args);
}
@@ -7678,8 +7685,9 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign(
src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
- dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst.getPointer() };
+ llvm::Value *dstVal =
+ CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+ llvm::Value *args[] = {src, dstVal};
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(),
args, "weakassign");
}
@@ -7689,8 +7697,8 @@ void CGObjCNonFragileABIMac::EmitGCMemmoveCollectable(
Address DestPtr,
Address SrcPtr,
llvm::Value *Size) {
- SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, ObjCTypes.Int8PtrTy);
- DestPtr = CGF.Builder.CreateBitCast(DestPtr, ObjCTypes.Int8PtrTy);
+ SrcPtr = CGF.Builder.CreateElementBitCast(SrcPtr, CGF.Int8Ty);
+ DestPtr = CGF.Builder.CreateElementBitCast(DestPtr, CGF.Int8Ty);
llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), Size };
CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args);
}
@@ -7702,10 +7710,11 @@ llvm::Value * CGObjCNonFragileABIMac::EmitObjCWeakRead(
CodeGen::CodeGenFunction &CGF,
Address AddrWeakObj) {
llvm::Type *DestTy = AddrWeakObj.getElementType();
- AddrWeakObj = CGF.Builder.CreateBitCast(AddrWeakObj, ObjCTypes.PtrObjectPtrTy);
+ llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast(
+ AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy);
llvm::Value *read_weak =
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(),
- AddrWeakObj.getPointer(), "weakread");
+ AddrWeakObjVal, "weakread");
read_weak = CGF.Builder.CreateBitCast(read_weak, DestTy);
return read_weak;
}
@@ -7724,8 +7733,9 @@ void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
- dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst.getPointer() };
+ llvm::Value *dstVal =
+ CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+ llvm::Value *args[] = {src, dstVal};
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(),
args, "weakassign");
}
@@ -7745,8 +7755,9 @@ void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
- dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst.getPointer() };
+ llvm::Value *dstVal =
+ CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+ llvm::Value *args[] = {src, dstVal};
if (!threadlocal)
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(),
args, "globalassign");
diff --git a/clang/lib/CodeGen/CGObjCRuntime.cpp b/clang/lib/CodeGen/CGObjCRuntime.cpp
index 33ae3c7c2b28..550fd3d70bdc 100644
--- a/clang/lib/CodeGen/CGObjCRuntime.cpp
+++ b/clang/lib/CodeGen/CGObjCRuntime.cpp
@@ -106,7 +106,7 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF,
CGF.CGM.getContext().toBits(StorageSize),
CharUnits::fromQuantity(0)));
- Address Addr(V, Alignment);
+ Address Addr = Address(V, CGF.Int8Ty, Alignment);
Addr = CGF.Builder.CreateElementBitCast(Addr,
llvm::Type::getIntNTy(CGF.getLLVMContext(),
Info->StorageSize));
diff --git a/clang/lib/CodeGen/CGOpenCLRuntime.cpp b/clang/lib/CodeGen/CGOpenCLRuntime.cpp
index dbe375294d17..ab8de7ecf50c 100644
--- a/clang/lib/CodeGen/CGOpenCLRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenCLRuntime.cpp
@@ -34,41 +34,46 @@ llvm::Type *CGOpenCLRuntime::convertOpenCLSpecificType(const Type *T) {
assert(T->isOpenCLSpecificType() &&
"Not an OpenCL specific type!");
- llvm::LLVMContext& Ctx = CGM.getLLVMContext();
- uint32_t AddrSpc = CGM.getContext().getTargetAddressSpace(
- CGM.getContext().getOpenCLTypeAddrSpace(T));
switch (cast<BuiltinType>(T)->getKind()) {
default:
llvm_unreachable("Unexpected opencl builtin type!");
return nullptr;
-#define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
- case BuiltinType::Id: \
- return llvm::PointerType::get( \
- llvm::StructType::create(Ctx, "opencl." #ImgType "_" #Suffix "_t"), \
- AddrSpc);
+#define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
+ case BuiltinType::Id: \
+ return getPointerType(T, "opencl." #ImgType "_" #Suffix "_t");
#include "clang/Basic/OpenCLImageTypes.def"
case BuiltinType::OCLSampler:
return getSamplerType(T);
case BuiltinType::OCLEvent:
- return llvm::PointerType::get(
- llvm::StructType::create(Ctx, "opencl.event_t"), AddrSpc);
+ return getPointerType(T, "opencl.event_t");
case BuiltinType::OCLClkEvent:
- return llvm::PointerType::get(
- llvm::StructType::create(Ctx, "opencl.clk_event_t"), AddrSpc);
+ return getPointerType(T, "opencl.clk_event_t");
case BuiltinType::OCLQueue:
- return llvm::PointerType::get(
- llvm::StructType::create(Ctx, "opencl.queue_t"), AddrSpc);
+ return getPointerType(T, "opencl.queue_t");
case BuiltinType::OCLReserveID:
- return llvm::PointerType::get(
- llvm::StructType::create(Ctx, "opencl.reserve_id_t"), AddrSpc);
-#define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
- case BuiltinType::Id: \
- return llvm::PointerType::get( \
- llvm::StructType::create(Ctx, "opencl." #ExtType), AddrSpc);
+ return getPointerType(T, "opencl.reserve_id_t");
+#define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
+ case BuiltinType::Id: \
+ return getPointerType(T, "opencl." #ExtType);
#include "clang/Basic/OpenCLExtensionTypes.def"
}
}
+llvm::PointerType *CGOpenCLRuntime::getPointerType(const Type *T,
+ StringRef Name) {
+ auto I = CachedTys.find(Name);
+ if (I != CachedTys.end())
+ return I->second;
+
+ llvm::LLVMContext &Ctx = CGM.getLLVMContext();
+ uint32_t AddrSpc = CGM.getContext().getTargetAddressSpace(
+ CGM.getContext().getOpenCLTypeAddrSpace(T));
+ auto *PTy =
+ llvm::PointerType::get(llvm::StructType::create(Ctx, Name), AddrSpc);
+ CachedTys[Name] = PTy;
+ return PTy;
+}
+
llvm::Type *CGOpenCLRuntime::getPipeType(const PipeType *T) {
if (T->isReadOnly())
return getPipeType(T, "opencl.pipe_ro_t", PipeROTy);
@@ -143,13 +148,14 @@ static const BlockExpr *getBlockExpr(const Expr *E) {
/// corresponding block expression.
void CGOpenCLRuntime::recordBlockInfo(const BlockExpr *E,
llvm::Function *InvokeF,
- llvm::Value *Block) {
+ llvm::Value *Block, llvm::Type *BlockTy) {
assert(EnqueuedBlockMap.find(E) == EnqueuedBlockMap.end() &&
"Block expression emitted twice");
assert(isa<llvm::Function>(InvokeF) && "Invalid invoke function");
assert(Block->getType()->isPointerTy() && "Invalid block literal type");
EnqueuedBlockMap[E].InvokeFunc = InvokeF;
EnqueuedBlockMap[E].BlockArg = Block;
+ EnqueuedBlockMap[E].BlockTy = BlockTy;
EnqueuedBlockMap[E].Kernel = nullptr;
}
@@ -174,8 +180,7 @@ CGOpenCLRuntime::emitOpenCLEnqueuedBlock(CodeGenFunction &CGF, const Expr *E) {
}
auto *F = CGF.getTargetHooks().createEnqueuedBlockKernel(
- CGF, EnqueuedBlockMap[Block].InvokeFunc,
- EnqueuedBlockMap[Block].BlockArg->stripPointerCasts());
+ CGF, EnqueuedBlockMap[Block].InvokeFunc, EnqueuedBlockMap[Block].BlockTy);
// The common part of the post-processing of the kernel goes here.
F->addFnAttr(llvm::Attribute::NoUnwind);
diff --git a/clang/lib/CodeGen/CGOpenCLRuntime.h b/clang/lib/CodeGen/CGOpenCLRuntime.h
index 3f7aa9b0d8dc..900644b3b93b 100644
--- a/clang/lib/CodeGen/CGOpenCLRuntime.h
+++ b/clang/lib/CodeGen/CGOpenCLRuntime.h
@@ -18,6 +18,7 @@
#include "clang/AST/Expr.h"
#include "clang/AST/Type.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringMap.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
@@ -38,18 +39,21 @@ protected:
llvm::Type *PipeROTy;
llvm::Type *PipeWOTy;
llvm::PointerType *SamplerTy;
+ llvm::StringMap<llvm::PointerType *> CachedTys;
/// Structure for enqueued block information.
struct EnqueuedBlockInfo {
llvm::Function *InvokeFunc; /// Block invoke function.
llvm::Function *Kernel; /// Enqueued block kernel.
llvm::Value *BlockArg; /// The first argument to enqueued block kernel.
+ llvm::Type *BlockTy; /// Type of the block argument.
};
/// Maps block expression to block information.
llvm::DenseMap<const Expr *, EnqueuedBlockInfo> EnqueuedBlockMap;
virtual llvm::Type *getPipeType(const PipeType *T, StringRef Name,
llvm::Type *&PipeTy);
+ llvm::PointerType *getPointerType(const Type *T, StringRef Name);
public:
CGOpenCLRuntime(CodeGenModule &CGM) : CGM(CGM),
@@ -90,7 +94,7 @@ public:
/// \param InvokeF invoke function emitted for the block expression.
/// \param Block block literal emitted for the block expression.
void recordBlockInfo(const BlockExpr *E, llvm::Function *InvokeF,
- llvm::Value *Block);
+ llvm::Value *Block, llvm::Type *BlockTy);
/// \return LLVM block invoke function emitted for an expression derived from
/// the block expression.
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index db1c3ca191ca..305040b01c08 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -29,11 +29,13 @@
#include "clang/CodeGen/ConstantInitBuilder.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Format.h"
@@ -369,8 +371,7 @@ public:
/*RefersToEnclosingVariableOrCapture=*/false,
VD->getType().getNonReferenceType(), VK_LValue,
C.getLocation());
- PrivScope.addPrivate(
- VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); });
+ PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF));
}
(void)PrivScope.Privatize();
}
@@ -633,10 +634,8 @@ static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
const auto *RHSDRE =
cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
- [=]() { return Private; });
- PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
- [=]() { return Original; });
+ PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()), Private);
+ PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original);
(void)PrivateScope.Privatize();
RValue Func = RValue::get(Reduction.second);
CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
@@ -719,14 +718,14 @@ static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
"omp.arraycpy.srcElementPast");
SrcElementPHI->addIncoming(SrcBegin, EntryBB);
SrcElementCurrent =
- Address(SrcElementPHI,
+ Address(SrcElementPHI, SrcAddr.getElementType(),
SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
}
llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
DestElementPHI->addIncoming(DestBegin, EntryBB);
Address DestElementCurrent =
- Address(DestElementPHI,
+ Address(DestElementPHI, DestAddr.getElementType(),
DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
// Emit copy.
@@ -825,9 +824,7 @@ void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {
}
void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
- const auto *PrivateVD =
- cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
- QualType PrivateType = PrivateVD->getType();
+ QualType PrivateType = getPrivateType(N);
bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
if (!PrivateType->isVariablyModifiedType()) {
Sizes.emplace_back(
@@ -862,9 +859,7 @@ void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
llvm::Value *Size) {
- const auto *PrivateVD =
- cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
- QualType PrivateType = PrivateVD->getType();
+ QualType PrivateType = getPrivateType(N);
if (!PrivateType->isVariablyModifiedType()) {
assert(!Size && !Sizes[N].second &&
"Size should be nullptr for non-variably modified reduction "
@@ -887,9 +882,6 @@ void ReductionCodeGen::emitInitialization(
cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
const OMPDeclareReductionDecl *DRD =
getReductionInit(ClausesData[N].ReductionOp);
- QualType PrivateType = PrivateVD->getType();
- PrivateAddr = CGF.Builder.CreateElementBitCast(
- PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
if (DRD && DRD->getInitializer())
(void)DefaultInit(CGF);
@@ -908,18 +900,14 @@ void ReductionCodeGen::emitInitialization(
}
bool ReductionCodeGen::needCleanups(unsigned N) {
- const auto *PrivateVD =
- cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
- QualType PrivateType = PrivateVD->getType();
+ QualType PrivateType = getPrivateType(N);
QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
return DTorKind != QualType::DK_none;
}
void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
Address PrivateAddr) {
- const auto *PrivateVD =
- cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
- QualType PrivateType = PrivateVD->getType();
+ QualType PrivateType = getPrivateType(N);
QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
if (needCleanups(N)) {
PrivateAddr = CGF.Builder.CreateElementBitCast(
@@ -949,8 +937,7 @@ static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
}
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
- llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
- llvm::Value *Addr) {
+ Address OriginalBaseAddress, llvm::Value *Addr) {
Address Tmp = Address::invalid();
Address TopTmp = Address::invalid();
Address MostTopTmp = Address::invalid();
@@ -965,15 +952,17 @@ static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
TopTmp = Tmp;
BaseTy = BaseTy->getPointeeType();
}
- llvm::Type *Ty = BaseLVType;
- if (Tmp.isValid())
- Ty = Tmp.getElementType();
- Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
+
if (Tmp.isValid()) {
+ Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ Addr, Tmp.getElementType());
CGF.Builder.CreateStore(Addr, Tmp);
return MostTopTmp;
}
- return Address(Addr, BaseLVAlignment);
+
+ Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ Addr, OriginalBaseAddress.getType());
+ return OriginalBaseAddress.withPointer(Addr);
}
static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
@@ -1016,8 +1005,7 @@ Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
SharedAddr.getElementType(), PrivatePointer, Adjustment);
return castToBase(CGF, OrigVD->getType(),
SharedAddresses[N].first.getType(),
- OriginalBaseLValue.getAddress(CGF).getType(),
- OriginalBaseLValue.getAlignment(), Ptr);
+ OriginalBaseLValue.getAddress(CGF), Ptr);
}
BaseDecls.emplace_back(
cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
@@ -1140,15 +1128,13 @@ emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
Out->getLocation());
CodeGenFunction::OMPPrivateScope Scope(CGF);
Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
- Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
- return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
- .getAddress(CGF);
- });
+ Scope.addPrivate(
+ In, CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
+ .getAddress(CGF));
Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
- Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
- return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
- .getAddress(CGF);
- });
+ Scope.addPrivate(
+ Out, CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
+ .getAddress(CGF));
(void)Scope.Privatize();
if (!IsCombiner && Out->hasInit() &&
!CGF.isTrivialInitializer(Out->getInit())) {
@@ -1343,51 +1329,6 @@ llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
return Res;
}
-static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
- const RecordDecl *RD, const CGRecordLayout &RL,
- ArrayRef<llvm::Constant *> Data) {
- llvm::StructType *StructTy = RL.getLLVMType();
- unsigned PrevIdx = 0;
- ConstantInitBuilder CIBuilder(CGM);
- auto DI = Data.begin();
- for (const FieldDecl *FD : RD->fields()) {
- unsigned Idx = RL.getLLVMFieldNo(FD);
- // Fill the alignment.
- for (unsigned I = PrevIdx; I < Idx; ++I)
- Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
- PrevIdx = Idx + 1;
- Fields.add(*DI);
- ++DI;
- }
-}
-
-template <class... As>
-static llvm::GlobalVariable *
-createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
- ArrayRef<llvm::Constant *> Data, const Twine &Name,
- As &&... Args) {
- const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
- const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
- ConstantInitBuilder CIBuilder(CGM);
- ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
- buildStructValue(Fields, CGM, RD, RL, Data);
- return Fields.finishAndCreateGlobal(
- Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
- std::forward<As>(Args)...);
-}
-
-template <typename T>
-static void
-createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
- ArrayRef<llvm::Constant *> Data,
- T &Parent) {
- const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
- const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
- ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
- buildStructValue(Fields, CGM, RD, RL, Data);
- Fields.finishAndAddTo(Parent);
-}
-
void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
bool AtCurrentPoint) {
auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
@@ -1703,10 +1644,10 @@ Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
OS << "_decl_tgt_ref_ptr";
}
llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
+ QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
+ llvm::Type *LlvmPtrTy = CGM.getTypes().ConvertTypeForMem(PtrTy);
if (!Ptr) {
- QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
- Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
- PtrName);
+ Ptr = getOrCreateInternalVariable(LlvmPtrTy, PtrName);
auto *GV = cast<llvm::GlobalVariable>(Ptr);
GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
@@ -1715,7 +1656,7 @@ Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
GV->setInitializer(CGM.GetAddrOfGlobal(VD));
registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
}
- return Address(Ptr, CGM.getContext().getDeclAlign(VD));
+ return Address(Ptr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD));
}
return Address::invalid();
}
@@ -1739,16 +1680,17 @@ Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
return VDAddr;
llvm::Type *VarTy = VDAddr.getElementType();
- llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
- CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
- CGM.Int8PtrTy),
- CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
- getOrCreateThreadPrivateCache(VD)};
- return Address(CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
- Args),
- VDAddr.getAlignment());
+ llvm::Value *Args[] = {
+ emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
+ CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy),
+ CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
+ getOrCreateThreadPrivateCache(VD)};
+ return Address(
+ CGF.EmitRuntimeCall(
+ OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
+ Args),
+ CGF.Int8Ty, VDAddr.getAlignment());
}
void CGOpenMPRuntime::emitThreadPrivateVarInit(
@@ -1805,7 +1747,7 @@ llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
CGM.getContext().VoidPtrTy, Dst.getLocation());
- Address Arg = Address(ArgVal, VDAddr.getAlignment());
+ Address Arg(ArgVal, CtorCGF.Int8Ty, VDAddr.getAlignment());
Arg = CtorCGF.Builder.CreateElementBitCast(
Arg, CtorCGF.ConvertTypeForMem(ASTTy));
CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
@@ -1841,9 +1783,10 @@ llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
DtorCGF.GetAddrOfLocalVar(&Dst),
/*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
- DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
- DtorCGF.getDestroyer(ASTTy.isDestructedType()),
- DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
+ DtorCGF.emitDestroy(
+ Address(ArgVal, DtorCGF.Int8Ty, VDAddr.getAlignment()), ASTTy,
+ DtorCGF.getDestroyer(ASTTy.isDestructedType()),
+ DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
DtorCGF.FinishFunction();
Dtor = Fn;
}
@@ -1938,19 +1881,27 @@ bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
llvm::Function *Fn = CGM.CreateGlobalInitOrCleanUpFunction(
- FTy, Twine(Buffer, "_ctor"), FI, Loc);
+ FTy, Twine(Buffer, "_ctor"), FI, Loc, false,
+ llvm::GlobalValue::WeakODRLinkage);
+ if (CGM.getTriple().isAMDGCN())
+ Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
FunctionArgList(), Loc, Loc);
auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
+ llvm::Constant *AddrInAS0 = Addr;
+ if (Addr->getAddressSpace() != 0)
+ AddrInAS0 = llvm::ConstantExpr::getAddrSpaceCast(
+ Addr, llvm::PointerType::getWithSamePointeeType(
+ cast<llvm::PointerType>(Addr->getType()), 0));
CtorCGF.EmitAnyExprToMem(Init,
- Address(Addr, CGM.getContext().getDeclAlign(VD)),
+ Address(AddrInAS0, Addr->getValueType(),
+ CGM.getContext().getDeclAlign(VD)),
Init->getType().getQualifiers(),
/*IsInitializer=*/true);
CtorCGF.FinishFunction();
Ctor = Fn;
ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
- CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
} else {
Ctor = new llvm::GlobalVariable(
CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
@@ -1976,20 +1927,28 @@ bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
llvm::Function *Fn = CGM.CreateGlobalInitOrCleanUpFunction(
- FTy, Twine(Buffer, "_dtor"), FI, Loc);
+ FTy, Twine(Buffer, "_dtor"), FI, Loc, false,
+ llvm::GlobalValue::WeakODRLinkage);
+ if (CGM.getTriple().isAMDGCN())
+ Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
FunctionArgList(), Loc, Loc);
// Create a scope with an artificial location for the body of this
// function.
auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
- DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
+ llvm::Constant *AddrInAS0 = Addr;
+ if (Addr->getAddressSpace() != 0)
+ AddrInAS0 = llvm::ConstantExpr::getAddrSpaceCast(
+ Addr, llvm::PointerType::getWithSamePointeeType(
+ cast<llvm::PointerType>(Addr->getType()), 0));
+ DtorCGF.emitDestroy(Address(AddrInAS0, Addr->getValueType(),
+ CGM.getContext().getDeclAlign(VD)),
ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
DtorCGF.FinishFunction();
Dtor = Fn;
ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
- CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
} else {
Dtor = new llvm::GlobalVariable(
CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
@@ -2035,7 +1994,7 @@ Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
Args),
VarLVType->getPointerTo(/*AddrSpace=*/0)),
- CGM.getContext().getTypeAlignInChars(VarType));
+ VarLVType, CGM.getContext().getTypeAlignInChars(VarType));
}
void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
@@ -2367,14 +2326,15 @@ static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
- Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
- Addr = CGF.Builder.CreateElementBitCast(
- Addr, CGF.ConvertTypeForMem(Var->getType()));
- return Addr;
+ llvm::Type *ElemTy = CGF.ConvertTypeForMem(Var->getType());
+ return Address(
+ CGF.Builder.CreateBitCast(
+ Ptr, ElemTy->getPointerTo(Ptr->getType()->getPointerAddressSpace())),
+ ElemTy, CGF.getContext().getDeclAlign(Var));
}
static llvm::Value *emitCopyprivateCopyFunction(
- CodeGenModule &CGM, llvm::Type *ArgsType,
+ CodeGenModule &CGM, llvm::Type *ArgsElemType,
ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
SourceLocation Loc) {
@@ -2401,11 +2361,13 @@ static llvm::Value *emitCopyprivateCopyFunction(
// Dest = (void*[n])(LHSArg);
// Src = (void*[n])(RHSArg);
Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
- ArgsType), CGF.getPointerAlign());
+ CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
+ ArgsElemType->getPointerTo()),
+ ArgsElemType, CGF.getPointerAlign());
Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
- ArgsType), CGF.getPointerAlign());
+ CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
+ ArgsElemType->getPointerTo()),
+ ArgsElemType, CGF.getPointerAlign());
// *(Type0*)Dst[0] = *(Type0*)Src[0];
// *(Type1*)Dst[1] = *(Type1*)Src[1];
// ...
@@ -2494,12 +2456,11 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
// Build function that copies private values from single region to all other
// threads in the corresponding parallel region.
llvm::Value *CpyFn = emitCopyprivateCopyFunction(
- CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
- CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
+ CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy), CopyprivateVars,
+ SrcExprs, DstExprs, AssignmentOps, Loc);
llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
- Address CL =
- CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
- CGF.VoidPtrTy);
+ Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CopyprivateList, CGF.VoidPtrTy, CGF.Int8Ty);
llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
llvm::Value *Args[] = {
emitUpdateLocation(CGF, Loc), // ident_t *<loc>
@@ -3130,32 +3091,7 @@ void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
void CGOpenMPRuntime::createOffloadEntry(
llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
llvm::GlobalValue::LinkageTypes Linkage) {
- StringRef Name = Addr->getName();
- llvm::Module &M = CGM.getModule();
- llvm::LLVMContext &C = M.getContext();
-
- // Create constant string with the name.
- llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
-
- std::string StringName = getName({"omp_offloading", "entry_name"});
- auto *Str = new llvm::GlobalVariable(
- M, StrPtrInit->getType(), /*isConstant=*/true,
- llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
- Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
-
- llvm::Constant *Data[] = {
- llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(ID, CGM.VoidPtrTy),
- llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(Str, CGM.Int8PtrTy),
- llvm::ConstantInt::get(CGM.SizeTy, Size),
- llvm::ConstantInt::get(CGM.Int32Ty, Flags),
- llvm::ConstantInt::get(CGM.Int32Ty, 0)};
- std::string EntryName = getName({"omp_offloading", "entry", ""});
- llvm::GlobalVariable *Entry = createGlobalStruct(
- CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
- Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
-
- // The entry has to be created in the section the linker expects it to be.
- Entry->setSection("omp_offloading_entries");
+ OMPBuilder.emitOffloadingEntry(ID, Addr->getName(), Size, Flags);
}
void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
@@ -3323,6 +3259,13 @@ void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
}
break;
}
+
+ // Hidden or internal symbols on the device are not externally visible. We
+ // should not attempt to register them by creating an offloading entry.
+ if (auto *GV = dyn_cast<llvm::GlobalValue>(CE->getAddress()))
+ if (GV->hasLocalLinkage() || GV->hasHiddenVisibility())
+ continue;
+
createOffloadEntry(CE->getAddress(), CE->getAddress(),
CE->getVarSize().getQuantity(), Flags,
CE->getLinkage());
@@ -3413,35 +3356,6 @@ void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
}
}
-QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
- // Make sure the type of the entry is already created. This is the type we
- // have to create:
- // struct __tgt_offload_entry{
- // void *addr; // Pointer to the offload entry info.
- // // (function or global)
- // char *name; // Name of the function or global.
- // size_t size; // Size of the entry info (0 if it a function).
- // int32_t flags; // Flags associated with the entry, e.g. 'link'.
- // int32_t reserved; // Reserved, to use by the runtime library.
- // };
- if (TgtOffloadEntryQTy.isNull()) {
- ASTContext &C = CGM.getContext();
- RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
- RD->startDefinition();
- addFieldToRecordDecl(C, RD, C.VoidPtrTy);
- addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
- addFieldToRecordDecl(C, RD, C.getSizeType());
- addFieldToRecordDecl(
- C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
- addFieldToRecordDecl(
- C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
- RD->completeDefinition();
- RD->addAttr(PackedAttr::CreateImplicit(C));
- TgtOffloadEntryQTy = C.getRecordType(RD);
- }
- return TgtOffloadEntryQTy;
-}
-
namespace {
struct PrivateHelpersTy {
PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original,
@@ -3642,12 +3556,12 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
}
- llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
- TaskPrivatesMap,
- CGF.Builder
- .CreatePointerBitCastOrAddrSpaceCast(
- TDBase.getAddress(CGF), CGF.VoidPtrTy)
- .getPointer()};
+ llvm::Value *CommonArgs[] = {
+ GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
+ CGF.Builder
+ .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(CGF),
+ CGF.VoidPtrTy, CGF.Int8Ty)
+ .getPointer()};
SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
std::end(CommonArgs));
if (isOpenMPTaskLoopDirective(Kind)) {
@@ -3867,7 +3781,8 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
(IsTargetTask && KmpTaskSharedsPtr.isValid())) {
SrcBase = CGF.MakeAddrLValue(
CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
+ KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy),
+ CGF.ConvertTypeForMem(SharedsTy)),
SharedsTy);
}
FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
@@ -3903,8 +3818,8 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
} else if (ForDup) {
SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
SharedRefLValue = CGF.MakeAddrLValue(
- Address(SharedRefLValue.getPointer(CGF),
- C.getDeclAlign(OriginalVD)),
+ SharedRefLValue.getAddress(CGF).withAlignment(
+ C.getDeclAlign(OriginalVD)),
SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
SharedRefLValue.getTBAAInfo());
} else if (CGF.LambdaCaptureFields.count(
@@ -3933,8 +3848,7 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
Address SrcElement) {
// Clean up any temporaries needed by the initialization.
CodeGenFunction::OMPPrivateScope InitScope(CGF);
- InitScope.addPrivate(
- Elem, [SrcElement]() -> Address { return SrcElement; });
+ InitScope.addPrivate(Elem, SrcElement);
(void)InitScope.Privatize();
// Emit initialization for single element.
CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
@@ -3946,9 +3860,7 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
}
} else {
CodeGenFunction::OMPPrivateScope InitScope(CGF);
- InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address {
- return SharedRefLValue.getAddress(CGF);
- });
+ InitScope.addPrivate(Elem, SharedRefLValue.getAddress(CGF));
(void)InitScope.Privatize();
CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
CGF.EmitExprAsInit(Init, VD, PrivateLValue,
@@ -3971,7 +3883,7 @@ static bool checkInitIsRequired(CodeGenFunction &CGF,
continue;
const VarDecl *VD = Pair.second.PrivateCopy;
const Expr *Init = VD->getAnyInitializer();
- InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
+ InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(Init) &&
!CGF.isTrivialInitializer(Init));
if (InitRequired)
break;
@@ -4051,7 +3963,7 @@ emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
Base, *std::next(KmpTaskTQTyRD->field_begin(),
KmpTaskTShareds)),
Loc),
- CGM.getNaturalTypeAlignment(SharedsTy));
+ CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy));
}
emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
@@ -4094,14 +4006,11 @@ public:
for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper));
const auto *VD = cast<VarDecl>(E->getIteratorDecl(I));
- addPrivate(VD, [&CGF, VD]() {
- return CGF.CreateMemTemp(VD->getType(), VD->getName());
- });
+ addPrivate(VD, CGF.CreateMemTemp(VD->getType(), VD->getName()));
const OMPIteratorHelperData &HelperData = E->getHelper(I);
- addPrivate(HelperData.CounterVD, [&CGF, &HelperData]() {
- return CGF.CreateMemTemp(HelperData.CounterVD->getType(),
- "counter.addr");
- });
+ addPrivate(
+ HelperData.CounterVD,
+ CGF.CreateMemTemp(HelperData.CounterVD->getType(), "counter.addr"));
}
Privatize();
@@ -4534,13 +4443,13 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
// Copy shareds if there are any.
Address KmpTaskSharedsPtr = Address::invalid();
if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
- KmpTaskSharedsPtr =
- Address(CGF.EmitLoadOfScalar(
- CGF.EmitLValueForField(
- TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
- KmpTaskTShareds)),
- Loc),
- CGM.getNaturalTypeAlignment(SharedsTy));
+ KmpTaskSharedsPtr = Address(
+ CGF.EmitLoadOfScalar(
+ CGF.EmitLValueForField(
+ TDBase,
+ *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
+ Loc),
+ CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy));
LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
@@ -4597,7 +4506,9 @@ namespace {
enum RTLDependenceKindTy {
DepIn = 0x01,
DepInOut = 0x3,
- DepMutexInOutSet = 0x4
+ DepMutexInOutSet = 0x4,
+ DepInOutSet = 0x8,
+ DepOmpAllMem = 0x80,
};
/// Fields ids in kmp_depend_info record.
enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
@@ -4618,9 +4529,16 @@ static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) {
case OMPC_DEPEND_mutexinoutset:
DepKind = DepMutexInOutSet;
break;
+ case OMPC_DEPEND_inoutset:
+ DepKind = DepInOutSet;
+ break;
+ case OMPC_DEPEND_outallmemory:
+ DepKind = DepOmpAllMem;
+ break;
case OMPC_DEPEND_source:
case OMPC_DEPEND_sink:
case OMPC_DEPEND_depobj:
+ case OMPC_DEPEND_inoutallmemory:
case OMPC_DEPEND_unknown:
llvm_unreachable("Unknown task dependence type");
}
@@ -4650,16 +4568,15 @@ CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
getDependTypes(C, KmpDependInfoTy, FlagsTy);
RecordDecl *KmpDependInfoRD =
cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
- LValue Base = CGF.EmitLoadOfPointerLValue(
- DepobjLVal.getAddress(CGF),
- C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
- Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
- Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
- Base.getTBAAInfo());
+ LValue Base = CGF.EmitLoadOfPointerLValue(
+ CGF.Builder.CreateElementBitCast(
+ DepobjLVal.getAddress(CGF),
+ CGF.ConvertTypeForMem(KmpDependInfoPtrTy)),
+ KmpDependInfoPtrTy->castAs<PointerType>());
Address DepObjAddr = CGF.Builder.CreateGEP(
- Addr, llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
+ Base.getAddress(CGF),
+ llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
LValue NumDepsBase = CGF.MakeAddrLValue(
DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo());
// NumDeps = deps[i].base_addr;
@@ -4688,12 +4605,21 @@ static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
for (const Expr *E : Data.DepExprs) {
llvm::Value *Addr;
llvm::Value *Size;
- std::tie(Addr, Size) = getPointerAndSize(CGF, E);
+
+ // The expression will be a nullptr in the 'omp_all_memory' case.
+ if (E) {
+ std::tie(Addr, Size) = getPointerAndSize(CGF, E);
+ Addr = CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy);
+ } else {
+ Addr = llvm::ConstantInt::get(CGF.IntPtrTy, 0);
+ Size = llvm::ConstantInt::get(CGF.SizeTy, 0);
+ }
LValue Base;
if (unsigned *P = Pos.dyn_cast<unsigned *>()) {
Base = CGF.MakeAddrLValue(
CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy);
} else {
+ assert(E && "Expected a non-null expression");
LValue &PosLVal = *Pos.get<LValue *>();
llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
Base = CGF.MakeAddrLValue(
@@ -4702,8 +4628,7 @@ static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
// deps[i].base_addr = &<Dependencies[i].second>;
LValue BaseAddrLVal = CGF.EmitLValueForField(
Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
- CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
- BaseAddrLVal);
+ CGF.EmitStoreOfScalar(Addr, BaseAddrLVal);
// deps[i].len = sizeof(<Dependencies[i].second>);
LValue LenLVal = CGF.EmitLValueForField(
Base, *std::next(KmpDependInfoRD->field_begin(), Len));
@@ -4726,43 +4651,25 @@ static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
}
}
-static SmallVector<llvm::Value *, 4>
-emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
- const OMPTaskDataTy::DependData &Data) {
+SmallVector<llvm::Value *, 4> CGOpenMPRuntime::emitDepobjElementsSizes(
+ CodeGenFunction &CGF, QualType &KmpDependInfoTy,
+ const OMPTaskDataTy::DependData &Data) {
assert(Data.DepKind == OMPC_DEPEND_depobj &&
"Expected depobj dependecy kind.");
SmallVector<llvm::Value *, 4> Sizes;
SmallVector<LValue, 4> SizeLVals;
ASTContext &C = CGF.getContext();
- QualType FlagsTy;
- getDependTypes(C, KmpDependInfoTy, FlagsTy);
- RecordDecl *KmpDependInfoRD =
- cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
- QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
- llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy);
{
OMPIteratorGeneratorScope IteratorScope(
CGF, cast_or_null<OMPIteratorExpr>(
Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
: nullptr));
for (const Expr *E : Data.DepExprs) {
+ llvm::Value *NumDeps;
+ LValue Base;
LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
- LValue Base = CGF.EmitLoadOfPointerLValue(
- DepobjLVal.getAddress(CGF),
- C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
- Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- Base.getAddress(CGF), KmpDependInfoPtrT);
- Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
- Base.getTBAAInfo());
- Address DepObjAddr = CGF.Builder.CreateGEP(
- Addr, llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
- LValue NumDepsBase = CGF.MakeAddrLValue(
- DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo());
- // NumDeps = deps[i].base_addr;
- LValue BaseAddrLVal = CGF.EmitLValueForField(
- NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
- llvm::Value *NumDeps =
- CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc());
+ std::tie(NumDeps, Base) =
+ getDepobjElements(CGF, DepobjLVal, E->getExprLoc());
LValue NumLVal = CGF.MakeAddrLValue(
CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"),
C.getUIntPtrType());
@@ -4782,19 +4689,13 @@ emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
return Sizes;
}
-static void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
- LValue PosLVal,
- const OMPTaskDataTy::DependData &Data,
- Address DependenciesArray) {
+void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF,
+ QualType &KmpDependInfoTy,
+ LValue PosLVal,
+ const OMPTaskDataTy::DependData &Data,
+ Address DependenciesArray) {
assert(Data.DepKind == OMPC_DEPEND_depobj &&
"Expected depobj dependecy kind.");
- ASTContext &C = CGF.getContext();
- QualType FlagsTy;
- getDependTypes(C, KmpDependInfoTy, FlagsTy);
- RecordDecl *KmpDependInfoRD =
- cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
- QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
- llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy);
llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy);
{
OMPIteratorGeneratorScope IteratorScope(
@@ -4803,25 +4704,11 @@ static void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
: nullptr));
for (unsigned I = 0, End = Data.DepExprs.size(); I < End; ++I) {
const Expr *E = Data.DepExprs[I];
+ llvm::Value *NumDeps;
+ LValue Base;
LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
- LValue Base = CGF.EmitLoadOfPointerLValue(
- DepobjLVal.getAddress(CGF),
- C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
- Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- Base.getAddress(CGF), KmpDependInfoPtrT);
- Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
- Base.getTBAAInfo());
-
- // Get number of elements in a single depobj.
- Address DepObjAddr = CGF.Builder.CreateGEP(
- Addr, llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
- LValue NumDepsBase = CGF.MakeAddrLValue(
- DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo());
- // NumDeps = deps[i].base_addr;
- LValue BaseAddrLVal = CGF.EmitLValueForField(
- NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
- llvm::Value *NumDeps =
- CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc());
+ std::tie(NumDeps, Base) =
+ getDepobjElements(CGF, DepobjLVal, E->getExprLoc());
// memcopy dependency data.
llvm::Value *Size = CGF.Builder.CreateNUWMul(
@@ -4959,7 +4846,7 @@ std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
}
}
DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- DependenciesArray, CGF.VoidPtrTy);
+ DependenciesArray, CGF.VoidPtrTy, CGF.Int8Ty);
return std::make_pair(NumOfElements, DependenciesArray);
}
@@ -5018,9 +4905,10 @@ Address CGOpenMPRuntime::emitDepobjDependClause(
CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), OMPRTL___kmpc_alloc),
Args, ".dep.arr.addr");
+ llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(KmpDependInfoTy);
Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- Addr, CGF.ConvertTypeForMem(KmpDependInfoTy)->getPointerTo());
- DependenciesArray = Address(Addr, Align);
+ Addr, KmpDependInfoLlvmTy->getPointerTo());
+ DependenciesArray = Address(Addr, KmpDependInfoLlvmTy, Align);
// Write number of elements in the first element of array for depobj.
LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy);
// deps[i].base_addr = NumDependencies;
@@ -5042,7 +4930,8 @@ Address CGOpenMPRuntime::emitDepobjDependClause(
}
emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray);
DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy);
+ CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy,
+ CGF.Int8Ty);
return DependenciesArray;
}
@@ -5052,11 +4941,11 @@ void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
QualType FlagsTy;
getDependTypes(C, KmpDependInfoTy, FlagsTy);
LValue Base = CGF.EmitLoadOfPointerLValue(
- DepobjLVal.getAddress(CGF),
- C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
+ DepobjLVal.getAddress(CGF), C.VoidPtrTy.castAs<PointerType>());
QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
+ Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy),
+ CGF.ConvertTypeForMem(KmpDependInfoTy));
llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
Addr.getElementType(), Addr.getPointer(),
llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
@@ -5098,7 +4987,7 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
llvm::PHINode *ElementPHI =
CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast");
ElementPHI->addIncoming(Begin.getPointer(), EntryBB);
- Begin = Address(ElementPHI, Begin.getAlignment());
+ Begin = Begin.withPointer(ElementPHI);
Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(),
Base.getTBAAInfo());
// deps[i].flags = NewDepKind;
@@ -5370,21 +5259,21 @@ static void EmitOMPAggregateReduction(
llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
RHSElementPHI->addIncoming(RHSBegin, EntryBB);
- Address RHSElementCurrent =
- Address(RHSElementPHI,
- RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
+ Address RHSElementCurrent(
+ RHSElementPHI, RHSAddr.getElementType(),
+ RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
LHSElementPHI->addIncoming(LHSBegin, EntryBB);
- Address LHSElementCurrent =
- Address(LHSElementPHI,
- LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
+ Address LHSElementCurrent(
+ LHSElementPHI, LHSAddr.getElementType(),
+ LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
// Emit copy.
CodeGenFunction::OMPPrivateScope Scope(CGF);
- Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
- Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
+ Scope.addPrivate(LHSVar, LHSElementCurrent);
+ Scope.addPrivate(RHSVar, RHSElementCurrent);
Scope.Privatize();
RedOpGen(CGF, XExpr, EExpr, UpExpr);
Scope.ForceCleanup();
@@ -5429,9 +5318,9 @@ static void emitReductionCombiner(CodeGenFunction &CGF,
}
llvm::Function *CGOpenMPRuntime::emitReductionFunction(
- SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
- ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
- ArrayRef<const Expr *> ReductionOps) {
+ SourceLocation Loc, llvm::Type *ArgsElemType,
+ ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
+ ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) {
ASTContext &C = CGM.getContext();
// void reduction_func(void *LHSArg, void *RHSArg);
@@ -5456,29 +5345,27 @@ llvm::Function *CGOpenMPRuntime::emitReductionFunction(
// Dst = (void*[n])(LHSArg);
// Src = (void*[n])(RHSArg);
Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
- ArgsType), CGF.getPointerAlign());
+ CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
+ ArgsElemType->getPointerTo()),
+ ArgsElemType, CGF.getPointerAlign());
Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
- ArgsType), CGF.getPointerAlign());
+ CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
+ ArgsElemType->getPointerTo()),
+ ArgsElemType, CGF.getPointerAlign());
// ...
// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
// ...
CodeGenFunction::OMPPrivateScope Scope(CGF);
- auto IPriv = Privates.begin();
+ const auto *IPriv = Privates.begin();
unsigned Idx = 0;
for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
const auto *RHSVar =
cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
- Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
- return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
- });
+ Scope.addPrivate(RHSVar, emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar));
const auto *LHSVar =
cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
- Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
- return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
- });
+ Scope.addPrivate(LHSVar, emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar));
QualType PrivTy = (*IPriv)->getType();
if (PrivTy->isVariablyModifiedType()) {
// Get array size and emit VLA type.
@@ -5495,8 +5382,8 @@ llvm::Function *CGOpenMPRuntime::emitReductionFunction(
}
Scope.Privatize();
IPriv = Privates.begin();
- auto ILHS = LHSExprs.begin();
- auto IRHS = RHSExprs.begin();
+ const auto *ILHS = LHSExprs.begin();
+ const auto *IRHS = RHSExprs.begin();
for (const Expr *E : ReductionOps) {
if ((*IPriv)->getType()->isArrayType()) {
// Emit reduction for array section.
@@ -5591,9 +5478,9 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
if (SimpleReduction) {
CodeGenFunction::RunCleanupsScope Scope(CGF);
- auto IPriv = Privates.begin();
- auto ILHS = LHSExprs.begin();
- auto IRHS = RHSExprs.begin();
+ const auto *IPriv = Privates.begin();
+ const auto *ILHS = LHSExprs.begin();
+ const auto *IRHS = RHSExprs.begin();
for (const Expr *E : ReductionOps) {
emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
cast<DeclRefExpr>(*IRHS));
@@ -5618,7 +5505,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
/*IndexTypeQuals=*/0);
Address ReductionList =
CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
- auto IPriv = Privates.begin();
+ const auto *IPriv = Privates.begin();
unsigned Idx = 0;
for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
@@ -5641,9 +5528,9 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
}
// 2. Emit reduce_func().
- llvm::Function *ReductionFn = emitReductionFunction(
- Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
- LHSExprs, RHSExprs, ReductionOps);
+ llvm::Function *ReductionFn =
+ emitReductionFunction(Loc, CGF.ConvertTypeForMem(ReductionArrayTy),
+ Privates, LHSExprs, RHSExprs, ReductionOps);
// 3. Create static kmp_critical_name lock = { 0 };
std::string Name = getName({"reduction"});
@@ -5695,9 +5582,9 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
CodeGenFunction &CGF, PrePostActionTy &Action) {
CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
- auto IPriv = Privates.begin();
- auto ILHS = LHSExprs.begin();
- auto IRHS = RHSExprs.begin();
+ const auto *IPriv = Privates.begin();
+ const auto *ILHS = LHSExprs.begin();
+ const auto *IRHS = RHSExprs.begin();
for (const Expr *E : ReductionOps) {
RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
cast<DeclRefExpr>(*IRHS));
@@ -5729,9 +5616,9 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
CodeGenFunction &CGF, PrePostActionTy &Action) {
- auto ILHS = LHSExprs.begin();
- auto IRHS = RHSExprs.begin();
- auto IPriv = Privates.begin();
+ const auto *ILHS = LHSExprs.begin();
+ const auto *IRHS = RHSExprs.begin();
+ const auto *IPriv = Privates.begin();
for (const Expr *E : ReductionOps) {
const Expr *XExpr = nullptr;
const Expr *EExpr = nullptr;
@@ -5773,14 +5660,11 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
llvm::AtomicOrdering::Monotonic, Loc,
[&CGF, UpExpr, VD, Loc](RValue XRValue) {
CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- PrivateScope.addPrivate(
- VD, [&CGF, VD, XRValue, Loc]() {
- Address LHSTemp = CGF.CreateMemTemp(VD->getType());
- CGF.emitOMPSimpleStore(
- CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
- VD->getType().getNonReferenceType(), Loc);
- return LHSTemp;
- });
+ Address LHSTemp = CGF.CreateMemTemp(VD->getType());
+ CGF.emitOMPSimpleStore(
+ CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
+ VD->getType().getNonReferenceType(), Loc);
+ PrivateScope.addPrivate(VD, LHSTemp);
(void)PrivateScope.Privatize();
return CGF.EmitAnyExpr(UpExpr);
});
@@ -5896,9 +5780,12 @@ static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
Fn->setDoesNotRecurse();
CodeGenFunction CGF(CGM);
CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
+ QualType PrivateType = RCG.getPrivateType(N);
Address PrivateAddr = CGF.EmitLoadOfPointer(
- CGF.GetAddrOfLocalVar(&Param),
- C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
+ CGF.Builder.CreateElementBitCast(
+ CGF.GetAddrOfLocalVar(&Param),
+ CGF.ConvertTypeForMem(PrivateType)->getPointerTo()),
+ C.getPointerType(PrivateType)->castAs<PointerType>());
llvm::Value *Size = nullptr;
// If the size of the reduction item is non-constant, load it from global
// threadprivate variable.
@@ -5980,22 +5867,22 @@ static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
// %lhs = bitcast void* %arg0 to <type>*
// %rhs = bitcast void* %arg1 to <type>*
CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- PrivateScope.addPrivate(LHSVD, [&C, &CGF, &ParamInOut, LHSVD]() {
- // Pull out the pointer to the variable.
- Address PtrAddr = CGF.EmitLoadOfPointer(
- CGF.GetAddrOfLocalVar(&ParamInOut),
- C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
- return CGF.Builder.CreateElementBitCast(
- PtrAddr, CGF.ConvertTypeForMem(LHSVD->getType()));
- });
- PrivateScope.addPrivate(RHSVD, [&C, &CGF, &ParamIn, RHSVD]() {
- // Pull out the pointer to the variable.
- Address PtrAddr = CGF.EmitLoadOfPointer(
- CGF.GetAddrOfLocalVar(&ParamIn),
- C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
- return CGF.Builder.CreateElementBitCast(
- PtrAddr, CGF.ConvertTypeForMem(RHSVD->getType()));
- });
+ PrivateScope.addPrivate(
+ LHSVD,
+ // Pull out the pointer to the variable.
+ CGF.EmitLoadOfPointer(
+ CGF.Builder.CreateElementBitCast(
+ CGF.GetAddrOfLocalVar(&ParamInOut),
+ CGF.ConvertTypeForMem(LHSVD->getType())->getPointerTo()),
+ C.getPointerType(LHSVD->getType())->castAs<PointerType>()));
+ PrivateScope.addPrivate(
+ RHSVD,
+ // Pull out the pointer to the variable.
+ CGF.EmitLoadOfPointer(
+ CGF.Builder.CreateElementBitCast(
+ CGF.GetAddrOfLocalVar(&ParamIn),
+ CGF.ConvertTypeForMem(RHSVD->getType())->getPointerTo()),
+ C.getPointerType(RHSVD->getType())->castAs<PointerType>()));
PrivateScope.Privatize();
// Emit the combiner body:
// %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)
@@ -6036,8 +5923,7 @@ static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,
CodeGenFunction CGF(CGM);
CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
Address PrivateAddr = CGF.EmitLoadOfPointer(
- CGF.GetAddrOfLocalVar(&Param),
- C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
+ CGF.GetAddrOfLocalVar(&Param), C.VoidPtrTy.castAs<PointerType>());
llvm::Value *Size = nullptr;
// If the size of the reduction item is non-constant, load it from global
// threadprivate variable.
@@ -6237,7 +6123,7 @@ Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), OMPRTL___kmpc_task_reduction_get_th_data),
Args),
- SharedLVal.getAlignment());
+ CGF.Int8Ty, SharedLVal.getAlignment());
}
void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
@@ -6478,7 +6364,7 @@ void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF,
.getLimitedValue());
LValue AllocatorTraitsLVal = CGF.EmitLValue(AllocatorTraits);
Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- AllocatorTraitsLVal.getAddress(CGF), CGF.VoidPtrPtrTy);
+ AllocatorTraitsLVal.getAddress(CGF), CGF.VoidPtrPtrTy, CGF.VoidPtrTy);
AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy,
AllocatorTraitsLVal.getBaseInfo(),
AllocatorTraitsLVal.getTBAAInfo());
@@ -6528,6 +6414,8 @@ void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
// mangled name of the function that encloses the target region and BB is the
// line number of the target region.
+ const bool BuildOutlinedFn = CGM.getLangOpts().OpenMPIsDevice ||
+ !CGM.getLangOpts().OpenMPOffloadMandatory;
unsigned DeviceID;
unsigned FileID;
unsigned Line;
@@ -6546,7 +6434,8 @@ void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
- OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS, D.getBeginLoc());
+ if (BuildOutlinedFn)
+ OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS, D.getBeginLoc());
// If this target outline function is not an offload entry, we don't need to
// register it.
@@ -6566,7 +6455,7 @@ void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
if (CGM.getLangOpts().OpenMPIsDevice) {
OutlinedFnID = llvm::ConstantExpr::getBitCast(OutlinedFn, CGM.Int8PtrTy);
- OutlinedFn->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
+ OutlinedFn->setLinkage(llvm::GlobalValue::WeakODRLinkage);
OutlinedFn->setDSOLocal(false);
if (CGM.getTriple().isAMDGCN())
OutlinedFn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
@@ -6578,26 +6467,38 @@ void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
llvm::Constant::getNullValue(CGM.Int8Ty), Name);
}
+ // If we do not allow host fallback we still need a named address to use.
+ llvm::Constant *TargetRegionEntryAddr = OutlinedFn;
+ if (!BuildOutlinedFn) {
+ assert(!CGM.getModule().getGlobalVariable(EntryFnName, true) &&
+ "Named kernel already exists?");
+ TargetRegionEntryAddr = new llvm::GlobalVariable(
+ CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
+ llvm::GlobalValue::InternalLinkage,
+ llvm::Constant::getNullValue(CGM.Int8Ty), EntryFnName);
+ }
+
// Register the information for the entry associated with this target region.
OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
- DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID,
+ DeviceID, FileID, ParentName, Line, TargetRegionEntryAddr, OutlinedFnID,
OffloadEntriesInfoManagerTy::OMPTargetRegionEntryTargetRegion);
// Add NumTeams and ThreadLimit attributes to the outlined GPU function
int32_t DefaultValTeams = -1;
getNumTeamsExprForTargetDirective(CGF, D, DefaultValTeams);
- if (DefaultValTeams > 0) {
+ if (DefaultValTeams > 0 && OutlinedFn) {
OutlinedFn->addFnAttr("omp_target_num_teams",
std::to_string(DefaultValTeams));
}
int32_t DefaultValThreads = -1;
getNumThreadsExprForTargetDirective(CGF, D, DefaultValThreads);
- if (DefaultValThreads > 0) {
+ if (DefaultValThreads > 0 && OutlinedFn) {
OutlinedFn->addFnAttr("omp_target_thread_limit",
std::to_string(DefaultValThreads));
}
- CGM.getTargetCodeGenInfo().setTargetAttributes(nullptr, OutlinedFn, CGM);
+ if (BuildOutlinedFn)
+ CGM.getTargetCodeGenInfo().setTargetAttributes(nullptr, OutlinedFn, CGM);
}
/// Checks if the expression is constant or does not have non-trivial function
@@ -7873,6 +7774,7 @@ private:
isa<CXXThisExpr>(OAShE->getBase()->IgnoreParenCasts())) {
BP = Address(
CGF.EmitScalarExpr(OAShE->getBase()),
+ CGF.ConvertTypeForMem(OAShE->getBase()->getType()->getPointeeType()),
CGF.getContext().getTypeAlignInChars(OAShE->getBase()->getType()));
} else {
// The base is the reference to the variable.
@@ -8042,9 +7944,12 @@ private:
return BaseLV;
};
if (OAShE) {
- LowestElem = LB = Address(CGF.EmitScalarExpr(OAShE->getBase()),
- CGF.getContext().getTypeAlignInChars(
- OAShE->getBase()->getType()));
+ LowestElem = LB =
+ Address(CGF.EmitScalarExpr(OAShE->getBase()),
+ CGF.ConvertTypeForMem(
+ OAShE->getBase()->getType()->getPointeeType()),
+ CGF.getContext().getTypeAlignInChars(
+ OAShE->getBase()->getType()));
} else if (IsMemberReference) {
const auto *ME = cast<MemberExpr>(I->getAssociatedExpression());
LValue BaseLVal = EmitMemberExprBase(CGF, ME);
@@ -8080,8 +7985,8 @@ private:
CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(
I->getAssociatedExpression()->getType());
Address HB = CGF.Builder.CreateConstGEP(
- CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(LowestElem,
- CGF.VoidPtrTy),
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ LowestElem, CGF.VoidPtrTy, CGF.Int8Ty),
TypeSize.getQuantity() - 1);
PartialStruct.HighestElem = {
std::numeric_limits<decltype(
@@ -8326,7 +8231,7 @@ private:
}
// Skip the dummy dimension since we have already have its information.
- auto DI = DimSizes.begin() + 1;
+ auto *DI = DimSizes.begin() + 1;
// Product of dimension.
llvm::Value *DimProd =
llvm::ConstantInt::get(CGF.CGM.Int64Ty, ElementTypeSize);
@@ -9037,15 +8942,13 @@ public:
void generateInfoForLambdaCaptures(
const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const {
- const auto *RD = VD->getType()
- .getCanonicalType()
- .getNonReferenceType()
- ->getAsCXXRecordDecl();
+ QualType VDType = VD->getType().getCanonicalType().getNonReferenceType();
+ const auto *RD = VDType->getAsCXXRecordDecl();
if (!RD || !RD->isLambda())
return;
- Address VDAddr = Address(Arg, CGF.getContext().getDeclAlign(VD));
- LValue VDLVal = CGF.MakeAddrLValue(
- VDAddr, VD->getType().getCanonicalType().getNonReferenceType());
+ Address VDAddr(Arg, CGF.ConvertTypeForMem(VDType),
+ CGF.getContext().getDeclAlign(VD));
+ LValue VDLVal = CGF.MakeAddrLValue(VDAddr, VDType);
llvm::DenseMap<const VarDecl *, FieldDecl *> Captures;
FieldDecl *ThisCapture = nullptr;
RD->getCaptureFields(Captures, ThisCapture);
@@ -9500,11 +9403,11 @@ static void emitNonContiguousDescriptor(
}
// args[I] = &dims
Address DAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- DimsAddr, CGM.Int8PtrTy);
+ DimsAddr, CGM.Int8PtrTy, CGM.Int8Ty);
llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32(
llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
Info.PointersArray, 0, I);
- Address PAddr(P, CGF.getPointerAlign());
+ Address PAddr(P, CGM.VoidPtrTy, CGF.getPointerAlign());
CGF.Builder.CreateStore(DAddr.getPointer(), PAddr);
++L;
}
@@ -9575,12 +9478,6 @@ static void emitOffloadingArrays(
if (Info.NumberOfPtrs) {
// Detect if we have any capture size requiring runtime evaluation of the
// size so that a constant array could be eventually used.
- bool hasRuntimeEvaluationCaptureSize = false;
- for (llvm::Value *S : CombinedInfo.Sizes)
- if (!isa<llvm::Constant>(S)) {
- hasRuntimeEvaluationCaptureSize = true;
- break;
- }
llvm::APInt PointerNumAP(32, Info.NumberOfPtrs, /*isSigned=*/true);
QualType PointerArrayType = Ctx.getConstantArrayType(
@@ -9600,35 +9497,56 @@ static void emitOffloadingArrays(
// need to fill up the arrays as we do for the pointers.
QualType Int64Ty =
Ctx.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
- if (hasRuntimeEvaluationCaptureSize) {
+ SmallVector<llvm::Constant *> ConstSizes(
+ CombinedInfo.Sizes.size(), llvm::ConstantInt::get(CGF.Int64Ty, 0));
+ llvm::SmallBitVector RuntimeSizes(CombinedInfo.Sizes.size());
+ for (unsigned I = 0, E = CombinedInfo.Sizes.size(); I < E; ++I) {
+ if (auto *CI = dyn_cast<llvm::Constant>(CombinedInfo.Sizes[I])) {
+ if (!isa<llvm::ConstantExpr>(CI) && !isa<llvm::GlobalValue>(CI)) {
+ if (IsNonContiguous && (CombinedInfo.Types[I] &
+ MappableExprsHandler::OMP_MAP_NON_CONTIG))
+ ConstSizes[I] = llvm::ConstantInt::get(
+ CGF.Int64Ty, CombinedInfo.NonContigInfo.Dims[I]);
+ else
+ ConstSizes[I] = CI;
+ continue;
+ }
+ }
+ RuntimeSizes.set(I);
+ }
+
+ if (RuntimeSizes.all()) {
QualType SizeArrayType = Ctx.getConstantArrayType(
Int64Ty, PointerNumAP, nullptr, ArrayType::Normal,
/*IndexTypeQuals=*/0);
Info.SizesArray =
CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer();
} else {
- // We expect all the sizes to be constant, so we collect them to create
- // a constant array.
- SmallVector<llvm::Constant *, 16> ConstSizes;
- for (unsigned I = 0, E = CombinedInfo.Sizes.size(); I < E; ++I) {
- if (IsNonContiguous &&
- (CombinedInfo.Types[I] & MappableExprsHandler::OMP_MAP_NON_CONTIG)) {
- ConstSizes.push_back(llvm::ConstantInt::get(
- CGF.Int64Ty, CombinedInfo.NonContigInfo.Dims[I]));
- } else {
- ConstSizes.push_back(cast<llvm::Constant>(CombinedInfo.Sizes[I]));
- }
- }
-
auto *SizesArrayInit = llvm::ConstantArray::get(
llvm::ArrayType::get(CGM.Int64Ty, ConstSizes.size()), ConstSizes);
std::string Name = CGM.getOpenMPRuntime().getName({"offload_sizes"});
auto *SizesArrayGbl = new llvm::GlobalVariable(
- CGM.getModule(), SizesArrayInit->getType(),
- /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
- SizesArrayInit, Name);
+ CGM.getModule(), SizesArrayInit->getType(), /*isConstant=*/true,
+ llvm::GlobalValue::PrivateLinkage, SizesArrayInit, Name);
SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
- Info.SizesArray = SizesArrayGbl;
+ if (RuntimeSizes.any()) {
+ QualType SizeArrayType = Ctx.getConstantArrayType(
+ Int64Ty, PointerNumAP, nullptr, ArrayType::Normal,
+ /*IndexTypeQuals=*/0);
+ Address Buffer = CGF.CreateMemTemp(SizeArrayType, ".offload_sizes");
+ llvm::Value *GblConstPtr =
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ SizesArrayGbl, CGM.Int64Ty->getPointerTo());
+ CGF.Builder.CreateMemCpy(
+ Buffer,
+ Address(GblConstPtr, CGM.Int64Ty,
+ CGM.getNaturalTypeAlignment(Ctx.getIntTypeForBitwidth(
+ /*DestWidth=*/64, /*Signed=*/false))),
+ CGF.getTypeSize(SizeArrayType));
+ Info.SizesArray = Buffer.getPointer();
+ } else {
+ Info.SizesArray = SizesArrayGbl;
+ }
}
// The map types are always constant so we don't need to generate code to
@@ -9683,7 +9601,8 @@ static void emitOffloadingArrays(
Info.BasePointersArray, 0, I);
BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
BP, BPVal->getType()->getPointerTo(/*AddrSpace=*/0));
- Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
+ Address BPAddr(BP, BPVal->getType(),
+ Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
CGF.Builder.CreateStore(BPVal, BPAddr);
if (Info.requiresDevicePointerInfo())
@@ -9697,16 +9616,16 @@ static void emitOffloadingArrays(
Info.PointersArray, 0, I);
P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
P, PVal->getType()->getPointerTo(/*AddrSpace=*/0));
- Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
+ Address PAddr(P, PVal->getType(), Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
CGF.Builder.CreateStore(PVal, PAddr);
- if (hasRuntimeEvaluationCaptureSize) {
+ if (RuntimeSizes.test(I)) {
llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32(
llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs),
Info.SizesArray,
/*Idx0=*/0,
/*Idx1=*/I);
- Address SAddr(S, Ctx.getTypeAlignInChars(Int64Ty));
+ Address SAddr(S, CGM.Int64Ty, Ctx.getTypeAlignInChars(Int64Ty));
CGF.Builder.CreateStore(CGF.Builder.CreateIntCast(CombinedInfo.Sizes[I],
CGM.Int64Ty,
/*isSigned=*/true),
@@ -9957,6 +9876,7 @@ void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
cast<VarDecl>(cast<DeclRefExpr>(D->getMapperVarRef())->getDecl());
SourceLocation Loc = D->getLocation();
CharUnits ElementSize = C.getTypeSizeInChars(Ty);
+ llvm::Type *ElemTy = CGM.getTypes().ConvertTypeForMem(Ty);
// Prepare mapper function arguments and attributes.
ImplicitParamDecl HandleArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
@@ -10011,8 +9931,7 @@ void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
Size, MapperCGF.Builder.getInt64(ElementSize.getQuantity()));
llvm::Value *PtrBegin = MapperCGF.Builder.CreateBitCast(
BeginIn, CGM.getTypes().ConvertTypeForMem(PtrTy));
- llvm::Value *PtrEnd = MapperCGF.Builder.CreateGEP(
- PtrBegin->getType()->getPointerElementType(), PtrBegin, Size);
+ llvm::Value *PtrEnd = MapperCGF.Builder.CreateGEP(ElemTy, PtrBegin, Size);
llvm::Value *MapType = MapperCGF.EmitLoadOfScalar(
MapperCGF.GetAddrOfLocalVar(&TypeArg), /*Volatile=*/false,
C.getPointerType(Int64Ty), Loc);
@@ -10044,13 +9963,13 @@ void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
llvm::PHINode *PtrPHI = MapperCGF.Builder.CreatePHI(
PtrBegin->getType(), 2, "omp.arraymap.ptrcurrent");
PtrPHI->addIncoming(PtrBegin, EntryBB);
- Address PtrCurrent =
- Address(PtrPHI, MapperCGF.GetAddrOfLocalVar(&BeginArg)
- .getAlignment()
- .alignmentOfArrayElement(ElementSize));
+ Address PtrCurrent(PtrPHI, ElemTy,
+ MapperCGF.GetAddrOfLocalVar(&BeginArg)
+ .getAlignment()
+ .alignmentOfArrayElement(ElementSize));
// Privatize the declared variable of mapper to be the current array element.
CodeGenFunction::OMPPrivateScope Scope(MapperCGF);
- Scope.addPrivate(MapperVarDecl, [PtrCurrent]() { return PtrCurrent; });
+ Scope.addPrivate(MapperVarDecl, PtrCurrent);
(void)Scope.Privatize();
// Get map clause information. Fill up the arrays with all mapped variables.
@@ -10169,7 +10088,6 @@ void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
// Update the pointer to point to the next element that needs to be mapped,
// and check whether we have mapped all elements.
- llvm::Type *ElemTy = PtrPHI->getType()->getPointerElementType();
llvm::Value *PtrNext = MapperCGF.Builder.CreateConstGEP1_32(
ElemTy, PtrPHI, /*Idx0=*/1, "omp.arraymap.next");
PtrPHI->addIncoming(PtrNext, LastBB);
@@ -10309,10 +10227,14 @@ void CGOpenMPRuntime::emitTargetCall(
if (!CGF.HaveInsertPoint())
return;
- assert(OutlinedFn && "Invalid outlined function!");
+ const bool OffloadingMandatory = !CGM.getLangOpts().OpenMPIsDevice &&
+ CGM.getLangOpts().OpenMPOffloadMandatory;
+
+ assert((OffloadingMandatory || OutlinedFn) && "Invalid outlined function!");
const bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>() ||
- D.hasClausesOfKind<OMPNowaitClause>();
+ D.hasClausesOfKind<OMPNowaitClause>() ||
+ D.hasClausesOfKind<OMPInReductionClause>();
llvm::SmallVector<llvm::Value *, 16> CapturedVars;
const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
@@ -10324,18 +10246,26 @@ void CGOpenMPRuntime::emitTargetCall(
CodeGenFunction::OMPTargetDataInfo InputInfo;
llvm::Value *MapTypesArray = nullptr;
llvm::Value *MapNamesArray = nullptr;
- // Fill up the pointer arrays and transfer execution to the device.
- auto &&ThenGen = [this, Device, OutlinedFn, OutlinedFnID, &D, &InputInfo,
- &MapTypesArray, &MapNamesArray, &CS, RequiresOuterTask,
- &CapturedVars,
- SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
- if (Device.getInt() == OMPC_DEVICE_ancestor) {
- // Reverse offloading is not supported, so just execute on the host.
+ // Generate code for the host fallback function.
+ auto &&FallbackGen = [this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask,
+ &CS, OffloadingMandatory](CodeGenFunction &CGF) {
+ if (OffloadingMandatory) {
+ CGF.Builder.CreateUnreachable();
+ } else {
if (RequiresOuterTask) {
CapturedVars.clear();
CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
}
emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars);
+ }
+ };
+ // Fill up the pointer arrays and transfer execution to the device.
+ auto &&ThenGen = [this, Device, OutlinedFnID, &D, &InputInfo, &MapTypesArray,
+ &MapNamesArray, SizeEmitter,
+ FallbackGen](CodeGenFunction &CGF, PrePostActionTy &) {
+ if (Device.getInt() == OMPC_DEVICE_ancestor) {
+ // Reverse offloading is not supported, so just execute on the host.
+ FallbackGen(CGF);
return;
}
@@ -10350,6 +10280,7 @@ void CGOpenMPRuntime::emitTargetCall(
// From this point on, we need to have an ID of the target region defined.
assert(OutlinedFnID && "Invalid outlined function ID!");
+ (void)OutlinedFnID;
// Emit device ID if any.
llvm::Value *DeviceID;
@@ -10479,25 +10410,16 @@ void CGOpenMPRuntime::emitTargetCall(
CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
CGF.EmitBlock(OffloadFailedBlock);
- if (RequiresOuterTask) {
- CapturedVars.clear();
- CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
- }
- emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars);
+ FallbackGen(CGF);
+
CGF.EmitBranch(OffloadContBlock);
CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true);
};
// Notify that the host version must be executed.
- auto &&ElseGen = [this, &D, OutlinedFn, &CS, &CapturedVars,
- RequiresOuterTask](CodeGenFunction &CGF,
- PrePostActionTy &) {
- if (RequiresOuterTask) {
- CapturedVars.clear();
- CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
- }
- emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars);
+ auto &&ElseGen = [FallbackGen](CodeGenFunction &CGF, PrePostActionTy &) {
+ FallbackGen(CGF);
};
auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,
@@ -10587,11 +10509,13 @@ void CGOpenMPRuntime::emitTargetCall(
InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
InputInfo.BasePointersArray =
- Address(Info.BasePointersArray, CGM.getPointerAlign());
+ Address(Info.BasePointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
InputInfo.PointersArray =
- Address(Info.PointersArray, CGM.getPointerAlign());
- InputInfo.SizesArray = Address(Info.SizesArray, CGM.getPointerAlign());
- InputInfo.MappersArray = Address(Info.MappersArray, CGM.getPointerAlign());
+ Address(Info.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
+ InputInfo.SizesArray =
+ Address(Info.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
+ InputInfo.MappersArray =
+ Address(Info.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
MapTypesArray = Info.MapTypesArray;
MapNamesArray = Info.MapNamesArray;
if (RequiresOuterTask)
@@ -10865,7 +10789,7 @@ void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD,
// If we have host/nohost variables, they do not need to be registered.
Optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
OMPDeclareTargetDeclAttr::getDeviceType(VD);
- if (DevTy && DevTy.getValue() != OMPDeclareTargetDeclAttr::DT_Any)
+ if (DevTy && *DevTy != OMPDeclareTargetDeclAttr::DT_Any)
return;
llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
@@ -11468,12 +11392,13 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
{/*ForEndCall=*/false});
InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
InputInfo.BasePointersArray =
- Address(Info.BasePointersArray, CGM.getPointerAlign());
+ Address(Info.BasePointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
InputInfo.PointersArray =
- Address(Info.PointersArray, CGM.getPointerAlign());
+ Address(Info.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
InputInfo.SizesArray =
- Address(Info.SizesArray, CGM.getPointerAlign());
- InputInfo.MappersArray = Address(Info.MappersArray, CGM.getPointerAlign());
+ Address(Info.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
+ InputInfo.MappersArray =
+ Address(Info.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
MapTypesArray = Info.MapTypesArray;
MapNamesArray = Info.MapNamesArray;
if (RequiresOuterTask)
@@ -11493,13 +11418,21 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
namespace {
/// Kind of parameter in a function with 'declare simd' directive.
- enum ParamKindTy { LinearWithVarStride, Linear, Uniform, Vector };
- /// Attribute set of the parameter.
- struct ParamAttrTy {
- ParamKindTy Kind = Vector;
- llvm::APSInt StrideOrArg;
- llvm::APSInt Alignment;
- };
+enum ParamKindTy {
+ Linear,
+ LinearRef,
+ LinearUVal,
+ LinearVal,
+ Uniform,
+ Vector,
+};
+/// Attribute set of the parameter.
+struct ParamAttrTy {
+ ParamKindTy Kind = Vector;
+ llvm::APSInt StrideOrArg;
+ llvm::APSInt Alignment;
+ bool HasVarStride = false;
+};
} // namespace
static unsigned evaluateCDTSize(const FunctionDecl *FD,
@@ -11554,6 +11487,52 @@ static unsigned evaluateCDTSize(const FunctionDecl *FD,
return C.getTypeSize(CDT);
}
+/// Mangle the parameter part of the vector function name according to
+/// their OpenMP classification. The mangling function is defined in
+/// section 4.5 of the AAVFABI(2021Q1).
+static std::string mangleVectorParameters(ArrayRef<ParamAttrTy> ParamAttrs) {
+ SmallString<256> Buffer;
+ llvm::raw_svector_ostream Out(Buffer);
+ for (const auto &ParamAttr : ParamAttrs) {
+ switch (ParamAttr.Kind) {
+ case Linear:
+ Out << 'l';
+ break;
+ case LinearRef:
+ Out << 'R';
+ break;
+ case LinearUVal:
+ Out << 'U';
+ break;
+ case LinearVal:
+ Out << 'L';
+ break;
+ case Uniform:
+ Out << 'u';
+ break;
+ case Vector:
+ Out << 'v';
+ break;
+ }
+ if (ParamAttr.HasVarStride)
+ Out << "s" << ParamAttr.StrideOrArg;
+ else if (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef ||
+ ParamAttr.Kind == LinearUVal || ParamAttr.Kind == LinearVal) {
+ // Don't print the step value if it is not present or if it is
+ // equal to 1.
+ if (ParamAttr.StrideOrArg < 0)
+ Out << 'n' << -ParamAttr.StrideOrArg;
+ else if (ParamAttr.StrideOrArg != 1)
+ Out << ParamAttr.StrideOrArg;
+ }
+
+ if (!!ParamAttr.Alignment)
+ Out << 'a' << ParamAttr.Alignment;
+ }
+
+ return std::string(Out.str());
+}
+
static void
emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn,
const llvm::APSInt &VLENVal,
@@ -11602,26 +11581,7 @@ emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn,
} else {
Out << VLENVal;
}
- for (const ParamAttrTy &ParamAttr : ParamAttrs) {
- switch (ParamAttr.Kind){
- case LinearWithVarStride:
- Out << 's' << ParamAttr.StrideOrArg;
- break;
- case Linear:
- Out << 'l';
- if (ParamAttr.StrideOrArg != 1)
- Out << ParamAttr.StrideOrArg;
- break;
- case Uniform:
- Out << 'u';
- break;
- case Vector:
- Out << 'v';
- break;
- }
- if (!!ParamAttr.Alignment)
- Out << 'a' << ParamAttr.Alignment;
- }
+ Out << mangleVectorParameters(ParamAttrs);
Out << '_' << Fn->getName();
Fn->addFnAttr(Out.str());
}
@@ -11634,11 +11594,7 @@ emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn,
// available at
// https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi.
-/// Maps To Vector (MTV), as defined in 3.1.1 of the AAVFABI.
-///
-/// TODO: Need to implement the behavior for reference marked with a
-/// var or no linear modifiers (1.b in the section). For this, we
-/// need to extend ParamKindTy to support the linear modifiers.
+/// Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
static bool getAArch64MTV(QualType QT, ParamKindTy Kind) {
QT = QT.getCanonicalType();
@@ -11648,12 +11604,11 @@ static bool getAArch64MTV(QualType QT, ParamKindTy Kind) {
if (Kind == ParamKindTy::Uniform)
return false;
- if (Kind == ParamKindTy::Linear)
+ if (Kind == ParamKindTy::LinearUVal || ParamKindTy::LinearRef)
return false;
- // TODO: Handle linear references with modifiers
-
- if (Kind == ParamKindTy::LinearWithVarStride)
+ if ((Kind == ParamKindTy::Linear || Kind == ParamKindTy::LinearVal) &&
+ !QT->isReferenceType())
return false;
return true;
@@ -11734,39 +11689,6 @@ getNDSWDS(const FunctionDecl *FD, ArrayRef<ParamAttrTy> ParamAttrs) {
OutputBecomesInput);
}
-/// Mangle the parameter part of the vector function name according to
-/// their OpenMP classification. The mangling function is defined in
-/// section 3.5 of the AAVFABI.
-static std::string mangleVectorParameters(ArrayRef<ParamAttrTy> ParamAttrs) {
- SmallString<256> Buffer;
- llvm::raw_svector_ostream Out(Buffer);
- for (const auto &ParamAttr : ParamAttrs) {
- switch (ParamAttr.Kind) {
- case LinearWithVarStride:
- Out << "ls" << ParamAttr.StrideOrArg;
- break;
- case Linear:
- Out << 'l';
- // Don't print the step value if it is not present or if it is
- // equal to 1.
- if (ParamAttr.StrideOrArg != 1)
- Out << ParamAttr.StrideOrArg;
- break;
- case Uniform:
- Out << 'u';
- break;
- case Vector:
- Out << 'v';
- break;
- }
-
- if (!!ParamAttr.Alignment)
- Out << 'a' << ParamAttr.Alignment;
- }
-
- return std::string(Out.str());
-}
-
// Function used to add the attribute. The parameter `VLEN` is
// templated to allow the use of "x" when targeting scalable functions
// for SVE.
@@ -11933,16 +11855,16 @@ void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
llvm::Function *Fn) {
ASTContext &C = CGM.getContext();
FD = FD->getMostRecentDecl();
- // Map params to their positions in function decl.
- llvm::DenseMap<const Decl *, unsigned> ParamPositions;
- if (isa<CXXMethodDecl>(FD))
- ParamPositions.try_emplace(FD, 0);
- unsigned ParamPos = ParamPositions.size();
- for (const ParmVarDecl *P : FD->parameters()) {
- ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos);
- ++ParamPos;
- }
while (FD) {
+ // Map params to their positions in function decl.
+ llvm::DenseMap<const Decl *, unsigned> ParamPositions;
+ if (isa<CXXMethodDecl>(FD))
+ ParamPositions.try_emplace(FD, 0);
+ unsigned ParamPos = ParamPositions.size();
+ for (const ParmVarDecl *P : FD->parameters()) {
+ ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos);
+ ++ParamPos;
+ }
for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
llvm::SmallVector<ParamAttrTy, 8> ParamAttrs(ParamPositions.size());
// Mark uniform parameters.
@@ -11954,12 +11876,14 @@ void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
} else {
const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
->getCanonicalDecl();
- Pos = ParamPositions[PVD];
+ auto It = ParamPositions.find(PVD);
+ assert(It != ParamPositions.end() && "Function parameter not found");
+ Pos = It->second;
}
ParamAttrs[Pos].Kind = Uniform;
}
// Get alignment info.
- auto NI = Attr->alignments_begin();
+ auto *NI = Attr->alignments_begin();
for (const Expr *E : Attr->aligneds()) {
E = E->IgnoreParenImpCasts();
unsigned Pos;
@@ -11970,7 +11894,9 @@ void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
} else {
const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
->getCanonicalDecl();
- Pos = ParamPositions[PVD];
+ auto It = ParamPositions.find(PVD);
+ assert(It != ParamPositions.end() && "Function parameter not found");
+ Pos = It->second;
ParmTy = PVD->getType();
}
ParamAttrs[Pos].Alignment =
@@ -11982,27 +11908,48 @@ void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
++NI;
}
// Mark linear parameters.
- auto SI = Attr->steps_begin();
- auto MI = Attr->modifiers_begin();
+ auto *SI = Attr->steps_begin();
+ auto *MI = Attr->modifiers_begin();
for (const Expr *E : Attr->linears()) {
E = E->IgnoreParenImpCasts();
unsigned Pos;
+ bool IsReferenceType = false;
// Rescaling factor needed to compute the linear parameter
// value in the mangled name.
unsigned PtrRescalingFactor = 1;
if (isa<CXXThisExpr>(E)) {
Pos = ParamPositions[FD];
+ auto *P = cast<PointerType>(E->getType());
+ PtrRescalingFactor = CGM.getContext()
+ .getTypeSizeInChars(P->getPointeeType())
+ .getQuantity();
} else {
const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
->getCanonicalDecl();
- Pos = ParamPositions[PVD];
+ auto It = ParamPositions.find(PVD);
+ assert(It != ParamPositions.end() && "Function parameter not found");
+ Pos = It->second;
if (auto *P = dyn_cast<PointerType>(PVD->getType()))
PtrRescalingFactor = CGM.getContext()
.getTypeSizeInChars(P->getPointeeType())
.getQuantity();
+ else if (PVD->getType()->isReferenceType()) {
+ IsReferenceType = true;
+ PtrRescalingFactor =
+ CGM.getContext()
+ .getTypeSizeInChars(PVD->getType().getNonReferenceType())
+ .getQuantity();
+ }
}
ParamAttrTy &ParamAttr = ParamAttrs[Pos];
- ParamAttr.Kind = Linear;
+ if (*MI == OMPC_LINEAR_ref)
+ ParamAttr.Kind = LinearRef;
+ else if (*MI == OMPC_LINEAR_uval)
+ ParamAttr.Kind = LinearUVal;
+ else if (IsReferenceType)
+ ParamAttr.Kind = LinearVal;
+ else
+ ParamAttr.Kind = Linear;
// Assuming a stride of 1, for `linear` without modifiers.
ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1);
if (*SI) {
@@ -12010,10 +11957,13 @@ void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
if (!(*SI)->EvaluateAsInt(Result, C, Expr::SE_AllowSideEffects)) {
if (const auto *DRE =
cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
- if (const auto *StridePVD = cast<ParmVarDecl>(DRE->getDecl())) {
- ParamAttr.Kind = LinearWithVarStride;
- ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(
- ParamPositions[StridePVD->getCanonicalDecl()]);
+ if (const auto *StridePVD =
+ dyn_cast<ParmVarDecl>(DRE->getDecl())) {
+ ParamAttr.HasVarStride = true;
+ auto It = ParamPositions.find(StridePVD->getCanonicalDecl());
+ assert(It != ParamPositions.end() &&
+ "Function parameter not found");
+ ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(It->second);
}
}
} else {
@@ -12023,7 +11973,8 @@ void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
// If we are using a linear clause on a pointer, we need to
// rescale the value of linear_step with the byte size of the
// pointee type.
- if (Linear == ParamAttr.Kind)
+ if (!ParamAttr.HasVarStride &&
+ (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef))
ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
++SI;
++MI;
@@ -12235,6 +12186,16 @@ static llvm::Value *getAllocatorVal(CodeGenFunction &CGF,
return AllocVal;
}
+/// Return the alignment from an allocate directive if present.
+static llvm::Value *getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD) {
+ llvm::Optional<CharUnits> AllocateAlignment = CGM.getOMPAllocateAlignment(VD);
+
+ if (!AllocateAlignment)
+ return nullptr;
+
+ return llvm::ConstantInt::get(CGM.SizeTy, AllocateAlignment->getQuantity());
+}
+
Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
const VarDecl *VD) {
if (!VD)
@@ -12273,11 +12234,7 @@ Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
const Expr *Allocator = AA->getAllocator();
llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator);
- llvm::Value *Alignment =
- AA->getAlignment()
- ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(AA->getAlignment()),
- CGM.SizeTy, /*isSigned=*/false)
- : nullptr;
+ llvm::Value *Alignment = getAlignmentValue(CGM, CVD);
SmallVector<llvm::Value *, 4> Args;
Args.push_back(ThreadID);
if (Alignment)
@@ -12324,7 +12281,9 @@ Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
}
};
Address VDAddr =
- UntiedRealAddr.isValid() ? UntiedRealAddr : Address(Addr, Align);
+ UntiedRealAddr.isValid()
+ ? UntiedRealAddr
+ : Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(
NormalAndEHCleanup, FiniRTLFn, CVD->getLocation().getRawEncoding(),
VDAddr, Allocator);
@@ -12772,7 +12731,8 @@ void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
LValue PrivLVal = CGF.EmitLValue(FoundE);
Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
PrivLVal.getAddress(CGF),
- CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy)));
+ CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy)),
+ CGF.ConvertTypeForMem(StructTy));
LValue BaseLVal =
CGF.MakeAddrLValue(StructAddr, StructTy, AlignmentSource::Decl);
LValue FiredLVal = CGF.EmitLValueForField(BaseLVal, FiredDecl);
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 19754b0cfacc..7fc6a7e278e5 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -218,6 +218,11 @@ public:
/// Returns true if the initialization of the reduction item uses initializer
/// from declare reduction construct.
bool usesReductionInitializer(unsigned N) const;
+ /// Return the type of the private item.
+ QualType getPrivateType(unsigned N) const {
+ return cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl())
+ ->getType();
+ }
};
class CGOpenMPRuntime {
@@ -513,15 +518,6 @@ private:
/// kmp_int64 st; // stride
/// };
QualType KmpDimTy;
- /// Type struct __tgt_offload_entry{
- /// void *addr; // Pointer to the offload entry info.
- /// // (function or global)
- /// char *name; // Name of the function or global.
- /// size_t size; // Size of the entry info (0 if it a function).
- /// int32_t flags;
- /// int32_t reserved;
- /// };
- QualType TgtOffloadEntryQTy;
/// Entity that registers the offloading constants that were emitted so
/// far.
class OffloadEntriesInfoManagerTy {
@@ -777,9 +773,6 @@ private:
/// metadata.
void loadOffloadInfoMetadata();
- /// Returns __tgt_offload_entry type.
- QualType getTgtOffloadEntryQTy();
-
/// Start scanning from statement \a S and and emit all target regions
/// found along the way.
/// \param S Starting statement.
@@ -915,6 +908,14 @@ private:
LValue DepobjLVal,
SourceLocation Loc);
+ SmallVector<llvm::Value *, 4>
+ emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
+ const OMPTaskDataTy::DependData &Data);
+
+ void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
+ LValue PosLVal, const OMPTaskDataTy::DependData &Data,
+ Address DependenciesArray);
+
public:
explicit CGOpenMPRuntime(CodeGenModule &CGM)
: CGOpenMPRuntime(CGM, ".", ".") {}
@@ -1406,14 +1407,14 @@ public:
bool HasCancel = false);
/// Emits reduction function.
- /// \param ArgsType Array type containing pointers to reduction variables.
+ /// \param ArgsElemType Array type containing pointers to reduction variables.
/// \param Privates List of private copies for original reduction arguments.
/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
/// or 'operator binop(LHS, RHS)'.
llvm::Function *emitReductionFunction(SourceLocation Loc,
- llvm::Type *ArgsType,
+ llvm::Type *ArgsElemType,
ArrayRef<const Expr *> Privates,
ArrayRef<const Expr *> LHSExprs,
ArrayRef<const Expr *> RHSExprs,
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 2d5511336851..6dea846f486f 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -1203,15 +1203,17 @@ CGOpenMPRuntimeGPU::CGOpenMPRuntimeGPU(CodeGenModule &CGM)
llvm_unreachable("OpenMP can only handle device code.");
llvm::OpenMPIRBuilder &OMPBuilder = getOMPBuilder();
- if (CGM.getLangOpts().OpenMPTargetNewRuntime &&
- !CGM.getLangOpts().OMPHostIRFile.empty()) {
- OMPBuilder.createGlobalFlag(CGM.getLangOpts().OpenMPTargetDebug,
- "__omp_rtl_debug_kind");
- OMPBuilder.createGlobalFlag(CGM.getLangOpts().OpenMPTeamSubscription,
- "__omp_rtl_assume_teams_oversubscription");
- OMPBuilder.createGlobalFlag(CGM.getLangOpts().OpenMPThreadSubscription,
- "__omp_rtl_assume_threads_oversubscription");
- }
+ if (CGM.getLangOpts().NoGPULib || CGM.getLangOpts().OMPHostIRFile.empty())
+ return;
+
+ OMPBuilder.createGlobalFlag(CGM.getLangOpts().OpenMPTargetDebug,
+ "__omp_rtl_debug_kind");
+ OMPBuilder.createGlobalFlag(CGM.getLangOpts().OpenMPTeamSubscription,
+ "__omp_rtl_assume_teams_oversubscription");
+ OMPBuilder.createGlobalFlag(CGM.getLangOpts().OpenMPThreadSubscription,
+ "__omp_rtl_assume_threads_oversubscription");
+ OMPBuilder.createGlobalFlag(CGM.getLangOpts().OpenMPNoThreadState,
+ "__omp_rtl_assume_no_thread_state");
}
void CGOpenMPRuntimeGPU::emitProcBindClause(CodeGenFunction &CGF,
@@ -1715,7 +1717,8 @@ static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val,
CastTy->hasSignedIntegerRepresentation());
Address CastItem = CGF.CreateMemTemp(CastTy);
Address ValCastItem = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CastItem, Val->getType()->getPointerTo(CastItem.getAddressSpace()));
+ CastItem, Val->getType()->getPointerTo(CastItem.getAddressSpace()),
+ Val->getType());
CGF.EmitStoreOfScalar(Val, ValCastItem, /*Volatile=*/false, ValTy,
LValueBaseInfo(AlignmentSource::Type),
TBAAAccessInfo());
@@ -1778,7 +1781,7 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr,
Address ElemPtr = DestAddr;
Address Ptr = SrcAddr;
Address PtrEnd = Bld.CreatePointerBitCastOrAddrSpaceCast(
- Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy);
+ Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy, CGF.Int8Ty);
for (int IntSize = 8; IntSize >= 1; IntSize /= 2) {
if (Size < CharUnits::fromQuantity(IntSize))
continue;
@@ -1786,9 +1789,10 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr,
CGF.getContext().toBits(CharUnits::fromQuantity(IntSize)),
/*Signed=*/1);
llvm::Type *IntTy = CGF.ConvertTypeForMem(IntType);
- Ptr = Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr, IntTy->getPointerTo());
- ElemPtr =
- Bld.CreatePointerBitCastOrAddrSpaceCast(ElemPtr, IntTy->getPointerTo());
+ Ptr = Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr, IntTy->getPointerTo(),
+ IntTy);
+ ElemPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
+ ElemPtr, IntTy->getPointerTo(), IntTy);
if (Size.getQuantity() / IntSize > 1) {
llvm::BasicBlock *PreCondBB = CGF.createBasicBlock(".shuffle.pre_cond");
llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".shuffle.then");
@@ -1801,8 +1805,9 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr,
llvm::PHINode *PhiDest =
Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2);
PhiDest->addIncoming(ElemPtr.getPointer(), CurrentBB);
- Ptr = Address(PhiSrc, Ptr.getAlignment());
- ElemPtr = Address(PhiDest, ElemPtr.getAlignment());
+ Ptr = Address(PhiSrc, Ptr.getElementType(), Ptr.getAlignment());
+ ElemPtr =
+ Address(PhiDest, ElemPtr.getElementType(), ElemPtr.getAlignment());
llvm::Value *PtrDiff = Bld.CreatePtrDiff(
CGF.Int8Ty, PtrEnd.getPointer(),
Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr.getPointer(),
@@ -1895,14 +1900,17 @@ static void emitReductionListCopy(
// new element.
bool IncrScratchpadSrc = false;
bool IncrScratchpadDest = false;
+ QualType PrivatePtrType = C.getPointerType(Private->getType());
+ llvm::Type *PrivateLlvmPtrType = CGF.ConvertType(PrivatePtrType);
switch (Action) {
case RemoteLaneToThread: {
// Step 1.1: Get the address for the src element in the Reduce list.
Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
- SrcElementAddr = CGF.EmitLoadOfPointer(
- SrcElementPtrAddr,
- C.getPointerType(Private->getType())->castAs<PointerType>());
+ SrcElementAddr =
+ CGF.EmitLoadOfPointer(CGF.Builder.CreateElementBitCast(
+ SrcElementPtrAddr, PrivateLlvmPtrType),
+ PrivatePtrType->castAs<PointerType>());
// Step 1.2: Create a temporary to store the element in the destination
// Reduce list.
@@ -1916,24 +1924,27 @@ static void emitReductionListCopy(
case ThreadCopy: {
// Step 1.1: Get the address for the src element in the Reduce list.
Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
- SrcElementAddr = CGF.EmitLoadOfPointer(
- SrcElementPtrAddr,
- C.getPointerType(Private->getType())->castAs<PointerType>());
+ SrcElementAddr =
+ CGF.EmitLoadOfPointer(CGF.Builder.CreateElementBitCast(
+ SrcElementPtrAddr, PrivateLlvmPtrType),
+ PrivatePtrType->castAs<PointerType>());
// Step 1.2: Get the address for dest element. The destination
// element has already been created on the thread's stack.
DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
- DestElementAddr = CGF.EmitLoadOfPointer(
- DestElementPtrAddr,
- C.getPointerType(Private->getType())->castAs<PointerType>());
+ DestElementAddr =
+ CGF.EmitLoadOfPointer(CGF.Builder.CreateElementBitCast(
+ DestElementPtrAddr, PrivateLlvmPtrType),
+ PrivatePtrType->castAs<PointerType>());
break;
}
case ThreadToScratchpad: {
// Step 1.1: Get the address for the src element in the Reduce list.
Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
- SrcElementAddr = CGF.EmitLoadOfPointer(
- SrcElementPtrAddr,
- C.getPointerType(Private->getType())->castAs<PointerType>());
+ SrcElementAddr =
+ CGF.EmitLoadOfPointer(CGF.Builder.CreateElementBitCast(
+ SrcElementPtrAddr, PrivateLlvmPtrType),
+ PrivatePtrType->castAs<PointerType>());
// Step 1.2: Get the address for dest element:
// address = base + index * ElementSizeInChars.
@@ -1944,7 +1955,7 @@ static void emitReductionListCopy(
Bld.CreateNUWAdd(DestBase.getPointer(), CurrentOffset);
ScratchPadElemAbsolutePtrVal =
Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
- DestElementAddr = Address(ScratchPadElemAbsolutePtrVal,
+ DestElementAddr = Address(ScratchPadElemAbsolutePtrVal, CGF.Int8Ty,
C.getTypeAlignInChars(Private->getType()));
IncrScratchpadDest = true;
break;
@@ -1959,7 +1970,7 @@ static void emitReductionListCopy(
Bld.CreateNUWAdd(SrcBase.getPointer(), CurrentOffset);
ScratchPadElemAbsolutePtrVal =
Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
- SrcElementAddr = Address(ScratchPadElemAbsolutePtrVal,
+ SrcElementAddr = Address(ScratchPadElemAbsolutePtrVal, CGF.Int8Ty,
C.getTypeAlignInChars(Private->getType()));
IncrScratchpadSrc = true;
@@ -2032,6 +2043,8 @@ static void emitReductionListCopy(
// address of the next element in scratchpad memory, unless we're currently
// processing the last one. Memory alignment is also taken care of here.
if ((IncrScratchpadDest || IncrScratchpadSrc) && (Idx + 1 < Size)) {
+ // FIXME: This code doesn't make any sense, it's trying to perform
+ // integer arithmetic on pointers.
llvm::Value *ScratchpadBasePtr =
IncrScratchpadDest ? DestBase.getPointer() : SrcBase.getPointer();
llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
@@ -2052,9 +2065,11 @@ static void emitReductionListCopy(
llvm::ConstantInt::get(CGM.SizeTy, GlobalMemoryAlignment));
if (IncrScratchpadDest)
- DestBase = Address(ScratchpadBasePtr, CGF.getPointerAlign());
+ DestBase =
+ Address(ScratchpadBasePtr, CGF.VoidPtrTy, CGF.getPointerAlign());
else /* IncrScratchpadSrc = true */
- SrcBase = Address(ScratchpadBasePtr, CGF.getPointerAlign());
+ SrcBase =
+ Address(ScratchpadBasePtr, CGF.VoidPtrTy, CGF.getPointerAlign());
}
++Idx;
@@ -2138,13 +2153,14 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
llvm::Value *WarpID = getNVPTXWarpID(CGF);
Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
+ llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy);
Address LocalReduceList(
Bld.CreatePointerBitCastOrAddrSpaceCast(
CGF.EmitLoadOfScalar(
AddrReduceListArg, /*Volatile=*/false, C.VoidPtrTy, Loc,
LValueBaseInfo(AlignmentSource::Type), TBAAAccessInfo()),
- CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()),
- CGF.getPointerAlign());
+ ElemTy->getPointerTo()),
+ ElemTy, CGF.getPointerAlign());
unsigned Idx = 0;
for (const Expr *Private : Privates) {
@@ -2202,7 +2218,7 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar(
ElemPtrPtrAddr, /*Volatile=*/false, C.VoidPtrTy, SourceLocation());
// elemptr = ((CopyType*)(elemptrptr)) + I
- Address ElemPtr = Address(ElemPtrPtr, Align);
+ Address ElemPtr(ElemPtrPtr, CGF.Int8Ty, Align);
ElemPtr = Bld.CreateElementBitCast(ElemPtr, CopyType);
if (NumIters > 1)
ElemPtr = Bld.CreateGEP(ElemPtr, Cnt);
@@ -2212,10 +2228,14 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
llvm::Value *MediumPtrVal = Bld.CreateInBoundsGEP(
TransferMedium->getValueType(), TransferMedium,
{llvm::Constant::getNullValue(CGM.Int64Ty), WarpID});
- Address MediumPtr(MediumPtrVal, Align);
// Casting to actual data type.
// MediumPtr = (CopyType*)MediumPtrAddr;
- MediumPtr = Bld.CreateElementBitCast(MediumPtr, CopyType);
+ Address MediumPtr(
+ Bld.CreateBitCast(
+ MediumPtrVal,
+ CopyType->getPointerTo(
+ MediumPtrVal->getType()->getPointerAddressSpace())),
+ CopyType, Align);
// elem = *elemptr
//*MediumPtr = elem
@@ -2261,15 +2281,19 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
llvm::Value *SrcMediumPtrVal = Bld.CreateInBoundsGEP(
TransferMedium->getValueType(), TransferMedium,
{llvm::Constant::getNullValue(CGM.Int64Ty), ThreadID});
- Address SrcMediumPtr(SrcMediumPtrVal, Align);
// SrcMediumVal = *SrcMediumPtr;
- SrcMediumPtr = Bld.CreateElementBitCast(SrcMediumPtr, CopyType);
+ Address SrcMediumPtr(
+ Bld.CreateBitCast(
+ SrcMediumPtrVal,
+ CopyType->getPointerTo(
+ SrcMediumPtrVal->getType()->getPointerAddressSpace())),
+ CopyType, Align);
// TargetElemPtr = (CopyType*)(SrcDataAddr[i]) + I
Address TargetElemPtrPtr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
llvm::Value *TargetElemPtrVal = CGF.EmitLoadOfScalar(
TargetElemPtrPtr, /*Volatile=*/false, C.VoidPtrTy, Loc);
- Address TargetElemPtr = Address(TargetElemPtrVal, Align);
+ Address TargetElemPtr(TargetElemPtrVal, CGF.Int8Ty, Align);
TargetElemPtr = Bld.CreateElementBitCast(TargetElemPtr, CopyType);
if (NumIters > 1)
TargetElemPtr = Bld.CreateGEP(TargetElemPtr, Cnt);
@@ -2405,12 +2429,13 @@ static llvm::Function *emitShuffleAndReduceFunction(
CGBuilderTy &Bld = CGF.Builder;
Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
+ llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy);
Address LocalReduceList(
Bld.CreatePointerBitCastOrAddrSpaceCast(
CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false,
C.VoidPtrTy, SourceLocation()),
- CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()),
- CGF.getPointerAlign());
+ ElemTy->getPointerTo()),
+ ElemTy, CGF.getPointerAlign());
Address AddrLaneIDArg = CGF.GetAddrOfLocalVar(&LaneIDArg);
llvm::Value *LaneIDArgVal = CGF.EmitLoadOfScalar(
@@ -2561,12 +2586,13 @@ static llvm::Value *emitListToGlobalCopyFunction(
Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg);
+ llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy);
Address LocalReduceList(
Bld.CreatePointerBitCastOrAddrSpaceCast(
CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false,
C.VoidPtrTy, Loc),
- CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()),
- CGF.getPointerAlign());
+ ElemTy->getPointerTo()),
+ ElemTy, CGF.getPointerAlign());
QualType StaticTy = C.getRecordType(TeamReductionRec);
llvm::Type *LLVMReductionsBufferTy =
CGM.getTypes().ConvertTypeForMem(StaticTy);
@@ -2584,19 +2610,22 @@ static llvm::Value *emitListToGlobalCopyFunction(
llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar(
ElemPtrPtrAddr, /*Volatile=*/false, C.VoidPtrTy, SourceLocation());
// elemptr = ((CopyType*)(elemptrptr)) + I
+ ElemTy = CGF.ConvertTypeForMem(Private->getType());
ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- ElemPtrPtr, CGF.ConvertTypeForMem(Private->getType())->getPointerTo());
+ ElemPtrPtr, ElemTy->getPointerTo());
Address ElemPtr =
- Address(ElemPtrPtr, C.getTypeAlignInChars(Private->getType()));
+ Address(ElemPtrPtr, ElemTy, C.getTypeAlignInChars(Private->getType()));
const ValueDecl *VD = cast<DeclRefExpr>(Private)->getDecl();
// Global = Buffer.VD[Idx];
const FieldDecl *FD = VarFieldMap.lookup(VD);
LValue GlobLVal = CGF.EmitLValueForField(
CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD);
Address GlobAddr = GlobLVal.getAddress(CGF);
- llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(
- GlobAddr.getElementType(), GlobAddr.getPointer(), Idxs);
- GlobLVal.setAddress(Address(BufferPtr, GlobAddr.getAlignment()));
+ llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(GlobAddr.getElementType(),
+ GlobAddr.getPointer(), Idxs);
+ GlobLVal.setAddress(Address(BufferPtr,
+ CGF.ConvertTypeForMem(Private->getType()),
+ GlobAddr.getAlignment()));
switch (CGF.getEvaluationKind(Private->getType())) {
case TEK_Scalar: {
llvm::Value *V = CGF.EmitLoadOfScalar(
@@ -2766,12 +2795,13 @@ static llvm::Value *emitGlobalToListCopyFunction(
Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg);
+ llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy);
Address LocalReduceList(
Bld.CreatePointerBitCastOrAddrSpaceCast(
CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false,
C.VoidPtrTy, Loc),
- CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()),
- CGF.getPointerAlign());
+ ElemTy->getPointerTo()),
+ ElemTy, CGF.getPointerAlign());
QualType StaticTy = C.getRecordType(TeamReductionRec);
llvm::Type *LLVMReductionsBufferTy =
CGM.getTypes().ConvertTypeForMem(StaticTy);
@@ -2790,19 +2820,22 @@ static llvm::Value *emitGlobalToListCopyFunction(
llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar(
ElemPtrPtrAddr, /*Volatile=*/false, C.VoidPtrTy, SourceLocation());
// elemptr = ((CopyType*)(elemptrptr)) + I
+ ElemTy = CGF.ConvertTypeForMem(Private->getType());
ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- ElemPtrPtr, CGF.ConvertTypeForMem(Private->getType())->getPointerTo());
+ ElemPtrPtr, ElemTy->getPointerTo());
Address ElemPtr =
- Address(ElemPtrPtr, C.getTypeAlignInChars(Private->getType()));
+ Address(ElemPtrPtr, ElemTy, C.getTypeAlignInChars(Private->getType()));
const ValueDecl *VD = cast<DeclRefExpr>(Private)->getDecl();
// Global = Buffer.VD[Idx];
const FieldDecl *FD = VarFieldMap.lookup(VD);
LValue GlobLVal = CGF.EmitLValueForField(
CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD);
Address GlobAddr = GlobLVal.getAddress(CGF);
- llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(
- GlobAddr.getElementType(), GlobAddr.getPointer(), Idxs);
- GlobLVal.setAddress(Address(BufferPtr, GlobAddr.getAlignment()));
+ llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(GlobAddr.getElementType(),
+ GlobAddr.getPointer(), Idxs);
+ GlobLVal.setAddress(Address(BufferPtr,
+ CGF.ConvertTypeForMem(Private->getType()),
+ GlobAddr.getAlignment()));
switch (CGF.getEvaluationKind(Private->getType())) {
case TEK_Scalar: {
llvm::Value *V = CGF.EmitLoadOfScalar(GlobLVal, Loc);
@@ -3242,9 +3275,9 @@ void CGOpenMPRuntimeGPU::emitReduction(
llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
ReductionList.getPointer(), CGF.VoidPtrTy);
- llvm::Function *ReductionFn = emitReductionFunction(
- Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
- LHSExprs, RHSExprs, ReductionOps);
+ llvm::Function *ReductionFn =
+ emitReductionFunction(Loc, CGF.ConvertTypeForMem(ReductionArrayTy),
+ Privates, LHSExprs, RHSExprs, ReductionOps);
llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
llvm::Function *ShuffleAndReduceFn = emitShuffleAndReduceFunction(
CGM, Privates, ReductionArrayTy, ReductionFn, Loc);
@@ -3284,7 +3317,7 @@ void CGOpenMPRuntimeGPU::emitReduction(
"_openmp_teams_reductions_buffer_$_$ptr");
}
llvm::Value *GlobalBufferPtr = CGF.EmitLoadOfScalar(
- Address(KernelTeamsReductionPtr, CGM.getPointerAlign()),
+ Address(KernelTeamsReductionPtr, CGF.VoidPtrTy, CGM.getPointerAlign()),
/*Volatile=*/false, C.getPointerType(C.VoidPtrTy), Loc);
llvm::Value *GlobalToBufferCpyFn = ::emitListToGlobalCopyFunction(
CGM, Privates, ReductionArrayTy, Loc, TeamReductionRec, VarFieldMap);
@@ -3526,15 +3559,14 @@ llvm::Function *CGOpenMPRuntimeGPU::createParallelDataSharingWrapper(
isOpenMPLoopBoundSharingDirective(D.getDirectiveKind())) {
SharedArgListAddress = CGF.EmitLoadOfPointer(
GlobalArgs, CGF.getContext()
- .getPointerType(CGF.getContext().getPointerType(
- CGF.getContext().VoidPtrTy))
+ .getPointerType(CGF.getContext().VoidPtrTy)
.castAs<PointerType>());
}
unsigned Idx = 0;
if (isOpenMPLoopBoundSharingDirective(D.getDirectiveKind())) {
Address Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx);
Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast(
- Src, CGF.SizeTy->getPointerTo());
+ Src, CGF.SizeTy->getPointerTo(), CGF.SizeTy);
llvm::Value *LB = CGF.EmitLoadOfScalar(
TypedAddress,
/*Volatile=*/false,
@@ -3544,7 +3576,7 @@ llvm::Function *CGOpenMPRuntimeGPU::createParallelDataSharingWrapper(
++Idx;
Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx);
TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast(
- Src, CGF.SizeTy->getPointerTo());
+ Src, CGF.SizeTy->getPointerTo(), CGF.SizeTy);
llvm::Value *UB = CGF.EmitLoadOfScalar(
TypedAddress,
/*Volatile=*/false,
@@ -3559,7 +3591,8 @@ llvm::Function *CGOpenMPRuntimeGPU::createParallelDataSharingWrapper(
QualType ElemTy = CurField->getType();
Address Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, I + Idx);
Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast(
- Src, CGF.ConvertTypeForMem(CGFContext.getPointerType(ElemTy)));
+ Src, CGF.ConvertTypeForMem(CGFContext.getPointerType(ElemTy)),
+ CGF.ConvertTypeForMem(ElemTy));
llvm::Value *Arg = CGF.EmitLoadOfScalar(TypedAddress,
/*Volatile=*/false,
CGFContext.getPointerType(ElemTy),
@@ -3630,7 +3663,7 @@ void CGOpenMPRuntimeGPU::emitFunctionProlog(CodeGenFunction &CGF,
CheckVarsEscapingDeclContext VarChecker(CGF, llvm::None);
VarChecker.Visit(Body);
I->getSecond().SecondaryLocalVarData.emplace();
- DeclToAddrMapTy &Data = I->getSecond().SecondaryLocalVarData.getValue();
+ DeclToAddrMapTy &Data = *I->getSecond().SecondaryLocalVarData;
for (const ValueDecl *VD : VarChecker.getEscapedDecls()) {
assert(VD->isCanonicalDecl() && "Expected canonical declaration");
Data.insert(std::make_pair(VD, MappedVarData()));
@@ -3691,7 +3724,7 @@ Address CGOpenMPRuntimeGPU::getAddressOfLocalVariable(CodeGenFunction &CGF,
CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
GV, VarTy->getPointerTo(CGM.getContext().getTargetAddressSpace(
VD->getType().getAddressSpace()))),
- Align);
+ VarTy, Align);
}
if (getDataSharingMode(CGM) != CGOpenMPRuntimeGPU::Generic)
@@ -3902,6 +3935,7 @@ void CGOpenMPRuntimeGPU::processRequiresDirective(
case CudaArch::GFX909:
case CudaArch::GFX90a:
case CudaArch::GFX90c:
+ case CudaArch::GFX940:
case CudaArch::GFX1010:
case CudaArch::GFX1011:
case CudaArch::GFX1012:
@@ -3912,6 +3946,11 @@ void CGOpenMPRuntimeGPU::processRequiresDirective(
case CudaArch::GFX1033:
case CudaArch::GFX1034:
case CudaArch::GFX1035:
+ case CudaArch::GFX1036:
+ case CudaArch::GFX1100:
+ case CudaArch::GFX1101:
+ case CudaArch::GFX1102:
+ case CudaArch::GFX1103:
case CudaArch::Generic:
case CudaArch::UNUSED:
case CudaArch::UNKNOWN:
diff --git a/clang/lib/CodeGen/CGRecordLayout.h b/clang/lib/CodeGen/CGRecordLayout.h
index 5a3bcdf72f7b..d5ea74922603 100644
--- a/clang/lib/CodeGen/CGRecordLayout.h
+++ b/clang/lib/CodeGen/CGRecordLayout.h
@@ -200,6 +200,12 @@ public:
return FieldInfo.lookup(FD);
}
+ // Return whether the following non virtual base has a corresponding
+ // entry in the LLVM struct.
+ bool hasNonVirtualBaseLLVMField(const CXXRecordDecl *RD) const {
+ return NonVirtualBases.count(RD);
+ }
+
unsigned getNonVirtualBaseLLVMFieldNo(const CXXRecordDecl *RD) const {
assert(NonVirtualBases.count(RD) && "Invalid non-virtual base!");
return NonVirtualBases.lookup(RD);
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 9e939bb545ad..df7e5608f8f0 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -314,18 +314,31 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
case Stmt::OMPMasterTaskLoopDirectiveClass:
EmitOMPMasterTaskLoopDirective(cast<OMPMasterTaskLoopDirective>(*S));
break;
+ case Stmt::OMPMaskedTaskLoopDirectiveClass:
+ llvm_unreachable("masked taskloop directive not supported yet.");
+ break;
case Stmt::OMPMasterTaskLoopSimdDirectiveClass:
EmitOMPMasterTaskLoopSimdDirective(
cast<OMPMasterTaskLoopSimdDirective>(*S));
break;
+ case Stmt::OMPMaskedTaskLoopSimdDirectiveClass:
+ llvm_unreachable("masked taskloop simd directive not supported yet.");
+ break;
case Stmt::OMPParallelMasterTaskLoopDirectiveClass:
EmitOMPParallelMasterTaskLoopDirective(
cast<OMPParallelMasterTaskLoopDirective>(*S));
break;
+ case Stmt::OMPParallelMaskedTaskLoopDirectiveClass:
+ llvm_unreachable("parallel masked taskloop directive not supported yet.");
+ break;
case Stmt::OMPParallelMasterTaskLoopSimdDirectiveClass:
EmitOMPParallelMasterTaskLoopSimdDirective(
cast<OMPParallelMasterTaskLoopSimdDirective>(*S));
break;
+ case Stmt::OMPParallelMaskedTaskLoopSimdDirectiveClass:
+ llvm_unreachable(
+ "parallel masked taskloop simd directive not supported yet.");
+ break;
case Stmt::OMPDistributeDirectiveClass:
EmitOMPDistributeDirective(cast<OMPDistributeDirective>(*S));
break;
@@ -396,6 +409,21 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
case Stmt::OMPGenericLoopDirectiveClass:
EmitOMPGenericLoopDirective(cast<OMPGenericLoopDirective>(*S));
break;
+ case Stmt::OMPTeamsGenericLoopDirectiveClass:
+ llvm_unreachable("teams loop directive not supported yet.");
+ break;
+ case Stmt::OMPTargetTeamsGenericLoopDirectiveClass:
+ llvm_unreachable("target teams loop directive not supported yet.");
+ break;
+ case Stmt::OMPParallelGenericLoopDirectiveClass:
+ llvm_unreachable("parallel loop directive not supported yet.");
+ break;
+ case Stmt::OMPTargetParallelGenericLoopDirectiveClass:
+ llvm_unreachable("target parallel loop directive not supported yet.");
+ break;
+ case Stmt::OMPParallelMaskedDirectiveClass:
+ llvm_unreachable("parallel masked directive not supported yet.");
+ break;
}
}
@@ -666,19 +694,34 @@ void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {
bool nomerge = false;
+ bool noinline = false;
+ bool alwaysinline = false;
const CallExpr *musttail = nullptr;
for (const auto *A : S.getAttrs()) {
- if (A->getKind() == attr::NoMerge) {
+ switch (A->getKind()) {
+ default:
+ break;
+ case attr::NoMerge:
nomerge = true;
- }
- if (A->getKind() == attr::MustTail) {
+ break;
+ case attr::NoInline:
+ noinline = true;
+ break;
+ case attr::AlwaysInline:
+ alwaysinline = true;
+ break;
+ case attr::MustTail:
const Stmt *Sub = S.getSubStmt();
const ReturnStmt *R = cast<ReturnStmt>(Sub);
musttail = cast<CallExpr>(R->getRetValue()->IgnoreParens());
+ break;
}
}
SaveAndRestore<bool> save_nomerge(InNoMergeAttributedStmt, nomerge);
+ SaveAndRestore<bool> save_noinline(InNoInlineAttributedStmt, noinline);
+ SaveAndRestore<bool> save_alwaysinline(InAlwaysInlineAttributedStmt,
+ alwaysinline);
SaveAndRestore<const CallExpr *> save_musttail(MustTailCall, musttail);
EmitStmt(S.getSubStmt(), S.getAttrs());
}
@@ -2121,10 +2164,9 @@ std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue(
if ((Size <= 64 && llvm::isPowerOf2_64(Size)) ||
getTargetHooks().isScalarizableAsmOperand(*this, Ty)) {
Ty = llvm::IntegerType::get(getLLVMContext(), Size);
- Ty = llvm::PointerType::getUnqual(Ty);
- return {Builder.CreateLoad(
- Builder.CreateBitCast(InputValue.getAddress(*this), Ty)),
+ return {Builder.CreateLoad(Builder.CreateElementBitCast(
+ InputValue.getAddress(*this), Ty)),
nullptr};
}
}
@@ -2260,6 +2302,9 @@ static void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,
}
void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
+ // Pop all cleanup blocks at the end of the asm statement.
+ CodeGenFunction::RunCleanupsScope Cleanups(*this);
+
// Assemble the final asm string.
std::string AsmString = S.generateAsmString(getContext());
@@ -2514,10 +2559,8 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
Arg = Builder.CreateZExt(Arg, OutputTy);
else if (isa<llvm::PointerType>(OutputTy))
Arg = Builder.CreateZExt(Arg, IntPtrTy);
- else {
- assert(OutputTy->isFloatingPointTy() && "Unexpected output type");
+ else if (OutputTy->isFloatingPointTy())
Arg = Builder.CreateFPExt(Arg, OutputTy);
- }
}
// Deal with the tied operands' constraint code in adjustInlineAsmType.
ReplaceConstraint = OutputConstraints[Output];
@@ -2713,8 +2756,8 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
// ResultTypeRequiresCast.size() elements of RegResults.
if ((i < ResultTypeRequiresCast.size()) && ResultTypeRequiresCast[i]) {
unsigned Size = getContext().getTypeSize(ResultRegQualTys[i]);
- Address A = Builder.CreateBitCast(Dest.getAddress(*this),
- ResultRegTypes[i]->getPointerTo());
+ Address A = Builder.CreateElementBitCast(Dest.getAddress(*this),
+ ResultRegTypes[i]);
if (getTargetHooks().isScalarizableAsmOperand(*this, TruncTy)) {
Builder.CreateStore(Tmp, A);
continue;
@@ -2723,9 +2766,8 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
QualType Ty = getContext().getIntTypeForBitwidth(Size, /*Signed*/ false);
if (Ty.isNull()) {
const Expr *OutExpr = S.getOutputExpr(i);
- CGM.Error(
- OutExpr->getExprLoc(),
- "impossible constraint in asm: can't store value into a register");
+ CGM.getDiags().Report(OutExpr->getExprLoc(),
+ diag::err_store_value_to_reg);
return;
}
Dest = MakeAddrLValue(A, Ty);
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 39dd4c00765d..301f5278df69 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -24,6 +24,7 @@
#include "clang/AST/StmtVisitor.h"
#include "clang/Basic/OpenMPKinds.h"
#include "clang/Basic/PrettyStackTrace.h"
+#include "llvm/ADT/SmallSet.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/Frontend/OpenMP/OMPConstants.h"
#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
@@ -79,7 +80,7 @@ public:
InlinedShareds(CGF) {
if (EmitPreInitStmt)
emitPreInitStmt(CGF, S);
- if (!CapturedRegion.hasValue())
+ if (!CapturedRegion)
return;
assert(S.hasAssociatedStmt() &&
"Expected associated statement for inlined directive.");
@@ -94,9 +95,7 @@ public:
isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
InlinedShareds.isGlobalVarCaptured(VD)),
VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
- InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
- return CGF.EmitLValue(&DRE).getAddress(CGF);
- });
+ InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF));
}
}
(void)InlinedShareds.Privatize();
@@ -154,11 +153,12 @@ class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
const auto *OrigVD =
cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
+ QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
(void)PreCondVars.setVarAddr(
CGF, OrigVD,
Address(llvm::UndefValue::get(CGF.ConvertTypeForMem(
- CGF.getContext().getPointerType(
- OrigVD->getType().getNonReferenceType()))),
+ CGF.getContext().getPointerType(OrigVDTy))),
+ CGF.ConvertTypeForMem(OrigVDTy),
CGF.getContext().getDeclAlign(OrigVD)));
}
}
@@ -271,9 +271,7 @@ public:
InlinedShareds.isGlobalVarCaptured(VD)),
VD->getType().getNonReferenceType(), VK_LValue,
C.getLocation());
- InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
- return CGF.EmitLValue(&DRE).getAddress(CGF);
- });
+ InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF));
}
}
CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
@@ -483,7 +481,11 @@ static llvm::Function *emitOutlinedFunctionPrologue(
if (ArgType->isVariablyModifiedType())
ArgType = getCanonicalParamType(Ctx, ArgType);
VarDecl *Arg;
- if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
+ if (CapVar && (CapVar->getTLSKind() != clang::VarDecl::TLS_None)) {
+ Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
+ II, ArgType,
+ ImplicitParamDecl::ThreadPrivateVar);
+ } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
Arg = ParmVarDecl::Create(
Ctx, DebugFunctionDecl,
CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
@@ -578,8 +580,7 @@ static llvm::Function *emitOutlinedFunctionPrologue(
}
if (!FO.RegisterCastedArgsOnly) {
LocalAddrs.insert(
- {Args[Cnt],
- {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
+ {Args[Cnt], {Var, ArgAddr.withAlignment(Ctx.getDeclAlign(Var))}});
}
} else if (I->capturesVariableByCopy()) {
assert(!FD->getType()->isAnyPointerType() &&
@@ -629,9 +630,8 @@ CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
CodeGenFunction::OMPPrivateScope LocalScope(*this);
for (const auto &LocalAddrPair : LocalAddrs) {
if (LocalAddrPair.second.first) {
- LocalScope.addPrivate(LocalAddrPair.second.first, [&LocalAddrPair]() {
- return LocalAddrPair.second.second;
- });
+ LocalScope.addPrivate(LocalAddrPair.second.first,
+ LocalAddrPair.second.second);
}
}
(void)LocalScope.Privatize();
@@ -669,7 +669,8 @@ CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
LV.setAddress(WrapperCGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
LV.getAddress(WrapperCGF),
PI->getType()->getPointerTo(
- LV.getAddress(WrapperCGF).getAddressSpace())));
+ LV.getAddress(WrapperCGF).getAddressSpace()),
+ PI->getType()));
CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
} else {
auto EI = VLASizes.find(Arg);
@@ -726,14 +727,14 @@ void CodeGenFunction::EmitOMPAggregateAssign(
Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
SrcElementPHI->addIncoming(SrcBegin, EntryBB);
Address SrcElementCurrent =
- Address(SrcElementPHI,
+ Address(SrcElementPHI, SrcAddr.getElementType(),
SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
llvm::PHINode *DestElementPHI = Builder.CreatePHI(
DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
DestElementPHI->addIncoming(DestBegin, EntryBB);
Address DestElementCurrent =
- Address(DestElementPHI,
+ Address(DestElementPHI, DestAddr.getElementType(),
DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
// Emit copy.
@@ -777,8 +778,8 @@ void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
// destination and source variables to corresponding array
// elements.
CodeGenFunction::OMPPrivateScope Remap(*this);
- Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
- Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
+ Remap.addPrivate(DestVD, DestElement);
+ Remap.addPrivate(SrcVD, SrcElement);
(void)Remap.Privatize();
EmitIgnoredExpr(Copy);
});
@@ -786,8 +787,8 @@ void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
} else {
// Remap pseudo source variable to private copy.
CodeGenFunction::OMPPrivateScope Remap(*this);
- Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
- Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
+ Remap.addPrivate(SrcVD, SrcAddr);
+ Remap.addPrivate(DestVD, DestAddr);
(void)Remap.Privatize();
// Emit copying of the whole variable.
EmitIgnoredExpr(Copy);
@@ -876,68 +877,56 @@ bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
// Emit VarDecl with copy init for arrays.
// Get the address of the original variable captured in current
// captured region.
- IsRegistered = PrivateScope.addPrivate(
- OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
- AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
- const Expr *Init = VD->getInit();
- if (!isa<CXXConstructExpr>(Init) ||
- isTrivialInitializer(Init)) {
- // Perform simple memcpy.
- LValue Dest =
- MakeAddrLValue(Emission.getAllocatedAddress(), Type);
- EmitAggregateAssign(Dest, OriginalLVal, Type);
- } else {
- EmitOMPAggregateAssign(
- Emission.getAllocatedAddress(),
- OriginalLVal.getAddress(*this), Type,
- [this, VDInit, Init](Address DestElement,
- Address SrcElement) {
- // Clean up any temporaries needed by the
- // initialization.
- RunCleanupsScope InitScope(*this);
- // Emit initialization for single element.
- setAddrOfLocalVar(VDInit, SrcElement);
- EmitAnyExprToMem(Init, DestElement,
- Init->getType().getQualifiers(),
- /*IsInitializer*/ false);
- LocalDeclMap.erase(VDInit);
- });
- }
- EmitAutoVarCleanups(Emission);
- return Emission.getAllocatedAddress();
- });
+ AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
+ const Expr *Init = VD->getInit();
+ if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
+ // Perform simple memcpy.
+ LValue Dest = MakeAddrLValue(Emission.getAllocatedAddress(), Type);
+ EmitAggregateAssign(Dest, OriginalLVal, Type);
+ } else {
+ EmitOMPAggregateAssign(
+ Emission.getAllocatedAddress(), OriginalLVal.getAddress(*this),
+ Type,
+ [this, VDInit, Init](Address DestElement, Address SrcElement) {
+ // Clean up any temporaries needed by the
+ // initialization.
+ RunCleanupsScope InitScope(*this);
+ // Emit initialization for single element.
+ setAddrOfLocalVar(VDInit, SrcElement);
+ EmitAnyExprToMem(Init, DestElement,
+ Init->getType().getQualifiers(),
+ /*IsInitializer*/ false);
+ LocalDeclMap.erase(VDInit);
+ });
+ }
+ EmitAutoVarCleanups(Emission);
+ IsRegistered =
+ PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());
} else {
Address OriginalAddr = OriginalLVal.getAddress(*this);
- IsRegistered =
- PrivateScope.addPrivate(OrigVD, [this, VDInit, OriginalAddr, VD,
- ThisFirstprivateIsLastprivate,
- OrigVD, &Lastprivates, IRef]() {
- // Emit private VarDecl with copy init.
- // Remap temp VDInit variable to the address of the original
- // variable (for proper handling of captured global variables).
- setAddrOfLocalVar(VDInit, OriginalAddr);
- EmitDecl(*VD);
- LocalDeclMap.erase(VDInit);
- if (ThisFirstprivateIsLastprivate &&
- Lastprivates[OrigVD->getCanonicalDecl()] ==
- OMPC_LASTPRIVATE_conditional) {
- // Create/init special variable for lastprivate conditionals.
- Address VDAddr =
- CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
- *this, OrigVD);
- llvm::Value *V = EmitLoadOfScalar(
- MakeAddrLValue(GetAddrOfLocalVar(VD), (*IRef)->getType(),
- AlignmentSource::Decl),
- (*IRef)->getExprLoc());
- EmitStoreOfScalar(V,
- MakeAddrLValue(VDAddr, (*IRef)->getType(),
- AlignmentSource::Decl));
- LocalDeclMap.erase(VD);
- setAddrOfLocalVar(VD, VDAddr);
- return VDAddr;
- }
- return GetAddrOfLocalVar(VD);
- });
+ // Emit private VarDecl with copy init.
+ // Remap temp VDInit variable to the address of the original
+ // variable (for proper handling of captured global variables).
+ setAddrOfLocalVar(VDInit, OriginalAddr);
+ EmitDecl(*VD);
+ LocalDeclMap.erase(VDInit);
+ Address VDAddr = GetAddrOfLocalVar(VD);
+ if (ThisFirstprivateIsLastprivate &&
+ Lastprivates[OrigVD->getCanonicalDecl()] ==
+ OMPC_LASTPRIVATE_conditional) {
+ // Create/init special variable for lastprivate conditionals.
+ llvm::Value *V =
+ EmitLoadOfScalar(MakeAddrLValue(VDAddr, (*IRef)->getType(),
+ AlignmentSource::Decl),
+ (*IRef)->getExprLoc());
+ VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
+ *this, OrigVD);
+ EmitStoreOfScalar(V, MakeAddrLValue(VDAddr, (*IRef)->getType(),
+ AlignmentSource::Decl));
+ LocalDeclMap.erase(VD);
+ setAddrOfLocalVar(VD, VDAddr);
+ }
+ IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
}
assert(IsRegistered &&
"firstprivate var already registered as private");
@@ -963,11 +952,10 @@ void CodeGenFunction::EmitOMPPrivateClause(
const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
- bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
- // Emit private VarDecl with copy init.
- EmitDecl(*VD);
- return GetAddrOfLocalVar(VD);
- });
+ EmitDecl(*VD);
+ // Emit private VarDecl with copy init.
+ bool IsRegistered =
+ PrivateScope.addPrivate(OrigVD, GetAddrOfLocalVar(VD));
assert(IsRegistered && "private var already registered as private");
// Silence the warning about unused variable.
(void)IsRegistered;
@@ -1010,6 +998,7 @@ bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
MasterAddr =
Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
: CGM.GetAddrOfGlobal(VD),
+ CGM.getTypes().ConvertTypeForMem(VD->getType()),
getContext().getDeclAlign(VD));
}
// Get the address of the threadprivate variable.
@@ -1078,31 +1067,27 @@ bool CodeGenFunction::EmitOMPLastprivateClauseInit(
if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
const auto *DestVD =
cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
- PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
- DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
- /*RefersToEnclosingVariableOrCapture=*/
- CapturedStmtInfo->lookup(OrigVD) != nullptr,
- (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
- return EmitLValue(&DRE).getAddress(*this);
- });
+ DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
+ /*RefersToEnclosingVariableOrCapture=*/
+ CapturedStmtInfo->lookup(OrigVD) != nullptr,
+ (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
+ PrivateScope.addPrivate(DestVD, EmitLValue(&DRE).getAddress(*this));
// Check if the variable is also a firstprivate: in this case IInit is
// not generated. Initialization of this variable will happen in codegen
// for 'firstprivate' clause.
if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
- bool IsRegistered =
- PrivateScope.addPrivate(OrigVD, [this, VD, C, OrigVD]() {
- if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
- Address VDAddr =
- CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
- *this, OrigVD);
- setAddrOfLocalVar(VD, VDAddr);
- return VDAddr;
- }
- // Emit private VarDecl with copy init.
- EmitDecl(*VD);
- return GetAddrOfLocalVar(VD);
- });
+ Address VDAddr = Address::invalid();
+ if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
+ VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
+ *this, OrigVD);
+ setAddrOfLocalVar(VD, VDAddr);
+ } else {
+ // Emit private VarDecl with copy init.
+ EmitDecl(*VD);
+ VDAddr = GetAddrOfLocalVar(VD);
+ }
+ bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
assert(IsRegistered &&
"lastprivate var already registered as private");
(void)IsRegistered;
@@ -1182,9 +1167,10 @@ void CodeGenFunction::EmitOMPLastprivateClauseFinal(
// Get the address of the private variable.
Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
- PrivateAddr =
- Address(Builder.CreateLoad(PrivateAddr),
- CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
+ PrivateAddr = Address(
+ Builder.CreateLoad(PrivateAddr),
+ CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),
+ CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
// Store the last value to the private copy in the last iteration.
if (C->getKind() == OMPC_LASTPRIVATE_conditional)
CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
@@ -1256,8 +1242,8 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
EmitAutoVarCleanups(Emission);
Address BaseAddr = RedCG.adjustPrivateAddress(
*this, Count, Emission.getAllocatedAddress());
- bool IsRegistered = PrivateScope.addPrivate(
- RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
+ bool IsRegistered =
+ PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);
assert(IsRegistered && "private var already registered as private");
// Silence the warning about unused variable.
(void)IsRegistered;
@@ -1269,23 +1255,19 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
// Store the address of the original variable associated with the LHS
// implicit variable.
- PrivateScope.addPrivate(LHSVD, [&RedCG, Count, this]() {
- return RedCG.getSharedLValue(Count).getAddress(*this);
- });
- PrivateScope.addPrivate(
- RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
+ PrivateScope.addPrivate(LHSVD,
+ RedCG.getSharedLValue(Count).getAddress(*this));
+ PrivateScope.addPrivate(RHSVD, GetAddrOfLocalVar(PrivateVD));
} else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
isa<ArraySubscriptExpr>(IRef)) {
// Store the address of the original variable associated with the LHS
// implicit variable.
- PrivateScope.addPrivate(LHSVD, [&RedCG, Count, this]() {
- return RedCG.getSharedLValue(Count).getAddress(*this);
- });
- PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
- return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
- ConvertTypeForMem(RHSVD->getType()),
- "rhs.begin");
- });
+ PrivateScope.addPrivate(LHSVD,
+ RedCG.getSharedLValue(Count).getAddress(*this));
+ PrivateScope.addPrivate(RHSVD, Builder.CreateElementBitCast(
+ GetAddrOfLocalVar(PrivateVD),
+ ConvertTypeForMem(RHSVD->getType()),
+ "rhs.begin"));
} else {
QualType Type = PrivateVD->getType();
bool IsArray = getContext().getAsArrayType(Type) != nullptr;
@@ -1296,13 +1278,12 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
OriginalAddr = Builder.CreateElementBitCast(
OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
}
- PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
- PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
- return IsArray ? Builder.CreateElementBitCast(
- GetAddrOfLocalVar(PrivateVD),
- ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
- : GetAddrOfLocalVar(PrivateVD);
- });
+ PrivateScope.addPrivate(LHSVD, OriginalAddr);
+ PrivateScope.addPrivate(
+ RHSVD, IsArray ? Builder.CreateElementBitCast(
+ GetAddrOfLocalVar(PrivateVD),
+ ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
+ : GetAddrOfLocalVar(PrivateVD));
}
++ILHS;
++IRHS;
@@ -1659,7 +1640,7 @@ Address CodeGenFunction::OMPBuilderCBHelpers::getAddressOfLocalVariable(
Addr,
CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())),
getNameWithSeparators({CVD->getName(), ".addr"}, ".", "."));
- return Address(Addr, Align);
+ return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
}
Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
@@ -1682,7 +1663,7 @@ Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
llvm::CallInst *ThreadPrivateCacheCall =
OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);
- return Address(ThreadPrivateCacheCall, VDAddr.getAlignment());
+ return Address(ThreadPrivateCacheCall, CGM.Int8Ty, VDAddr.getAlignment());
}
std::string CodeGenFunction::OMPBuilderCBHelpers::getNameWithSeparators(
@@ -1696,6 +1677,41 @@ std::string CodeGenFunction::OMPBuilderCBHelpers::getNameWithSeparators(
}
return OS.str().str();
}
+
+void CodeGenFunction::OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
+ CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
+ InsertPointTy CodeGenIP, Twine RegionName) {
+ CGBuilderTy &Builder = CGF.Builder;
+ Builder.restoreIP(CodeGenIP);
+ llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
+ "." + RegionName + ".after");
+
+ {
+ OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
+ CGF.EmitStmt(RegionBodyStmt);
+ }
+
+ if (Builder.saveIP().isSet())
+ Builder.CreateBr(FiniBB);
+}
+
+void CodeGenFunction::OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(
+ CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
+ InsertPointTy CodeGenIP, Twine RegionName) {
+ CGBuilderTy &Builder = CGF.Builder;
+ Builder.restoreIP(CodeGenIP);
+ llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
+ "." + RegionName + ".after");
+
+ {
+ OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
+ CGF.EmitStmt(RegionBodyStmt);
+ }
+
+ if (Builder.saveIP().isSet())
+ Builder.CreateBr(FiniBB);
+}
+
void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
if (CGM.getLangOpts().OpenMPIRBuilder) {
llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
@@ -1738,13 +1754,10 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
- auto BodyGenCB = [ParallelRegionBodyStmt,
- this](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
- llvm::BasicBlock &ContinuationBB) {
- OMPBuilderCBHelpers::OutlinedRegionBodyRAII ORB(*this, AllocaIP,
- ContinuationBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, ParallelRegionBodyStmt,
- CodeGenIP, ContinuationBB);
+ auto BodyGenCB = [&, this](InsertPointTy AllocaIP,
+ InsertPointTy CodeGenIP) {
+ OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(
+ *this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP, "parallel");
};
CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
@@ -2181,7 +2194,7 @@ void CodeGenFunction::EmitOMPLinearClauseFinal(
(*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
Address OrigAddr = EmitLValue(&DRE).getAddress(*this);
CodeGenFunction::OMPPrivateScope VarScope(*this);
- VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
+ VarScope.addPrivate(OrigVD, OrigAddr);
(void)VarScope.Privatize();
EmitIgnoredExpr(F);
++IC;
@@ -2240,20 +2253,15 @@ void CodeGenFunction::EmitOMPPrivateLoopCounters(
AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
EmitAutoVarCleanups(VarEmission);
LocalDeclMap.erase(PrivateVD);
- (void)LoopScope.addPrivate(
- VD, [&VarEmission]() { return VarEmission.getAllocatedAddress(); });
+ (void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());
if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
VD->hasGlobalStorage()) {
- (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
- DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
- LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
- E->getType(), VK_LValue, E->getExprLoc());
- return EmitLValue(&DRE).getAddress(*this);
- });
+ DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
+ LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
+ E->getType(), VK_LValue, E->getExprLoc());
+ (void)LoopScope.addPrivate(PrivateVD, EmitLValue(&DRE).getAddress(*this));
} else {
- (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
- return VarEmission.getAllocatedAddress();
- });
+ (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());
}
++I;
}
@@ -2268,9 +2276,8 @@ void CodeGenFunction::EmitOMPPrivateLoopCounters(
// Override only those variables that can be captured to avoid re-emission
// of the variables declared within the loops.
if (DRE->refersToEnclosingVariableOrCapture()) {
- (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
- return CreateMemTemp(DRE->getType(), VD->getName());
- });
+ (void)LoopScope.addPrivate(
+ VD, CreateMemTemp(DRE->getType(), VD->getName()));
}
}
}
@@ -2333,11 +2340,10 @@ void CodeGenFunction::EmitOMPLinearClause(
const auto *PrivateVD =
cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
- bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
- // Emit private VarDecl with copy init.
- EmitVarDecl(*PrivateVD);
- return GetAddrOfLocalVar(PrivateVD);
- });
+ // Emit private VarDecl with copy init.
+ EmitVarDecl(*PrivateVD);
+ bool IsRegistered =
+ PrivateScope.addPrivate(VD, GetAddrOfLocalVar(PrivateVD));
assert(IsRegistered && "linear var already registered as private");
// Silence the warning about unused variable.
(void)IsRegistered;
@@ -2428,7 +2434,7 @@ void CodeGenFunction::EmitOMPSimdFinal(
OrigAddr = EmitLValue(&DRE).getAddress(*this);
}
OMPPrivateScope VarScope(*this);
- VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
+ VarScope.addPrivate(OrigVD, OrigAddr);
(void)VarScope.Privatize();
EmitIgnoredExpr(F);
}
@@ -3509,6 +3515,57 @@ static void emitScanBasedDirectiveDecls(
}
}
+/// Copies final inscan reductions values to the original variables.
+/// The code is the following:
+/// \code
+/// <orig_var> = buffer[num_iters-1];
+/// \endcode
+static void emitScanBasedDirectiveFinals(
+ CodeGenFunction &CGF, const OMPLoopDirective &S,
+ llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
+ llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
+ NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
+ SmallVector<const Expr *, 4> Shareds;
+ SmallVector<const Expr *, 4> LHSs;
+ SmallVector<const Expr *, 4> RHSs;
+ SmallVector<const Expr *, 4> Privates;
+ SmallVector<const Expr *, 4> CopyOps;
+ SmallVector<const Expr *, 4> CopyArrayElems;
+ for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
+ assert(C->getModifier() == OMPC_REDUCTION_inscan &&
+ "Only inscan reductions are expected.");
+ Shareds.append(C->varlist_begin(), C->varlist_end());
+ LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
+ RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
+ Privates.append(C->privates().begin(), C->privates().end());
+ CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
+ CopyArrayElems.append(C->copy_array_elems().begin(),
+ C->copy_array_elems().end());
+ }
+ // Create temp var and copy LHS value to this temp value.
+ // LHS = TMP[LastIter];
+ llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
+ OMPScanNumIterations,
+ llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false));
+ for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
+ const Expr *PrivateExpr = Privates[I];
+ const Expr *OrigExpr = Shareds[I];
+ const Expr *CopyArrayElem = CopyArrayElems[I];
+ CodeGenFunction::OpaqueValueMapping IdxMapping(
+ CGF,
+ cast<OpaqueValueExpr>(
+ cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
+ RValue::get(OMPLast));
+ LValue DestLVal = CGF.EmitLValue(OrigExpr);
+ LValue SrcLVal = CGF.EmitLValue(CopyArrayElem);
+ CGF.EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(CGF),
+ SrcLVal.getAddress(CGF),
+ cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
+ cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
+ CopyOps[I]);
+ }
+}
+
/// Emits the code for the directive with inscan reductions.
/// The code is the following:
/// \code
@@ -3617,7 +3674,7 @@ static void emitScanBasedDirective(
RValue::get(IVal));
LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
}
- PrivScope.addPrivate(LHSVD, [LHSAddr]() { return LHSAddr; });
+ PrivScope.addPrivate(LHSVD, LHSAddr);
Address RHSAddr = Address::invalid();
{
llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
@@ -3628,7 +3685,7 @@ static void emitScanBasedDirective(
RValue::get(OffsetIVal));
RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
}
- PrivScope.addPrivate(RHSVD, [RHSAddr]() { return RHSAddr; });
+ PrivScope.addPrivate(RHSVD, RHSAddr);
++ILHS;
++IRHS;
}
@@ -3703,6 +3760,8 @@ static bool emitWorksharingDirective(CodeGenFunction &CGF,
if (!isOpenMPParallelDirective(S.getDirectiveKind()))
emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
+ if (!isOpenMPParallelDirective(S.getDirectiveKind()))
+ emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);
} else {
CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
HasCancel);
@@ -3716,13 +3775,52 @@ static bool emitWorksharingDirective(CodeGenFunction &CGF,
static bool isSupportedByOpenMPIRBuilder(const OMPForDirective &S) {
if (S.hasCancel())
return false;
- for (OMPClause *C : S.clauses())
- if (!isa<OMPNowaitClause>(C))
- return false;
+ for (OMPClause *C : S.clauses()) {
+ if (isa<OMPNowaitClause>(C))
+ continue;
+
+ if (auto *SC = dyn_cast<OMPScheduleClause>(C)) {
+ if (SC->getFirstScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
+ return false;
+ if (SC->getSecondScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
+ return false;
+ switch (SC->getScheduleKind()) {
+ case OMPC_SCHEDULE_auto:
+ case OMPC_SCHEDULE_dynamic:
+ case OMPC_SCHEDULE_runtime:
+ case OMPC_SCHEDULE_guided:
+ case OMPC_SCHEDULE_static:
+ continue;
+ case OMPC_SCHEDULE_unknown:
+ return false;
+ }
+ }
+
+ return false;
+ }
return true;
}
+static llvm::omp::ScheduleKind
+convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind) {
+ switch (ScheduleClauseKind) {
+ case OMPC_SCHEDULE_unknown:
+ return llvm::omp::OMP_SCHEDULE_Default;
+ case OMPC_SCHEDULE_auto:
+ return llvm::omp::OMP_SCHEDULE_Auto;
+ case OMPC_SCHEDULE_dynamic:
+ return llvm::omp::OMP_SCHEDULE_Dynamic;
+ case OMPC_SCHEDULE_guided:
+ return llvm::omp::OMP_SCHEDULE_Guided;
+ case OMPC_SCHEDULE_runtime:
+ return llvm::omp::OMP_SCHEDULE_Runtime;
+ case OMPC_SCHEDULE_static:
+ return llvm::omp::OMP_SCHEDULE_Static;
+ }
+ llvm_unreachable("Unhandled schedule kind");
+}
+
void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
bool HasLastprivates = false;
bool UseOMPIRBuilder =
@@ -3731,18 +3829,31 @@ void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
// Use the OpenMPIRBuilder if enabled.
if (UseOMPIRBuilder) {
+ bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
+
+ llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
+ llvm::Value *ChunkSize = nullptr;
+ if (auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {
+ SchedKind =
+ convertClauseKindToSchedKind(SchedClause->getScheduleKind());
+ if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
+ ChunkSize = EmitScalarExpr(ChunkSizeExpr);
+ }
+
// Emit the associated statement and get its loop representation.
const Stmt *Inner = S.getRawStmt();
llvm::CanonicalLoopInfo *CLI =
EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
- bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
- OMPBuilder.applyWorkshareLoop(Builder.getCurrentDebugLocation(), CLI,
- AllocaIP, NeedsBarrier);
+ OMPBuilder.applyWorkshareLoop(
+ Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
+ SchedKind, ChunkSize, /*HasSimdModifier=*/false,
+ /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,
+ /*HasOrderedClause=*/false);
return;
}
@@ -3957,22 +4068,17 @@ void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
if (CS) {
for (const Stmt *SubStmt : CS->children()) {
auto SectionCB = [this, SubStmt](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP,
- FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, SubStmt, CodeGenIP,
- FiniBB);
+ InsertPointTy CodeGenIP) {
+ OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
+ *this, SubStmt, AllocaIP, CodeGenIP, "section");
};
SectionCBVector.push_back(SectionCB);
}
} else {
auto SectionCB = [this, CapturedStmt](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP, FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, CapturedStmt, CodeGenIP,
- FiniBB);
+ InsertPointTy CodeGenIP) {
+ OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
+ *this, CapturedStmt, AllocaIP, CodeGenIP, "section");
};
SectionCBVector.push_back(SectionCB);
}
@@ -4025,11 +4131,9 @@ void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
};
auto BodyGenCB = [SectionRegionBodyStmt, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP, FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, SectionRegionBodyStmt,
- CodeGenIP, FiniBB);
+ InsertPointTy CodeGenIP) {
+ OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
+ *this, SectionRegionBodyStmt, AllocaIP, CodeGenIP, "section");
};
LexicalScope Scope(*this, S.getSourceRange());
@@ -4108,11 +4212,9 @@ void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
};
auto BodyGenCB = [MasterRegionBodyStmt, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP, FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, MasterRegionBodyStmt,
- CodeGenIP, FiniBB);
+ InsertPointTy CodeGenIP) {
+ OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
+ *this, MasterRegionBodyStmt, AllocaIP, CodeGenIP, "master");
};
LexicalScope Scope(*this, S.getSourceRange());
@@ -4156,11 +4258,9 @@ void CodeGenFunction::EmitOMPMaskedDirective(const OMPMaskedDirective &S) {
};
auto BodyGenCB = [MaskedRegionBodyStmt, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP, FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, MaskedRegionBodyStmt,
- CodeGenIP, FiniBB);
+ InsertPointTy CodeGenIP) {
+ OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
+ *this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP, "masked");
};
LexicalScope Scope(*this, S.getSourceRange());
@@ -4198,11 +4298,9 @@ void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
};
auto BodyGenCB = [CriticalRegionBodyStmt, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP, FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, CriticalRegionBodyStmt,
- CodeGenIP, FiniBB);
+ InsertPointTy CodeGenIP) {
+ OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
+ *this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP, "critical");
};
LexicalScope Scope(*this, S.getSourceRange());
@@ -4237,23 +4335,25 @@ void CodeGenFunction::EmitOMPParallelForDirective(
(void)emitWorksharingDirective(CGF, S, S.hasCancel());
};
{
- if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
+ const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
+ CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
+ CGCapturedStmtInfo CGSI(CR_OpenMP);
+ CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
+ OMPLoopScope LoopScope(CGF, S);
+ return CGF.EmitScalarExpr(S.getNumIterations());
+ };
+ bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
[](const OMPReductionClause *C) {
return C->getModifier() == OMPC_REDUCTION_inscan;
- })) {
- const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
- CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
- CGCapturedStmtInfo CGSI(CR_OpenMP);
- CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
- OMPLoopScope LoopScope(CGF, S);
- return CGF.EmitScalarExpr(S.getNumIterations());
- };
+ });
+ if (IsInscan)
emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
- }
auto LPCRegion =
CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
emitEmptyBoundParameters);
+ if (IsInscan)
+ emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
}
// Check for outer lastprivate conditional update.
checkForLastprivateConditionalUpdate(*this, S);
@@ -4268,23 +4368,25 @@ void CodeGenFunction::EmitOMPParallelForSimdDirective(
(void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
};
{
- if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
+ const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
+ CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
+ CGCapturedStmtInfo CGSI(CR_OpenMP);
+ CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
+ OMPLoopScope LoopScope(CGF, S);
+ return CGF.EmitScalarExpr(S.getNumIterations());
+ };
+ bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
[](const OMPReductionClause *C) {
return C->getModifier() == OMPC_REDUCTION_inscan;
- })) {
- const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
- CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
- CGCapturedStmtInfo CGSI(CR_OpenMP);
- CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
- OMPLoopScope LoopScope(CGF, S);
- return CGF.EmitScalarExpr(S.getNumIterations());
- };
+ });
+ if (IsInscan)
emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
- }
auto LPCRegion =
CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
emitCommonOMPParallelDirective(*this, S, OMPD_for_simd, CodeGen,
emitEmptyBoundParameters);
+ if (IsInscan)
+ emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
}
// Check for outer lastprivate conditional update.
checkForLastprivateConditionalUpdate(*this, S);
@@ -4379,6 +4481,40 @@ public:
};
} // anonymous namespace
+static void buildDependences(const OMPExecutableDirective &S,
+ OMPTaskDataTy &Data) {
+
+ // First look for 'omp_all_memory' and add this first.
+ bool OmpAllMemory = false;
+ if (llvm::any_of(
+ S.getClausesOfKind<OMPDependClause>(), [](const OMPDependClause *C) {
+ return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
+ C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
+ })) {
+ OmpAllMemory = true;
+ // Since both OMPC_DEPEND_outallmemory and OMPC_DEPEND_inoutallmemory are
+ // equivalent to the runtime, always use OMPC_DEPEND_outallmemory to
+ // simplify.
+ OMPTaskDataTy::DependData &DD =
+ Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
+ /*IteratorExpr=*/nullptr);
+ // Add a nullptr Expr to simplify the codegen in emitDependData.
+ DD.DepExprs.push_back(nullptr);
+ }
+ // Add remaining dependences skipping any 'out' or 'inout' if they are
+ // overridden by 'omp_all_memory'.
+ for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
+ OpenMPDependClauseKind Kind = C->getDependencyKind();
+ if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
+ continue;
+ if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
+ continue;
+ OMPTaskDataTy::DependData &DD =
+ Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
+ DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
+ }
+}
+
void CodeGenFunction::EmitOMPTaskBasedDirective(
const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
@@ -4474,11 +4610,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
*this, S.getBeginLoc(), LHSs, RHSs, Data);
// Build list of dependences.
- for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
- OMPTaskDataTy::DependData &DD =
- Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
- DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
- }
+ buildDependences(S, Data);
// Get list of local vars for untied tasks.
if (!Data.Tied) {
CheckVarsEscapingUntiedTaskDeclContext Checker;
@@ -4613,14 +4745,14 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
Pair.second->getType(), VK_LValue,
Pair.second->getExprLoc());
- Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
- return CGF.EmitLValue(&DRE).getAddress(CGF);
- });
+ Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress(CGF));
}
for (const auto &Pair : PrivatePtrs) {
- Address Replacement(CGF.Builder.CreateLoad(Pair.second),
- CGF.getContext().getDeclAlign(Pair.first));
- Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
+ Address Replacement = Address(
+ CGF.Builder.CreateLoad(Pair.second),
+ CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
+ CGF.getContext().getDeclAlign(Pair.first));
+ Scope.addPrivate(Pair.first, Replacement);
if (auto *DI = CGF.getDebugInfo())
if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
(void)DI->EmitDeclareOfAutoVariable(
@@ -4630,16 +4762,22 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
// Adjust mapping for internal locals by mapping actual memory instead of
// a pointer to this memory.
for (auto &Pair : UntiedLocalVars) {
+ QualType VDType = Pair.first->getType().getNonReferenceType();
if (isAllocatableDecl(Pair.first)) {
llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
- Address Replacement(Ptr, CGF.getPointerAlign());
+ Address Replacement(
+ Ptr,
+ CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
+ CGF.getPointerAlign());
Pair.second.first = Replacement;
Ptr = CGF.Builder.CreateLoad(Replacement);
- Replacement = Address(Ptr, CGF.getContext().getDeclAlign(Pair.first));
+ Replacement = Address(Ptr, CGF.ConvertTypeForMem(VDType),
+ CGF.getContext().getDeclAlign(Pair.first));
Pair.second.second = Replacement;
} else {
llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
- Address Replacement(Ptr, CGF.getContext().getDeclAlign(Pair.first));
+ Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
+ CGF.getContext().getDeclAlign(Pair.first));
Pair.second.first = Replacement;
}
}
@@ -4647,10 +4785,11 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
if (Data.Reductions) {
OMPPrivateScope FirstprivateScope(CGF);
for (const auto &Pair : FirstprivatePtrs) {
- Address Replacement(CGF.Builder.CreateLoad(Pair.second),
- CGF.getContext().getDeclAlign(Pair.first));
- FirstprivateScope.addPrivate(Pair.first,
- [Replacement]() { return Replacement; });
+ Address Replacement(
+ CGF.Builder.CreateLoad(Pair.second),
+ CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
+ CGF.getContext().getDeclAlign(Pair.first));
+ FirstprivateScope.addPrivate(Pair.first, Replacement);
}
(void)FirstprivateScope.Privatize();
OMPLexicalScope LexScope(CGF, S, CapturedRegion);
@@ -4674,10 +4813,10 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
CGF.getContext().getPointerType(
Data.ReductionCopies[Cnt]->getType()),
Data.ReductionCopies[Cnt]->getExprLoc()),
+ CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
Replacement.getAlignment());
Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
- Scope.addPrivate(RedCG.getBaseDecl(Cnt),
- [Replacement]() { return Replacement; });
+ Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
}
}
// Privatize all private variables except for in_reduction items.
@@ -4729,10 +4868,10 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
Replacement.getPointer(), CGF.getContext().VoidPtrTy,
CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
InRedPrivs[Cnt]->getExprLoc()),
+ CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
Replacement.getAlignment());
Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
- InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
- [Replacement]() { return Replacement; });
+ InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
}
}
(void)InRedScope.Privatize();
@@ -4806,6 +4945,17 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
++IElemInitRef;
}
}
+ SmallVector<const Expr *, 4> LHSs;
+ SmallVector<const Expr *, 4> RHSs;
+ for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
+ Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
+ Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
+ Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
+ Data.ReductionOps.append(C->reduction_ops().begin(),
+ C->reduction_ops().end());
+ LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
+ RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
+ }
OMPPrivateScope TargetScope(*this);
VarDecl *BPVD = nullptr;
VarDecl *PVD = nullptr;
@@ -4828,29 +4978,20 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
/*IndexTypeQuals=*/0);
SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
S.getBeginLoc());
- TargetScope.addPrivate(
- BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
- TargetScope.addPrivate(PVD,
- [&InputInfo]() { return InputInfo.PointersArray; });
- TargetScope.addPrivate(SVD,
- [&InputInfo]() { return InputInfo.SizesArray; });
+ TargetScope.addPrivate(BPVD, InputInfo.BasePointersArray);
+ TargetScope.addPrivate(PVD, InputInfo.PointersArray);
+ TargetScope.addPrivate(SVD, InputInfo.SizesArray);
// If there is no user-defined mapper, the mapper array will be nullptr. In
// this case, we don't need to privatize it.
if (!isa_and_nonnull<llvm::ConstantPointerNull>(
InputInfo.MappersArray.getPointer())) {
MVD = createImplicitFirstprivateForType(
getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
- TargetScope.addPrivate(MVD,
- [&InputInfo]() { return InputInfo.MappersArray; });
+ TargetScope.addPrivate(MVD, InputInfo.MappersArray);
}
}
(void)TargetScope.Privatize();
- // Build list of dependences.
- for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
- OMPTaskDataTy::DependData &DD =
- Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
- DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
- }
+ buildDependences(S, Data);
auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD,
&InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
// Set proper addresses for generated private copies.
@@ -4883,13 +5024,14 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
for (const auto &Pair : PrivatePtrs) {
- Address Replacement(CGF.Builder.CreateLoad(Pair.second),
- CGF.getContext().getDeclAlign(Pair.first));
- Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
+ Address Replacement(
+ CGF.Builder.CreateLoad(Pair.second),
+ CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
+ CGF.getContext().getDeclAlign(Pair.first));
+ Scope.addPrivate(Pair.first, Replacement);
}
}
- // Privatize all private variables except for in_reduction items.
- (void)Scope.Privatize();
+ CGF.processInReduction(S, Data, CGF, CS, Scope);
if (InputInfo.NumberOfTargetItems > 0) {
InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
@@ -4914,11 +5056,97 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
IntegerLiteral IfCond(getContext(), TrueOrFalse,
getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
SourceLocation());
-
CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
SharedsTy, CapturedStruct, &IfCond, Data);
}
+void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,
+ OMPTaskDataTy &Data,
+ CodeGenFunction &CGF,
+ const CapturedStmt *CS,
+ OMPPrivateScope &Scope) {
+ if (Data.Reductions) {
+ OpenMPDirectiveKind CapturedRegion = S.getDirectiveKind();
+ OMPLexicalScope LexScope(CGF, S, CapturedRegion);
+ ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
+ Data.ReductionCopies, Data.ReductionOps);
+ llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
+ CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(4)));
+ for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
+ RedCG.emitSharedOrigLValue(CGF, Cnt);
+ RedCG.emitAggregateType(CGF, Cnt);
+ // FIXME: This must removed once the runtime library is fixed.
+ // Emit required threadprivate variables for
+ // initializer/combiner/finalizer.
+ CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
+ RedCG, Cnt);
+ Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
+ CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
+ Replacement =
+ Address(CGF.EmitScalarConversion(
+ Replacement.getPointer(), CGF.getContext().VoidPtrTy,
+ CGF.getContext().getPointerType(
+ Data.ReductionCopies[Cnt]->getType()),
+ Data.ReductionCopies[Cnt]->getExprLoc()),
+ CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
+ Replacement.getAlignment());
+ Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
+ Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
+ }
+ }
+ (void)Scope.Privatize();
+ SmallVector<const Expr *, 4> InRedVars;
+ SmallVector<const Expr *, 4> InRedPrivs;
+ SmallVector<const Expr *, 4> InRedOps;
+ SmallVector<const Expr *, 4> TaskgroupDescriptors;
+ for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
+ auto IPriv = C->privates().begin();
+ auto IRed = C->reduction_ops().begin();
+ auto ITD = C->taskgroup_descriptors().begin();
+ for (const Expr *Ref : C->varlists()) {
+ InRedVars.emplace_back(Ref);
+ InRedPrivs.emplace_back(*IPriv);
+ InRedOps.emplace_back(*IRed);
+ TaskgroupDescriptors.emplace_back(*ITD);
+ std::advance(IPriv, 1);
+ std::advance(IRed, 1);
+ std::advance(ITD, 1);
+ }
+ }
+ OMPPrivateScope InRedScope(CGF);
+ if (!InRedVars.empty()) {
+ ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
+ for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
+ RedCG.emitSharedOrigLValue(CGF, Cnt);
+ RedCG.emitAggregateType(CGF, Cnt);
+ // FIXME: This must removed once the runtime library is fixed.
+ // Emit required threadprivate variables for
+ // initializer/combiner/finalizer.
+ CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
+ RedCG, Cnt);
+ llvm::Value *ReductionsPtr;
+ if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
+ ReductionsPtr =
+ CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), TRExpr->getExprLoc());
+ } else {
+ ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ }
+ Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
+ CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
+ Replacement = Address(
+ CGF.EmitScalarConversion(
+ Replacement.getPointer(), CGF.getContext().VoidPtrTy,
+ CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
+ InRedPrivs[Cnt]->getExprLoc()),
+ CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
+ Replacement.getAlignment());
+ Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
+ InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
+ }
+ }
+ (void)InRedScope.Privatize();
+}
+
void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
// Emit outlined function for task construct.
const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
@@ -4963,11 +5191,7 @@ void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
OMPTaskDataTy Data;
// Build list of dependences
- for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
- OMPTaskDataTy::DependData &DD =
- Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
- DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
- }
+ buildDependences(S, Data);
CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(), Data);
}
@@ -5533,10 +5757,13 @@ void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
};
auto BodyGenCB = [&S, C, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
+ InsertPointTy CodeGenIP) {
+ Builder.restoreIP(CodeGenIP);
+
const CapturedStmt *CS = S.getInnermostCapturedStmt();
if (C) {
+ llvm::BasicBlock *FiniBB = splitBBWithSuffix(
+ Builder, /*CreateBranch=*/false, ".ordered.after");
llvm::SmallVector<llvm::Value *, 16> CapturedVars;
GenerateOpenMPCapturedVars(*CS, CapturedVars);
llvm::Function *OutlinedFn =
@@ -5544,13 +5771,11 @@ void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
assert(S.getBeginLoc().isValid() &&
"Outlined function call location must be valid.");
ApplyDebugLocation::CreateDefaultArtificial(*this, S.getBeginLoc());
- OMPBuilderCBHelpers::EmitCaptureStmt(*this, CodeGenIP, FiniBB,
+ OMPBuilderCBHelpers::EmitCaptureStmt(*this, CodeGenIP, *FiniBB,
OutlinedFn, CapturedVars);
} else {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP,
- FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, CS->getCapturedStmt(),
- CodeGenIP, FiniBB);
+ OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
+ *this, CS->getCapturedStmt(), AllocaIP, CodeGenIP, "ordered");
}
};
@@ -5730,25 +5955,38 @@ static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
// Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
// expression is simple and atomic is allowed for the given type for the
// target platform.
- if (BO == BO_Comma || !Update.isScalar() ||
- !Update.getScalarVal()->getType()->isIntegerTy() || !X.isSimple() ||
+ if (BO == BO_Comma || !Update.isScalar() || !X.isSimple() ||
(!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
(Update.getScalarVal()->getType() !=
X.getAddress(CGF).getElementType())) ||
- !X.getAddress(CGF).getElementType()->isIntegerTy() ||
!Context.getTargetInfo().hasBuiltinAtomic(
Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
return std::make_pair(false, RValue::get(nullptr));
+ auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) {
+ if (T->isIntegerTy())
+ return true;
+
+ if (T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
+ return llvm::isPowerOf2_64(CGF.CGM.getDataLayout().getTypeStoreSize(T));
+
+ return false;
+ };
+
+ if (!CheckAtomicSupport(Update.getScalarVal()->getType(), BO) ||
+ !CheckAtomicSupport(X.getAddress(CGF).getElementType(), BO))
+ return std::make_pair(false, RValue::get(nullptr));
+
+ bool IsInteger = X.getAddress(CGF).getElementType()->isIntegerTy();
llvm::AtomicRMWInst::BinOp RMWOp;
switch (BO) {
case BO_Add:
- RMWOp = llvm::AtomicRMWInst::Add;
+ RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
break;
case BO_Sub:
if (!IsXLHSInRHSPart)
return std::make_pair(false, RValue::get(nullptr));
- RMWOp = llvm::AtomicRMWInst::Sub;
+ RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
break;
case BO_And:
RMWOp = llvm::AtomicRMWInst::And;
@@ -5806,9 +6044,13 @@ static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
}
llvm::Value *UpdateVal = Update.getScalarVal();
if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
- UpdateVal = CGF.Builder.CreateIntCast(
- IC, X.getAddress(CGF).getElementType(),
- X.getType()->hasSignedIntegerRepresentation());
+ if (IsInteger)
+ UpdateVal = CGF.Builder.CreateIntCast(
+ IC, X.getAddress(CGF).getElementType(),
+ X.getType()->hasSignedIntegerRepresentation());
+ else
+ UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
+ X.getAddress(CGF).getElementType());
}
llvm::Value *Res =
CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(CGF), UpdateVal, AO);
@@ -6011,11 +6253,88 @@ static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF,
}
}
+static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF,
+ llvm::AtomicOrdering AO, const Expr *X,
+ const Expr *V, const Expr *R,
+ const Expr *E, const Expr *D,
+ const Expr *CE, bool IsXBinopExpr,
+ bool IsPostfixUpdate, bool IsFailOnly,
+ SourceLocation Loc) {
+ llvm::OpenMPIRBuilder &OMPBuilder =
+ CGF.CGM.getOpenMPRuntime().getOMPBuilder();
+
+ OMPAtomicCompareOp Op;
+ assert(isa<BinaryOperator>(CE) && "CE is not a BinaryOperator");
+ switch (cast<BinaryOperator>(CE)->getOpcode()) {
+ case BO_EQ:
+ Op = OMPAtomicCompareOp::EQ;
+ break;
+ case BO_LT:
+ Op = OMPAtomicCompareOp::MIN;
+ break;
+ case BO_GT:
+ Op = OMPAtomicCompareOp::MAX;
+ break;
+ default:
+ llvm_unreachable("unsupported atomic compare binary operator");
+ }
+
+ LValue XLVal = CGF.EmitLValue(X);
+ Address XAddr = XLVal.getAddress(CGF);
+
+ auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) {
+ if (X->getType() == E->getType())
+ return CGF.EmitScalarExpr(E);
+ const Expr *NewE = E->IgnoreImplicitAsWritten();
+ llvm::Value *V = CGF.EmitScalarExpr(NewE);
+ if (NewE->getType() == X->getType())
+ return V;
+ return CGF.EmitScalarConversion(V, NewE->getType(), X->getType(), Loc);
+ };
+
+ llvm::Value *EVal = EmitRValueWithCastIfNeeded(X, E);
+ llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(X, D) : nullptr;
+ if (auto *CI = dyn_cast<llvm::ConstantInt>(EVal))
+ EVal = CGF.Builder.CreateIntCast(
+ CI, XLVal.getAddress(CGF).getElementType(),
+ E->getType()->hasSignedIntegerRepresentation());
+ if (DVal)
+ if (auto *CI = dyn_cast<llvm::ConstantInt>(DVal))
+ DVal = CGF.Builder.CreateIntCast(
+ CI, XLVal.getAddress(CGF).getElementType(),
+ D->getType()->hasSignedIntegerRepresentation());
+
+ llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
+ XAddr.getPointer(), XAddr.getElementType(),
+ X->getType()->hasSignedIntegerRepresentation(),
+ X->getType().isVolatileQualified()};
+ llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
+ if (V) {
+ LValue LV = CGF.EmitLValue(V);
+ Address Addr = LV.getAddress(CGF);
+ VOpVal = {Addr.getPointer(), Addr.getElementType(),
+ V->getType()->hasSignedIntegerRepresentation(),
+ V->getType().isVolatileQualified()};
+ }
+ if (R) {
+ LValue LV = CGF.EmitLValue(R);
+ Address Addr = LV.getAddress(CGF);
+ ROpVal = {Addr.getPointer(), Addr.getElementType(),
+ R->getType()->hasSignedIntegerRepresentation(),
+ R->getType().isVolatileQualified()};
+ }
+
+ CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
+ CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
+ IsPostfixUpdate, IsFailOnly));
+}
+
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
llvm::AtomicOrdering AO, bool IsPostfixUpdate,
- const Expr *X, const Expr *V, const Expr *E,
- const Expr *UE, bool IsXLHSInRHSPart,
- SourceLocation Loc) {
+ const Expr *X, const Expr *V, const Expr *R,
+ const Expr *E, const Expr *UE, const Expr *D,
+ const Expr *CE, bool IsXLHSInRHSPart,
+ bool IsFailOnly, SourceLocation Loc) {
switch (Kind) {
case OMPC_read:
emitOMPAtomicReadExpr(CGF, AO, X, V, Loc);
@@ -6031,9 +6350,11 @@ static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE,
IsXLHSInRHSPart, Loc);
break;
- case OMPC_compare:
- // Do nothing here as we already emit an error.
+ case OMPC_compare: {
+ emitOMPAtomicCompareExpr(CGF, AO, X, V, R, E, D, CE, IsXLHSInRHSPart,
+ IsPostfixUpdate, IsFailOnly, Loc);
break;
+ }
case OMPC_if:
case OMPC_final:
case OMPC_num_threads:
@@ -6091,6 +6412,7 @@ static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
case OMPC_use_device_ptr:
case OMPC_use_device_addr:
case OMPC_is_device_ptr:
+ case OMPC_has_device_addr:
case OMPC_unified_address:
case OMPC_unified_shared_memory:
case OMPC_reverse_offload:
@@ -6121,6 +6443,7 @@ static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
case OMPC_memory_order:
case OMPC_bind:
case OMPC_align:
+ case OMPC_cancellation_construct_type:
llvm_unreachable("Clause is not allowed in 'omp atomic'.");
}
}
@@ -6144,19 +6467,24 @@ void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
AO = llvm::AtomicOrdering::Monotonic;
MemOrderingSpecified = true;
}
+ llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
OpenMPClauseKind Kind = OMPC_unknown;
for (const OMPClause *C : S.clauses()) {
// Find first clause (skip seq_cst|acq_rel|aqcuire|release|relaxed clause,
// if it is first).
- if (C->getClauseKind() != OMPC_seq_cst &&
- C->getClauseKind() != OMPC_acq_rel &&
- C->getClauseKind() != OMPC_acquire &&
- C->getClauseKind() != OMPC_release &&
- C->getClauseKind() != OMPC_relaxed && C->getClauseKind() != OMPC_hint) {
- Kind = C->getClauseKind();
- break;
- }
+ OpenMPClauseKind K = C->getClauseKind();
+ if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
+ K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
+ continue;
+ Kind = K;
+ KindsEncountered.insert(K);
}
+ // We just need to correct Kind here. No need to set a bool saying it is
+ // actually compare capture because we can tell from whether V and R are
+ // nullptr.
+ if (KindsEncountered.contains(OMPC_compare) &&
+ KindsEncountered.contains(OMPC_capture))
+ Kind = OMPC_compare;
if (!MemOrderingSpecified) {
llvm::AtomicOrdering DefaultOrder =
CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
@@ -6178,7 +6506,8 @@ void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
LexicalScope Scope(*this, S.getSourceRange());
EmitStopPoint(S.getAssociatedStmt());
emitOMPAtomicExpr(*this, Kind, AO, S.isPostfixUpdate(), S.getX(), S.getV(),
- S.getExpr(), S.getUpdateExpr(), S.isXLHSInRHSPart(),
+ S.getR(), S.getExpr(), S.getUpdateExpr(), S.getD(),
+ S.getCondExpr(), S.isXLHSInRHSPart(), S.isFailOnly(),
S.getBeginLoc());
}
@@ -6230,6 +6559,13 @@ static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
if (CGM.getLangOpts().OMPTargetTriples.empty())
IsOffloadEntry = false;
+ if (CGM.getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
+ unsigned DiagID = CGM.getDiags().getCustomDiagID(
+ DiagnosticsEngine::Error,
+ "No offloading entry generated while offloading is mandatory.");
+ CGM.getDiags().Report(DiagID);
+ }
+
assert(CGF.CurFuncDecl && "No parent declaration for target region!");
StringRef ParentName;
// In case we have Ctors/Dtors we use the complete type variant to produce
@@ -6806,30 +7142,26 @@ void CodeGenFunction::EmitOMPUseDevicePtrClause(
if (InitAddrIt == CaptureDeviceAddrMap.end())
continue;
- bool IsRegistered = PrivateScope.addPrivate(
- OrigVD, [this, OrigVD, InitAddrIt, InitVD, PvtVD]() {
- // Initialize the temporary initialization variable with the address
- // we get from the runtime library. We have to cast the source address
- // because it is always a void *. References are materialized in the
- // privatization scope, so the initialization here disregards the fact
- // the original variable is a reference.
- QualType AddrQTy = getContext().getPointerType(
- OrigVD->getType().getNonReferenceType());
- llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
- Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
- setAddrOfLocalVar(InitVD, InitAddr);
+ // Initialize the temporary initialization variable with the address
+ // we get from the runtime library. We have to cast the source address
+ // because it is always a void *. References are materialized in the
+ // privatization scope, so the initialization here disregards the fact
+ // the original variable is a reference.
+ llvm::Type *Ty = ConvertTypeForMem(OrigVD->getType().getNonReferenceType());
+ Address InitAddr = Builder.CreateElementBitCast(InitAddrIt->second, Ty);
+ setAddrOfLocalVar(InitVD, InitAddr);
- // Emit private declaration, it will be initialized by the value we
- // declaration we just added to the local declarations map.
- EmitDecl(*PvtVD);
+ // Emit private declaration, it will be initialized by the value we
+ // declaration we just added to the local declarations map.
+ EmitDecl(*PvtVD);
- // The initialization variables reached its purpose in the emission
- // of the previous declaration, so we don't need it anymore.
- LocalDeclMap.erase(InitVD);
+ // The initialization variables reached its purpose in the emission
+ // of the previous declaration, so we don't need it anymore.
+ LocalDeclMap.erase(InitVD);
- // Return the address of the private variable.
- return GetAddrOfLocalVar(PvtVD);
- });
+ // Return the address of the private variable.
+ bool IsRegistered =
+ PrivateScope.addPrivate(OrigVD, GetAddrOfLocalVar(PvtVD));
assert(IsRegistered && "firstprivate var already registered as private");
// Silence the warning about unused variable.
(void)IsRegistered;
@@ -6879,17 +7211,15 @@ void CodeGenFunction::EmitOMPUseDeviceAddrClause(
// For declrefs and variable length array need to load the pointer for
// correct mapping, since the pointer to the data was passed to the runtime.
if (isa<DeclRefExpr>(Ref->IgnoreParenImpCasts()) ||
- MatchingVD->getType()->isArrayType())
- PrivAddr =
- EmitLoadOfPointer(PrivAddr, getContext()
- .getPointerType(OrigVD->getType())
- ->castAs<PointerType>());
- llvm::Type *RealTy =
- ConvertTypeForMem(OrigVD->getType().getNonReferenceType())
- ->getPointerTo();
- PrivAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(PrivAddr, RealTy);
+ MatchingVD->getType()->isArrayType()) {
+ QualType PtrTy = getContext().getPointerType(
+ OrigVD->getType().getNonReferenceType());
+ PrivAddr = EmitLoadOfPointer(
+ Builder.CreateElementBitCast(PrivAddr, ConvertTypeForMem(PtrTy)),
+ PtrTy->castAs<PointerType>());
+ }
- (void)PrivateScope.addPrivate(OrigVD, [PrivAddr]() { return PrivAddr; });
+ (void)PrivateScope.addPrivate(OrigVD, PrivAddr);
}
}
@@ -7161,8 +7491,7 @@ static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
const ImplicitParamDecl *PVD,
CodeGenFunction::OMPPrivateScope &Privates) {
const auto *VDecl = cast<VarDecl>(Helper->getDecl());
- Privates.addPrivate(VDecl,
- [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
+ Privates.addPrivate(VDecl, CGF.GetAddrOfLocalVar(PVD));
}
void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
@@ -7454,8 +7783,7 @@ void CodeGenFunction::EmitSimpleOMPExecutableDirective(
continue;
if (!CGF.LocalDeclMap.count(VD)) {
LValue GlobLVal = CGF.EmitLValue(Ref);
- GlobalsScope.addPrivate(
- VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); });
+ GlobalsScope.addPrivate(VD, GlobLVal.getAddress(CGF));
}
}
}
@@ -7470,8 +7798,7 @@ void CodeGenFunction::EmitSimpleOMPExecutableDirective(
const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
LValue GlobLVal = CGF.EmitLValue(E);
- GlobalsScope.addPrivate(
- VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); });
+ GlobalsScope.addPrivate(VD, GlobLVal.getAddress(CGF));
}
if (isa<OMPCapturedExprDecl>(VD)) {
// Emit only those that were not explicitly referenced in clauses.
diff --git a/clang/lib/CodeGen/CGVTT.cpp b/clang/lib/CodeGen/CGVTT.cpp
index 564d9f354e64..ebac9196df02 100644
--- a/clang/lib/CodeGen/CGVTT.cpp
+++ b/clang/lib/CodeGen/CGVTT.cpp
@@ -96,9 +96,6 @@ CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT,
if (CGM.supportsCOMDAT() && VTT->isWeakForLinker())
VTT->setComdat(CGM.getModule().getOrInsertComdat(VTT->getName()));
-
- // Set the right visibility.
- CGM.setGVProperties(VTT, RD);
}
llvm::GlobalVariable *CodeGenVTables::GetAddrOfVTT(const CXXRecordDecl *RD) {
@@ -122,6 +119,7 @@ llvm::GlobalVariable *CodeGenVTables::GetAddrOfVTT(const CXXRecordDecl *RD) {
llvm::GlobalVariable *GV = CGM.CreateOrReplaceCXXRuntimeVariable(
Name, ArrayType, llvm::GlobalValue::ExternalLinkage, Align);
GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
+ CGM.setGVProperties(GV, RD);
return GV;
}
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
index c839376880c4..cdd40d2a6a2e 100644
--- a/clang/lib/CodeGen/CGVTables.cpp
+++ b/clang/lib/CodeGen/CGVTables.cpp
@@ -90,9 +90,11 @@ static RValue PerformReturnAdjustment(CodeGenFunction &CGF,
auto ClassDecl = ResultType->getPointeeType()->getAsCXXRecordDecl();
auto ClassAlign = CGF.CGM.getClassPointerAlignment(ClassDecl);
- ReturnValue = CGF.CGM.getCXXABI().performReturnAdjustment(CGF,
- Address(ReturnValue, ClassAlign),
- Thunk.Return);
+ ReturnValue = CGF.CGM.getCXXABI().performReturnAdjustment(
+ CGF,
+ Address(ReturnValue, CGF.ConvertTypeForMem(ResultType->getPointeeType()),
+ ClassAlign),
+ Thunk.Return);
if (NullCheckValue) {
CGF.Builder.CreateBr(AdjustEnd);
@@ -198,7 +200,9 @@ CodeGenFunction::GenerateVarArgsThunk(llvm::Function *Fn,
// Find the first store of "this", which will be to the alloca associated
// with "this".
- Address ThisPtr(&*AI, CGM.getClassPointerAlignment(MD->getParent()));
+ Address ThisPtr =
+ Address(&*AI, ConvertTypeForMem(MD->getThisType()->getPointeeType()),
+ CGM.getClassPointerAlignment(MD->getParent()));
llvm::BasicBlock *EntryBB = &Fn->front();
llvm::BasicBlock::iterator ThisStore =
llvm::find_if(*EntryBB, [&](llvm::Instruction &I) {
@@ -396,9 +400,7 @@ void CodeGenFunction::EmitMustTailThunk(GlobalDecl GD,
// to translate AST arguments into LLVM IR arguments. For thunks, we know
// that the caller prototype more or less matches the callee prototype with
// the exception of 'this'.
- SmallVector<llvm::Value *, 8> Args;
- for (llvm::Argument &A : CurFn->args())
- Args.push_back(&A);
+ SmallVector<llvm::Value *, 8> Args(llvm::make_pointer_range(CurFn->args()));
// Set the adjusted 'this' pointer.
const ABIArgInfo &ThisAI = CurFnInfo->arg_begin()->info;
@@ -1173,7 +1175,10 @@ void CodeGenModule::EmitDeferredVTables() {
DeferredVTables.clear();
}
-bool CodeGenModule::HasLTOVisibilityPublicStd(const CXXRecordDecl *RD) {
+bool CodeGenModule::AlwaysHasLTOVisibilityPublic(const CXXRecordDecl *RD) {
+ if (RD->hasAttr<LTOVisibilityPublicAttr>() || RD->hasAttr<UuidAttr>())
+ return true;
+
if (!getCodeGenOpts().LTOVisibilityPublicStd)
return false;
@@ -1198,9 +1203,6 @@ bool CodeGenModule::HasHiddenLTOVisibility(const CXXRecordDecl *RD) {
if (!isExternallyVisible(LV.getLinkage()))
return true;
- if (RD->hasAttr<LTOVisibilityPublicAttr>() || RD->hasAttr<UuidAttr>())
- return false;
-
if (getTriple().isOSBinFormatCOFF()) {
if (RD->hasAttr<DLLExportAttr>() || RD->hasAttr<DLLImportAttr>())
return false;
@@ -1209,7 +1211,7 @@ bool CodeGenModule::HasHiddenLTOVisibility(const CXXRecordDecl *RD) {
return false;
}
- return !HasLTOVisibilityPublicStd(RD);
+ return !AlwaysHasLTOVisibilityPublic(RD);
}
llvm::GlobalObject::VCallVisibility CodeGenModule::GetVCallVisibilityLevel(
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index c2c508dedb09..4ffbecdf2741 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -340,6 +340,15 @@ namespace clang {
CodeGenOpts.getProfileUse() != CodeGenOptions::ProfileNone)
Ctx.setDiagnosticsHotnessRequested(true);
+ if (CodeGenOpts.MisExpect) {
+ Ctx.setMisExpectWarningRequested(true);
+ }
+
+ if (CodeGenOpts.DiagnosticsMisExpectTolerance) {
+ Ctx.setDiagnosticsMisExpectTolerance(
+ CodeGenOpts.DiagnosticsMisExpectTolerance);
+ }
+
// Link each LinkModule into our module.
if (LinkInModules())
return;
@@ -440,6 +449,9 @@ namespace clang {
void OptimizationFailureHandler(
const llvm::DiagnosticInfoOptimizationFailure &D);
void DontCallDiagHandler(const DiagnosticInfoDontCall &D);
+ /// Specialized handler for misexpect warnings.
+ /// Note that misexpect remarks are emitted through ORE
+ void MisExpectDiagHandler(const llvm::DiagnosticInfoMisExpect &D);
};
void BackendConsumer::anchor() {}
@@ -821,6 +833,25 @@ void BackendConsumer::DontCallDiagHandler(const DiagnosticInfoDontCall &D) {
<< llvm::demangle(D.getFunctionName().str()) << D.getNote();
}
+void BackendConsumer::MisExpectDiagHandler(
+ const llvm::DiagnosticInfoMisExpect &D) {
+ StringRef Filename;
+ unsigned Line, Column;
+ bool BadDebugInfo = false;
+ FullSourceLoc Loc =
+ getBestLocationFromDebugLoc(D, BadDebugInfo, Filename, Line, Column);
+
+ Diags.Report(Loc, diag::warn_profile_data_misexpect) << D.getMsg().str();
+
+ if (BadDebugInfo)
+ // If we were not able to translate the file:line:col information
+ // back to a SourceLocation, at least emit a note stating that
+ // we could not translate this location. This can happen in the
+ // case of #line directives.
+ Diags.Report(Loc, diag::note_fe_backend_invalid_loc)
+ << Filename << Line << Column;
+}
+
/// This function is invoked when the backend needs
/// to report something to the user.
void BackendConsumer::DiagnosticHandlerImpl(const DiagnosticInfo &DI) {
@@ -895,6 +926,9 @@ void BackendConsumer::DiagnosticHandlerImpl(const DiagnosticInfo &DI) {
case llvm::DK_DontCall:
DontCallDiagHandler(cast<DiagnosticInfoDontCall>(DI));
return;
+ case llvm::DK_MisExpect:
+ MisExpectDiagHandler(cast<DiagnosticInfoMisExpect>(DI));
+ return;
default:
// Plugin IDs are not bound to any value as they are set dynamically.
ComputeDiagRemarkID(Severity, backend_plugin, DiagID);
@@ -983,6 +1017,8 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
if (BA != Backend_EmitNothing && !OS)
return nullptr;
+ VMContext->setOpaquePointers(CI.getCodeGenOpts().OpaquePointers);
+
// Load bitcode modules to link with, if we need to.
if (LinkModules.empty())
for (const CodeGenOptions::BitcodeFileToLink &F :
@@ -1113,7 +1149,7 @@ void CodeGenAction::ExecuteAction() {
auto &CodeGenOpts = CI.getCodeGenOpts();
auto &Diagnostics = CI.getDiagnostics();
std::unique_ptr<raw_pwrite_stream> OS =
- GetOutputStream(CI, getCurrentFile(), BA);
+ GetOutputStream(CI, getCurrentFileOrBufferName(), BA);
if (BA != Backend_EmitNothing && !OS)
return;
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 50e1638924d1..05942f462dd1 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -105,8 +105,9 @@ clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
case LangOptions::FPE_Ignore: return llvm::fp::ebIgnore;
case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
case LangOptions::FPE_Strict: return llvm::fp::ebStrict;
+ default:
+ llvm_unreachable("Unsupported FP Exception Behavior");
}
- llvm_unreachable("Unsupported FP Exception Behavior");
}
void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
@@ -145,12 +146,11 @@ void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
FMFGuard.emplace(CGF.Builder);
- llvm::RoundingMode NewRoundingBehavior =
- static_cast<llvm::RoundingMode>(FPFeatures.getRoundingMode());
+ llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode();
CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
auto NewExceptionBehavior =
ToConstrainedExceptMD(static_cast<LangOptions::FPExceptionModeKind>(
- FPFeatures.getFPExceptionMode()));
+ FPFeatures.getExceptionMode()));
CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
CGF.SetFastMathFlags(FPFeatures);
@@ -383,9 +383,6 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
"__cyg_profile_func_exit");
}
- if (ShouldSkipSanitizerInstrumentation())
- CurFn->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation);
-
// Emit debug descriptor for function end.
if (CGDebugInfo *DI = getDebugInfo())
DI->EmitFunctionEnd(Builder, CurFn);
@@ -488,6 +485,9 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
std::max((uint64_t)LargestVectorWidth,
VT->getPrimitiveSizeInBits().getKnownMinSize());
+ if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth)
+ LargestVectorWidth = CurFnInfo->getMaxVectorWidth();
+
// Add the required-vector-width attribute. This contains the max width from:
// 1. min-vector-width attribute used in the source program.
// 2. Any builtins used that have a vector width specified.
@@ -502,8 +502,7 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
getContext().getTargetInfo().getVScaleRange(getLangOpts());
if (VScaleRange) {
CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(
- getLLVMContext(), VScaleRange.getValue().first,
- VScaleRange.getValue().second));
+ getLLVMContext(), VScaleRange->first, VScaleRange->second));
}
// If we generated an unreachable return block, delete it now.
@@ -560,29 +559,6 @@ bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
XRayInstrKind::Typed);
}
-llvm::Constant *
-CodeGenFunction::EncodeAddrForUseInPrologue(llvm::Function *F,
- llvm::Constant *Addr) {
- // Addresses stored in prologue data can't require run-time fixups and must
- // be PC-relative. Run-time fixups are undesirable because they necessitate
- // writable text segments, which are unsafe. And absolute addresses are
- // undesirable because they break PIE mode.
-
- // Add a layer of indirection through a private global. Taking its address
- // won't result in a run-time fixup, even if Addr has linkonce_odr linkage.
- auto *GV = new llvm::GlobalVariable(CGM.getModule(), Addr->getType(),
- /*isConstant=*/true,
- llvm::GlobalValue::PrivateLinkage, Addr);
-
- // Create a PC-relative address.
- auto *GOTAsInt = llvm::ConstantExpr::getPtrToInt(GV, IntPtrTy);
- auto *FuncAsInt = llvm::ConstantExpr::getPtrToInt(F, IntPtrTy);
- auto *PCRelAsInt = llvm::ConstantExpr::getSub(GOTAsInt, FuncAsInt);
- return (IntPtrTy == Int32Ty)
- ? PCRelAsInt
- : llvm::ConstantExpr::getTrunc(PCRelAsInt, Int32Ty);
-}
-
llvm::Value *
CodeGenFunction::DecodeAddrUsedInPrologue(llvm::Value *F,
llvm::Value *EncodedAddr) {
@@ -593,19 +569,21 @@ CodeGenFunction::DecodeAddrUsedInPrologue(llvm::Value *F,
auto *GOTAddr = Builder.CreateIntToPtr(GOTAsInt, Int8PtrPtrTy, "global_addr");
// Load the original pointer through the global.
- return Builder.CreateLoad(Address(GOTAddr, getPointerAlign()),
+ return Builder.CreateLoad(Address(GOTAddr, Int8PtrTy, getPointerAlign()),
"decoded_addr");
}
-void CodeGenFunction::EmitOpenCLKernelMetadata(const FunctionDecl *FD,
- llvm::Function *Fn)
-{
- if (!FD->hasAttr<OpenCLKernelAttr>())
+void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,
+ llvm::Function *Fn) {
+ if (!FD->hasAttr<OpenCLKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
return;
llvm::LLVMContext &Context = getLLVMContext();
- CGM.GenOpenCLArgMetadata(Fn, FD, this);
+ CGM.GenKernelArgMetadata(Fn, FD, this);
+
+ if (!getLangOpts().OpenCL)
+ return;
if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
QualType HintQTy = A->getTypeHint();
@@ -743,6 +721,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
} while (false);
if (D) {
+ const bool SanitizeBounds = SanOpts.hasOneOf(SanitizerKind::Bounds);
bool NoSanitizeCoverage = false;
for (auto Attr : D->specific_attrs<NoSanitizeAttr>()) {
@@ -763,21 +742,29 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
NoSanitizeCoverage = true;
}
+ if (SanitizeBounds && !SanOpts.hasOneOf(SanitizerKind::Bounds))
+ Fn->addFnAttr(llvm::Attribute::NoSanitizeBounds);
+
if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())
Fn->addFnAttr(llvm::Attribute::NoSanitizeCoverage);
}
- // Apply sanitizer attributes to the function.
- if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress))
- Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
- if (SanOpts.hasOneOf(SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress))
- Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
- if (SanOpts.has(SanitizerKind::MemTag))
- Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
- if (SanOpts.has(SanitizerKind::Thread))
- Fn->addFnAttr(llvm::Attribute::SanitizeThread);
- if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory))
- Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
+ if (ShouldSkipSanitizerInstrumentation()) {
+ CurFn->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation);
+ } else {
+ // Apply sanitizer attributes to the function.
+ if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress))
+ Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
+ if (SanOpts.hasOneOf(SanitizerKind::HWAddress |
+ SanitizerKind::KernelHWAddress))
+ Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
+ if (SanOpts.has(SanitizerKind::MemtagStack))
+ Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
+ if (SanOpts.has(SanitizerKind::Thread))
+ Fn->addFnAttr(llvm::Attribute::SanitizeThread);
+ if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory))
+ Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
+ }
if (SanOpts.has(SanitizerKind::SafeStack))
Fn->addFnAttr(llvm::Attribute::SafeStack);
if (SanOpts.has(SanitizerKind::ShadowCallStack))
@@ -911,9 +898,10 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
if (D && D->hasAttr<NoProfileFunctionAttr>())
Fn->addFnAttr(llvm::Attribute::NoProfile);
- if (FD && getLangOpts().OpenCL) {
+ if (FD && (getLangOpts().OpenCL ||
+ (getLangOpts().HIP && getLangOpts().CUDAIsDevice))) {
// Add metadata for a kernel function.
- EmitOpenCLKernelMetadata(FD, Fn);
+ EmitKernelMetadata(FD, Fn);
}
// If we are checking function types, emit a function type signature as
@@ -926,12 +914,13 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
FD->getType(), EST_None);
llvm::Constant *FTRTTIConst =
CGM.GetAddrOfRTTIDescriptor(ProtoTy, /*ForEH=*/true);
- llvm::Constant *FTRTTIConstEncoded =
- EncodeAddrForUseInPrologue(Fn, FTRTTIConst);
- llvm::Constant *PrologueStructElems[] = {PrologueSig, FTRTTIConstEncoded};
- llvm::Constant *PrologueStructConst =
- llvm::ConstantStruct::getAnon(PrologueStructElems, /*Packed=*/true);
- Fn->setPrologueData(PrologueStructConst);
+ llvm::GlobalVariable *FTRTTIProxy =
+ CGM.GetOrCreateRTTIProxyGlobalVariable(FTRTTIConst);
+ llvm::LLVMContext &Ctx = Fn->getContext();
+ llvm::MDBuilder MDB(Ctx);
+ Fn->setMetadata(llvm::LLVMContext::MD_func_sanitize,
+ MDB.createRTTIPointerPrologue(PrologueSig, FTRTTIProxy));
+ CGM.addCompilerUsedGlobal(FTRTTIProxy);
}
}
@@ -963,9 +952,9 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
(getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>())))
Fn->addFnAttr(llvm::Attribute::NoRecurse);
- llvm::RoundingMode RM = getLangOpts().getFPRoundingMode();
+ llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode();
llvm::fp::ExceptionBehavior FPExceptionBehavior =
- ToConstrainedExceptMD(getLangOpts().getFPExceptionMode());
+ ToConstrainedExceptMD(getLangOpts().getDefaultExceptionMode());
Builder.setDefaultConstrainedRounding(RM);
Builder.setDefaultConstrainedExcept(FPExceptionBehavior);
if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) ||
@@ -981,6 +970,10 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
CGM.getCodeGenOpts().StackAlignment))
Fn->addFnAttr("stackrealign");
+ // "main" doesn't need to zero out call-used registers.
+ if (FD && FD->isMain())
+ Fn->removeFnAttr("zero-call-used-regs");
+
llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);
// Create a marker to make it easy to insert allocas into the entryblock
@@ -1094,12 +1087,13 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
llvm::Function::arg_iterator EI = CurFn->arg_end();
--EI;
llvm::Value *Addr = Builder.CreateStructGEP(
- EI->getType()->getPointerElementType(), &*EI, Idx);
+ CurFnInfo->getArgStruct(), &*EI, Idx);
llvm::Type *Ty =
cast<llvm::GetElementPtrInst>(Addr)->getResultElementType();
- ReturnValuePointer = Address(Addr, getPointerAlign());
+ ReturnValuePointer = Address(Addr, Ty, getPointerAlign());
Addr = Builder.CreateAlignedLoad(Ty, Addr, getPointerAlign(), "agg.result");
- ReturnValue = Address(Addr, CGM.getNaturalTypeAlignment(RetTy));
+ ReturnValue =
+ Address(Addr, ConvertType(RetTy), CGM.getNaturalTypeAlignment(RetTy));
} else {
ReturnValue = CreateIRTemp(RetTy, "retval");
@@ -1122,7 +1116,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
EmitFunctionProlog(*CurFnInfo, CurFn, Args);
- if (D && isa<CXXMethodDecl>(D) && cast<CXXMethodDecl>(D)->isInstance()) {
+ if (isa_and_nonnull<CXXMethodDecl>(D) &&
+ cast<CXXMethodDecl>(D)->isInstance()) {
CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
const CXXMethodDecl *MD = cast<CXXMethodDecl>(D);
if (MD->getParent()->isLambda() &&
@@ -1183,27 +1178,26 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
}
// If any of the arguments have a variably modified type, make sure to
- // emit the type size.
- for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
- i != e; ++i) {
- const VarDecl *VD = *i;
-
- // Dig out the type as written from ParmVarDecls; it's unclear whether
- // the standard (C99 6.9.1p10) requires this, but we're following the
- // precedent set by gcc.
- QualType Ty;
- if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD))
- Ty = PVD->getOriginalType();
- else
- Ty = VD->getType();
+ // emit the type size, but only if the function is not naked. Naked functions
+ // have no prolog to run this evaluation.
+ if (!FD || !FD->hasAttr<NakedAttr>()) {
+ for (const VarDecl *VD : Args) {
+ // Dig out the type as written from ParmVarDecls; it's unclear whether
+ // the standard (C99 6.9.1p10) requires this, but we're following the
+ // precedent set by gcc.
+ QualType Ty;
+ if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD))
+ Ty = PVD->getOriginalType();
+ else
+ Ty = VD->getType();
- if (Ty->isVariablyModifiedType())
- EmitVariablyModifiedType(Ty);
+ if (Ty->isVariablyModifiedType())
+ EmitVariablyModifiedType(Ty);
+ }
}
// Emit a location at the end of the prologue.
if (CGDebugInfo *DI = getDebugInfo())
DI->EmitLocation(Builder, StartLoc);
-
// TODO: Do we need to handle this in two places like we do with
// target-features/target-cpu?
if (CurFuncDecl)
@@ -1398,8 +1392,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
// Save parameters for coroutine function.
if (Body && isa_and_nonnull<CoroutineBodyStmt>(Body))
- for (const auto *ParamDecl : FD->parameters())
- FnArgs.push_back(ParamDecl);
+ llvm::append_range(FnArgs, FD->parameters());
// Generate the body of the function.
PGO.assignRegionCounters(GD, CurFn);
@@ -1924,7 +1917,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
dest.getAlignment().alignmentOfArrayElement(baseSize);
// memcpy the individual element bit-pattern.
- Builder.CreateMemCpy(Address(cur, curAlign), src, baseSizeInChars,
+ Builder.CreateMemCpy(Address(cur, CGF.Int8Ty, curAlign), src, baseSizeInChars,
/*volatile*/ false);
// Go to the next element.
@@ -1997,7 +1990,7 @@ CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
CharUnits NullAlign = DestPtr.getAlignment();
NullVariable->setAlignment(NullAlign.getAsAlign());
Address SrcPtr(Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy()),
- NullAlign);
+ Builder.getInt8Ty(), NullAlign);
if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal);
@@ -2299,6 +2292,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
case Type::TypeOf:
case Type::UnaryTransform:
case Type::Attributed:
+ case Type::BTFTagAttributed:
case Type::SubstTemplateTypeParm:
case Type::MacroQualified:
// Keep walking after single level desugaring.
@@ -2475,7 +2469,7 @@ Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
V = Builder.CreateBitCast(V, VTy);
}
- return Address(V, Addr.getAlignment());
+ return Address(V, Addr.getElementType(), Addr.getAlignment());
}
CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
@@ -2536,16 +2530,13 @@ void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
llvm::StringMap<bool> CallerFeatureMap;
CGM.getContext().getFunctionFeatureMap(CallerFeatureMap, FD);
if (BuiltinID) {
- StringRef FeatureList(
- CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID));
- // Return if the builtin doesn't have any required features.
- if (FeatureList.empty())
- return;
- assert(!FeatureList.contains(' ') && "Space in feature list");
- TargetFeatures TF(CallerFeatureMap);
- if (!TF.hasRequiredFeatures(FeatureList))
+ StringRef FeatureList(CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID));
+ if (!Builtin::evaluateRequiredTargetFeatures(
+ FeatureList, CallerFeatureMap)) {
CGM.getDiags().Report(Loc, diag::err_builtin_needs_feature)
- << TargetDecl->getDeclName() << FeatureList;
+ << TargetDecl->getDeclName()
+ << FeatureList;
+ }
} else if (!TargetDecl->isMultiVersion() &&
TargetDecl->hasAttr<TargetAttr>()) {
// Get the required features for the callee.
@@ -2614,9 +2605,8 @@ static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
return;
}
- llvm::SmallVector<llvm::Value *, 10> Args;
- llvm::for_each(Resolver->args(),
- [&](llvm::Argument &Arg) { Args.push_back(&Arg); });
+ llvm::SmallVector<llvm::Value *, 10> Args(
+ llvm::make_pointer_range(Resolver->args()));
llvm::CallInst *Result = Builder.CreateCall(FuncToReturn, Args);
Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
@@ -2754,3 +2744,19 @@ CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
}
llvm_unreachable("Unknown Likelihood");
}
+
+llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec,
+ unsigned NumElementsDst,
+ const llvm::Twine &Name) {
+ auto *SrcTy = cast<llvm::FixedVectorType>(SrcVec->getType());
+ unsigned NumElementsSrc = SrcTy->getNumElements();
+ if (NumElementsSrc == NumElementsDst)
+ return SrcVec;
+
+ std::vector<int> ShuffleMask(NumElementsDst, -1);
+ for (unsigned MaskIdx = 0;
+ MaskIdx < std::min<>(NumElementsDst, NumElementsSrc); ++MaskIdx)
+ ShuffleMask[MaskIdx] = MaskIdx;
+
+ return Builder.CreateShuffleVector(SrcVec, ShuffleMask, Name);
+}
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index df99cd9a1b79..fe0890f433e8 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -201,10 +201,11 @@ template <> struct DominatingValue<RValue> {
AggregateAddress, ComplexAddress };
llvm::Value *Value;
+ llvm::Type *ElementType;
unsigned K : 3;
unsigned Align : 29;
- saved_type(llvm::Value *v, Kind k, unsigned a = 0)
- : Value(v), K(k), Align(a) {}
+ saved_type(llvm::Value *v, llvm::Type *e, Kind k, unsigned a = 0)
+ : Value(v), ElementType(e), K(k), Align(a) {}
public:
static bool needsSaving(RValue value);
@@ -551,6 +552,12 @@ public:
/// True if the current statement has nomerge attribute.
bool InNoMergeAttributedStmt = false;
+ /// True if the current statement has noinline attribute.
+ bool InNoInlineAttributedStmt = false;
+
+ /// True if the current statement has always_inline attribute.
+ bool InAlwaysInlineAttributedStmt = false;
+
// The CallExpr within the current statement that the musttail attribute
// applies to. nullptr if there is no 'musttail' on the current statement.
const CallExpr *MustTailCall = nullptr;
@@ -1065,15 +1072,14 @@ public:
/// Enter a new OpenMP private scope.
explicit OMPPrivateScope(CodeGenFunction &CGF) : RunCleanupsScope(CGF) {}
- /// Registers \p LocalVD variable as a private and apply \p PrivateGen
- /// function for it to generate corresponding private variable. \p
- /// PrivateGen returns an address of the generated private variable.
+ /// Registers \p LocalVD variable as a private with \p Addr as the address
+ /// of the corresponding private variable. \p
+ /// PrivateGen is the address of the generated private variable.
/// \return true if the variable is registered as private, false if it has
/// been privatized already.
- bool addPrivate(const VarDecl *LocalVD,
- const llvm::function_ref<Address()> PrivateGen) {
+ bool addPrivate(const VarDecl *LocalVD, Address Addr) {
assert(PerformCleanup && "adding private to dead scope");
- return MappedVars.setVarAddr(CGF, LocalVD, PrivateGen());
+ return MappedVars.setVarAddr(CGF, LocalVD, Addr);
}
/// Privatizes local variables previously registered as private.
@@ -1523,10 +1529,7 @@ public:
/// Get the profiler's count for the given statement.
uint64_t getProfileCount(const Stmt *S) {
- Optional<uint64_t> Count = PGO.getStmtCount(S);
- if (!Count.hasValue())
- return 0;
- return *Count;
+ return PGO.getStmtCount(S).value_or(0);
}
/// Set the profiler's current count.
@@ -1785,26 +1788,17 @@ public:
}
/// Emit the body of an OMP region
- /// \param CGF The Codegen function this belongs to
- /// \param RegionBodyStmt The body statement for the OpenMP region being
- /// generated
- /// \param CodeGenIP Insertion point for generating the body code.
- /// \param FiniBB The finalization basic block
- static void EmitOMPRegionBody(CodeGenFunction &CGF,
- const Stmt *RegionBodyStmt,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
- if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator())
- CodeGenIPBBTI->eraseFromParent();
-
- CGF.Builder.SetInsertPoint(CodeGenIPBB);
-
- CGF.EmitStmt(RegionBodyStmt);
-
- if (CGF.Builder.saveIP().isSet())
- CGF.Builder.CreateBr(&FiniBB);
- }
+ /// \param CGF The Codegen function this belongs to
+ /// \param RegionBodyStmt The body statement for the OpenMP region being
+ /// generated
+ /// \param AllocaIP Where to insert alloca instructions
+ /// \param CodeGenIP Where to insert the region code
+ /// \param RegionName Name to be used for new blocks
+ static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF,
+ const Stmt *RegionBodyStmt,
+ InsertPointTy AllocaIP,
+ InsertPointTy CodeGenIP,
+ Twine RegionName);
static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP,
llvm::BasicBlock &FiniBB, llvm::Function *Fn,
@@ -1824,12 +1818,25 @@ public:
CGF.Builder.CreateBr(&FiniBB);
}
+ /// Emit the body of an OMP region that will be outlined in
+ /// OpenMPIRBuilder::finalize().
+ /// \param CGF The Codegen function this belongs to
+ /// \param RegionBodyStmt The body statement for the OpenMP region being
+ /// generated
+ /// \param AllocaIP Where to insert alloca instructions
+ /// \param CodeGenIP Where to insert the region code
+ /// \param RegionName Name to be used for new blocks
+ static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF,
+ const Stmt *RegionBodyStmt,
+ InsertPointTy AllocaIP,
+ InsertPointTy CodeGenIP,
+ Twine RegionName);
+
/// RAII for preserving necessary info during Outlined region body codegen.
class OutlinedRegionBodyRAII {
llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
CodeGenFunction::JumpDest OldReturnBlock;
- CGBuilderTy::InsertPoint IP;
CodeGenFunction &CGF;
public:
@@ -1840,7 +1847,6 @@ public:
"Must specify Insertion point for allocas of outlined function");
OldAllocaIP = CGF.AllocaInsertPt;
CGF.AllocaInsertPt = &*AllocaIP.getPoint();
- IP = CGF.Builder.saveIP();
OldReturnBlock = CGF.ReturnBlock;
CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(&RetBB);
@@ -1849,7 +1855,6 @@ public:
~OutlinedRegionBodyRAII() {
CGF.AllocaInsertPt = OldAllocaIP;
CGF.ReturnBlock = OldReturnBlock;
- CGF.Builder.restoreIP(IP);
}
};
@@ -1963,8 +1968,7 @@ private:
/// Add OpenCL kernel arg metadata and the kernel attribute metadata to
/// the function metadata.
- void EmitOpenCLKernelMetadata(const FunctionDecl *FD,
- llvm::Function *Fn);
+ void EmitKernelMetadata(const FunctionDecl *FD, llvm::Function *Fn);
public:
CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext=false);
@@ -2296,9 +2300,8 @@ public:
/// Derived is the presumed address of an object of type T after a
/// cast. If T is a polymorphic class type, emit a check that the virtual
/// table for Derived belongs to a class derived from T.
- void EmitVTablePtrCheckForCast(QualType T, llvm::Value *Derived,
- bool MayBeNull, CFITypeCheckKind TCK,
- SourceLocation Loc);
+ void EmitVTablePtrCheckForCast(QualType T, Address Derived, bool MayBeNull,
+ CFITypeCheckKind TCK, SourceLocation Loc);
/// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
/// If vptr CFI is enabled, emit a check that VTable is valid.
@@ -2322,7 +2325,9 @@ public:
bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD);
/// Emit a type checked load from the given vtable.
- llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD, llvm::Value *VTable,
+ llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD,
+ llvm::Value *VTable,
+ llvm::Type *VTableTy,
uint64_t VTableByteOffset);
/// EnterDtorCleanups - Enter the cleanups necessary to complete the
@@ -2351,10 +2356,6 @@ public:
/// XRay typed event handling calls.
bool AlwaysEmitXRayTypedEvents() const;
- /// Encode an address into a form suitable for use in a function prologue.
- llvm::Constant *EncodeAddrForUseInPrologue(llvm::Function *F,
- llvm::Constant *Addr);
-
/// Decode an address used in a function prologue, encoded by \c
/// EncodeAddrForUseInPrologue.
llvm::Value *DecodeAddrUsedInPrologue(llvm::Value *F,
@@ -2520,6 +2521,9 @@ public:
return EmitLoadOfReferenceLValue(RefLVal);
}
+ /// Load a pointer with type \p PtrTy stored at address \p Ptr.
+ /// Note that \p PtrTy is the type of the loaded pointer, not the addresses
+ /// it is loaded from.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy,
LValueBaseInfo *BaseInfo = nullptr,
TBAAAccessInfo *TBAAInfo = nullptr);
@@ -3486,7 +3490,11 @@ public:
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,
const RegionCodeGenTy &BodyGen,
OMPTargetDataInfo &InputInfo);
-
+ void processInReduction(const OMPExecutableDirective &S,
+ OMPTaskDataTy &Data,
+ CodeGenFunction &CGF,
+ const CapturedStmt *CS,
+ OMPPrivateScope &Scope);
void EmitOMPMetaDirective(const OMPMetaDirective &S);
void EmitOMPParallelDirective(const OMPParallelDirective &S);
void EmitOMPSimdDirective(const OMPSimdDirective &S);
@@ -3905,6 +3913,7 @@ public:
LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);
LValue EmitInitListLValue(const InitListExpr *E);
+ void EmitIgnoredConditionalOperator(const AbstractConditionalOperator *E);
LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);
LValue EmitCastLValue(const CastExpr *E);
LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
@@ -4643,6 +4652,11 @@ public:
/// Set the codegen fast-math flags.
void SetFastMathFlags(FPOptions FPFeatures);
+ // Truncate or extend a boolean vector to the requested number of elements.
+ llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec,
+ unsigned NumElementsDst,
+ const llvm::Twine &Name = "");
+
private:
llvm::MDNode *getRangeForLoadFromType(QualType Ty);
void EmitReturnOfRValue(RValue RV, QualType Ty);
@@ -4796,76 +4810,6 @@ private:
llvm::Value *FormResolverCondition(const MultiVersionResolverOption &RO);
};
-/// TargetFeatures - This class is used to check whether the builtin function
-/// has the required tagert specific features. It is able to support the
-/// combination of ','(and), '|'(or), and '()'. By default, the priority of
-/// ',' is higher than that of '|' .
-/// E.g:
-/// A,B|C means the builtin function requires both A and B, or C.
-/// If we want the builtin function requires both A and B, or both A and C,
-/// there are two ways: A,B|A,C or A,(B|C).
-/// The FeaturesList should not contain spaces, and brackets must appear in
-/// pairs.
-class TargetFeatures {
- struct FeatureListStatus {
- bool HasFeatures;
- StringRef CurFeaturesList;
- };
-
- const llvm::StringMap<bool> &CallerFeatureMap;
-
- FeatureListStatus getAndFeatures(StringRef FeatureList) {
- int InParentheses = 0;
- bool HasFeatures = true;
- size_t SubexpressionStart = 0;
- for (size_t i = 0, e = FeatureList.size(); i < e; ++i) {
- char CurrentToken = FeatureList[i];
- switch (CurrentToken) {
- default:
- break;
- case '(':
- if (InParentheses == 0)
- SubexpressionStart = i + 1;
- ++InParentheses;
- break;
- case ')':
- --InParentheses;
- assert(InParentheses >= 0 && "Parentheses are not in pair");
- LLVM_FALLTHROUGH;
- case '|':
- case ',':
- if (InParentheses == 0) {
- if (HasFeatures && i != SubexpressionStart) {
- StringRef F = FeatureList.slice(SubexpressionStart, i);
- HasFeatures = CurrentToken == ')' ? hasRequiredFeatures(F)
- : CallerFeatureMap.lookup(F);
- }
- SubexpressionStart = i + 1;
- if (CurrentToken == '|') {
- return {HasFeatures, FeatureList.substr(SubexpressionStart)};
- }
- }
- break;
- }
- }
- assert(InParentheses == 0 && "Parentheses are not in pair");
- if (HasFeatures && SubexpressionStart != FeatureList.size())
- HasFeatures =
- CallerFeatureMap.lookup(FeatureList.substr(SubexpressionStart));
- return {HasFeatures, StringRef()};
- }
-
-public:
- bool hasRequiredFeatures(StringRef FeatureList) {
- FeatureListStatus FS = {false, FeatureList};
- while (!FS.HasFeatures && !FS.CurFeaturesList.empty())
- FS = getAndFeatures(FS.CurFeaturesList);
- return FS.HasFeatures;
- }
-
- TargetFeatures(const llvm::StringMap<bool> &CallerFeatureMap)
- : CallerFeatureMap(CallerFeatureMap) {}
-};
inline DominatingLLVMValue::saved_type
DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 2346176a1562..56ed59d1e3f1 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -16,6 +16,7 @@
#include "CGCXXABI.h"
#include "CGCall.h"
#include "CGDebugInfo.h"
+#include "CGHLSLRuntime.h"
#include "CGObjCRuntime.h"
#include "CGOpenCLRuntime.h"
#include "CGOpenMPRuntime.h"
@@ -43,6 +44,7 @@
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/Basic/Version.h"
+#include "clang/CodeGen/BackendUtil.h"
#include "clang/CodeGen/ConstantInitBuilder.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "llvm/ADT/StringSwitch.h"
@@ -68,9 +70,8 @@ using namespace clang;
using namespace CodeGen;
static llvm::cl::opt<bool> LimitedCoverage(
- "limited-coverage-experimental", llvm::cl::ZeroOrMore, llvm::cl::Hidden,
- llvm::cl::desc("Emit limited coverage mapping information (experimental)"),
- llvm::cl::init(false));
+ "limited-coverage-experimental", llvm::cl::Hidden,
+ llvm::cl::desc("Emit limited coverage mapping information (experimental)"));
static const char AnnotationSection[] = "llvm.metadata";
@@ -145,6 +146,8 @@ CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO,
createOpenMPRuntime();
if (LangOpts.CUDA)
createCUDARuntime();
+ if (LangOpts.HLSL)
+ createHLSLRuntime();
// Enable TBAA unless it's suppressed. ThreadSanitizer needs TBAA even at O0.
if (LangOpts.Sanitize.has(SanitizerKind::Thread) ||
@@ -261,6 +264,10 @@ void CodeGenModule::createCUDARuntime() {
CUDARuntime.reset(CreateNVCUDARuntime(*this));
}
+void CodeGenModule::createHLSLRuntime() {
+ HLSLRuntime.reset(new CGHLSLRuntime(*this));
+}
+
void CodeGenModule::addReplacement(StringRef Name, llvm::Constant *C) {
Replacements[Name] = C;
}
@@ -506,7 +513,6 @@ void CodeGenModule::Release() {
EmitVTablesOpportunistically();
applyGlobalValReplacements();
applyReplacements();
- checkAliases();
emitMultiVersionFunctions();
EmitCXXGlobalInitFunc();
EmitCXXGlobalCleanUpFunc();
@@ -538,6 +544,7 @@ void CodeGenModule::Release() {
EmitCtorList(GlobalDtors, "llvm.global_dtors");
EmitGlobalAnnotations();
EmitStaticExternCAliases();
+ checkAliases();
EmitDeferredUnusedCoverageMappings();
CodeGenPGO(*this).setValueProfilingFlag(getModule());
if (CoverageMapping)
@@ -547,27 +554,57 @@ void CodeGenModule::Release() {
CodeGenFunction(*this).EmitCfiCheckStub();
}
emitAtAvailableLinkGuard();
- if (Context.getTargetInfo().getTriple().isWasm() &&
- !Context.getTargetInfo().getTriple().isOSEmscripten()) {
+ if (Context.getTargetInfo().getTriple().isWasm())
EmitMainVoidAlias();
+
+ if (getTriple().isAMDGPU()) {
+ // Emit reference of __amdgpu_device_library_preserve_asan_functions to
+ // preserve ASAN functions in bitcode libraries.
+ if (LangOpts.Sanitize.has(SanitizerKind::Address)) {
+ auto *FT = llvm::FunctionType::get(VoidTy, {});
+ auto *F = llvm::Function::Create(
+ FT, llvm::GlobalValue::ExternalLinkage,
+ "__amdgpu_device_library_preserve_asan_functions", &getModule());
+ auto *Var = new llvm::GlobalVariable(
+ getModule(), FT->getPointerTo(),
+ /*isConstant=*/true, llvm::GlobalValue::WeakAnyLinkage, F,
+ "__amdgpu_device_library_preserve_asan_functions_ptr", nullptr,
+ llvm::GlobalVariable::NotThreadLocal);
+ addCompilerUsedGlobal(Var);
+ }
+ // Emit amdgpu_code_object_version module flag, which is code object version
+ // times 100.
+ // ToDo: Enable module flag for all code object version when ROCm device
+ // library is ready.
+ if (getTarget().getTargetOpts().CodeObjectVersion == TargetOptions::COV_5) {
+ getModule().addModuleFlag(llvm::Module::Error,
+ "amdgpu_code_object_version",
+ getTarget().getTargetOpts().CodeObjectVersion);
+ }
}
- // Emit reference of __amdgpu_device_library_preserve_asan_functions to
- // preserve ASAN functions in bitcode libraries.
- if (LangOpts.Sanitize.has(SanitizerKind::Address) && getTriple().isAMDGPU()) {
- auto *FT = llvm::FunctionType::get(VoidTy, {});
- auto *F = llvm::Function::Create(
- FT, llvm::GlobalValue::ExternalLinkage,
- "__amdgpu_device_library_preserve_asan_functions", &getModule());
- auto *Var = new llvm::GlobalVariable(
- getModule(), FT->getPointerTo(),
- /*isConstant=*/true, llvm::GlobalValue::WeakAnyLinkage, F,
- "__amdgpu_device_library_preserve_asan_functions_ptr", nullptr,
- llvm::GlobalVariable::NotThreadLocal);
- addCompilerUsedGlobal(Var);
- if (!getModule().getModuleFlag("amdgpu_hostcall")) {
- getModule().addModuleFlag(llvm::Module::Override, "amdgpu_hostcall", 1);
+ // Emit a global array containing all external kernels or device variables
+ // used by host functions and mark it as used for CUDA/HIP. This is necessary
+ // to get kernels or device variables in archives linked in even if these
+ // kernels or device variables are only used in host functions.
+ if (!Context.CUDAExternalDeviceDeclODRUsedByHost.empty()) {
+ SmallVector<llvm::Constant *, 8> UsedArray;
+ for (auto D : Context.CUDAExternalDeviceDeclODRUsedByHost) {
+ GlobalDecl GD;
+ if (auto *FD = dyn_cast<FunctionDecl>(D))
+ GD = GlobalDecl(FD, KernelReferenceKind::Kernel);
+ else
+ GD = GlobalDecl(D);
+ UsedArray.push_back(llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
+ GetAddrOfGlobal(GD), Int8PtrTy));
}
+
+ llvm::ArrayType *ATy = llvm::ArrayType::get(Int8PtrTy, UsedArray.size());
+
+ auto *GV = new llvm::GlobalVariable(
+ getModule(), ATy, false, llvm::GlobalValue::InternalLinkage,
+ llvm::ConstantArray::get(ATy, UsedArray), "__clang_gpu_used_external");
+ addCompilerUsedGlobal(GV);
}
emitLLVMUsed();
@@ -720,13 +757,16 @@ void CodeGenModule::Release() {
// attributes, but we use module metadata to emit build attributes. This is
// needed for LTO, where the function attributes are inside bitcode
// serialised into a global variable by the time build attributes are
- // emitted, so we can't access them.
+ // emitted, so we can't access them. LTO objects could be compiled with
+ // different flags therefore module flags are set to "Min" behavior to achieve
+ // the same end result of the normal build where e.g BTI is off if any object
+ // doesn't support it.
if (Context.getTargetInfo().hasFeature("ptrauth") &&
LangOpts.getSignReturnAddressScope() !=
LangOptions::SignReturnAddressScopeKind::None)
getModule().addModuleFlag(llvm::Module::Override,
"sign-return-address-buildattr", 1);
- if (LangOpts.Sanitize.has(SanitizerKind::MemTag))
+ if (LangOpts.Sanitize.has(SanitizerKind::MemtagStack))
getModule().addModuleFlag(llvm::Module::Override,
"tag-stack-memory-buildattr", 1);
@@ -734,16 +774,16 @@ void CodeGenModule::Release() {
Arch == llvm::Triple::arm || Arch == llvm::Triple::armeb ||
Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_32 ||
Arch == llvm::Triple::aarch64_be) {
- getModule().addModuleFlag(llvm::Module::Error, "branch-target-enforcement",
+ getModule().addModuleFlag(llvm::Module::Min, "branch-target-enforcement",
LangOpts.BranchTargetEnforcement);
- getModule().addModuleFlag(llvm::Module::Error, "sign-return-address",
+ getModule().addModuleFlag(llvm::Module::Min, "sign-return-address",
LangOpts.hasSignReturnAddress());
- getModule().addModuleFlag(llvm::Module::Error, "sign-return-address-all",
+ getModule().addModuleFlag(llvm::Module::Min, "sign-return-address-all",
LangOpts.isSignReturnAddressScopeAll());
- getModule().addModuleFlag(llvm::Module::Error,
+ getModule().addModuleFlag(llvm::Module::Min,
"sign-return-address-with-bkey",
!LangOpts.isSignReturnAddressWithAKey());
}
@@ -775,7 +815,7 @@ void CodeGenModule::Release() {
LangOpts.OpenMP);
// Emit OpenCL specific module metadata: OpenCL/SPIR version.
- if (LangOpts.OpenCL) {
+ if (LangOpts.OpenCL || (LangOpts.CUDAIsDevice && getTriple().isSPIRV())) {
EmitOpenCLMetadata();
// Emit SPIR version.
if (getTriple().isSPIR()) {
@@ -796,6 +836,10 @@ void CodeGenModule::Release() {
}
}
+ // HLSL related end of code gen work items.
+ if (LangOpts.HLSL)
+ getHLSLRuntime().finishCodeGen();
+
if (uint32_t PLevel = Context.getLangOpts().PICLevel) {
assert(PLevel < 3 && "Invalid PIC Level");
getModule().setPICLevel(static_cast<llvm::PICLevel::Level>(PLevel));
@@ -820,7 +864,7 @@ void CodeGenModule::Release() {
if (CodeGenOpts.NoPLT)
getModule().setRtLibUseGOT();
if (CodeGenOpts.UnwindTables)
- getModule().setUwtable();
+ getModule().setUwtable(llvm::UWTableKind(CodeGenOpts.UnwindTables));
switch (CodeGenOpts.getFramePointer()) {
case CodeGenOptions::FramePointerKind::None:
@@ -868,6 +912,9 @@ void CodeGenModule::Release() {
EmitBackendOptionsMetadata(getCodeGenOpts());
+ // If there is device offloading code embed it in the host now.
+ EmbedObject(&getModule(), CodeGenOpts, getDiags());
+
// Set visibility from DLL storage class
// We do this at the end of LLVM IR generation; after any operation
// that might affect the DLL storage class or the visibility, and
@@ -1167,7 +1214,9 @@ void CodeGenModule::setDLLImportDLLExport(llvm::GlobalValue *GV,
if (D && D->isExternallyVisible()) {
if (D->hasAttr<DLLImportAttr>())
GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
- else if (D->hasAttr<DLLExportAttr>() && !GV->isDeclarationForLinker())
+ else if ((D->hasAttr<DLLExportAttr>() ||
+ shouldMapVisibilityToDLLExport(D)) &&
+ !GV->isDeclarationForLinker())
GV->setDLLStorageClass(llvm::GlobalVariable::DLLExportStorageClass);
}
}
@@ -1364,10 +1413,11 @@ static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD,
}
// Make unique name for device side static file-scope variable for HIP.
- if (CGM.getContext().shouldExternalizeStaticVar(ND) &&
+ if (CGM.getContext().shouldExternalize(ND) &&
CGM.getLangOpts().GPURelocatableDeviceCode &&
- CGM.getLangOpts().CUDAIsDevice && !CGM.getLangOpts().CUID.empty())
- CGM.printPostfixForExternalizedStaticVar(Out);
+ CGM.getLangOpts().CUDAIsDevice)
+ CGM.printPostfixForExternalizedDecl(Out, ND);
+
return std::string(Out.str());
}
@@ -1434,8 +1484,7 @@ StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
// static device variable depends on whether the variable is referenced by
// a host or device host function. Therefore the mangled name cannot be
// cached.
- if (!LangOpts.CUDAIsDevice ||
- !getContext().mayExternalizeStaticVar(GD.getDecl())) {
+ if (!LangOpts.CUDAIsDevice || !getContext().mayExternalize(GD.getDecl())) {
auto FoundName = MangledDeclNames.find(CanonicalGD);
if (FoundName != MangledDeclNames.end())
return FoundName->second;
@@ -1455,7 +1504,7 @@ StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
// directly between host- and device-compilations, the host- and
// device-mangling in host compilation could help catching certain ones.
assert(!isa<FunctionDecl>(ND) || !ND->hasAttr<CUDAGlobalAttr>() ||
- getLangOpts().CUDAIsDevice ||
+ getContext().shouldExternalize(ND) || getLangOpts().CUDAIsDevice ||
(getContext().getAuxTargetInfo() &&
(getContext().getAuxTargetInfo()->getCXXABI() !=
getContext().getTargetInfo().getCXXABI())) ||
@@ -1490,6 +1539,16 @@ StringRef CodeGenModule::getBlockMangledName(GlobalDecl GD,
return Result.first->first();
}
+const GlobalDecl CodeGenModule::getMangledNameDecl(StringRef Name) {
+ auto it = MangledDeclNames.begin();
+ while (it != MangledDeclNames.end()) {
+ if (it->second == Name)
+ return it->first;
+ it++;
+ }
+ return GlobalDecl();
+}
+
llvm::GlobalValue *CodeGenModule::GetGlobalValue(StringRef Name) {
return getModule().getNamedValue(Name);
}
@@ -1638,7 +1697,7 @@ static unsigned ArgInfoAddressSpace(LangAS AS) {
}
}
-void CodeGenModule::GenOpenCLArgMetadata(llvm::Function *Fn,
+void CodeGenModule::GenKernelArgMetadata(llvm::Function *Fn,
const FunctionDecl *FD,
CodeGenFunction *CGF) {
assert(((FD && CGF) || (!FD && !CGF)) &&
@@ -1670,6 +1729,11 @@ void CodeGenModule::GenOpenCLArgMetadata(llvm::Function *Fn,
if (FD && CGF)
for (unsigned i = 0, e = FD->getNumParams(); i != e; ++i) {
const ParmVarDecl *parm = FD->getParamDecl(i);
+ // Get argument name.
+ argNames.push_back(llvm::MDString::get(VMContext, parm->getName()));
+
+ if (!getLangOpts().OpenCL)
+ continue;
QualType ty = parm->getType();
std::string typeQuals;
@@ -1688,9 +1752,6 @@ void CodeGenModule::GenOpenCLArgMetadata(llvm::Function *Fn,
} else
accessQuals.push_back(llvm::MDString::get(VMContext, "none"));
- // Get argument name.
- argNames.push_back(llvm::MDString::get(VMContext, parm->getName()));
-
auto getTypeSpelling = [&](QualType Ty) {
auto typeName = Ty.getUnqualifiedType().getAsString(Policy);
@@ -1763,17 +1824,20 @@ void CodeGenModule::GenOpenCLArgMetadata(llvm::Function *Fn,
argTypeQuals.push_back(llvm::MDString::get(VMContext, typeQuals));
}
- Fn->setMetadata("kernel_arg_addr_space",
- llvm::MDNode::get(VMContext, addressQuals));
- Fn->setMetadata("kernel_arg_access_qual",
- llvm::MDNode::get(VMContext, accessQuals));
- Fn->setMetadata("kernel_arg_type",
- llvm::MDNode::get(VMContext, argTypeNames));
- Fn->setMetadata("kernel_arg_base_type",
- llvm::MDNode::get(VMContext, argBaseTypeNames));
- Fn->setMetadata("kernel_arg_type_qual",
- llvm::MDNode::get(VMContext, argTypeQuals));
- if (getCodeGenOpts().EmitOpenCLArgMetadata)
+ if (getLangOpts().OpenCL) {
+ Fn->setMetadata("kernel_arg_addr_space",
+ llvm::MDNode::get(VMContext, addressQuals));
+ Fn->setMetadata("kernel_arg_access_qual",
+ llvm::MDNode::get(VMContext, accessQuals));
+ Fn->setMetadata("kernel_arg_type",
+ llvm::MDNode::get(VMContext, argTypeNames));
+ Fn->setMetadata("kernel_arg_base_type",
+ llvm::MDNode::get(VMContext, argBaseTypeNames));
+ Fn->setMetadata("kernel_arg_type_qual",
+ llvm::MDNode::get(VMContext, argTypeQuals));
+ }
+ if (getCodeGenOpts().EmitOpenCLArgMetadata ||
+ getCodeGenOpts().HIPSaveKernelArgName)
Fn->setMetadata("kernel_arg_name",
llvm::MDNode::get(VMContext, argNames));
}
@@ -1826,12 +1890,28 @@ CodeGenModule::getMostBaseClasses(const CXXRecordDecl *RD) {
return MostBases.takeVector();
}
+llvm::GlobalVariable *
+CodeGenModule::GetOrCreateRTTIProxyGlobalVariable(llvm::Constant *Addr) {
+ auto It = RTTIProxyMap.find(Addr);
+ if (It != RTTIProxyMap.end())
+ return It->second;
+
+ auto *FTRTTIProxy = new llvm::GlobalVariable(
+ TheModule, Addr->getType(),
+ /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, Addr,
+ "__llvm_rtti_proxy");
+ FTRTTIProxy->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
+
+ RTTIProxyMap[Addr] = FTRTTIProxy;
+ return FTRTTIProxy;
+}
+
void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
llvm::Function *F) {
llvm::AttrBuilder B(F->getContext());
if (CodeGenOpts.UnwindTables)
- B.addAttribute(llvm::Attribute::UWTable);
+ B.addUWTableAttr(llvm::UWTableKind(CodeGenOpts.UnwindTables));
if (CodeGenOpts.StackClashProtector)
B.addAttribute("probe-stack", "inline-asm");
@@ -2052,6 +2132,13 @@ bool CodeGenModule::GetCPUAndFeaturesAttributes(GlobalDecl GD,
getTarget().isValidCPUName(ParsedAttr.Tune))
TuneCPU = ParsedAttr.Tune;
}
+
+ if (SD) {
+ // Apply the given CPU name as the 'tune-cpu' so that the optimizer can
+ // favor this processor.
+ TuneCPU = getTarget().getCPUSpecificTuneName(
+ SD->getCPUName(GD.getMultiVersionIndex())->getName());
+ }
} else {
// Otherwise just add the existing target cpu and target features to the
// function.
@@ -2489,8 +2576,8 @@ void CodeGenModule::EmitDeferred() {
// Note we should not clear CUDADeviceVarODRUsedByHost since it is still
// needed for further handling.
if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice)
- for (const auto *V : getContext().CUDADeviceVarODRUsedByHost)
- DeferredDeclsToEmit.push_back(V);
+ llvm::append_range(DeferredDeclsToEmit,
+ getContext().CUDADeviceVarODRUsedByHost);
// Stop if we're out of both deferred vtables and deferred declarations.
if (DeferredDeclsToEmit.empty())
@@ -2694,21 +2781,14 @@ bool CodeGenModule::isInNoSanitizeList(SanitizerMask Kind, llvm::Function *Fn,
return false;
}
-bool CodeGenModule::isInNoSanitizeList(llvm::GlobalVariable *GV,
+bool CodeGenModule::isInNoSanitizeList(SanitizerMask Kind,
+ llvm::GlobalVariable *GV,
SourceLocation Loc, QualType Ty,
StringRef Category) const {
- // For now globals can be ignored only in ASan and KASan.
- const SanitizerMask EnabledAsanMask =
- LangOpts.Sanitize.Mask &
- (SanitizerKind::Address | SanitizerKind::KernelAddress |
- SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress |
- SanitizerKind::MemTag);
- if (!EnabledAsanMask)
- return false;
const auto &NoSanitizeL = getContext().getNoSanitizeList();
- if (NoSanitizeL.containsGlobal(EnabledAsanMask, GV->getName(), Category))
+ if (NoSanitizeL.containsGlobal(Kind, GV->getName(), Category))
return true;
- if (NoSanitizeL.containsLocation(EnabledAsanMask, Loc, Category))
+ if (NoSanitizeL.containsLocation(Kind, Loc, Category))
return true;
// Check global type.
if (!Ty.isNull()) {
@@ -2720,7 +2800,7 @@ bool CodeGenModule::isInNoSanitizeList(llvm::GlobalVariable *GV,
// Only record types (classes, structs etc.) are ignored.
if (Ty->isRecordType()) {
std::string TypeStr = Ty.getAsString(getContext().getPrintingPolicy());
- if (NoSanitizeL.containsType(EnabledAsanMask, TypeStr, Category))
+ if (NoSanitizeL.containsType(Kind, TypeStr, Category))
return true;
}
}
@@ -2762,12 +2842,12 @@ bool CodeGenModule::isProfileInstrExcluded(llvm::Function *Fn,
CodeGenOptions::ProfileInstrKind Kind = getCodeGenOpts().getProfileInstr();
// First, check the function name.
Optional<bool> V = ProfileList.isFunctionExcluded(Fn->getName(), Kind);
- if (V.hasValue())
+ if (V)
return *V;
// Next, check the source location.
if (Loc.isValid()) {
Optional<bool> V = ProfileList.isLocationExcluded(Loc, Kind);
- if (V.hasValue())
+ if (V)
return *V;
}
// If location is unknown, this may be a compiler-generated function. Assume
@@ -2775,7 +2855,7 @@ bool CodeGenModule::isProfileInstrExcluded(llvm::Function *Fn,
auto &SM = Context.getSourceManager();
if (const auto *MainFile = SM.getFileEntryForID(SM.getMainFileID())) {
Optional<bool> V = ProfileList.isFileExcluded(MainFile->getName(), Kind);
- if (V.hasValue())
+ if (V)
return *V;
}
return ProfileList.getDefault();
@@ -2886,6 +2966,37 @@ ConstantAddress CodeGenModule::GetAddrOfMSGuidDecl(const MSGuidDecl *GD) {
return ConstantAddress(Addr, Ty, Alignment);
}
+ConstantAddress CodeGenModule::GetAddrOfUnnamedGlobalConstantDecl(
+ const UnnamedGlobalConstantDecl *GCD) {
+ CharUnits Alignment = getContext().getTypeAlignInChars(GCD->getType());
+
+ llvm::GlobalVariable **Entry = nullptr;
+ Entry = &UnnamedGlobalConstantDeclMap[GCD];
+ if (*Entry)
+ return ConstantAddress(*Entry, (*Entry)->getValueType(), Alignment);
+
+ ConstantEmitter Emitter(*this);
+ llvm::Constant *Init;
+
+ const APValue &V = GCD->getValue();
+
+ assert(!V.isAbsent());
+ Init = Emitter.emitForInitializer(V, GCD->getType().getAddressSpace(),
+ GCD->getType());
+
+ auto *GV = new llvm::GlobalVariable(getModule(), Init->getType(),
+ /*isConstant=*/true,
+ llvm::GlobalValue::PrivateLinkage, Init,
+ ".constant");
+ GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
+ GV->setAlignment(Alignment.getAsAlign());
+
+ Emitter.finalize(GV);
+
+ *Entry = GV;
+ return ConstantAddress(GV, GV->getValueType(), Alignment);
+}
+
ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject(
const TemplateParamObjectDecl *TPO) {
StringRef Name = getMangledName(TPO);
@@ -3270,13 +3381,13 @@ void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD,
auto *Spec = FD->getAttr<CPUSpecificAttr>();
for (unsigned I = 0; I < Spec->cpus_size(); ++I)
EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
- // Requires multiple emits.
} else if (FD->isTargetClonesMultiVersion()) {
auto *Clone = FD->getAttr<TargetClonesAttr>();
for (unsigned I = 0; I < Clone->featuresStrs_size(); ++I)
if (Clone->isFirstOfVersion(I))
EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
- EmitTargetClonesResolver(GD);
+ // Ensure that the resolver function is also emitted.
+ GetOrCreateMultiVersionResolver(GD);
} else
EmitGlobalFunctionDefinition(GD, GV);
}
@@ -3358,112 +3469,94 @@ llvm::GlobalValue::LinkageTypes getMultiversionLinkage(CodeGenModule &CGM,
return llvm::GlobalValue::WeakODRLinkage;
}
-void CodeGenModule::EmitTargetClonesResolver(GlobalDecl GD) {
- const auto *FD = cast<FunctionDecl>(GD.getDecl());
- assert(FD && "Not a FunctionDecl?");
- const auto *TC = FD->getAttr<TargetClonesAttr>();
- assert(TC && "Not a target_clones Function?");
-
- QualType CanonTy = Context.getCanonicalType(FD->getType());
- llvm::Type *DeclTy = getTypes().ConvertType(CanonTy);
-
- if (const auto *CXXFD = dyn_cast<CXXMethodDecl>(FD)) {
- const CGFunctionInfo &FInfo = getTypes().arrangeCXXMethodDeclaration(CXXFD);
- DeclTy = getTypes().GetFunctionType(FInfo);
- }
-
- llvm::Function *ResolverFunc;
- if (getTarget().supportsIFunc()) {
- auto *IFunc = cast<llvm::GlobalIFunc>(
- GetOrCreateMultiVersionResolver(GD, DeclTy, FD));
- ResolverFunc = cast<llvm::Function>(IFunc->getResolver());
- } else
- ResolverFunc =
- cast<llvm::Function>(GetOrCreateMultiVersionResolver(GD, DeclTy, FD));
-
- SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
- for (unsigned VersionIndex = 0; VersionIndex < TC->featuresStrs_size();
- ++VersionIndex) {
- if (!TC->isFirstOfVersion(VersionIndex))
- continue;
- StringRef Version = TC->getFeatureStr(VersionIndex);
- StringRef MangledName =
- getMangledName(GD.getWithMultiVersionIndex(VersionIndex));
- llvm::Constant *Func = GetGlobalValue(MangledName);
- assert(Func &&
- "Should have already been created before calling resolver emit");
-
- StringRef Architecture;
- llvm::SmallVector<StringRef, 1> Feature;
-
- if (Version.startswith("arch="))
- Architecture = Version.drop_front(sizeof("arch=") - 1);
- else if (Version != "default")
- Feature.push_back(Version);
-
- Options.emplace_back(cast<llvm::Function>(Func), Architecture, Feature);
- }
-
- const TargetInfo &TI = getTarget();
- std::stable_sort(
- Options.begin(), Options.end(),
- [&TI](const CodeGenFunction::MultiVersionResolverOption &LHS,
- const CodeGenFunction::MultiVersionResolverOption &RHS) {
- return TargetMVPriority(TI, LHS) > TargetMVPriority(TI, RHS);
- });
- CodeGenFunction CGF(*this);
- CGF.EmitMultiVersionResolver(ResolverFunc, Options);
-}
-
void CodeGenModule::emitMultiVersionFunctions() {
std::vector<GlobalDecl> MVFuncsToEmit;
MultiVersionFuncs.swap(MVFuncsToEmit);
for (GlobalDecl GD : MVFuncsToEmit) {
+ const auto *FD = cast<FunctionDecl>(GD.getDecl());
+ assert(FD && "Expected a FunctionDecl");
+
SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
- const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
- getContext().forEachMultiversionedFunctionVersion(
- FD, [this, &GD, &Options](const FunctionDecl *CurFD) {
- GlobalDecl CurGD{
- (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)};
- StringRef MangledName = getMangledName(CurGD);
- llvm::Constant *Func = GetGlobalValue(MangledName);
- if (!Func) {
- if (CurFD->isDefined()) {
- EmitGlobalFunctionDefinition(CurGD, nullptr);
- Func = GetGlobalValue(MangledName);
- } else {
- const CGFunctionInfo &FI =
- getTypes().arrangeGlobalDeclaration(GD);
- llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
- Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false,
- /*DontDefer=*/false, ForDefinition);
+ if (FD->isTargetMultiVersion()) {
+ getContext().forEachMultiversionedFunctionVersion(
+ FD, [this, &GD, &Options](const FunctionDecl *CurFD) {
+ GlobalDecl CurGD{
+ (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)};
+ StringRef MangledName = getMangledName(CurGD);
+ llvm::Constant *Func = GetGlobalValue(MangledName);
+ if (!Func) {
+ if (CurFD->isDefined()) {
+ EmitGlobalFunctionDefinition(CurGD, nullptr);
+ Func = GetGlobalValue(MangledName);
+ } else {
+ const CGFunctionInfo &FI =
+ getTypes().arrangeGlobalDeclaration(GD);
+ llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
+ Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false,
+ /*DontDefer=*/false, ForDefinition);
+ }
+ assert(Func && "This should have just been created");
}
- assert(Func && "This should have just been created");
- }
- const auto *TA = CurFD->getAttr<TargetAttr>();
- llvm::SmallVector<StringRef, 8> Feats;
- TA->getAddedFeatures(Feats);
+ const auto *TA = CurFD->getAttr<TargetAttr>();
+ llvm::SmallVector<StringRef, 8> Feats;
+ TA->getAddedFeatures(Feats);
- Options.emplace_back(cast<llvm::Function>(Func),
- TA->getArchitecture(), Feats);
- });
+ Options.emplace_back(cast<llvm::Function>(Func),
+ TA->getArchitecture(), Feats);
+ });
+ } else if (FD->isTargetClonesMultiVersion()) {
+ const auto *TC = FD->getAttr<TargetClonesAttr>();
+ for (unsigned VersionIndex = 0; VersionIndex < TC->featuresStrs_size();
+ ++VersionIndex) {
+ if (!TC->isFirstOfVersion(VersionIndex))
+ continue;
+ GlobalDecl CurGD{(FD->isDefined() ? FD->getDefinition() : FD),
+ VersionIndex};
+ StringRef Version = TC->getFeatureStr(VersionIndex);
+ StringRef MangledName = getMangledName(CurGD);
+ llvm::Constant *Func = GetGlobalValue(MangledName);
+ if (!Func) {
+ if (FD->isDefined()) {
+ EmitGlobalFunctionDefinition(CurGD, nullptr);
+ Func = GetGlobalValue(MangledName);
+ } else {
+ const CGFunctionInfo &FI =
+ getTypes().arrangeGlobalDeclaration(CurGD);
+ llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
+ Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false,
+ /*DontDefer=*/false, ForDefinition);
+ }
+ assert(Func && "This should have just been created");
+ }
- llvm::Function *ResolverFunc;
- const TargetInfo &TI = getTarget();
+ StringRef Architecture;
+ llvm::SmallVector<StringRef, 1> Feature;
+
+ if (Version.startswith("arch="))
+ Architecture = Version.drop_front(sizeof("arch=") - 1);
+ else if (Version != "default")
+ Feature.push_back(Version);
- if (TI.supportsIFunc() || FD->isTargetMultiVersion()) {
- ResolverFunc = cast<llvm::Function>(
- GetGlobalValue((getMangledName(GD) + ".resolver").str()));
- ResolverFunc->setLinkage(getMultiversionLinkage(*this, GD));
+ Options.emplace_back(cast<llvm::Function>(Func), Architecture, Feature);
+ }
} else {
- ResolverFunc = cast<llvm::Function>(GetGlobalValue(getMangledName(GD)));
+ assert(0 && "Expected a target or target_clones multiversion function");
+ continue;
}
+ llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD);
+ if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(ResolverConstant))
+ ResolverConstant = IFunc->getResolver();
+ llvm::Function *ResolverFunc = cast<llvm::Function>(ResolverConstant);
+
+ ResolverFunc->setLinkage(getMultiversionLinkage(*this, GD));
+
if (supportsCOMDAT())
ResolverFunc->setComdat(
getModule().getOrInsertComdat(ResolverFunc->getName()));
+ const TargetInfo &TI = getTarget();
llvm::stable_sort(
Options, [&TI](const CodeGenFunction::MultiVersionResolverOption &LHS,
const CodeGenFunction::MultiVersionResolverOption &RHS) {
@@ -3491,12 +3584,9 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
assert(FD->isCPUDispatchMultiVersion() && "Not a multiversion function?");
const auto *DD = FD->getAttr<CPUDispatchAttr>();
assert(DD && "Not a cpu_dispatch Function?");
- llvm::Type *DeclTy = getTypes().ConvertType(FD->getType());
- if (const auto *CXXFD = dyn_cast<CXXMethodDecl>(FD)) {
- const CGFunctionInfo &FInfo = getTypes().arrangeCXXMethodDeclaration(CXXFD);
- DeclTy = getTypes().GetFunctionType(FInfo);
- }
+ const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
+ llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI);
StringRef ResolverName = getMangledName(GD);
UpdateMultiVersionNames(GD, FD, ResolverName);
@@ -3585,16 +3675,27 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
CGF.EmitMultiVersionResolver(ResolverFunc, Options);
if (getTarget().supportsIFunc()) {
+ llvm::GlobalValue::LinkageTypes Linkage = getMultiversionLinkage(*this, GD);
+ auto *IFunc = cast<llvm::GlobalValue>(GetOrCreateMultiVersionResolver(GD));
+
+ // Fix up function declarations that were created for cpu_specific before
+ // cpu_dispatch was known
+ if (!isa<llvm::GlobalIFunc>(IFunc)) {
+ assert(cast<llvm::Function>(IFunc)->isDeclaration());
+ auto *GI = llvm::GlobalIFunc::create(DeclTy, 0, Linkage, "", ResolverFunc,
+ &getModule());
+ GI->takeName(IFunc);
+ IFunc->replaceAllUsesWith(GI);
+ IFunc->eraseFromParent();
+ IFunc = GI;
+ }
+
std::string AliasName = getMangledNameImpl(
*this, GD, FD, /*OmitMultiVersionMangling=*/true);
llvm::Constant *AliasFunc = GetGlobalValue(AliasName);
if (!AliasFunc) {
- auto *IFunc = cast<llvm::GlobalIFunc>(GetOrCreateLLVMFunction(
- AliasName, DeclTy, GD, /*ForVTable=*/false, /*DontDefer=*/true,
- /*IsThunk=*/false, llvm::AttributeList(), NotForDefinition));
- auto *GA = llvm::GlobalAlias::create(DeclTy, 0,
- getMultiversionLinkage(*this, GD),
- AliasName, IFunc, &getModule());
+ auto *GA = llvm::GlobalAlias::create(DeclTy, 0, Linkage, AliasName, IFunc,
+ &getModule());
SetCommonAttributes(GD, GA);
}
}
@@ -3602,8 +3703,10 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
/// If a dispatcher for the specified mangled name is not in the module, create
/// and return an llvm Function with the specified type.
-llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(
- GlobalDecl GD, llvm::Type *DeclTy, const FunctionDecl *FD) {
+llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) {
+ const auto *FD = cast<FunctionDecl>(GD.getDecl());
+ assert(FD && "Not a FunctionDecl?");
+
std::string MangledName =
getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true);
@@ -3615,34 +3718,21 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(
else if (FD->isTargetMultiVersion())
ResolverName += ".resolver";
- // If this already exists, just return that one.
+ // If the resolver has already been created, just return it.
if (llvm::GlobalValue *ResolverGV = GetGlobalValue(ResolverName))
return ResolverGV;
- // Since this is the first time we've created this IFunc, make sure
- // that we put this multiversioned function into the list to be
- // replaced later if necessary (target multiversioning only).
- if (FD->isTargetMultiVersion())
+ const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
+ llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI);
+
+ // The resolver needs to be created. For target and target_clones, defer
+ // creation until the end of the TU.
+ if (FD->isTargetMultiVersion() || FD->isTargetClonesMultiVersion())
MultiVersionFuncs.push_back(GD);
- else if (FD->isTargetClonesMultiVersion()) {
- // In target_clones multiversioning, make sure we emit this if used.
- auto DDI =
- DeferredDecls.find(getMangledName(GD.getWithMultiVersionIndex(0)));
- if (DDI != DeferredDecls.end()) {
- addDeferredDeclToEmit(GD);
- DeferredDecls.erase(DDI);
- } else {
- // Emit the symbol of the 1st variant, so that the deferred decls know we
- // need it, otherwise the only global value will be the resolver/ifunc,
- // which end up getting broken if we search for them with GetGlobalValue'.
- GetOrCreateLLVMFunction(
- getMangledName(GD.getWithMultiVersionIndex(0)), DeclTy, FD,
- /*ForVTable=*/false, /*DontDefer=*/true,
- /*IsThunk=*/false, llvm::AttributeList(), ForDefinition);
- }
- }
- if (getTarget().supportsIFunc()) {
+ // For cpu_specific, don't create an ifunc yet because we don't know if the
+ // cpu_dispatch will be emitted in this translation unit.
+ if (getTarget().supportsIFunc() && !FD->isCPUSpecificMultiVersion()) {
llvm::Type *ResolverType = llvm::FunctionType::get(
llvm::PointerType::get(
DeclTy, getContext().getTargetAddressSpace(FD->getType())),
@@ -3700,9 +3790,9 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(
}
if (FD->isMultiVersion()) {
- UpdateMultiVersionNames(GD, FD, MangledName);
+ UpdateMultiVersionNames(GD, FD, MangledName);
if (!IsForDefinition)
- return GetOrCreateMultiVersionResolver(GD, Ty, FD);
+ return GetOrCreateMultiVersionResolver(GD);
}
}
@@ -3716,7 +3806,8 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(
}
// Handle dropped DLL attributes.
- if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>()) {
+ if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>() &&
+ !shouldMapVisibilityToDLLExport(cast_or_null<NamedDecl>(D))) {
Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
setDSOLocal(Entry);
}
@@ -4028,7 +4119,8 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty,
}
// Handle dropped DLL attributes.
- if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>())
+ if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>() &&
+ !shouldMapVisibilityToDLLExport(D))
Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
if (LangOpts.OpenMP && !LangOpts.OpenMPSimd && D)
@@ -4381,8 +4473,16 @@ LangAS CodeGenModule::GetGlobalConstantAddressSpace() const {
return LangAS::opencl_constant;
if (LangOpts.SYCLIsDevice)
return LangAS::sycl_global;
+ if (LangOpts.HIP && LangOpts.CUDAIsDevice && getTriple().isSPIRV())
+ // For HIPSPV map literals to cuda_device (maps to CrossWorkGroup in SPIR-V)
+ // instead of default AS (maps to Generic in SPIR-V). Otherwise, we end up
+ // with OpVariable instructions with Generic storage class which is not
+ // allowed (SPIR-V V1.6 s3.42.8). Also, mapping literals to SPIR-V
+ // UniformConstant storage class is not viable as pointers to it may not be
+ // casted to Generic pointers which are used to model HIP's "flat" pointers.
+ return LangAS::cuda_device;
if (auto AS = getTarget().getConstantAddressSpace())
- return AS.getValue();
+ return *AS;
return LangAS::Default;
}
@@ -4542,6 +4642,8 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
T = D->getType();
if (getLangOpts().CPlusPlus) {
+ if (InitDecl->hasFlexibleArrayInit(getContext()))
+ ErrorUnsupported(D, "flexible array initializer");
Init = EmitNullConstant(T);
NeedsGlobalCtor = true;
} else {
@@ -4555,6 +4657,14 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
// also don't need to register a destructor.
if (getLangOpts().CPlusPlus && !NeedsGlobalDtor)
DelayedCXXInitPosition.erase(D);
+
+#ifndef NDEBUG
+ CharUnits VarSize = getContext().getTypeSizeInChars(ASTTy) +
+ InitDecl->getFlexibleArrayInitChars(getContext());
+ CharUnits CstSize = CharUnits::fromQuantity(
+ getDataLayout().getTypeAllocSize(Init->getType()));
+ assert(VarSize == CstSize && "Emitted constant has unexpected size");
+#endif
}
}
@@ -4643,7 +4753,12 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
GV->setConstant(true);
}
- GV->setAlignment(getContext().getDeclAlign(D).getAsAlign());
+ CharUnits AlignVal = getContext().getDeclAlign(D);
+ // Check for alignment specifed in an 'omp allocate' directive.
+ if (llvm::Optional<CharUnits> AlignValFromAllocate =
+ getOMPAllocateAlignment(D))
+ AlignVal = *AlignValFromAllocate;
+ GV->setAlignment(AlignVal.getAsAlign());
// On Darwin, unlike other Itanium C++ ABI platforms, the thread-wrapper
// function is only defined alongside the variable, not also alongside
@@ -4698,7 +4813,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
if (NeedsGlobalCtor || NeedsGlobalDtor)
EmitCXXGlobalVarDeclInitFunc(D, GV, NeedsGlobalCtor);
- SanitizerMD->reportGlobalToASan(GV, *D, NeedsGlobalCtor);
+ SanitizerMD->reportGlobal(GV, *D, NeedsGlobalCtor);
// Emit global variable debug information.
if (CGDebugInfo *DI = getModuleDebugInfo())
@@ -4799,12 +4914,8 @@ llvm::GlobalValue::LinkageTypes CodeGenModule::getLLVMLinkageForDeclarator(
if (Linkage == GVA_Internal)
return llvm::Function::InternalLinkage;
- if (D->hasAttr<WeakAttr>()) {
- if (IsConstantVariable)
- return llvm::GlobalVariable::WeakODRLinkage;
- else
- return llvm::GlobalVariable::WeakAnyLinkage;
- }
+ if (D->hasAttr<WeakAttr>())
+ return llvm::GlobalVariable::WeakAnyLinkage;
if (const auto *FD = D->getAsFunction())
if (FD->isMultiVersion() && Linkage == GVA_AvailableExternally)
@@ -5134,6 +5245,11 @@ void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) {
setTLSMode(GA, *VD);
SetCommonAttributes(GD, GA);
+
+ // Emit global alias debug information.
+ if (isa<VarDecl>(D))
+ if (CGDebugInfo *DI = getModuleDebugInfo())
+ DI->EmitGlobalAlias(cast<llvm::GlobalValue>(GA->getAliasee()), GD);
}
void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) {
@@ -5335,7 +5451,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
auto Fields = Builder.beginStruct(STy);
// Class pointer.
- Fields.add(cast<llvm::ConstantExpr>(CFConstantStringClassRef));
+ Fields.add(cast<llvm::Constant>(CFConstantStringClassRef));
// Flags.
if (IsSwiftABI) {
@@ -5414,10 +5530,11 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
switch (Triple.getObjectFormat()) {
case llvm::Triple::UnknownObjectFormat:
llvm_unreachable("unknown file format");
+ case llvm::Triple::DXContainer:
case llvm::Triple::GOFF:
- llvm_unreachable("GOFF is not yet implemented");
+ case llvm::Triple::SPIRV:
case llvm::Triple::XCOFF:
- llvm_unreachable("XCOFF is not yet implemented");
+ llvm_unreachable("unimplemented");
case llvm::Triple::COFF:
case llvm::Triple::ELF:
case llvm::Triple::Wasm:
@@ -5570,11 +5687,15 @@ CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S,
}
auto GV = GenerateStringLiteral(C, LT, *this, GlobalVariableName, Alignment);
+
+ CGDebugInfo *DI = getModuleDebugInfo();
+ if (DI && getCodeGenOpts().hasReducedDebugInfo())
+ DI->AddStringLiteralDebugInfo(GV, S);
+
if (Entry)
*Entry = GV;
- SanitizerMD->reportGlobalToASan(GV, S->getStrTokenLoc(0), "<string literal>",
- QualType());
+ SanitizerMD->reportGlobal(GV, S->getStrTokenLoc(0), "<string literal>");
return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
GV->getValueType(), Alignment);
@@ -5654,9 +5775,11 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary(
getModule(), Type, false, llvm::GlobalVariable::InternalLinkage,
nullptr);
}
- return ConstantAddress(
- InsertResult.first->second,
- InsertResult.first->second->getType()->getPointerElementType(), Align);
+ return ConstantAddress(InsertResult.first->second,
+ llvm::cast<llvm::GlobalVariable>(
+ InsertResult.first->second->stripPointerCasts())
+ ->getValueType(),
+ Align);
}
// FIXME: If an externally-visible declaration extends multiple temporaries,
@@ -5725,6 +5848,9 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary(
/*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, TargetAS);
if (emitter) emitter->finalize(GV);
setGVProperties(GV, VD);
+ if (GV->getDLLStorageClass() == llvm::GlobalVariable::DLLExportStorageClass)
+ // The reference temporary should never be dllexport.
+ GV->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass);
GV->setAlignment(Align.getAsAlign());
if (supportsCOMDAT() && GV->isWeakForLinker())
GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
@@ -6241,8 +6367,10 @@ void CodeGenModule::EmitMainVoidAlias() {
// new-style no-argument main is in used.
if (llvm::Function *F = getModule().getFunction("main")) {
if (!F->isDeclaration() && F->arg_size() == 0 && !F->isVarArg() &&
- F->getReturnType()->isIntegerTy(Context.getTargetInfo().getIntWidth()))
- addUsedGlobal(llvm::GlobalAlias::create("__main_void", F));
+ F->getReturnType()->isIntegerTy(Context.getTargetInfo().getIntWidth())) {
+ auto *GA = llvm::GlobalAlias::create("__main_void", F);
+ GA->setVisibility(llvm::GlobalValue::HiddenVisibility);
+ }
}
}
@@ -6269,6 +6397,72 @@ static void EmitGlobalDeclMetadata(CodeGenModule &CGM,
GlobalMetadata->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops));
}
+bool CodeGenModule::CheckAndReplaceExternCIFuncs(llvm::GlobalValue *Elem,
+ llvm::GlobalValue *CppFunc) {
+ // Store the list of ifuncs we need to replace uses in.
+ llvm::SmallVector<llvm::GlobalIFunc *> IFuncs;
+ // List of ConstantExprs that we should be able to delete when we're done
+ // here.
+ llvm::SmallVector<llvm::ConstantExpr *> CEs;
+
+ // It isn't valid to replace the extern-C ifuncs if all we find is itself!
+ if (Elem == CppFunc)
+ return false;
+
+ // First make sure that all users of this are ifuncs (or ifuncs via a
+ // bitcast), and collect the list of ifuncs and CEs so we can work on them
+ // later.
+ for (llvm::User *User : Elem->users()) {
+ // Users can either be a bitcast ConstExpr that is used by the ifuncs, OR an
+ // ifunc directly. In any other case, just give up, as we don't know what we
+ // could break by changing those.
+ if (auto *ConstExpr = dyn_cast<llvm::ConstantExpr>(User)) {
+ if (ConstExpr->getOpcode() != llvm::Instruction::BitCast)
+ return false;
+
+ for (llvm::User *CEUser : ConstExpr->users()) {
+ if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(CEUser)) {
+ IFuncs.push_back(IFunc);
+ } else {
+ return false;
+ }
+ }
+ CEs.push_back(ConstExpr);
+ } else if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(User)) {
+ IFuncs.push_back(IFunc);
+ } else {
+ // This user is one we don't know how to handle, so fail redirection. This
+ // will result in an ifunc retaining a resolver name that will ultimately
+ // fail to be resolved to a defined function.
+ return false;
+ }
+ }
+
+ // Now we know this is a valid case where we can do this alias replacement, we
+ // need to remove all of the references to Elem (and the bitcasts!) so we can
+ // delete it.
+ for (llvm::GlobalIFunc *IFunc : IFuncs)
+ IFunc->setResolver(nullptr);
+ for (llvm::ConstantExpr *ConstExpr : CEs)
+ ConstExpr->destroyConstant();
+
+ // We should now be out of uses for the 'old' version of this function, so we
+ // can erase it as well.
+ Elem->eraseFromParent();
+
+ for (llvm::GlobalIFunc *IFunc : IFuncs) {
+ // The type of the resolver is always just a function-type that returns the
+ // type of the IFunc, so create that here. If the type of the actual
+ // resolver doesn't match, it just gets bitcast to the right thing.
+ auto *ResolverTy =
+ llvm::FunctionType::get(IFunc->getType(), /*isVarArg*/ false);
+ llvm::Constant *Resolver = GetOrCreateLLVMFunction(
+ CppFunc->getName(), ResolverTy, {}, /*ForVTable*/ false);
+ IFunc->setResolver(Resolver);
+ }
+ return true;
+}
+
/// For each function which is declared within an extern "C" region and marked
/// as 'used', but has internal linkage, create an alias from the unmangled
/// name to the mangled name if possible. People expect to be able to refer
@@ -6280,7 +6474,19 @@ void CodeGenModule::EmitStaticExternCAliases() {
for (auto &I : StaticExternCValues) {
IdentifierInfo *Name = I.first;
llvm::GlobalValue *Val = I.second;
- if (Val && !getModule().getNamedValue(Name->getName()))
+
+ // If Val is null, that implies there were multiple declarations that each
+ // had a claim to the unmangled name. In this case, generation of the alias
+ // is suppressed. See CodeGenModule::MaybeHandleStaticInExternC.
+ if (!Val)
+ break;
+
+ llvm::GlobalValue *ExistingElem =
+ getModule().getNamedValue(Name->getName());
+
+ // If there is either not something already by this name, or we were able to
+ // replace all uses from IFuncs, create the alias.
+ if (!ExistingElem || CheckAndReplaceExternCIFuncs(ExistingElem, Val))
addCompilerUsedGlobal(llvm::GlobalAlias::create(Name->getName(), Val));
}
}
@@ -6410,7 +6616,9 @@ void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) {
!VD->getAnyInitializer()->isConstantInitializer(getContext(),
/*ForRef=*/false);
- Address Addr(GetAddrOfGlobalVar(VD), getContext().getDeclAlign(VD));
+ Address Addr(GetAddrOfGlobalVar(VD),
+ getTypes().ConvertTypeForMem(VD->getType()),
+ getContext().getDeclAlign(VD));
if (auto InitFunction = getOpenMPRuntime().emitThreadPrivateVarDefinition(
VD, Addr, RefExpr->getBeginLoc(), PerformInit))
CXXGlobalInits.push_back(InitFunction);
@@ -6637,7 +6845,40 @@ bool CodeGenModule::stopAutoInit() {
return false;
}
-void CodeGenModule::printPostfixForExternalizedStaticVar(
- llvm::raw_ostream &OS) const {
- OS << "__static__" << getContext().getCUIDHash();
+void CodeGenModule::printPostfixForExternalizedDecl(llvm::raw_ostream &OS,
+ const Decl *D) const {
+ // ptxas does not allow '.' in symbol names. On the other hand, HIP prefers
+ // postfix beginning with '.' since the symbol name can be demangled.
+ if (LangOpts.HIP)
+ OS << (isa<VarDecl>(D) ? ".static." : ".intern.");
+ else
+ OS << (isa<VarDecl>(D) ? "__static__" : "__intern__");
+
+ // If the CUID is not specified we try to generate a unique postfix.
+ if (getLangOpts().CUID.empty()) {
+ SourceManager &SM = getContext().getSourceManager();
+ PresumedLoc PLoc = SM.getPresumedLoc(D->getLocation());
+ assert(PLoc.isValid() && "Source location is expected to be valid.");
+
+ // Get the hash of the user defined macros.
+ llvm::MD5 Hash;
+ llvm::MD5::MD5Result Result;
+ for (const auto &Arg : PreprocessorOpts.Macros)
+ Hash.update(Arg.first);
+ Hash.final(Result);
+
+ // Get the UniqueID for the file containing the decl.
+ llvm::sys::fs::UniqueID ID;
+ if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID)) {
+ PLoc = SM.getPresumedLoc(D->getLocation(), /*UseLineDirectives=*/false);
+ assert(PLoc.isValid() && "Source location is expected to be valid.");
+ if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
+ SM.getDiagnostics().Report(diag::err_cannot_open_file)
+ << PLoc.getFilename() << EC.message();
+ }
+ OS << llvm::format("%x", ID.getFile()) << llvm::format("%x", ID.getDevice())
+ << "_" << llvm::utohexstr(Result.low(), /*LowerCase=*/true, /*Width=*/8);
+ } else {
+ OS << getContext().getCUIDHash();
+ }
}
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index 1fcd5d4d808a..da43b9616c88 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -85,6 +85,7 @@ class CGObjCRuntime;
class CGOpenCLRuntime;
class CGOpenMPRuntime;
class CGCUDARuntime;
+class CGHLSLRuntime;
class CoverageMappingModuleGen;
class TargetCodeGenInfo;
@@ -319,6 +320,7 @@ private:
std::unique_ptr<CGOpenCLRuntime> OpenCLRuntime;
std::unique_ptr<CGOpenMPRuntime> OpenMPRuntime;
std::unique_ptr<CGCUDARuntime> CUDARuntime;
+ std::unique_ptr<CGHLSLRuntime> HLSLRuntime;
std::unique_ptr<CGDebugInfo> DebugInfo;
std::unique_ptr<ObjCEntrypoints> ObjCData;
llvm::MDNode *NoObjCARCExceptionsMetadata = nullptr;
@@ -348,8 +350,9 @@ private:
/// is defined once we get to the end of the of the translation unit.
std::vector<GlobalDecl> Aliases;
- /// List of multiversion functions that have to be emitted. Used to make sure
- /// we properly emit the iFunc.
+ /// List of multiversion functions to be emitted. This list is processed in
+ /// conjunction with other deferred symbols and is used to ensure that
+ /// multiversion function resolvers and ifuncs are defined and emitted.
std::vector<GlobalDecl> MultiVersionFuncs;
typedef llvm::StringMap<llvm::TrackingVH<llvm::Constant> > ReplacementsTy;
@@ -406,6 +409,8 @@ private:
llvm::StringMap<llvm::GlobalVariable *> CFConstantStringMap;
llvm::DenseMap<llvm::Constant *, llvm::GlobalVariable *> ConstantStringMap;
+ llvm::DenseMap<const UnnamedGlobalConstantDecl *, llvm::GlobalVariable *>
+ UnnamedGlobalConstantDeclMap;
llvm::DenseMap<const Decl*, llvm::Constant *> StaticLocalDeclMap;
llvm::DenseMap<const Decl*, llvm::GlobalVariable*> StaticLocalDeclGuardMap;
llvm::DenseMap<const Expr*, llvm::Constant *> MaterializedGlobalTemporaryMap;
@@ -509,6 +514,7 @@ private:
void createOpenCLRuntime();
void createOpenMPRuntime();
void createCUDARuntime();
+ void createHLSLRuntime();
bool isTriviallyRecursive(const FunctionDecl *F);
bool shouldEmitFunction(GlobalDecl GD);
@@ -561,6 +567,8 @@ private:
MetadataTypeMap VirtualMetadataIdMap;
MetadataTypeMap GeneralizedMetadataIdMap;
+ llvm::DenseMap<const llvm::Constant *, llvm::GlobalVariable *> RTTIProxyMap;
+
public:
CodeGenModule(ASTContext &C, const HeaderSearchOptions &headersearchopts,
const PreprocessorOptions &ppopts,
@@ -607,6 +615,12 @@ public:
return *CUDARuntime;
}
+ /// Return a reference to the configured HLSL runtime.
+ CGHLSLRuntime &getHLSLRuntime() {
+ assert(HLSLRuntime != nullptr);
+ return *HLSLRuntime;
+ }
+
ObjCEntrypoints &getObjCEntrypoints() const {
assert(ObjCData != nullptr);
return *ObjCData;
@@ -786,6 +800,14 @@ public:
void setDSOLocal(llvm::GlobalValue *GV) const;
+ bool shouldMapVisibilityToDLLExport(const NamedDecl *D) const {
+ return getLangOpts().hasDefaultVisibilityExportMapping() && D &&
+ (D->getLinkageAndVisibility().getVisibility() ==
+ DefaultVisibility) &&
+ (getLangOpts().isAllDefaultVisibilityExportMapping() ||
+ (getLangOpts().isExplicitDefaultVisibilityExportMapping() &&
+ D->getLinkageAndVisibility().isVisibilityExplicit()));
+ }
void setDLLImportDLLExport(llvm::GlobalValue *GV, GlobalDecl D) const;
void setDLLImportDLLExport(llvm::GlobalValue *GV, const NamedDecl *D) const;
/// Set visibility, dllimport/dllexport and dso_local.
@@ -826,7 +848,9 @@ public:
llvm::Function *CreateGlobalInitOrCleanUpFunction(
llvm::FunctionType *ty, const Twine &name, const CGFunctionInfo &FI,
- SourceLocation Loc = SourceLocation(), bool TLS = false);
+ SourceLocation Loc = SourceLocation(), bool TLS = false,
+ llvm::GlobalVariable::LinkageTypes Linkage =
+ llvm::GlobalVariable::InternalLinkage);
/// Return the AST address space of the underlying global variable for D, as
/// determined by its declaration. Normally this is the same as the address
@@ -873,6 +897,10 @@ public:
/// Get the address of a GUID.
ConstantAddress GetAddrOfMSGuidDecl(const MSGuidDecl *GD);
+ /// Get the address of a UnnamedGlobalConstant
+ ConstantAddress
+ GetAddrOfUnnamedGlobalConstantDecl(const UnnamedGlobalConstantDecl *GCD);
+
/// Get the address of a template parameter object.
ConstantAddress
GetAddrOfTemplateParamObject(const TemplateParamObjectDecl *TPO);
@@ -1213,6 +1241,7 @@ public:
StringRef getMangledName(GlobalDecl GD);
StringRef getBlockMangledName(GlobalDecl GD, const BlockDecl *BD);
+ const GlobalDecl getMangledNameDecl(StringRef);
void EmitTentativeDefinition(const VarDecl *D);
@@ -1287,8 +1316,9 @@ public:
bool isInNoSanitizeList(SanitizerMask Kind, llvm::Function *Fn,
SourceLocation Loc) const;
- bool isInNoSanitizeList(llvm::GlobalVariable *GV, SourceLocation Loc,
- QualType Ty, StringRef Category = StringRef()) const;
+ bool isInNoSanitizeList(SanitizerMask Kind, llvm::GlobalVariable *GV,
+ SourceLocation Loc, QualType Ty,
+ StringRef Category = StringRef()) const;
/// Imbue XRay attributes to a function, applying the always/never attribute
/// lists in the process. Returns true if we did imbue attributes this way,
@@ -1346,15 +1376,18 @@ public:
/// \param D The allocate declaration
void EmitOMPAllocateDecl(const OMPAllocateDecl *D);
+ /// Return the alignment specified in an allocate directive, if present.
+ llvm::Optional<CharUnits> getOMPAllocateAlignment(const VarDecl *VD);
+
/// Returns whether the given record has hidden LTO visibility and therefore
/// may participate in (single-module) CFI and whole-program vtable
/// optimization.
bool HasHiddenLTOVisibility(const CXXRecordDecl *RD);
- /// Returns whether the given record has public std LTO visibility
- /// and therefore may not participate in (single-module) CFI and whole-program
- /// vtable optimization.
- bool HasLTOVisibilityPublicStd(const CXXRecordDecl *RD);
+ /// Returns whether the given record has public LTO visibility (regardless of
+ /// -lto-whole-program-visibility) and therefore may not participate in
+ /// (single-module) CFI and whole-program vtable optimization.
+ bool AlwaysHasLTOVisibilityPublic(const CXXRecordDecl *RD);
/// Returns the vcall visibility of the given type. This is the scope in which
/// a virtual function call could be made which ends up being dispatched to a
@@ -1411,6 +1444,9 @@ public:
std::vector<const CXXRecordDecl *>
getMostBaseClasses(const CXXRecordDecl *RD);
+ llvm::GlobalVariable *
+ GetOrCreateRTTIProxyGlobalVariable(llvm::Constant *Addr);
+
/// Get the declaration of std::terminate for the platform.
llvm::FunctionCallee getTerminateFn();
@@ -1429,7 +1465,7 @@ public:
/// \param FN is a pointer to IR function being generated.
/// \param FD is a pointer to function declaration if any.
/// \param CGF is a pointer to CodeGenFunction that generates this function.
- void GenOpenCLArgMetadata(llvm::Function *FN,
+ void GenKernelArgMetadata(llvm::Function *FN,
const FunctionDecl *FD = nullptr,
CodeGenFunction *CGF = nullptr);
@@ -1447,9 +1483,40 @@ public:
TBAAAccessInfo *TBAAInfo = nullptr);
bool stopAutoInit();
- /// Print the postfix for externalized static variable for single source
- /// offloading languages CUDA and HIP.
- void printPostfixForExternalizedStaticVar(llvm::raw_ostream &OS) const;
+ /// Print the postfix for externalized static variable or kernels for single
+ /// source offloading languages CUDA and HIP. The unique postfix is created
+ /// using either the CUID argument, or the file's UniqueID and active macros.
+ /// The fallback method without a CUID requires that the offloading toolchain
+ /// does not define separate macros via the -cc1 options.
+ void printPostfixForExternalizedDecl(llvm::raw_ostream &OS,
+ const Decl *D) const;
+
+ /// Move some lazily-emitted states to the NewBuilder. This is especially
+ /// essential for the incremental parsing environment like Clang Interpreter,
+ /// because we'll lose all important information after each repl.
+ void moveLazyEmissionStates(CodeGenModule *NewBuilder) {
+ assert(DeferredDeclsToEmit.empty() &&
+ "Should have emitted all decls deferred to emit.");
+ assert(NewBuilder->DeferredDecls.empty() &&
+ "Newly created module should not have deferred decls");
+ NewBuilder->DeferredDecls = std::move(DeferredDecls);
+
+ assert(NewBuilder->DeferredVTables.empty() &&
+ "Newly created module should not have deferred vtables");
+ NewBuilder->DeferredVTables = std::move(DeferredVTables);
+
+ assert(NewBuilder->MangledDeclNames.empty() &&
+ "Newly created module should not have mangled decl names");
+ assert(NewBuilder->Manglings.empty() &&
+ "Newly created module should not have manglings");
+ NewBuilder->Manglings = std::move(Manglings);
+
+ assert(WeakRefReferences.empty() &&
+ "Not all WeakRefRefs have been applied");
+ NewBuilder->WeakRefReferences = std::move(WeakRefReferences);
+
+ NewBuilder->TBAA = std::move(TBAA);
+ }
private:
llvm::Constant *GetOrCreateLLVMFunction(
@@ -1458,9 +1525,18 @@ private:
llvm::AttributeList ExtraAttrs = llvm::AttributeList(),
ForDefinition_t IsForDefinition = NotForDefinition);
- llvm::Constant *GetOrCreateMultiVersionResolver(GlobalDecl GD,
- llvm::Type *DeclTy,
- const FunctionDecl *FD);
+ // References to multiversion functions are resolved through an implicitly
+ // defined resolver function. This function is responsible for creating
+ // the resolver symbol for the provided declaration. The value returned
+ // will be for an ifunc (llvm::GlobalIFunc) if the current target supports
+ // that feature and for a regular function (llvm::GlobalValue) otherwise.
+ llvm::Constant *GetOrCreateMultiVersionResolver(GlobalDecl GD);
+
+ // In scenarios where a function is not known to be a multiversion function
+ // until a later declaration, it is sometimes necessary to change the
+ // previously created mangled name to align with requirements of whatever
+ // multiversion function kind the function is now known to be. This function
+ // is responsible for performing such mangled name updates.
void UpdateMultiVersionNames(GlobalDecl GD, const FunctionDecl *FD,
StringRef &CurName);
@@ -1487,7 +1563,6 @@ private:
void EmitAliasDefinition(GlobalDecl GD);
void emitIFuncDefinition(GlobalDecl GD);
void emitCPUDispatchDefinition(GlobalDecl GD);
- void EmitTargetClonesResolver(GlobalDecl GD);
void EmitObjCPropertyImplementations(const ObjCImplementationDecl *D);
void EmitObjCIvarInitializations(ObjCImplementationDecl *D);
@@ -1553,6 +1628,7 @@ private:
// registered by the atexit subroutine using unatexit.
void unregisterGlobalDtorsWithUnAtExit();
+ /// Emit deferred multiversion function resolvers and associated variants.
void emitMultiVersionFunctions();
/// Emit any vtables which we deferred and still have a use for.
@@ -1568,6 +1644,16 @@ private:
/// Emit the link options introduced by imported modules.
void EmitModuleLinkOptions();
+ /// Helper function for EmitStaticExternCAliases() to redirect ifuncs that
+ /// have a resolver name that matches 'Elem' to instead resolve to the name of
+ /// 'CppFunc'. This redirection is necessary in cases where 'Elem' has a name
+ /// that will be emitted as an alias of the name bound to 'CppFunc'; ifuncs
+ /// may not reference aliases. Redirection is only performed if 'Elem' is only
+ /// used by ifuncs in which case, 'Elem' is destroyed. 'true' is returned if
+ /// redirection is successful, and 'false' is returned otherwise.
+ bool CheckAndReplaceExternCIFuncs(llvm::GlobalValue *Elem,
+ llvm::GlobalValue *CppFunc);
+
/// Emit aliases for internal-linkage declarations inside "C" language
/// linkage specifications, giving them the "expected" name where possible.
void EmitStaticExternCAliases();
diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp
index 6657f2a91e3d..587bcef78ee5 100644
--- a/clang/lib/CodeGen/CodeGenPGO.cpp
+++ b/clang/lib/CodeGen/CodeGenPGO.cpp
@@ -23,7 +23,7 @@
#include "llvm/Support/MD5.h"
static llvm::cl::opt<bool>
- EnableValueProfiling("enable-value-profiling", llvm::cl::ZeroOrMore,
+ EnableValueProfiling("enable-value-profiling",
llvm::cl::desc("Enable value profiling"),
llvm::cl::Hidden, llvm::cl::init(false));
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index 4839e22c4b14..fcce424747f1 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -25,6 +25,7 @@
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Module.h"
+
using namespace clang;
using namespace CodeGen;
@@ -97,6 +98,14 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T, bool ForBitField) {
llvm::Type *R = ConvertType(T);
+ // Check for the boolean vector case.
+ if (T->isExtVectorBoolType()) {
+ auto *FixedVT = cast<llvm::FixedVectorType>(R);
+ // Pad to at least one byte.
+ uint64_t BytePadded = std::max<uint64_t>(FixedVT->getNumElements(), 8);
+ return llvm::IntegerType::get(FixedVT->getContext(), BytePadded);
+ }
+
// If this is a bool type, or a bit-precise integer type in a bitfield
// representation, map this integer to the target-specified size.
if ((ForBitField && T->isBitIntType()) ||
@@ -382,9 +391,6 @@ llvm::Type *CodeGenTypes::ConvertFunctionTypeInternal(QualType QFT) {
RecordsBeingLaidOut.erase(Ty);
- if (SkippedLayout)
- TypeCache.clear();
-
if (RecordsBeingLaidOut.empty())
while (!DeferredRecords.empty())
ConvertRecordDeclType(DeferredRecords.pop_back_val());
@@ -415,11 +421,27 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
if (const RecordType *RT = dyn_cast<RecordType>(Ty))
return ConvertRecordDeclType(RT->getDecl());
- // See if type is already cached.
- llvm::DenseMap<const Type *, llvm::Type *>::iterator TCI = TypeCache.find(Ty);
- // If type is found in map then use it. Otherwise, convert type T.
- if (TCI != TypeCache.end())
- return TCI->second;
+ // The LLVM type we return for a given Clang type may not always be the same,
+ // most notably when dealing with recursive structs. We mark these potential
+ // cases with ShouldUseCache below. Builtin types cannot be recursive.
+ // TODO: when clang uses LLVM opaque pointers we won't be able to represent
+ // recursive types with LLVM types, making this logic much simpler.
+ llvm::Type *CachedType = nullptr;
+ bool ShouldUseCache =
+ Ty->isBuiltinType() ||
+ (noRecordsBeingLaidOut() && FunctionsBeingProcessed.empty());
+ if (ShouldUseCache) {
+ llvm::DenseMap<const Type *, llvm::Type *>::iterator TCI =
+ TypeCache.find(Ty);
+ if (TCI != TypeCache.end())
+ CachedType = TCI->second;
+ // With expensive checks, check that the type we compute matches the
+ // cached type.
+#ifndef EXPENSIVE_CHECKS
+ if (CachedType)
+ return CachedType;
+#endif
+ }
// If we don't have it in the cache, convert it now.
llvm::Type *ResultType = nullptr;
@@ -687,9 +709,12 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
}
case Type::ExtVector:
case Type::Vector: {
- const VectorType *VT = cast<VectorType>(Ty);
- ResultType = llvm::FixedVectorType::get(ConvertType(VT->getElementType()),
- VT->getNumElements());
+ const auto *VT = cast<VectorType>(Ty);
+ // An ext_vector_type of Bool is really a vector of bits.
+ llvm::Type *IRElemTy = VT->isExtVectorBoolType()
+ ? llvm::Type::getInt1Ty(getLLVMContext())
+ : ConvertType(VT->getElementType());
+ ResultType = llvm::FixedVectorType::get(IRElemTy, VT->getNumElements());
break;
}
case Type::ConstantMatrix: {
@@ -758,8 +783,11 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
case Type::MemberPointer: {
auto *MPTy = cast<MemberPointerType>(Ty);
if (!getCXXABI().isMemberPointerConvertible(MPTy)) {
- RecordsWithOpaqueMemberPointers.insert(MPTy->getClass());
- ResultType = llvm::StructType::create(getLLVMContext());
+ auto *C = MPTy->getClass();
+ auto Insertion = RecordsWithOpaqueMemberPointers.insert({C, nullptr});
+ if (Insertion.second)
+ Insertion.first->second = llvm::StructType::create(getLLVMContext());
+ ResultType = Insertion.first->second;
} else {
ResultType = getCXXABI().ConvertMemberPointerType(MPTy);
}
@@ -796,8 +824,11 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
}
assert(ResultType && "Didn't convert a type?");
+ assert((!CachedType || CachedType == ResultType) &&
+ "Cached type doesn't match computed type");
- TypeCache[Ty] = ResultType;
+ if (ShouldUseCache)
+ TypeCache[Ty] = ResultType;
return ResultType;
}
diff --git a/clang/lib/CodeGen/CodeGenTypes.h b/clang/lib/CodeGen/CodeGenTypes.h
index 28b831222943..cd20563cbf75 100644
--- a/clang/lib/CodeGen/CodeGenTypes.h
+++ b/clang/lib/CodeGen/CodeGenTypes.h
@@ -76,7 +76,7 @@ class CodeGenTypes {
llvm::DenseMap<const Type*, llvm::StructType *> RecordDeclTypes;
/// Hold memoized CGFunctionInfo results.
- llvm::FoldingSet<CGFunctionInfo> FunctionInfos;
+ llvm::FoldingSet<CGFunctionInfo> FunctionInfos{FunctionInfosLog2InitSize};
/// This set keeps track of records that we're currently converting
/// to an IR type. For example, when converting:
@@ -96,8 +96,9 @@ class CodeGenTypes {
/// corresponding llvm::Type.
llvm::DenseMap<const Type *, llvm::Type *> TypeCache;
- llvm::SmallSet<const Type *, 8> RecordsWithOpaqueMemberPointers;
+ llvm::DenseMap<const Type *, llvm::Type *> RecordsWithOpaqueMemberPointers;
+ static constexpr unsigned FunctionInfosLog2InitSize = 9;
/// Helper for ConvertType.
llvm::Type *ConvertFunctionTypeInternal(QualType FT);
diff --git a/clang/lib/CodeGen/ConstantInitBuilder.cpp b/clang/lib/CodeGen/ConstantInitBuilder.cpp
index 24e3ca19709c..06d3e44f01b1 100644
--- a/clang/lib/CodeGen/ConstantInitBuilder.cpp
+++ b/clang/lib/CodeGen/ConstantInitBuilder.cpp
@@ -114,7 +114,7 @@ void ConstantInitBuilderBase::abandon(size_t newEnd) {
if (newEnd == 0) {
for (auto &entry : SelfReferences) {
auto dummy = entry.Dummy;
- dummy->replaceAllUsesWith(llvm::UndefValue::get(dummy->getType()));
+ dummy->replaceAllUsesWith(llvm::PoisonValue::get(dummy->getType()));
dummy->eraseFromParent();
}
SelfReferences.clear();
diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp
index 9b81c8a670f5..0fe084b628da 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.cpp
+++ b/clang/lib/CodeGen/CoverageMappingGen.cpp
@@ -60,26 +60,27 @@ CoverageMappingModuleGen::setUpCoverageCallbacks(Preprocessor &PP) {
return CoverageInfo;
}
-void CoverageSourceInfo::AddSkippedRange(SourceRange Range) {
+void CoverageSourceInfo::AddSkippedRange(SourceRange Range,
+ SkippedRange::Kind RangeKind) {
if (EmptyLineCommentCoverage && !SkippedRanges.empty() &&
PrevTokLoc == SkippedRanges.back().PrevTokLoc &&
SourceMgr.isWrittenInSameFile(SkippedRanges.back().Range.getEnd(),
Range.getBegin()))
SkippedRanges.back().Range.setEnd(Range.getEnd());
else
- SkippedRanges.push_back({Range, PrevTokLoc});
+ SkippedRanges.push_back({Range, RangeKind, PrevTokLoc});
}
void CoverageSourceInfo::SourceRangeSkipped(SourceRange Range, SourceLocation) {
- AddSkippedRange(Range);
+ AddSkippedRange(Range, SkippedRange::PPIfElse);
}
void CoverageSourceInfo::HandleEmptyline(SourceRange Range) {
- AddSkippedRange(Range);
+ AddSkippedRange(Range, SkippedRange::EmptyLine);
}
bool CoverageSourceInfo::HandleComment(Preprocessor &PP, SourceRange Range) {
- AddSkippedRange(Range);
+ AddSkippedRange(Range, SkippedRange::Comment);
return false;
}
@@ -129,7 +130,7 @@ public:
void setCounter(Counter C) { Count = C; }
- bool hasStartLoc() const { return LocStart.hasValue(); }
+ bool hasStartLoc() const { return LocStart.has_value(); }
void setStartLoc(SourceLocation Loc) { LocStart = Loc; }
@@ -138,7 +139,7 @@ public:
return *LocStart;
}
- bool hasEndLoc() const { return LocEnd.hasValue(); }
+ bool hasEndLoc() const { return LocEnd.has_value(); }
void setEndLoc(SourceLocation Loc) {
assert(Loc.isValid() && "Setting an invalid end location");
@@ -154,7 +155,7 @@ public:
void setGap(bool Gap) { GapRegion = Gap; }
- bool isBranch() const { return FalseCount.hasValue(); }
+ bool isBranch() const { return FalseCount.has_value(); }
};
/// Spelling locations for the start and end of a source region.
@@ -335,6 +336,8 @@ public:
/// This shrinks the skipped range if it spans a line that contains a
/// non-comment token. If shrinking the skipped range would make it empty,
/// this returns None.
+ /// Note this function can potentially be expensive because
+ /// getSpellingLineNumber uses getLineNumber, which is expensive.
Optional<SpellingRegion> adjustSkippedRange(SourceManager &SM,
SourceLocation LocStart,
SourceLocation LocEnd,
@@ -382,9 +385,14 @@ public:
auto CovFileID = getCoverageFileID(LocStart);
if (!CovFileID)
continue;
- Optional<SpellingRegion> SR =
- adjustSkippedRange(SM, LocStart, LocEnd, I.PrevTokLoc, I.NextTokLoc);
- if (!SR.hasValue())
+ Optional<SpellingRegion> SR;
+ if (I.isComment())
+ SR = adjustSkippedRange(SM, LocStart, LocEnd, I.PrevTokLoc,
+ I.NextTokLoc);
+ else if (I.isPPIfElse() || I.isEmptyLine())
+ SR = {SM, LocStart, LocEnd};
+
+ if (!SR)
continue;
auto Region = CounterMappingRegion::makeSkipped(
*CovFileID, SR->LineStart, SR->ColumnStart, SR->LineEnd,
@@ -550,17 +558,18 @@ struct CounterCoverageMappingBuilder
Counter GapRegionCounter;
/// Return a counter for the subtraction of \c RHS from \c LHS
- Counter subtractCounters(Counter LHS, Counter RHS) {
- return Builder.subtract(LHS, RHS);
+ Counter subtractCounters(Counter LHS, Counter RHS, bool Simplify = true) {
+ return Builder.subtract(LHS, RHS, Simplify);
}
/// Return a counter for the sum of \c LHS and \c RHS.
- Counter addCounters(Counter LHS, Counter RHS) {
- return Builder.add(LHS, RHS);
+ Counter addCounters(Counter LHS, Counter RHS, bool Simplify = true) {
+ return Builder.add(LHS, RHS, Simplify);
}
- Counter addCounters(Counter C1, Counter C2, Counter C3) {
- return addCounters(addCounters(C1, C2), C3);
+ Counter addCounters(Counter C1, Counter C2, Counter C3,
+ bool Simplify = true) {
+ return addCounters(addCounters(C1, C2, Simplify), C3, Simplify);
}
/// Return the region counter for the given statement.
@@ -578,7 +587,7 @@ struct CounterCoverageMappingBuilder
Optional<SourceLocation> EndLoc = None,
Optional<Counter> FalseCount = None) {
- if (StartLoc && !FalseCount.hasValue()) {
+ if (StartLoc && !FalseCount) {
MostRecentLocation = *StartLoc;
}
@@ -1317,11 +1326,16 @@ struct CounterCoverageMappingBuilder
const SwitchCase *Case = S->getSwitchCaseList();
for (; Case; Case = Case->getNextSwitchCase()) {
HasDefaultCase = HasDefaultCase || isa<DefaultStmt>(Case);
- CaseCountSum = addCounters(CaseCountSum, getRegionCounter(Case));
+ CaseCountSum =
+ addCounters(CaseCountSum, getRegionCounter(Case), /*Simplify=*/false);
createSwitchCaseRegion(
Case, getRegionCounter(Case),
subtractCounters(ParentCount, getRegionCounter(Case)));
}
+ // Simplify is skipped while building the counters above: it can get really
+ // slow on top of switches with thousands of cases. Instead, trigger
+ // simplification by adding zero to the last counter.
+ CaseCountSum = addCounters(CaseCountSum, Counter::getZero());
// If no explicit default case exists, create a branch region to represent
// the hidden branch, which will be added later by the CodeGen. This region
diff --git a/clang/lib/CodeGen/CoverageMappingGen.h b/clang/lib/CodeGen/CoverageMappingGen.h
index ae4f435d4ff3..f5282601b640 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.h
+++ b/clang/lib/CodeGen/CoverageMappingGen.h
@@ -31,15 +31,29 @@ class Decl;
class Stmt;
struct SkippedRange {
+ enum Kind {
+ PPIfElse, // Preprocessor #if/#else ...
+ EmptyLine,
+ Comment,
+ };
+
SourceRange Range;
// The location of token before the skipped source range.
SourceLocation PrevTokLoc;
// The location of token after the skipped source range.
SourceLocation NextTokLoc;
+ // The nature of this skipped range
+ Kind RangeKind;
+
+ bool isComment() { return RangeKind == Comment; }
+ bool isEmptyLine() { return RangeKind == EmptyLine; }
+ bool isPPIfElse() { return RangeKind == PPIfElse; }
- SkippedRange(SourceRange Range, SourceLocation PrevTokLoc = SourceLocation(),
+ SkippedRange(SourceRange Range, Kind K,
+ SourceLocation PrevTokLoc = SourceLocation(),
SourceLocation NextTokLoc = SourceLocation())
- : Range(Range), PrevTokLoc(PrevTokLoc), NextTokLoc(NextTokLoc) {}
+ : Range(Range), PrevTokLoc(PrevTokLoc), NextTokLoc(NextTokLoc),
+ RangeKind(K) {}
};
/// Stores additional source code information like skipped ranges which
@@ -62,7 +76,7 @@ public:
std::vector<SkippedRange> &getSkippedRanges() { return SkippedRanges; }
- void AddSkippedRange(SourceRange Range);
+ void AddSkippedRange(SourceRange Range, SkippedRange::Kind RangeKind);
void SourceRangeSkipped(SourceRange Range, SourceLocation EndifLoc) override;
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 2979d92c8417..f0003c4aab78 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -668,8 +668,8 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
CGM.HasHiddenLTOVisibility(RD);
bool ShouldEmitWPDInfo =
CGM.getCodeGenOpts().WholeProgramVTables &&
- // Don't insert type tests if we are forcing public std visibility.
- !CGM.HasLTOVisibilityPublicStd(RD);
+ // Don't insert type tests if we are forcing public visibility.
+ !CGM.AlwaysHasLTOVisibilityPublic(RD);
llvm::Value *VirtualFn = nullptr;
{
@@ -955,14 +955,16 @@ ItaniumCXXABI::EmitMemberPointerConversion(const CastExpr *E,
adj = llvm::ConstantInt::get(adj->getType(), offset);
}
- llvm::Constant *srcAdj = llvm::ConstantExpr::getExtractValue(src, 1);
+ llvm::Constant *srcAdj = src->getAggregateElement(1);
llvm::Constant *dstAdj;
if (isDerivedToBase)
dstAdj = llvm::ConstantExpr::getNSWSub(srcAdj, adj);
else
dstAdj = llvm::ConstantExpr::getNSWAdd(srcAdj, adj);
- return llvm::ConstantExpr::getInsertValue(src, dstAdj, 1);
+ llvm::Constant *res = ConstantFoldInsertValueInstruction(src, dstAdj, 1);
+ assert(res != nullptr && "Folding must succeed");
+ return res;
}
llvm::Constant *
@@ -1925,7 +1927,7 @@ CGCallee ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
llvm::Value *VFunc;
if (CGF.ShouldEmitVTableTypeCheckedLoad(MethodDecl->getParent())) {
VFunc = CGF.EmitVTableTypeCheckedLoad(
- MethodDecl->getParent(), VTable,
+ MethodDecl->getParent(), VTable, TyPtr,
VTableIndex * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8);
} else {
CGF.EmitTypeMetadataCodeForVCall(MethodDecl->getParent(), VTable, Loc);
@@ -3680,12 +3682,14 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty) {
llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass =
llvm::GlobalValue::DefaultStorageClass;
- if (CGM.getTriple().isWindowsItaniumEnvironment()) {
- auto RD = Ty->getAsCXXRecordDecl();
- if (RD && RD->hasAttr<DLLExportAttr>())
+ if (auto RD = Ty->getAsCXXRecordDecl()) {
+ if ((CGM.getTriple().isWindowsItaniumEnvironment() &&
+ RD->hasAttr<DLLExportAttr>()) ||
+ (CGM.shouldMapVisibilityToDLLExport(RD) &&
+ !llvm::GlobalValue::isLocalLinkage(Linkage) &&
+ llvmVisibility == llvm::GlobalValue::DefaultVisibility))
DLLStorageClass = llvm::GlobalValue::DLLExportStorageClass;
}
-
return BuildTypeInfo(Ty, Linkage, llvmVisibility, DLLStorageClass);
}
@@ -4168,9 +4172,9 @@ void ItaniumCXXABI::EmitFundamentalRTTIDescriptors(const CXXRecordDecl *RD) {
getContext().Char32Ty
};
llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass =
- RD->hasAttr<DLLExportAttr>()
- ? llvm::GlobalValue::DLLExportStorageClass
- : llvm::GlobalValue::DefaultStorageClass;
+ RD->hasAttr<DLLExportAttr>() || CGM.shouldMapVisibilityToDLLExport(RD)
+ ? llvm::GlobalValue::DLLExportStorageClass
+ : llvm::GlobalValue::DefaultStorageClass;
llvm::GlobalValue::VisibilityTypes Visibility =
CodeGenModule::GetLLVMVisibility(RD->getVisibility());
for (const QualType &FundamentalType : FundamentalTypes) {
@@ -4472,7 +4476,7 @@ static void InitCatchParam(CodeGenFunction &CGF,
// pad. The best solution is to fix the personality function.
} else {
// Pull the pointer for the reference type off.
- llvm::Type *PtrTy = LLVMCatchTy->getPointerElementType();
+ llvm::Type *PtrTy = CGF.ConvertTypeForMem(CaughtType);
// Create the temporary and write the adjusted pointer into it.
Address ExnPtrTmp =
@@ -4555,7 +4559,7 @@ static void InitCatchParam(CodeGenFunction &CGF,
if (!copyExpr) {
llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, true);
Address adjustedExn(CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy),
- caughtExnAlignment);
+ LLVMCatchTy, caughtExnAlignment);
LValue Dest = CGF.MakeAddrLValue(ParamAddr, CatchType);
LValue Src = CGF.MakeAddrLValue(adjustedExn, CatchType);
CGF.EmitAggregateCopy(Dest, Src, CatchType, AggValueSlot::DoesNotOverlap);
@@ -4569,7 +4573,7 @@ static void InitCatchParam(CodeGenFunction &CGF,
// Cast that to the appropriate type.
Address adjustedExn(CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy),
- caughtExnAlignment);
+ LLVMCatchTy, caughtExnAlignment);
// The copy expression is defined in terms of an OpaqueValueExpr.
// Find it and map it to the adjusted expression.
diff --git a/clang/lib/CodeGen/MacroPPCallbacks.cpp b/clang/lib/CodeGen/MacroPPCallbacks.cpp
index 92800e738b62..2f09fd2b6c15 100644
--- a/clang/lib/CodeGen/MacroPPCallbacks.cpp
+++ b/clang/lib/CodeGen/MacroPPCallbacks.cpp
@@ -167,7 +167,7 @@ void MacroPPCallbacks::FileChanged(SourceLocation Loc, FileChangeReason Reason,
void MacroPPCallbacks::InclusionDirective(
SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
- bool IsAngled, CharSourceRange FilenameRange, const FileEntry *File,
+ bool IsAngled, CharSourceRange FilenameRange, Optional<FileEntryRef> File,
StringRef SearchPath, StringRef RelativePath, const Module *Imported,
SrcMgr::CharacteristicKind FileType) {
diff --git a/clang/lib/CodeGen/MacroPPCallbacks.h b/clang/lib/CodeGen/MacroPPCallbacks.h
index d249b5b0eb88..01041b16e4b7 100644
--- a/clang/lib/CodeGen/MacroPPCallbacks.h
+++ b/clang/lib/CodeGen/MacroPPCallbacks.h
@@ -100,9 +100,9 @@ public:
/// Callback invoked whenever a directive (#xxx) is processed.
void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
StringRef FileName, bool IsAngled,
- CharSourceRange FilenameRange, const FileEntry *File,
- StringRef SearchPath, StringRef RelativePath,
- const Module *Imported,
+ CharSourceRange FilenameRange,
+ Optional<FileEntryRef> File, StringRef SearchPath,
+ StringRef RelativePath, const Module *Imported,
SrcMgr::CharacteristicKind FileType) override;
/// Hook called whenever a macro definition is seen.
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index e00ff2b68719..2bc1e8e8c5b9 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -47,7 +47,11 @@ public:
: CGCXXABI(CGM), BaseClassDescriptorType(nullptr),
ClassHierarchyDescriptorType(nullptr),
CompleteObjectLocatorType(nullptr), CatchableTypeType(nullptr),
- ThrowInfoType(nullptr) {}
+ ThrowInfoType(nullptr) {
+ assert(!(CGM.getLangOpts().isExplicitDefaultVisibilityExportMapping() ||
+ CGM.getLangOpts().isAllDefaultVisibilityExportMapping()) &&
+ "visibility export mapping option unimplemented in this ABI");
+ }
bool HasThisReturn(GlobalDecl GD) const override;
bool hasMostDerivedReturn(GlobalDecl GD) const override;
@@ -948,7 +952,8 @@ MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value,
Value.getElementType(), Value.getPointer(), Offset);
CharUnits VBaseAlign =
CGF.CGM.getVBaseAlignment(Value.getAlignment(), SrcDecl, PolymorphicBase);
- return std::make_tuple(Address(Ptr, VBaseAlign), Offset, PolymorphicBase);
+ return std::make_tuple(Address(Ptr, CGF.Int8Ty, VBaseAlign), Offset,
+ PolymorphicBase);
}
bool MicrosoftCXXABI::shouldTypeidBeNullChecked(bool IsDeref,
@@ -1470,7 +1475,7 @@ Address MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall(
Result.getElementType(), Result.getPointer(), VBaseOffset);
CharUnits VBaseAlign =
CGF.CGM.getVBaseAlignment(Result.getAlignment(), Derived, VBase);
- Result = Address(VBasePtr, VBaseAlign);
+ Result = Address(VBasePtr, CGF.Int8Ty, VBaseAlign);
}
if (!StaticOffset.isZero()) {
assert(StaticOffset.isPositive());
@@ -1946,7 +1951,7 @@ CGCallee MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
llvm::Value *VFunc;
if (CGF.ShouldEmitVTableTypeCheckedLoad(MethodDecl->getParent())) {
VFunc = CGF.EmitVTableTypeCheckedLoad(
- getObjectWithVPtr(), VTable,
+ getObjectWithVPtr(), VTable, Ty,
ML.Index * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8);
} else {
if (CGM.getCodeGenOpts().PrepareForLTO)
@@ -2217,10 +2222,10 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF,
assert(TA.Virtual.Microsoft.VBPtrOffset > 0);
assert(TA.Virtual.Microsoft.VBOffsetOffset >= 0);
llvm::Value *VBPtr;
- llvm::Value *VBaseOffset =
- GetVBaseOffsetFromVBPtr(CGF, Address(V, CGF.getPointerAlign()),
- -TA.Virtual.Microsoft.VBPtrOffset,
- TA.Virtual.Microsoft.VBOffsetOffset, &VBPtr);
+ llvm::Value *VBaseOffset = GetVBaseOffsetFromVBPtr(
+ CGF, Address(V, CGF.Int8Ty, CGF.getPointerAlign()),
+ -TA.Virtual.Microsoft.VBPtrOffset,
+ TA.Virtual.Microsoft.VBOffsetOffset, &VBPtr);
V = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, VBPtr, VBaseOffset);
}
}
@@ -2432,7 +2437,7 @@ static void emitTlsGuardCheck(CodeGenFunction &CGF, llvm::GlobalValue *TlsGuard,
llvm::BasicBlock *DynInitBB,
llvm::BasicBlock *ContinueBB) {
llvm::LoadInst *TlsGuardValue =
- CGF.Builder.CreateLoad(Address(TlsGuard, CharUnits::One()));
+ CGF.Builder.CreateLoad(Address(TlsGuard, CGF.Int8Ty, CharUnits::One()));
llvm::Value *CmpResult =
CGF.Builder.CreateICmpEQ(TlsGuardValue, CGF.Builder.getInt8(0));
CGF.Builder.CreateCondBr(CmpResult, DynInitBB, ContinueBB);
@@ -2483,7 +2488,7 @@ LValue MicrosoftCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF,
V = CGF.Builder.CreateBitCast(V, RealVarTy->getPointerTo(AS));
CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
- Address Addr(V, Alignment);
+ Address Addr(V, RealVarTy, Alignment);
LValue LV = VD->getType()->isReferenceType()
? CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
diff --git a/clang/lib/CodeGen/ModuleBuilder.cpp b/clang/lib/CodeGen/ModuleBuilder.cpp
index f6642a79e1e4..8e97a298ce7f 100644
--- a/clang/lib/CodeGen/ModuleBuilder.cpp
+++ b/clang/lib/CodeGen/ModuleBuilder.cpp
@@ -134,7 +134,14 @@ namespace {
llvm::LLVMContext &C) {
assert(!M && "Replacing existing Module?");
M.reset(new llvm::Module(ExpandModuleName(ModuleName, CodeGenOpts), C));
+
+ std::unique_ptr<CodeGenModule> OldBuilder = std::move(Builder);
+
Initialize(*Ctx);
+
+ if (OldBuilder)
+ OldBuilder->moveLazyEmissionStates(Builder.get());
+
return M.get();
}
@@ -146,6 +153,11 @@ namespace {
const auto &SDKVersion = Ctx->getTargetInfo().getSDKVersion();
if (!SDKVersion.empty())
M->setSDKVersion(SDKVersion);
+ if (const auto *TVT = Ctx->getTargetInfo().getDarwinTargetVariantTriple())
+ M->setDarwinTargetVariantTriple(TVT->getTriple());
+ if (auto TVSDKVersion =
+ Ctx->getTargetInfo().getDarwinTargetVariantSDKVersion())
+ M->setDarwinTargetVariantSDKVersion(*TVSDKVersion);
Builder.reset(new CodeGen::CodeGenModule(Context, HeaderSearchOpts,
PreprocessorOpts, CodeGenOpts,
*M, Diags, CoverageInfo));
diff --git a/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp b/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
index 9fe7e5d1f5c3..d03e5bd50873 100644
--- a/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
+++ b/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
@@ -96,6 +96,10 @@ class PCHContainerGenerator : public ASTConsumer {
}
bool VisitFunctionDecl(FunctionDecl *D) {
+ // Skip deduction guides.
+ if (isa<CXXDeductionGuideDecl>(D))
+ return true;
+
if (isa<CXXMethodDecl>(D))
// This is not yet supported. Constructing the `this' argument
// mandates a CodeGenFunction.
diff --git a/clang/lib/CodeGen/SanitizerMetadata.cpp b/clang/lib/CodeGen/SanitizerMetadata.cpp
index 009965a36c39..5f4eb9be981f 100644
--- a/clang/lib/CodeGen/SanitizerMetadata.cpp
+++ b/clang/lib/CodeGen/SanitizerMetadata.cpp
@@ -22,84 +22,87 @@ using namespace CodeGen;
SanitizerMetadata::SanitizerMetadata(CodeGenModule &CGM) : CGM(CGM) {}
-static bool isAsanHwasanOrMemTag(const SanitizerSet& SS) {
+static bool isAsanHwasanOrMemTag(const SanitizerSet &SS) {
return SS.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress |
- SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress |
- SanitizerKind::MemTag);
+ SanitizerKind::HWAddress | SanitizerKind::MemTag);
}
-void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV,
- SourceLocation Loc, StringRef Name,
- QualType Ty, bool IsDynInit,
- bool IsExcluded) {
- if (!isAsanHwasanOrMemTag(CGM.getLangOpts().Sanitize))
+SanitizerMask expandKernelSanitizerMasks(SanitizerMask Mask) {
+ if (Mask & (SanitizerKind::Address | SanitizerKind::KernelAddress))
+ Mask |= SanitizerKind::Address | SanitizerKind::KernelAddress;
+ // Note: KHWASan doesn't support globals.
+ return Mask;
+}
+
+void SanitizerMetadata::reportGlobal(llvm::GlobalVariable *GV,
+ SourceLocation Loc, StringRef Name,
+ QualType Ty,
+ SanitizerMask NoSanitizeAttrMask,
+ bool IsDynInit) {
+ SanitizerSet FsanitizeArgument = CGM.getLangOpts().Sanitize;
+ if (!isAsanHwasanOrMemTag(FsanitizeArgument))
return;
- IsDynInit &= !CGM.isInNoSanitizeList(GV, Loc, Ty, "init");
- IsExcluded |= CGM.isInNoSanitizeList(GV, Loc, Ty);
- llvm::Metadata *LocDescr = nullptr;
- llvm::Metadata *GlobalName = nullptr;
- llvm::LLVMContext &VMContext = CGM.getLLVMContext();
- if (!IsExcluded) {
- // Don't generate source location and global name if it is on
- // the NoSanitizeList - it won't be instrumented anyway.
- LocDescr = getLocationMetadata(Loc);
- if (!Name.empty())
- GlobalName = llvm::MDString::get(VMContext, Name);
- }
+ FsanitizeArgument.Mask = expandKernelSanitizerMasks(FsanitizeArgument.Mask);
+ NoSanitizeAttrMask = expandKernelSanitizerMasks(NoSanitizeAttrMask);
+ SanitizerSet NoSanitizeAttrSet = {NoSanitizeAttrMask &
+ FsanitizeArgument.Mask};
+
+ llvm::GlobalVariable::SanitizerMetadata Meta;
+ if (GV->hasSanitizerMetadata())
+ Meta = GV->getSanitizerMetadata();
+
+ Meta.NoAddress |= NoSanitizeAttrSet.hasOneOf(SanitizerKind::Address);
+ Meta.NoAddress |= CGM.isInNoSanitizeList(
+ FsanitizeArgument.Mask & SanitizerKind::Address, GV, Loc, Ty);
+
+ Meta.NoHWAddress |= NoSanitizeAttrSet.hasOneOf(SanitizerKind::HWAddress);
+ Meta.NoHWAddress |= CGM.isInNoSanitizeList(
+ FsanitizeArgument.Mask & SanitizerKind::HWAddress, GV, Loc, Ty);
+
+ Meta.NoMemtag |= NoSanitizeAttrSet.hasOneOf(SanitizerKind::MemTag);
+ Meta.NoMemtag |= CGM.isInNoSanitizeList(
+ FsanitizeArgument.Mask & SanitizerKind::MemTag, GV, Loc, Ty);
- llvm::Metadata *GlobalMetadata[] = {
- llvm::ConstantAsMetadata::get(GV), LocDescr, GlobalName,
- llvm::ConstantAsMetadata::get(
- llvm::ConstantInt::get(llvm::Type::getInt1Ty(VMContext), IsDynInit)),
- llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
- llvm::Type::getInt1Ty(VMContext), IsExcluded))};
+ if (FsanitizeArgument.has(SanitizerKind::Address)) {
+ // TODO(hctim): Make this conditional when we migrate off llvm.asan.globals.
+ IsDynInit &= !CGM.isInNoSanitizeList(SanitizerKind::Address |
+ SanitizerKind::KernelAddress,
+ GV, Loc, Ty, "init");
+ Meta.IsDynInit = IsDynInit;
+ }
- llvm::MDNode *ThisGlobal = llvm::MDNode::get(VMContext, GlobalMetadata);
- llvm::NamedMDNode *AsanGlobals =
- CGM.getModule().getOrInsertNamedMetadata("llvm.asan.globals");
- AsanGlobals->addOperand(ThisGlobal);
+ GV->setSanitizerMetadata(Meta);
}
-void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV,
- const VarDecl &D, bool IsDynInit) {
+void SanitizerMetadata::reportGlobal(llvm::GlobalVariable *GV, const VarDecl &D,
+ bool IsDynInit) {
if (!isAsanHwasanOrMemTag(CGM.getLangOpts().Sanitize))
return;
std::string QualName;
llvm::raw_string_ostream OS(QualName);
D.printQualifiedName(OS);
- bool IsExcluded = false;
- for (auto Attr : D.specific_attrs<NoSanitizeAttr>())
- if (Attr->getMask() & SanitizerKind::Address)
- IsExcluded = true;
- reportGlobalToASan(GV, D.getLocation(), OS.str(), D.getType(), IsDynInit,
- IsExcluded);
+ auto getNoSanitizeMask = [](const VarDecl &D) {
+ if (D.hasAttr<DisableSanitizerInstrumentationAttr>())
+ return SanitizerKind::All;
+
+ SanitizerMask NoSanitizeMask;
+ for (auto *Attr : D.specific_attrs<NoSanitizeAttr>())
+ NoSanitizeMask |= Attr->getMask();
+
+ return NoSanitizeMask;
+ };
+
+ reportGlobal(GV, D.getLocation(), OS.str(), D.getType(), getNoSanitizeMask(D),
+ IsDynInit);
}
void SanitizerMetadata::disableSanitizerForGlobal(llvm::GlobalVariable *GV) {
- // For now, just make sure the global is not modified by the ASan
- // instrumentation.
- if (isAsanHwasanOrMemTag(CGM.getLangOpts().Sanitize))
- reportGlobalToASan(GV, SourceLocation(), "", QualType(), false, true);
+ reportGlobal(GV, SourceLocation(), "", QualType(), SanitizerKind::All);
}
void SanitizerMetadata::disableSanitizerForInstruction(llvm::Instruction *I) {
- I->setMetadata(CGM.getModule().getMDKindID("nosanitize"),
+ I->setMetadata(llvm::LLVMContext::MD_nosanitize,
llvm::MDNode::get(CGM.getLLVMContext(), None));
}
-
-llvm::MDNode *SanitizerMetadata::getLocationMetadata(SourceLocation Loc) {
- PresumedLoc PLoc = CGM.getContext().getSourceManager().getPresumedLoc(Loc);
- if (!PLoc.isValid())
- return nullptr;
- llvm::LLVMContext &VMContext = CGM.getLLVMContext();
- llvm::Metadata *LocMetadata[] = {
- llvm::MDString::get(VMContext, PLoc.getFilename()),
- llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
- llvm::Type::getInt32Ty(VMContext), PLoc.getLine())),
- llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
- llvm::Type::getInt32Ty(VMContext), PLoc.getColumn())),
- };
- return llvm::MDNode::get(VMContext, LocMetadata);
-}
diff --git a/clang/lib/CodeGen/SanitizerMetadata.h b/clang/lib/CodeGen/SanitizerMetadata.h
index 440a54590acc..bcad32ce31df 100644
--- a/clang/lib/CodeGen/SanitizerMetadata.h
+++ b/clang/lib/CodeGen/SanitizerMetadata.h
@@ -14,13 +14,14 @@
#include "clang/AST/Type.h"
#include "clang/Basic/LLVM.h"
+#include "clang/Basic/Sanitizers.h"
#include "clang/Basic/SourceLocation.h"
namespace llvm {
class GlobalVariable;
class Instruction;
class MDNode;
-}
+} // namespace llvm
namespace clang {
class VarDecl;
@@ -34,19 +35,19 @@ class SanitizerMetadata {
void operator=(const SanitizerMetadata &) = delete;
CodeGenModule &CGM;
+
public:
SanitizerMetadata(CodeGenModule &CGM);
- void reportGlobalToASan(llvm::GlobalVariable *GV, const VarDecl &D,
- bool IsDynInit = false);
- void reportGlobalToASan(llvm::GlobalVariable *GV, SourceLocation Loc,
- StringRef Name, QualType Ty, bool IsDynInit = false,
- bool IsExcluded = false);
+ void reportGlobal(llvm::GlobalVariable *GV, const VarDecl &D,
+ bool IsDynInit = false);
+ void reportGlobal(llvm::GlobalVariable *GV, SourceLocation Loc,
+ StringRef Name, QualType Ty = {},
+ SanitizerMask NoSanitizeAttrMask = {},
+ bool IsDynInit = false);
void disableSanitizerForGlobal(llvm::GlobalVariable *GV);
void disableSanitizerForInstruction(llvm::Instruction *I);
-private:
- llvm::MDNode *getLocationMetadata(SourceLocation Loc);
};
-} // end namespace CodeGen
-} // end namespace clang
+} // end namespace CodeGen
+} // end namespace clang
#endif
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index 8a0150218a7a..8eaed1db8e7d 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -19,9 +19,9 @@
#include "CodeGenFunction.h"
#include "clang/AST/Attr.h"
#include "clang/AST/RecordLayout.h"
+#include "clang/Basic/Builtins.h"
#include "clang/Basic/CodeGenOptions.h"
#include "clang/Basic/DiagnosticFrontend.h"
-#include "clang/Basic/Builtins.h"
#include "clang/CodeGen/CGFunctionInfo.h"
#include "clang/CodeGen/SwiftCallingConv.h"
#include "llvm/ADT/SmallBitVector.h"
@@ -33,6 +33,7 @@
#include "llvm/IR/IntrinsicsNVPTX.h"
#include "llvm/IR/IntrinsicsS390.h"
#include "llvm/IR/Type.h"
+#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm> // std::sort
@@ -100,6 +101,11 @@ Address ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
return Address::invalid();
}
+static llvm::Type *getVAListElementType(CodeGenFunction &CGF) {
+ return CGF.ConvertTypeForMem(
+ CGF.getContext().getBuiltinVaListType()->getPointeeType());
+}
+
bool ABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
if (Ty->isPromotableIntegerType())
return true;
@@ -234,6 +240,11 @@ bool ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
return false;
}
+bool ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate() const {
+ // For compatibility with GCC, ignore empty bitfields in C++ mode.
+ return getContext().getLangOpts().CPlusPlus;
+}
+
LLVM_DUMP_METHOD void ABIArgInfo::dump() const {
raw_ostream &OS = llvm::errs();
OS << "(ABIArgInfo Kind=";
@@ -324,9 +335,9 @@ static Address emitVoidPtrDirectVAArg(CodeGenFunction &CGF,
Address Addr = Address::invalid();
if (AllowHigherAlign && DirectAlign > SlotSize) {
Addr = Address(emitRoundPointerUpToAlignment(CGF, Ptr, DirectAlign),
- DirectAlign);
+ CGF.Int8Ty, DirectAlign);
} else {
- Addr = Address(Ptr, SlotSize);
+ Addr = Address(Ptr, CGF.Int8Ty, SlotSize);
}
// Advance the pointer past the argument, then store that back.
@@ -375,21 +386,19 @@ static Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr,
}
// Cast the address we've calculated to the right type.
- llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy);
+ llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy), *ElementTy = DirectTy;
if (IsIndirect)
DirectTy = DirectTy->getPointerTo(0);
- Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy,
- DirectSize, DirectAlign,
- SlotSizeAndAlign,
- AllowHigherAlign);
+ Address Addr =
+ emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy, DirectSize, DirectAlign,
+ SlotSizeAndAlign, AllowHigherAlign);
if (IsIndirect) {
- Addr = Address(CGF.Builder.CreateLoad(Addr), ValueInfo.Align);
+ Addr = Address(CGF.Builder.CreateLoad(Addr), ElementTy, ValueInfo.Align);
}
return Addr;
-
}
static Address complexTempStructure(CodeGenFunction &CGF, Address VAListAddr,
@@ -690,11 +699,11 @@ Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
auto TyInfo = CGF.getContext().getTypeInfoInChars(Ty);
CharUnits TyAlignForABI = TyInfo.Align;
- llvm::Type *BaseTy =
- llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty));
+ llvm::Type *ElementTy = CGF.ConvertTypeForMem(Ty);
+ llvm::Type *BaseTy = llvm::PointerType::getUnqual(ElementTy);
llvm::Value *Addr =
CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy);
- return Address(Addr, TyAlignForABI);
+ return Address(Addr, ElementTy, TyAlignForABI);
} else {
assert((AI.isDirect() || AI.isExtend()) &&
"Unexpected ArgInfo Kind in generic VAArg emitter!");
@@ -709,7 +718,8 @@ Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
"Unexpected CoerceToType seen in arginfo in generic VAArg emitter!");
Address Temp = CGF.CreateMemTemp(Ty, "varet");
- Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(), CGF.ConvertType(Ty));
+ Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(),
+ CGF.ConvertTypeForMem(Ty));
CGF.Builder.CreateStore(Val, Temp);
return Temp;
}
@@ -1351,8 +1361,8 @@ void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
ResultTruncRegTypes.push_back(CoerceTy);
// Coerce the integer by bitcasting the return slot pointer.
- ReturnSlot.setAddress(CGF.Builder.CreateBitCast(ReturnSlot.getAddress(CGF),
- CoerceTy->getPointerTo()));
+ ReturnSlot.setAddress(
+ CGF.Builder.CreateElementBitCast(ReturnSlot.getAddress(CGF), CoerceTy));
ResultRegDests.push_back(ReturnSlot);
rewriteInputConstraintReferences(NumOutputs, 1, AsmString);
@@ -2292,6 +2302,8 @@ class X86_64ABIInfo : public SwiftABIInfo {
/// \param isNamedArg - Whether the argument in question is a "named"
/// argument, as used in AMD64-ABI 3.5.7.
///
+ /// \param IsRegCall - Whether the calling conversion is regcall.
+ ///
/// If a word is unused its result will be NoClass; if a type should
/// be passed in Memory then at least the classification of \arg Lo
/// will be Memory.
@@ -2301,7 +2313,7 @@ class X86_64ABIInfo : public SwiftABIInfo {
/// If the \arg Lo class is ComplexX87, then the \arg Hi class will
/// also be ComplexX87.
void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
- bool isNamedArg) const;
+ bool isNamedArg, bool IsRegCall = false) const;
llvm::Type *GetByteVectorType(QualType Ty) const;
llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
@@ -2326,13 +2338,16 @@ class X86_64ABIInfo : public SwiftABIInfo {
ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
unsigned &neededInt, unsigned &neededSSE,
- bool isNamedArg) const;
+ bool isNamedArg,
+ bool IsRegCall = false) const;
ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
- unsigned &NeededSSE) const;
+ unsigned &NeededSSE,
+ unsigned &MaxVectorWidth) const;
ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
- unsigned &NeededSSE) const;
+ unsigned &NeededSSE,
+ unsigned &MaxVectorWidth) const;
bool IsIllegalVectorType(QualType Ty) const;
@@ -2354,7 +2369,7 @@ class X86_64ABIInfo : public SwiftABIInfo {
return false;
const llvm::Triple &Triple = getTarget().getTriple();
- if (Triple.isOSDarwin() || Triple.getOS() == llvm::Triple::PS4)
+ if (Triple.isOSDarwin() || Triple.isPS())
return false;
if (Triple.isOSFreeBSD() && Triple.getOSMajorVersion() >= 10)
return false;
@@ -2827,8 +2842,8 @@ X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
return SSE;
}
-void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
- Class &Lo, Class &Hi, bool isNamedArg) const {
+void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
+ Class &Hi, bool isNamedArg, bool IsRegCall) const {
// FIXME: This code can be simplified by introducing a simple value class for
// Class pairs with appropriate constructor methods for the various
// situations.
@@ -3026,7 +3041,9 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
// AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
// than eight eightbytes, ..., it has class MEMORY.
- if (Size > 512)
+ // regcall ABI doesn't have limitation to an object. The only limitation
+ // is the free registers, which will be checked in computeInfo.
+ if (!IsRegCall && Size > 512)
return;
// AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
@@ -3119,7 +3136,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
unsigned idx = 0;
bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=
LangOptions::ClangABI::Ver11 ||
- getContext().getTargetInfo().getTriple().isPS4();
+ getContext().getTargetInfo().getTriple().isPS();
bool IsUnion = RT->isUnionType() && !UseClang11Compat;
for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
@@ -3733,15 +3750,14 @@ classifyReturnType(QualType RetTy) const {
return ABIArgInfo::getDirect(ResType);
}
-ABIArgInfo X86_64ABIInfo::classifyArgumentType(
- QualType Ty, unsigned freeIntRegs, unsigned &neededInt, unsigned &neededSSE,
- bool isNamedArg)
- const
-{
+ABIArgInfo
+X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
+ unsigned &neededInt, unsigned &neededSSE,
+ bool isNamedArg, bool IsRegCall) const {
Ty = useFirstFieldIfTransparentUnion(Ty);
X86_64ABIInfo::Class Lo, Hi;
- classify(Ty, 0, Lo, Hi, isNamedArg);
+ classify(Ty, 0, Lo, Hi, isNamedArg, IsRegCall);
// Check some invariants.
// FIXME: Enforce these by construction.
@@ -3864,7 +3880,8 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType(
ABIArgInfo
X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
- unsigned &NeededSSE) const {
+ unsigned &NeededSSE,
+ unsigned &MaxVectorWidth) const {
auto RT = Ty->getAs<RecordType>();
assert(RT && "classifyRegCallStructType only valid with struct types");
@@ -3879,7 +3896,8 @@ X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
}
for (const auto &I : CXXRD->bases())
- if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE)
+ if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE,
+ MaxVectorWidth)
.isIndirect()) {
NeededInt = NeededSSE = 0;
return getIndirectReturnResult(Ty);
@@ -3888,20 +3906,27 @@ X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
// Sum up members
for (const auto *FD : RT->getDecl()->fields()) {
- if (FD->getType()->isRecordType() && !FD->getType()->isUnionType()) {
- if (classifyRegCallStructTypeImpl(FD->getType(), NeededInt, NeededSSE)
+ QualType MTy = FD->getType();
+ if (MTy->isRecordType() && !MTy->isUnionType()) {
+ if (classifyRegCallStructTypeImpl(MTy, NeededInt, NeededSSE,
+ MaxVectorWidth)
.isIndirect()) {
NeededInt = NeededSSE = 0;
return getIndirectReturnResult(Ty);
}
} else {
unsigned LocalNeededInt, LocalNeededSSE;
- if (classifyArgumentType(FD->getType(), UINT_MAX, LocalNeededInt,
- LocalNeededSSE, true)
+ if (classifyArgumentType(MTy, UINT_MAX, LocalNeededInt, LocalNeededSSE,
+ true, true)
.isIndirect()) {
NeededInt = NeededSSE = 0;
return getIndirectReturnResult(Ty);
}
+ if (const auto *AT = getContext().getAsConstantArrayType(MTy))
+ MTy = AT->getElementType();
+ if (const auto *VT = MTy->getAs<VectorType>())
+ if (getContext().getTypeSize(VT) > MaxVectorWidth)
+ MaxVectorWidth = getContext().getTypeSize(VT);
NeededInt += LocalNeededInt;
NeededSSE += LocalNeededSSE;
}
@@ -3910,14 +3935,17 @@ X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
return ABIArgInfo::getDirect();
}
-ABIArgInfo X86_64ABIInfo::classifyRegCallStructType(QualType Ty,
- unsigned &NeededInt,
- unsigned &NeededSSE) const {
+ABIArgInfo
+X86_64ABIInfo::classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
+ unsigned &NeededSSE,
+ unsigned &MaxVectorWidth) const {
NeededInt = 0;
NeededSSE = 0;
+ MaxVectorWidth = 0;
- return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE);
+ return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE,
+ MaxVectorWidth);
}
void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
@@ -3937,13 +3965,13 @@ void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
// Keep track of the number of assigned registers.
unsigned FreeIntRegs = IsRegCall ? 11 : 6;
unsigned FreeSSERegs = IsRegCall ? 16 : 8;
- unsigned NeededInt, NeededSSE;
+ unsigned NeededInt = 0, NeededSSE = 0, MaxVectorWidth = 0;
if (!::classifyReturnType(getCXXABI(), FI, *this)) {
if (IsRegCall && FI.getReturnType()->getTypePtr()->isRecordType() &&
!FI.getReturnType()->getTypePtr()->isUnionType()) {
- FI.getReturnInfo() =
- classifyRegCallStructType(FI.getReturnType(), NeededInt, NeededSSE);
+ FI.getReturnInfo() = classifyRegCallStructType(
+ FI.getReturnType(), NeededInt, NeededSSE, MaxVectorWidth);
if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
FreeIntRegs -= NeededInt;
FreeSSERegs -= NeededSSE;
@@ -3966,6 +3994,8 @@ void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
// integer register.
if (FI.getReturnInfo().isIndirect())
--FreeIntRegs;
+ else if (NeededSSE && MaxVectorWidth > 0)
+ FI.setMaxVectorWidth(MaxVectorWidth);
// The chain argument effectively gives us another free register.
if (FI.isChainCall())
@@ -3980,7 +4010,8 @@ void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
bool IsNamedArg = ArgNo < NumRequiredArgs;
if (IsRegCall && it->type->isStructureOrClassType())
- it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE);
+ it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE,
+ MaxVectorWidth);
else
it->info = classifyArgumentType(it->type, FreeIntRegs, NeededInt,
NeededSSE, IsNamedArg);
@@ -3992,6 +4023,8 @@ void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
FreeIntRegs -= NeededInt;
FreeSSERegs -= NeededSSE;
+ if (MaxVectorWidth > FI.getMaxVectorWidth())
+ FI.setMaxVectorWidth(MaxVectorWidth);
} else {
it->info = getIndirectResult(it->type, FreeIntRegs);
}
@@ -4827,7 +4860,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
Builder.CreateCondBr(CC, UsingRegs, UsingOverflow);
- llvm::Type *DirectTy = CGF.ConvertType(Ty);
+ llvm::Type *DirectTy = CGF.ConvertType(Ty), *ElementTy = DirectTy;
if (isIndirect) DirectTy = DirectTy->getPointerTo(0);
// Case 1: consume registers.
@@ -4836,7 +4869,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
CGF.EmitBlock(UsingRegs);
Address RegSaveAreaPtr = Builder.CreateStructGEP(VAList, 4);
- RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr),
+ RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr), CGF.Int8Ty,
CharUnits::fromQuantity(8));
assert(RegAddr.getElementType() == CGF.Int8Ty);
@@ -4850,10 +4883,10 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
// registers we've used by the number of
CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8);
llvm::Value *RegOffset =
- Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
- RegAddr = Address(Builder.CreateInBoundsGEP(CGF.Int8Ty,
- RegAddr.getPointer(), RegOffset),
- RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
+ Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
+ RegAddr = Address(
+ Builder.CreateInBoundsGEP(CGF.Int8Ty, RegAddr.getPointer(), RegOffset),
+ CGF.Int8Ty, RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
RegAddr = Builder.CreateElementBitCast(RegAddr, DirectTy);
// Increase the used-register count.
@@ -4884,14 +4917,15 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
}
Address OverflowAreaAddr = Builder.CreateStructGEP(VAList, 3);
- Address OverflowArea(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"),
- OverflowAreaAlign);
+ Address OverflowArea =
+ Address(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"), CGF.Int8Ty,
+ OverflowAreaAlign);
// Round up address of argument to alignment
CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
if (Align > OverflowAreaAlign) {
llvm::Value *Ptr = OverflowArea.getPointer();
OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
- Align);
+ OverflowArea.getElementType(), Align);
}
MemAddr = Builder.CreateElementBitCast(OverflowArea, DirectTy);
@@ -4910,7 +4944,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
// Load the pointer if the argument was passed indirectly.
if (isIndirect) {
- Result = Address(Builder.CreateLoad(Result, "aggr"),
+ Result = Address(Builder.CreateLoad(Result, "aggr"), ElementTy,
getContext().getTypeAlignInChars(Ty));
}
@@ -5184,8 +5218,7 @@ bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base,
if (isEmptyRecord(getContext(), FT, true))
continue;
- // For compatibility with GCC, ignore empty bitfields in C++ mode.
- if (getContext().getLangOpts().CPlusPlus &&
+ if (isZeroLengthBitfieldPermittedInHomogeneousAggregate() &&
FD->isZeroLengthBitField(getContext()))
continue;
@@ -5482,6 +5515,7 @@ private:
bool isHomogeneousAggregateBaseType(QualType Ty) const override;
bool isHomogeneousAggregateSmallEnough(const Type *Ty,
uint64_t Members) const override;
+ bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;
bool isIllegalVectorType(QualType Ty) const;
@@ -5941,6 +5975,16 @@ bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
return Members <= 4;
}
+bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()
+ const {
+ // AAPCS64 says that the rule for whether something is a homogeneous
+ // aggregate is applied to the output of the data layout decision. So
+ // anything that doesn't affect the data layout also does not affect
+ // homogeneity. In particular, zero-length bitfields don't stop a struct
+ // being homogeneous.
+ return true;
+}
+
Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
CodeGenFunction &CGF) const {
ABIArgInfo AI = classifyArgumentType(Ty, /*IsVariadic=*/true,
@@ -6059,9 +6103,9 @@ Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
CGF.Builder.CreateStructGEP(VAListAddr, reg_top_index, "reg_top_p");
reg_top = CGF.Builder.CreateLoad(reg_top_p, "reg_top");
Address BaseAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, reg_top, reg_offs),
- CharUnits::fromQuantity(IsFPR ? 16 : 8));
+ CGF.Int8Ty, CharUnits::fromQuantity(IsFPR ? 16 : 8));
Address RegAddr = Address::invalid();
- llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty);
+ llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty), *ElementTy = MemTy;
if (IsIndirect) {
// If it's been passed indirectly (actually a struct), whatever we find from
@@ -6144,8 +6188,8 @@ Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
OnStackPtr = CGF.Builder.CreateIntToPtr(OnStackPtr, CGF.Int8PtrTy);
}
- Address OnStackAddr(OnStackPtr,
- std::max(CharUnits::fromQuantity(8), TyAlign));
+ Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty,
+ std::max(CharUnits::fromQuantity(8), TyAlign));
// All stack slots are multiples of 8 bytes.
CharUnits StackSlotSize = CharUnits::fromQuantity(8);
@@ -6177,11 +6221,11 @@ Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
//=======================================
CGF.EmitBlock(ContBlock);
- Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock,
- OnStackAddr, OnStackBlock, "vaargs.addr");
+ Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, OnStackAddr,
+ OnStackBlock, "vaargs.addr");
if (IsIndirect)
- return Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"),
+ return Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"), ElementTy,
TyAlign);
return ResAddr;
@@ -6200,7 +6244,8 @@ Address AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,
// Empty records are ignored for parameter passing purposes.
if (isEmptyRecord(getContext(), Ty, true)) {
- Address Addr(CGF.Builder.CreateLoad(VAListAddr, "ap.cur"), SlotSize);
+ Address Addr = Address(CGF.Builder.CreateLoad(VAListAddr, "ap.cur"),
+ getVAListElementType(CGF), SlotSize);
Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
return Addr;
}
@@ -6309,6 +6354,7 @@ private:
bool isHomogeneousAggregateBaseType(QualType Ty) const override;
bool isHomogeneousAggregateSmallEnough(const Type *Ty,
uint64_t Members) const override;
+ bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;
bool isEffectivelyAAPCS_VFP(unsigned callConvention, bool acceptHalf) const;
@@ -6972,6 +7018,15 @@ bool ARMABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
return Members <= 4;
}
+bool ARMABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate() const {
+ // AAPCS32 says that the rule for whether something is a homogeneous
+ // aggregate is applied to the output of the data layout decision. So
+ // anything that doesn't affect the data layout also does not affect
+ // homogeneity. In particular, zero-length bitfields don't stop a struct
+ // being homogeneous.
+ return true;
+}
+
bool ARMABIInfo::isEffectivelyAAPCS_VFP(unsigned callConvention,
bool acceptHalf) const {
// Give precedence to user-specified calling conventions.
@@ -6988,7 +7043,8 @@ Address ARMABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
// Empty records are ignored for parameter passing purposes.
if (isEmptyRecord(getContext(), Ty, true)) {
- Address Addr(CGF.Builder.CreateLoad(VAListAddr), SlotSize);
+ Address Addr = Address(CGF.Builder.CreateLoad(VAListAddr),
+ getVAListElementType(CGF), SlotSize);
Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
return Addr;
}
@@ -7483,12 +7539,9 @@ QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const {
// Check the fields.
for (const auto *FD : RD->fields()) {
- // For compatibility with GCC, ignore empty bitfields in C++ mode.
// Unlike isSingleElementStruct(), empty structure and array fields
// do count. So do anonymous bitfields that aren't zero-sized.
- if (getContext().getLangOpts().CPlusPlus &&
- FD->isZeroLengthBitField(getContext()))
- continue;
+
// Like isSingleElementStruct(), ignore C++20 empty data members.
if (FD->hasAttr<NoUniqueAddressAttr>() &&
isEmptyRecord(getContext(), FD->getType(), true))
@@ -7562,16 +7615,15 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
Address OverflowArgAreaPtr =
CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_ptr");
Address OverflowArgArea =
- Address(CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"),
- TyInfo.Align);
+ Address(CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"),
+ CGF.Int8Ty, TyInfo.Align);
Address MemAddr =
- CGF.Builder.CreateElementBitCast(OverflowArgArea, DirectTy, "mem_addr");
+ CGF.Builder.CreateElementBitCast(OverflowArgArea, DirectTy, "mem_addr");
// Update overflow_arg_area_ptr pointer
- llvm::Value *NewOverflowArgArea =
- CGF.Builder.CreateGEP(OverflowArgArea.getElementType(),
- OverflowArgArea.getPointer(), PaddedSizeV,
- "overflow_arg_area");
+ llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP(
+ OverflowArgArea.getElementType(), OverflowArgArea.getPointer(),
+ PaddedSizeV, "overflow_arg_area");
CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr);
return MemAddr;
@@ -7619,12 +7671,12 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
Address RegSaveAreaPtr =
CGF.Builder.CreateStructGEP(VAListAddr, 3, "reg_save_area_ptr");
llvm::Value *RegSaveArea =
- CGF.Builder.CreateLoad(RegSaveAreaPtr, "reg_save_area");
- Address RawRegAddr(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, RegOffset,
- "raw_reg_addr"),
- PaddedSize);
+ CGF.Builder.CreateLoad(RegSaveAreaPtr, "reg_save_area");
+ Address RawRegAddr(
+ CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, RegOffset, "raw_reg_addr"),
+ CGF.Int8Ty, PaddedSize);
Address RegAddr =
- CGF.Builder.CreateElementBitCast(RawRegAddr, DirectTy, "reg_addr");
+ CGF.Builder.CreateElementBitCast(RawRegAddr, DirectTy, "reg_addr");
// Update the register count
llvm::Value *One = llvm::ConstantInt::get(IndexTy, 1);
@@ -7640,10 +7692,10 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
Address OverflowArgAreaPtr =
CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_ptr");
Address OverflowArgArea =
- Address(CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"),
- PaddedSize);
+ Address(CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"),
+ CGF.Int8Ty, PaddedSize);
Address RawMemAddr =
- CGF.Builder.CreateConstByteGEP(OverflowArgArea, Padding, "raw_mem_addr");
+ CGF.Builder.CreateConstByteGEP(OverflowArgArea, Padding, "raw_mem_addr");
Address MemAddr =
CGF.Builder.CreateElementBitCast(RawMemAddr, DirectTy, "mem_addr");
@@ -7657,11 +7709,11 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
// Return the appropriate result.
CGF.EmitBlock(ContBlock);
- Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock,
- MemAddr, InMemBlock, "va_arg.addr");
+ Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,
+ "va_arg.addr");
if (IsIndirect)
- ResAddr = Address(CGF.Builder.CreateLoad(ResAddr, "indirect_arg"),
+ ResAddr = Address(CGF.Builder.CreateLoad(ResAddr, "indirect_arg"), ArgTy,
TyInfo.Align);
return ResAddr;
@@ -8272,32 +8324,93 @@ void M68kTargetCodeGenInfo::setTargetAttributes(
namespace {
class AVRABIInfo : public DefaultABIInfo {
+private:
+ // The total amount of registers can be used to pass parameters. It is 18 on
+ // AVR, or 6 on AVRTiny.
+ const unsigned ParamRegs;
+ // The total amount of registers can be used to pass return value. It is 8 on
+ // AVR, or 4 on AVRTiny.
+ const unsigned RetRegs;
+
public:
- AVRABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
+ AVRABIInfo(CodeGenTypes &CGT, unsigned NPR, unsigned NRR)
+ : DefaultABIInfo(CGT), ParamRegs(NPR), RetRegs(NRR) {}
+
+ ABIArgInfo classifyReturnType(QualType Ty, bool &LargeRet) const {
+ if (isAggregateTypeForABI(Ty)) {
+ // On AVR, a return struct with size less than or equals to 8 bytes is
+ // returned directly via registers R18-R25. On AVRTiny, a return struct
+ // with size less than or equals to 4 bytes is returned directly via
+ // registers R22-R25.
+ if (getContext().getTypeSize(Ty) <= RetRegs * 8)
+ return ABIArgInfo::getDirect();
+ // A return struct with larger size is returned via a stack
+ // slot, along with a pointer to it as the function's implicit argument.
+ LargeRet = true;
+ return getNaturalAlignIndirect(Ty);
+ }
+ // Otherwise we follow the default way which is compatible.
+ return DefaultABIInfo::classifyReturnType(Ty);
+ }
+
+ ABIArgInfo classifyArgumentType(QualType Ty, unsigned &NumRegs) const {
+ unsigned TySize = getContext().getTypeSize(Ty);
- ABIArgInfo classifyReturnType(QualType Ty) const {
- // A return struct with size less than or equal to 8 bytes is returned
- // directly via registers R18-R25.
- if (isAggregateTypeForABI(Ty) && getContext().getTypeSize(Ty) <= 64)
+ // An int8 type argument always costs two registers like an int16.
+ if (TySize == 8 && NumRegs >= 2) {
+ NumRegs -= 2;
+ return ABIArgInfo::getExtend(Ty);
+ }
+
+ // If the argument size is an odd number of bytes, round up the size
+ // to the next even number.
+ TySize = llvm::alignTo(TySize, 16);
+
+ // Any type including an array/struct type can be passed in rgisters,
+ // if there are enough registers left.
+ if (TySize <= NumRegs * 8) {
+ NumRegs -= TySize / 8;
return ABIArgInfo::getDirect();
- else
- return DefaultABIInfo::classifyReturnType(Ty);
+ }
+
+ // An argument is passed either completely in registers or completely in
+ // memory. Since there are not enough registers left, current argument
+ // and all other unprocessed arguments should be passed in memory.
+ // However we still need to return `ABIArgInfo::getDirect()` other than
+ // `ABIInfo::getNaturalAlignIndirect(Ty)`, otherwise an extra stack slot
+ // will be allocated, so the stack frame layout will be incompatible with
+ // avr-gcc.
+ NumRegs = 0;
+ return ABIArgInfo::getDirect();
}
- // Just copy the original implementation of DefaultABIInfo::computeInfo(),
- // since DefaultABIInfo::classify{Return,Argument}Type() are not virtual.
void computeInfo(CGFunctionInfo &FI) const override {
+ // Decide the return type.
+ bool LargeRet = false;
if (!getCXXABI().classifyReturnType(FI))
- FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
+ FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), LargeRet);
+
+ // Decide each argument type. The total number of registers can be used for
+ // arguments depends on several factors:
+ // 1. Arguments of varargs functions are passed on the stack. This applies
+ // even to the named arguments. So no register can be used.
+ // 2. Total 18 registers can be used on avr and 6 ones on avrtiny.
+ // 3. If the return type is a struct with too large size, two registers
+ // (out of 18/6) will be cost as an implicit pointer argument.
+ unsigned NumRegs = ParamRegs;
+ if (FI.isVariadic())
+ NumRegs = 0;
+ else if (LargeRet)
+ NumRegs -= 2;
for (auto &I : FI.arguments())
- I.info = classifyArgumentType(I.type);
+ I.info = classifyArgumentType(I.type, NumRegs);
}
};
class AVRTargetCodeGenInfo : public TargetCodeGenInfo {
public:
- AVRTargetCodeGenInfo(CodeGenTypes &CGT)
- : TargetCodeGenInfo(std::make_unique<AVRABIInfo>(CGT)) {}
+ AVRTargetCodeGenInfo(CodeGenTypes &CGT, unsigned NPR, unsigned NRR)
+ : TargetCodeGenInfo(std::make_unique<AVRABIInfo>(CGT, NPR, NRR)) {}
LangAS getGlobalVarAddressSpace(CodeGenModule &CGM,
const VarDecl *D) const override {
@@ -8599,9 +8712,10 @@ Address HexagonABIInfo::EmitVAArgFromMemory(CodeGenFunction &CGF,
// Get the type of the argument from memory and bitcast
// overflow area pointer to the argument type.
llvm::Type *PTy = CGF.ConvertTypeForMem(Ty);
- Address AddrTyped = CGF.Builder.CreateBitCast(
- Address(__overflow_area_pointer, CharUnits::fromQuantity(Align)),
- llvm::PointerType::getUnqual(PTy));
+ Address AddrTyped = CGF.Builder.CreateElementBitCast(
+ Address(__overflow_area_pointer, CGF.Int8Ty,
+ CharUnits::fromQuantity(Align)),
+ PTy);
// Round up to the minimum stack alignment for varargs which is 4 bytes.
uint64_t Offset = llvm::alignTo(CGF.getContext().getTypeSize(Ty) / 8, 4);
@@ -8620,9 +8734,8 @@ Address HexagonABIInfo::EmitVAArgForHexagon(CodeGenFunction &CGF,
QualType Ty) const {
// FIXME: Need to handle alignment
llvm::Type *BP = CGF.Int8PtrTy;
- llvm::Type *BPP = CGF.Int8PtrPtrTy;
CGBuilderTy &Builder = CGF.Builder;
- Address VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap");
+ Address VAListAddrAsBPP = Builder.CreateElementBitCast(VAListAddr, BP, "ap");
llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
// Handle address alignment for type alignment > 32 bits
uint64_t TyAlign = CGF.getContext().getTypeAlign(Ty) / 8;
@@ -8633,9 +8746,9 @@ Address HexagonABIInfo::EmitVAArgForHexagon(CodeGenFunction &CGF,
AddrAsInt = Builder.CreateAnd(AddrAsInt, Builder.getInt32(~(TyAlign - 1)));
Addr = Builder.CreateIntToPtr(AddrAsInt, BP);
}
- llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
- Address AddrTyped = Builder.CreateBitCast(
- Address(Addr, CharUnits::fromQuantity(TyAlign)), PTy);
+ Address AddrTyped = Builder.CreateElementBitCast(
+ Address(Addr, CGF.Int8Ty, CharUnits::fromQuantity(TyAlign)),
+ CGF.ConvertType(Ty));
uint64_t Offset = llvm::alignTo(CGF.getContext().getTypeSize(Ty) / 8, 4);
llvm::Value *NextAddr = Builder.CreateGEP(
@@ -8788,12 +8901,13 @@ Address HexagonABIInfo::EmitVAArgForHexagonLinux(CodeGenFunction &CGF,
// Implement the ContBlock
CGF.EmitBlock(ContBlock);
- llvm::Type *MemPTy = llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty));
+ llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty);
+ llvm::Type *MemPTy = llvm::PointerType::getUnqual(MemTy);
llvm::PHINode *ArgAddr = CGF.Builder.CreatePHI(MemPTy, 2, "vaarg.addr");
ArgAddr->addIncoming(__saved_reg_area_p, InRegBlock);
ArgAddr->addIncoming(__overflow_area_p, OnStackBlock);
- return Address(ArgAddr, CharUnits::fromQuantity(ArgAlign));
+ return Address(ArgAddr, MemTy, CharUnits::fromQuantity(ArgAlign));
}
Address HexagonABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
@@ -9213,7 +9327,7 @@ public:
llvm::Function *
createEnqueuedBlockKernel(CodeGenFunction &CGF,
llvm::Function *BlockInvokeFunc,
- llvm::Value *BlockLiteral) const override;
+ llvm::Type *BlockTy) const override;
bool shouldEmitStaticExternCAliases() const override;
void setCUDAKernelCallingConvention(const FunctionType *&FT) const override;
};
@@ -9381,7 +9495,7 @@ AMDGPUTargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
if (CGM.isTypeConstant(D->getType(), false) &&
D->hasConstantInitialization()) {
if (auto ConstAS = CGM.getTarget().getConstantAddressSpace())
- return ConstAS.getValue();
+ return *ConstAS;
}
return DefaultGlobalAS;
}
@@ -9474,6 +9588,28 @@ class SparcV8TargetCodeGenInfo : public TargetCodeGenInfo {
public:
SparcV8TargetCodeGenInfo(CodeGenTypes &CGT)
: TargetCodeGenInfo(std::make_unique<SparcV8ABIInfo>(CGT)) {}
+
+ llvm::Value *decodeReturnAddress(CodeGen::CodeGenFunction &CGF,
+ llvm::Value *Address) const override {
+ int Offset;
+ if (isAggregateTypeForABI(CGF.CurFnInfo->getReturnType()))
+ Offset = 12;
+ else
+ Offset = 8;
+ return CGF.Builder.CreateGEP(CGF.Int8Ty, Address,
+ llvm::ConstantInt::get(CGF.Int32Ty, Offset));
+ }
+
+ llvm::Value *encodeReturnAddress(CodeGen::CodeGenFunction &CGF,
+ llvm::Value *Address) const override {
+ int Offset;
+ if (isAggregateTypeForABI(CGF.CurFnInfo->getReturnType()))
+ Offset = -12;
+ else
+ Offset = -8;
+ return CGF.Builder.CreateGEP(CGF.Int8Ty, Address,
+ llvm::ConstantInt::get(CGF.Int32Ty, Offset));
+ }
};
} // end anonymous namespace
@@ -9684,7 +9820,8 @@ Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
CharUnits SlotSize = CharUnits::fromQuantity(8);
CGBuilderTy &Builder = CGF.Builder;
- Address Addr(Builder.CreateLoad(VAListAddr, "ap.cur"), SlotSize);
+ Address Addr = Address(Builder.CreateLoad(VAListAddr, "ap.cur"),
+ getVAListElementType(CGF), SlotSize);
llvm::Type *ArgPtrTy = llvm::PointerType::getUnqual(ArgTy);
auto TypeInfo = getContext().getTypeInfoInChars(Ty);
@@ -9715,19 +9852,19 @@ Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
case ABIArgInfo::IndirectAliased:
Stride = SlotSize;
ArgAddr = Builder.CreateElementBitCast(Addr, ArgPtrTy, "indirect");
- ArgAddr = Address(Builder.CreateLoad(ArgAddr, "indirect.arg"),
+ ArgAddr = Address(Builder.CreateLoad(ArgAddr, "indirect.arg"), ArgTy,
TypeInfo.Align);
break;
case ABIArgInfo::Ignore:
- return Address(llvm::UndefValue::get(ArgPtrTy), TypeInfo.Align);
+ return Address(llvm::UndefValue::get(ArgPtrTy), ArgTy, TypeInfo.Align);
}
// Update VAList.
Address NextPtr = Builder.CreateConstInBoundsByteGEP(Addr, Stride, "ap.next");
Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
- return Builder.CreateBitCast(ArgAddr, ArgPtrTy, "arg.addr");
+ return Builder.CreateElementBitCast(ArgAddr, ArgTy, "arg.addr");
}
void SparcV9ABIInfo::computeInfo(CGFunctionInfo &FI) const {
@@ -9748,6 +9885,18 @@ public:
bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
llvm::Value *Address) const override;
+
+ llvm::Value *decodeReturnAddress(CodeGen::CodeGenFunction &CGF,
+ llvm::Value *Address) const override {
+ return CGF.Builder.CreateGEP(CGF.Int8Ty, Address,
+ llvm::ConstantInt::get(CGF.Int32Ty, 8));
+ }
+
+ llvm::Value *encodeReturnAddress(CodeGen::CodeGenFunction &CGF,
+ llvm::Value *Address) const override {
+ return CGF.Builder.CreateGEP(CGF.Int8Ty, Address,
+ llvm::ConstantInt::get(CGF.Int32Ty, -8));
+ }
};
} // end anonymous namespace
@@ -10049,7 +10198,8 @@ Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
// Get the VAList.
CharUnits SlotSize = CharUnits::fromQuantity(4);
- Address AP(Builder.CreateLoad(VAListAddr), SlotSize);
+ Address AP = Address(Builder.CreateLoad(VAListAddr),
+ getVAListElementType(CGF), SlotSize);
// Handle the argument.
ABIArgInfo AI = classifyArgumentType(Ty);
@@ -10067,20 +10217,20 @@ Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
case ABIArgInfo::InAlloca:
llvm_unreachable("Unsupported ABI kind for va_arg");
case ABIArgInfo::Ignore:
- Val = Address(llvm::UndefValue::get(ArgPtrTy), TypeAlign);
+ Val = Address(llvm::UndefValue::get(ArgPtrTy), ArgTy, TypeAlign);
ArgSize = CharUnits::Zero();
break;
case ABIArgInfo::Extend:
case ABIArgInfo::Direct:
- Val = Builder.CreateBitCast(AP, ArgPtrTy);
+ Val = Builder.CreateElementBitCast(AP, ArgTy);
ArgSize = CharUnits::fromQuantity(
- getDataLayout().getTypeAllocSize(AI.getCoerceToType()));
+ getDataLayout().getTypeAllocSize(AI.getCoerceToType()));
ArgSize = ArgSize.alignTo(SlotSize);
break;
case ABIArgInfo::Indirect:
case ABIArgInfo::IndirectAliased:
Val = Builder.CreateElementBitCast(AP, ArgPtrTy);
- Val = Address(Builder.CreateLoad(Val), TypeAlign);
+ Val = Address(Builder.CreateLoad(Val), ArgTy, TypeAlign);
ArgSize = SlotSize;
break;
}
@@ -10284,10 +10434,10 @@ void CommonSPIRABIInfo::setCCs() {
}
ABIArgInfo SPIRVABIInfo::classifyKernelArgumentType(QualType Ty) const {
- if (getContext().getLangOpts().HIP) {
+ if (getContext().getLangOpts().CUDAIsDevice) {
// Coerce pointer arguments with default address space to CrossWorkGroup
- // pointers for HIPSPV. When the language mode is HIP, the SPIRTargetInfo
- // maps cuda_device to SPIR-V's CrossWorkGroup address space.
+ // pointers for HIPSPV/CUDASPV. When the language mode is HIP/CUDA, the
+ // SPIRTargetInfo maps cuda_device to SPIR-V's CrossWorkGroup address space.
llvm::Type *LTy = CGT.ConvertType(Ty);
auto DefaultAS = getContext().getTargetAddressSpace(LangAS::Default);
auto GlobalAS = getContext().getTargetAddressSpace(LangAS::cuda_device);
@@ -11094,7 +11244,8 @@ Address RISCVABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
// Empty records are ignored for parameter passing purposes.
if (isEmptyRecord(getContext(), Ty, true)) {
- Address Addr(CGF.Builder.CreateLoad(VAListAddr), SlotSize);
+ Address Addr = Address(CGF.Builder.CreateLoad(VAListAddr),
+ getVAListElementType(CGF), SlotSize);
Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
return Addr;
}
@@ -11200,6 +11351,165 @@ public:
} // end anonymous namespace
//===----------------------------------------------------------------------===//
+// CSKY ABI Implementation
+//===----------------------------------------------------------------------===//
+namespace {
+class CSKYABIInfo : public DefaultABIInfo {
+ static const int NumArgGPRs = 4;
+ static const int NumArgFPRs = 4;
+
+ static const unsigned XLen = 32;
+ unsigned FLen;
+
+public:
+ CSKYABIInfo(CodeGen::CodeGenTypes &CGT, unsigned FLen)
+ : DefaultABIInfo(CGT), FLen(FLen) {}
+
+ void computeInfo(CGFunctionInfo &FI) const override;
+ ABIArgInfo classifyArgumentType(QualType Ty, int &ArgGPRsLeft,
+ int &ArgFPRsLeft,
+ bool isReturnType = false) const;
+ ABIArgInfo classifyReturnType(QualType RetTy) const;
+
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
+};
+
+} // end anonymous namespace
+
+void CSKYABIInfo::computeInfo(CGFunctionInfo &FI) const {
+ QualType RetTy = FI.getReturnType();
+ if (!getCXXABI().classifyReturnType(FI))
+ FI.getReturnInfo() = classifyReturnType(RetTy);
+
+ bool IsRetIndirect = FI.getReturnInfo().getKind() == ABIArgInfo::Indirect;
+
+ // We must track the number of GPRs used in order to conform to the CSKY
+ // ABI, as integer scalars passed in registers should have signext/zeroext
+ // when promoted.
+ int ArgGPRsLeft = IsRetIndirect ? NumArgGPRs - 1 : NumArgGPRs;
+ int ArgFPRsLeft = FLen ? NumArgFPRs : 0;
+
+ for (auto &ArgInfo : FI.arguments()) {
+ ArgInfo.info = classifyArgumentType(ArgInfo.type, ArgGPRsLeft, ArgFPRsLeft);
+ }
+}
+
+Address CSKYABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
+ CharUnits SlotSize = CharUnits::fromQuantity(XLen / 8);
+
+ // Empty records are ignored for parameter passing purposes.
+ if (isEmptyRecord(getContext(), Ty, true)) {
+ Address Addr = Address(CGF.Builder.CreateLoad(VAListAddr),
+ getVAListElementType(CGF), SlotSize);
+ Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
+ return Addr;
+ }
+
+ auto TInfo = getContext().getTypeInfoInChars(Ty);
+
+ return emitVoidPtrVAArg(CGF, VAListAddr, Ty, false, TInfo, SlotSize,
+ /*AllowHigherAlign=*/true);
+}
+
+ABIArgInfo CSKYABIInfo::classifyArgumentType(QualType Ty, int &ArgGPRsLeft,
+ int &ArgFPRsLeft,
+ bool isReturnType) const {
+ assert(ArgGPRsLeft <= NumArgGPRs && "Arg GPR tracking underflow");
+ Ty = useFirstFieldIfTransparentUnion(Ty);
+
+ // Structures with either a non-trivial destructor or a non-trivial
+ // copy constructor are always passed indirectly.
+ if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
+ if (ArgGPRsLeft)
+ ArgGPRsLeft -= 1;
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/RAA ==
+ CGCXXABI::RAA_DirectInMemory);
+ }
+
+ // Ignore empty structs/unions.
+ if (isEmptyRecord(getContext(), Ty, true))
+ return ABIArgInfo::getIgnore();
+
+ if (!Ty->getAsUnionType())
+ if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
+ return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
+
+ uint64_t Size = getContext().getTypeSize(Ty);
+ // Pass floating point values via FPRs if possible.
+ if (Ty->isFloatingType() && !Ty->isComplexType() && FLen >= Size &&
+ ArgFPRsLeft) {
+ ArgFPRsLeft--;
+ return ABIArgInfo::getDirect();
+ }
+
+ // Complex types for the hard float ABI must be passed direct rather than
+ // using CoerceAndExpand.
+ if (Ty->isComplexType() && FLen && !isReturnType) {
+ QualType EltTy = Ty->castAs<ComplexType>()->getElementType();
+ if (getContext().getTypeSize(EltTy) <= FLen) {
+ ArgFPRsLeft -= 2;
+ return ABIArgInfo::getDirect();
+ }
+ }
+
+ if (!isAggregateTypeForABI(Ty)) {
+ // Treat an enum type as its underlying type.
+ if (const EnumType *EnumTy = Ty->getAs<EnumType>())
+ Ty = EnumTy->getDecl()->getIntegerType();
+
+ // All integral types are promoted to XLen width, unless passed on the
+ // stack.
+ if (Size < XLen && Ty->isIntegralOrEnumerationType())
+ return ABIArgInfo::getExtend(Ty);
+
+ if (const auto *EIT = Ty->getAs<BitIntType>()) {
+ if (EIT->getNumBits() < XLen)
+ return ABIArgInfo::getExtend(Ty);
+ }
+
+ return ABIArgInfo::getDirect();
+ }
+
+ // For argument type, the first 4*XLen parts of aggregate will be passed
+ // in registers, and the rest will be passed in stack.
+ // So we can coerce to integers directly and let backend handle it correctly.
+ // For return type, aggregate which <= 2*XLen will be returned in registers.
+ // Otherwise, aggregate will be returned indirectly.
+ if (!isReturnType || (isReturnType && Size <= 2 * XLen)) {
+ if (Size <= XLen) {
+ return ABIArgInfo::getDirect(
+ llvm::IntegerType::get(getVMContext(), XLen));
+ } else {
+ return ABIArgInfo::getDirect(llvm::ArrayType::get(
+ llvm::IntegerType::get(getVMContext(), XLen), (Size + 31) / XLen));
+ }
+ }
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
+}
+
+ABIArgInfo CSKYABIInfo::classifyReturnType(QualType RetTy) const {
+ if (RetTy->isVoidType())
+ return ABIArgInfo::getIgnore();
+
+ int ArgGPRsLeft = 2;
+ int ArgFPRsLeft = FLen ? 1 : 0;
+
+ // The rules for return and argument types are the same, so defer to
+ // classifyArgumentType.
+ return classifyArgumentType(RetTy, ArgGPRsLeft, ArgFPRsLeft, true);
+}
+
+namespace {
+class CSKYTargetCodeGenInfo : public TargetCodeGenInfo {
+public:
+ CSKYTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, unsigned FLen)
+ : TargetCodeGenInfo(std::make_unique<CSKYABIInfo>(CGT, FLen)) {}
+};
+} // end anonymous namespace
+
+//===----------------------------------------------------------------------===//
// Driver code
//===----------------------------------------------------------------------===//
@@ -11236,8 +11546,14 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
case llvm::Triple::mips64el:
return SetCGInfo(new MIPSTargetCodeGenInfo(Types, false));
- case llvm::Triple::avr:
- return SetCGInfo(new AVRTargetCodeGenInfo(Types));
+ case llvm::Triple::avr: {
+ // For passing parameters, R8~R25 are used on avr, and R18~R25 are used
+ // on avrtiny. For passing return value, R18~R25 are used on avr, and
+ // R22~R25 are used on avrtiny.
+ unsigned NPR = getTarget().getABI() == "avrtiny" ? 6 : 18;
+ unsigned NRR = getTarget().getABI() == "avrtiny" ? 4 : 8;
+ return SetCGInfo(new AVRTargetCodeGenInfo(Types, NPR, NRR));
+ }
case llvm::Triple::aarch64:
case llvm::Triple::aarch64_32:
@@ -11413,6 +11729,14 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
return SetCGInfo(new SPIRVTargetCodeGenInfo(Types));
case llvm::Triple::ve:
return SetCGInfo(new VETargetCodeGenInfo(Types));
+ case llvm::Triple::csky: {
+ bool IsSoftFloat = !getTarget().hasFeature("hard-float-abi");
+ bool hasFP64 = getTarget().hasFeature("fpuv2_df") ||
+ getTarget().hasFeature("fpuv3_df");
+ return SetCGInfo(new CSKYTargetCodeGenInfo(Types, IsSoftFloat ? 0
+ : hasFP64 ? 64
+ : 32));
+ }
}
}
@@ -11424,23 +11748,19 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
llvm::Function *
TargetCodeGenInfo::createEnqueuedBlockKernel(CodeGenFunction &CGF,
llvm::Function *Invoke,
- llvm::Value *BlockLiteral) const {
+ llvm::Type *BlockTy) const {
auto *InvokeFT = Invoke->getFunctionType();
- llvm::SmallVector<llvm::Type *, 2> ArgTys;
- for (auto &P : InvokeFT->params())
- ArgTys.push_back(P);
auto &C = CGF.getLLVMContext();
std::string Name = Invoke->getName().str() + "_kernel";
- auto *FT = llvm::FunctionType::get(llvm::Type::getVoidTy(C), ArgTys, false);
+ auto *FT = llvm::FunctionType::get(llvm::Type::getVoidTy(C),
+ InvokeFT->params(), false);
auto *F = llvm::Function::Create(FT, llvm::GlobalValue::ExternalLinkage, Name,
&CGF.CGM.getModule());
auto IP = CGF.Builder.saveIP();
auto *BB = llvm::BasicBlock::Create(C, "entry", F);
auto &Builder = CGF.Builder;
Builder.SetInsertPoint(BB);
- llvm::SmallVector<llvm::Value *, 2> Args;
- for (auto &A : F->args())
- Args.push_back(&A);
+ llvm::SmallVector<llvm::Value *, 2> Args(llvm::make_pointer_range(F->args()));
llvm::CallInst *call = Builder.CreateCall(Invoke, Args);
call->setCallingConv(Invoke->getCallingConv());
Builder.CreateRetVoid();
@@ -11458,11 +11778,10 @@ TargetCodeGenInfo::createEnqueuedBlockKernel(CodeGenFunction &CGF,
/// has "enqueued-block" function attribute and kernel argument metadata.
llvm::Function *AMDGPUTargetCodeGenInfo::createEnqueuedBlockKernel(
CodeGenFunction &CGF, llvm::Function *Invoke,
- llvm::Value *BlockLiteral) const {
+ llvm::Type *BlockTy) const {
auto &Builder = CGF.Builder;
auto &C = CGF.getLLVMContext();
- auto *BlockTy = BlockLiteral->getType()->getPointerElementType();
auto *InvokeFT = Invoke->getFunctionType();
llvm::SmallVector<llvm::Type *, 2> ArgTys;
llvm::SmallVector<llvm::Metadata *, 8> AddressQuals;
diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h
index dfdb2f5f55bb..bdd64977b475 100644
--- a/clang/lib/CodeGen/TargetInfo.h
+++ b/clang/lib/CodeGen/TargetInfo.h
@@ -329,7 +329,7 @@ public:
virtual llvm::Function *
createEnqueuedBlockKernel(CodeGenFunction &CGF,
llvm::Function *BlockInvokeFunc,
- llvm::Value *BlockLiteral) const;
+ llvm::Type *BlockTy) const;
/// \return true if the target supports alias from the unmangled name to the
/// mangled name of functions declared within an extern "C" region and marked
generated by cgit v1.3 (git 2.53.0) at 2026-09-09 18:00:19 +0000