[Transforms] Use std::nullopt instead of None (NFC)
This patch mechanically replaces None with std::nullopt where the compiler would warn if None were deprecated. The intent is to reduce the amount of manual work required in migrating from Optional to std::optional. This is part of an effort to migrate from llvm::Optional to std::optional: https://discourse.llvm.org/t/deprecating-llvm-optional-x-hasvalue-getvalue-getvalueor/63716
This commit is contained in:
parent
998960ee1f
commit
343de6856e
|
@ -1348,7 +1348,7 @@ struct AttributorConfig {
|
||||||
DenseSet<const char *> *Allowed = nullptr;
|
DenseSet<const char *> *Allowed = nullptr;
|
||||||
|
|
||||||
/// Maximum number of iterations to run until fixpoint.
|
/// Maximum number of iterations to run until fixpoint.
|
||||||
std::optional<unsigned> MaxFixpointIterations = None;
|
std::optional<unsigned> MaxFixpointIterations = std::nullopt;
|
||||||
|
|
||||||
/// A callback function that returns an ORE object from a Function pointer.
|
/// A callback function that returns an ORE object from a Function pointer.
|
||||||
///{
|
///{
|
||||||
|
@ -4392,7 +4392,7 @@ struct AAValueConstantRange
|
||||||
AA::getWithType(*ConstantInt::get(Ty->getContext(), *C), *Ty));
|
AA::getWithType(*ConstantInt::get(Ty->getContext(), *C), *Ty));
|
||||||
}
|
}
|
||||||
if (RangeV.isEmptySet())
|
if (RangeV.isEmptySet())
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -4647,7 +4647,7 @@ struct AAPotentialConstantValues
|
||||||
if (getAssumedSet().size() == 0) {
|
if (getAssumedSet().size() == 0) {
|
||||||
if (undefIsContained())
|
if (undefIsContained())
|
||||||
return UndefValue::get(getAssociatedValue().getType());
|
return UndefValue::get(getAssociatedValue().getType());
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
return nullptr;
|
return nullptr;
|
||||||
|
|
|
@ -72,11 +72,11 @@ class GVNLegacyPass;
|
||||||
/// Intended use is to create a default object, modify parameters with
|
/// Intended use is to create a default object, modify parameters with
|
||||||
/// additional setters and then pass it to GVN.
|
/// additional setters and then pass it to GVN.
|
||||||
struct GVNOptions {
|
struct GVNOptions {
|
||||||
std::optional<bool> AllowPRE = None;
|
std::optional<bool> AllowPRE = std::nullopt;
|
||||||
std::optional<bool> AllowLoadPRE = None;
|
std::optional<bool> AllowLoadPRE = std::nullopt;
|
||||||
std::optional<bool> AllowLoadInLoopPRE = None;
|
std::optional<bool> AllowLoadInLoopPRE = std::nullopt;
|
||||||
std::optional<bool> AllowLoadPRESplitBackedge = None;
|
std::optional<bool> AllowLoadPRESplitBackedge = std::nullopt;
|
||||||
std::optional<bool> AllowMemDep = None;
|
std::optional<bool> AllowMemDep = std::nullopt;
|
||||||
|
|
||||||
GVNOptions() = default;
|
GVNOptions() = default;
|
||||||
|
|
||||||
|
|
|
@ -403,7 +403,7 @@ Optional<PreservedAnalyses> LoopPassManager::runSinglePass(
|
||||||
// Check the PassInstrumentation's BeforePass callbacks before running the
|
// Check the PassInstrumentation's BeforePass callbacks before running the
|
||||||
// pass, skip its execution completely if asked to (callback returns false).
|
// pass, skip its execution completely if asked to (callback returns false).
|
||||||
if (!PI.runBeforePass<Loop>(*Pass, L))
|
if (!PI.runBeforePass<Loop>(*Pass, L))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
PreservedAnalyses PA = Pass->run(IR, AM, AR, U);
|
PreservedAnalyses PA = Pass->run(IR, AM, AR, U);
|
||||||
|
|
||||||
|
|
|
@ -574,12 +574,11 @@ bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI,
|
||||||
/// for the caller to accomplish, since each specific use of this function
|
/// for the caller to accomplish, since each specific use of this function
|
||||||
/// may have additional information which simplifies this fixup. For example,
|
/// may have additional information which simplifies this fixup. For example,
|
||||||
/// see restoreSSA() in the UnifyLoopExits pass.
|
/// see restoreSSA() in the UnifyLoopExits pass.
|
||||||
BasicBlock *CreateControlFlowHub(DomTreeUpdater *DTU,
|
BasicBlock *CreateControlFlowHub(
|
||||||
SmallVectorImpl<BasicBlock *> &GuardBlocks,
|
DomTreeUpdater *DTU, SmallVectorImpl<BasicBlock *> &GuardBlocks,
|
||||||
const SetVector<BasicBlock *> &Predecessors,
|
const SetVector<BasicBlock *> &Predecessors,
|
||||||
const SetVector<BasicBlock *> &Successors,
|
const SetVector<BasicBlock *> &Successors, const StringRef Prefix,
|
||||||
const StringRef Prefix,
|
Optional<unsigned> MaxControlFlowBooleans = std::nullopt);
|
||||||
Optional<unsigned> MaxControlFlowBooleans = None);
|
|
||||||
|
|
||||||
} // end namespace llvm
|
} // end namespace llvm
|
||||||
|
|
||||||
|
|
|
@ -89,9 +89,9 @@ private:
|
||||||
/// parameter. These are used by an implementation to opt-into stricter
|
/// parameter. These are used by an implementation to opt-into stricter
|
||||||
/// checking.
|
/// checking.
|
||||||
bool isFortifiedCallFoldable(CallInst *CI, unsigned ObjSizeOp,
|
bool isFortifiedCallFoldable(CallInst *CI, unsigned ObjSizeOp,
|
||||||
Optional<unsigned> SizeOp = None,
|
Optional<unsigned> SizeOp = std::nullopt,
|
||||||
Optional<unsigned> StrOp = None,
|
Optional<unsigned> StrOp = std::nullopt,
|
||||||
Optional<unsigned> FlagsOp = None);
|
Optional<unsigned> FlagsOp = std::nullopt);
|
||||||
};
|
};
|
||||||
|
|
||||||
/// LibCallSimplifier - This class implements a collection of optimizations
|
/// LibCallSimplifier - This class implements a collection of optimizations
|
||||||
|
|
|
@ -107,7 +107,7 @@ getFrameLayout(Function *Resume) {
|
||||||
// Pull information from the function attributes.
|
// Pull information from the function attributes.
|
||||||
auto Size = Resume->getParamDereferenceableBytes(0);
|
auto Size = Resume->getParamDereferenceableBytes(0);
|
||||||
if (!Size)
|
if (!Size)
|
||||||
return None;
|
return std::nullopt;
|
||||||
return std::make_pair(Size, Resume->getParamAlign(0).valueOrOne());
|
return std::make_pair(Size, Resume->getParamAlign(0).valueOrOne());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -892,7 +892,7 @@ static DIType *solveDIType(DIBuilder &Builder, Type *Ty,
|
||||||
RetType =
|
RetType =
|
||||||
Builder.createPointerType(nullptr, Layout.getTypeSizeInBits(Ty),
|
Builder.createPointerType(nullptr, Layout.getTypeSizeInBits(Ty),
|
||||||
Layout.getABITypeAlignment(Ty) * CHAR_BIT,
|
Layout.getABITypeAlignment(Ty) * CHAR_BIT,
|
||||||
/*DWARFAddressSpace=*/None, Name);
|
/*DWARFAddressSpace=*/std::nullopt, Name);
|
||||||
} else if (Ty->isStructTy()) {
|
} else if (Ty->isStructTy()) {
|
||||||
auto *DIStruct = Builder.createStructType(
|
auto *DIStruct = Builder.createStructType(
|
||||||
Scope, Name, Scope->getFile(), LineNum, Layout.getTypeSizeInBits(Ty),
|
Scope, Name, Scope->getFile(), LineNum, Layout.getTypeSizeInBits(Ty),
|
||||||
|
@ -1149,8 +1149,8 @@ static StructType *buildFrameType(Function &F, coro::Shape &Shape,
|
||||||
|
|
||||||
// Add header fields for the resume and destroy functions.
|
// Add header fields for the resume and destroy functions.
|
||||||
// We can rely on these being perfectly packed.
|
// We can rely on these being perfectly packed.
|
||||||
(void)B.addField(FnPtrTy, None, /*header*/ true);
|
(void)B.addField(FnPtrTy, std::nullopt, /*header*/ true);
|
||||||
(void)B.addField(FnPtrTy, None, /*header*/ true);
|
(void)B.addField(FnPtrTy, std::nullopt, /*header*/ true);
|
||||||
|
|
||||||
// PromiseAlloca field needs to be explicitly added here because it's
|
// PromiseAlloca field needs to be explicitly added here because it's
|
||||||
// a header field with a fixed offset based on its alignment. Hence it
|
// a header field with a fixed offset based on its alignment. Hence it
|
||||||
|
@ -1164,7 +1164,7 @@ static StructType *buildFrameType(Function &F, coro::Shape &Shape,
|
||||||
unsigned IndexBits = std::max(1U, Log2_64_Ceil(Shape.CoroSuspends.size()));
|
unsigned IndexBits = std::max(1U, Log2_64_Ceil(Shape.CoroSuspends.size()));
|
||||||
Type *IndexType = Type::getIntNTy(C, IndexBits);
|
Type *IndexType = Type::getIntNTy(C, IndexBits);
|
||||||
|
|
||||||
SwitchIndexFieldId = B.addField(IndexType, None);
|
SwitchIndexFieldId = B.addField(IndexType, std::nullopt);
|
||||||
} else {
|
} else {
|
||||||
assert(PromiseAlloca == nullptr && "lowering doesn't support promises");
|
assert(PromiseAlloca == nullptr && "lowering doesn't support promises");
|
||||||
}
|
}
|
||||||
|
@ -1189,8 +1189,8 @@ static StructType *buildFrameType(Function &F, coro::Shape &Shape,
|
||||||
if (const Argument *A = dyn_cast<Argument>(S.first))
|
if (const Argument *A = dyn_cast<Argument>(S.first))
|
||||||
if (A->hasByValAttr())
|
if (A->hasByValAttr())
|
||||||
FieldType = A->getParamByValType();
|
FieldType = A->getParamByValType();
|
||||||
FieldIDType Id =
|
FieldIDType Id = B.addField(FieldType, std::nullopt, false /*header*/,
|
||||||
B.addField(FieldType, None, false /*header*/, true /*IsSpillOfValue*/);
|
true /*IsSpillOfValue*/);
|
||||||
FrameData.setFieldIndex(S.first, Id);
|
FrameData.setFieldIndex(S.first, Id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -489,7 +489,7 @@ static bool findArgParts(Argument *Arg, const DataLayout &DL, AAResults &AAR,
|
||||||
Ptr = Ptr->stripAndAccumulateConstantOffsets(DL, Offset,
|
Ptr = Ptr->stripAndAccumulateConstantOffsets(DL, Offset,
|
||||||
/* AllowNonInbounds */ true);
|
/* AllowNonInbounds */ true);
|
||||||
if (Ptr != Arg)
|
if (Ptr != Arg)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
if (Offset.getSignificantBits() >= 64)
|
if (Offset.getSignificantBits() >= 64)
|
||||||
return false;
|
return false;
|
||||||
|
|
|
@ -1079,7 +1079,7 @@ Attributor::getAssumedConstant(const IRPosition &IRP,
|
||||||
for (auto &CB : SimplificationCallbacks.lookup(IRP)) {
|
for (auto &CB : SimplificationCallbacks.lookup(IRP)) {
|
||||||
Optional<Value *> SimplifiedV = CB(IRP, &AA, UsedAssumedInformation);
|
Optional<Value *> SimplifiedV = CB(IRP, &AA, UsedAssumedInformation);
|
||||||
if (!SimplifiedV)
|
if (!SimplifiedV)
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
if (isa_and_nonnull<Constant>(*SimplifiedV))
|
if (isa_and_nonnull<Constant>(*SimplifiedV))
|
||||||
return cast<Constant>(*SimplifiedV);
|
return cast<Constant>(*SimplifiedV);
|
||||||
return nullptr;
|
return nullptr;
|
||||||
|
@ -1091,7 +1091,7 @@ Attributor::getAssumedConstant(const IRPosition &IRP,
|
||||||
AA::ValueScope::Interprocedural,
|
AA::ValueScope::Interprocedural,
|
||||||
UsedAssumedInformation)) {
|
UsedAssumedInformation)) {
|
||||||
if (Values.empty())
|
if (Values.empty())
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
if (auto *C = dyn_cast_or_null<Constant>(
|
if (auto *C = dyn_cast_or_null<Constant>(
|
||||||
AAPotentialValues::getSingleValue(*this, AA, IRP, Values)))
|
AAPotentialValues::getSingleValue(*this, AA, IRP, Values)))
|
||||||
return C;
|
return C;
|
||||||
|
@ -1113,7 +1113,7 @@ Optional<Value *> Attributor::getAssumedSimplified(const IRPosition &IRP,
|
||||||
if (!getAssumedSimplifiedValues(IRP, AA, Values, S, UsedAssumedInformation))
|
if (!getAssumedSimplifiedValues(IRP, AA, Values, S, UsedAssumedInformation))
|
||||||
return &IRP.getAssociatedValue();
|
return &IRP.getAssociatedValue();
|
||||||
if (Values.empty())
|
if (Values.empty())
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
if (AA)
|
if (AA)
|
||||||
if (Value *V = AAPotentialValues::getSingleValue(*this, *AA, IRP, Values))
|
if (Value *V = AAPotentialValues::getSingleValue(*this, *AA, IRP, Values))
|
||||||
return V;
|
return V;
|
||||||
|
|
|
@ -2861,7 +2861,7 @@ private:
|
||||||
// If it is known (which we tested above) but it doesn't have a value,
|
// If it is known (which we tested above) but it doesn't have a value,
|
||||||
// then we can assume `undef` and hence the instruction is UB.
|
// then we can assume `undef` and hence the instruction is UB.
|
||||||
KnownUBInsts.insert(I);
|
KnownUBInsts.insert(I);
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
if (!*SimplifiedV)
|
if (!*SimplifiedV)
|
||||||
return nullptr;
|
return nullptr;
|
||||||
|
@ -2869,7 +2869,7 @@ private:
|
||||||
}
|
}
|
||||||
if (isa<UndefValue>(V)) {
|
if (isa<UndefValue>(V)) {
|
||||||
KnownUBInsts.insert(I);
|
KnownUBInsts.insert(I);
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
return V;
|
return V;
|
||||||
}
|
}
|
||||||
|
@ -5534,7 +5534,7 @@ struct AAValueSimplifyImpl : AAValueSimplify {
|
||||||
Optional<Constant *> COpt = AA.getAssumedConstant(A);
|
Optional<Constant *> COpt = AA.getAssumedConstant(A);
|
||||||
|
|
||||||
if (!COpt) {
|
if (!COpt) {
|
||||||
SimplifiedAssociatedValue = llvm::None;
|
SimplifiedAssociatedValue = std::nullopt;
|
||||||
A.recordDependence(AA, *this, DepClassTy::OPTIONAL);
|
A.recordDependence(AA, *this, DepClassTy::OPTIONAL);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -6091,7 +6091,7 @@ struct AAHeapToStackFunction final : public AAHeapToStack {
|
||||||
if (!isa<UndefValue>(InitVal)) {
|
if (!isa<UndefValue>(InitVal)) {
|
||||||
IRBuilder<> Builder(Alloca->getNextNode());
|
IRBuilder<> Builder(Alloca->getNextNode());
|
||||||
// TODO: Use alignment above if align!=1
|
// TODO: Use alignment above if align!=1
|
||||||
Builder.CreateMemSet(Alloca, InitVal, Size, None);
|
Builder.CreateMemSet(Alloca, InitVal, Size, std::nullopt);
|
||||||
}
|
}
|
||||||
HasChanged = ChangeStatus::CHANGED;
|
HasChanged = ChangeStatus::CHANGED;
|
||||||
}
|
}
|
||||||
|
@ -6108,7 +6108,7 @@ struct AAHeapToStackFunction final : public AAHeapToStack {
|
||||||
return APInt(64, 0);
|
return APInt(64, 0);
|
||||||
if (auto *CI = dyn_cast_or_null<ConstantInt>(SimpleV.value()))
|
if (auto *CI = dyn_cast_or_null<ConstantInt>(SimpleV.value()))
|
||||||
return CI->getValue();
|
return CI->getValue();
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
Optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
|
Optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
|
||||||
|
@ -6439,7 +6439,7 @@ ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
|
||||||
namespace {
|
namespace {
|
||||||
struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
|
struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
|
||||||
AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
|
AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
|
||||||
: AAPrivatizablePtr(IRP, A), PrivatizableType(llvm::None) {}
|
: AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
|
||||||
|
|
||||||
ChangeStatus indicatePessimisticFixpoint() override {
|
ChangeStatus indicatePessimisticFixpoint() override {
|
||||||
AAPrivatizablePtr::indicatePessimisticFixpoint();
|
AAPrivatizablePtr::indicatePessimisticFixpoint();
|
||||||
|
@ -9630,7 +9630,7 @@ private:
|
||||||
if (Unreachable.count(&Fn))
|
if (Unreachable.count(&Fn))
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Set of functions that we know for sure is reachable.
|
/// Set of functions that we know for sure is reachable.
|
||||||
|
@ -9909,7 +9909,7 @@ askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA,
|
||||||
|
|
||||||
if (!COpt.has_value()) {
|
if (!COpt.has_value()) {
|
||||||
A.recordDependence(AA, QueryingAA, DepClassTy::OPTIONAL);
|
A.recordDependence(AA, QueryingAA, DepClassTy::OPTIONAL);
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
if (auto *C = COpt.value()) {
|
if (auto *C = COpt.value()) {
|
||||||
A.recordDependence(AA, QueryingAA, DepClassTy::OPTIONAL);
|
A.recordDependence(AA, QueryingAA, DepClassTy::OPTIONAL);
|
||||||
|
@ -9972,7 +9972,7 @@ struct AAPotentialValuesImpl : AAPotentialValues {
|
||||||
return &IRP.getAssociatedValue();
|
return &IRP.getAssociatedValue();
|
||||||
Optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
|
Optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
|
||||||
if (!C)
|
if (!C)
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
if (C.value())
|
if (C.value())
|
||||||
if (auto *CC = AA::getWithType(**C, Ty))
|
if (auto *CC = AA::getWithType(**C, Ty))
|
||||||
return CC;
|
return CC;
|
||||||
|
|
|
@ -883,7 +883,7 @@ OptimizeGlobalAddressOfAllocation(GlobalVariable *GV, CallInst *CI,
|
||||||
if (!isa<UndefValue>(InitVal)) {
|
if (!isa<UndefValue>(InitVal)) {
|
||||||
IRBuilder<> Builder(CI->getNextNode());
|
IRBuilder<> Builder(CI->getNextNode());
|
||||||
// TODO: Use alignment above if align!=1
|
// TODO: Use alignment above if align!=1
|
||||||
Builder.CreateMemSet(NewGV, InitVal, AllocSize, None);
|
Builder.CreateMemSet(NewGV, InitVal, AllocSize, std::nullopt);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update users of the allocation to use the new global instead.
|
// Update users of the allocation to use the new global instead.
|
||||||
|
|
|
@ -469,7 +469,7 @@ constantMatches(Value *V, unsigned GVN,
|
||||||
// See if we have a constants
|
// See if we have a constants
|
||||||
Constant *CST = dyn_cast<Constant>(V);
|
Constant *CST = dyn_cast<Constant>(V);
|
||||||
if (!CST)
|
if (!CST)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Holds a mapping from a global value number to a Constant.
|
// Holds a mapping from a global value number to a Constant.
|
||||||
DenseMap<unsigned, Constant *>::iterator GVNToConstantIt;
|
DenseMap<unsigned, Constant *>::iterator GVNToConstantIt;
|
||||||
|
@ -684,7 +684,8 @@ Function *IROutliner::createFunction(Module &M, OutlinableGroup &Group,
|
||||||
Unit /* Context */, F->getName(), MangledNameStream.str(),
|
Unit /* Context */, F->getName(), MangledNameStream.str(),
|
||||||
Unit /* File */,
|
Unit /* File */,
|
||||||
0 /* Line 0 is reserved for compiler-generated code. */,
|
0 /* Line 0 is reserved for compiler-generated code. */,
|
||||||
DB.createSubroutineType(DB.getOrCreateTypeArray(None)), /* void type */
|
DB.createSubroutineType(
|
||||||
|
DB.getOrCreateTypeArray(std::nullopt)), /* void type */
|
||||||
0, /* Line 0 is reserved for compiler-generated code. */
|
0, /* Line 0 is reserved for compiler-generated code. */
|
||||||
DINode::DIFlags::FlagArtificial /* Compiler-generated code. */,
|
DINode::DIFlags::FlagArtificial /* Compiler-generated code. */,
|
||||||
/* Outlined code is optimized code by definition. */
|
/* Outlined code is optimized code by definition. */
|
||||||
|
@ -1193,7 +1194,7 @@ static Optional<unsigned> getGVNForPHINode(OutlinableRegion &Region,
|
||||||
Optional<unsigned> OGVN = Cand.getGVN(Incoming);
|
Optional<unsigned> OGVN = Cand.getGVN(Incoming);
|
||||||
if (!OGVN && Blocks.contains(IncomingBlock)) {
|
if (!OGVN && Blocks.contains(IncomingBlock)) {
|
||||||
Region.IgnoreRegion = true;
|
Region.IgnoreRegion = true;
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// If the incoming block isn't in the region, we don't have to worry about
|
// If the incoming block isn't in the region, we don't have to worry about
|
||||||
|
@ -2013,7 +2014,7 @@ Optional<unsigned> findDuplicateOutputBlock(
|
||||||
MatchingNum++;
|
MatchingNum++;
|
||||||
}
|
}
|
||||||
|
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Remove empty output blocks from the outlined region.
|
/// Remove empty output blocks from the outlined region.
|
||||||
|
|
|
@ -2342,7 +2342,7 @@ struct AAICVTracker : public StateWrapper<BooleanState, AbstractAttribute> {
|
||||||
virtual Optional<Value *> getReplacementValue(InternalControlVar ICV,
|
virtual Optional<Value *> getReplacementValue(InternalControlVar ICV,
|
||||||
const Instruction *I,
|
const Instruction *I,
|
||||||
Attributor &A) const {
|
Attributor &A) const {
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return an assumed unique ICV value if a single candidate is found. If
|
/// Return an assumed unique ICV value if a single candidate is found. If
|
||||||
|
@ -2447,7 +2447,7 @@ struct AAICVTrackerFunction : public AAICVTracker {
|
||||||
const auto *CB = dyn_cast<CallBase>(&I);
|
const auto *CB = dyn_cast<CallBase>(&I);
|
||||||
if (!CB || CB->hasFnAttr("no_openmp") ||
|
if (!CB || CB->hasFnAttr("no_openmp") ||
|
||||||
CB->hasFnAttr("no_openmp_routines"))
|
CB->hasFnAttr("no_openmp_routines"))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
|
auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
|
||||||
auto &GetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Getter];
|
auto &GetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Getter];
|
||||||
|
@ -2458,7 +2458,7 @@ struct AAICVTrackerFunction : public AAICVTracker {
|
||||||
if (CalledFunction == nullptr)
|
if (CalledFunction == nullptr)
|
||||||
return nullptr;
|
return nullptr;
|
||||||
if (CalledFunction == GetterRFI.Declaration)
|
if (CalledFunction == GetterRFI.Declaration)
|
||||||
return None;
|
return std::nullopt;
|
||||||
if (CalledFunction == SetterRFI.Declaration) {
|
if (CalledFunction == SetterRFI.Declaration) {
|
||||||
if (ICVReplacementValuesMap[ICV].count(&I))
|
if (ICVReplacementValuesMap[ICV].count(&I))
|
||||||
return ICVReplacementValuesMap[ICV].lookup(&I);
|
return ICVReplacementValuesMap[ICV].lookup(&I);
|
||||||
|
@ -2487,7 +2487,7 @@ struct AAICVTrackerFunction : public AAICVTracker {
|
||||||
// We don't check unique value for a function, so return None.
|
// We don't check unique value for a function, so return None.
|
||||||
Optional<Value *>
|
Optional<Value *>
|
||||||
getUniqueReplacementValue(InternalControlVar ICV) const override {
|
getUniqueReplacementValue(InternalControlVar ICV) const override {
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return the value with which \p I can be replaced for specific \p ICV.
|
/// Return the value with which \p I can be replaced for specific \p ICV.
|
||||||
|
|
|
@ -814,8 +814,8 @@ void updatePublicTypeTestCalls(Module &M,
|
||||||
for (Use &U : make_early_inc_range(PublicTypeTestFunc->uses())) {
|
for (Use &U : make_early_inc_range(PublicTypeTestFunc->uses())) {
|
||||||
auto *CI = cast<CallInst>(U.getUser());
|
auto *CI = cast<CallInst>(U.getUser());
|
||||||
auto *NewCI = CallInst::Create(
|
auto *NewCI = CallInst::Create(
|
||||||
TypeTestFunc, {CI->getArgOperand(0), CI->getArgOperand(1)}, None, "",
|
TypeTestFunc, {CI->getArgOperand(0), CI->getArgOperand(1)},
|
||||||
CI);
|
std::nullopt, "", CI);
|
||||||
CI->replaceAllUsesWith(NewCI);
|
CI->replaceAllUsesWith(NewCI);
|
||||||
CI->eraseFromParent();
|
CI->eraseFromParent();
|
||||||
}
|
}
|
||||||
|
|
|
@ -243,7 +243,7 @@ getMaskedTypeForICmpPair(Value *&A, Value *&B, Value *&C,
|
||||||
// Don't allow pointers. Splat vectors are fine.
|
// Don't allow pointers. Splat vectors are fine.
|
||||||
if (!LHS->getOperand(0)->getType()->isIntOrIntVectorTy() ||
|
if (!LHS->getOperand(0)->getType()->isIntOrIntVectorTy() ||
|
||||||
!RHS->getOperand(0)->getType()->isIntOrIntVectorTy())
|
!RHS->getOperand(0)->getType()->isIntOrIntVectorTy())
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Here comes the tricky part:
|
// Here comes the tricky part:
|
||||||
// LHS might be of the form L11 & L12 == X, X == L21 & L22,
|
// LHS might be of the form L11 & L12 == X, X == L21 & L22,
|
||||||
|
@ -274,7 +274,7 @@ getMaskedTypeForICmpPair(Value *&A, Value *&B, Value *&C,
|
||||||
|
|
||||||
// Bail if LHS was a icmp that can't be decomposed into an equality.
|
// Bail if LHS was a icmp that can't be decomposed into an equality.
|
||||||
if (!ICmpInst::isEquality(PredL))
|
if (!ICmpInst::isEquality(PredL))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
Value *R1 = RHS->getOperand(0);
|
Value *R1 = RHS->getOperand(0);
|
||||||
Value *R2 = RHS->getOperand(1);
|
Value *R2 = RHS->getOperand(1);
|
||||||
|
@ -288,7 +288,7 @@ getMaskedTypeForICmpPair(Value *&A, Value *&B, Value *&C,
|
||||||
A = R12;
|
A = R12;
|
||||||
D = R11;
|
D = R11;
|
||||||
} else {
|
} else {
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
E = R2;
|
E = R2;
|
||||||
R1 = nullptr;
|
R1 = nullptr;
|
||||||
|
@ -316,7 +316,7 @@ getMaskedTypeForICmpPair(Value *&A, Value *&B, Value *&C,
|
||||||
|
|
||||||
// Bail if RHS was a icmp that can't be decomposed into an equality.
|
// Bail if RHS was a icmp that can't be decomposed into an equality.
|
||||||
if (!ICmpInst::isEquality(PredR))
|
if (!ICmpInst::isEquality(PredR))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Look for ANDs on the right side of the RHS icmp.
|
// Look for ANDs on the right side of the RHS icmp.
|
||||||
if (!Ok) {
|
if (!Ok) {
|
||||||
|
@ -336,7 +336,7 @@ getMaskedTypeForICmpPair(Value *&A, Value *&B, Value *&C,
|
||||||
E = R1;
|
E = R1;
|
||||||
Ok = true;
|
Ok = true;
|
||||||
} else {
|
} else {
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
assert(Ok && "Failed to find AND on the right side of the RHS icmp.");
|
assert(Ok && "Failed to find AND on the right side of the RHS icmp.");
|
||||||
|
@ -1019,7 +1019,7 @@ struct IntPart {
|
||||||
static Optional<IntPart> matchIntPart(Value *V) {
|
static Optional<IntPart> matchIntPart(Value *V) {
|
||||||
Value *X;
|
Value *X;
|
||||||
if (!match(V, m_OneUse(m_Trunc(m_Value(X)))))
|
if (!match(V, m_OneUse(m_Trunc(m_Value(X)))))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
unsigned NumOriginalBits = X->getType()->getScalarSizeInBits();
|
unsigned NumOriginalBits = X->getType()->getScalarSizeInBits();
|
||||||
unsigned NumExtractedBits = V->getType()->getScalarSizeInBits();
|
unsigned NumExtractedBits = V->getType()->getScalarSizeInBits();
|
||||||
|
|
|
@ -2346,7 +2346,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
|
||||||
Pred == ICmpInst::ICMP_NE && LHS->getOpcode() == Instruction::Load &&
|
Pred == ICmpInst::ICMP_NE && LHS->getOpcode() == Instruction::Load &&
|
||||||
LHS->getType()->isPointerTy() &&
|
LHS->getType()->isPointerTy() &&
|
||||||
isValidAssumeForContext(II, LHS, &DT)) {
|
isValidAssumeForContext(II, LHS, &DT)) {
|
||||||
MDNode *MD = MDNode::get(II->getContext(), None);
|
MDNode *MD = MDNode::get(II->getContext(), std::nullopt);
|
||||||
LHS->setMetadata(LLVMContext::MD_nonnull, MD);
|
LHS->setMetadata(LLVMContext::MD_nonnull, MD);
|
||||||
return RemoveConditionFromAssume(II);
|
return RemoveConditionFromAssume(II);
|
||||||
|
|
||||||
|
|
|
@ -5873,13 +5873,13 @@ InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred,
|
||||||
if (auto *CI = dyn_cast<ConstantInt>(C)) {
|
if (auto *CI = dyn_cast<ConstantInt>(C)) {
|
||||||
// Bail out if the constant can't be safely incremented/decremented.
|
// Bail out if the constant can't be safely incremented/decremented.
|
||||||
if (!ConstantIsOk(CI))
|
if (!ConstantIsOk(CI))
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
} else if (auto *FVTy = dyn_cast<FixedVectorType>(Type)) {
|
} else if (auto *FVTy = dyn_cast<FixedVectorType>(Type)) {
|
||||||
unsigned NumElts = FVTy->getNumElements();
|
unsigned NumElts = FVTy->getNumElements();
|
||||||
for (unsigned i = 0; i != NumElts; ++i) {
|
for (unsigned i = 0; i != NumElts; ++i) {
|
||||||
Constant *Elt = C->getAggregateElement(i);
|
Constant *Elt = C->getAggregateElement(i);
|
||||||
if (!Elt)
|
if (!Elt)
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
|
|
||||||
if (isa<UndefValue>(Elt))
|
if (isa<UndefValue>(Elt))
|
||||||
continue;
|
continue;
|
||||||
|
@ -5888,14 +5888,14 @@ InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred,
|
||||||
// know that this constant is min/max.
|
// know that this constant is min/max.
|
||||||
auto *CI = dyn_cast<ConstantInt>(Elt);
|
auto *CI = dyn_cast<ConstantInt>(Elt);
|
||||||
if (!CI || !ConstantIsOk(CI))
|
if (!CI || !ConstantIsOk(CI))
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
|
|
||||||
if (!SafeReplacementConstant)
|
if (!SafeReplacementConstant)
|
||||||
SafeReplacementConstant = CI;
|
SafeReplacementConstant = CI;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// ConstantExpr?
|
// ConstantExpr?
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// It may not be safe to change a compare predicate in the presence of
|
// It may not be safe to change a compare predicate in the presence of
|
||||||
|
|
|
@ -522,7 +522,7 @@ std::array<Value *, 2> Negator::getSortedOperandsOfBinOp(Instruction *I) {
|
||||||
// endless combine looping.
|
// endless combine looping.
|
||||||
for (Instruction *I : llvm::reverse(NewInstructions))
|
for (Instruction *I : llvm::reverse(NewInstructions))
|
||||||
I->eraseFromParent();
|
I->eraseFromParent();
|
||||||
return llvm::None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
return std::make_pair(ArrayRef<Instruction *>(NewInstructions), Negated);
|
return std::make_pair(ArrayRef<Instruction *>(NewInstructions), Negated);
|
||||||
}
|
}
|
||||||
|
|
|
@ -871,7 +871,7 @@ Instruction *InstCombinerImpl::foldAggregateConstructionIntoAggregateReuse(
|
||||||
if (NumAggElts > 2)
|
if (NumAggElts > 2)
|
||||||
return nullptr;
|
return nullptr;
|
||||||
|
|
||||||
static constexpr auto NotFound = None;
|
static constexpr auto NotFound = std::nullopt;
|
||||||
static constexpr auto FoundMismatch = nullptr;
|
static constexpr auto FoundMismatch = nullptr;
|
||||||
|
|
||||||
// Try to find a value of each element of an aggregate.
|
// Try to find a value of each element of an aggregate.
|
||||||
|
@ -1032,7 +1032,8 @@ Instruction *InstCombinerImpl::foldAggregateConstructionIntoAggregateReuse(
|
||||||
Optional<Value *> SourceAggregate;
|
Optional<Value *> SourceAggregate;
|
||||||
|
|
||||||
// Can we find the source aggregate without looking at predecessors?
|
// Can we find the source aggregate without looking at predecessors?
|
||||||
SourceAggregate = FindCommonSourceAggregate(/*UseBB=*/None, /*PredBB=*/None);
|
SourceAggregate = FindCommonSourceAggregate(/*UseBB=*/std::nullopt,
|
||||||
|
/*PredBB=*/std::nullopt);
|
||||||
if (Describe(SourceAggregate) != AggregateDescription::NotFound) {
|
if (Describe(SourceAggregate) != AggregateDescription::NotFound) {
|
||||||
if (Describe(SourceAggregate) == AggregateDescription::FoundMismatch)
|
if (Describe(SourceAggregate) == AggregateDescription::FoundMismatch)
|
||||||
return nullptr; // Conflicting source aggregates!
|
return nullptr; // Conflicting source aggregates!
|
||||||
|
|
|
@ -2888,7 +2888,7 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
|
||||||
Module *M = II->getModule();
|
Module *M = II->getModule();
|
||||||
Function *F = Intrinsic::getDeclaration(M, Intrinsic::donothing);
|
Function *F = Intrinsic::getDeclaration(M, Intrinsic::donothing);
|
||||||
InvokeInst::Create(F, II->getNormalDest(), II->getUnwindDest(),
|
InvokeInst::Create(F, II->getNormalDest(), II->getUnwindDest(),
|
||||||
None, "", II->getParent());
|
std::nullopt, "", II->getParent());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove debug intrinsics which describe the value contained within the
|
// Remove debug intrinsics which describe the value contained within the
|
||||||
|
@ -4214,7 +4214,7 @@ bool InstCombinerImpl::run() {
|
||||||
auto getOptionalSinkBlockForInst =
|
auto getOptionalSinkBlockForInst =
|
||||||
[this](Instruction *I) -> std::optional<BasicBlock *> {
|
[this](Instruction *I) -> std::optional<BasicBlock *> {
|
||||||
if (!EnableCodeSinking)
|
if (!EnableCodeSinking)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
BasicBlock *BB = I->getParent();
|
BasicBlock *BB = I->getParent();
|
||||||
BasicBlock *UserParent = nullptr;
|
BasicBlock *UserParent = nullptr;
|
||||||
|
@ -4224,7 +4224,7 @@ bool InstCombinerImpl::run() {
|
||||||
if (U->isDroppable())
|
if (U->isDroppable())
|
||||||
continue;
|
continue;
|
||||||
if (NumUsers > MaxSinkNumUsers)
|
if (NumUsers > MaxSinkNumUsers)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
Instruction *UserInst = cast<Instruction>(U);
|
Instruction *UserInst = cast<Instruction>(U);
|
||||||
// Special handling for Phi nodes - get the block the use occurs in.
|
// Special handling for Phi nodes - get the block the use occurs in.
|
||||||
|
@ -4235,14 +4235,14 @@ bool InstCombinerImpl::run() {
|
||||||
// sophisticated analysis (i.e finding NearestCommonDominator of
|
// sophisticated analysis (i.e finding NearestCommonDominator of
|
||||||
// these use blocks).
|
// these use blocks).
|
||||||
if (UserParent && UserParent != PN->getIncomingBlock(i))
|
if (UserParent && UserParent != PN->getIncomingBlock(i))
|
||||||
return None;
|
return std::nullopt;
|
||||||
UserParent = PN->getIncomingBlock(i);
|
UserParent = PN->getIncomingBlock(i);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
assert(UserParent && "expected to find user block!");
|
assert(UserParent && "expected to find user block!");
|
||||||
} else {
|
} else {
|
||||||
if (UserParent && UserParent != UserInst->getParent())
|
if (UserParent && UserParent != UserInst->getParent())
|
||||||
return None;
|
return std::nullopt;
|
||||||
UserParent = UserInst->getParent();
|
UserParent = UserInst->getParent();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -4252,7 +4252,7 @@ bool InstCombinerImpl::run() {
|
||||||
// Try sinking to another block. If that block is unreachable, then do
|
// Try sinking to another block. If that block is unreachable, then do
|
||||||
// not bother. SimplifyCFG should handle it.
|
// not bother. SimplifyCFG should handle it.
|
||||||
if (UserParent == BB || !DT.isReachableFromEntry(UserParent))
|
if (UserParent == BB || !DT.isReachableFromEntry(UserParent))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
auto *Term = UserParent->getTerminator();
|
auto *Term = UserParent->getTerminator();
|
||||||
// See if the user is one of our successors that has only one
|
// See if the user is one of our successors that has only one
|
||||||
|
@ -4264,7 +4264,7 @@ bool InstCombinerImpl::run() {
|
||||||
// - the User will be executed at most once.
|
// - the User will be executed at most once.
|
||||||
// So sinking I down to User is always profitable or neutral.
|
// So sinking I down to User is always profitable or neutral.
|
||||||
if (UserParent->getUniquePredecessor() != BB && !succ_empty(Term))
|
if (UserParent->getUniquePredecessor() != BB && !succ_empty(Term))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
assert(DT.dominates(BB, UserParent) && "Dominance relation broken?");
|
assert(DT.dominates(BB, UserParent) && "Dominance relation broken?");
|
||||||
}
|
}
|
||||||
|
@ -4274,7 +4274,7 @@ bool InstCombinerImpl::run() {
|
||||||
|
|
||||||
// No user or only has droppable users.
|
// No user or only has droppable users.
|
||||||
if (!UserParent)
|
if (!UserParent)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
return UserParent;
|
return UserParent;
|
||||||
};
|
};
|
||||||
|
|
|
@ -1305,12 +1305,13 @@ void AddressSanitizer::getInterestingMemoryOperands(
|
||||||
if (!ClInstrumentAtomics || ignoreAccess(I, RMW->getPointerOperand()))
|
if (!ClInstrumentAtomics || ignoreAccess(I, RMW->getPointerOperand()))
|
||||||
return;
|
return;
|
||||||
Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
|
Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
|
||||||
RMW->getValOperand()->getType(), None);
|
RMW->getValOperand()->getType(), std::nullopt);
|
||||||
} else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
|
} else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
|
||||||
if (!ClInstrumentAtomics || ignoreAccess(I, XCHG->getPointerOperand()))
|
if (!ClInstrumentAtomics || ignoreAccess(I, XCHG->getPointerOperand()))
|
||||||
return;
|
return;
|
||||||
Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
|
Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
|
||||||
XCHG->getCompareOperand()->getType(), None);
|
XCHG->getCompareOperand()->getType(),
|
||||||
|
std::nullopt);
|
||||||
} else if (auto CI = dyn_cast<CallInst>(I)) {
|
} else if (auto CI = dyn_cast<CallInst>(I)) {
|
||||||
if (CI->getIntrinsicID() == Intrinsic::masked_load ||
|
if (CI->getIntrinsicID() == Intrinsic::masked_load ||
|
||||||
CI->getIntrinsicID() == Intrinsic::masked_store) {
|
CI->getIntrinsicID() == Intrinsic::masked_store) {
|
||||||
|
|
|
@ -1087,8 +1087,8 @@ bool DataFlowSanitizer::initializeModule(Module &M) {
|
||||||
Type::getInt8PtrTy(*Ctx), IntptrTy};
|
Type::getInt8PtrTy(*Ctx), IntptrTy};
|
||||||
DFSanSetLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx),
|
DFSanSetLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx),
|
||||||
DFSanSetLabelArgs, /*isVarArg=*/false);
|
DFSanSetLabelArgs, /*isVarArg=*/false);
|
||||||
DFSanNonzeroLabelFnTy =
|
DFSanNonzeroLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx), std::nullopt,
|
||||||
FunctionType::get(Type::getVoidTy(*Ctx), None, /*isVarArg=*/false);
|
/*isVarArg=*/false);
|
||||||
DFSanVarargWrapperFnTy = FunctionType::get(
|
DFSanVarargWrapperFnTy = FunctionType::get(
|
||||||
Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false);
|
Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false);
|
||||||
DFSanConditionalCallbackFnTy =
|
DFSanConditionalCallbackFnTy =
|
||||||
|
|
|
@ -757,12 +757,13 @@ void HWAddressSanitizer::getInterestingMemoryOperands(
|
||||||
if (!ClInstrumentAtomics || ignoreAccess(I, RMW->getPointerOperand()))
|
if (!ClInstrumentAtomics || ignoreAccess(I, RMW->getPointerOperand()))
|
||||||
return;
|
return;
|
||||||
Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
|
Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
|
||||||
RMW->getValOperand()->getType(), None);
|
RMW->getValOperand()->getType(), std::nullopt);
|
||||||
} else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
|
} else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
|
||||||
if (!ClInstrumentAtomics || ignoreAccess(I, XCHG->getPointerOperand()))
|
if (!ClInstrumentAtomics || ignoreAccess(I, XCHG->getPointerOperand()))
|
||||||
return;
|
return;
|
||||||
Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
|
Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
|
||||||
XCHG->getCompareOperand()->getType(), None);
|
XCHG->getCompareOperand()->getType(),
|
||||||
|
std::nullopt);
|
||||||
} else if (auto CI = dyn_cast<CallInst>(I)) {
|
} else if (auto CI = dyn_cast<CallInst>(I)) {
|
||||||
for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
|
for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
|
||||||
if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
|
if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
|
||||||
|
|
|
@ -271,31 +271,31 @@ Optional<InterestingMemoryAccess>
|
||||||
MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
|
MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
|
||||||
// Do not instrument the load fetching the dynamic shadow address.
|
// Do not instrument the load fetching the dynamic shadow address.
|
||||||
if (DynamicShadowOffset == I)
|
if (DynamicShadowOffset == I)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
InterestingMemoryAccess Access;
|
InterestingMemoryAccess Access;
|
||||||
|
|
||||||
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
|
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
|
||||||
if (!ClInstrumentReads)
|
if (!ClInstrumentReads)
|
||||||
return None;
|
return std::nullopt;
|
||||||
Access.IsWrite = false;
|
Access.IsWrite = false;
|
||||||
Access.AccessTy = LI->getType();
|
Access.AccessTy = LI->getType();
|
||||||
Access.Addr = LI->getPointerOperand();
|
Access.Addr = LI->getPointerOperand();
|
||||||
} else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
|
} else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
|
||||||
if (!ClInstrumentWrites)
|
if (!ClInstrumentWrites)
|
||||||
return None;
|
return std::nullopt;
|
||||||
Access.IsWrite = true;
|
Access.IsWrite = true;
|
||||||
Access.AccessTy = SI->getValueOperand()->getType();
|
Access.AccessTy = SI->getValueOperand()->getType();
|
||||||
Access.Addr = SI->getPointerOperand();
|
Access.Addr = SI->getPointerOperand();
|
||||||
} else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
|
} else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
|
||||||
if (!ClInstrumentAtomics)
|
if (!ClInstrumentAtomics)
|
||||||
return None;
|
return std::nullopt;
|
||||||
Access.IsWrite = true;
|
Access.IsWrite = true;
|
||||||
Access.AccessTy = RMW->getValOperand()->getType();
|
Access.AccessTy = RMW->getValOperand()->getType();
|
||||||
Access.Addr = RMW->getPointerOperand();
|
Access.Addr = RMW->getPointerOperand();
|
||||||
} else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
|
} else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
|
||||||
if (!ClInstrumentAtomics)
|
if (!ClInstrumentAtomics)
|
||||||
return None;
|
return std::nullopt;
|
||||||
Access.IsWrite = true;
|
Access.IsWrite = true;
|
||||||
Access.AccessTy = XCHG->getCompareOperand()->getType();
|
Access.AccessTy = XCHG->getCompareOperand()->getType();
|
||||||
Access.Addr = XCHG->getPointerOperand();
|
Access.Addr = XCHG->getPointerOperand();
|
||||||
|
@ -306,14 +306,14 @@ MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
|
||||||
unsigned OpOffset = 0;
|
unsigned OpOffset = 0;
|
||||||
if (F->getIntrinsicID() == Intrinsic::masked_store) {
|
if (F->getIntrinsicID() == Intrinsic::masked_store) {
|
||||||
if (!ClInstrumentWrites)
|
if (!ClInstrumentWrites)
|
||||||
return None;
|
return std::nullopt;
|
||||||
// Masked store has an initial operand for the value.
|
// Masked store has an initial operand for the value.
|
||||||
OpOffset = 1;
|
OpOffset = 1;
|
||||||
Access.AccessTy = CI->getArgOperand(0)->getType();
|
Access.AccessTy = CI->getArgOperand(0)->getType();
|
||||||
Access.IsWrite = true;
|
Access.IsWrite = true;
|
||||||
} else {
|
} else {
|
||||||
if (!ClInstrumentReads)
|
if (!ClInstrumentReads)
|
||||||
return None;
|
return std::nullopt;
|
||||||
Access.AccessTy = CI->getType();
|
Access.AccessTy = CI->getType();
|
||||||
Access.IsWrite = false;
|
Access.IsWrite = false;
|
||||||
}
|
}
|
||||||
|
@ -325,20 +325,20 @@ MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!Access.Addr)
|
if (!Access.Addr)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Do not instrument accesses from different address spaces; we cannot deal
|
// Do not instrument accesses from different address spaces; we cannot deal
|
||||||
// with them.
|
// with them.
|
||||||
Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
|
Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
|
||||||
if (PtrTy->getPointerAddressSpace() != 0)
|
if (PtrTy->getPointerAddressSpace() != 0)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Ignore swifterror addresses.
|
// Ignore swifterror addresses.
|
||||||
// swifterror memory addresses are mem2reg promoted by instruction
|
// swifterror memory addresses are mem2reg promoted by instruction
|
||||||
// selection. As such they cannot have regular uses like an instrumentation
|
// selection. As such they cannot have regular uses like an instrumentation
|
||||||
// function and it makes no sense to track them as memory.
|
// function and it makes no sense to track them as memory.
|
||||||
if (Access.Addr->isSwiftError())
|
if (Access.Addr->isSwiftError())
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Peel off GEPs and BitCasts.
|
// Peel off GEPs and BitCasts.
|
||||||
auto *Addr = Access.Addr->stripInBoundsOffsets();
|
auto *Addr = Access.Addr->stripInBoundsOffsets();
|
||||||
|
@ -351,12 +351,12 @@ MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
|
||||||
auto OF = Triple(I->getModule()->getTargetTriple()).getObjectFormat();
|
auto OF = Triple(I->getModule()->getTargetTriple()).getObjectFormat();
|
||||||
if (SectionName.endswith(
|
if (SectionName.endswith(
|
||||||
getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
|
getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Do not instrument accesses to LLVM internal variables.
|
// Do not instrument accesses to LLVM internal variables.
|
||||||
if (GV->getName().startswith("__llvm"))
|
if (GV->getName().startswith("__llvm"))
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DataLayout &DL = I->getModule()->getDataLayout();
|
const DataLayout &DL = I->getModule()->getDataLayout();
|
||||||
|
|
|
@ -4128,7 +4128,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
|
||||||
if (ArgOffset + Size > kParamTLSSize)
|
if (ArgOffset + Size > kParamTLSSize)
|
||||||
break;
|
break;
|
||||||
const MaybeAlign ParamAlignment(CB.getParamAlign(i));
|
const MaybeAlign ParamAlignment(CB.getParamAlign(i));
|
||||||
MaybeAlign Alignment = llvm::None;
|
MaybeAlign Alignment = std::nullopt;
|
||||||
if (ParamAlignment)
|
if (ParamAlignment)
|
||||||
Alignment = std::min(*ParamAlignment, kShadowTLSAlignment);
|
Alignment = std::min(*ParamAlignment, kShadowTLSAlignment);
|
||||||
Value *AShadowPtr, *AOriginPtr;
|
Value *AShadowPtr, *AOriginPtr;
|
||||||
|
|
|
@ -251,7 +251,7 @@ private:
|
||||||
Type *Ty);
|
Type *Ty);
|
||||||
|
|
||||||
void SetNoSanitizeMetadata(Instruction *I) {
|
void SetNoSanitizeMetadata(Instruction *I) {
|
||||||
I->setMetadata(LLVMContext::MD_nosanitize, MDNode::get(*C, None));
|
I->setMetadata(LLVMContext::MD_nosanitize, MDNode::get(*C, std::nullopt));
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string getSectionName(const std::string &Section) const;
|
std::string getSectionName(const std::string &Section) const;
|
||||||
|
|
|
@ -472,7 +472,8 @@ bool ObjCARCContract::tryToPeepholeInstruction(
|
||||||
RVInstMarker->getString(),
|
RVInstMarker->getString(),
|
||||||
/*Constraints=*/"", /*hasSideEffects=*/true);
|
/*Constraints=*/"", /*hasSideEffects=*/true);
|
||||||
|
|
||||||
objcarc::createCallInstWithColors(IA, None, "", Inst, BlockColors);
|
objcarc::createCallInstWithColors(IA, std::nullopt, "", Inst,
|
||||||
|
BlockColors);
|
||||||
}
|
}
|
||||||
decline_rv_optimization:
|
decline_rv_optimization:
|
||||||
return false;
|
return false;
|
||||||
|
|
|
@ -999,7 +999,7 @@ void ObjCARCOpt::OptimizeIndividualCallImpl(
|
||||||
CallInst *NewCall =
|
CallInst *NewCall =
|
||||||
CallInst::Create(Decl, Call->getArgOperand(0), "", Call);
|
CallInst::Create(Decl, Call->getArgOperand(0), "", Call);
|
||||||
NewCall->setMetadata(MDKindCache.get(ARCMDKindID::ImpreciseRelease),
|
NewCall->setMetadata(MDKindCache.get(ARCMDKindID::ImpreciseRelease),
|
||||||
MDNode::get(C, None));
|
MDNode::get(C, std::nullopt));
|
||||||
|
|
||||||
LLVM_DEBUG(dbgs() << "Replacing autorelease{,RV}(x) with objc_release(x) "
|
LLVM_DEBUG(dbgs() << "Replacing autorelease{,RV}(x) with objc_release(x) "
|
||||||
"since x is otherwise unused.\nOld: "
|
"since x is otherwise unused.\nOld: "
|
||||||
|
|
|
@ -116,7 +116,7 @@ static MaybeAlign getNewAlignmentDiff(const SCEV *DiffSCEV,
|
||||||
return Align(DiffUnitsAbs);
|
return Align(DiffUnitsAbs);
|
||||||
}
|
}
|
||||||
|
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// There is an address given by an offset OffSCEV from AASCEV which has an
|
// There is an address given by an offset OffSCEV from AASCEV which has an
|
||||||
|
|
|
@ -497,8 +497,8 @@ static void shortenAssignment(Instruction *Inst, uint64_t OldOffsetInBits,
|
||||||
// FIXME: This should be using the DIExpression in the Alloca's dbg.assign
|
// FIXME: This should be using the DIExpression in the Alloca's dbg.assign
|
||||||
// for the variable, since that could also contain a fragment?
|
// for the variable, since that could also contain a fragment?
|
||||||
return *DIExpression::createFragmentExpression(
|
return *DIExpression::createFragmentExpression(
|
||||||
DIExpression::get(Inst->getContext(), None), DeadFragment.OffsetInBits,
|
DIExpression::get(Inst->getContext(), std::nullopt),
|
||||||
DeadFragment.SizeInBits);
|
DeadFragment.OffsetInBits, DeadFragment.SizeInBits);
|
||||||
};
|
};
|
||||||
|
|
||||||
// A DIAssignID to use so that the inserted dbg.assign intrinsics do not
|
// A DIAssignID to use so that the inserted dbg.assign intrinsics do not
|
||||||
|
@ -1303,7 +1303,7 @@ struct DSEState {
|
||||||
bool IsMemTerm, unsigned &PartialLimit) {
|
bool IsMemTerm, unsigned &PartialLimit) {
|
||||||
if (ScanLimit == 0 || WalkerStepLimit == 0) {
|
if (ScanLimit == 0 || WalkerStepLimit == 0) {
|
||||||
LLVM_DEBUG(dbgs() << "\n ... hit scan limit\n");
|
LLVM_DEBUG(dbgs() << "\n ... hit scan limit\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
MemoryAccess *Current = StartAccess;
|
MemoryAccess *Current = StartAccess;
|
||||||
|
@ -1336,7 +1336,7 @@ struct DSEState {
|
||||||
if (CanOptimize && Current != KillingDef->getDefiningAccess())
|
if (CanOptimize && Current != KillingDef->getDefiningAccess())
|
||||||
// The first clobbering def is... none.
|
// The first clobbering def is... none.
|
||||||
KillingDef->setOptimized(Current);
|
KillingDef->setOptimized(Current);
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cost of a step. Accesses in the same block are more likely to be valid
|
// Cost of a step. Accesses in the same block are more likely to be valid
|
||||||
|
@ -1346,7 +1346,7 @@ struct DSEState {
|
||||||
: MemorySSAOtherBBStepCost;
|
: MemorySSAOtherBBStepCost;
|
||||||
if (WalkerStepLimit <= StepCost) {
|
if (WalkerStepLimit <= StepCost) {
|
||||||
LLVM_DEBUG(dbgs() << " ... hit walker step limit\n");
|
LLVM_DEBUG(dbgs() << " ... hit walker step limit\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
WalkerStepLimit -= StepCost;
|
WalkerStepLimit -= StepCost;
|
||||||
|
|
||||||
|
@ -1371,14 +1371,14 @@ struct DSEState {
|
||||||
// instructions that block us from DSEing
|
// instructions that block us from DSEing
|
||||||
if (mayThrowBetween(KillingI, CurrentI, KillingUndObj)) {
|
if (mayThrowBetween(KillingI, CurrentI, KillingUndObj)) {
|
||||||
LLVM_DEBUG(dbgs() << " ... skip, may throw!\n");
|
LLVM_DEBUG(dbgs() << " ... skip, may throw!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for anything that looks like it will be a barrier to further
|
// Check for anything that looks like it will be a barrier to further
|
||||||
// removal
|
// removal
|
||||||
if (isDSEBarrier(KillingUndObj, CurrentI)) {
|
if (isDSEBarrier(KillingUndObj, CurrentI)) {
|
||||||
LLVM_DEBUG(dbgs() << " ... skip, barrier\n");
|
LLVM_DEBUG(dbgs() << " ... skip, barrier\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// If Current is known to be on path that reads DefLoc or is a read
|
// If Current is known to be on path that reads DefLoc or is a read
|
||||||
|
@ -1386,7 +1386,7 @@ struct DSEState {
|
||||||
// for intrinsic calls, because the code knows how to handle memcpy
|
// for intrinsic calls, because the code knows how to handle memcpy
|
||||||
// intrinsics.
|
// intrinsics.
|
||||||
if (!isa<IntrinsicInst>(CurrentI) && isReadClobber(KillingLoc, CurrentI))
|
if (!isa<IntrinsicInst>(CurrentI) && isReadClobber(KillingLoc, CurrentI))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Quick check if there are direct uses that are read-clobbers.
|
// Quick check if there are direct uses that are read-clobbers.
|
||||||
if (any_of(Current->uses(), [this, &KillingLoc, StartAccess](Use &U) {
|
if (any_of(Current->uses(), [this, &KillingLoc, StartAccess](Use &U) {
|
||||||
|
@ -1396,7 +1396,7 @@ struct DSEState {
|
||||||
return false;
|
return false;
|
||||||
})) {
|
})) {
|
||||||
LLVM_DEBUG(dbgs() << " ... found a read clobber\n");
|
LLVM_DEBUG(dbgs() << " ... found a read clobber\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// If Current does not have an analyzable write location or is not
|
// If Current does not have an analyzable write location or is not
|
||||||
|
@ -1490,7 +1490,7 @@ struct DSEState {
|
||||||
// Bail out if the number of accesses to check exceeds the scan limit.
|
// Bail out if the number of accesses to check exceeds the scan limit.
|
||||||
if (ScanLimit < (WorkList.size() - I)) {
|
if (ScanLimit < (WorkList.size() - I)) {
|
||||||
LLVM_DEBUG(dbgs() << "\n ... hit scan limit\n");
|
LLVM_DEBUG(dbgs() << "\n ... hit scan limit\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
--ScanLimit;
|
--ScanLimit;
|
||||||
NumDomMemDefChecks++;
|
NumDomMemDefChecks++;
|
||||||
|
@ -1535,14 +1535,14 @@ struct DSEState {
|
||||||
|
|
||||||
if (UseInst->mayThrow() && !isInvisibleToCallerOnUnwind(KillingUndObj)) {
|
if (UseInst->mayThrow() && !isInvisibleToCallerOnUnwind(KillingUndObj)) {
|
||||||
LLVM_DEBUG(dbgs() << " ... found throwing instruction\n");
|
LLVM_DEBUG(dbgs() << " ... found throwing instruction\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Uses which may read the original MemoryDef mean we cannot eliminate the
|
// Uses which may read the original MemoryDef mean we cannot eliminate the
|
||||||
// original MD. Stop walk.
|
// original MD. Stop walk.
|
||||||
if (isReadClobber(MaybeDeadLoc, UseInst)) {
|
if (isReadClobber(MaybeDeadLoc, UseInst)) {
|
||||||
LLVM_DEBUG(dbgs() << " ... found read clobber\n");
|
LLVM_DEBUG(dbgs() << " ... found read clobber\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// If this worklist walks back to the original memory access (and the
|
// If this worklist walks back to the original memory access (and the
|
||||||
|
@ -1551,7 +1551,7 @@ struct DSEState {
|
||||||
if (MaybeDeadAccess == UseAccess &&
|
if (MaybeDeadAccess == UseAccess &&
|
||||||
!isGuaranteedLoopInvariant(MaybeDeadLoc.Ptr)) {
|
!isGuaranteedLoopInvariant(MaybeDeadLoc.Ptr)) {
|
||||||
LLVM_DEBUG(dbgs() << " ... found not loop invariant self access\n");
|
LLVM_DEBUG(dbgs() << " ... found not loop invariant self access\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
// Otherwise, for the KillingDef and MaybeDeadAccess we only have to check
|
// Otherwise, for the KillingDef and MaybeDeadAccess we only have to check
|
||||||
// if it reads the memory location.
|
// if it reads the memory location.
|
||||||
|
@ -1585,7 +1585,7 @@ struct DSEState {
|
||||||
} else {
|
} else {
|
||||||
LLVM_DEBUG(dbgs()
|
LLVM_DEBUG(dbgs()
|
||||||
<< " ... found preceeding def " << *UseInst << "\n");
|
<< " ... found preceeding def " << *UseInst << "\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
} else
|
} else
|
||||||
PushMemUses(UseDef);
|
PushMemUses(UseDef);
|
||||||
|
@ -1615,7 +1615,7 @@ struct DSEState {
|
||||||
// killing block.
|
// killing block.
|
||||||
if (!PDT.dominates(CommonPred, MaybeDeadAccess->getBlock())) {
|
if (!PDT.dominates(CommonPred, MaybeDeadAccess->getBlock())) {
|
||||||
if (!AnyUnreachableExit)
|
if (!AnyUnreachableExit)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Fall back to CFG scan starting at all non-unreachable roots if not
|
// Fall back to CFG scan starting at all non-unreachable roots if not
|
||||||
// all paths to the exit go through CommonPred.
|
// all paths to the exit go through CommonPred.
|
||||||
|
@ -1646,7 +1646,7 @@ struct DSEState {
|
||||||
if (KillingBlocks.count(Current))
|
if (KillingBlocks.count(Current))
|
||||||
continue;
|
continue;
|
||||||
if (Current == MaybeDeadAccess->getBlock())
|
if (Current == MaybeDeadAccess->getBlock())
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// MaybeDeadAccess is reachable from the entry, so we don't have to
|
// MaybeDeadAccess is reachable from the entry, so we don't have to
|
||||||
// explore unreachable blocks further.
|
// explore unreachable blocks further.
|
||||||
|
@ -1657,7 +1657,7 @@ struct DSEState {
|
||||||
WorkList.insert(Pred);
|
WorkList.insert(Pred);
|
||||||
|
|
||||||
if (WorkList.size() >= MemorySSAPathCheckLimit)
|
if (WorkList.size() >= MemorySSAPathCheckLimit)
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
NumCFGSuccess++;
|
NumCFGSuccess++;
|
||||||
}
|
}
|
||||||
|
|
|
@ -243,7 +243,7 @@ Optional<ConstantRange> Float2IntPass::calcRange(Instruction *I) {
|
||||||
auto OpIt = SeenInsts.find(OI);
|
auto OpIt = SeenInsts.find(OI);
|
||||||
assert(OpIt != SeenInsts.end() && "def not seen before use!");
|
assert(OpIt != SeenInsts.end() && "def not seen before use!");
|
||||||
if (OpIt->second == unknownRange())
|
if (OpIt->second == unknownRange())
|
||||||
return None; // Wait until operand range has been calculated.
|
return std::nullopt; // Wait until operand range has been calculated.
|
||||||
OpRanges.push_back(OpIt->second);
|
OpRanges.push_back(OpIt->second);
|
||||||
} else if (ConstantFP *CF = dyn_cast<ConstantFP>(O)) {
|
} else if (ConstantFP *CF = dyn_cast<ConstantFP>(O)) {
|
||||||
// Work out if the floating point number can be losslessly represented
|
// Work out if the floating point number can be losslessly represented
|
||||||
|
|
|
@ -763,14 +763,14 @@ void GVNPass::printPipeline(
|
||||||
OS, MapClassName2PassName);
|
OS, MapClassName2PassName);
|
||||||
|
|
||||||
OS << "<";
|
OS << "<";
|
||||||
if (Options.AllowPRE != None)
|
if (Options.AllowPRE != std::nullopt)
|
||||||
OS << (Options.AllowPRE.value() ? "" : "no-") << "pre;";
|
OS << (Options.AllowPRE.value() ? "" : "no-") << "pre;";
|
||||||
if (Options.AllowLoadPRE != None)
|
if (Options.AllowLoadPRE != std::nullopt)
|
||||||
OS << (Options.AllowLoadPRE.value() ? "" : "no-") << "load-pre;";
|
OS << (Options.AllowLoadPRE.value() ? "" : "no-") << "load-pre;";
|
||||||
if (Options.AllowLoadPRESplitBackedge != None)
|
if (Options.AllowLoadPRESplitBackedge != std::nullopt)
|
||||||
OS << (Options.AllowLoadPRESplitBackedge.value() ? "" : "no-")
|
OS << (Options.AllowLoadPRESplitBackedge.value() ? "" : "no-")
|
||||||
<< "split-backedge-load-pre;";
|
<< "split-backedge-load-pre;";
|
||||||
if (Options.AllowMemDep != None)
|
if (Options.AllowMemDep != std::nullopt)
|
||||||
OS << (Options.AllowMemDep.value() ? "" : "no-") << "memdep";
|
OS << (Options.AllowMemDep.value() ? "" : "no-") << "memdep";
|
||||||
OS << ">";
|
OS << ">";
|
||||||
}
|
}
|
||||||
|
@ -1129,12 +1129,12 @@ tryToConvertLoadOfPtrSelect(BasicBlock *DepBB, BasicBlock::iterator End,
|
||||||
|
|
||||||
auto *Sel = dyn_cast_or_null<SelectInst>(Address);
|
auto *Sel = dyn_cast_or_null<SelectInst>(Address);
|
||||||
if (!Sel || DepBB != Sel->getParent())
|
if (!Sel || DepBB != Sel->getParent())
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
LoadInst *L1 = findDominatingLoad(Sel->getOperand(1), LoadTy, Sel, DT);
|
LoadInst *L1 = findDominatingLoad(Sel->getOperand(1), LoadTy, Sel, DT);
|
||||||
LoadInst *L2 = findDominatingLoad(Sel->getOperand(2), LoadTy, Sel, DT);
|
LoadInst *L2 = findDominatingLoad(Sel->getOperand(2), LoadTy, Sel, DT);
|
||||||
if (!L1 || !L2)
|
if (!L1 || !L2)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Ensure there are no accesses that may modify the locations referenced by
|
// Ensure there are no accesses that may modify the locations referenced by
|
||||||
// either L1 or L2 between L1, L2 and the specified End iterator.
|
// either L1 or L2 between L1, L2 and the specified End iterator.
|
||||||
|
@ -1145,7 +1145,7 @@ tryToConvertLoadOfPtrSelect(BasicBlock *DepBB, BasicBlock::iterator End,
|
||||||
return isModSet(AA->getModRefInfo(&I, L1Loc)) ||
|
return isModSet(AA->getModRefInfo(&I, L1Loc)) ||
|
||||||
isModSet(AA->getModRefInfo(&I, L2Loc));
|
isModSet(AA->getModRefInfo(&I, L2Loc));
|
||||||
}))
|
}))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
return AvailableValue::getSelect(Sel);
|
return AvailableValue::getSelect(Sel);
|
||||||
}
|
}
|
||||||
|
@ -1204,7 +1204,9 @@ bool GVNPass::AnalyzeLoadAvailability(LoadInst *Load, MemDepResult DepInfo,
|
||||||
canCoerceMustAliasedValueToLoad(DepLoad, LoadType, DL)) {
|
canCoerceMustAliasedValueToLoad(DepLoad, LoadType, DL)) {
|
||||||
const auto ClobberOff = MD->getClobberOffset(DepLoad);
|
const auto ClobberOff = MD->getClobberOffset(DepLoad);
|
||||||
// GVN has no deal with a negative offset.
|
// GVN has no deal with a negative offset.
|
||||||
Offset = (ClobberOff == None || *ClobberOff < 0) ? -1 : *ClobberOff;
|
Offset = (ClobberOff == std::nullopt || *ClobberOff < 0)
|
||||||
|
? -1
|
||||||
|
: *ClobberOff;
|
||||||
}
|
}
|
||||||
if (Offset == -1)
|
if (Offset == -1)
|
||||||
Offset =
|
Offset =
|
||||||
|
|
|
@ -654,7 +654,7 @@ Optional<SinkingInstructionCandidate> GVNSink::analyzeInstructionForSinking(
|
||||||
uint32_t N = VN.lookupOrAdd(I);
|
uint32_t N = VN.lookupOrAdd(I);
|
||||||
LLVM_DEBUG(dbgs() << " VN=" << Twine::utohexstr(N) << " for" << *I << "\n");
|
LLVM_DEBUG(dbgs() << " VN=" << Twine::utohexstr(N) << " for" << *I << "\n");
|
||||||
if (N == ~0U)
|
if (N == ~0U)
|
||||||
return None;
|
return std::nullopt;
|
||||||
VNums[N]++;
|
VNums[N]++;
|
||||||
}
|
}
|
||||||
unsigned VNumToSink =
|
unsigned VNumToSink =
|
||||||
|
@ -662,7 +662,7 @@ Optional<SinkingInstructionCandidate> GVNSink::analyzeInstructionForSinking(
|
||||||
|
|
||||||
if (VNums[VNumToSink] == 1)
|
if (VNums[VNumToSink] == 1)
|
||||||
// Can't sink anything!
|
// Can't sink anything!
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Now restrict the number of incoming blocks down to only those with
|
// Now restrict the number of incoming blocks down to only those with
|
||||||
// VNumToSink.
|
// VNumToSink.
|
||||||
|
@ -677,7 +677,7 @@ Optional<SinkingInstructionCandidate> GVNSink::analyzeInstructionForSinking(
|
||||||
}
|
}
|
||||||
for (auto *I : NewInsts)
|
for (auto *I : NewInsts)
|
||||||
if (shouldAvoidSinkingInstruction(I))
|
if (shouldAvoidSinkingInstruction(I))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// If we've restricted the incoming blocks, restrict all needed PHIs also
|
// If we've restricted the incoming blocks, restrict all needed PHIs also
|
||||||
// to that set.
|
// to that set.
|
||||||
|
@ -715,7 +715,7 @@ Optional<SinkingInstructionCandidate> GVNSink::analyzeInstructionForSinking(
|
||||||
// V exists in this PHI, but the whole PHI is different to NewPHI
|
// V exists in this PHI, but the whole PHI is different to NewPHI
|
||||||
// (else it would have been removed earlier). We cannot continue
|
// (else it would have been removed earlier). We cannot continue
|
||||||
// because this isn't representable.
|
// because this isn't representable.
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Which operands need PHIs?
|
// Which operands need PHIs?
|
||||||
// FIXME: If any of these fail, we should partition up the candidates to
|
// FIXME: If any of these fail, we should partition up the candidates to
|
||||||
|
@ -728,7 +728,7 @@ Optional<SinkingInstructionCandidate> GVNSink::analyzeInstructionForSinking(
|
||||||
return I->getNumOperands() != I0->getNumOperands();
|
return I->getNumOperands() != I0->getNumOperands();
|
||||||
};
|
};
|
||||||
if (any_of(NewInsts, hasDifferentNumOperands))
|
if (any_of(NewInsts, hasDifferentNumOperands))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
for (unsigned OpNum = 0, E = I0->getNumOperands(); OpNum != E; ++OpNum) {
|
for (unsigned OpNum = 0, E = I0->getNumOperands(); OpNum != E; ++OpNum) {
|
||||||
ModelledPHI PHI(NewInsts, OpNum, ActivePreds);
|
ModelledPHI PHI(NewInsts, OpNum, ActivePreds);
|
||||||
|
@ -736,15 +736,15 @@ Optional<SinkingInstructionCandidate> GVNSink::analyzeInstructionForSinking(
|
||||||
continue;
|
continue;
|
||||||
if (!canReplaceOperandWithVariable(I0, OpNum))
|
if (!canReplaceOperandWithVariable(I0, OpNum))
|
||||||
// We can 't create a PHI from this instruction!
|
// We can 't create a PHI from this instruction!
|
||||||
return None;
|
return std::nullopt;
|
||||||
if (NeededPHIs.count(PHI))
|
if (NeededPHIs.count(PHI))
|
||||||
continue;
|
continue;
|
||||||
if (!PHI.areAllIncomingValuesSameType())
|
if (!PHI.areAllIncomingValuesSameType())
|
||||||
return None;
|
return std::nullopt;
|
||||||
// Don't create indirect calls! The called value is the final operand.
|
// Don't create indirect calls! The called value is the final operand.
|
||||||
if ((isa<CallInst>(I0) || isa<InvokeInst>(I0)) && OpNum == E - 1 &&
|
if ((isa<CallInst>(I0) || isa<InvokeInst>(I0)) && OpNum == E - 1 &&
|
||||||
PHI.areAnyIncomingValuesConstant())
|
PHI.areAnyIncomingValuesConstant())
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
NeededPHIs.reserve(NeededPHIs.size());
|
NeededPHIs.reserve(NeededPHIs.size());
|
||||||
NeededPHIs.insert(PHI);
|
NeededPHIs.insert(PHI);
|
||||||
|
|
|
@ -246,7 +246,7 @@ class InductiveRangeCheckElimination {
|
||||||
public:
|
public:
|
||||||
InductiveRangeCheckElimination(ScalarEvolution &SE,
|
InductiveRangeCheckElimination(ScalarEvolution &SE,
|
||||||
BranchProbabilityInfo *BPI, DominatorTree &DT,
|
BranchProbabilityInfo *BPI, DominatorTree &DT,
|
||||||
LoopInfo &LI, GetBFIFunc GetBFI = None)
|
LoopInfo &LI, GetBFIFunc GetBFI = std::nullopt)
|
||||||
: SE(SE), BPI(BPI), DT(DT), LI(LI), GetBFI(GetBFI) {}
|
: SE(SE), BPI(BPI), DT(DT), LI(LI), GetBFI(GetBFI) {}
|
||||||
|
|
||||||
bool run(Loop *L, function_ref<void(Loop *, bool)> LPMAddNewLoop);
|
bool run(Loop *L, function_ref<void(Loop *, bool)> LPMAddNewLoop);
|
||||||
|
@ -753,7 +753,7 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
|
||||||
const char *&FailureReason) {
|
const char *&FailureReason) {
|
||||||
if (!L.isLoopSimplifyForm()) {
|
if (!L.isLoopSimplifyForm()) {
|
||||||
FailureReason = "loop not in LoopSimplify form";
|
FailureReason = "loop not in LoopSimplify form";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
BasicBlock *Latch = L.getLoopLatch();
|
BasicBlock *Latch = L.getLoopLatch();
|
||||||
|
@ -761,25 +761,25 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
|
||||||
|
|
||||||
if (Latch->getTerminator()->getMetadata(ClonedLoopTag)) {
|
if (Latch->getTerminator()->getMetadata(ClonedLoopTag)) {
|
||||||
FailureReason = "loop has already been cloned";
|
FailureReason = "loop has already been cloned";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!L.isLoopExiting(Latch)) {
|
if (!L.isLoopExiting(Latch)) {
|
||||||
FailureReason = "no loop latch";
|
FailureReason = "no loop latch";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
BasicBlock *Header = L.getHeader();
|
BasicBlock *Header = L.getHeader();
|
||||||
BasicBlock *Preheader = L.getLoopPreheader();
|
BasicBlock *Preheader = L.getLoopPreheader();
|
||||||
if (!Preheader) {
|
if (!Preheader) {
|
||||||
FailureReason = "no preheader";
|
FailureReason = "no preheader";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
BranchInst *LatchBr = dyn_cast<BranchInst>(Latch->getTerminator());
|
BranchInst *LatchBr = dyn_cast<BranchInst>(Latch->getTerminator());
|
||||||
if (!LatchBr || LatchBr->isUnconditional()) {
|
if (!LatchBr || LatchBr->isUnconditional()) {
|
||||||
FailureReason = "latch terminator not conditional branch";
|
FailureReason = "latch terminator not conditional branch";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
unsigned LatchBrExitIdx = LatchBr->getSuccessor(0) == Header ? 1 : 0;
|
unsigned LatchBrExitIdx = LatchBr->getSuccessor(0) == Header ? 1 : 0;
|
||||||
|
@ -787,13 +787,13 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
|
||||||
ICmpInst *ICI = dyn_cast<ICmpInst>(LatchBr->getCondition());
|
ICmpInst *ICI = dyn_cast<ICmpInst>(LatchBr->getCondition());
|
||||||
if (!ICI || !isa<IntegerType>(ICI->getOperand(0)->getType())) {
|
if (!ICI || !isa<IntegerType>(ICI->getOperand(0)->getType())) {
|
||||||
FailureReason = "latch terminator branch not conditional on integral icmp";
|
FailureReason = "latch terminator branch not conditional on integral icmp";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
const SCEV *LatchCount = SE.getExitCount(&L, Latch);
|
const SCEV *LatchCount = SE.getExitCount(&L, Latch);
|
||||||
if (isa<SCEVCouldNotCompute>(LatchCount)) {
|
if (isa<SCEVCouldNotCompute>(LatchCount)) {
|
||||||
FailureReason = "could not compute latch count";
|
FailureReason = "could not compute latch count";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
ICmpInst::Predicate Pred = ICI->getPredicate();
|
ICmpInst::Predicate Pred = ICI->getPredicate();
|
||||||
|
@ -812,7 +812,7 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
|
||||||
Pred = ICmpInst::getSwappedPredicate(Pred);
|
Pred = ICmpInst::getSwappedPredicate(Pred);
|
||||||
} else {
|
} else {
|
||||||
FailureReason = "no add recurrences in the icmp";
|
FailureReason = "no add recurrences in the icmp";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -848,22 +848,22 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
|
||||||
const SCEVAddRecExpr *IndVarBase = cast<SCEVAddRecExpr>(LeftSCEV);
|
const SCEVAddRecExpr *IndVarBase = cast<SCEVAddRecExpr>(LeftSCEV);
|
||||||
if (IndVarBase->getLoop() != &L) {
|
if (IndVarBase->getLoop() != &L) {
|
||||||
FailureReason = "LHS in cmp is not an AddRec for this loop";
|
FailureReason = "LHS in cmp is not an AddRec for this loop";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
if (!IndVarBase->isAffine()) {
|
if (!IndVarBase->isAffine()) {
|
||||||
FailureReason = "LHS in icmp not induction variable";
|
FailureReason = "LHS in icmp not induction variable";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
const SCEV* StepRec = IndVarBase->getStepRecurrence(SE);
|
const SCEV* StepRec = IndVarBase->getStepRecurrence(SE);
|
||||||
if (!isa<SCEVConstant>(StepRec)) {
|
if (!isa<SCEVConstant>(StepRec)) {
|
||||||
FailureReason = "LHS in icmp not induction variable";
|
FailureReason = "LHS in icmp not induction variable";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
ConstantInt *StepCI = cast<SCEVConstant>(StepRec)->getValue();
|
ConstantInt *StepCI = cast<SCEVConstant>(StepRec)->getValue();
|
||||||
|
|
||||||
if (ICI->isEquality() && !HasNoSignedWrap(IndVarBase)) {
|
if (ICI->isEquality() && !HasNoSignedWrap(IndVarBase)) {
|
||||||
FailureReason = "LHS in icmp needs nsw for equality predicates";
|
FailureReason = "LHS in icmp needs nsw for equality predicates";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
assert(!StepCI->isZero() && "Zero step?");
|
assert(!StepCI->isZero() && "Zero step?");
|
||||||
|
@ -926,19 +926,19 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
|
||||||
|
|
||||||
if (!FoundExpectedPred) {
|
if (!FoundExpectedPred) {
|
||||||
FailureReason = "expected icmp slt semantically, found something else";
|
FailureReason = "expected icmp slt semantically, found something else";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
IsSignedPredicate = ICmpInst::isSigned(Pred);
|
IsSignedPredicate = ICmpInst::isSigned(Pred);
|
||||||
if (!IsSignedPredicate && !AllowUnsignedLatchCondition) {
|
if (!IsSignedPredicate && !AllowUnsignedLatchCondition) {
|
||||||
FailureReason = "unsigned latch conditions are explicitly prohibited";
|
FailureReason = "unsigned latch conditions are explicitly prohibited";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isSafeIncreasingBound(IndVarStart, RightSCEV, Step, Pred,
|
if (!isSafeIncreasingBound(IndVarStart, RightSCEV, Step, Pred,
|
||||||
LatchBrExitIdx, &L, SE)) {
|
LatchBrExitIdx, &L, SE)) {
|
||||||
FailureReason = "Unsafe loop bounds";
|
FailureReason = "Unsafe loop bounds";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
if (LatchBrExitIdx == 0) {
|
if (LatchBrExitIdx == 0) {
|
||||||
// We need to increase the right value unless we have already decreased
|
// We need to increase the right value unless we have already decreased
|
||||||
|
@ -989,7 +989,7 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
|
||||||
|
|
||||||
if (!FoundExpectedPred) {
|
if (!FoundExpectedPred) {
|
||||||
FailureReason = "expected icmp sgt semantically, found something else";
|
FailureReason = "expected icmp sgt semantically, found something else";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
IsSignedPredicate =
|
IsSignedPredicate =
|
||||||
|
@ -997,13 +997,13 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
|
||||||
|
|
||||||
if (!IsSignedPredicate && !AllowUnsignedLatchCondition) {
|
if (!IsSignedPredicate && !AllowUnsignedLatchCondition) {
|
||||||
FailureReason = "unsigned latch conditions are explicitly prohibited";
|
FailureReason = "unsigned latch conditions are explicitly prohibited";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isSafeDecreasingBound(IndVarStart, RightSCEV, Step, Pred,
|
if (!isSafeDecreasingBound(IndVarStart, RightSCEV, Step, Pred,
|
||||||
LatchBrExitIdx, &L, SE)) {
|
LatchBrExitIdx, &L, SE)) {
|
||||||
FailureReason = "Unsafe bounds";
|
FailureReason = "Unsafe bounds";
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (LatchBrExitIdx == 0) {
|
if (LatchBrExitIdx == 0) {
|
||||||
|
@ -1070,9 +1070,9 @@ LoopConstrainer::calculateSubRanges(bool IsSignedPredicate) const {
|
||||||
|
|
||||||
// We only support wide range checks and narrow latches.
|
// We only support wide range checks and narrow latches.
|
||||||
if (!AllowNarrowLatchCondition && RTy != Ty)
|
if (!AllowNarrowLatchCondition && RTy != Ty)
|
||||||
return None;
|
return std::nullopt;
|
||||||
if (RTy->getBitWidth() < Ty->getBitWidth())
|
if (RTy->getBitWidth() < Ty->getBitWidth())
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
LoopConstrainer::SubRanges Result;
|
LoopConstrainer::SubRanges Result;
|
||||||
|
|
||||||
|
@ -1592,9 +1592,9 @@ InductiveRangeCheck::computeSafeIterationSpace(
|
||||||
auto *RCType = dyn_cast<IntegerType>(getBegin()->getType());
|
auto *RCType = dyn_cast<IntegerType>(getBegin()->getType());
|
||||||
// Do not work with pointer types.
|
// Do not work with pointer types.
|
||||||
if (!IVType || !RCType)
|
if (!IVType || !RCType)
|
||||||
return None;
|
return std::nullopt;
|
||||||
if (IVType->getBitWidth() > RCType->getBitWidth())
|
if (IVType->getBitWidth() > RCType->getBitWidth())
|
||||||
return None;
|
return std::nullopt;
|
||||||
// IndVar is of the form "A + B * I" (where "I" is the canonical induction
|
// IndVar is of the form "A + B * I" (where "I" is the canonical induction
|
||||||
// variable, that may or may not exist as a real llvm::Value in the loop) and
|
// variable, that may or may not exist as a real llvm::Value in the loop) and
|
||||||
// this inductive range check is a range check on the "C + D * I" ("C" is
|
// this inductive range check is a range check on the "C + D * I" ("C" is
|
||||||
|
@ -1616,19 +1616,19 @@ InductiveRangeCheck::computeSafeIterationSpace(
|
||||||
// to deal with overflown values.
|
// to deal with overflown values.
|
||||||
|
|
||||||
if (!IndVar->isAffine())
|
if (!IndVar->isAffine())
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
const SCEV *A = NoopOrExtend(IndVar->getStart(), RCType, SE, IsLatchSigned);
|
const SCEV *A = NoopOrExtend(IndVar->getStart(), RCType, SE, IsLatchSigned);
|
||||||
const SCEVConstant *B = dyn_cast<SCEVConstant>(
|
const SCEVConstant *B = dyn_cast<SCEVConstant>(
|
||||||
NoopOrExtend(IndVar->getStepRecurrence(SE), RCType, SE, IsLatchSigned));
|
NoopOrExtend(IndVar->getStepRecurrence(SE), RCType, SE, IsLatchSigned));
|
||||||
if (!B)
|
if (!B)
|
||||||
return None;
|
return std::nullopt;
|
||||||
assert(!B->isZero() && "Recurrence with zero step?");
|
assert(!B->isZero() && "Recurrence with zero step?");
|
||||||
|
|
||||||
const SCEV *C = getBegin();
|
const SCEV *C = getBegin();
|
||||||
const SCEVConstant *D = dyn_cast<SCEVConstant>(getStep());
|
const SCEVConstant *D = dyn_cast<SCEVConstant>(getStep());
|
||||||
if (D != B)
|
if (D != B)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
assert(!D->getValue()->isZero() && "Recurrence with zero step?");
|
assert(!D->getValue()->isZero() && "Recurrence with zero step?");
|
||||||
unsigned BitWidth = RCType->getBitWidth();
|
unsigned BitWidth = RCType->getBitWidth();
|
||||||
|
@ -1716,7 +1716,7 @@ IntersectSignedRange(ScalarEvolution &SE,
|
||||||
const Optional<InductiveRangeCheck::Range> &R1,
|
const Optional<InductiveRangeCheck::Range> &R1,
|
||||||
const InductiveRangeCheck::Range &R2) {
|
const InductiveRangeCheck::Range &R2) {
|
||||||
if (R2.isEmpty(SE, /* IsSigned */ true))
|
if (R2.isEmpty(SE, /* IsSigned */ true))
|
||||||
return None;
|
return std::nullopt;
|
||||||
if (!R1)
|
if (!R1)
|
||||||
return R2;
|
return R2;
|
||||||
auto &R1Value = R1.value();
|
auto &R1Value = R1.value();
|
||||||
|
@ -1728,7 +1728,7 @@ IntersectSignedRange(ScalarEvolution &SE,
|
||||||
// TODO: we could widen the smaller range and have this work; but for now we
|
// TODO: we could widen the smaller range and have this work; but for now we
|
||||||
// bail out to keep things simple.
|
// bail out to keep things simple.
|
||||||
if (R1Value.getType() != R2.getType())
|
if (R1Value.getType() != R2.getType())
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
const SCEV *NewBegin = SE.getSMaxExpr(R1Value.getBegin(), R2.getBegin());
|
const SCEV *NewBegin = SE.getSMaxExpr(R1Value.getBegin(), R2.getBegin());
|
||||||
const SCEV *NewEnd = SE.getSMinExpr(R1Value.getEnd(), R2.getEnd());
|
const SCEV *NewEnd = SE.getSMinExpr(R1Value.getEnd(), R2.getEnd());
|
||||||
|
@ -1736,7 +1736,7 @@ IntersectSignedRange(ScalarEvolution &SE,
|
||||||
// If the resulting range is empty, just return None.
|
// If the resulting range is empty, just return None.
|
||||||
auto Ret = InductiveRangeCheck::Range(NewBegin, NewEnd);
|
auto Ret = InductiveRangeCheck::Range(NewBegin, NewEnd);
|
||||||
if (Ret.isEmpty(SE, /* IsSigned */ true))
|
if (Ret.isEmpty(SE, /* IsSigned */ true))
|
||||||
return None;
|
return std::nullopt;
|
||||||
return Ret;
|
return Ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1745,7 +1745,7 @@ IntersectUnsignedRange(ScalarEvolution &SE,
|
||||||
const Optional<InductiveRangeCheck::Range> &R1,
|
const Optional<InductiveRangeCheck::Range> &R1,
|
||||||
const InductiveRangeCheck::Range &R2) {
|
const InductiveRangeCheck::Range &R2) {
|
||||||
if (R2.isEmpty(SE, /* IsSigned */ false))
|
if (R2.isEmpty(SE, /* IsSigned */ false))
|
||||||
return None;
|
return std::nullopt;
|
||||||
if (!R1)
|
if (!R1)
|
||||||
return R2;
|
return R2;
|
||||||
auto &R1Value = R1.value();
|
auto &R1Value = R1.value();
|
||||||
|
@ -1757,7 +1757,7 @@ IntersectUnsignedRange(ScalarEvolution &SE,
|
||||||
// TODO: we could widen the smaller range and have this work; but for now we
|
// TODO: we could widen the smaller range and have this work; but for now we
|
||||||
// bail out to keep things simple.
|
// bail out to keep things simple.
|
||||||
if (R1Value.getType() != R2.getType())
|
if (R1Value.getType() != R2.getType())
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
const SCEV *NewBegin = SE.getUMaxExpr(R1Value.getBegin(), R2.getBegin());
|
const SCEV *NewBegin = SE.getUMaxExpr(R1Value.getBegin(), R2.getBegin());
|
||||||
const SCEV *NewEnd = SE.getUMinExpr(R1Value.getEnd(), R2.getEnd());
|
const SCEV *NewEnd = SE.getUMinExpr(R1Value.getEnd(), R2.getEnd());
|
||||||
|
@ -1765,7 +1765,7 @@ IntersectUnsignedRange(ScalarEvolution &SE,
|
||||||
// If the resulting range is empty, just return None.
|
// If the resulting range is empty, just return None.
|
||||||
auto Ret = InductiveRangeCheck::Range(NewBegin, NewEnd);
|
auto Ret = InductiveRangeCheck::Range(NewBegin, NewEnd);
|
||||||
if (Ret.isEmpty(SE, /* IsSigned */ false))
|
if (Ret.isEmpty(SE, /* IsSigned */ false))
|
||||||
return None;
|
return std::nullopt;
|
||||||
return Ret;
|
return Ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -645,7 +645,7 @@ private:
|
||||||
void collectFusionCandidates(const LoopVector &LV) {
|
void collectFusionCandidates(const LoopVector &LV) {
|
||||||
for (Loop *L : LV) {
|
for (Loop *L : LV) {
|
||||||
TTI::PeelingPreferences PP =
|
TTI::PeelingPreferences PP =
|
||||||
gatherPeelingPreferences(L, SE, TTI, None, None);
|
gatherPeelingPreferences(L, SE, TTI, std::nullopt, std::nullopt);
|
||||||
FusionCandidate CurrCand(L, DT, &PDT, ORE, PP);
|
FusionCandidate CurrCand(L, DT, &PDT, ORE, PP);
|
||||||
if (!CurrCand.isEligibleForFusion(SE))
|
if (!CurrCand.isEligibleForFusion(SE))
|
||||||
continue;
|
continue;
|
||||||
|
@ -708,14 +708,14 @@ private:
|
||||||
if (isa<SCEVCouldNotCompute>(TripCount0)) {
|
if (isa<SCEVCouldNotCompute>(TripCount0)) {
|
||||||
UncomputableTripCount++;
|
UncomputableTripCount++;
|
||||||
LLVM_DEBUG(dbgs() << "Trip count of first loop could not be computed!");
|
LLVM_DEBUG(dbgs() << "Trip count of first loop could not be computed!");
|
||||||
return {false, None};
|
return {false, std::nullopt};
|
||||||
}
|
}
|
||||||
|
|
||||||
const SCEV *TripCount1 = SE.getBackedgeTakenCount(FC1.L);
|
const SCEV *TripCount1 = SE.getBackedgeTakenCount(FC1.L);
|
||||||
if (isa<SCEVCouldNotCompute>(TripCount1)) {
|
if (isa<SCEVCouldNotCompute>(TripCount1)) {
|
||||||
UncomputableTripCount++;
|
UncomputableTripCount++;
|
||||||
LLVM_DEBUG(dbgs() << "Trip count of second loop could not be computed!");
|
LLVM_DEBUG(dbgs() << "Trip count of second loop could not be computed!");
|
||||||
return {false, None};
|
return {false, std::nullopt};
|
||||||
}
|
}
|
||||||
|
|
||||||
LLVM_DEBUG(dbgs() << "\tTrip counts: " << *TripCount0 << " & "
|
LLVM_DEBUG(dbgs() << "\tTrip counts: " << *TripCount0 << " & "
|
||||||
|
@ -740,7 +740,7 @@ private:
|
||||||
LLVM_DEBUG(dbgs() << "Loop(s) do not have a single exit point or do not "
|
LLVM_DEBUG(dbgs() << "Loop(s) do not have a single exit point or do not "
|
||||||
"have a constant number of iterations. Peeling "
|
"have a constant number of iterations. Peeling "
|
||||||
"is not benefical\n");
|
"is not benefical\n");
|
||||||
return {false, None};
|
return {false, std::nullopt};
|
||||||
}
|
}
|
||||||
|
|
||||||
Optional<unsigned> Difference;
|
Optional<unsigned> Difference;
|
||||||
|
|
|
@ -392,10 +392,10 @@ LoopPredication::parseLoopICmp(ICmpInst *ICI) {
|
||||||
|
|
||||||
const SCEV *LHSS = SE->getSCEV(LHS);
|
const SCEV *LHSS = SE->getSCEV(LHS);
|
||||||
if (isa<SCEVCouldNotCompute>(LHSS))
|
if (isa<SCEVCouldNotCompute>(LHSS))
|
||||||
return None;
|
return std::nullopt;
|
||||||
const SCEV *RHSS = SE->getSCEV(RHS);
|
const SCEV *RHSS = SE->getSCEV(RHS);
|
||||||
if (isa<SCEVCouldNotCompute>(RHSS))
|
if (isa<SCEVCouldNotCompute>(RHSS))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Canonicalize RHS to be loop invariant bound, LHS - a loop computable IV
|
// Canonicalize RHS to be loop invariant bound, LHS - a loop computable IV
|
||||||
if (SE->isLoopInvariant(LHSS, L)) {
|
if (SE->isLoopInvariant(LHSS, L)) {
|
||||||
|
@ -406,7 +406,7 @@ LoopPredication::parseLoopICmp(ICmpInst *ICI) {
|
||||||
|
|
||||||
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHSS);
|
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHSS);
|
||||||
if (!AR || AR->getLoop() != L)
|
if (!AR || AR->getLoop() != L)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
return LoopICmp(Pred, AR, RHSS);
|
return LoopICmp(Pred, AR, RHSS);
|
||||||
}
|
}
|
||||||
|
@ -494,9 +494,9 @@ static std::optional<LoopICmp> generateLoopLatchCheck(const DataLayout &DL,
|
||||||
// For now, bail out if latch type is narrower than range type.
|
// For now, bail out if latch type is narrower than range type.
|
||||||
if (DL.getTypeSizeInBits(LatchType).getFixedSize() <
|
if (DL.getTypeSizeInBits(LatchType).getFixedSize() <
|
||||||
DL.getTypeSizeInBits(RangeCheckType).getFixedSize())
|
DL.getTypeSizeInBits(RangeCheckType).getFixedSize())
|
||||||
return None;
|
return std::nullopt;
|
||||||
if (!isSafeToTruncateWideIVType(DL, SE, LatchCheck, RangeCheckType))
|
if (!isSafeToTruncateWideIVType(DL, SE, LatchCheck, RangeCheckType))
|
||||||
return None;
|
return std::nullopt;
|
||||||
// We can now safely identify the truncated version of the IV and limit for
|
// We can now safely identify the truncated version of the IV and limit for
|
||||||
// RangeCheckType.
|
// RangeCheckType.
|
||||||
LoopICmp NewLatchCheck;
|
LoopICmp NewLatchCheck;
|
||||||
|
@ -504,7 +504,7 @@ static std::optional<LoopICmp> generateLoopLatchCheck(const DataLayout &DL,
|
||||||
NewLatchCheck.IV = dyn_cast<SCEVAddRecExpr>(
|
NewLatchCheck.IV = dyn_cast<SCEVAddRecExpr>(
|
||||||
SE.getTruncateExpr(LatchCheck.IV, RangeCheckType));
|
SE.getTruncateExpr(LatchCheck.IV, RangeCheckType));
|
||||||
if (!NewLatchCheck.IV)
|
if (!NewLatchCheck.IV)
|
||||||
return None;
|
return std::nullopt;
|
||||||
NewLatchCheck.Limit = SE.getTruncateExpr(LatchCheck.Limit, RangeCheckType);
|
NewLatchCheck.Limit = SE.getTruncateExpr(LatchCheck.Limit, RangeCheckType);
|
||||||
LLVM_DEBUG(dbgs() << "IV of type: " << *LatchType
|
LLVM_DEBUG(dbgs() << "IV of type: " << *LatchType
|
||||||
<< "can be represented as range check type:"
|
<< "can be represented as range check type:"
|
||||||
|
@ -598,12 +598,12 @@ Optional<Value *> LoopPredication::widenICmpRangeCheckIncrementingLoop(
|
||||||
!isLoopInvariantValue(LatchStart) ||
|
!isLoopInvariantValue(LatchStart) ||
|
||||||
!isLoopInvariantValue(LatchLimit)) {
|
!isLoopInvariantValue(LatchLimit)) {
|
||||||
LLVM_DEBUG(dbgs() << "Can't expand limit check!\n");
|
LLVM_DEBUG(dbgs() << "Can't expand limit check!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
if (!Expander.isSafeToExpandAt(LatchStart, Guard) ||
|
if (!Expander.isSafeToExpandAt(LatchStart, Guard) ||
|
||||||
!Expander.isSafeToExpandAt(LatchLimit, Guard)) {
|
!Expander.isSafeToExpandAt(LatchLimit, Guard)) {
|
||||||
LLVM_DEBUG(dbgs() << "Can't expand limit check!\n");
|
LLVM_DEBUG(dbgs() << "Can't expand limit check!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// guardLimit - guardStart + latchStart - 1
|
// guardLimit - guardStart + latchStart - 1
|
||||||
|
@ -641,12 +641,12 @@ Optional<Value *> LoopPredication::widenICmpRangeCheckDecrementingLoop(
|
||||||
!isLoopInvariantValue(LatchStart) ||
|
!isLoopInvariantValue(LatchStart) ||
|
||||||
!isLoopInvariantValue(LatchLimit)) {
|
!isLoopInvariantValue(LatchLimit)) {
|
||||||
LLVM_DEBUG(dbgs() << "Can't expand limit check!\n");
|
LLVM_DEBUG(dbgs() << "Can't expand limit check!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
if (!Expander.isSafeToExpandAt(LatchStart, Guard) ||
|
if (!Expander.isSafeToExpandAt(LatchStart, Guard) ||
|
||||||
!Expander.isSafeToExpandAt(LatchLimit, Guard)) {
|
!Expander.isSafeToExpandAt(LatchLimit, Guard)) {
|
||||||
LLVM_DEBUG(dbgs() << "Can't expand limit check!\n");
|
LLVM_DEBUG(dbgs() << "Can't expand limit check!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
// The decrement of the latch check IV should be the same as the
|
// The decrement of the latch check IV should be the same as the
|
||||||
// rangeCheckIV.
|
// rangeCheckIV.
|
||||||
|
@ -655,7 +655,7 @@ Optional<Value *> LoopPredication::widenICmpRangeCheckDecrementingLoop(
|
||||||
LLVM_DEBUG(dbgs() << "Not the same. PostDecLatchCheckIV: "
|
LLVM_DEBUG(dbgs() << "Not the same. PostDecLatchCheckIV: "
|
||||||
<< *PostDecLatchCheckIV
|
<< *PostDecLatchCheckIV
|
||||||
<< " and RangeCheckIV: " << *RangeCheck.IV << "\n");
|
<< " and RangeCheckIV: " << *RangeCheck.IV << "\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate the widened condition for CountDownLoop:
|
// Generate the widened condition for CountDownLoop:
|
||||||
|
@ -701,26 +701,26 @@ Optional<Value *> LoopPredication::widenICmpRangeCheck(ICmpInst *ICI,
|
||||||
auto RangeCheck = parseLoopICmp(ICI);
|
auto RangeCheck = parseLoopICmp(ICI);
|
||||||
if (!RangeCheck) {
|
if (!RangeCheck) {
|
||||||
LLVM_DEBUG(dbgs() << "Failed to parse the loop latch condition!\n");
|
LLVM_DEBUG(dbgs() << "Failed to parse the loop latch condition!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
LLVM_DEBUG(dbgs() << "Guard check:\n");
|
LLVM_DEBUG(dbgs() << "Guard check:\n");
|
||||||
LLVM_DEBUG(RangeCheck->dump());
|
LLVM_DEBUG(RangeCheck->dump());
|
||||||
if (RangeCheck->Pred != ICmpInst::ICMP_ULT) {
|
if (RangeCheck->Pred != ICmpInst::ICMP_ULT) {
|
||||||
LLVM_DEBUG(dbgs() << "Unsupported range check predicate("
|
LLVM_DEBUG(dbgs() << "Unsupported range check predicate("
|
||||||
<< RangeCheck->Pred << ")!\n");
|
<< RangeCheck->Pred << ")!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
auto *RangeCheckIV = RangeCheck->IV;
|
auto *RangeCheckIV = RangeCheck->IV;
|
||||||
if (!RangeCheckIV->isAffine()) {
|
if (!RangeCheckIV->isAffine()) {
|
||||||
LLVM_DEBUG(dbgs() << "Range check IV is not affine!\n");
|
LLVM_DEBUG(dbgs() << "Range check IV is not affine!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
auto *Step = RangeCheckIV->getStepRecurrence(*SE);
|
auto *Step = RangeCheckIV->getStepRecurrence(*SE);
|
||||||
// We cannot just compare with latch IV step because the latch and range IVs
|
// We cannot just compare with latch IV step because the latch and range IVs
|
||||||
// may have different types.
|
// may have different types.
|
||||||
if (!isSupportedStep(Step)) {
|
if (!isSupportedStep(Step)) {
|
||||||
LLVM_DEBUG(dbgs() << "Range check and latch have IVs different steps!\n");
|
LLVM_DEBUG(dbgs() << "Range check and latch have IVs different steps!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
auto *Ty = RangeCheckIV->getType();
|
auto *Ty = RangeCheckIV->getType();
|
||||||
auto CurrLatchCheckOpt = generateLoopLatchCheck(*DL, *SE, LatchCheck, Ty);
|
auto CurrLatchCheckOpt = generateLoopLatchCheck(*DL, *SE, LatchCheck, Ty);
|
||||||
|
@ -728,7 +728,7 @@ Optional<Value *> LoopPredication::widenICmpRangeCheck(ICmpInst *ICI,
|
||||||
LLVM_DEBUG(dbgs() << "Failed to generate a loop latch check "
|
LLVM_DEBUG(dbgs() << "Failed to generate a loop latch check "
|
||||||
"corresponding to range type: "
|
"corresponding to range type: "
|
||||||
<< *Ty << "\n");
|
<< *Ty << "\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
LoopICmp CurrLatchCheck = *CurrLatchCheckOpt;
|
LoopICmp CurrLatchCheck = *CurrLatchCheckOpt;
|
||||||
|
@ -739,7 +739,7 @@ Optional<Value *> LoopPredication::widenICmpRangeCheck(ICmpInst *ICI,
|
||||||
"Range and latch steps should be of same type!");
|
"Range and latch steps should be of same type!");
|
||||||
if (Step != CurrLatchCheck.IV->getStepRecurrence(*SE)) {
|
if (Step != CurrLatchCheck.IV->getStepRecurrence(*SE)) {
|
||||||
LLVM_DEBUG(dbgs() << "Range and latch have different step values!\n");
|
LLVM_DEBUG(dbgs() << "Range and latch have different step values!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (Step->isOne())
|
if (Step->isOne())
|
||||||
|
@ -879,13 +879,13 @@ Optional<LoopICmp> LoopPredication::parseLoopLatchICmp() {
|
||||||
BasicBlock *LoopLatch = L->getLoopLatch();
|
BasicBlock *LoopLatch = L->getLoopLatch();
|
||||||
if (!LoopLatch) {
|
if (!LoopLatch) {
|
||||||
LLVM_DEBUG(dbgs() << "The loop doesn't have a single latch!\n");
|
LLVM_DEBUG(dbgs() << "The loop doesn't have a single latch!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto *BI = dyn_cast<BranchInst>(LoopLatch->getTerminator());
|
auto *BI = dyn_cast<BranchInst>(LoopLatch->getTerminator());
|
||||||
if (!BI || !BI->isConditional()) {
|
if (!BI || !BI->isConditional()) {
|
||||||
LLVM_DEBUG(dbgs() << "Failed to match the latch terminator!\n");
|
LLVM_DEBUG(dbgs() << "Failed to match the latch terminator!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
BasicBlock *TrueDest = BI->getSuccessor(0);
|
BasicBlock *TrueDest = BI->getSuccessor(0);
|
||||||
assert(
|
assert(
|
||||||
|
@ -895,12 +895,12 @@ Optional<LoopICmp> LoopPredication::parseLoopLatchICmp() {
|
||||||
auto *ICI = dyn_cast<ICmpInst>(BI->getCondition());
|
auto *ICI = dyn_cast<ICmpInst>(BI->getCondition());
|
||||||
if (!ICI) {
|
if (!ICI) {
|
||||||
LLVM_DEBUG(dbgs() << "Failed to match the latch condition!\n");
|
LLVM_DEBUG(dbgs() << "Failed to match the latch condition!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
auto Result = parseLoopICmp(ICI);
|
auto Result = parseLoopICmp(ICI);
|
||||||
if (!Result) {
|
if (!Result) {
|
||||||
LLVM_DEBUG(dbgs() << "Failed to parse the loop latch condition!\n");
|
LLVM_DEBUG(dbgs() << "Failed to parse the loop latch condition!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (TrueDest != L->getHeader())
|
if (TrueDest != L->getHeader())
|
||||||
|
@ -910,13 +910,13 @@ Optional<LoopICmp> LoopPredication::parseLoopLatchICmp() {
|
||||||
// recurrence.
|
// recurrence.
|
||||||
if (!Result->IV->isAffine()) {
|
if (!Result->IV->isAffine()) {
|
||||||
LLVM_DEBUG(dbgs() << "The induction variable is not affine!\n");
|
LLVM_DEBUG(dbgs() << "The induction variable is not affine!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto *Step = Result->IV->getStepRecurrence(*SE);
|
auto *Step = Result->IV->getStepRecurrence(*SE);
|
||||||
if (!isSupportedStep(Step)) {
|
if (!isSupportedStep(Step)) {
|
||||||
LLVM_DEBUG(dbgs() << "Unsupported loop stride(" << *Step << ")!\n");
|
LLVM_DEBUG(dbgs() << "Unsupported loop stride(" << *Step << ")!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto IsUnsupportedPredicate = [](const SCEV *Step, ICmpInst::Predicate Pred) {
|
auto IsUnsupportedPredicate = [](const SCEV *Step, ICmpInst::Predicate Pred) {
|
||||||
|
@ -934,7 +934,7 @@ Optional<LoopICmp> LoopPredication::parseLoopLatchICmp() {
|
||||||
if (IsUnsupportedPredicate(Step, Result->Pred)) {
|
if (IsUnsupportedPredicate(Step, Result->Pred)) {
|
||||||
LLVM_DEBUG(dbgs() << "Unsupported loop latch predicate(" << Result->Pred
|
LLVM_DEBUG(dbgs() << "Unsupported loop latch predicate(" << Result->Pred
|
||||||
<< ")!\n");
|
<< ")!\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
return Result;
|
return Result;
|
||||||
|
|
|
@ -6620,17 +6620,17 @@ canFoldTermCondOfLoop(Loop *L, ScalarEvolution &SE, DominatorTree &DT,
|
||||||
const LoopInfo &LI) {
|
const LoopInfo &LI) {
|
||||||
if (!L->isInnermost()) {
|
if (!L->isInnermost()) {
|
||||||
LLVM_DEBUG(dbgs() << "Cannot fold on non-innermost loop\n");
|
LLVM_DEBUG(dbgs() << "Cannot fold on non-innermost loop\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
// Only inspect on simple loop structure
|
// Only inspect on simple loop structure
|
||||||
if (!L->isLoopSimplifyForm()) {
|
if (!L->isLoopSimplifyForm()) {
|
||||||
LLVM_DEBUG(dbgs() << "Cannot fold on non-simple loop\n");
|
LLVM_DEBUG(dbgs() << "Cannot fold on non-simple loop\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!SE.hasLoopInvariantBackedgeTakenCount(L)) {
|
if (!SE.hasLoopInvariantBackedgeTakenCount(L)) {
|
||||||
LLVM_DEBUG(dbgs() << "Cannot fold on backedge that is loop variant\n");
|
LLVM_DEBUG(dbgs() << "Cannot fold on backedge that is loop variant\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
BasicBlock *LoopPreheader = L->getLoopPreheader();
|
BasicBlock *LoopPreheader = L->getLoopPreheader();
|
||||||
|
@ -6640,18 +6640,18 @@ canFoldTermCondOfLoop(Loop *L, ScalarEvolution &SE, DominatorTree &DT,
|
||||||
// Terminating condition is foldable when it is an eq/ne icmp
|
// Terminating condition is foldable when it is an eq/ne icmp
|
||||||
BranchInst *BI = cast<BranchInst>(LoopLatch->getTerminator());
|
BranchInst *BI = cast<BranchInst>(LoopLatch->getTerminator());
|
||||||
if (BI->isUnconditional())
|
if (BI->isUnconditional())
|
||||||
return None;
|
return std::nullopt;
|
||||||
Value *TermCond = BI->getCondition();
|
Value *TermCond = BI->getCondition();
|
||||||
if (!isa<ICmpInst>(TermCond) || !cast<ICmpInst>(TermCond)->isEquality()) {
|
if (!isa<ICmpInst>(TermCond) || !cast<ICmpInst>(TermCond)->isEquality()) {
|
||||||
LLVM_DEBUG(dbgs() << "Cannot fold on branching condition that is not an "
|
LLVM_DEBUG(dbgs() << "Cannot fold on branching condition that is not an "
|
||||||
"ICmpInst::eq / ICmpInst::ne\n");
|
"ICmpInst::eq / ICmpInst::ne\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
if (!TermCond->hasOneUse()) {
|
if (!TermCond->hasOneUse()) {
|
||||||
LLVM_DEBUG(
|
LLVM_DEBUG(
|
||||||
dbgs()
|
dbgs()
|
||||||
<< "Cannot replace terminating condition with more than one use\n");
|
<< "Cannot replace terminating condition with more than one use\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// For `IsToFold`, a primary IV can be replaced by other affine AddRec when it
|
// For `IsToFold`, a primary IV can be replaced by other affine AddRec when it
|
||||||
|
@ -6769,7 +6769,7 @@ canFoldTermCondOfLoop(Loop *L, ScalarEvolution &SE, DominatorTree &DT,
|
||||||
<< " ToHelpFold: " << *ToHelpFold << "\n");
|
<< " ToHelpFold: " << *ToHelpFold << "\n");
|
||||||
|
|
||||||
if (!ToFold || !ToHelpFold)
|
if (!ToFold || !ToHelpFold)
|
||||||
return None;
|
return std::nullopt;
|
||||||
return {{ToFold, {ToHelpFold, TermValueS}}};
|
return {{ToFold, {ToHelpFold, TermValueS}}};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -284,11 +284,11 @@ tryToUnrollAndJamLoop(Loop *L, DominatorTree &DT, LoopInfo *LI,
|
||||||
ScalarEvolution &SE, const TargetTransformInfo &TTI,
|
ScalarEvolution &SE, const TargetTransformInfo &TTI,
|
||||||
AssumptionCache &AC, DependenceInfo &DI,
|
AssumptionCache &AC, DependenceInfo &DI,
|
||||||
OptimizationRemarkEmitter &ORE, int OptLevel) {
|
OptimizationRemarkEmitter &ORE, int OptLevel) {
|
||||||
TargetTransformInfo::UnrollingPreferences UP =
|
TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(
|
||||||
gatherUnrollingPreferences(L, SE, TTI, nullptr, nullptr, ORE, OptLevel,
|
L, SE, TTI, nullptr, nullptr, ORE, OptLevel, std::nullopt, std::nullopt,
|
||||||
None, None, None, None, None, None);
|
std::nullopt, std::nullopt, std::nullopt, std::nullopt);
|
||||||
TargetTransformInfo::PeelingPreferences PP =
|
TargetTransformInfo::PeelingPreferences PP =
|
||||||
gatherPeelingPreferences(L, SE, TTI, None, None);
|
gatherPeelingPreferences(L, SE, TTI, std::nullopt, std::nullopt);
|
||||||
|
|
||||||
TransformationMode EnableMode = hasUnrollAndJamTransformation(L);
|
TransformationMode EnableMode = hasUnrollAndJamTransformation(L);
|
||||||
if (EnableMode & TM_Disable)
|
if (EnableMode & TM_Disable)
|
||||||
|
|
|
@ -359,11 +359,11 @@ static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost(
|
||||||
// Only analyze inner loops. We can't properly estimate cost of nested loops
|
// Only analyze inner loops. We can't properly estimate cost of nested loops
|
||||||
// and we won't visit inner loops again anyway.
|
// and we won't visit inner loops again anyway.
|
||||||
if (!L->isInnermost())
|
if (!L->isInnermost())
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Don't simulate loops with a big or unknown tripcount
|
// Don't simulate loops with a big or unknown tripcount
|
||||||
if (!TripCount || TripCount > MaxIterationsCountToAnalyze)
|
if (!TripCount || TripCount > MaxIterationsCountToAnalyze)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
SmallSetVector<BasicBlock *, 16> BBWorklist;
|
SmallSetVector<BasicBlock *, 16> BBWorklist;
|
||||||
SmallSetVector<std::pair<BasicBlock *, BasicBlock *>, 4> ExitWorklist;
|
SmallSetVector<std::pair<BasicBlock *, BasicBlock *>, 4> ExitWorklist;
|
||||||
|
@ -559,7 +559,7 @@ static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost(
|
||||||
const Function *Callee = CI->getCalledFunction();
|
const Function *Callee = CI->getCalledFunction();
|
||||||
if (!Callee || TTI.isLoweredToCall(Callee)) {
|
if (!Callee || TTI.isLoweredToCall(Callee)) {
|
||||||
LLVM_DEBUG(dbgs() << "Can't analyze cost of loop with call\n");
|
LLVM_DEBUG(dbgs() << "Can't analyze cost of loop with call\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -574,7 +574,7 @@ static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost(
|
||||||
<< " UnrolledCost: " << UnrolledCost
|
<< " UnrolledCost: " << UnrolledCost
|
||||||
<< ", MaxUnrolledLoopSize: " << MaxUnrolledLoopSize
|
<< ", MaxUnrolledLoopSize: " << MaxUnrolledLoopSize
|
||||||
<< "\n");
|
<< "\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -632,7 +632,7 @@ static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost(
|
||||||
if (UnrolledCost == RolledDynamicCost) {
|
if (UnrolledCost == RolledDynamicCost) {
|
||||||
LLVM_DEBUG(dbgs() << " No opportunities found.. exiting.\n"
|
LLVM_DEBUG(dbgs() << " No opportunities found.. exiting.\n"
|
||||||
<< " UnrolledCost: " << UnrolledCost << "\n");
|
<< " UnrolledCost: " << UnrolledCost << "\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -798,7 +798,7 @@ shouldPragmaUnroll(Loop *L, const PragmaInfo &PInfo,
|
||||||
return TripCount;
|
return TripCount;
|
||||||
|
|
||||||
// if didn't return until here, should continue to other priorties
|
// if didn't return until here, should continue to other priorties
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
static std::optional<unsigned> shouldFullUnroll(
|
static std::optional<unsigned> shouldFullUnroll(
|
||||||
|
@ -809,7 +809,7 @@ static std::optional<unsigned> shouldFullUnroll(
|
||||||
assert(FullUnrollTripCount && "should be non-zero!");
|
assert(FullUnrollTripCount && "should be non-zero!");
|
||||||
|
|
||||||
if (FullUnrollTripCount > UP.FullUnrollMaxCount)
|
if (FullUnrollTripCount > UP.FullUnrollMaxCount)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// When computing the unrolled size, note that BEInsns are not replicated
|
// When computing the unrolled size, note that BEInsns are not replicated
|
||||||
// like the rest of the loop body.
|
// like the rest of the loop body.
|
||||||
|
@ -828,7 +828,7 @@ static std::optional<unsigned> shouldFullUnroll(
|
||||||
if (Cost->UnrolledCost < UP.Threshold * Boost / 100)
|
if (Cost->UnrolledCost < UP.Threshold * Boost / 100)
|
||||||
return FullUnrollTripCount;
|
return FullUnrollTripCount;
|
||||||
}
|
}
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
static std::optional<unsigned>
|
static std::optional<unsigned>
|
||||||
|
@ -837,7 +837,7 @@ shouldPartialUnroll(const unsigned LoopSize, const unsigned TripCount,
|
||||||
const TargetTransformInfo::UnrollingPreferences &UP) {
|
const TargetTransformInfo::UnrollingPreferences &UP) {
|
||||||
|
|
||||||
if (!TripCount)
|
if (!TripCount)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
if (!UP.Partial) {
|
if (!UP.Partial) {
|
||||||
LLVM_DEBUG(dbgs() << " will not try to unroll partially because "
|
LLVM_DEBUG(dbgs() << " will not try to unroll partially because "
|
||||||
|
@ -1378,13 +1378,15 @@ public:
|
||||||
Optional<unsigned> ProvidedFullUnrollMaxCount;
|
Optional<unsigned> ProvidedFullUnrollMaxCount;
|
||||||
|
|
||||||
LoopUnroll(int OptLevel = 2, bool OnlyWhenForced = false,
|
LoopUnroll(int OptLevel = 2, bool OnlyWhenForced = false,
|
||||||
bool ForgetAllSCEV = false, Optional<unsigned> Threshold = None,
|
bool ForgetAllSCEV = false,
|
||||||
Optional<unsigned> Count = None,
|
Optional<unsigned> Threshold = std::nullopt,
|
||||||
Optional<bool> AllowPartial = None, Optional<bool> Runtime = None,
|
Optional<unsigned> Count = std::nullopt,
|
||||||
Optional<bool> UpperBound = None,
|
Optional<bool> AllowPartial = std::nullopt,
|
||||||
Optional<bool> AllowPeeling = None,
|
Optional<bool> Runtime = std::nullopt,
|
||||||
Optional<bool> AllowProfileBasedPeeling = None,
|
Optional<bool> UpperBound = std::nullopt,
|
||||||
Optional<unsigned> ProvidedFullUnrollMaxCount = None)
|
Optional<bool> AllowPeeling = std::nullopt,
|
||||||
|
Optional<bool> AllowProfileBasedPeeling = std::nullopt,
|
||||||
|
Optional<unsigned> ProvidedFullUnrollMaxCount = std::nullopt)
|
||||||
: LoopPass(ID), OptLevel(OptLevel), OnlyWhenForced(OnlyWhenForced),
|
: LoopPass(ID), OptLevel(OptLevel), OnlyWhenForced(OnlyWhenForced),
|
||||||
ForgetAllSCEV(ForgetAllSCEV), ProvidedCount(std::move(Count)),
|
ForgetAllSCEV(ForgetAllSCEV), ProvidedCount(std::move(Count)),
|
||||||
ProvidedThreshold(Threshold), ProvidedAllowPartial(AllowPartial),
|
ProvidedThreshold(Threshold), ProvidedAllowPartial(AllowPartial),
|
||||||
|
@ -1456,12 +1458,12 @@ Pass *llvm::createLoopUnrollPass(int OptLevel, bool OnlyWhenForced,
|
||||||
// callers.
|
// callers.
|
||||||
return new LoopUnroll(
|
return new LoopUnroll(
|
||||||
OptLevel, OnlyWhenForced, ForgetAllSCEV,
|
OptLevel, OnlyWhenForced, ForgetAllSCEV,
|
||||||
Threshold == -1 ? None : Optional<unsigned>(Threshold),
|
Threshold == -1 ? std::nullopt : Optional<unsigned>(Threshold),
|
||||||
Count == -1 ? None : Optional<unsigned>(Count),
|
Count == -1 ? std::nullopt : Optional<unsigned>(Count),
|
||||||
AllowPartial == -1 ? None : Optional<bool>(AllowPartial),
|
AllowPartial == -1 ? std::nullopt : Optional<bool>(AllowPartial),
|
||||||
Runtime == -1 ? None : Optional<bool>(Runtime),
|
Runtime == -1 ? std::nullopt : Optional<bool>(Runtime),
|
||||||
UpperBound == -1 ? None : Optional<bool>(UpperBound),
|
UpperBound == -1 ? std::nullopt : Optional<bool>(UpperBound),
|
||||||
AllowPeeling == -1 ? None : Optional<bool>(AllowPeeling));
|
AllowPeeling == -1 ? std::nullopt : Optional<bool>(AllowPeeling));
|
||||||
}
|
}
|
||||||
|
|
||||||
Pass *llvm::createSimpleLoopUnrollPass(int OptLevel, bool OnlyWhenForced,
|
Pass *llvm::createSimpleLoopUnrollPass(int OptLevel, bool OnlyWhenForced,
|
||||||
|
@ -1489,16 +1491,17 @@ PreservedAnalyses LoopFullUnrollPass::run(Loop &L, LoopAnalysisManager &AM,
|
||||||
|
|
||||||
std::string LoopName = std::string(L.getName());
|
std::string LoopName = std::string(L.getName());
|
||||||
|
|
||||||
bool Changed = tryToUnrollLoop(&L, AR.DT, &AR.LI, AR.SE, AR.TTI, AR.AC, ORE,
|
bool Changed =
|
||||||
/*BFI*/ nullptr, /*PSI*/ nullptr,
|
tryToUnrollLoop(&L, AR.DT, &AR.LI, AR.SE, AR.TTI, AR.AC, ORE,
|
||||||
/*PreserveLCSSA*/ true, OptLevel,
|
/*BFI*/ nullptr, /*PSI*/ nullptr,
|
||||||
OnlyWhenForced, ForgetSCEV, /*Count*/ None,
|
/*PreserveLCSSA*/ true, OptLevel, OnlyWhenForced,
|
||||||
/*Threshold*/ None, /*AllowPartial*/ false,
|
ForgetSCEV, /*Count*/ std::nullopt,
|
||||||
/*Runtime*/ false, /*UpperBound*/ false,
|
/*Threshold*/ std::nullopt, /*AllowPartial*/ false,
|
||||||
/*AllowPeeling*/ true,
|
/*Runtime*/ false, /*UpperBound*/ false,
|
||||||
/*AllowProfileBasedPeeling*/ false,
|
/*AllowPeeling*/ true,
|
||||||
/*FullUnrollMaxCount*/ None) !=
|
/*AllowProfileBasedPeeling*/ false,
|
||||||
LoopUnrollResult::Unmodified;
|
/*FullUnrollMaxCount*/ std::nullopt) !=
|
||||||
|
LoopUnrollResult::Unmodified;
|
||||||
if (!Changed)
|
if (!Changed)
|
||||||
return PreservedAnalyses::all();
|
return PreservedAnalyses::all();
|
||||||
|
|
||||||
|
@ -1618,9 +1621,9 @@ PreservedAnalyses LoopUnrollPass::run(Function &F,
|
||||||
LoopUnrollResult Result = tryToUnrollLoop(
|
LoopUnrollResult Result = tryToUnrollLoop(
|
||||||
&L, DT, &LI, SE, TTI, AC, ORE, BFI, PSI,
|
&L, DT, &LI, SE, TTI, AC, ORE, BFI, PSI,
|
||||||
/*PreserveLCSSA*/ true, UnrollOpts.OptLevel, UnrollOpts.OnlyWhenForced,
|
/*PreserveLCSSA*/ true, UnrollOpts.OptLevel, UnrollOpts.OnlyWhenForced,
|
||||||
UnrollOpts.ForgetSCEV, /*Count*/ None,
|
UnrollOpts.ForgetSCEV, /*Count*/ std::nullopt,
|
||||||
/*Threshold*/ None, UnrollOpts.AllowPartial, UnrollOpts.AllowRuntime,
|
/*Threshold*/ std::nullopt, UnrollOpts.AllowPartial,
|
||||||
UnrollOpts.AllowUpperBound, LocalAllowPeeling,
|
UnrollOpts.AllowRuntime, UnrollOpts.AllowUpperBound, LocalAllowPeeling,
|
||||||
UnrollOpts.AllowProfileBasedPeeling, UnrollOpts.FullUnrollMaxCount);
|
UnrollOpts.AllowProfileBasedPeeling, UnrollOpts.FullUnrollMaxCount);
|
||||||
Changed |= Result != LoopUnrollResult::Unmodified;
|
Changed |= Result != LoopUnrollResult::Unmodified;
|
||||||
|
|
||||||
|
@ -1646,18 +1649,18 @@ void LoopUnrollPass::printPipeline(
|
||||||
static_cast<PassInfoMixin<LoopUnrollPass> *>(this)->printPipeline(
|
static_cast<PassInfoMixin<LoopUnrollPass> *>(this)->printPipeline(
|
||||||
OS, MapClassName2PassName);
|
OS, MapClassName2PassName);
|
||||||
OS << "<";
|
OS << "<";
|
||||||
if (UnrollOpts.AllowPartial != None)
|
if (UnrollOpts.AllowPartial != std::nullopt)
|
||||||
OS << (UnrollOpts.AllowPartial.value() ? "" : "no-") << "partial;";
|
OS << (UnrollOpts.AllowPartial.value() ? "" : "no-") << "partial;";
|
||||||
if (UnrollOpts.AllowPeeling != None)
|
if (UnrollOpts.AllowPeeling != std::nullopt)
|
||||||
OS << (UnrollOpts.AllowPeeling.value() ? "" : "no-") << "peeling;";
|
OS << (UnrollOpts.AllowPeeling.value() ? "" : "no-") << "peeling;";
|
||||||
if (UnrollOpts.AllowRuntime != None)
|
if (UnrollOpts.AllowRuntime != std::nullopt)
|
||||||
OS << (UnrollOpts.AllowRuntime.value() ? "" : "no-") << "runtime;";
|
OS << (UnrollOpts.AllowRuntime.value() ? "" : "no-") << "runtime;";
|
||||||
if (UnrollOpts.AllowUpperBound != None)
|
if (UnrollOpts.AllowUpperBound != std::nullopt)
|
||||||
OS << (UnrollOpts.AllowUpperBound.value() ? "" : "no-") << "upperbound;";
|
OS << (UnrollOpts.AllowUpperBound.value() ? "" : "no-") << "upperbound;";
|
||||||
if (UnrollOpts.AllowProfileBasedPeeling != None)
|
if (UnrollOpts.AllowProfileBasedPeeling != std::nullopt)
|
||||||
OS << (UnrollOpts.AllowProfileBasedPeeling.value() ? "" : "no-")
|
OS << (UnrollOpts.AllowProfileBasedPeeling.value() ? "" : "no-")
|
||||||
<< "profile-peeling;";
|
<< "profile-peeling;";
|
||||||
if (UnrollOpts.FullUnrollMaxCount != None)
|
if (UnrollOpts.FullUnrollMaxCount != std::nullopt)
|
||||||
OS << "full-unroll-max=" << UnrollOpts.FullUnrollMaxCount << ";";
|
OS << "full-unroll-max=" << UnrollOpts.FullUnrollMaxCount << ";";
|
||||||
OS << "O" << UnrollOpts.OptLevel;
|
OS << "O" << UnrollOpts.OptLevel;
|
||||||
OS << ">";
|
OS << ">";
|
||||||
|
|
|
@ -590,7 +590,7 @@ bool MemCpyOptPass::moveUp(StoreInst *SI, Instruction *P, const LoadInst *LI) {
|
||||||
if (!isGuaranteedToTransferExecutionToSuccessor(C))
|
if (!isGuaranteedToTransferExecutionToSuccessor(C))
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
bool MayAlias = isModOrRefSet(AA->getModRefInfo(C, None));
|
bool MayAlias = isModOrRefSet(AA->getModRefInfo(C, std::nullopt));
|
||||||
|
|
||||||
bool NeedLift = false;
|
bool NeedLift = false;
|
||||||
if (Args.erase(C))
|
if (Args.erase(C))
|
||||||
|
|
|
@ -310,19 +310,19 @@ Optional<BCECmp> visitICmp(const ICmpInst *const CmpI,
|
||||||
// other comparisons as we would create an orphan use of the value.
|
// other comparisons as we would create an orphan use of the value.
|
||||||
if (!CmpI->hasOneUse()) {
|
if (!CmpI->hasOneUse()) {
|
||||||
LLVM_DEBUG(dbgs() << "cmp has several uses\n");
|
LLVM_DEBUG(dbgs() << "cmp has several uses\n");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
if (CmpI->getPredicate() != ExpectedPredicate)
|
if (CmpI->getPredicate() != ExpectedPredicate)
|
||||||
return None;
|
return std::nullopt;
|
||||||
LLVM_DEBUG(dbgs() << "cmp "
|
LLVM_DEBUG(dbgs() << "cmp "
|
||||||
<< (ExpectedPredicate == ICmpInst::ICMP_EQ ? "eq" : "ne")
|
<< (ExpectedPredicate == ICmpInst::ICMP_EQ ? "eq" : "ne")
|
||||||
<< "\n");
|
<< "\n");
|
||||||
auto Lhs = visitICmpLoadOperand(CmpI->getOperand(0), BaseId);
|
auto Lhs = visitICmpLoadOperand(CmpI->getOperand(0), BaseId);
|
||||||
if (!Lhs.BaseId)
|
if (!Lhs.BaseId)
|
||||||
return None;
|
return std::nullopt;
|
||||||
auto Rhs = visitICmpLoadOperand(CmpI->getOperand(1), BaseId);
|
auto Rhs = visitICmpLoadOperand(CmpI->getOperand(1), BaseId);
|
||||||
if (!Rhs.BaseId)
|
if (!Rhs.BaseId)
|
||||||
return None;
|
return std::nullopt;
|
||||||
const auto &DL = CmpI->getModule()->getDataLayout();
|
const auto &DL = CmpI->getModule()->getDataLayout();
|
||||||
return BCECmp(std::move(Lhs), std::move(Rhs),
|
return BCECmp(std::move(Lhs), std::move(Rhs),
|
||||||
DL.getTypeSizeInBits(CmpI->getOperand(0)->getType()), CmpI);
|
DL.getTypeSizeInBits(CmpI->getOperand(0)->getType()), CmpI);
|
||||||
|
@ -333,9 +333,11 @@ Optional<BCECmp> visitICmp(const ICmpInst *const CmpI,
|
||||||
Optional<BCECmpBlock> visitCmpBlock(Value *const Val, BasicBlock *const Block,
|
Optional<BCECmpBlock> visitCmpBlock(Value *const Val, BasicBlock *const Block,
|
||||||
const BasicBlock *const PhiBlock,
|
const BasicBlock *const PhiBlock,
|
||||||
BaseIdentifier &BaseId) {
|
BaseIdentifier &BaseId) {
|
||||||
if (Block->empty()) return None;
|
if (Block->empty())
|
||||||
|
return std::nullopt;
|
||||||
auto *const BranchI = dyn_cast<BranchInst>(Block->getTerminator());
|
auto *const BranchI = dyn_cast<BranchInst>(Block->getTerminator());
|
||||||
if (!BranchI) return None;
|
if (!BranchI)
|
||||||
|
return std::nullopt;
|
||||||
LLVM_DEBUG(dbgs() << "branch\n");
|
LLVM_DEBUG(dbgs() << "branch\n");
|
||||||
Value *Cond;
|
Value *Cond;
|
||||||
ICmpInst::Predicate ExpectedPredicate;
|
ICmpInst::Predicate ExpectedPredicate;
|
||||||
|
@ -351,7 +353,8 @@ Optional<BCECmpBlock> visitCmpBlock(Value *const Val, BasicBlock *const Block,
|
||||||
// chained).
|
// chained).
|
||||||
const auto *const Const = cast<ConstantInt>(Val);
|
const auto *const Const = cast<ConstantInt>(Val);
|
||||||
LLVM_DEBUG(dbgs() << "const\n");
|
LLVM_DEBUG(dbgs() << "const\n");
|
||||||
if (!Const->isZero()) return None;
|
if (!Const->isZero())
|
||||||
|
return std::nullopt;
|
||||||
LLVM_DEBUG(dbgs() << "false\n");
|
LLVM_DEBUG(dbgs() << "false\n");
|
||||||
assert(BranchI->getNumSuccessors() == 2 && "expecting a cond branch");
|
assert(BranchI->getNumSuccessors() == 2 && "expecting a cond branch");
|
||||||
BasicBlock *const FalseBlock = BranchI->getSuccessor(1);
|
BasicBlock *const FalseBlock = BranchI->getSuccessor(1);
|
||||||
|
@ -361,12 +364,13 @@ Optional<BCECmpBlock> visitCmpBlock(Value *const Val, BasicBlock *const Block,
|
||||||
}
|
}
|
||||||
|
|
||||||
auto *CmpI = dyn_cast<ICmpInst>(Cond);
|
auto *CmpI = dyn_cast<ICmpInst>(Cond);
|
||||||
if (!CmpI) return None;
|
if (!CmpI)
|
||||||
|
return std::nullopt;
|
||||||
LLVM_DEBUG(dbgs() << "icmp\n");
|
LLVM_DEBUG(dbgs() << "icmp\n");
|
||||||
|
|
||||||
Optional<BCECmp> Result = visitICmp(CmpI, ExpectedPredicate, BaseId);
|
Optional<BCECmp> Result = visitICmp(CmpI, ExpectedPredicate, BaseId);
|
||||||
if (!Result)
|
if (!Result)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
BCECmpBlock::InstructionSet BlockInsts(
|
BCECmpBlock::InstructionSet BlockInsts(
|
||||||
{Result->Lhs.LoadI, Result->Rhs.LoadI, Result->CmpI, BranchI});
|
{Result->Lhs.LoadI, Result->Rhs.LoadI, Result->CmpI, BranchI});
|
||||||
|
|
|
@ -301,7 +301,7 @@ static ArrayRef<Use> GetDeoptBundleOperands(const CallBase *Call) {
|
||||||
if (!DeoptBundle) {
|
if (!DeoptBundle) {
|
||||||
assert(AllowStatepointWithNoDeoptInfo &&
|
assert(AllowStatepointWithNoDeoptInfo &&
|
||||||
"Found non-leaf call without deopt info!");
|
"Found non-leaf call without deopt info!");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
return DeoptBundle->Inputs;
|
return DeoptBundle->Inputs;
|
||||||
|
|
|
@ -489,11 +489,11 @@ ScalarizerVisitor::getVectorLayout(Type *Ty, Align Alignment,
|
||||||
// Make sure we're dealing with a vector.
|
// Make sure we're dealing with a vector.
|
||||||
Layout.VecTy = dyn_cast<VectorType>(Ty);
|
Layout.VecTy = dyn_cast<VectorType>(Ty);
|
||||||
if (!Layout.VecTy)
|
if (!Layout.VecTy)
|
||||||
return None;
|
return std::nullopt;
|
||||||
// Check that we're dealing with full-byte elements.
|
// Check that we're dealing with full-byte elements.
|
||||||
Layout.ElemTy = Layout.VecTy->getElementType();
|
Layout.ElemTy = Layout.VecTy->getElementType();
|
||||||
if (!DL.typeSizeEqualsStoreSize(Layout.ElemTy))
|
if (!DL.typeSizeEqualsStoreSize(Layout.ElemTy))
|
||||||
return None;
|
return std::nullopt;
|
||||||
Layout.VecAlign = Alignment;
|
Layout.VecAlign = Alignment;
|
||||||
Layout.ElemSize = DL.getTypeStoreSize(Layout.ElemTy);
|
Layout.ElemSize = DL.getTypeStoreSize(Layout.ElemTy);
|
||||||
return Layout;
|
return Layout;
|
||||||
|
|
|
@ -124,8 +124,8 @@ struct NonTrivialUnswitchCandidate {
|
||||||
TinyPtrVector<Value *> Invariants;
|
TinyPtrVector<Value *> Invariants;
|
||||||
Optional<InstructionCost> Cost;
|
Optional<InstructionCost> Cost;
|
||||||
NonTrivialUnswitchCandidate(Instruction *TI, ArrayRef<Value *> Invariants,
|
NonTrivialUnswitchCandidate(Instruction *TI, ArrayRef<Value *> Invariants,
|
||||||
Optional<InstructionCost> Cost = None)
|
Optional<InstructionCost> Cost = std::nullopt)
|
||||||
: TI(TI), Invariants(Invariants), Cost(Cost) {};
|
: TI(TI), Invariants(Invariants), Cost(Cost){};
|
||||||
};
|
};
|
||||||
} // end anonymous namespace.
|
} // end anonymous namespace.
|
||||||
|
|
||||||
|
|
|
@ -427,7 +427,7 @@ static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
|
||||||
VariableMap;
|
VariableMap;
|
||||||
for (auto &I : *BB) {
|
for (auto &I : *BB) {
|
||||||
if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
|
if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
|
||||||
DebugVariable Key(DVI->getVariable(), None,
|
DebugVariable Key(DVI->getVariable(), std::nullopt,
|
||||||
DVI->getDebugLoc()->getInlinedAt());
|
DVI->getDebugLoc()->getInlinedAt());
|
||||||
auto VMI = VariableMap.find(Key);
|
auto VMI = VariableMap.find(Key);
|
||||||
auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
|
auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
|
||||||
|
@ -488,7 +488,7 @@ static bool remomveUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
|
||||||
DenseSet<DebugVariable> SeenDefForAggregate;
|
DenseSet<DebugVariable> SeenDefForAggregate;
|
||||||
// Returns the DebugVariable for DVI with no fragment info.
|
// Returns the DebugVariable for DVI with no fragment info.
|
||||||
auto GetAggregateVariable = [](DbgValueInst *DVI) {
|
auto GetAggregateVariable = [](DbgValueInst *DVI) {
|
||||||
return DebugVariable(DVI->getVariable(), None,
|
return DebugVariable(DVI->getVariable(), std::nullopt,
|
||||||
DVI->getDebugLoc()->getInlinedAt());
|
DVI->getDebugLoc()->getInlinedAt());
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -357,12 +357,12 @@ Optional<QuotRemPair> FastDivInsertionTask::insertFastDivAndRem() {
|
||||||
VisitedSetTy SetL;
|
VisitedSetTy SetL;
|
||||||
ValueRange DividendRange = getValueRange(Dividend, SetL);
|
ValueRange DividendRange = getValueRange(Dividend, SetL);
|
||||||
if (DividendRange == VALRNG_LIKELY_LONG)
|
if (DividendRange == VALRNG_LIKELY_LONG)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
VisitedSetTy SetR;
|
VisitedSetTy SetR;
|
||||||
ValueRange DivisorRange = getValueRange(Divisor, SetR);
|
ValueRange DivisorRange = getValueRange(Divisor, SetR);
|
||||||
if (DivisorRange == VALRNG_LIKELY_LONG)
|
if (DivisorRange == VALRNG_LIKELY_LONG)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
bool DividendShort = (DividendRange == VALRNG_KNOWN_SHORT);
|
bool DividendShort = (DividendRange == VALRNG_KNOWN_SHORT);
|
||||||
bool DivisorShort = (DivisorRange == VALRNG_KNOWN_SHORT);
|
bool DivisorShort = (DivisorRange == VALRNG_KNOWN_SHORT);
|
||||||
|
@ -387,7 +387,7 @@ Optional<QuotRemPair> FastDivInsertionTask::insertFastDivAndRem() {
|
||||||
// If the divisor is not a constant, DAGCombiner will convert it to a
|
// If the divisor is not a constant, DAGCombiner will convert it to a
|
||||||
// multiplication by a magic constant. It isn't clear if it is worth
|
// multiplication by a magic constant. It isn't clear if it is worth
|
||||||
// introducing control flow to get a narrower multiply.
|
// introducing control flow to get a narrower multiply.
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// After Constant Hoisting pass, long constants may be represented as
|
// After Constant Hoisting pass, long constants may be represented as
|
||||||
|
@ -397,7 +397,7 @@ Optional<QuotRemPair> FastDivInsertionTask::insertFastDivAndRem() {
|
||||||
if (auto *BCI = dyn_cast<BitCastInst>(Divisor))
|
if (auto *BCI = dyn_cast<BitCastInst>(Divisor))
|
||||||
if (BCI->getParent() == SlowDivOrRem->getParent() &&
|
if (BCI->getParent() == SlowDivOrRem->getParent() &&
|
||||||
isa<ConstantInt>(BCI->getOperand(0)))
|
isa<ConstantInt>(BCI->getOperand(0)))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
IRBuilder<> Builder(MainBB, MainBB->end());
|
IRBuilder<> Builder(MainBB, MainBB->end());
|
||||||
Builder.SetCurrentDebugLocation(SlowDivOrRem->getDebugLoc());
|
Builder.SetCurrentDebugLocation(SlowDivOrRem->getDebugLoc());
|
||||||
|
|
|
@ -1539,7 +1539,8 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
|
||||||
assert(OldSP->getUnit() && "Missing compile unit for subprogram");
|
assert(OldSP->getUnit() && "Missing compile unit for subprogram");
|
||||||
DIBuilder DIB(*OldFunc.getParent(), /*AllowUnresolved=*/false,
|
DIBuilder DIB(*OldFunc.getParent(), /*AllowUnresolved=*/false,
|
||||||
OldSP->getUnit());
|
OldSP->getUnit());
|
||||||
auto SPType = DIB.createSubroutineType(DIB.getOrCreateTypeArray(None));
|
auto SPType =
|
||||||
|
DIB.createSubroutineType(DIB.getOrCreateTypeArray(std::nullopt));
|
||||||
DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagDefinition |
|
DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagDefinition |
|
||||||
DISubprogram::SPFlagOptimized |
|
DISubprogram::SPFlagOptimized |
|
||||||
DISubprogram::SPFlagLocalToUnit;
|
DISubprogram::SPFlagLocalToUnit;
|
||||||
|
|
|
@ -129,7 +129,7 @@ const Optional<ControlConditions> ControlConditions::collectControlConditions(
|
||||||
// Limitation: can only handle branch instruction currently.
|
// Limitation: can only handle branch instruction currently.
|
||||||
const BranchInst *BI = dyn_cast<BranchInst>(IDom->getTerminator());
|
const BranchInst *BI = dyn_cast<BranchInst>(IDom->getTerminator());
|
||||||
if (!BI)
|
if (!BI)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
bool Inserted = false;
|
bool Inserted = false;
|
||||||
if (PDT.dominates(CurBlock, IDom)) {
|
if (PDT.dominates(CurBlock, IDom)) {
|
||||||
|
@ -149,13 +149,13 @@ const Optional<ControlConditions> ControlConditions::collectControlConditions(
|
||||||
Inserted = Conditions.addControlCondition(
|
Inserted = Conditions.addControlCondition(
|
||||||
ControlCondition(BI->getCondition(), false));
|
ControlCondition(BI->getCondition(), false));
|
||||||
} else
|
} else
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
if (Inserted)
|
if (Inserted)
|
||||||
++NumConditions;
|
++NumConditions;
|
||||||
|
|
||||||
if (MaxLookup != 0 && NumConditions > MaxLookup)
|
if (MaxLookup != 0 && NumConditions > MaxLookup)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
CurBlock = IDom;
|
CurBlock = IDom;
|
||||||
} while (CurBlock != &Dominator);
|
} while (CurBlock != &Dominator);
|
||||||
|
@ -252,13 +252,13 @@ bool llvm::isControlFlowEquivalent(const BasicBlock &BB0, const BasicBlock &BB1,
|
||||||
const Optional<ControlConditions> BB0Conditions =
|
const Optional<ControlConditions> BB0Conditions =
|
||||||
ControlConditions::collectControlConditions(BB0, *CommonDominator, DT,
|
ControlConditions::collectControlConditions(BB0, *CommonDominator, DT,
|
||||||
PDT);
|
PDT);
|
||||||
if (BB0Conditions == None)
|
if (BB0Conditions == std::nullopt)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
const Optional<ControlConditions> BB1Conditions =
|
const Optional<ControlConditions> BB1Conditions =
|
||||||
ControlConditions::collectControlConditions(BB1, *CommonDominator, DT,
|
ControlConditions::collectControlConditions(BB1, *CommonDominator, DT,
|
||||||
PDT);
|
PDT);
|
||||||
if (BB1Conditions == None)
|
if (BB1Conditions == std::nullopt)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
return BB0Conditions->isEquivalent(*BB1Conditions);
|
return BB0Conditions->isEquivalent(*BB1Conditions);
|
||||||
|
|
|
@ -114,7 +114,8 @@ bool llvm::applyDebugifyMetadata(
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
bool InsertedDbgVal = false;
|
bool InsertedDbgVal = false;
|
||||||
auto SPType = DIB.createSubroutineType(DIB.getOrCreateTypeArray(None));
|
auto SPType =
|
||||||
|
DIB.createSubroutineType(DIB.getOrCreateTypeArray(std::nullopt));
|
||||||
DISubprogram::DISPFlags SPFlags =
|
DISubprogram::DISPFlags SPFlags =
|
||||||
DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized;
|
DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized;
|
||||||
if (F.hasPrivateLinkage() || F.hasInternalLinkage())
|
if (F.hasPrivateLinkage() || F.hasInternalLinkage())
|
||||||
|
|
|
@ -1083,7 +1083,7 @@ void ScopedAliasMetadataDeepCloner::clone() {
|
||||||
|
|
||||||
SmallVector<TempMDTuple, 16> DummyNodes;
|
SmallVector<TempMDTuple, 16> DummyNodes;
|
||||||
for (const MDNode *I : MD) {
|
for (const MDNode *I : MD) {
|
||||||
DummyNodes.push_back(MDTuple::getTemporary(I->getContext(), None));
|
DummyNodes.push_back(MDTuple::getTemporary(I->getContext(), std::nullopt));
|
||||||
MDMap[I].reset(DummyNodes.back().get());
|
MDMap[I].reset(DummyNodes.back().get());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1896,7 +1896,8 @@ static void updateCallProfile(Function *Callee, const ValueToValueMapTy &VMap,
|
||||||
BlockFrequencyInfo *CallerBFI) {
|
BlockFrequencyInfo *CallerBFI) {
|
||||||
if (CalleeEntryCount.isSynthetic() || CalleeEntryCount.getCount() < 1)
|
if (CalleeEntryCount.isSynthetic() || CalleeEntryCount.getCount() < 1)
|
||||||
return;
|
return;
|
||||||
auto CallSiteCount = PSI ? PSI->getProfileCount(TheCall, CallerBFI) : None;
|
auto CallSiteCount =
|
||||||
|
PSI ? PSI->getProfileCount(TheCall, CallerBFI) : std::nullopt;
|
||||||
int64_t CallCount =
|
int64_t CallCount =
|
||||||
std::min(CallSiteCount.value_or(0), CalleeEntryCount.getCount());
|
std::min(CallSiteCount.value_or(0), CalleeEntryCount.getCount());
|
||||||
updateProfileCallee(Callee, -CallCount, &VMap);
|
updateProfileCallee(Callee, -CallCount, &VMap);
|
||||||
|
|
|
@ -2174,7 +2174,7 @@ bool llvm::replaceAllDbgUsesWith(Instruction &From, Value &To,
|
||||||
// Without knowing signedness, sign/zero extension isn't possible.
|
// Without knowing signedness, sign/zero extension isn't possible.
|
||||||
auto Signedness = Var->getSignedness();
|
auto Signedness = Var->getSignedness();
|
||||||
if (!Signedness)
|
if (!Signedness)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
bool Signed = *Signedness == DIBasicType::Signedness::Signed;
|
bool Signed = *Signedness == DIBasicType::Signedness::Signed;
|
||||||
return DIExpression::appendExt(DII.getExpression(), ToBits, FromBits,
|
return DIExpression::appendExt(DII.getExpression(), ToBits, FromBits,
|
||||||
|
@ -2912,7 +2912,7 @@ void llvm::copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
|
||||||
|
|
||||||
unsigned BitWidth = DL.getPointerTypeSizeInBits(NewTy);
|
unsigned BitWidth = DL.getPointerTypeSizeInBits(NewTy);
|
||||||
if (!getConstantRangeFromMetadata(*N).contains(APInt(BitWidth, 0))) {
|
if (!getConstantRangeFromMetadata(*N).contains(APInt(BitWidth, 0))) {
|
||||||
MDNode *NN = MDNode::get(OldLI.getContext(), None);
|
MDNode *NN = MDNode::get(OldLI.getContext(), std::nullopt);
|
||||||
NewLI.setMetadata(LLVMContext::MD_nonnull, NN);
|
NewLI.setMetadata(LLVMContext::MD_nonnull, NN);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -3021,7 +3021,7 @@ collectBitParts(Value *V, bool MatchBSwaps, bool MatchBitReversals,
|
||||||
if (I != BPS.end())
|
if (I != BPS.end())
|
||||||
return I->second;
|
return I->second;
|
||||||
|
|
||||||
auto &Result = BPS[V] = None;
|
auto &Result = BPS[V] = std::nullopt;
|
||||||
auto BitWidth = V->getType()->getScalarSizeInBits();
|
auto BitWidth = V->getType()->getScalarSizeInBits();
|
||||||
|
|
||||||
// Can't do integer/elements > 128 bits.
|
// Can't do integer/elements > 128 bits.
|
||||||
|
@ -3057,7 +3057,7 @@ collectBitParts(Value *V, bool MatchBSwaps, bool MatchBitReversals,
|
||||||
if (A->Provenance[BitIdx] != BitPart::Unset &&
|
if (A->Provenance[BitIdx] != BitPart::Unset &&
|
||||||
B->Provenance[BitIdx] != BitPart::Unset &&
|
B->Provenance[BitIdx] != BitPart::Unset &&
|
||||||
A->Provenance[BitIdx] != B->Provenance[BitIdx])
|
A->Provenance[BitIdx] != B->Provenance[BitIdx])
|
||||||
return Result = None;
|
return Result = std::nullopt;
|
||||||
|
|
||||||
if (A->Provenance[BitIdx] == BitPart::Unset)
|
if (A->Provenance[BitIdx] == BitPart::Unset)
|
||||||
Result->Provenance[BitIdx] = B->Provenance[BitIdx];
|
Result->Provenance[BitIdx] = B->Provenance[BitIdx];
|
||||||
|
|
|
@ -165,7 +165,7 @@ public:
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
using PeelCounter = std::optional<unsigned>;
|
using PeelCounter = std::optional<unsigned>;
|
||||||
const PeelCounter Unknown = None;
|
const PeelCounter Unknown = std::nullopt;
|
||||||
|
|
||||||
// Add 1 respecting Unknown and return Unknown if result over MaxIterations
|
// Add 1 respecting Unknown and return Unknown if result over MaxIterations
|
||||||
PeelCounter addOne(PeelCounter PC) const {
|
PeelCounter addOne(PeelCounter PC) const {
|
||||||
|
@ -250,7 +250,7 @@ Optional<unsigned> PhiAnalyzer::calculateIterationsToPeel() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
assert((Iterations <= MaxIterations) && "bad result in phi analysis");
|
assert((Iterations <= MaxIterations) && "bad result in phi analysis");
|
||||||
return Iterations ? Optional<unsigned>(Iterations) : None;
|
return Iterations ? Optional<unsigned>(Iterations) : std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
} // unnamed namespace
|
} // unnamed namespace
|
||||||
|
|
|
@ -703,7 +703,7 @@ LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
|
||||||
if (CompletelyUnroll) {
|
if (CompletelyUnroll) {
|
||||||
if (PreserveOnlyFirst) {
|
if (PreserveOnlyFirst) {
|
||||||
if (i == 0)
|
if (i == 0)
|
||||||
return None;
|
return std::nullopt;
|
||||||
return j == 0;
|
return j == 0;
|
||||||
}
|
}
|
||||||
// Complete (but possibly inexact) unrolling
|
// Complete (but possibly inexact) unrolling
|
||||||
|
@ -711,7 +711,7 @@ LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
|
||||||
return true;
|
return true;
|
||||||
if (Info.TripCount && j != Info.TripCount)
|
if (Info.TripCount && j != Info.TripCount)
|
||||||
return false;
|
return false;
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (ULO.Runtime) {
|
if (ULO.Runtime) {
|
||||||
|
@ -719,7 +719,7 @@ LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
|
||||||
// exits may be stale.
|
// exits may be stale.
|
||||||
if (IsLatch && j != 0)
|
if (IsLatch && j != 0)
|
||||||
return false;
|
return false;
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (j != Info.BreakoutTrip &&
|
if (j != Info.BreakoutTrip &&
|
||||||
|
@ -728,7 +728,7 @@ LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
|
||||||
// unconditional branch for some iterations.
|
// unconditional branch for some iterations.
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return None;
|
return std::nullopt;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Fold branches for iterations where we know that they will exit or not
|
// Fold branches for iterations where we know that they will exit or not
|
||||||
|
|
|
@ -258,7 +258,7 @@ llvm::getOptionalElementCountLoopAttribute(const Loop *TheLoop) {
|
||||||
return ElementCount::get(*Width, IsScalable.value_or(false));
|
return ElementCount::get(*Width, IsScalable.value_or(false));
|
||||||
}
|
}
|
||||||
|
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
Optional<MDNode *> llvm::makeFollowupLoopID(
|
Optional<MDNode *> llvm::makeFollowupLoopID(
|
||||||
|
@ -267,7 +267,7 @@ Optional<MDNode *> llvm::makeFollowupLoopID(
|
||||||
if (!OrigLoopID) {
|
if (!OrigLoopID) {
|
||||||
if (AlwaysNew)
|
if (AlwaysNew)
|
||||||
return nullptr;
|
return nullptr;
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
assert(OrigLoopID->getOperand(0) == OrigLoopID);
|
assert(OrigLoopID->getOperand(0) == OrigLoopID);
|
||||||
|
@ -326,7 +326,7 @@ Optional<MDNode *> llvm::makeFollowupLoopID(
|
||||||
// Attributes of the followup loop not specified explicity, so signal to the
|
// Attributes of the followup loop not specified explicity, so signal to the
|
||||||
// transformation pass to add suitable attributes.
|
// transformation pass to add suitable attributes.
|
||||||
if (!AlwaysNew && !HasAnyFollowup)
|
if (!AlwaysNew && !HasAnyFollowup)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// If no attributes were added or remove, the previous loop Id can be reused.
|
// If no attributes were added or remove, the previous loop Id can be reused.
|
||||||
if (!AlwaysNew && !Changed)
|
if (!AlwaysNew && !Changed)
|
||||||
|
@ -790,14 +790,14 @@ getEstimatedTripCount(BranchInst *ExitingBranch, Loop *L,
|
||||||
// we exited the loop.
|
// we exited the loop.
|
||||||
uint64_t LoopWeight, ExitWeight;
|
uint64_t LoopWeight, ExitWeight;
|
||||||
if (!extractBranchWeights(*ExitingBranch, LoopWeight, ExitWeight))
|
if (!extractBranchWeights(*ExitingBranch, LoopWeight, ExitWeight))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
if (L->contains(ExitingBranch->getSuccessor(1)))
|
if (L->contains(ExitingBranch->getSuccessor(1)))
|
||||||
std::swap(LoopWeight, ExitWeight);
|
std::swap(LoopWeight, ExitWeight);
|
||||||
|
|
||||||
if (!ExitWeight)
|
if (!ExitWeight)
|
||||||
// Don't have a way to return predicated infinite
|
// Don't have a way to return predicated infinite
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
OrigExitWeight = ExitWeight;
|
OrigExitWeight = ExitWeight;
|
||||||
|
|
||||||
|
@ -824,7 +824,7 @@ llvm::getLoopEstimatedTripCount(Loop *L,
|
||||||
return *EstTripCount;
|
return *EstTripCount;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool llvm::setLoopEstimatedTripCount(Loop *L, unsigned EstimatedTripCount,
|
bool llvm::setLoopEstimatedTripCount(Loop *L, unsigned EstimatedTripCount,
|
||||||
|
|
|
@ -146,7 +146,7 @@ static void inlineVolatileOrAtomicWithExtraArgs(bool *Inline, bool Volatile,
|
||||||
|
|
||||||
static Optional<uint64_t> getSizeInBytes(Optional<uint64_t> SizeInBits) {
|
static Optional<uint64_t> getSizeInBytes(Optional<uint64_t> SizeInBits) {
|
||||||
if (!SizeInBits || *SizeInBits % 8 != 0)
|
if (!SizeInBits || *SizeInBits % 8 != 0)
|
||||||
return None;
|
return std::nullopt;
|
||||||
return *SizeInBits / 8;
|
return *SizeInBits / 8;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -300,7 +300,7 @@ void MemoryOpRemark::visitSizeOperand(Value *V, DiagnosticInfoIROptimization &R)
|
||||||
static Optional<StringRef> nameOrNone(const Value *V) {
|
static Optional<StringRef> nameOrNone(const Value *V) {
|
||||||
if (V->hasName())
|
if (V->hasName())
|
||||||
return V->getName();
|
return V->getName();
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
void MemoryOpRemark::visitVariable(const Value *V,
|
void MemoryOpRemark::visitVariable(const Value *V,
|
||||||
|
@ -341,7 +341,7 @@ void MemoryOpRemark::visitVariable(const Value *V,
|
||||||
// If not, get it from the alloca.
|
// If not, get it from the alloca.
|
||||||
Optional<TypeSize> TySize = AI->getAllocationSizeInBits(DL);
|
Optional<TypeSize> TySize = AI->getAllocationSizeInBits(DL);
|
||||||
Optional<uint64_t> Size =
|
Optional<uint64_t> Size =
|
||||||
TySize ? getSizeInBytes(TySize->getFixedSize()) : None;
|
TySize ? getSizeInBytes(TySize->getFixedSize()) : std::nullopt;
|
||||||
VariableInfo Var{nameOrNone(AI), Size};
|
VariableInfo Var{nameOrNone(AI), Size};
|
||||||
if (!Var.isEmpty())
|
if (!Var.isEmpty())
|
||||||
Result.push_back(std::move(Var));
|
Result.push_back(std::move(Var));
|
||||||
|
@ -361,7 +361,7 @@ void MemoryOpRemark::visitPtr(Value *Ptr, bool IsRead, DiagnosticInfoIROptimizat
|
||||||
uint64_t Size = Ptr->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
|
uint64_t Size = Ptr->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
|
||||||
if (!Size)
|
if (!Size)
|
||||||
return;
|
return;
|
||||||
VIs.push_back({None, Size});
|
VIs.push_back({std::nullopt, Size});
|
||||||
}
|
}
|
||||||
|
|
||||||
R << (IsRead ? "\n Read Variables: " : "\n Written Variables: ");
|
R << (IsRead ? "\n Read Variables: " : "\n Written Variables: ");
|
||||||
|
|
|
@ -800,7 +800,7 @@ Optional<PredicateConstraint> PredicateBase::getConstraint() const {
|
||||||
CmpInst *Cmp = dyn_cast<CmpInst>(Condition);
|
CmpInst *Cmp = dyn_cast<CmpInst>(Condition);
|
||||||
if (!Cmp) {
|
if (!Cmp) {
|
||||||
// TODO: Make this an assertion once RenamedOp is fully accurate.
|
// TODO: Make this an assertion once RenamedOp is fully accurate.
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
CmpInst::Predicate Pred;
|
CmpInst::Predicate Pred;
|
||||||
|
@ -813,7 +813,7 @@ Optional<PredicateConstraint> PredicateBase::getConstraint() const {
|
||||||
OtherOp = Cmp->getOperand(0);
|
OtherOp = Cmp->getOperand(0);
|
||||||
} else {
|
} else {
|
||||||
// TODO: Make this an assertion once RenamedOp is fully accurate.
|
// TODO: Make this an assertion once RenamedOp is fully accurate.
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Invert predicate along false edge.
|
// Invert predicate along false edge.
|
||||||
|
@ -825,7 +825,7 @@ Optional<PredicateConstraint> PredicateBase::getConstraint() const {
|
||||||
case PT_Switch:
|
case PT_Switch:
|
||||||
if (Condition != RenamedOp) {
|
if (Condition != RenamedOp) {
|
||||||
// TODO: Make this an assertion once RenamedOp is fully accurate.
|
// TODO: Make this an assertion once RenamedOp is fully accurate.
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
return {{CmpInst::ICMP_EQ, cast<PredicateSwitch>(this)->CaseValue}};
|
return {{CmpInst::ICMP_EQ, cast<PredicateSwitch>(this)->CaseValue}};
|
||||||
|
|
|
@ -3267,7 +3267,7 @@ FoldCondBranchOnValueKnownInPredecessorImpl(BranchInst *BI, DomTreeUpdater *DTU,
|
||||||
MergeBlockIntoPredecessor(EdgeBB, DTU);
|
MergeBlockIntoPredecessor(EdgeBB, DTU);
|
||||||
|
|
||||||
// Signal repeat, simplifying any other constants.
|
// Signal repeat, simplifying any other constants.
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
|
@ -3282,8 +3282,8 @@ static bool FoldCondBranchOnValueKnownInPredecessor(BranchInst *BI,
|
||||||
do {
|
do {
|
||||||
// Note that None means "we changed things, but recurse further."
|
// Note that None means "we changed things, but recurse further."
|
||||||
Result = FoldCondBranchOnValueKnownInPredecessorImpl(BI, DTU, DL, AC);
|
Result = FoldCondBranchOnValueKnownInPredecessorImpl(BI, DTU, DL, AC);
|
||||||
EverChanged |= Result == None || *Result;
|
EverChanged |= Result == std::nullopt || *Result;
|
||||||
} while (Result == None);
|
} while (Result == std::nullopt);
|
||||||
return EverChanged;
|
return EverChanged;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3558,7 +3558,7 @@ shouldFoldCondBranchesToCommonDestination(BranchInst *BI, BranchInst *PBI,
|
||||||
if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
|
if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
|
||||||
return {{Instruction::Or, true}};
|
return {{Instruction::Or, true}};
|
||||||
}
|
}
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
|
static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
|
||||||
|
|
|
@ -1042,9 +1042,8 @@ class WidenIV {
|
||||||
Instruction *UseI) {
|
Instruction *UseI) {
|
||||||
DefUserPair Key(Def, UseI);
|
DefUserPair Key(Def, UseI);
|
||||||
auto It = PostIncRangeInfos.find(Key);
|
auto It = PostIncRangeInfos.find(Key);
|
||||||
return It == PostIncRangeInfos.end()
|
return It == PostIncRangeInfos.end() ? Optional<ConstantRange>(std::nullopt)
|
||||||
? Optional<ConstantRange>(None)
|
: Optional<ConstantRange>(It->second);
|
||||||
: Optional<ConstantRange>(It->second);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void calculatePostIncRanges(PHINode *OrigPhi);
|
void calculatePostIncRanges(PHINode *OrigPhi);
|
||||||
|
|
|
@ -3855,7 +3855,7 @@ Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
|
||||||
// st[rp]cpy_chk call which may fail at runtime if the size is too long.
|
// st[rp]cpy_chk call which may fail at runtime if the size is too long.
|
||||||
// TODO: It might be nice to get a maximum length out of the possible
|
// TODO: It might be nice to get a maximum length out of the possible
|
||||||
// string lengths for varying.
|
// string lengths for varying.
|
||||||
if (isFortifiedCallFoldable(CI, 2, None, 1)) {
|
if (isFortifiedCallFoldable(CI, 2, std::nullopt, 1)) {
|
||||||
if (Func == LibFunc_strcpy_chk)
|
if (Func == LibFunc_strcpy_chk)
|
||||||
return copyFlags(*CI, emitStrCpy(Dst, Src, B, TLI));
|
return copyFlags(*CI, emitStrCpy(Dst, Src, B, TLI));
|
||||||
else
|
else
|
||||||
|
@ -3886,7 +3886,7 @@ Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
|
||||||
|
|
||||||
Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
|
Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
|
||||||
IRBuilderBase &B) {
|
IRBuilderBase &B) {
|
||||||
if (isFortifiedCallFoldable(CI, 1, None, 0))
|
if (isFortifiedCallFoldable(CI, 1, std::nullopt, 0))
|
||||||
return copyFlags(*CI, emitStrLen(CI->getArgOperand(0), B,
|
return copyFlags(*CI, emitStrLen(CI->getArgOperand(0), B,
|
||||||
CI->getModule()->getDataLayout(), TLI));
|
CI->getModule()->getDataLayout(), TLI));
|
||||||
return nullptr;
|
return nullptr;
|
||||||
|
@ -3921,7 +3921,7 @@ Value *FortifiedLibCallSimplifier::optimizeMemCCpyChk(CallInst *CI,
|
||||||
|
|
||||||
Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
|
Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
|
||||||
IRBuilderBase &B) {
|
IRBuilderBase &B) {
|
||||||
if (isFortifiedCallFoldable(CI, 3, 1, None, 2)) {
|
if (isFortifiedCallFoldable(CI, 3, 1, std::nullopt, 2)) {
|
||||||
SmallVector<Value *, 8> VariadicArgs(drop_begin(CI->args(), 5));
|
SmallVector<Value *, 8> VariadicArgs(drop_begin(CI->args(), 5));
|
||||||
return copyFlags(*CI,
|
return copyFlags(*CI,
|
||||||
emitSNPrintf(CI->getArgOperand(0), CI->getArgOperand(1),
|
emitSNPrintf(CI->getArgOperand(0), CI->getArgOperand(1),
|
||||||
|
@ -3933,7 +3933,7 @@ Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
|
||||||
|
|
||||||
Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
|
Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
|
||||||
IRBuilderBase &B) {
|
IRBuilderBase &B) {
|
||||||
if (isFortifiedCallFoldable(CI, 2, None, None, 1)) {
|
if (isFortifiedCallFoldable(CI, 2, std::nullopt, std::nullopt, 1)) {
|
||||||
SmallVector<Value *, 8> VariadicArgs(drop_begin(CI->args(), 4));
|
SmallVector<Value *, 8> VariadicArgs(drop_begin(CI->args(), 4));
|
||||||
return copyFlags(*CI,
|
return copyFlags(*CI,
|
||||||
emitSPrintf(CI->getArgOperand(0), CI->getArgOperand(3),
|
emitSPrintf(CI->getArgOperand(0), CI->getArgOperand(3),
|
||||||
|
@ -3984,7 +3984,7 @@ Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
|
||||||
|
|
||||||
Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
|
Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
|
||||||
IRBuilderBase &B) {
|
IRBuilderBase &B) {
|
||||||
if (isFortifiedCallFoldable(CI, 3, 1, None, 2))
|
if (isFortifiedCallFoldable(CI, 3, 1, std::nullopt, 2))
|
||||||
return copyFlags(
|
return copyFlags(
|
||||||
*CI, emitVSNPrintf(CI->getArgOperand(0), CI->getArgOperand(1),
|
*CI, emitVSNPrintf(CI->getArgOperand(0), CI->getArgOperand(1),
|
||||||
CI->getArgOperand(4), CI->getArgOperand(5), B, TLI));
|
CI->getArgOperand(4), CI->getArgOperand(5), B, TLI));
|
||||||
|
@ -3994,7 +3994,7 @@ Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
|
||||||
|
|
||||||
Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
|
Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
|
||||||
IRBuilderBase &B) {
|
IRBuilderBase &B) {
|
||||||
if (isFortifiedCallFoldable(CI, 2, None, None, 1))
|
if (isFortifiedCallFoldable(CI, 2, std::nullopt, std::nullopt, 1))
|
||||||
return copyFlags(*CI,
|
return copyFlags(*CI,
|
||||||
emitVSPrintf(CI->getArgOperand(0), CI->getArgOperand(3),
|
emitVSPrintf(CI->getArgOperand(0), CI->getArgOperand(3),
|
||||||
CI->getArgOperand(4), B, TLI));
|
CI->getArgOperand(4), B, TLI));
|
||||||
|
|
|
@ -391,8 +391,9 @@ Value *Mapper::mapValue(const Value *V) {
|
||||||
// ensures metadata operands only reference defined SSA values.
|
// ensures metadata operands only reference defined SSA values.
|
||||||
return (Flags & RF_IgnoreMissingLocals)
|
return (Flags & RF_IgnoreMissingLocals)
|
||||||
? nullptr
|
? nullptr
|
||||||
: MetadataAsValue::get(V->getContext(),
|
: MetadataAsValue::get(
|
||||||
MDTuple::get(V->getContext(), None));
|
V->getContext(),
|
||||||
|
MDTuple::get(V->getContext(), std::nullopt));
|
||||||
}
|
}
|
||||||
if (auto *AL = dyn_cast<DIArgList>(MD)) {
|
if (auto *AL = dyn_cast<DIArgList>(MD)) {
|
||||||
SmallVector<ValueAsMetadata *, 4> MappedArgs;
|
SmallVector<ValueAsMetadata *, 4> MappedArgs;
|
||||||
|
@ -578,7 +579,7 @@ Optional<Metadata *> MDNodeMapper::tryToMapOperand(const Metadata *Op) {
|
||||||
const MDNode &N = *cast<MDNode>(Op);
|
const MDNode &N = *cast<MDNode>(Op);
|
||||||
if (N.isDistinct())
|
if (N.isDistinct())
|
||||||
return mapDistinctNode(N);
|
return mapDistinctNode(N);
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
MDNode *MDNodeMapper::mapDistinctNode(const MDNode &N) {
|
MDNode *MDNodeMapper::mapDistinctNode(const MDNode &N) {
|
||||||
|
@ -619,7 +620,7 @@ Optional<Metadata *> MDNodeMapper::getMappedOp(const Metadata *Op) const {
|
||||||
if (auto *CMD = dyn_cast<ConstantAsMetadata>(Op))
|
if (auto *CMD = dyn_cast<ConstantAsMetadata>(Op))
|
||||||
return wrapConstantAsMetadata(*CMD, M.getVM().lookup(CMD->getValue()));
|
return wrapConstantAsMetadata(*CMD, M.getVM().lookup(CMD->getValue()));
|
||||||
|
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
Metadata &MDNodeMapper::UniquedGraph::getFwdReference(MDNode &Op) {
|
Metadata &MDNodeMapper::UniquedGraph::getFwdReference(MDNode &Op) {
|
||||||
|
@ -848,7 +849,7 @@ Optional<Metadata *> Mapper::mapSimpleMetadata(const Metadata *MD) {
|
||||||
|
|
||||||
assert(isa<MDNode>(MD) && "Expected a metadata node");
|
assert(isa<MDNode>(MD) && "Expected a metadata node");
|
||||||
|
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
Metadata *Mapper::mapMetadata(const Metadata *MD) {
|
Metadata *Mapper::mapMetadata(const Metadata *MD) {
|
||||||
|
|
|
@ -417,7 +417,7 @@ static Optional<unsigned> getSmallBestKnownTC(ScalarEvolution &SE, Loop *L) {
|
||||||
if (unsigned ExpectedTC = SE.getSmallConstantMaxTripCount(L))
|
if (unsigned ExpectedTC = SE.getSmallConstantMaxTripCount(L))
|
||||||
return ExpectedTC;
|
return ExpectedTC;
|
||||||
|
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Forward declare GeneratedRTChecks.
|
// Forward declare GeneratedRTChecks.
|
||||||
|
@ -6452,8 +6452,8 @@ LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
|
||||||
|
|
||||||
bool Reverse = ConsecutiveStride < 0;
|
bool Reverse = ConsecutiveStride < 0;
|
||||||
if (Reverse)
|
if (Reverse)
|
||||||
Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, None,
|
Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy,
|
||||||
CostKind, 0);
|
std::nullopt, CostKind, 0);
|
||||||
return Cost;
|
return Cost;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -6537,8 +6537,8 @@ LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
|
||||||
assert(!Legal->isMaskRequired(I) &&
|
assert(!Legal->isMaskRequired(I) &&
|
||||||
"Reverse masked interleaved access not supported.");
|
"Reverse masked interleaved access not supported.");
|
||||||
Cost += Group->getNumMembers() *
|
Cost += Group->getNumMembers() *
|
||||||
TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, None,
|
TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy,
|
||||||
CostKind, 0);
|
std::nullopt, CostKind, 0);
|
||||||
}
|
}
|
||||||
return Cost;
|
return Cost;
|
||||||
}
|
}
|
||||||
|
@ -6548,7 +6548,7 @@ Optional<InstructionCost> LoopVectorizationCostModel::getReductionPatternCost(
|
||||||
using namespace llvm::PatternMatch;
|
using namespace llvm::PatternMatch;
|
||||||
// Early exit for no inloop reductions
|
// Early exit for no inloop reductions
|
||||||
if (InLoopReductionChains.empty() || VF.isScalar() || !isa<VectorType>(Ty))
|
if (InLoopReductionChains.empty() || VF.isScalar() || !isa<VectorType>(Ty))
|
||||||
return None;
|
return std::nullopt;
|
||||||
auto *VectorTy = cast<VectorType>(Ty);
|
auto *VectorTy = cast<VectorType>(Ty);
|
||||||
|
|
||||||
// We are looking for a pattern of, and finding the minimal acceptable cost:
|
// We are looking for a pattern of, and finding the minimal acceptable cost:
|
||||||
|
@ -6566,7 +6566,7 @@ Optional<InstructionCost> LoopVectorizationCostModel::getReductionPatternCost(
|
||||||
Instruction *RetI = I;
|
Instruction *RetI = I;
|
||||||
if (match(RetI, m_ZExtOrSExt(m_Value()))) {
|
if (match(RetI, m_ZExtOrSExt(m_Value()))) {
|
||||||
if (!RetI->hasOneUser())
|
if (!RetI->hasOneUser())
|
||||||
return None;
|
return std::nullopt;
|
||||||
RetI = RetI->user_back();
|
RetI = RetI->user_back();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -6578,7 +6578,7 @@ Optional<InstructionCost> LoopVectorizationCostModel::getReductionPatternCost(
|
||||||
// Test if the found instruction is a reduction, and if not return an invalid
|
// Test if the found instruction is a reduction, and if not return an invalid
|
||||||
// cost specifying the parent to use the original cost modelling.
|
// cost specifying the parent to use the original cost modelling.
|
||||||
if (!InLoopReductionImmediateChains.count(RetI))
|
if (!InLoopReductionImmediateChains.count(RetI))
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Find the reduction this chain is a part of and calculate the basic cost of
|
// Find the reduction this chain is a part of and calculate the basic cost of
|
||||||
// the reduction on its own.
|
// the reduction on its own.
|
||||||
|
@ -6712,7 +6712,7 @@ Optional<InstructionCost> LoopVectorizationCostModel::getReductionPatternCost(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return I == RetI ? Optional<InstructionCost>(BaseCost) : None;
|
return I == RetI ? Optional<InstructionCost>(BaseCost) : std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
InstructionCost
|
InstructionCost
|
||||||
|
@ -7498,7 +7498,7 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
|
||||||
assert(OrigLoop->isInnermost() && "Inner loop expected.");
|
assert(OrigLoop->isInnermost() && "Inner loop expected.");
|
||||||
FixedScalableVFPair MaxFactors = CM.computeMaxVF(UserVF, UserIC);
|
FixedScalableVFPair MaxFactors = CM.computeMaxVF(UserVF, UserIC);
|
||||||
if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
|
if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// Invalidate interleave groups if all blocks of loop will be predicated.
|
// Invalidate interleave groups if all blocks of loop will be predicated.
|
||||||
if (CM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
|
if (CM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
|
||||||
|
|
|
@ -293,12 +293,12 @@ static Optional<unsigned> getInsertIndex(const Value *InsertInst,
|
||||||
if (const auto *IE = dyn_cast<InsertElementInst>(InsertInst)) {
|
if (const auto *IE = dyn_cast<InsertElementInst>(InsertInst)) {
|
||||||
const auto *VT = dyn_cast<FixedVectorType>(IE->getType());
|
const auto *VT = dyn_cast<FixedVectorType>(IE->getType());
|
||||||
if (!VT)
|
if (!VT)
|
||||||
return None;
|
return std::nullopt;
|
||||||
const auto *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
|
const auto *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
|
||||||
if (!CI)
|
if (!CI)
|
||||||
return None;
|
return std::nullopt;
|
||||||
if (CI->getValue().uge(VT->getNumElements()))
|
if (CI->getValue().uge(VT->getNumElements()))
|
||||||
return None;
|
return std::nullopt;
|
||||||
Index *= VT->getNumElements();
|
Index *= VT->getNumElements();
|
||||||
Index += CI->getZExtValue();
|
Index += CI->getZExtValue();
|
||||||
return Index;
|
return Index;
|
||||||
|
@ -314,7 +314,7 @@ static Optional<unsigned> getInsertIndex(const Value *InsertInst,
|
||||||
Index *= AT->getNumElements();
|
Index *= AT->getNumElements();
|
||||||
CurrentType = AT->getElementType();
|
CurrentType = AT->getElementType();
|
||||||
} else {
|
} else {
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
Index += I;
|
Index += I;
|
||||||
}
|
}
|
||||||
|
@ -326,7 +326,7 @@ static Optional<unsigned> getInsertIndex(const Value *InsertInst,
|
||||||
/// elements actually mask the insertelement buildvector, if any.
|
/// elements actually mask the insertelement buildvector, if any.
|
||||||
template <bool IsPoisonOnly = false>
|
template <bool IsPoisonOnly = false>
|
||||||
static SmallBitVector isUndefVector(const Value *V,
|
static SmallBitVector isUndefVector(const Value *V,
|
||||||
ArrayRef<int> ShuffleMask = None) {
|
ArrayRef<int> ShuffleMask = std::nullopt) {
|
||||||
SmallBitVector Res(ShuffleMask.empty() ? 1 : ShuffleMask.size(), true);
|
SmallBitVector Res(ShuffleMask.empty() ? 1 : ShuffleMask.size(), true);
|
||||||
using T = std::conditional_t<IsPoisonOnly, PoisonValue, UndefValue>;
|
using T = std::conditional_t<IsPoisonOnly, PoisonValue, UndefValue>;
|
||||||
if (isa<T>(V))
|
if (isa<T>(V))
|
||||||
|
@ -417,10 +417,10 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask) {
|
||||||
const auto *It =
|
const auto *It =
|
||||||
find_if(VL, [](Value *V) { return isa<ExtractElementInst>(V); });
|
find_if(VL, [](Value *V) { return isa<ExtractElementInst>(V); });
|
||||||
if (It == VL.end())
|
if (It == VL.end())
|
||||||
return None;
|
return std::nullopt;
|
||||||
auto *EI0 = cast<ExtractElementInst>(*It);
|
auto *EI0 = cast<ExtractElementInst>(*It);
|
||||||
if (isa<ScalableVectorType>(EI0->getVectorOperandType()))
|
if (isa<ScalableVectorType>(EI0->getVectorOperandType()))
|
||||||
return None;
|
return std::nullopt;
|
||||||
unsigned Size =
|
unsigned Size =
|
||||||
cast<FixedVectorType>(EI0->getVectorOperandType())->getNumElements();
|
cast<FixedVectorType>(EI0->getVectorOperandType())->getNumElements();
|
||||||
Value *Vec1 = nullptr;
|
Value *Vec1 = nullptr;
|
||||||
|
@ -434,19 +434,19 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask) {
|
||||||
continue;
|
continue;
|
||||||
auto *EI = cast<ExtractElementInst>(VL[I]);
|
auto *EI = cast<ExtractElementInst>(VL[I]);
|
||||||
if (isa<ScalableVectorType>(EI->getVectorOperandType()))
|
if (isa<ScalableVectorType>(EI->getVectorOperandType()))
|
||||||
return None;
|
return std::nullopt;
|
||||||
auto *Vec = EI->getVectorOperand();
|
auto *Vec = EI->getVectorOperand();
|
||||||
// We can extractelement from undef or poison vector.
|
// We can extractelement from undef or poison vector.
|
||||||
if (isUndefVector(Vec).all())
|
if (isUndefVector(Vec).all())
|
||||||
continue;
|
continue;
|
||||||
// All vector operands must have the same number of vector elements.
|
// All vector operands must have the same number of vector elements.
|
||||||
if (cast<FixedVectorType>(Vec->getType())->getNumElements() != Size)
|
if (cast<FixedVectorType>(Vec->getType())->getNumElements() != Size)
|
||||||
return None;
|
return std::nullopt;
|
||||||
if (isa<UndefValue>(EI->getIndexOperand()))
|
if (isa<UndefValue>(EI->getIndexOperand()))
|
||||||
continue;
|
continue;
|
||||||
auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
|
auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
|
||||||
if (!Idx)
|
if (!Idx)
|
||||||
return None;
|
return std::nullopt;
|
||||||
// Undefined behavior if Idx is negative or >= Size.
|
// Undefined behavior if Idx is negative or >= Size.
|
||||||
if (Idx->getValue().uge(Size))
|
if (Idx->getValue().uge(Size))
|
||||||
continue;
|
continue;
|
||||||
|
@ -460,7 +460,7 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask) {
|
||||||
Vec2 = Vec;
|
Vec2 = Vec;
|
||||||
Mask[I] += Size;
|
Mask[I] += Size;
|
||||||
} else {
|
} else {
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
if (CommonShuffleMode == Permute)
|
if (CommonShuffleMode == Permute)
|
||||||
continue;
|
continue;
|
||||||
|
@ -737,12 +737,12 @@ static Optional<unsigned> getExtractIndex(Instruction *E) {
|
||||||
if (Opcode == Instruction::ExtractElement) {
|
if (Opcode == Instruction::ExtractElement) {
|
||||||
auto *CI = dyn_cast<ConstantInt>(E->getOperand(1));
|
auto *CI = dyn_cast<ConstantInt>(E->getOperand(1));
|
||||||
if (!CI)
|
if (!CI)
|
||||||
return None;
|
return std::nullopt;
|
||||||
return CI->getZExtValue();
|
return CI->getZExtValue();
|
||||||
}
|
}
|
||||||
ExtractValueInst *EI = cast<ExtractValueInst>(E);
|
ExtractValueInst *EI = cast<ExtractValueInst>(E);
|
||||||
if (EI->getNumIndices() != 1)
|
if (EI->getNumIndices() != 1)
|
||||||
return None;
|
return std::nullopt;
|
||||||
return *EI->idx_begin();
|
return *EI->idx_begin();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -980,7 +980,7 @@ public:
|
||||||
|
|
||||||
/// \returns the vectorization cost of the subtree that starts at \p VL.
|
/// \returns the vectorization cost of the subtree that starts at \p VL.
|
||||||
/// A negative number means that this is profitable.
|
/// A negative number means that this is profitable.
|
||||||
InstructionCost getTreeCost(ArrayRef<Value *> VectorizedVals = None);
|
InstructionCost getTreeCost(ArrayRef<Value *> VectorizedVals = std::nullopt);
|
||||||
|
|
||||||
/// Construct a vectorizable tree that starts at \p Roots, ignoring users for
|
/// Construct a vectorizable tree that starts at \p Roots, ignoring users for
|
||||||
/// the purpose of scheduling and extraction in the \p UserIgnoreLst.
|
/// the purpose of scheduling and extraction in the \p UserIgnoreLst.
|
||||||
|
@ -1420,7 +1420,7 @@ public:
|
||||||
// Recursively calculate the cost at each level
|
// Recursively calculate the cost at each level
|
||||||
int TmpScore =
|
int TmpScore =
|
||||||
getScoreAtLevelRec(I1->getOperand(OpIdx1), I2->getOperand(OpIdx2),
|
getScoreAtLevelRec(I1->getOperand(OpIdx1), I2->getOperand(OpIdx2),
|
||||||
I1, I2, CurrLevel + 1, None);
|
I1, I2, CurrLevel + 1, std::nullopt);
|
||||||
// Look for the best score.
|
// Look for the best score.
|
||||||
if (TmpScore > LookAheadHeuristics::ScoreFail &&
|
if (TmpScore > LookAheadHeuristics::ScoreFail &&
|
||||||
TmpScore > MaxTmpScore) {
|
TmpScore > MaxTmpScore) {
|
||||||
|
@ -1585,7 +1585,7 @@ public:
|
||||||
auto *IdxLaneI = dyn_cast<Instruction>(IdxLaneV);
|
auto *IdxLaneI = dyn_cast<Instruction>(IdxLaneV);
|
||||||
if (!IdxLaneI || !isa<Instruction>(OpIdxLaneV))
|
if (!IdxLaneI || !isa<Instruction>(OpIdxLaneV))
|
||||||
return 0;
|
return 0;
|
||||||
return R.areAllUsersVectorized(IdxLaneI, None)
|
return R.areAllUsersVectorized(IdxLaneI, std::nullopt)
|
||||||
? LookAheadHeuristics::ScoreAllUserVectorized
|
? LookAheadHeuristics::ScoreAllUserVectorized
|
||||||
: 0;
|
: 0;
|
||||||
}
|
}
|
||||||
|
@ -1653,7 +1653,7 @@ public:
|
||||||
// Our strategy mode for OpIdx.
|
// Our strategy mode for OpIdx.
|
||||||
ReorderingMode RMode = ReorderingModes[OpIdx];
|
ReorderingMode RMode = ReorderingModes[OpIdx];
|
||||||
if (RMode == ReorderingMode::Failed)
|
if (RMode == ReorderingMode::Failed)
|
||||||
return None;
|
return std::nullopt;
|
||||||
|
|
||||||
// The linearized opcode of the operand at OpIdx, Lane.
|
// The linearized opcode of the operand at OpIdx, Lane.
|
||||||
bool OpIdxAPO = getData(OpIdx, Lane).APO;
|
bool OpIdxAPO = getData(OpIdx, Lane).APO;
|
||||||
|
@ -1662,7 +1662,7 @@ public:
|
||||||
// Sometimes we have more than one option (e.g., Opcode and Undefs), so we
|
// Sometimes we have more than one option (e.g., Opcode and Undefs), so we
|
||||||
// are using the score to differentiate between the two.
|
// are using the score to differentiate between the two.
|
||||||
struct BestOpData {
|
struct BestOpData {
|
||||||
Optional<unsigned> Idx = None;
|
Optional<unsigned> Idx = std::nullopt;
|
||||||
unsigned Score = 0;
|
unsigned Score = 0;
|
||||||
} BestOp;
|
} BestOp;
|
||||||
BestOp.Score =
|
BestOp.Score =
|
||||||
|
@ -1722,7 +1722,7 @@ public:
|
||||||
return BestOp.Idx;
|
return BestOp.Idx;
|
||||||
}
|
}
|
||||||
// If we could not find a good match return None.
|
// If we could not find a good match return None.
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Helper for reorderOperandVecs.
|
/// Helper for reorderOperandVecs.
|
||||||
|
@ -2153,7 +2153,7 @@ public:
|
||||||
int Score = LookAhead.getScoreAtLevelRec(Candidates[I].first,
|
int Score = LookAhead.getScoreAtLevelRec(Candidates[I].first,
|
||||||
Candidates[I].second,
|
Candidates[I].second,
|
||||||
/*U1=*/nullptr, /*U2=*/nullptr,
|
/*U1=*/nullptr, /*U2=*/nullptr,
|
||||||
/*Level=*/1, None);
|
/*Level=*/1, std::nullopt);
|
||||||
if (Score > BestScore) {
|
if (Score > BestScore) {
|
||||||
BestScore = Score;
|
BestScore = Score;
|
||||||
Index = I;
|
Index = I;
|
||||||
|
@ -2655,8 +2655,8 @@ private:
|
||||||
TreeEntry *newTreeEntry(ArrayRef<Value *> VL, Optional<ScheduleData *> Bundle,
|
TreeEntry *newTreeEntry(ArrayRef<Value *> VL, Optional<ScheduleData *> Bundle,
|
||||||
const InstructionsState &S,
|
const InstructionsState &S,
|
||||||
const EdgeInfo &UserTreeIdx,
|
const EdgeInfo &UserTreeIdx,
|
||||||
ArrayRef<int> ReuseShuffleIndices = None,
|
ArrayRef<int> ReuseShuffleIndices = std::nullopt,
|
||||||
ArrayRef<unsigned> ReorderIndices = None) {
|
ArrayRef<unsigned> ReorderIndices = std::nullopt) {
|
||||||
TreeEntry::EntryState EntryState =
|
TreeEntry::EntryState EntryState =
|
||||||
Bundle ? TreeEntry::Vectorize : TreeEntry::NeedToGather;
|
Bundle ? TreeEntry::Vectorize : TreeEntry::NeedToGather;
|
||||||
return newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
|
return newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
|
||||||
|
@ -2668,8 +2668,8 @@ private:
|
||||||
Optional<ScheduleData *> Bundle,
|
Optional<ScheduleData *> Bundle,
|
||||||
const InstructionsState &S,
|
const InstructionsState &S,
|
||||||
const EdgeInfo &UserTreeIdx,
|
const EdgeInfo &UserTreeIdx,
|
||||||
ArrayRef<int> ReuseShuffleIndices = None,
|
ArrayRef<int> ReuseShuffleIndices = std::nullopt,
|
||||||
ArrayRef<unsigned> ReorderIndices = None) {
|
ArrayRef<unsigned> ReorderIndices = std::nullopt) {
|
||||||
assert(((!Bundle && EntryState == TreeEntry::NeedToGather) ||
|
assert(((!Bundle && EntryState == TreeEntry::NeedToGather) ||
|
||||||
(Bundle && EntryState != TreeEntry::NeedToGather)) &&
|
(Bundle && EntryState != TreeEntry::NeedToGather)) &&
|
||||||
"Need to vectorize gather entry?");
|
"Need to vectorize gather entry?");
|
||||||
|
@ -3580,11 +3580,11 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) {
|
||||||
STE = LocalSTE;
|
STE = LocalSTE;
|
||||||
else if (STE != LocalSTE)
|
else if (STE != LocalSTE)
|
||||||
// Take the order only from the single vector node.
|
// Take the order only from the single vector node.
|
||||||
return None;
|
return std::nullopt;
|
||||||
unsigned Lane =
|
unsigned Lane =
|
||||||
std::distance(STE->Scalars.begin(), find(STE->Scalars, V));
|
std::distance(STE->Scalars.begin(), find(STE->Scalars, V));
|
||||||
if (Lane >= NumScalars)
|
if (Lane >= NumScalars)
|
||||||
return None;
|
return std::nullopt;
|
||||||
if (CurrentOrder[Lane] != NumScalars) {
|
if (CurrentOrder[Lane] != NumScalars) {
|
||||||
if (Lane != I)
|
if (Lane != I)
|
||||||
continue;
|
continue;
|
||||||
|
@ -3623,7 +3623,7 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) {
|
||||||
}
|
}
|
||||||
return CurrentOrder;
|
return CurrentOrder;
|
||||||
}
|
}
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
@ -3808,14 +3808,14 @@ BoUpSLP::findPartiallyOrderedLoads(const BoUpSLP::TreeEntry &TE) {
|
||||||
for (Value *V : TE.Scalars) {
|
for (Value *V : TE.Scalars) {
|
||||||
auto *L = dyn_cast<LoadInst>(V);
|
auto *L = dyn_cast<LoadInst>(V);
|
||||||
if (!L || !L->isSimple())
|
if (!L || !L->isSimple())
|
||||||
return None;
|
return std::nullopt;
|
||||||
Ptrs.push_back(L->getPointerOperand());
|
Ptrs.push_back(L->getPointerOperand());
|
||||||
}
|
}
|
||||||
|
|
||||||
BoUpSLP::OrdersType Order;
|
BoUpSLP::OrdersType Order;
|
||||||
if (clusterSortPtrAccesses(Ptrs, ScalarTy, *DL, *SE, Order))
|
if (clusterSortPtrAccesses(Ptrs, ScalarTy, *DL, *SE, Order))
|
||||||
return Order;
|
return Order;
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if two insertelement instructions are from the same buildvector.
|
/// Check if two insertelement instructions are from the same buildvector.
|
||||||
|
@ -3835,7 +3835,7 @@ static bool areTwoInsertFromSameBuildVector(
|
||||||
auto *IE2 = V;
|
auto *IE2 = V;
|
||||||
Optional<unsigned> Idx1 = getInsertIndex(IE1);
|
Optional<unsigned> Idx1 = getInsertIndex(IE1);
|
||||||
Optional<unsigned> Idx2 = getInsertIndex(IE2);
|
Optional<unsigned> Idx2 = getInsertIndex(IE2);
|
||||||
if (Idx1 == None || Idx2 == None)
|
if (Idx1 == std::nullopt || Idx2 == std::nullopt)
|
||||||
return false;
|
return false;
|
||||||
// Go through the vector operand of insertelement instructions trying to find
|
// Go through the vector operand of insertelement instructions trying to find
|
||||||
// either VU as the original vector for IE2 or V as the original vector for
|
// either VU as the original vector for IE2 or V as the original vector for
|
||||||
|
@ -3878,7 +3878,7 @@ Optional<BoUpSLP::OrdersType> BoUpSLP::getReorderingData(const TreeEntry &TE,
|
||||||
unsigned Sz = TE.Scalars.size();
|
unsigned Sz = TE.Scalars.size();
|
||||||
if (!ShuffleVectorInst::isOneUseSingleSourceMask(TE.ReuseShuffleIndices,
|
if (!ShuffleVectorInst::isOneUseSingleSourceMask(TE.ReuseShuffleIndices,
|
||||||
Sz))
|
Sz))
|
||||||
return None;
|
return std::nullopt;
|
||||||
unsigned VF = TE.getVectorFactor();
|
unsigned VF = TE.getVectorFactor();
|
||||||
// Try build correct order for extractelement instructions.
|
// Try build correct order for extractelement instructions.
|
||||||
SmallVector<int> ReusedMask(TE.ReuseShuffleIndices.begin(),
|
SmallVector<int> ReusedMask(TE.ReuseShuffleIndices.begin(),
|
||||||
|
@ -3940,7 +3940,7 @@ Optional<BoUpSLP::OrdersType> BoUpSLP::getReorderingData(const TreeEntry &TE,
|
||||||
return false;
|
return false;
|
||||||
Optional<unsigned> Idx1 = getInsertIndex(IE1);
|
Optional<unsigned> Idx1 = getInsertIndex(IE1);
|
||||||
Optional<unsigned> Idx2 = getInsertIndex(IE2);
|
Optional<unsigned> Idx2 = getInsertIndex(IE2);
|
||||||
if (Idx1 == None || Idx2 == None)
|
if (Idx1 == std::nullopt || Idx2 == std::nullopt)
|
||||||
return false;
|
return false;
|
||||||
return *Idx1 < *Idx2;
|
return *Idx1 < *Idx2;
|
||||||
}
|
}
|
||||||
|
@ -3950,7 +3950,7 @@ Optional<BoUpSLP::OrdersType> BoUpSLP::getReorderingData(const TreeEntry &TE,
|
||||||
return false;
|
return false;
|
||||||
Optional<unsigned> Idx1 = getExtractIndex(EE1);
|
Optional<unsigned> Idx1 = getExtractIndex(EE1);
|
||||||
Optional<unsigned> Idx2 = getExtractIndex(EE2);
|
Optional<unsigned> Idx2 = getExtractIndex(EE2);
|
||||||
if (Idx1 == None || Idx2 == None)
|
if (Idx1 == std::nullopt || Idx2 == std::nullopt)
|
||||||
return false;
|
return false;
|
||||||
return *Idx1 < *Idx2;
|
return *Idx1 < *Idx2;
|
||||||
}
|
}
|
||||||
|
@ -4011,7 +4011,7 @@ Optional<BoUpSLP::OrdersType> BoUpSLP::getReorderingData(const TreeEntry &TE,
|
||||||
if (Optional<OrdersType> Order = findPartiallyOrderedLoads(TE))
|
if (Optional<OrdersType> Order = findPartiallyOrderedLoads(TE))
|
||||||
return Order;
|
return Order;
|
||||||
}
|
}
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Checks if the given mask is a "clustered" mask with the same clusters of
|
/// Checks if the given mask is a "clustered" mask with the same clusters of
|
||||||
|
@ -4961,7 +4961,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
})) ||
|
})) ||
|
||||||
!llvm::isPowerOf2_32(NumUniqueScalarValues)) {
|
!llvm::isPowerOf2_32(NumUniqueScalarValues)) {
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
VL = UniqueValues;
|
VL = UniqueValues;
|
||||||
|
@ -4985,7 +4985,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
})))) {
|
})))) {
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
|
||||||
if (TryToFindDuplicates(S))
|
if (TryToFindDuplicates(S))
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -4996,7 +4996,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
cast<ExtractElementInst>(S.OpValue)->getVectorOperandType())) {
|
cast<ExtractElementInst>(S.OpValue)->getVectorOperandType())) {
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
|
||||||
if (TryToFindDuplicates(S))
|
if (TryToFindDuplicates(S))
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -5005,14 +5005,14 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
if (S.OpValue->getType()->isVectorTy() &&
|
if (S.OpValue->getType()->isVectorTy() &&
|
||||||
!isa<InsertElementInst>(S.OpValue)) {
|
!isa<InsertElementInst>(S.OpValue)) {
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
|
if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
|
||||||
if (SI->getValueOperand()->getType()->isVectorTy()) {
|
if (SI->getValueOperand()->getType()->isVectorTy()) {
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n");
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5100,7 +5100,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
NotProfitableForVectorization(VL)) {
|
NotProfitableForVectorization(VL)) {
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O, small shuffle. \n");
|
LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O, small shuffle. \n");
|
||||||
if (TryToFindDuplicates(S))
|
if (TryToFindDuplicates(S))
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -5114,7 +5114,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
if (EphValues.count(V)) {
|
if (EphValues.count(V)) {
|
||||||
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
|
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
|
||||||
<< ") is ephemeral.\n");
|
<< ") is ephemeral.\n");
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -5126,7 +5126,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
if (!E->isSame(VL)) {
|
if (!E->isSame(VL)) {
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
|
||||||
if (TryToFindDuplicates(S))
|
if (TryToFindDuplicates(S))
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -5146,7 +5146,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
|
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
|
||||||
<< ") is already in tree.\n");
|
<< ") is already in tree.\n");
|
||||||
if (TryToFindDuplicates(S))
|
if (TryToFindDuplicates(S))
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -5158,7 +5158,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
if (UserIgnoreList && UserIgnoreList->contains(V)) {
|
if (UserIgnoreList && UserIgnoreList->contains(V)) {
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
|
||||||
if (TryToFindDuplicates(S))
|
if (TryToFindDuplicates(S))
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -5189,7 +5189,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
// Don't go into unreachable blocks. They may contain instructions with
|
// Don't go into unreachable blocks. They may contain instructions with
|
||||||
// dependency cycles which confuse the final scheduling.
|
// dependency cycles which confuse the final scheduling.
|
||||||
LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
|
LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5198,7 +5198,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
// place to insert a shuffle if we need to, so just avoid that issue.
|
// place to insert a shuffle if we need to, so just avoid that issue.
|
||||||
if (isa<CatchSwitchInst>(BB->getTerminator())) {
|
if (isa<CatchSwitchInst>(BB->getTerminator())) {
|
||||||
LLVM_DEBUG(dbgs() << "SLP: bundle in catchswitch block.\n");
|
LLVM_DEBUG(dbgs() << "SLP: bundle in catchswitch block.\n");
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5222,7 +5222,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
assert((!BS.getScheduleData(VL0) ||
|
assert((!BS.getScheduleData(VL0) ||
|
||||||
!BS.getScheduleData(VL0)->isPartOfBundle()) &&
|
!BS.getScheduleData(VL0)->isPartOfBundle()) &&
|
||||||
"tryScheduleBundle should cancelScheduling on failure");
|
"tryScheduleBundle should cancelScheduling on failure");
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -5242,7 +5242,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
LLVM_DEBUG(dbgs()
|
LLVM_DEBUG(dbgs()
|
||||||
<< "SLP: Need to swizzle PHINodes (terminator use).\n");
|
<< "SLP: Need to swizzle PHINodes (terminator use).\n");
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -5309,7 +5309,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n");
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
return;
|
return;
|
||||||
|
@ -5322,7 +5322,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
ValueSet SourceVectors;
|
ValueSet SourceVectors;
|
||||||
for (Value *V : VL) {
|
for (Value *V : VL) {
|
||||||
SourceVectors.insert(cast<Instruction>(V)->getOperand(0));
|
SourceVectors.insert(cast<Instruction>(V)->getOperand(0));
|
||||||
assert(getInsertIndex(V) != None && "Non-constant or undef index?");
|
assert(getInsertIndex(V) != std::nullopt &&
|
||||||
|
"Non-constant or undef index?");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (count_if(VL, [&SourceVectors](Value *V) {
|
if (count_if(VL, [&SourceVectors](Value *V) {
|
||||||
|
@ -5331,7 +5332,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
// Found 2nd source vector - cancel.
|
// Found 2nd source vector - cancel.
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Gather of insertelement vectors with "
|
LLVM_DEBUG(dbgs() << "SLP: Gather of insertelement vectors with "
|
||||||
"different source vectors.\n");
|
"different source vectors.\n");
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -5357,7 +5358,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
if (IsIdentity)
|
if (IsIdentity)
|
||||||
CurrentOrder.clear();
|
CurrentOrder.clear();
|
||||||
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
|
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
|
||||||
None, CurrentOrder);
|
std::nullopt, CurrentOrder);
|
||||||
LLVM_DEBUG(dbgs() << "SLP: added inserts bundle.\n");
|
LLVM_DEBUG(dbgs() << "SLP: added inserts bundle.\n");
|
||||||
|
|
||||||
constexpr int NumOps = 2;
|
constexpr int NumOps = 2;
|
||||||
|
@ -5408,7 +5409,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
break;
|
break;
|
||||||
case LoadsState::Gather:
|
case LoadsState::Gather:
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
#ifndef NDEBUG
|
#ifndef NDEBUG
|
||||||
Type *ScalarTy = VL0->getType();
|
Type *ScalarTy = VL0->getType();
|
||||||
|
@ -5443,7 +5444,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
Type *Ty = cast<Instruction>(V)->getOperand(0)->getType();
|
Type *Ty = cast<Instruction>(V)->getOperand(0)->getType();
|
||||||
if (Ty != SrcTy || !isValidElementType(Ty)) {
|
if (Ty != SrcTy || !isValidElementType(Ty)) {
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
LLVM_DEBUG(dbgs()
|
LLVM_DEBUG(dbgs()
|
||||||
<< "SLP: Gathering casts with different src types.\n");
|
<< "SLP: Gathering casts with different src types.\n");
|
||||||
|
@ -5476,7 +5477,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
if ((Cmp->getPredicate() != P0 && Cmp->getPredicate() != SwapP0) ||
|
if ((Cmp->getPredicate() != P0 && Cmp->getPredicate() != SwapP0) ||
|
||||||
Cmp->getOperand(0)->getType() != ComparedTy) {
|
Cmp->getOperand(0)->getType() != ComparedTy) {
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
LLVM_DEBUG(dbgs()
|
LLVM_DEBUG(dbgs()
|
||||||
<< "SLP: Gathering cmp with different predicate.\n");
|
<< "SLP: Gathering cmp with different predicate.\n");
|
||||||
|
@ -5568,7 +5569,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
if (I->getNumOperands() != 2) {
|
if (I->getNumOperands() != 2) {
|
||||||
LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
|
LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -5586,7 +5587,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
LLVM_DEBUG(dbgs()
|
LLVM_DEBUG(dbgs()
|
||||||
<< "SLP: not-vectorizable GEP (different types).\n");
|
<< "SLP: not-vectorizable GEP (different types).\n");
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -5608,7 +5609,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
LLVM_DEBUG(dbgs()
|
LLVM_DEBUG(dbgs()
|
||||||
<< "SLP: not-vectorizable GEP (non-constant indexes).\n");
|
<< "SLP: not-vectorizable GEP (non-constant indexes).\n");
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -5676,7 +5677,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
if (DL->getTypeSizeInBits(ScalarTy) !=
|
if (DL->getTypeSizeInBits(ScalarTy) !=
|
||||||
DL->getTypeAllocSizeInBits(ScalarTy)) {
|
DL->getTypeAllocSizeInBits(ScalarTy)) {
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Gathering stores of non-packed type.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Gathering stores of non-packed type.\n");
|
||||||
return;
|
return;
|
||||||
|
@ -5691,7 +5692,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
auto *SI = cast<StoreInst>(V);
|
auto *SI = cast<StoreInst>(V);
|
||||||
if (!SI->isSimple()) {
|
if (!SI->isSimple()) {
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple stores.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple stores.\n");
|
||||||
return;
|
return;
|
||||||
|
@ -5739,7 +5740,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
}
|
}
|
||||||
|
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
|
||||||
return;
|
return;
|
||||||
|
@ -5757,7 +5758,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
|
|
||||||
if (!VecFunc && !isTriviallyVectorizable(ID)) {
|
if (!VecFunc && !isTriviallyVectorizable(ID)) {
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
|
||||||
return;
|
return;
|
||||||
|
@ -5776,7 +5777,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
VecFunc != VFDatabase(*CI2).getVectorizedFunction(Shape)) ||
|
VecFunc != VFDatabase(*CI2).getVectorizedFunction(Shape)) ||
|
||||||
!CI->hasIdenticalOperandBundleSchema(*CI2)) {
|
!CI->hasIdenticalOperandBundleSchema(*CI2)) {
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *V
|
LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *V
|
||||||
<< "\n");
|
<< "\n");
|
||||||
|
@ -5789,7 +5790,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
Value *A1J = CI2->getArgOperand(j);
|
Value *A1J = CI2->getArgOperand(j);
|
||||||
if (ScalarArgs[j] != A1J) {
|
if (ScalarArgs[j] != A1J) {
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
LLVM_DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI
|
LLVM_DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI
|
||||||
<< " argument " << ScalarArgs[j] << "!=" << A1J
|
<< " argument " << ScalarArgs[j] << "!=" << A1J
|
||||||
|
@ -5804,7 +5805,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
CI->op_begin() + CI->getBundleOperandsEndIndex(),
|
CI->op_begin() + CI->getBundleOperandsEndIndex(),
|
||||||
CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
|
CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:"
|
LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:"
|
||||||
<< *CI << "!=" << *V << '\n');
|
<< *CI << "!=" << *V << '\n');
|
||||||
|
@ -5835,7 +5836,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
// then do not vectorize this instruction.
|
// then do not vectorize this instruction.
|
||||||
if (!S.isAltShuffle()) {
|
if (!S.isAltShuffle()) {
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
|
LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
|
||||||
return;
|
return;
|
||||||
|
@ -5898,7 +5899,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
BS.cancelScheduling(VL, VL0);
|
BS.cancelScheduling(VL, VL0);
|
||||||
newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
|
newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
|
||||||
ReuseShuffleIndicies);
|
ReuseShuffleIndicies);
|
||||||
LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
|
LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
|
||||||
return;
|
return;
|
||||||
|
@ -6336,20 +6337,22 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
|
||||||
unsigned Idx = (Data.second / NumElts) * NumElts;
|
unsigned Idx = (Data.second / NumElts) * NumElts;
|
||||||
unsigned EENumElts = EEVTy->getNumElements();
|
unsigned EENumElts = EEVTy->getNumElements();
|
||||||
if (Idx + NumElts <= EENumElts) {
|
if (Idx + NumElts <= EENumElts) {
|
||||||
Cost += TTI->getShuffleCost(TargetTransformInfo::SK_ExtractSubvector,
|
Cost +=
|
||||||
EEVTy, None, CostKind, Idx, VecTy);
|
TTI->getShuffleCost(TargetTransformInfo::SK_ExtractSubvector,
|
||||||
|
EEVTy, std::nullopt, CostKind, Idx, VecTy);
|
||||||
} else {
|
} else {
|
||||||
// Need to round up the subvector type vectorization factor to avoid a
|
// Need to round up the subvector type vectorization factor to avoid a
|
||||||
// crash in cost model functions. Make SubVT so that Idx + VF of SubVT
|
// crash in cost model functions. Make SubVT so that Idx + VF of SubVT
|
||||||
// <= EENumElts.
|
// <= EENumElts.
|
||||||
auto *SubVT =
|
auto *SubVT =
|
||||||
FixedVectorType::get(VecTy->getElementType(), EENumElts - Idx);
|
FixedVectorType::get(VecTy->getElementType(), EENumElts - Idx);
|
||||||
Cost += TTI->getShuffleCost(TargetTransformInfo::SK_ExtractSubvector,
|
Cost +=
|
||||||
EEVTy, None, CostKind, Idx, SubVT);
|
TTI->getShuffleCost(TargetTransformInfo::SK_ExtractSubvector,
|
||||||
|
EEVTy, std::nullopt, CostKind, Idx, SubVT);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
Cost += TTI->getShuffleCost(TargetTransformInfo::SK_InsertSubvector,
|
Cost += TTI->getShuffleCost(TargetTransformInfo::SK_InsertSubvector,
|
||||||
VecTy, None, CostKind, 0, EEVTy);
|
VecTy, std::nullopt, CostKind, 0, EEVTy);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -6417,7 +6420,7 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
|
||||||
assert(VecTy == FinalVecTy &&
|
assert(VecTy == FinalVecTy &&
|
||||||
"No reused scalars expected for broadcast.");
|
"No reused scalars expected for broadcast.");
|
||||||
return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy,
|
return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy,
|
||||||
/*Mask=*/None, CostKind, /*Index=*/0,
|
/*Mask=*/std::nullopt, CostKind, /*Index=*/0,
|
||||||
/*SubTp=*/nullptr, /*Args=*/VL[0]);
|
/*SubTp=*/nullptr, /*Args=*/VL[0]);
|
||||||
}
|
}
|
||||||
InstructionCost ReuseShuffleCost = 0;
|
InstructionCost ReuseShuffleCost = 0;
|
||||||
|
@ -6507,8 +6510,9 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
|
||||||
if (NeedInsertSubvectorAnalysis) {
|
if (NeedInsertSubvectorAnalysis) {
|
||||||
// Add the cost for the subvectors insert.
|
// Add the cost for the subvectors insert.
|
||||||
for (int I = VF, E = VL.size(); I < E; I += VF)
|
for (int I = VF, E = VL.size(); I < E; I += VF)
|
||||||
GatherCost += TTI->getShuffleCost(TTI::SK_InsertSubvector, VecTy,
|
GatherCost +=
|
||||||
None, CostKind, I, LoadTy);
|
TTI->getShuffleCost(TTI::SK_InsertSubvector, VecTy,
|
||||||
|
std::nullopt, CostKind, I, LoadTy);
|
||||||
}
|
}
|
||||||
return ReuseShuffleCost + GatherCost - ScalarsCost;
|
return ReuseShuffleCost + GatherCost - ScalarsCost;
|
||||||
}
|
}
|
||||||
|
@ -6721,8 +6725,9 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
|
||||||
if (InsertVecSz != VecSz) {
|
if (InsertVecSz != VecSz) {
|
||||||
auto *ActualVecTy =
|
auto *ActualVecTy =
|
||||||
FixedVectorType::get(SrcVecTy->getElementType(), VecSz);
|
FixedVectorType::get(SrcVecTy->getElementType(), VecSz);
|
||||||
Cost += TTI->getShuffleCost(TTI::SK_InsertSubvector, ActualVecTy, None,
|
Cost += TTI->getShuffleCost(TTI::SK_InsertSubvector, ActualVecTy,
|
||||||
CostKind, OffsetBeg - Offset, InsertVecTy);
|
std::nullopt, CostKind, OffsetBeg - Offset,
|
||||||
|
InsertVecTy);
|
||||||
} else {
|
} else {
|
||||||
for (unsigned I = 0, End = OffsetBeg - Offset; I < End; ++I)
|
for (unsigned I = 0, End = OffsetBeg - Offset; I < End; ++I)
|
||||||
Mask[I] = InMask.test(I) ? UndefMaskElem : I;
|
Mask[I] = InMask.test(I) ? UndefMaskElem : I;
|
||||||
|
@ -7693,7 +7698,7 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, SmallVectorImpl<int> &Mask,
|
||||||
if (const TreeEntry *VTE = getTreeEntry(V))
|
if (const TreeEntry *VTE = getTreeEntry(V))
|
||||||
VToTEs.insert(VTE);
|
VToTEs.insert(VTE);
|
||||||
if (VToTEs.empty())
|
if (VToTEs.empty())
|
||||||
return None;
|
return std::nullopt;
|
||||||
if (UsedTEs.empty()) {
|
if (UsedTEs.empty()) {
|
||||||
// The first iteration, just insert the list of nodes to vector.
|
// The first iteration, just insert the list of nodes to vector.
|
||||||
UsedTEs.push_back(VToTEs);
|
UsedTEs.push_back(VToTEs);
|
||||||
|
@ -7722,7 +7727,7 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, SmallVectorImpl<int> &Mask,
|
||||||
// If the number of input vectors is greater than 2 - not a permutation,
|
// If the number of input vectors is greater than 2 - not a permutation,
|
||||||
// fallback to the regular gather.
|
// fallback to the regular gather.
|
||||||
if (UsedTEs.size() == 2)
|
if (UsedTEs.size() == 2)
|
||||||
return None;
|
return std::nullopt;
|
||||||
UsedTEs.push_back(SavedVToTEs);
|
UsedTEs.push_back(SavedVToTEs);
|
||||||
Idx = UsedTEs.size() - 1;
|
Idx = UsedTEs.size() - 1;
|
||||||
}
|
}
|
||||||
|
@ -7733,7 +7738,7 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, SmallVectorImpl<int> &Mask,
|
||||||
if (UsedTEs.empty()) {
|
if (UsedTEs.empty()) {
|
||||||
assert(all_of(TE->Scalars, UndefValue::classof) &&
|
assert(all_of(TE->Scalars, UndefValue::classof) &&
|
||||||
"Expected vector of undefs only.");
|
"Expected vector of undefs only.");
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
unsigned VF = 0;
|
unsigned VF = 0;
|
||||||
|
@ -7767,7 +7772,7 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, SmallVectorImpl<int> &Mask,
|
||||||
// No 2 source vectors with the same vector factor - give up and do regular
|
// No 2 source vectors with the same vector factor - give up and do regular
|
||||||
// gather.
|
// gather.
|
||||||
if (Entries.empty())
|
if (Entries.empty())
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build a shuffle mask for better cost estimation and vector emission.
|
// Build a shuffle mask for better cost estimation and vector emission.
|
||||||
|
@ -7782,7 +7787,7 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, SmallVectorImpl<int> &Mask,
|
||||||
// Extra check required by isSingleSourceMaskImpl function (called by
|
// Extra check required by isSingleSourceMaskImpl function (called by
|
||||||
// ShuffleVectorInst::isSingleSourceMask).
|
// ShuffleVectorInst::isSingleSourceMask).
|
||||||
if (Mask[I] >= 2 * E)
|
if (Mask[I] >= 2 * E)
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
switch (Entries.size()) {
|
switch (Entries.size()) {
|
||||||
case 1:
|
case 1:
|
||||||
|
@ -7792,7 +7797,7 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, SmallVectorImpl<int> &Mask,
|
||||||
default:
|
default:
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
InstructionCost BoUpSLP::getGatherCost(FixedVectorType *Ty,
|
InstructionCost BoUpSLP::getGatherCost(FixedVectorType *Ty,
|
||||||
|
@ -9744,7 +9749,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
|
||||||
// dependencies and emit instruction in the wrong order at the actual
|
// dependencies and emit instruction in the wrong order at the actual
|
||||||
// scheduling.
|
// scheduling.
|
||||||
TryScheduleBundleImpl(/*ReSchedule=*/false, nullptr);
|
TryScheduleBundleImpl(/*ReSchedule=*/false, nullptr);
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -9774,7 +9779,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
|
||||||
TryScheduleBundleImpl(ReSchedule, Bundle);
|
TryScheduleBundleImpl(ReSchedule, Bundle);
|
||||||
if (!Bundle->isReady()) {
|
if (!Bundle->isReady()) {
|
||||||
cancelScheduling(VL, S.OpValue);
|
cancelScheduling(VL, S.OpValue);
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
return Bundle;
|
return Bundle;
|
||||||
}
|
}
|
||||||
|
@ -12259,7 +12264,7 @@ static Optional<unsigned> getAggregateSize(Instruction *InsertInst) {
|
||||||
if (auto *ST = dyn_cast<StructType>(CurrentType)) {
|
if (auto *ST = dyn_cast<StructType>(CurrentType)) {
|
||||||
for (auto *Elt : ST->elements())
|
for (auto *Elt : ST->elements())
|
||||||
if (Elt != ST->getElementType(0)) // check homogeneity
|
if (Elt != ST->getElementType(0)) // check homogeneity
|
||||||
return None;
|
return std::nullopt;
|
||||||
AggregateSize *= ST->getNumElements();
|
AggregateSize *= ST->getNumElements();
|
||||||
CurrentType = ST->getElementType(0);
|
CurrentType = ST->getElementType(0);
|
||||||
} else if (auto *AT = dyn_cast<ArrayType>(CurrentType)) {
|
} else if (auto *AT = dyn_cast<ArrayType>(CurrentType)) {
|
||||||
|
@ -12271,7 +12276,7 @@ static Optional<unsigned> getAggregateSize(Instruction *InsertInst) {
|
||||||
} else if (CurrentType->isSingleValueType()) {
|
} else if (CurrentType->isSingleValueType()) {
|
||||||
return AggregateSize;
|
return AggregateSize;
|
||||||
} else {
|
} else {
|
||||||
return None;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
} while (true);
|
} while (true);
|
||||||
}
|
}
|
||||||
|
|
|
@ -193,7 +193,7 @@ static std::optional<unsigned> getOpcode(ArrayRef<VPValue *> Values) {
|
||||||
if (any_of(Values, [Opcode](VPValue *V) {
|
if (any_of(Values, [Opcode](VPValue *V) {
|
||||||
return cast<VPInstruction>(V)->getOpcode() != Opcode;
|
return cast<VPInstruction>(V)->getOpcode() != Opcode;
|
||||||
}))
|
}))
|
||||||
return None;
|
return std::nullopt;
|
||||||
return {Opcode};
|
return {Opcode};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue