OSchip
/
llvm-project
mirror of https://github.com/THU-DSP-LAB/llvm-project.git
13
0
Fork 6
Code Issues Packages Projects Releases Wiki Activity

[LV] Use info from State in some helper functions (NFC). 

This updates several helper functions to use information provided by
VPTransformState instead of ILV directly, to help with the transition
out of ILV.
This commit is contained in:
Florian Hahn 2021-12-12 20:48:38 +00:00
parent 24b28db8cc
commit b6a2ddb6c8
No known key found for this signature in database
GPG Key ID: EEF712BB5E80EBBA
1 changed files with 29 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -2335,6 +2335,7 @@ Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {
void InnerLoopVectorizer::createVectorIntOrFpInductionPHI( void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
const InductionDescriptor &II, Value *Step, Value *Start, const InductionDescriptor &II, Value *Step, Value *Start,
Instruction *EntryVal, VPValue *Def, VPTransformState &State) { Instruction *EntryVal, VPValue *Def, VPTransformState &State) {
IRBuilder<> &Builder = State.Builder;
assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
"Expected either an induction phi-node or a truncate of it!"); "Expected either an induction phi-node or a truncate of it!");
@ -2350,7 +2351,7 @@ void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
} }
Value *Zero = getSignedIntOrFpConstant(Start->getType(), 0); Value *Zero = getSignedIntOrFpConstant(Start->getType(), 0);
Value *SplatStart = Builder.CreateVectorSplat(VF, Start); Value *SplatStart = Builder.CreateVectorSplat(State.VF, Start);
Value *SteppedStart = Value *SteppedStart =
getStepVector(SplatStart, Zero, Step, II.getInductionOpcode()); getStepVector(SplatStart, Zero, Step, II.getInductionOpcode());
@ -2371,9 +2372,9 @@ void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
Type *StepType = Step->getType(); Type *StepType = Step->getType();
Value *RuntimeVF; Value *RuntimeVF;
if (Step->getType()->isFloatingPointTy()) if (Step->getType()->isFloatingPointTy())
RuntimeVF = getRuntimeVFAsFloat(Builder, StepType, VF); RuntimeVF = getRuntimeVFAsFloat(Builder, StepType, State.VF);
else else
RuntimeVF = getRuntimeVF(Builder, StepType, VF); RuntimeVF = getRuntimeVF(Builder, StepType, State.VF);
Value *Mul = Builder.CreateBinOp(MulOp, Step, RuntimeVF); Value *Mul = Builder.CreateBinOp(MulOp, Step, RuntimeVF);
// Create a vector splat to use in the induction update. // Create a vector splat to use in the induction update.
@ -2382,8 +2383,8 @@ void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
// IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't // IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
// handle a constant vector splat. // handle a constant vector splat.
Value *SplatVF = isa<Constant>(Mul) Value *SplatVF = isa<Constant>(Mul)
? ConstantVector::getSplat(VF, cast<Constant>(Mul)) ? ConstantVector::getSplat(State.VF, cast<Constant>(Mul))
: Builder.CreateVectorSplat(VF, Mul); : Builder.CreateVectorSplat(State.VF, Mul);
Builder.restoreIP(CurrIP); Builder.restoreIP(CurrIP);
// We may need to add the step a number of times, depending on the unroll // We may need to add the step a number of times, depending on the unroll
@ -2435,6 +2436,7 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV,
Value *Start, TruncInst *Trunc, Value *Start, TruncInst *Trunc,
VPValue *Def, VPValue *Def,
VPTransformState &State) { VPTransformState &State) {
IRBuilder<> &Builder = State.Builder;
assert((IV->getType()->isIntegerTy() || IV != OldInduction) && assert((IV->getType()->isIntegerTy() || IV != OldInduction) &&
"Primary induction variable must have an integer type"); "Primary induction variable must have an integer type");
assert(IV->getType() == ID.getStartValue()->getType() && "Types must match"); assert(IV->getType() == ID.getStartValue()->getType() && "Types must match");
@ -2443,7 +2445,7 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV,
// variable. // variable.
Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV; Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
auto &DL = OrigLoop->getHeader()->getModule()->getDataLayout(); auto &DL = EntryVal->getModule()->getDataLayout();
// Generate code for the induction step. Note that induction steps are // Generate code for the induction step. Note that induction steps are
// required to be loop-invariant // required to be loop-invariant
@ -2453,7 +2455,7 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV,
if (PSE.getSE()->isSCEVable(IV->getType())) { if (PSE.getSE()->isSCEVable(IV->getType())) {
SCEVExpander Exp(*PSE.getSE(), DL, "induction"); SCEVExpander Exp(*PSE.getSE(), DL, "induction");
return Exp.expandCodeFor(Step, Step->getType(), return Exp.expandCodeFor(Step, Step->getType(),
LoopVectorPreHeader->getTerminator()); State.CFG.VectorPreHeader->getTerminator());
} }
return cast<SCEVUnknown>(Step)->getValue(); return cast<SCEVUnknown>(Step)->getValue();
}; };
@ -2487,12 +2489,13 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV,
auto CreateSplatIV = [&](Value *ScalarIV, Value *Step) { auto CreateSplatIV = [&](Value *ScalarIV, Value *Step) {
Value *Broadcasted = getBroadcastInstrs(ScalarIV); Value *Broadcasted = getBroadcastInstrs(ScalarIV);
for (unsigned Part = 0; Part < UF; ++Part) { for (unsigned Part = 0; Part < UF; ++Part) {
assert(!VF.isScalable() && "scalable vectors not yet supported."); assert(!State.VF.isScalable() && "scalable vectors not yet supported.");
Value *StartIdx; Value *StartIdx;
if (Step->getType()->isFloatingPointTy()) if (Step->getType()->isFloatingPointTy())
StartIdx = getRuntimeVFAsFloat(Builder, Step->getType(), VF * Part); StartIdx =
getRuntimeVFAsFloat(Builder, Step->getType(), State.VF * Part);
else else
StartIdx = getRuntimeVF(Builder, Step->getType(), VF * Part); StartIdx = getRuntimeVF(Builder, Step->getType(), State.VF * Part);
Value *EntryPart = Value *EntryPart =
getStepVector(Broadcasted, StartIdx, Step, ID.getInductionOpcode()); getStepVector(Broadcasted, StartIdx, Step, ID.getInductionOpcode());
@ -2509,7 +2512,7 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV,
// Now do the actual transformations, and start with creating the step value. // Now do the actual transformations, and start with creating the step value.
Value *Step = CreateStepValue(ID.getStep()); Value *Step = CreateStepValue(ID.getStep());
if (VF.isZero() || VF.isScalar()) { if (State.VF.isZero() || State.VF.isScalar()) {
Value *ScalarIV = CreateScalarIV(Step); Value *ScalarIV = CreateScalarIV(Step);
CreateSplatIV(ScalarIV, Step); CreateSplatIV(ScalarIV, Step);
return; return;
@ -2600,8 +2603,9 @@ void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
const InductionDescriptor &ID, const InductionDescriptor &ID,
VPValue *Def, VPValue *Def,
VPTransformState &State) { VPTransformState &State) {
IRBuilder<> &Builder = State.Builder;
// We shouldn't have to build scalar steps if we aren't vectorizing. // We shouldn't have to build scalar steps if we aren't vectorizing.
assert(VF.isVector() && "VF should be greater than one"); assert(State.VF.isVector() && "VF should be greater than one");
// Get the value type and ensure it and the step have the same integer type. // Get the value type and ensure it and the step have the same integer type.
Type *ScalarIVTy = ScalarIV->getType()->getScalarType(); Type *ScalarIVTy = ScalarIV->getType()->getScalarType();
assert(ScalarIVTy == Step->getType() && assert(ScalarIVTy == Step->getType() &&
@ -2623,25 +2627,26 @@ void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
// iteration. If EntryVal is uniform, we only need to generate the first // iteration. If EntryVal is uniform, we only need to generate the first
// lane. Otherwise, we generate all VF values. // lane. Otherwise, we generate all VF values.
bool IsUniform = bool IsUniform =
Cost->isUniformAfterVectorization(cast<Instruction>(EntryVal), VF); Cost->isUniformAfterVectorization(cast<Instruction>(EntryVal), State.VF);
unsigned Lanes = IsUniform ? 1 : VF.getKnownMinValue(); unsigned Lanes = IsUniform ? 1 : State.VF.getKnownMinValue();
// Compute the scalar steps and save the results in State. // Compute the scalar steps and save the results in State.
Type *IntStepTy = IntegerType::get(ScalarIVTy->getContext(), Type *IntStepTy = IntegerType::get(ScalarIVTy->getContext(),
ScalarIVTy->getScalarSizeInBits()); ScalarIVTy->getScalarSizeInBits());
Type *VecIVTy = nullptr; Type *VecIVTy = nullptr;
Value *UnitStepVec = nullptr, *SplatStep = nullptr, *SplatIV = nullptr; Value *UnitStepVec = nullptr, *SplatStep = nullptr, *SplatIV = nullptr;
if (!IsUniform && VF.isScalable()) { if (!IsUniform && State.VF.isScalable()) {
VecIVTy = VectorType::get(ScalarIVTy, VF); VecIVTy = VectorType::get(ScalarIVTy, State.VF);
UnitStepVec = Builder.CreateStepVector(VectorType::get(IntStepTy, VF)); UnitStepVec =
SplatStep = Builder.CreateVectorSplat(VF, Step); Builder.CreateStepVector(VectorType::get(IntStepTy, State.VF));
SplatIV = Builder.CreateVectorSplat(VF, ScalarIV); SplatStep = Builder.CreateVectorSplat(State.VF, Step);
SplatIV = Builder.CreateVectorSplat(State.VF, ScalarIV);
} }
for (unsigned Part = 0; Part < UF; ++Part) { for (unsigned Part = 0; Part < State.UF; ++Part) {
Value *StartIdx0 = createStepForVF(Builder, IntStepTy, VF, Part); Value *StartIdx0 = createStepForVF(Builder, IntStepTy, State.VF, Part);
if (!IsUniform && VF.isScalable()) { if (!IsUniform && State.VF.isScalable()) {
auto *SplatStartIdx = Builder.CreateVectorSplat(VF, StartIdx0); auto *SplatStartIdx = Builder.CreateVectorSplat(State.VF, StartIdx0);
auto *InitVec = Builder.CreateAdd(SplatStartIdx, UnitStepVec); auto *InitVec = Builder.CreateAdd(SplatStartIdx, UnitStepVec);
if (ScalarIVTy->isFloatingPointTy()) if (ScalarIVTy->isFloatingPointTy())
InitVec = Builder.CreateSIToFP(InitVec, VecIVTy); InitVec = Builder.CreateSIToFP(InitVec, VecIVTy);
@ -2661,7 +2666,7 @@ void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
AddOp, StartIdx0, getSignedIntOrFpConstant(ScalarIVTy, Lane)); AddOp, StartIdx0, getSignedIntOrFpConstant(ScalarIVTy, Lane));
// The step returned by `createStepForVF` is a runtime-evaluated value // The step returned by `createStepForVF` is a runtime-evaluated value
// when VF is scalable. Otherwise, it should be folded into a Constant. // when VF is scalable. Otherwise, it should be folded into a Constant.
assert((VF.isScalable() || isa<Constant>(StartIdx)) && assert((State.VF.isScalable() || isa<Constant>(StartIdx)) &&
"Expected StartIdx to be folded to a constant when VF is not " "Expected StartIdx to be folded to a constant when VF is not "
"scalable"); "scalable");
auto *Mul = Builder.CreateBinOp(MulOp, StartIdx, Step); auto *Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);