Skip to content

Commit 56853f5

Browse files
fhahnfhahn
committed
[VPlan] Move getCanonicalIV to VPRegionBlock (NFC).
The canonical IV is tied to region blocks; move getCanonicalIV there and update all users.
1 parent ae7b15f commit 56853f5

File tree

7 files changed

+72
-55
lines changed

7 files changed

+72
-55
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8242,12 +8242,12 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
82428242
// count is >= increment and a multiple of the increment.
82438243
bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
82448244
if (!HasNUW) {
8245-
auto *IVInc = Plan->getVectorLoopRegion()
8246-
->getExitingBasicBlock()
8247-
->getTerminator()
8248-
->getOperand(0);
8249-
assert(match(IVInc, m_VPInstruction<Instruction::Add>(
8250-
m_Specific(Plan->getCanonicalIV()), m_VPValue())) &&
8245+
VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
8246+
auto *IVInc =
8247+
LoopRegion->getExitingBasicBlock()->getTerminator()->getOperand(0);
8248+
assert(match(IVInc,
8249+
m_VPInstruction<Instruction::Add>(
8250+
m_Specific(LoopRegion->getCanonicalIV()), m_VPValue())) &&
82518251
"Did not find the canonical IV increment");
82528252
cast<VPRecipeWithIRFlags>(IVInc)->dropPoisonGeneratingFlags();
82538253
}
@@ -8377,8 +8377,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
83778377
for (VPValue *Old : Old2New.keys())
83788378
Old->getDefiningRecipe()->eraseFromParent();
83798379

8380-
assert(isa<VPRegionBlock>(Plan->getVectorLoopRegion()) &&
8381-
!Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() &&
8380+
assert(isa<VPRegionBlock>(LoopRegion) &&
8381+
!LoopRegion->getEntryBasicBlock()->empty() &&
83828382
"entry block must be set to a VPRegionBlock having a non-empty entry "
83838383
"VPBasicBlock");
83848384

@@ -9320,8 +9320,9 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
93209320
if (ResumePhiIter == MainScalarPH->phis().end()) {
93219321
VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
93229322
ResumePhi = ScalarPHBuilder.createScalarPhi(
9323-
{VectorTC, MainPlan.getCanonicalIV()->getStartValue()}, {},
9324-
"vec.epilog.resume.val");
9323+
{VectorTC,
9324+
MainPlan.getVectorLoopRegion()->getCanonicalIV()->getStartValue()},
9325+
{}, "vec.epilog.resume.val");
93259326
} else {
93269327
ResumePhi = cast<VPPhi>(&*ResumePhiIter);
93279328
if (MainScalarPH->begin() == MainScalarPH->end())
@@ -9350,7 +9351,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
93509351

93519352
// Ensure that the start values for all header phi recipes are updated before
93529353
// vectorizing the epilogue loop.
9353-
VPCanonicalIVPHIRecipe *IV = Plan.getCanonicalIV();
9354+
VPCanonicalIVPHIRecipe *IV = VectorLoop->getCanonicalIV();
93549355
// When vectorizing the epilogue loop, the canonical induction start
93559356
// value needs to be changed from zero to the value after the main
93569357
// vector loop. Find the resume value created during execution of the main

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4066,6 +4066,16 @@ class LLVM_ABI_FOR_TEST VPRegionBlock : public VPBlockBase {
40664066
/// Remove the current region from its VPlan, connecting its predecessor to
40674067
/// its entry, and its exiting block to its successor.
40684068
void dissolveToCFGLoop();
4069+
4070+
/// Returns the canonical induction recipe of the region.
4071+
VPCanonicalIVPHIRecipe *getCanonicalIV() {
4072+
VPBasicBlock *EntryVPBB = getEntryBasicBlock();
4073+
if (EntryVPBB->empty()) {
4074+
// VPlan native path.
4075+
EntryVPBB = cast<VPBasicBlock>(EntryVPBB->getSingleSuccessor());
4076+
}
4077+
return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->begin());
4078+
}
40694079
};
40704080

40714081
/// VPlan models a candidate for vectorization, encoding various decisions take
@@ -4377,16 +4387,6 @@ class VPlan {
43774387
LLVM_DUMP_METHOD void dump() const;
43784388
#endif
43794389

4380-
/// Returns the canonical induction recipe of the vector loop.
4381-
VPCanonicalIVPHIRecipe *getCanonicalIV() {
4382-
VPBasicBlock *EntryVPBB = getVectorLoopRegion()->getEntryBasicBlock();
4383-
if (EntryVPBB->empty()) {
4384-
// VPlan native path.
4385-
EntryVPBB = cast<VPBasicBlock>(EntryVPBB->getSingleSuccessor());
4386-
}
4387-
return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->begin());
4388-
}
4389-
43904390
VPValue *getSCEVExpansion(const SCEV *S) const {
43914391
return SCEVToExpansion.lookup(S);
43924392
}

‎llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -661,9 +661,11 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
661661
}
662662

663663
VPIRMetadata VPBranchWeights;
664-
auto *Term = VPBuilder(CheckBlockVPBB)
665-
.createNaryOp(VPInstruction::BranchOnCond, {CondVPV},
666-
Plan.getCanonicalIV()->getDebugLoc());
664+
auto *Term =
665+
VPBuilder(CheckBlockVPBB)
666+
.createNaryOp(
667+
VPInstruction::BranchOnCond, {CondVPV},
668+
Plan.getVectorLoopRegion()->getCanonicalIV()->getDebugLoc());
667669
if (AddBranchWeights) {
668670
MDBuilder MDB(Plan.getContext());
669671
MDNode *BranchWeights =
@@ -925,8 +927,8 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
925927
if (auto *DerivedIV = dyn_cast<VPDerivedIVRecipe>(VecV)) {
926928
if (DerivedIV->getNumUsers() == 1 &&
927929
DerivedIV->getOperand(1) == &Plan.getVectorTripCount()) {
928-
auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(),
929-
&Plan.getVectorTripCount());
930+
auto *NewSel = Builder.createSelect(
931+
AnyNaN, LoopRegion->getCanonicalIV(), &Plan.getVectorTripCount());
930932
DerivedIV->moveAfter(&*Builder.getInsertPoint());
931933
DerivedIV->setOperand(1, NewSel);
932934
continue;
@@ -939,7 +941,8 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
939941
"FMaxNum/FMinNum reduction.\n");
940942
return false;
941943
}
942-
auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), VecV);
944+
auto *NewSel =
945+
Builder.createSelect(AnyNaN, LoopRegion->getCanonicalIV(), VecV);
943946
ResumeR->setOperand(0, NewSel);
944947
}
945948

‎llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,8 @@ void VPPredicator::createHeaderMask(VPBasicBlock *HeaderVPBB, bool FoldTail) {
168168
// non-phi instructions.
169169

170170
auto &Plan = *HeaderVPBB->getPlan();
171-
auto *IV = new VPWidenCanonicalIVRecipe(Plan.getCanonicalIV());
171+
auto *IV = new VPWidenCanonicalIVRecipe(
172+
Plan.getVectorLoopRegion()->getCanonicalIV());
172173
Builder.setInsertPoint(HeaderVPBB, HeaderVPBB->getFirstNonPhi());
173174
Builder.insert(IV);
174175

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 31 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -501,7 +501,8 @@ static void removeRedundantInductionCasts(VPlan &Plan) {
501501
/// Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV
502502
/// recipe, if it exists.
503503
static void removeRedundantCanonicalIVs(VPlan &Plan) {
504-
VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
504+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
505+
VPCanonicalIVPHIRecipe *CanonicalIV = LoopRegion->getCanonicalIV();
505506
VPWidenCanonicalIVRecipe *WidenNewIV = nullptr;
506507
for (VPUser *U : CanonicalIV->users()) {
507508
WidenNewIV = dyn_cast<VPWidenCanonicalIVRecipe>(U);
@@ -512,7 +513,7 @@ static void removeRedundantCanonicalIVs(VPlan &Plan) {
512513
if (!WidenNewIV)
513514
return;
514515

515-
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
516+
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
516517
for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
517518
auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
518519

@@ -582,8 +583,9 @@ createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
582583
FPMathOperator *FPBinOp, Instruction *TruncI,
583584
VPValue *StartV, VPValue *Step, DebugLoc DL,
584585
VPBuilder &Builder) {
585-
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
586-
VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
586+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
587+
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
588+
VPCanonicalIVPHIRecipe *CanonicalIV = LoopRegion->getCanonicalIV();
587589
VPSingleDefRecipe *BaseIV = Builder.createDerivedIV(
588590
Kind, FPBinOp, StartV, CanonicalIV, Step, "offset.idx");
589591

@@ -800,8 +802,9 @@ static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan,
800802
return nullptr;
801803

802804
// Calculate the final index.
803-
VPValue *EndValue = Plan.getCanonicalIV();
804-
auto CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
805+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
806+
VPValue *EndValue = LoopRegion->getCanonicalIV();
807+
auto CanonicalIVType = LoopRegion->getCanonicalIV()->getScalarType();
805808
VPBuilder B(cast<VPBasicBlock>(PredVPBB));
806809

807810
DebugLoc DL = cast<VPInstruction>(Op)->getDebugLoc();
@@ -1520,7 +1523,7 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
15201523
return isConditionTrueViaVFAndUF(C, Plan, BestVF, BestUF, SE);
15211524
});
15221525

1523-
auto *CanIV = Plan.getCanonicalIV();
1526+
auto *CanIV = Plan.getVectorLoopRegion()->getCanonicalIV();
15241527
if (!match(Cond, m_SpecificICmp(CmpInst::ICMP_EQ,
15251528
m_Specific(CanIV->getBackedgeValue()),
15261529
m_Specific(&Plan.getVectorTripCount()))))
@@ -2304,7 +2307,7 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
23042307
VPlan &Plan, bool DataAndControlFlowWithoutRuntimeCheck) {
23052308
VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
23062309
VPBasicBlock *EB = TopRegion->getExitingBasicBlock();
2307-
auto *CanonicalIVPHI = Plan.getCanonicalIV();
2310+
auto *CanonicalIVPHI = TopRegion->getCanonicalIV();
23082311
VPValue *StartV = CanonicalIVPHI->getStartValue();
23092312

23102313
auto *CanonicalIVIncrement =
@@ -2343,7 +2346,7 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
23432346

23442347
// Create the active lane mask instruction in the VPlan preheader.
23452348
VPValue *ALMMultiplier = Plan.getOrAddLiveIn(
2346-
ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1));
2349+
ConstantInt::get(TopRegion->getCanonicalIV()->getScalarType(), 1));
23472350
auto *EntryALM = Builder.createNaryOp(VPInstruction::ActiveLaneMask,
23482351
{EntryIncrement, TC, ALMMultiplier}, DL,
23492352
"active.lane.mask.entry");
@@ -2379,21 +2382,23 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
23792382
/// TODO: Introduce explicit recipe for header-mask instead of searching
23802383
/// for the header-mask pattern manually.
23812384
static VPSingleDefRecipe *findHeaderMask(VPlan &Plan) {
2385+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
23822386
SmallVector<VPValue *> WideCanonicalIVs;
2383-
auto *FoundWidenCanonicalIVUser = find_if(Plan.getCanonicalIV()->users(),
2384-
IsaPred<VPWidenCanonicalIVRecipe>);
2385-
assert(count_if(Plan.getCanonicalIV()->users(),
2387+
auto *FoundWidenCanonicalIVUser = find_if(
2388+
LoopRegion->getCanonicalIV()->users(), IsaPred<VPWidenCanonicalIVRecipe>);
2389+
assert(count_if(LoopRegion->getCanonicalIV()->users(),
23862390
IsaPred<VPWidenCanonicalIVRecipe>) <= 1 &&
23872391
"Must have at most one VPWideCanonicalIVRecipe");
2388-
if (FoundWidenCanonicalIVUser != Plan.getCanonicalIV()->users().end()) {
2392+
if (FoundWidenCanonicalIVUser !=
2393+
LoopRegion->getCanonicalIV()->users().end()) {
23892394
auto *WideCanonicalIV =
23902395
cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
23912396
WideCanonicalIVs.push_back(WideCanonicalIV);
23922397
}
23932398

23942399
// Also include VPWidenIntOrFpInductionRecipes that represent a widened
23952400
// version of the canonical induction.
2396-
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
2401+
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
23972402
for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
23982403
auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
23992404
if (WidenOriginalIV && WidenOriginalIV->isCanonical())
@@ -2426,8 +2431,9 @@ void VPlanTransforms::addActiveLaneMask(
24262431
"DataAndControlFlowWithoutRuntimeCheck implies "
24272432
"UseActiveLaneMaskForControlFlow");
24282433

2429-
auto *FoundWidenCanonicalIVUser = find_if(Plan.getCanonicalIV()->users(),
2430-
IsaPred<VPWidenCanonicalIVRecipe>);
2434+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
2435+
auto *FoundWidenCanonicalIVUser = find_if(
2436+
LoopRegion->getCanonicalIV()->users(), IsaPred<VPWidenCanonicalIVRecipe>);
24312437
assert(FoundWidenCanonicalIVUser &&
24322438
"Must have widened canonical IV when tail folding!");
24332439
VPSingleDefRecipe *HeaderMask = findHeaderMask(Plan);
@@ -2440,7 +2446,7 @@ void VPlanTransforms::addActiveLaneMask(
24402446
} else {
24412447
VPBuilder B = VPBuilder::getToInsertAfter(WideCanonicalIV);
24422448
VPValue *ALMMultiplier = Plan.getOrAddLiveIn(
2443-
ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1));
2449+
ConstantInt::get(LoopRegion->getCanonicalIV()->getScalarType(), 1));
24442450
LaneMask =
24452451
B.createNaryOp(VPInstruction::ActiveLaneMask,
24462452
{WideCanonicalIV, Plan.getTripCount(), ALMMultiplier},
@@ -2550,9 +2556,10 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
25502556
});
25512557

25522558
assert(all_of(Plan.getVFxUF().users(),
2553-
[&Plan](VPUser *U) {
2554-
return match(U, m_c_Add(m_Specific(Plan.getCanonicalIV()),
2555-
m_Specific(&Plan.getVFxUF()))) ||
2559+
[&LoopRegion, &Plan](VPUser *U) {
2560+
return match(U,
2561+
m_c_Add(m_Specific(LoopRegion->getCanonicalIV()),
2562+
m_Specific(&Plan.getVFxUF()))) ||
25562563
isa<VPWidenPointerInductionRecipe>(U);
25572564
}) &&
25582565
"Only users of VFxUF should be VPWidenPointerInductionRecipe and the "
@@ -2707,9 +2714,10 @@ void VPlanTransforms::addExplicitVectorLength(
27072714
VPlan &Plan, const std::optional<unsigned> &MaxSafeElements) {
27082715
if (Plan.hasScalarVFOnly())
27092716
return;
2710-
VPBasicBlock *Header = Plan.getVectorLoopRegion()->getEntryBasicBlock();
2717+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
2718+
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
27112719

2712-
auto *CanonicalIVPHI = Plan.getCanonicalIV();
2720+
auto *CanonicalIVPHI = LoopRegion->getCanonicalIV();
27132721
auto *CanIVTy = CanonicalIVPHI->getScalarType();
27142722
VPValue *StartV = CanonicalIVPHI->getStartValue();
27152723

@@ -4153,7 +4161,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
41534161

41544162
// Adjust induction to reflect that the transformed plan only processes one
41554163
// original iteration.
4156-
auto *CanIV = Plan.getCanonicalIV();
4164+
auto *CanIV = VectorLoop->getCanonicalIV();
41574165
auto *Inc = cast<VPInstruction>(CanIV->getBackedgeValue());
41584166
VPBuilder PHBuilder(Plan.getVectorPreheader());
41594167

‎llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ class UnrollState {
6969
VPBasicBlock::iterator InsertPtForPhi);
7070

7171
VPValue *getConstantVPV(unsigned Part) {
72-
Type *CanIVIntTy = Plan.getCanonicalIV()->getScalarType();
72+
Type *CanIVIntTy =
73+
Plan.getVectorLoopRegion()->getCanonicalIV()->getScalarType();
7374
return Plan.getOrAddLiveIn(ConstantInt::get(CanIVIntTy, Part));
7475
}
7576

‎llvm/lib/Transforms/Vectorize/VPlanUtils.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,10 @@ bool vputils::isHeaderMask(const VPValue *V, VPlan &Plan) {
6767

6868
if (match(V, m_ActiveLaneMask(m_VPValue(A), m_VPValue(B), m_One())))
6969
return B == Plan.getTripCount() &&
70-
(match(A, m_ScalarIVSteps(m_Specific(Plan.getCanonicalIV()), m_One(),
71-
m_Specific(&Plan.getVF()))) ||
70+
(match(A,
71+
m_ScalarIVSteps(
72+
m_Specific(Plan.getVectorLoopRegion()->getCanonicalIV()),
73+
m_One(), m_Specific(&Plan.getVF()))) ||
7274
IsWideCanonicalIV(A));
7375

7476
return match(V, m_ICmp(m_VPValue(A), m_VPValue(B))) && IsWideCanonicalIV(A) &&
@@ -102,7 +104,8 @@ bool vputils::isUniformAcrossVFsAndUFs(VPValue *V) {
102104
return all_of(R->operands(), isUniformAcrossVFsAndUFs);
103105
}
104106

105-
auto *CanonicalIV = R->getParent()->getPlan()->getCanonicalIV();
107+
auto *CanonicalIV =
108+
R->getParent()->getPlan()->getVectorLoopRegion()->getCanonicalIV();
106109
// Canonical IV chain is uniform.
107110
if (V == CanonicalIV || V == CanonicalIV->getBackedgeValue())
108111
return true;

0 commit comments

Comments
 (0)