Skip to content

Commit 00bbfbb

Browse files
arcbbbarcbbb
committed
Add MemIntrinsicCostAttributes
1 parent 8104387 commit 00bbfbb

File tree

7 files changed

+158
-107
lines changed

7 files changed

+158
-107
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h‎

Lines changed: 50 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,53 @@ struct HardwareLoopInfo {
123123
LLVM_ABI bool canAnalyze(LoopInfo &LI);
124124
};
125125

126+
/// Information for Memory intrinsic cost model.
127+
class MemIntrinsicCostAttributes {
128+
/// Optional original context instruction, if one exists, e.g. the
129+
/// load/store to transform to the intrinsic
130+
const Instruction *I = nullptr;
131+
132+
/// Address in memory
133+
const Value *Ptr = nullptr;
134+
135+
/// Vector type of the data to be loaded or stored.
136+
Type *DataTy = nullptr;
137+
Intrinsic::ID IID;
138+
139+
/// True when the memory access is predicated with a mask
140+
/// that is not a compile-time constant
141+
bool VariableMask = true;
142+
unsigned AddressSpace = 0;
143+
144+
/// Alignment of single element
145+
Align Alignment;
146+
147+
public:
148+
LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
149+
const Value *Ptr, bool VariableMask,
150+
Align Alignment,
151+
const Instruction *I = nullptr)
152+
: I(I), Ptr(Ptr), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
153+
Alignment(Alignment) {}
154+
LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
155+
Align Alignment, unsigned AddressSpace)
156+
: DataTy(DataTy), IID(Id), AddressSpace(AddressSpace),
157+
Alignment(Alignment) {}
158+
LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
159+
bool VariableMask, Align Alignment,
160+
const Instruction *I = nullptr)
161+
: I(I), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
162+
Alignment(Alignment) {}
163+
164+
Intrinsic::ID getID() const { return IID; }
165+
const Instruction *getInst() const { return I; }
166+
const Value *getPointer() const { return Ptr; }
167+
Type *getDataType() const { return DataTy; }
168+
bool getVariableMask() const { return VariableMask; }
169+
unsigned getAddressSpace() const { return AddressSpace; }
170+
Align getAlignment() const { return Alignment; }
171+
};
172+
126173
class IntrinsicCostAttributes {
127174
const IntrinsicInst *II = nullptr;
128175
Type *RetTy = nullptr;
@@ -1634,17 +1681,9 @@ class TargetTransformInfo {
16341681

16351682
/// \returns The cost of memory intrinsic instructions.
16361683
/// Used when IntrinsicInst is not materialized.
1637-
/// \p DataTy - a vector type of the data to be loaded or stored
1638-
/// \p Ptr - pointer [or vector of pointers] - address[es] in memory
1639-
/// \p VariableMask - true when the memory access is predicated with a mask
1640-
/// that is not a compile-time constant
1641-
/// \p Alignment - alignment of single element
1642-
/// \p I - the optional original context instruction, if one exists, e.g. the
1643-
/// load/store to transform or the call to the gather/scatter intrinsic
1644-
LLVM_ABI InstructionCost getMemIntrinsicInstrCost(
1645-
Intrinsic::ID Id, Type *DataTy, const Value *Ptr, bool VariableMask,
1646-
Align Alignment, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
1647-
const Instruction *I = nullptr) const;
1684+
LLVM_ABI InstructionCost
1685+
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
1686+
TTI::TargetCostKind CostKind) const;
16481687

16491688
/// \returns The cost of Call instructions.
16501689
LLVM_ABI InstructionCost getCallInstrCost(

‎llvm/include/llvm/Analysis/TargetTransformInfoImpl.h‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -927,10 +927,8 @@ class TargetTransformInfoImplBase {
927927
}
928928

929929
virtual InstructionCost
930-
getMemIntrinsicInstrCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
931-
bool VariableMask, Align Alignment,
932-
TTI::TargetCostKind CostKind,
933-
const Instruction *I = nullptr) const {
930+
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
931+
TTI::TargetCostKind CostKind) const {
934932
return 1;
935933
}
936934
virtual InstructionCost getCallInstrCost(Function *F, Type *RetTy,

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h‎

Lines changed: 45 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1610,8 +1610,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
16101610
unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
16111611
: Intrinsic::masked_store;
16121612
Cost = thisT()->getMemIntrinsicInstrCost(
1613-
IID, VecTy, /*Ptr*/ nullptr, /*VariableMask*/ true, Alignment,
1614-
CostKind, /*Instruction*/ nullptr);
1613+
{IID, VecTy, Alignment, AddressSpace}, CostKind);
16151614
} else
16161615
Cost = thisT()->getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace,
16171616
CostKind);
@@ -1812,8 +1811,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
18121811
Alignment = VPI->getPointerAlignment().valueOrOne();
18131812
bool VarMask = isa<Constant>(ICA.getArgs()[2]);
18141813
return thisT()->getMemIntrinsicInstrCost(
1815-
Intrinsic::vp_scatter, ICA.getArgTypes()[0], ICA.getArgs()[1],
1816-
VarMask, Alignment, CostKind, nullptr);
1814+
{Intrinsic::vp_scatter, ICA.getArgTypes()[0], ICA.getArgs()[1],
1815+
VarMask, Alignment, nullptr},
1816+
CostKind);
18171817
}
18181818
if (ICA.getID() == Intrinsic::vp_gather) {
18191819
if (ICA.isTypeBasedOnly()) {
@@ -1828,8 +1828,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
18281828
Alignment = VPI->getPointerAlignment().valueOrOne();
18291829
bool VarMask = isa<Constant>(ICA.getArgs()[1]);
18301830
return thisT()->getMemIntrinsicInstrCost(
1831-
Intrinsic::vp_gather, ICA.getReturnType(), ICA.getArgs()[0],
1832-
VarMask, Alignment, CostKind, nullptr);
1831+
{Intrinsic::vp_gather, ICA.getReturnType(), ICA.getArgs()[0],
1832+
VarMask, Alignment, nullptr},
1833+
CostKind);
18331834
}
18341835

18351836
if (ICA.getID() == Intrinsic::vp_select ||
@@ -1934,32 +1935,35 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
19341935
const Value *Mask = Args[2];
19351936
bool VarMask = !isa<Constant>(Mask);
19361937
Align Alignment = I->getParamAlign(1).valueOrOne();
1937-
return thisT()->getMemIntrinsicInstrCost(Intrinsic::masked_scatter,
1938-
ICA.getArgTypes()[0], Args[1],
1939-
VarMask, Alignment, CostKind, I);
1938+
return thisT()->getMemIntrinsicInstrCost({Intrinsic::masked_scatter,
1939+
ICA.getArgTypes()[0], Args[1],
1940+
VarMask, Alignment, I},
1941+
CostKind);
19401942
}
19411943
case Intrinsic::masked_gather: {
19421944
const Value *Mask = Args[1];
19431945
bool VarMask = !isa<Constant>(Mask);
19441946
Align Alignment = I->getParamAlign(0).valueOrOne();
1945-
return thisT()->getMemIntrinsicInstrCost(Intrinsic::masked_gather, RetTy,
1946-
Args[0], VarMask, Alignment,
1947-
CostKind, I);
1947+
return thisT()->getMemIntrinsicInstrCost(
1948+
{Intrinsic::masked_gather, RetTy, Args[0], VarMask, Alignment, I},
1949+
CostKind);
19481950
}
19491951
case Intrinsic::masked_compressstore: {
19501952
const Value *Data = Args[0];
19511953
const Value *Mask = Args[2];
19521954
Align Alignment = I->getParamAlign(1).valueOrOne();
19531955
return thisT()->getMemIntrinsicInstrCost(
1954-
Intrinsic::masked_compressstore, Data->getType(), Args[1],
1955-
!isa<Constant>(Mask), Alignment, CostKind, I);
1956+
{Intrinsic::masked_compressstore, Data->getType(),
1957+
!isa<Constant>(Mask), Alignment, I},
1958+
CostKind);
19561959
}
19571960
case Intrinsic::masked_expandload: {
19581961
const Value *Mask = Args[1];
19591962
Align Alignment = I->getParamAlign(0).valueOrOne();
1960-
return thisT()->getMemIntrinsicInstrCost(
1961-
Intrinsic::masked_expandload, RetTy, Args[0], !isa<Constant>(Mask),
1962-
Alignment, CostKind, I);
1963+
return thisT()->getMemIntrinsicInstrCost({Intrinsic::masked_expandload,
1964+
RetTy, !isa<Constant>(Mask),
1965+
Alignment, I},
1966+
CostKind);
19631967
}
19641968
case Intrinsic::experimental_vp_strided_store: {
19651969
const Value *Data = Args[0];
@@ -1970,8 +1974,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
19701974
Type *EltTy = cast<VectorType>(Data->getType())->getElementType();
19711975
Align Alignment =
19721976
I->getParamAlign(1).value_or(thisT()->DL.getABITypeAlign(EltTy));
1973-
return thisT()->getMemIntrinsicInstrCost(IID, Data->getType(), Ptr,
1974-
VarMask, Alignment, CostKind, I);
1977+
return thisT()->getMemIntrinsicInstrCost(
1978+
{IID, Data->getType(), Ptr, VarMask, Alignment, I}, CostKind);
19751979
}
19761980
case Intrinsic::experimental_vp_strided_load: {
19771981
const Value *Ptr = Args[0];
@@ -1981,8 +1985,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
19811985
Type *EltTy = cast<VectorType>(RetTy)->getElementType();
19821986
Align Alignment =
19831987
I->getParamAlign(0).value_or(thisT()->DL.getABITypeAlign(EltTy));
1984-
return thisT()->getMemIntrinsicInstrCost(IID, RetTy, Ptr, VarMask,
1985-
Alignment, CostKind, I);
1988+
return thisT()->getMemIntrinsicInstrCost(
1989+
{IID, RetTy, Ptr, VarMask, Alignment, I}, CostKind);
19861990
}
19871991
case Intrinsic::stepvector: {
19881992
if (isa<ScalableVectorType>(RetTy))
@@ -2412,29 +2416,29 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
24122416
Type *Ty = Tys[0];
24132417
Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
24142418
return thisT()->getMemIntrinsicInstrCost(
2415-
Intrinsic::masked_store, Ty,
2416-
/*Ptr*/ nullptr, /*VariableMask*/ true, TyAlign, CostKind, nullptr);
2419+
{Intrinsic::masked_store, Ty, TyAlign, 0}, CostKind);
24172420
}
24182421
case Intrinsic::masked_load: {
24192422
Type *Ty = RetTy;
24202423
Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
24212424
return thisT()->getMemIntrinsicInstrCost(
2422-
Intrinsic::masked_load, Ty,
2423-
/*Ptr*/ nullptr, /*VariableMask*/ true, TyAlign, CostKind, nullptr);
2425+
{Intrinsic::masked_load, Ty, TyAlign, 0}, CostKind);
24242426
}
24252427
case Intrinsic::experimental_vp_strided_store: {
24262428
auto *Ty = cast<VectorType>(ICA.getArgTypes()[0]);
24272429
Align Alignment = thisT()->DL.getABITypeAlign(Ty->getElementType());
2428-
return thisT()->getMemIntrinsicInstrCost(IID, Ty, /*Ptr=*/nullptr,
2429-
/*VariableMask=*/true, Alignment,
2430-
CostKind, ICA.getInst());
2430+
return thisT()->getMemIntrinsicInstrCost({IID, Ty, /*Ptr=*/nullptr,
2431+
/*VariableMask=*/true,
2432+
Alignment, ICA.getInst()},
2433+
CostKind);
24312434
}
24322435
case Intrinsic::experimental_vp_strided_load: {
24332436
auto *Ty = cast<VectorType>(ICA.getReturnType());
24342437
Align Alignment = thisT()->DL.getABITypeAlign(Ty->getElementType());
2435-
return thisT()->getMemIntrinsicInstrCost(IID, Ty, /*Ptr=*/nullptr,
2436-
/*VariableMask=*/true, Alignment,
2437-
CostKind, ICA.getInst());
2438+
return thisT()->getMemIntrinsicInstrCost({IID, Ty, /*Ptr=*/nullptr,
2439+
/*VariableMask=*/true,
2440+
Alignment, ICA.getInst()},
2441+
CostKind);
24382442
}
24392443
case Intrinsic::vector_reduce_add:
24402444
case Intrinsic::vector_reduce_mul:
@@ -3024,10 +3028,16 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
30243028

30253029
/// Get memory intrinsic cost based on arguments.
30263030
InstructionCost
3027-
getMemIntrinsicInstrCost(Intrinsic::ID Id, Type *DataTy, const Value *Ptr,
3028-
bool VariableMask, Align Alignment,
3029-
TTI::TargetCostKind CostKind,
3030-
const Instruction *I) const override {
3031+
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
3032+
TTI::TargetCostKind CostKind) const override {
3033+
unsigned Id = MICA.getID();
3034+
Type *DataTy = MICA.getDataType();
3035+
const Value *Ptr = MICA.getPointer();
3036+
const Instruction *I = MICA.getInst();
3037+
bool VariableMask = MICA.getVariableMask();
3038+
unsigned AS = MICA.getAddressSpace();
3039+
Align Alignment = MICA.getAlignment();
3040+
30313041
switch (Id) {
30323042
case Intrinsic::experimental_vp_strided_load:
30333043
case Intrinsic::experimental_vp_strided_store: {
@@ -3052,7 +3062,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
30523062
case Intrinsic::masked_store: {
30533063
unsigned Opcode =
30543064
Id == Intrinsic::masked_load ? Instruction::Load : Instruction::Store;
3055-
unsigned AS = Ptr ? Ptr->getType()->getPointerAddressSpace() : 0;
30563065
return thisT()->getMaskedMemoryOpCost(Opcode, DataTy, Alignment, AS,
30573066
CostKind);
30583067
}

‎llvm/lib/Analysis/TargetTransformInfo.cpp‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1202,10 +1202,9 @@ TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
12021202
}
12031203

12041204
InstructionCost TargetTransformInfo::getMemIntrinsicInstrCost(
1205-
unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1206-
Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
1207-
InstructionCost Cost = TTIImpl->getMemIntrinsicInstrCost(
1208-
Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
1205+
const MemIntrinsicCostAttributes &MICA,
1206+
TTI::TargetCostKind CostKind) const {
1207+
InstructionCost Cost = TTIImpl->getMemIntrinsicInstrCost(MICA, CostKind);
12091208
assert(Cost >= 0 && "TTI should not produce negative costs!");
12101209
return Cost;
12111210
}

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5200,8 +5200,8 @@ LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
52005200
unsigned IID = I->getOpcode() == Instruction::Load
52015201
? Intrinsic::masked_load
52025202
: Intrinsic::masked_store;
5203-
Cost += TTI.getMemIntrinsicInstrCost(
5204-
IID, VectorTy, Ptr, /*VariableMask*/ true, Alignment, CostKind);
5203+
Cost +=
5204+
TTI.getMemIntrinsicInstrCost({IID, VectorTy, Alignment, AS}, CostKind);
52055205
} else {
52065206
TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
52075207
Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
@@ -5264,9 +5264,9 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
52645264
? Intrinsic::masked_gather
52655265
: Intrinsic::masked_scatter;
52665266
return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5267-
TTI.getMemIntrinsicInstrCost(IID, VectorTy, Ptr,
5268-
Legal->isMaskRequired(I), Alignment,
5269-
CostKind, I);
5267+
TTI.getMemIntrinsicInstrCost(
5268+
{IID, VectorTy, Ptr, Legal->isMaskRequired(I), Alignment, I},
5269+
CostKind);
52705270
}
52715271

52725272
InstructionCost

0 commit comments

Comments
 (0)