[TTI] Remove getMaskedMemoryOpCost from Analysis/TargetTransformInfo

Author: arcbbb

llvm/include/llvm/Analysis/TargetTransformInfo.h

@@ -1554,11 +1554,6 @@ class TargetTransformInfo {
  OperandValueInfo OpdInfo = {OK_AnyValue, OP_None},
  const Instruction *I = nullptr) const;
 
- /// \return The cost of masked Load and Store instructions.
- LLVM_ABI InstructionCost getMaskedMemoryOpCost(
- unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
- TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;
-
  /// \return The cost of Expand Load or Compress Store operation
  /// \p Opcode - is a type of memory access Load or Store
  /// \p Src - a vector type of the data to be loaded or stored

llvm/include/llvm/CodeGen/BasicTTIImpl.h

@@ -1606,10 +1606,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
  // Firstly, the cost of load/store operation.
  InstructionCost Cost;
- if (UseMaskForCond || UseMaskForGaps)
- Cost = thisT()->getMaskedMemoryOpCost(Opcode, VecTy, Alignment,
- AddressSpace, CostKind);
- else
+ if (UseMaskForCond || UseMaskForGaps) {
+ unsigned IID = (Opcode == Instruction::Load) ? Intrinsic::masked_load
+ : Intrinsic::masked_store;
+ Cost = thisT()->getMemIntrinsicInstrCost(
+ IID, VecTy, /*Ptr*/ nullptr, /*VariableMask*/ false, Alignment,
+ CostKind, /*Instruction*/ nullptr);
+ } else
  Cost = thisT()->getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace,
  CostKind);
 
@@ -2408,14 +2411,16 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
  case Intrinsic::masked_store: {
  Type *Ty = Tys[0];
  Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
- return thisT()->getMaskedMemoryOpCost(Instruction::Store, Ty, TyAlign, 0,
- CostKind);
+ return thisT()->getMemIntrinsicInstrCost(
+ Intrinsic::masked_store, Ty,
+ /*Ptr*/ nullptr, /*VariableMask*/ false, TyAlign, CostKind, nullptr);
  }
  case Intrinsic::masked_load: {
  Type *Ty = RetTy;
  Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
- return thisT()->getMaskedMemoryOpCost(Instruction::Load, Ty, TyAlign, 0,
- CostKind);
+ return thisT()->getMemIntrinsicInstrCost(
+ Intrinsic::masked_load, Ty,
+ /*Ptr*/ nullptr, /*VariableMask*/ false, TyAlign, CostKind, nullptr);
  }
  case Intrinsic::experimental_vp_strided_store: {
  auto *Ty = cast<VectorType>(ICA.getArgTypes()[0]);
@@ -3043,6 +3048,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
  return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
  Alignment, CostKind, I);
  }
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store: {
+ unsigned Opcode = (Id == Intrinsic::masked_load) ? Instruction::Load
+ : Instruction::Store;
+ unsigned AS = Ptr ? Ptr->getType()->getPointerAddressSpace() : 0;
+ return thisT()->getMaskedMemoryOpCost(Opcode, DataTy, Alignment, AS,
+ CostKind);
+ }
  default:
  llvm_unreachable("unexpected intrinsic");
  }

llvm/lib/Analysis/TargetTransformInfo.cpp

@@ -1182,15 +1182,6 @@ InstructionCost TargetTransformInfo::getMemoryOpCost(
  return Cost;
 }
 
-InstructionCost TargetTransformInfo::getMaskedMemoryOpCost(
- unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
- TTI::TargetCostKind CostKind) const {
- InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment,
- AddressSpace, CostKind);
- assert(Cost >= 0 && "TTI should not produce negative costs!");
- return Cost;
-}
-
 InstructionCost TargetTransformInfo::getExpandCompressMemoryOpCost(
  unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
  TTI::TargetCostKind CostKind, const Instruction *I) const {

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -5197,8 +5197,11 @@ LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
  const Align Alignment = getLoadStoreAlignment(I);
  InstructionCost Cost = 0;
  if (Legal->isMaskRequired(I)) {
- Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
- CostKind);
+ unsigned IID = (I->getOpcode() == Instruction::Load)
+ ? Intrinsic::masked_load
+ : Intrinsic::masked_store;
+ Cost += TTI.getMemIntrinsicInstrCost(
+ IID, VectorTy, Ptr, /*VariableMask*/ false, Alignment, CostKind);
  } else {
  TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
  Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

@@ -6848,9 +6848,9 @@ static bool isMaskedLoadCompress(
  ScalarLoadsCost;
  InstructionCost LoadCost = 0;
  if (IsMasked) {
- LoadCost =
- TTI.getMaskedMemoryOpCost(Instruction::Load, LoadVecTy, CommonAlignment,
-  LI->getPointerAddressSpace(), CostKind);
+ LoadCost = TTI.getMemIntrinsicInstrCost(
+ Intrinsic::masked_load, LoadVecTy, LI->getPointerOperand(),
+ /*VariableMask*/ false, CommonAlignment, CostKind);
  } else {
  LoadCost =
  TTI.getMemoryOpCost(Instruction::Load, LoadVecTy, CommonAlignment,
@@ -7249,12 +7249,13 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
  VectorGEPCost;
  break;
  case LoadsState::CompressVectorize:
- VecLdCost += TTI.getMaskedMemoryOpCost(
- Instruction::Load, SubVecTy, CommonAlignment,
- LI0->getPointerAddressSpace(), CostKind) +
- VectorGEPCost +
- ::getShuffleCost(TTI, TTI::SK_PermuteSingleSrc, SubVecTy,
- {}, CostKind);
+ VecLdCost +=
+ TTI.getMemIntrinsicInstrCost(
+ Intrinsic::masked_load, SubVecTy, LI0->getPointerOperand(),
+ /*VariableMask*/ false, CommonAlignment, CostKind) +
+ VectorGEPCost +
+ ::getShuffleCost(TTI, TTI::SK_PermuteSingleSrc, SubVecTy, {},
+ CostKind);
  break;
  case LoadsState::ScatterVectorize:
  VecLdCost +=
@@ -15044,9 +15045,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
  Instruction::Load, LoadVecTy, InterleaveFactor, {},
  CommonAlignment, LI0->getPointerAddressSpace(), CostKind);
  } else if (IsMasked) {
- VecLdCost = TTI->getMaskedMemoryOpCost(
- Instruction::Load, LoadVecTy, CommonAlignment,
- LI0->getPointerAddressSpace(), CostKind);
+ VecLdCost = TTI->getMemIntrinsicInstrCost(
+ Intrinsic::masked_load, LoadVecTy, LI0->getPointerOperand(),
+ /*VariableMask*/ false, CommonAlignment, CostKind);
  // TODO: include this cost into CommonCost.
  VecLdCost += ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
  LoadVecTy, CompressMask, CostKind);

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -3570,8 +3570,11 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
 
  InstructionCost Cost = 0;
  if (IsMasked) {
- Cost +=
- Ctx.TTI.getMaskedMemoryOpCost(Opcode, Ty, Alignment, AS, Ctx.CostKind);
+ unsigned IID = isa<VPWidenLoadRecipe>(this) ? Intrinsic::masked_load
+ : Intrinsic::masked_store;
+ const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
+ Cost += Ctx.TTI.getMemIntrinsicInstrCost(
+ IID, Ty, Ptr, /*VariableMask*/ false, Alignment, Ctx.CostKind);
  } else {
  TTI::OperandValueInfo OpInfo = Ctx.getOperandInfo(
  isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(this) ? getOperand(0)
@@ -3681,16 +3684,17 @@ InstructionCost VPWidenLoadEVLRecipe::computeCost(ElementCount VF,
  if (!Consecutive || IsMasked)
  return VPWidenMemoryRecipe::computeCost(VF, Ctx);
 
- // We need to use the getMaskedMemoryOpCost() instead of getMemoryOpCost()
+ // We need to use the getMemIntrinsicInstrCost() instead of getMemoryOpCost()
  // here because the EVL recipes using EVL to replace the tail mask. But in the
  // legacy model, it will always calculate the cost of mask.
- // TODO: Using getMemoryOpCost() instead of getMaskedMemoryOpCost when we
+ // TODO: Using getMemoryOpCost() instead of getMemIntrinsicInstrCost when we
  // don't need to compare to the legacy cost model.
  Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
- unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
- ->getAddressSpace();
- InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(
- Instruction::Load, Ty, Alignment, AS, Ctx.CostKind);
+ const Align Alignment = getLoadStoreAlignment(&Ingredient);
+ const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
+ InstructionCost Cost = Ctx.TTI.getMemIntrinsicInstrCost(
+ Intrinsic::masked_load, Ty, Ptr, /*VariableMask*/ false, Alignment,
+ Ctx.CostKind);
  if (!Reverse)
  return Cost;
 
@@ -3790,16 +3794,17 @@ InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
  if (!Consecutive || IsMasked)
  return VPWidenMemoryRecipe::computeCost(VF, Ctx);
 
- // We need to use the getMaskedMemoryOpCost() instead of getMemoryOpCost()
+ // We need to use the getMemIntrinsicInstrCost() instead of getMemoryOpCost()
  // here because the EVL recipes using EVL to replace the tail mask. But in the
  // legacy model, it will always calculate the cost of mask.
- // TODO: Using getMemoryOpCost() instead of getMaskedMemoryOpCost when we
+ // TODO: Using getMemoryOpCost() instead of getMemIntrinsicInstrCost when we
  // don't need to compare to the legacy cost model.
  Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
- unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
- ->getAddressSpace();
- InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(
- Instruction::Store, Ty, Alignment, AS, Ctx.CostKind);
+ const Align Alignment = getLoadStoreAlignment(&Ingredient);
+ const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
+ InstructionCost Cost = Ctx.TTI.getMemIntrinsicInstrCost(
+ Intrinsic::masked_store, Ty, Ptr, /*VariableMask*/ false, Alignment,
+ Ctx.CostKind);
  if (!Reverse)
  return Cost;