[CSSPGO] Support of CS profiles in extended binary format.

This change brings up support of context-sensitive profiles in the format of extended binary. Existing sample profile reader/writer/merger code is being tweaked to reflect the fact of bracketed input contexts, like (`[...]`). The paired brackets are also needed in extbinary profiles because we don't yet have an otherwise good way to tell calling contexts apart from regular function names since the context delimiter `@` can somehow serve as a part of the C++ mangled names. Reviewed By: wmi, wenlei Differential Revision: https://reviews.llvm.org/D95547

Author: htyu

llvm/include/llvm/ProfileData/SampleProf.h

@@ -439,16 +439,19 @@ class SampleContext {
  void clearState(ContextStateMask S) { State &= (uint32_t)~S; }
  bool hasContext() const { return State != UnknownContext; }
  bool isBaseContext() const { return CallingContext.empty(); }
- StringRef getName() const { return Name; }
+ StringRef getNameWithoutContext() const { return Name; }
  StringRef getCallingContext() const { return CallingContext; }
- StringRef getNameWithContext() const { return FullContext; }
+ StringRef getNameWithContext(bool WithBracket = false) const {
+ return WithBracket ? InputContext : FullContext;
+ }
 
 private:
  // Give a context string, decode and populate internal states like
  // Function name, Calling context and context state. Example of input
  // `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`
  void setContext(StringRef ContextStr, ContextStateMask CState) {
  assert(!ContextStr.empty());
+ InputContext = ContextStr;
  // Note that `[]` wrapped input indicates a full context string, otherwise
  // it's treated as context-less function name only.
  bool HasContext = ContextStr.startswith("[");
@@ -480,6 +483,9 @@ class SampleContext {
  }
  }
 
+ // Input context string including bracketed calling context and leaf function
+ // name
+ StringRef InputContext;
  // Full context string including calling context and leaf function name
  StringRef FullContext;
  // Function name for the associated sample profile
@@ -676,7 +682,8 @@ class FunctionSamples {
  Name = Other.getName();
  if (!GUIDToFuncNameMap)
  GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
-
+ if (Context.getNameWithContext(true).empty())
+ Context = Other.getContext();
  if (FunctionHash == 0) {
  // Set the function hash code for the target profile.
  FunctionHash = Other.getFunctionHash();
@@ -743,8 +750,10 @@ class FunctionSamples {
  StringRef getName() const { return Name; }
 
  /// Return function name with context.
- StringRef getNameWithContext() const {
- return FunctionSamples::ProfileIsCS ? Context.getNameWithContext() : Name;
+ StringRef getNameWithContext(bool WithBracket = false) const {
+ return FunctionSamples::ProfileIsCS
+ ? Context.getNameWithContext(WithBracket)
+ : Name;
  }
 
  /// Return the original function name.

llvm/include/llvm/ProfileData/SampleProfReader.h

@@ -488,8 +488,12 @@ class SampleProfileReader {
  /// \brief Whether samples are collected based on pseudo probes.
  bool ProfileIsProbeBased = false;
 
+ /// Whether function profiles are context-sensitive.
  bool ProfileIsCS = false;
 
+ /// Number of context-sensitive profiles.
+ uint32_t CSProfileCount = 0;
+
  /// \brief The format of sample.
  SampleProfileFormat Format = SPF_None;
 };

llvm/lib/ProfileData/SampleProfReader.cpp

@@ -222,8 +222,6 @@ std::error_code SampleProfileReaderText::readImpl() {
  sampleprof_error Result = sampleprof_error::success;
 
  InlineCallStack InlineStack;
- int CSProfileCount = 0;
- int RegularProfileCount = 0;
  uint32_t ProbeProfileCount = 0;
 
  // SeenMetadata tracks whether we have processed metadata for the current
@@ -257,11 +255,9 @@ std::error_code SampleProfileReaderText::readImpl() {
  SampleContext FContext(FName);
  if (FContext.hasContext())
  ++CSProfileCount;
- else
- ++RegularProfileCount;
  Profiles[FContext] = FunctionSamples();
  FunctionSamples &FProfile = Profiles[FContext];
- FProfile.setName(FContext.getName());
+ FProfile.setName(FContext.getNameWithoutContext());
  FProfile.setContext(FContext);
  MergeResult(Result, FProfile.addTotalSamples(NumSamples));
  MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples));
@@ -324,13 +320,14 @@ std::error_code SampleProfileReaderText::readImpl() {
  }
  }
 
- assert((RegularProfileCount == 0 || CSProfileCount == 0) &&
+ assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
  "Cannot have both context-sensitive and regular profile");
  ProfileIsCS = (CSProfileCount > 0);
  assert((ProbeProfileCount == 0 || ProbeProfileCount == Profiles.size()) &&
  "Cannot have both probe-based profiles and regular profiles");
  ProfileIsProbeBased = (ProbeProfileCount > 0);
  FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
+ FunctionSamples::ProfileIsCS = ProfileIsCS;
 
  if (Result == sampleprof_error::success)
  computeSummary();
@@ -546,12 +543,16 @@ SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
  if (std::error_code EC = FName.getError())
  return EC;
 
- Profiles[*FName] = FunctionSamples();
- FunctionSamples &FProfile = Profiles[*FName];
- FProfile.setName(*FName);
-
+ SampleContext FContext(*FName);
+ Profiles[FContext] = FunctionSamples();
+ FunctionSamples &FProfile = Profiles[FContext];
+ FProfile.setName(FContext.getNameWithoutContext());
+ FProfile.setContext(FContext);
  FProfile.addHeadSamples(*NumHeadSamples);
 
+ if (FContext.hasContext())
+ CSProfileCount++;
+
  if (std::error_code EC = readProfile(FProfile))
  return EC;
  return sampleprof_error::success;
@@ -654,40 +655,44 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
  return EC;
  }
  assert(Data == End && "More data is read than expected");
- return sampleprof_error::success;
- }
-
- if (Remapper) {
- for (auto Name : FuncsToUse) {
- Remapper->insert(Name);
+ } else {
+ if (Remapper) {
+ for (auto Name : FuncsToUse) {
+ Remapper->insert(Name);
+ }
  }
- }
 
- if (useMD5()) {
- for (auto Name : FuncsToUse) {
- auto GUID = std::to_string(MD5Hash(Name));
- auto iter = FuncOffsetTable.find(StringRef(GUID));
- if (iter == FuncOffsetTable.end())
- continue;
- const uint8_t *FuncProfileAddr = Start + iter->second;
- assert(FuncProfileAddr < End && "out of LBRProfile section");
- if (std::error_code EC = readFuncProfile(FuncProfileAddr))
- return EC;
- }
- } else {
- for (auto NameOffset : FuncOffsetTable) {
- auto FuncName = NameOffset.first;
- if (!FuncsToUse.count(FuncName) &&
- (!Remapper || !Remapper->exist(FuncName)))
- continue;
- const uint8_t *FuncProfileAddr = Start + NameOffset.second;
- assert(FuncProfileAddr < End && "out of LBRProfile section");
- if (std::error_code EC = readFuncProfile(FuncProfileAddr))
- return EC;
+ if (useMD5()) {
+ for (auto Name : FuncsToUse) {
+ auto GUID = std::to_string(MD5Hash(Name));
+ auto iter = FuncOffsetTable.find(StringRef(GUID));
+ if (iter == FuncOffsetTable.end())
+ continue;
+ const uint8_t *FuncProfileAddr = Start + iter->second;
+ assert(FuncProfileAddr < End && "out of LBRProfile section");
+ if (std::error_code EC = readFuncProfile(FuncProfileAddr))
+ return EC;
+ }
+ } else {
+ for (auto NameOffset : FuncOffsetTable) {
+ SampleContext FContext(NameOffset.first);
+ auto FuncName = FContext.getNameWithoutContext();
+ if (!FuncsToUse.count(FuncName) &&
+ (!Remapper || !Remapper->exist(FuncName)))
+ continue;
+ const uint8_t *FuncProfileAddr = Start + NameOffset.second;
+ assert(FuncProfileAddr < End && "out of LBRProfile section");
+ if (std::error_code EC = readFuncProfile(FuncProfileAddr))
+ return EC;
+ }
  }
+ Data = End;
  }
 
- Data = End;
+ assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
+ "Cannot have both context-sensitive and regular profile");
+ ProfileIsCS = (CSProfileCount > 0);
+ FunctionSamples::ProfileIsCS = ProfileIsCS;
  return sampleprof_error::success;
 }
 
@@ -887,7 +892,8 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() {
  if (std::error_code EC = Checksum.getError())
  return EC;
 
- Profiles[*FName].setFunctionHash(*Checksum);
+ SampleContext FContext(*FName);
+ Profiles[FContext].setFunctionHash(*Checksum);
  }
  return sampleprof_error::success;
 }

llvm/lib/ProfileData/SampleProfWriter.cpp

@@ -147,7 +147,7 @@ std::error_code SampleProfileWriterExtBinaryBase::write(
 std::error_code
 SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
  uint64_t Offset = OutputStream->tell();
- StringRef Name = S.getName();
+ StringRef Name = S.getNameWithContext(true);
  FuncOffsetTable[Name] = Offset - SecLBRProfileStart;
  encodeULEB128(S.getHeadSamples(), *OutputStream);
  return writeBody(S);
@@ -635,7 +635,7 @@ std::error_code SampleProfileWriterBinary::writeSummary() {
 std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
  auto &OS = *OutputStream;
 
- if (std::error_code EC = writeNameIdx(S.getName()))
+ if (std::error_code EC = writeNameIdx(S.getNameWithContext(true)))
  return EC;
 
  encodeULEB128(S.getTotalSamples(), OS);

llvm/lib/Transforms/IPO/SampleContextTracker.cpp

@@ -179,7 +179,7 @@ SampleContextTracker::SampleContextTracker(
  SampleContext Context(FuncSample.first(), RawContext);
  LLVM_DEBUG(dbgs() << "Tracking Context for function: " << Context << "\n");
  if (!Context.isBaseContext())
- FuncToCtxtProfileSet[Context.getName()].insert(FSamples);
+ FuncToCtxtProfileSet[Context.getNameWithoutContext()].insert(FSamples);
  ContextTrieNode *NewNode = getOrCreateContextPath(Context, true);
  assert(!NewNode->getFunctionSamples() &&
  "New node can't have sample profile");

llvm/test/Transforms/SampleProfile/profile-context-tracker.ll

@@ -1,18 +1,22 @@
 ; Test for CSSPGO's SampleContextTracker to make sure context profile tree is promoted and merged properly
 ; based on inline decision, so post inline counts are accurate.
 
+; RUN: llvm-profdata merge --sample --extbinary %S/Inputs/profile-context-tracker.prof -o %t
+
 ; Note that we need new pass manager to enable top-down processing for sample profile loader
 ; Testwe we inlined the following in top-down order and entry counts accurate reflects post-inline base profile
 ; main:3 @ _Z5funcAi
 ; main:3 @ _Z5funcAi:1 @ _Z8funcLeafi
 ; _Z5funcBi:1 @ _Z8funcLeafi
 ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-context-tracker.prof -sample-profile-inline-size -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-ALL
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%t -sample-profile-inline-size -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-ALL
 
 ; Testwe we inlined the following in top-down order and entry counts accurate reflects post-inline base profile
 ; main:3 @ _Z5funcAi
 ; _Z5funcAi:1 @ _Z8funcLeafi
 ; _Z5funcBi:1 @ _Z8funcLeafi
 ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-context-tracker.prof -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-HOT
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%t -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-HOT
 
 
 @factor = dso_local global i32 3, align 4, !dbg !0

llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext

@@ -0,0 +1,36 @@
+[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]:1467299:11
+ 0: 6
+ 1: 6
+ 3: 287884
+ 4: 287864 _Z3fibi:315608
+ 15: 23
+[main:3.1 @ _Z5funcBi:1 @ _Z8funcLeafi]:500853:20
+ 0: 15
+ 1: 15
+ 3: 74946
+ 4: 74941 _Z3fibi:82359
+ 10: 23324
+ 11: 23327 _Z3fibi:25228
+ 15: 11
+[main]:154:0
+ 2: 12
+ 3: 18 _Z5funcAi:11
+ 3.1: 18 _Z5funcBi:19
+[external:12 @ main]:154:12
+ 2: 12
+ 3: 10 _Z5funcAi:7
+ 3.1: 10 _Z5funcBi:11
+[main:3.1 @ _Z5funcBi]:120:19
+ 0: 19
+ 1: 19 _Z8funcLeafi:20
+ 3: 12
+[externalA:17 @ _Z5funcBi]:120:3
+ 0: 3
+ 1: 3
+[external:10 @ _Z5funcBi]:120:10
+ 0: 10
+ 1: 10
+[main:3 @ _Z5funcAi]:99:11
+ 0: 10
+ 1: 10 _Z8funcLeafi:11
+ 3: 24

llvm/test/tools/llvm-profdata/cs-sample-profile.test

@@ -0,0 +1,4 @@
+RUN: llvm-profdata merge --sample --text -output=%t.proftext %S/Inputs/cs-sample.proftext
+RUN: diff -b %t.proftext %S/Inputs/cs-sample.proftext
+RUN: llvm-profdata merge --sample --extbinary %p/Inputs/cs-sample.proftext -o %t.prof && llvm-profdata merge --sample --text %t.prof -o %t1.proftext
+RUN: diff -b %t1.proftext %S/Inputs/cs-sample.proftext

llvm/tools/llvm-profdata/llvm-profdata.cpp

@@ -696,7 +696,7 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
  Remapper ? remapSamples(I->second, *Remapper, Result)
  : FunctionSamples();
  FunctionSamples &Samples = Remapper ? Remapped : I->second;
- StringRef FName = Samples.getName();
+ StringRef FName = Samples.getNameWithContext(true);
  MergeResult(Result, ProfileMap[FName].merge(Samples, Input.Weight));
  if (Result != sampleprof_error::success) {
  std::error_code EC = make_error_code(Result);

llvm/tools/llvm-profgen/ProfileGenerator.cpp

@@ -164,7 +164,7 @@ CSProfileGenerator::getFunctionProfileForContext(StringRef ContextStr) {
  if (Ret.second) {
  SampleContext FContext(Ret.first->first(), RawContext);
  FunctionSamples &FProfile = Ret.first->second;
- FProfile.setName(FContext.getName());
+ FProfile.setName(FContext.getNameWithoutContext());
  FProfile.setContext(FContext);
  }
  return Ret.first->second;