[JIT] Add support to inline the field access of primitive types marked with TLS

src/coreclr/inc/corinfo.h

@@ -546,6 +546,7 @@ enum CorInfoHelpFunc
  CORINFO_HELP_GETSHARED_NONGCTHREADSTATIC_BASE_NOCTOR,
  CORINFO_HELP_GETSHARED_GCTHREADSTATIC_BASE_DYNAMICCLASS,
  CORINFO_HELP_GETSHARED_NONGCTHREADSTATIC_BASE_DYNAMICCLASS,
+ CORINFO_HELP_GETSHARED_NONGCTHREADSTATIC_BASE_NOCTOR_OPTIMIZED,
 
  /* Debugger */
 
@@ -1686,6 +1687,7 @@ enum CORINFO_FIELD_ACCESSOR
  CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER, // static field access using the "generic static" helper (argument is MethodTable *)
  CORINFO_FIELD_STATIC_ADDR_HELPER, // static field accessed using address-of helper (argument is FieldDesc *)
  CORINFO_FIELD_STATIC_TLS, // unmanaged TLS access
+ CORINFO_FIELD_STATIC_TLS_MANAGED, // managed TLS access
  CORINFO_FIELD_STATIC_READYTORUN_HELPER, // static field access using a runtime lookup helper
  CORINFO_FIELD_STATIC_RELOCATABLE, // static field access using relocation (used in AOT)
  CORINFO_FIELD_INTRINSIC_ZERO, // intrinsic zero (IntPtr.Zero, UIntPtr.Zero)
@@ -1725,6 +1727,17 @@ struct CORINFO_FIELD_INFO
  CORINFO_CONST_LOOKUP fieldLookup; // Used by Ready-to-Run
 };
 
+//----------------------------------------------------------------------------
+// getThreadLocalStaticBlocksInfo and CORINFO_THREAD_STATIC_BLOCKS_INFO: The EE instructs the JIT about how to access a thread local field
+
+struct CORINFO_THREAD_STATIC_BLOCKS_INFO
+{
+ CORINFO_CONST_LOOKUP tlsIndex;
+ uint32_t offsetOfThreadLocalStoragePointer;
+ uint32_t offsetOfMaxThreadStaticBlocks;
+ uint32_t offsetOfThreadStaticBlocks;
+};
+
 //----------------------------------------------------------------------------
 // Exception handling
 
@@ -2743,6 +2756,12 @@ class ICorStaticInfo
  CORINFO_FIELD_INFO *pResult
  ) = 0;
 
+ virtual uint32_t getThreadLocalFieldInfo (
+ CORINFO_FIELD_HANDLE field) = 0;
+
+ virtual void getThreadLocalStaticBlocksInfo (
+ CORINFO_THREAD_STATIC_BLOCKS_INFO* pInfo) = 0;
+
  // Returns true iff "fldHnd" represents a static field.
  virtual bool isFieldStatic(CORINFO_FIELD_HANDLE fldHnd) = 0;
 

src/coreclr/inc/icorjitinfoimpl_generated.h

@@ -414,6 +414,12 @@ void getFieldInfo(
  CORINFO_ACCESS_FLAGS flags,
  CORINFO_FIELD_INFO* pResult) override;
 
+uint32_t getThreadLocalFieldInfo(
+ CORINFO_FIELD_HANDLE field) override;
+
+void getThreadLocalStaticBlocksInfo(
+ CORINFO_THREAD_STATIC_BLOCKS_INFO* pInfo) override;
+
 bool isFieldStatic(
  CORINFO_FIELD_HANDLE fldHnd) override;
 

src/coreclr/inc/jiteeversionguid.h

@@ -43,11 +43,11 @@ typedef const GUID *LPCGUID;
 #define GUID_DEFINED
 #endif // !GUID_DEFINED
 
-constexpr GUID JITEEVersionIdentifier =  { /* 3054e9ba-bcfe-417c-9043-92ccc8738b80 */
- 0x3054e9ba,
- 0xbcfe,
- 0x417c,
- {0x90, 0x43, 0x92, 0xcc, 0xc8, 0x73, 0x8b, 0x80}
+constexpr GUID JITEEVersionIdentifier = { /* 236d7997-3d71-45f9-b7d7-5241ad89a56f */
+ 0x236d7997,
+ 0x3d71,
+ 0x45f9,
+ { 0xb7, 0xd7, 0x52, 0x41, 0xad, 0x89, 0xa5, 0x6f }
  };
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////

src/coreclr/inc/jithelpers.h

@@ -203,6 +203,7 @@
  JITHELPER(CORINFO_HELP_GETSHARED_NONGCTHREADSTATIC_BASE_NOCTOR, JIT_GetSharedNonGCThreadStaticBase, CORINFO_HELP_SIG_REG_ONLY)
  JITHELPER(CORINFO_HELP_GETSHARED_GCTHREADSTATIC_BASE_DYNAMICCLASS, JIT_GetSharedGCThreadStaticBaseDynamicClass, CORINFO_HELP_SIG_REG_ONLY)
  JITHELPER(CORINFO_HELP_GETSHARED_NONGCTHREADSTATIC_BASE_DYNAMICCLASS, JIT_GetSharedNonGCThreadStaticBaseDynamicClass, CORINFO_HELP_SIG_REG_ONLY)
+ JITHELPER(CORINFO_HELP_GETSHARED_NONGCTHREADSTATIC_BASE_NOCTOR_OPTIMIZED, JIT_GetSharedNonGCThreadStaticBaseOptimized, CORINFO_HELP_SIG_REG_ONLY)
 
  // Debugger
  JITHELPER(CORINFO_HELP_DBG_IS_JUST_MY_CODE, JIT_DbgIsJustMyCode,CORINFO_HELP_SIG_REG_ONLY)

src/coreclr/jit/CMakeLists.txt

@@ -129,7 +129,7 @@ set( JIT_SOURCES
  hwintrinsic.cpp
  hostallocator.cpp
  ifconversion.cpp
- runtimelookup.cpp
+ helperexpansion.cpp
  indirectcalltransformer.cpp
  importercalls.cpp
  importer.cpp

src/coreclr/jit/ICorJitInfo_names_generated.h

@@ -103,6 +103,8 @@ DEF_CLR_API(getFieldClass)
 DEF_CLR_API(getFieldType)
 DEF_CLR_API(getFieldOffset)
 DEF_CLR_API(getFieldInfo)
+DEF_CLR_API(getThreadLocalFieldInfo)
+DEF_CLR_API(getThreadLocalStaticBlocksInfo)
 DEF_CLR_API(isFieldStatic)
 DEF_CLR_API(getArrayOrStringLength)
 DEF_CLR_API(getBoundaries)

src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp

@@ -981,6 +981,23 @@ void WrapICorJitInfo::getFieldInfo(
  API_LEAVE(getFieldInfo);
 }
 
+uint32_t WrapICorJitInfo::getThreadLocalFieldInfo(
+ CORINFO_FIELD_HANDLE field)
+{
+ API_ENTER(getThreadLocalFieldInfo);
+ uint32_t temp = wrapHnd->getThreadLocalFieldInfo(field);
+ API_LEAVE(getThreadLocalFieldInfo);
+ return temp;
+}
+
+void WrapICorJitInfo::getThreadLocalStaticBlocksInfo(
+ CORINFO_THREAD_STATIC_BLOCKS_INFO* pInfo)
+{
+ API_ENTER(getThreadLocalStaticBlocksInfo);
+ wrapHnd->getThreadLocalStaticBlocksInfo(pInfo);
+ API_LEAVE(getThreadLocalStaticBlocksInfo);
+}
+
 bool WrapICorJitInfo::isFieldStatic(
  CORINFO_FIELD_HANDLE fldHnd)
 {

src/coreclr/jit/codegenxarch.cpp

@@ -5420,8 +5420,13 @@ void CodeGen::genCodeForIndir(GenTreeIndir* tree)
  if (addr->IsIconHandle(GTF_ICON_TLS_HDL))
  {
  noway_assert(EA_ATTR(genTypeSize(targetType)) == EA_PTRSIZE);
+#if TARGET_64BIT
+ emit->emitIns_R_C(ins_Load(TYP_I_IMPL), EA_PTRSIZE, tree->GetRegNum(), FLD_GLOBAL_GS,
+ (int)addr->AsIntCon()->gtIconVal);
+#else
  emit->emitIns_R_C(ins_Load(TYP_I_IMPL), EA_PTRSIZE, tree->GetRegNum(), FLD_GLOBAL_FS,
  (int)addr->AsIntCon()->gtIconVal);
+#endif
  }
  else
  {

src/coreclr/jit/compiler.cpp

@@ -5040,6 +5040,12 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
  // Partially inline static initializations
  DoPhase(this, PHASE_EXPAND_STATIC_INIT, &Compiler::fgExpandStaticInit);
 
+ if (TargetOS::IsWindows)
+ {
+ // Currently this is only applicable for Windows
+ DoPhase(this, PHASE_EXPAND_TLS, &Compiler::fgExpandThreadLocalAccess);
+ }
+
  // Insert GC Polls
  DoPhase(this, PHASE_INSERT_GC_POLLS, &Compiler::fgInsertGCPolls);
 

src/coreclr/jit/compiler.h

@@ -4862,7 +4862,7 @@ class Compiler
 
  GenTree* fgInitThisClass();
 
- GenTreeCall* fgGetStaticsCCtorHelper(CORINFO_CLASS_HANDLE cls, CorInfoHelpFunc helper);
+ GenTreeCall* fgGetStaticsCCtorHelper(CORINFO_CLASS_HANDLE cls, CorInfoHelpFunc helper, uint32_t typeIndex = 0);
 
  GenTreeCall* fgGetSharedCCtor(CORINFO_CLASS_HANDLE cls);
 
@@ -5311,11 +5311,21 @@ class Compiler
  PhaseStatus StressSplitTree();
  void SplitTreesRandomly();
  void SplitTreesRemoveCommas();
+
+ template <bool (Compiler::*ExpansionFunction)(BasicBlock*, Statement*, GenTreeCall*)>
+ PhaseStatus fgExpandHelper(bool skipRarelyRunBlocks);
+
+ template <bool (Compiler::*ExpansionFunction)(BasicBlock*, Statement*, GenTreeCall*)>
+ bool fgExpandHelperForBlock(BasicBlock* block);
+
  PhaseStatus fgExpandRuntimeLookups();
+ bool fgExpandRuntimeLookupsForCall(BasicBlock* block, Statement* stmt, GenTreeCall* call);
+
+ PhaseStatus fgExpandThreadLocalAccess();
+ bool fgExpandThreadLocalAccessForCall(BasicBlock* block, Statement* stmt, GenTreeCall* call);
 
- bool fgExpandStaticInitForBlock(BasicBlock* block);
- bool fgExpandStaticInitForCall(BasicBlock* block, Statement* stmt, GenTreeCall* call);
  PhaseStatus fgExpandStaticInit();
+ bool fgExpandStaticInitForCall(BasicBlock* block, Statement* stmt, GenTreeCall* call);
 
  PhaseStatus fgInsertGCPolls();
  BasicBlock* fgCreateGCPoll(GCPollType pollType, BasicBlock* block);
@@ -7039,6 +7049,7 @@ class Compiler
 #define OMF_HAS_MDNEWARRAY 0x00002000 // Method contains 'new' of an MD array
 #define OMF_HAS_MDARRAYREF 0x00004000 // Method contains multi-dimensional intrinsic array element loads or stores.
 #define OMF_HAS_STATIC_INIT 0x00008000 // Method has static initializations we might want to partially inline
+#define OMF_HAS_TLS_FIELD 0x00010000 // Method contains TLS field access
 
  // clang-format on
 
@@ -7089,6 +7100,16 @@ class Compiler
  optMethodFlags |= OMF_HAS_GUARDEDDEVIRT;
  }
 
+ bool doesMethodHasTlsFieldAccess()
+ {
+ return (optMethodFlags & OMF_HAS_TLS_FIELD) != 0;
+ }
+
+ void setMethodHasTlsFieldAccess()
+ {
+ optMethodFlags |= OMF_HAS_TLS_FIELD;
+ }
+
  void pickGDV(GenTreeCall* call,
  IL_OFFSET ilOffset,
  bool isInterface,
@@ -7790,6 +7811,7 @@ class Compiler
  void eeGetFieldInfo(CORINFO_RESOLVED_TOKEN* pResolvedToken,
  CORINFO_ACCESS_FLAGS flags,
  CORINFO_FIELD_INFO* pResult);
+ uint32_t eeGetThreadLocalFieldInfo(CORINFO_FIELD_HANDLE field);
 
  // Get the flags
 

src/coreclr/jit/compiler.hpp

@@ -3635,6 +3635,7 @@ inline bool Compiler::IsSharedStaticHelper(GenTree* tree)
  helper == CORINFO_HELP_GETSHARED_NONGCTHREADSTATIC_BASE ||
  helper == CORINFO_HELP_GETSHARED_GCTHREADSTATIC_BASE_NOCTOR ||
  helper == CORINFO_HELP_GETSHARED_NONGCTHREADSTATIC_BASE_NOCTOR ||
+ helper == CORINFO_HELP_GETSHARED_NONGCTHREADSTATIC_BASE_NOCTOR_OPTIMIZED ||
  helper == CORINFO_HELP_GETSHARED_GCTHREADSTATIC_BASE_DYNAMICCLASS ||
  helper == CORINFO_HELP_GETSHARED_NONGCTHREADSTATIC_BASE_DYNAMICCLASS ||
 #ifdef FEATURE_READYTORUN
@@ -3684,12 +3685,12 @@ inline bool Compiler::IsGcSafePoint(GenTreeCall* call)
 // Note that we want to have two special FIELD_HANDLES that will both
 // be considered non-Data Offset handles
 //
-// The special values that we use are FLD_GLOBAL_DS and FLD_GLOBAL_FS
+// The special values that we use are FLD_GLOBAL_DS, FLD_GLOBAL_FS or FLD_GLOBAL_GS.
 //
 
 inline bool jitStaticFldIsGlobAddr(CORINFO_FIELD_HANDLE fldHnd)
 {
- return (fldHnd == FLD_GLOBAL_DS || fldHnd == FLD_GLOBAL_FS);
+ return (fldHnd == FLD_GLOBAL_DS || fldHnd == FLD_GLOBAL_FS || fldHnd == FLD_GLOBAL_GS);
 }
 
 /*

src/coreclr/jit/compphases.h

@@ -93,6 +93,7 @@ CompPhaseNameMacro(PHASE_COMPUTE_EDGE_WEIGHTS2, "Compute edge weights (2, f
 CompPhaseNameMacro(PHASE_STRESS_SPLIT_TREE, "Stress gtSplitTree", false, -1, false)
 CompPhaseNameMacro(PHASE_EXPAND_RTLOOKUPS, "Expand runtime lookups", false, -1, true)
 CompPhaseNameMacro(PHASE_EXPAND_STATIC_INIT, "Expand static init", false, -1, true)
+CompPhaseNameMacro(PHASE_EXPAND_TLS, "Expand TLS access", false, -1, true)
 CompPhaseNameMacro(PHASE_INSERT_GC_POLLS, "Insert GC Polls", false, -1, true)
 CompPhaseNameMacro(PHASE_DETERMINE_FIRST_COLD_BLOCK, "Determine first cold block", false, -1, true)
 CompPhaseNameMacro(PHASE_RATIONALIZE, "Rationalize IR", false, -1, false)

src/coreclr/jit/ee_il_dll.hpp

@@ -44,6 +44,12 @@ void Compiler::eeGetFieldInfo(CORINFO_RESOLVED_TOKEN* pResolvedToken,
  info.compCompHnd->getFieldInfo(pResolvedToken, info.compMethodHnd, accessFlags, pResult);
 }
 
+FORCEINLINE
+uint32_t Compiler::eeGetThreadLocalFieldInfo(CORINFO_FIELD_HANDLE field)
+{
+ return info.compCompHnd->getThreadLocalFieldInfo(field);
+}
+
 /*****************************************************************************
  *
  * VOS info, method sigs, etc

src/coreclr/jit/emitarm64.cpp

@@ -13804,7 +13804,7 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR
 
  if (addr->isContained())
  {
- assert(addr->OperIs(GT_CLS_VAR_ADDR, GT_LCL_ADDR, GT_LEA));
+ assert(addr->OperIs(GT_CLS_VAR_ADDR, GT_LCL_ADDR, GT_LEA) || (addr->IsIconHandle(GTF_ICON_TLS_HDL)));
 
  int offset = 0;
  DWORD lsl = 0;
@@ -13927,6 +13927,11 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR
  emitIns_R_S(ins, attr, dataReg, lclNum, offset);
  }
  }
+ else if (addr->IsIconHandle(GTF_ICON_TLS_HDL))
+ {
+ // On Arm64, TEB is in r18, so load from the r18 as base.
+ emitIns_R_R_I(ins, attr, dataReg, REG_R18, addr->AsIntCon()->IconValue());
+ }
  else if (emitIns_valid_imm_for_ldst_offset(offset, emitTypeSize(indir->TypeGet())))
  {
  // Then load/store dataReg from/to [memBase + offset]

src/coreclr/jit/emitxarch.cpp

@@ -7260,6 +7260,10 @@ void emitter::emitIns_R_C(instruction ins, emitAttr attr, regNumber reg, CORINFO
  {
  sz += 1;
  }
+ else if (fldHnd == FLD_GLOBAL_GS)
+ {
+ sz += 2; // Needs SIB byte as well.
+ }
  }
 
  id->idCodeSize(sz);
@@ -7328,7 +7332,7 @@ void emitter::emitIns_C_R(instruction ins, emitAttr attr, CORINFO_FIELD_HANDLE f
  }
 
  // Special case: mov reg, fs:[ddd]
- if (fldHnd == FLD_GLOBAL_FS)
+ if ((fldHnd == FLD_GLOBAL_FS) || (fldHnd == FLD_GLOBAL_GS))
  {
  sz += 1;
  }
@@ -10001,6 +10005,12 @@ void emitter::emitDispClsVar(CORINFO_FIELD_HANDLE fldHnd, ssize_t offs, bool rel
  return;
  }
 
+ if (fldHnd == FLD_GLOBAL_GS)
+ {
+ printf("GS:[0x%04X]", (unsigned)offs);
+ return;
+ }
+
  if (fldHnd == FLD_GLOBAL_DS)
  {
  printf("[0x%04X]", (unsigned)offs);
@@ -13336,11 +13346,16 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc)
  fldh = id->idAddr()->iiaFieldHnd;
  offs = emitGetInsDsp(id);
 
- // Special case: mov reg, fs:[ddd]
  if (fldh == FLD_GLOBAL_FS)
  {
+ // Special case: mov reg, fs:[ddd]
  dst += emitOutputByte(dst, 0x64);
  }
+ else if (fldh == FLD_GLOBAL_GS)
+ {
+ // Special case: mov reg, gs:[ddd]
+ dst += emitOutputByte(dst, 0x65);
+ }
 
  // Compute VEX/EVEX prefix
  // Some of its callers already add EVEX/VEX prefix and then call this routine.
@@ -13550,6 +13565,11 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc)
  }
  }
 
+ if (fldh == FLD_GLOBAL_GS)
+ {
+ dst += emitOutputByte(dst, 0x25);
+ }
+
  // Do we have a constant or a static data member?
  doff = Compiler::eeGetJitDataOffs(fldh);
  if (doff >= 0)
@@ -13629,12 +13649,18 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc)
 #endif // TARGET_AMD64
 
 #ifdef TARGET_AMD64
- // All static field and data section constant accesses should be marked as relocatable
- noway_assert(id->idIsDspReloc());
- dst += emitOutputLong(dst, 0);
-#else // TARGET_X86
+ if (id->idIsDspReloc())
+ {
+ // All static field and data section constant accesses should be marked as relocatable
+ dst += emitOutputLong(dst, 0);
+ }
+ else
+ {
+ dst += emitOutputLong(dst, (ssize_t)target);
+ }
+#else
  dst += emitOutputLong(dst, (int)(ssize_t)target);
-#endif // TARGET_X86
+#endif // TARGET_AMD64
 
  if (id->idIsDspReloc())
  {
@@ -16447,8 +16473,16 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp)
  code = insEncodeReg3456(id, id->idReg1(), size, code);
  }
 
- regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8);
- dst = emitOutputCV(dst, id, code | regcode | 0x0500);
+ regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8);
+ CORINFO_FIELD_HANDLE fldh = id->idAddr()->iiaFieldHnd;
+ if (fldh == FLD_GLOBAL_GS)
+ {
+ dst = emitOutputCV(dst, id, code | regcode | 0x0400);
+ }
+ else
+ {
+ dst = emitOutputCV(dst, id, code | regcode | 0x0500);
+ }
  }
 
  sz = emitSizeOfInsDsc(id);