Original implementation of the dynamic dictionary expansion feature

Author: Fadi Hanna

docs/design/coreclr/botr/shared-generics.md

@@ -6992,7 +6992,7 @@ NativeImageDumper::DumpMethodTable( PTR_MethodTable mt, const char * name,
  //if there is a layout, use it to compute
  //the size, otherwise there is just the one
  //entry.
- DictionaryLayout::GetFirstDictionaryBucketSize(mt->GetNumGenericArgs(), layout),
+ DictionaryLayout::GetDictionarySizeFromLayout(mt->GetNumGenericArgs(), layout),
  METHODTABLES );
 
  DisplayStartArrayWithOffset( m_pEntries, NULL, Dictionary,
@@ -8018,7 +8018,7 @@ void NativeImageDumper::DumpMethodDesc( PTR_MethodDesc md, PTR_Module module )
  {
  PTR_DictionaryLayout layout(wrapped->IsSharedByGenericMethodInstantiations()
  ? dac_cast<TADDR>(wrapped->GetDictLayoutRaw()) : NULL );
- dictSize = DictionaryLayout::GetFirstDictionaryBucketSize(imd->GetNumGenericMethodArgs(),
+ dictSize = DictionaryLayout::GetDictionarySizeFromLayout(imd->GetNumGenericMethodArgs(), 
  layout);
  }
  }

src/coreclr/src/debug/daccess/nidump.cpp

@@ -2011,6 +2011,10 @@ void SystemDomain::LoadBaseSystemClasses()
  g_pDelegateClass = MscorlibBinder::GetClass(CLASS__DELEGATE);
  g_pMulticastDelegateClass = MscorlibBinder::GetClass(CLASS__MULTICAST_DELEGATE);
 
+#ifndef CROSSGEN_COMPILE
+ CrossLoaderAllocatorHashSetup::EnsureTypesLoaded();
+#endif
+
  // used by IsImplicitInterfaceOfSZArray
  MscorlibBinder::GetClass(CLASS__IENUMERABLEGENERIC);
  MscorlibBinder::GetClass(CLASS__ICOLLECTIONGENERIC);
@@ -2026,10 +2030,6 @@ void SystemDomain::LoadBaseSystemClasses()
  g_pUtf8StringClass = MscorlibBinder::GetClass(CLASS__UTF8_STRING);
 #endif // FEATURE_UTF8STRING
 
-#ifndef CROSSGEN_COMPILE
- CrossLoaderAllocatorHashSetup::EnsureTypesLoaded();
-#endif
-
 #ifndef CROSSGEN_COMPILE
  ECall::PopulateManagedStringConstructors();
 #endif // CROSSGEN_COMPILE

src/coreclr/src/vm/appdomain.cpp

@@ -556,6 +556,7 @@ void Module::Initialize(AllocMemTracker *pamTracker, LPCWSTR szName)
  m_FixupCrst.Init(CrstModuleFixup, (CrstFlags)(CRST_HOST_BREAKABLE|CRST_REENTRANCY));
  m_InstMethodHashTableCrst.Init(CrstInstMethodHashTable, CRST_REENTRANCY);
  m_ISymUnmanagedReaderCrst.Init(CrstISymUnmanagedReader, CRST_DEBUGGER_THREAD);
+ m_DictionaryCrst.Init(CrstDomainLocalBlock);
 
  if (!m_file->HasNativeImage())
  {
@@ -687,8 +688,12 @@ void Module::Initialize(AllocMemTracker *pamTracker, LPCWSTR szName)
  }
 #endif // defined (PROFILING_SUPPORTED) &&!defined(DACCESS_COMPILE) && !defined(CROSSGEN_COMPILE)
 
- LOG((LF_CLASSLOADER, LL_INFO10, "Loaded pModule: \"%ws\".\n", GetDebugName()));
+#ifndef CROSSGEN_COMPILE
+ m_dynamicSlotsHashForTypes.Init(GetLoaderAllocator());
+ m_dynamicSlotsHashForMethods.Init(GetLoaderAllocator());
+#endif
 
+ LOG((LF_CLASSLOADER, LL_INFO10, "Loaded pModule: \"%ws\".\n", GetDebugName()));
 }
 
 #endif // DACCESS_COMPILE
@@ -13212,6 +13217,326 @@ void ReflectionModule::ResumeMetadataCapture()
  CaptureModuleMetaDataToMemory();
 }
 
+TypeHandle* AllocateNewMethodDictionaryForExpansion(InstantiatedMethodDesc* pIMD, DWORD cbSize)
+{
+ TypeHandle* pInstOrPerInstInfo = (TypeHandle*)(void*)pIMD->GetLoaderAllocator()->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(cbSize + sizeof(void*)));
+ ZeroMemory(pInstOrPerInstInfo, cbSize + sizeof(void*));
+
+ // Slot[-1] points at previous dictionary to help with diagnostics when investigating crashes
+ *(byte**)pInstOrPerInstInfo = (byte*)pIMD->m_pPerInstInfo.GetValue() + 1;
+ pInstOrPerInstInfo++;
+
+ // Copy old dictionary entry contents
+ memcpy(pInstOrPerInstInfo, (const void*)pIMD->m_pPerInstInfo.GetValue(), pIMD->GetDictionarySlotsSize());
+
+ ULONG_PTR* pSizeSlot = ((ULONG_PTR*)pInstOrPerInstInfo) + pIMD->GetNumGenericMethodArgs();
+ *pSizeSlot = cbSize;
+
+ return pInstOrPerInstInfo;
+}
+
+TypeHandle* AllocateNewTypeDictionaryForExpansion(MethodTable* pMT, DWORD cbSize)
+{
+ TypeHandle* pInstOrPerInstInfo = (TypeHandle*)(void*)pMT->GetLoaderAllocator()->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(cbSize + sizeof(void*)));
+ ZeroMemory(pInstOrPerInstInfo, cbSize + sizeof(void*));
+
+ // Slot[-1] points at previous dictionary to help with diagnostics when investigating crashes
+ *(byte**)pInstOrPerInstInfo = (byte*)pMT->GetPerInstInfo()[pMT->GetNumDicts() - 1].GetValue() + 1;
+ pInstOrPerInstInfo++;
+
+ // Copy old dictionary entry contents
+ memcpy(pInstOrPerInstInfo, (const void*)pMT->GetPerInstInfo()[pMT->GetNumDicts() - 1].GetValue(), pMT->GetDictionarySlotsSize());
+
+ ULONG_PTR* pSizeSlot = ((ULONG_PTR*)pInstOrPerInstInfo) + pMT->GetNumGenericArgs();
+ *pSizeSlot = cbSize;
+
+ return pInstOrPerInstInfo;
+}
+
+#ifdef _DEBUG
+void Module::EnsureTypeRecorded(MethodTable* pMT)
+{
+ _ASSERTE(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
+
+ BOOL typeExistsInHashtable = FALSE;
+ auto lamda = [&typeExistsInHashtable, pMT](OBJECTREF obj, MethodTable* pMTKey, MethodTable* pMTValue)
+ {
+ typeExistsInHashtable = (pMT == pMTValue);
+ return pMT != pMTValue;
+ };
+
+ m_dynamicSlotsHashForTypes.VisitValuesOfKey(pMT->GetCanonicalMethodTable(), lamda);
+ _ASSERTE(typeExistsInHashtable);
+}
+
+void Module::EnsureMethodRecorded(MethodDesc* pMD)
+{
+ _ASSERTE(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
+
+ BOOL methodExistsInHashtable = FALSE;
+ auto lamda = [&methodExistsInHashtable, pMD](OBJECTREF obj, MethodDesc* pMDKey, MethodDesc* pMDValue)
+ {
+ methodExistsInHashtable = (pMD== pMDValue);
+ return pMD != pMDValue;
+ };
+
+ m_dynamicSlotsHashForMethods.VisitValuesOfKey(pMD->GetExistingWrappedMethodDesc(), lamda);
+ _ASSERTE(methodExistsInHashtable);
+}
+#endif
+
+void Module::RecordTypeForDictionaryExpansion_Locked(MethodTable* pGenericParentMT, MethodTable* pDependencyMT)
+{
+ CONTRACTL
+ {
+ GC_TRIGGERS;
+ PRECONDITION(CheckPointer(pGenericParentMT) && CheckPointer(pDependencyMT));
+ PRECONDITION(pGenericParentMT->HasInstantiation() && pGenericParentMT != pGenericParentMT->GetCanonicalMethodTable());
+ PRECONDITION(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
+ }
+ CONTRACTL_END
+
+ DictionaryLayout* pDictLayout = pDependencyMT->GetClass()->GetDictionaryLayout();
+ if (pDictLayout != NULL && pDictLayout->GetMaxSlots() > 0)
+ {
+ DWORD sizeFromDictLayout = DictionaryLayout::GetDictionarySizeFromLayout(pDependencyMT->GetNumGenericArgs(), pDictLayout);
+ if (pDependencyMT->GetDictionarySlotsSize() != sizeFromDictLayout)
+ {
+ _ASSERT(pDependencyMT->GetDictionarySlotsSize() < sizeFromDictLayout);
+
+ //
+ // Another thread got a chance to expand the dictionary layout and expand the dictionary slots of
+ // other types, but not for this one yet because we're still in the process of recording it for 
+ // expansions.
+ // Expand the dictionary slots here before finally adding the type to the hashtable.
+ //
+
+ TypeHandle* pInstOrPerInstInfo = AllocateNewTypeDictionaryForExpansion(pDependencyMT, sizeFromDictLayout);
+
+ // Publish the new dictionary slots to the type.
+ TypeHandle** pPerInstInfo = (TypeHandle**)pDependencyMT->GetPerInstInfo()->GetValuePtr();
+ FastInterlockExchangePointer(pPerInstInfo + (pDependencyMT->GetNumDicts() - 1), pInstOrPerInstInfo);
+
+ FlushProcessWriteBuffers();
+ }
+ }
+
+ GCX_COOP();
+ m_dynamicSlotsHashForTypes.Add(pGenericParentMT->GetCanonicalMethodTable(), pDependencyMT, GetLoaderAllocator());
+}
+
+void Module::RecordMethodForDictionaryExpansion_Locked(MethodDesc* pDependencyMD)
+{
+ CONTRACTL
+ {
+ GC_TRIGGERS;
+ PRECONDITION(CheckPointer(pDependencyMD) && pDependencyMD->HasMethodInstantiation() && pDependencyMD->IsInstantiatingStub());
+ PRECONDITION(pDependencyMD->GetDictionaryLayout() != NULL && pDependencyMD->GetDictionaryLayout()->GetMaxSlots() > 0);
+ PRECONDITION(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
+ }
+ CONTRACTL_END
+
+ DictionaryLayout* pDictLayout = pDependencyMD->GetDictionaryLayout();
+ InstantiatedMethodDesc* pIMDDependency = pDependencyMD->AsInstantiatedMethodDesc();
+
+ DWORD sizeFromDictLayout = DictionaryLayout::GetDictionarySizeFromLayout(pDependencyMD->GetNumGenericMethodArgs(), pDictLayout);
+ if (pIMDDependency->GetDictionarySlotsSize() != sizeFromDictLayout)
+ {
+ _ASSERT(pIMDDependency->GetDictionarySlotsSize() < sizeFromDictLayout);
+
+ //
+ // Another thread got a chance to expand the dictionary layout and expand the dictionary slots of
+ // other methods, but not for this one yet because we're still in the process of recording it for 
+ // expansions.
+ // Expand the dictionary slots here before finally adding the method to the hashtable.
+ //
+
+ TypeHandle* pInstOrPerInstInfo = AllocateNewMethodDictionaryForExpansion(pIMDDependency, sizeFromDictLayout);
+
+ FastInterlockExchangePointer((TypeHandle**)pIMDDependency->m_pPerInstInfo.GetValuePtr(), pInstOrPerInstInfo);
+
+ FlushProcessWriteBuffers();
+ }
+
+ GCX_COOP();
+
+ _ASSERTE(pDependencyMD->GetExistingWrappedMethodDesc() != NULL);
+ m_dynamicSlotsHashForMethods.Add(pDependencyMD->GetExistingWrappedMethodDesc(), pDependencyMD, GetLoaderAllocator());
+}
+
+void Module::ExpandTypeDictionaries_Locked(MethodTable* pMT, DictionaryLayout* pOldLayout, DictionaryLayout* pNewLayout)
+{
+ CONTRACTL
+ {
+ STANDARD_VM_CHECK;
+ INJECT_FAULT(ThrowOutOfMemory(););
+ PRECONDITION(CheckPointer(pOldLayout) && CheckPointer(pNewLayout));
+ PRECONDITION(CheckPointer(pMT) && pMT->HasInstantiation() && pMT->GetClass()->GetDictionaryLayout() == pOldLayout);
+ PRECONDITION(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
+ }
+ CONTRACTL_END
+
+ GCX_COOP();
+
+ MethodTable* pCanonMT = pMT->GetCanonicalMethodTable();
+ DWORD oldInfoSize = DictionaryLayout::GetDictionarySizeFromLayout(pMT->GetNumGenericArgs(), pOldLayout);
+ DWORD newInfoSize = DictionaryLayout::GetDictionarySizeFromLayout(pMT->GetNumGenericArgs(), pNewLayout);
+
+ //
+ // Dictionary expansion for types needs to be done in multiple steps, given how derived types do not directly embed dictionaries
+ // from parent types, but instead reference them from directly from the parent types. Also, this is necessary to ensure correctness
+ // for lock-free read operations for dictionary slots:
+ // 1) Allocate new dictionaries for all instantiated types of the same typedef as the one being expanded.
+ // 2) After all allocations and initializations are completed, publish the dictionaries to the types in #1 after 
+ // flushing write buffers
+ // 3) For all types that derive from #1, update the embedded dictinoary pointer to the newly allocated one.
+ //
+
+ struct NewDictionary
+ {
+ MethodTable* pMT;
+ TypeHandle* pDictSlots;
+ };
+ StackSArray<NewDictionary> dictionaryUpdates;
+
+#ifdef _DEBUG
+ auto expandPerInstInfos = [oldInfoSize, newInfoSize, &dictionaryUpdates](OBJECTREF obj, MethodTable* pMTKey, MethodTable* pMTToUpdate)
+#else
+ auto expandPerInstInfos = [newInfoSize, &dictionaryUpdates](OBJECTREF obj, MethodTable* pMTKey, MethodTable* pMTToUpdate)
+#endif
+ {
+ if (!pMTToUpdate->HasSameTypeDefAs(pMTKey))
+ return true;
+
+ _ASSERTE(pMTToUpdate != pMTToUpdate->GetCanonicalMethodTable() && pMTToUpdate->GetCanonicalMethodTable() == pMTKey);
+ _ASSERTE(pMTToUpdate->GetDictionarySlotsSize() == oldInfoSize);
+
+ TypeHandle* pInstOrPerInstInfo = AllocateNewTypeDictionaryForExpansion(pMTToUpdate, newInfoSize);
+
+ NewDictionary entry;
+ entry.pMT = pMTToUpdate;
+ entry.pDictSlots = pInstOrPerInstInfo;
+ dictionaryUpdates.Append(entry);
+
+ return true; // Keep walking
+ };
+
+ m_dynamicSlotsHashForTypes.VisitValuesOfKey(pCanonMT, expandPerInstInfos);
+
+ // Flush write buffers to ensure new dictionaries are fully writted and initalized before publishing them
+ FlushProcessWriteBuffers();
+
+ for (SArray<NewDictionary>::Iterator i = dictionaryUpdates.Begin(); i != dictionaryUpdates.End(); i++)
+ {
+ MethodTable* pMT = i->pMT;
+ TypeHandle** pPerInstInfo = (TypeHandle**)pMT->GetPerInstInfo()->GetValuePtr();
+ FastInterlockExchangePointer(pPerInstInfo + (pMT->GetNumDicts() - 1), i->pDictSlots);
+ _ASSERTE(pMT->GetDictionarySlotsSize() == newInfoSize);
+ _ASSERTE((TypeHandle*)pMT->GetPerInstInfo()[pMT->GetNumDicts() - 1].GetValue() == i->pDictSlots);
+ }
+
+ auto updateDependentDicts = [](OBJECTREF obj, MethodTable* pMTKey, MethodTable* pMTToUpdate)
+ {
+ if (pMTToUpdate->HasSameTypeDefAs(pMTKey))
+ return true;
+
+ MethodTable* pCurrentMT = pMTToUpdate->GetParentMethodTable();
+ while (pCurrentMT)
+ {
+ if (pCurrentMT->HasSameTypeDefAs(pMTKey))
+ {
+ DWORD dictToUpdate = pCurrentMT->GetNumDicts() - 1;
+ Dictionary* pUpdatedParentDict = pCurrentMT->GetPerInstInfo()[dictToUpdate].GetValue();
+ TypeHandle** pPerInstInfo = (TypeHandle**)pMTToUpdate->GetPerInstInfo()->GetValuePtr();
+ FastInterlockExchangePointer(pPerInstInfo + dictToUpdate, (TypeHandle*)pUpdatedParentDict);
+ _ASSERTE(pMTToUpdate->GetPerInstInfo()[dictToUpdate].GetValue() == pUpdatedParentDict);
+
+ return true; // Keep walking
+ }
+ pCurrentMT = pCurrentMT->GetParentMethodTable();
+ }
+
+ UNREACHABLE();
+ };
+
+ m_dynamicSlotsHashForTypes.VisitValuesOfKey(pCanonMT, updateDependentDicts);
+
+ // Ensure no other thread uses old dictionary pointers
+ FlushProcessWriteBuffers();
+}
+
+void Module::ExpandMethodDictionaries_Locked(MethodDesc* pMD, DictionaryLayout* pOldLayout, DictionaryLayout* pNewLayout)
+{
+ CONTRACTL
+ {
+ STANDARD_VM_CHECK;
+ INJECT_FAULT(ThrowOutOfMemory(););
+ PRECONDITION(CheckPointer(pOldLayout) && CheckPointer(pNewLayout));
+ PRECONDITION(CheckPointer(pMD));
+ PRECONDITION(pMD->HasMethodInstantiation() && pMD->GetDictionaryLayout() == pOldLayout);
+ PRECONDITION(SystemDomain::SystemModule()->m_DictionaryCrst.OwnedByCurrentThread());
+ }
+ CONTRACTL_END
+
+ GCX_COOP();
+
+ //
+ // Dictionary expansion for methods needs to be done in two steps to ensure correctness for lock-free read operations 
+ // for dictionary slots:
+ // 1) Allocate new dictionaries for all instantiated methods sharing the same canonical form as the input method
+ // 2) After all allocations and initializations are completed, publish the dictionaries to the methods after 
+ // flushing write buffers
+ //
+
+ MethodDesc* pCanonMD = pMD->IsInstantiatingStub() ? pMD->GetExistingWrappedMethodDesc() : pMD;
+ _ASSERTE(pCanonMD != NULL);
+ DWORD oldInfoSize = DictionaryLayout::GetDictionarySizeFromLayout(pMD->GetNumGenericMethodArgs(), pOldLayout);
+ DWORD newInfoSize = DictionaryLayout::GetDictionarySizeFromLayout(pMD->GetNumGenericMethodArgs(), pNewLayout);
+
+ struct NewDictionary
+ {
+ InstantiatedMethodDesc* pIMD;
+ TypeHandle* pDictSlots;
+ };
+ StackSArray<NewDictionary> dictionaryUpdates;
+
+#ifdef _DEBUG
+ auto lambda = [oldInfoSize, newInfoSize, &dictionaryUpdates](OBJECTREF obj, MethodDesc* pMDKey, MethodDesc* pMDToUpdate)
+#else
+ auto lambda = [newInfoSize, &dictionaryUpdates](OBJECTREF obj, MethodDesc* pMDKey, MethodDesc* pMDToUpdate)
+#endif
+ {
+ // Update m_pPerInstInfo for the pMethodDesc being visited here
+ _ASSERTE(pMDToUpdate->IsInstantiatingStub() && pMDToUpdate->GetExistingWrappedMethodDesc() == pMDKey);
+
+ InstantiatedMethodDesc* pInstantiatedMD = pMDToUpdate->AsInstantiatedMethodDesc();
+ _ASSERTE(pInstantiatedMD->GetDictionarySlotsSize() == oldInfoSize);
+
+ TypeHandle* pInstOrPerInstInfo = AllocateNewMethodDictionaryForExpansion(pInstantiatedMD, newInfoSize);
+
+ NewDictionary entry;
+ entry.pIMD = pInstantiatedMD;
+ entry.pDictSlots = pInstOrPerInstInfo;
+ dictionaryUpdates.Append(entry);
+
+ return true; // Keep walking
+ };
+
+ m_dynamicSlotsHashForMethods.VisitValuesOfKey(pCanonMD, lambda);
+
+ // Flush write buffers to ensure new dictionaries are fully writted and initalized before publishing them
+ FlushProcessWriteBuffers();
+
+ for (SArray<NewDictionary>::Iterator i = dictionaryUpdates.Begin(); i != dictionaryUpdates.End(); i++)
+ {
+ FastInterlockExchangePointer((TypeHandle**)i->pIMD->m_pPerInstInfo.GetValuePtr(), i->pDictSlots);
+ _ASSERTE((TypeHandle*)i->pIMD->m_pPerInstInfo.GetValue() == i->pDictSlots);
+ _ASSERTE(i->pIMD->GetDictionarySlotsSize() == newInfoSize);
+ }
+
+ // Ensure no other thread uses old dictionary pointers
+ FlushProcessWriteBuffers();
+}
 #endif // !CROSSGEN_COMPILE
 
 #endif // !DACCESS_COMPILE