support ic

Author: timotheecour

compiler/ic/ic.nim

@@ -36,6 +36,7 @@ type
  bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position.
  #producedGenerics*: Table[GenericKey, SymId]
  exports*: seq[(LitId, int32)]
+ hidden*: seq[(LitId, int32)]
  reexports*: seq[(LitId, PackedItemId)]
  compilerProcs*: seq[(LitId, int32)]
  converters*, methods*, trmacros*, pureEnums*: seq[int32]
@@ -177,6 +178,10 @@ proc addIncludeFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex)
 proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) =
  m.imports.add toLitId(f, c, m)
 
+proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+ let nameId = getOrIncl(m.sh.strings, s.name.s)
+ m.hidden.add((nameId, s.itemId.item))
+
 proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
  let nameId = getOrIncl(m.sh.strings, s.name.s)
  m.exports.add((nameId, s.itemId.item))
@@ -524,7 +529,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
  loadTabSection numbersSection, m.sh.numbers
 
  loadSeqSection exportsSection, m.exports
-
+ loadSeqSection hiddenSection, m.hidden
  loadSeqSection reexportsSection, m.reexports
 
  loadSeqSection compilerProcsSection, m.compilerProcs
@@ -589,7 +594,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
  storeTabSection numbersSection, m.sh.numbers
 
  storeSeqSection exportsSection, m.exports
-
+ storeSeqSection hiddenSection, m.hidden
  storeSeqSection reexportsSection, m.reexports
 
  storeSeqSection compilerProcsSection, m.compilerProcs
@@ -655,7 +660,9 @@ type
  syms: seq[PSym] # indexed by itemId
  types: seq[PType]
  module*: PSym # the one true module symbol.
- iface: Table[PIdent, seq[PackedItemId]] # PackedItemId so that it works with reexported symbols too
+ iface, ifaceHidden: Table[PIdent, seq[PackedItemId]]
+ # PackedItemId so that it works with reexported symbols too
+ # ifaceHidden includes private symbols
 
  PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex
 
@@ -882,12 +889,18 @@ proc newPackage(config: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym
 proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
  fileIdx: FileIndex; m: var LoadedModule) =
  m.iface = initTable[PIdent, seq[PackedItemId]]()
+ m.ifaceHidden = initTable[PIdent, seq[PackedItemId]]()
  for e in m.fromDisk.exports:
  let nameLit = e[0]
  m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(PackedItemId(module: LitId(0), item: e[1]))
  for re in m.fromDisk.reexports:
  let nameLit = re[0]
  m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(re[1])
+ m.ifaceHidden.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(re[1])
+
+ for e in m.fromDisk.hidden:
+ let nameLit = e[0]
+ m.ifaceHidden.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(PackedItemId(module: LitId(0), item: e[1]))
 
  let filename = AbsoluteFile toFullPath(conf, fileIdx)
  # We cannot call ``newSym`` here, because we have to circumvent the ID
@@ -1053,24 +1066,29 @@ type
  values: seq[PackedItemId]
  i, module: int
 
+template interfSelect(a: LoadedModule, importHidden: bool): auto =
+ var ret = a.iface.addr
+ if importHidden: ret = a.ifaceHidden.addr
+ ret[]
+
 proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache;
  g: var PackedModuleGraph; module: FileIndex;
- name: PIdent): PSym =
+ name: PIdent, importHidden: bool): PSym =
  it.decoder = PackedDecoder(
  lastModule: int32(-1),
  lastLit: LitId(0),
  lastFile: FileIndex(-1),
  config: config,
  cache: cache)
- it.values = g[int module].iface.getOrDefault(name)
+ it.values = g[int module].interfSelect(importHidden).getOrDefault(name)
  it.i = 0
  it.module = int(module)
  if it.i < it.values.len:
  result = loadSym(it.decoder, g, int(module), it.values[it.i])
  inc it.i
 
 proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache;
- g: var PackedModuleGraph; module: FileIndex): PSym =
+ g: var PackedModuleGraph; module: FileIndex, importHidden: bool): PSym =
  it.decoder = PackedDecoder(
  lastModule: int32(-1),
  lastLit: LitId(0),
@@ -1079,7 +1097,7 @@ proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache;
  cache: cache)
  it.values = @[]
  it.module = int(module)
- for v in g[int module].iface.values:
+ for v in g[int module].interfSelect(importHidden).values:
  it.values.add v
  it.i = 0
  if it.i < it.values.len:
@@ -1093,19 +1111,19 @@ proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym =
 
 iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache;
  g: var PackedModuleGraph; module: FileIndex;
- name: PIdent): PSym =
+ name: PIdent, importHidden: bool): PSym =
  setupDecoder()
- let values = g[int module].iface.getOrDefault(name)
+ let values = g[int module].interfSelect(importHidden).getOrDefault(name)
  for pid in values:
  let s = loadSym(decoder, g, int(module), pid)
  assert s != nil
  yield s
 
 proc interfaceSymbol*(config: ConfigRef, cache: IdentCache;
  g: var PackedModuleGraph; module: FileIndex;
- name: PIdent): PSym =
+ name: PIdent, importHidden: bool): PSym =
  setupDecoder()
- let values = g[int module].iface.getOrDefault(name)
+ let values = g[int module].interfSelect(importHidden).getOrDefault(name)
  result = loadSym(decoder, g, int(module), values[0])
 
 proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator =
@@ -1126,7 +1144,7 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
  let err = loadRodFile(rodfile, m, config, ignoreConfig=true)
  if err != ok:
  echo "Error: could not load: ", rodfile.string, " reason: ", err
- quit 1
+ doAssert false # quit would prevent getting a stacktrace
 
  when true:
  echo "exports:"
@@ -1140,6 +1158,10 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
  echo " ", m.sh.strings[ex[0]]
  # reexports*: seq[(LitId, PackedItemId)]
 
+ echo "hidden: " & $m.hidden.len
+ for ex in m.hidden:
+ echo " ", m.sh.strings[ex[0]], " local ID: ", ex[1]
+
  echo "all symbols"
  for i in 0..high(m.sh.syms):
  if m.sh.syms[i].name != LitId(0):

compiler/ic/rodfiles.nim

@@ -18,6 +18,7 @@ type
  depsSection
  numbersSection
  exportsSection
+ hiddenSection
  reexportsSection
  compilerProcsSection
  trmacrosSection

compiler/modulegraphs.nim

@@ -29,7 +29,7 @@ type
  patterns*: seq[LazySym]
  pureEnums*: seq[LazySym]
  interf: TStrTable
- interfAll: TStrTable
+ interfHidden: TStrTable
  uniqueName*: Rope
 
  Operators* = object
@@ -162,16 +162,15 @@ proc toBase64a(s: cstring, len: int): string =
  result.add cb64[(a and 3) shl 4]
 
 template interfSelect(iface: Iface, importHidden: bool): TStrTable =
- var ret: ptr TStrTable # without intermediate ptr, it creates a copy and compiler becomes 15x slower!
- if importHidden: ret = iface.interfAll.addr
- else: ret = iface.interf.addr
+ var ret = iface.interf.addr # without intermediate ptr, it creates a copy and compiler becomes 15x slower!
+ if importHidden: ret = iface.interfHidden.addr
  ret[]
 
 template semtab(g: ModuleGraph, m: PSym): TStrTable =
  g.ifaces[m.position].interf
 
 template semtabAll*(g: ModuleGraph, m: PSym): TStrTable =
- g.ifaces[m.position].interfAll
+ g.ifaces[m.position].interfHidden
 
 proc initStrTables*(g: ModuleGraph, m: PSym) =
  initStrTable(semtab(g, m))
@@ -213,7 +212,7 @@ proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent):
  mi.fromRod = isCachedModule(g, mi.modIndex)
  mi.importHidden = optImportHidden in m.options
  if mi.fromRod:
- result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name)
+ result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name, mi.importHidden)
  else:
  result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden), name)
 
@@ -224,22 +223,24 @@ proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym =
  result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden))
 
 iterator allSyms*(g: ModuleGraph; m: PSym): PSym =
+ let importHidden = optImportHidden in m.options
  if isCachedModule(g, m):
  var rodIt: RodIter
- var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position)
+ var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position, importHidden)
  while r != nil:
  yield r
  r = nextRodIter(rodIt, g.packed)
  else:
- for s in g.ifaces[m.position].interfSelect(optImportHidden in m.options).data:
+ for s in g.ifaces[m.position].interfSelect(importHidden).data:
  if s != nil:
  yield s
 
 proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym =
+ let importHidden = optImportHidden in m.options
  if isCachedModule(g, m):
- result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name)
+ result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name, importHidden)
  else:
- result = strTableGet(g.ifaces[m.position].interfSelect(optImportHidden in m.options), name)
+ result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name)
 
 proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym =
  result = someSym(g, g.systemModule, name)
@@ -376,7 +377,10 @@ template onDefAux(info: TLineInfo; s0: PSym, c0: untyped, isFwd: bool) =
  # unfortunately, can't use `c.isTopLevel` because the scope isn't closed yet
  top = c.currentScope.depthLevel <= 3
  else: top = c.currentScope.depthLevel <= 2
- if top and c.module != nil: strTableAdd(semtabAll(c.graph, c.module), s0)
+ if top and c.module != nil:
+ strTableAdd(semtabAll(c.graph, c.module), s0)
+ if c.config.symbolFiles != disabledSf:
+ addHidden(c.encoder, c.packedRepr, s0)
 
 when defined(nimfind):
  template onUse*(info: TLineInfo; s: PSym) =

testament/categories.nim

@@ -500,6 +500,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string)
 
  const tempExt = "_temp.nim"
  for it in walkDirRec(testsDir / "ic"):
+ # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test
  if isTestFile(it) and not it.endsWith(tempExt):
  let nimcache = nimcacheDir(it, options, getTestSpecTarget())
  removeDir(nimcache)

tests/ic/mimports.nim

@@ -0,0 +1,9 @@
+from mimportsb {.all.} import fnb1, hfnb3
+
+proc fn1*(): int = 1
+proc fn2*(): int = 2
+proc hfn3(): int = 3
+proc hfn4(): int = 4
+proc hfn5(): int = 5
+
+export mimportsb.fnb2, hfnb3

tests/ic/mimportsb.nim

@@ -0,0 +1,4 @@
+proc fnb1*(): int = 1
+proc fnb2*(): int = 2
+proc hfnb3(): int = 3
+proc hfnb4(): int = 4

tests/ic/timports.nim

@@ -0,0 +1,29 @@
+import mimports
+doAssert fn1() == 1
+doAssert not declared(hfn3)
+
+#!EDIT!#
+
+import mimports {.all.}
+doAssert fn1() == 1
+doAssert declared(hfn3)
+doAssert hfn3() == 3
+doAssert mimports.hfn4() == 4
+
+# reexports
+doAssert not declared(fnb1)
+doAssert not declared(hfnb4)
+doAssert fnb2() == 2
+doAssert hfnb3() == 3
+
+#!EDIT!#
+
+from mimports {.all.} import hfn3
+doAssert not declared(fn1)
+from mimports {.all.} as bar import fn1
+doAssert fn1() == 1
+doAssert hfn3() == 3
+doAssert not declared(hfn4)
+doAssert declared(mimports.hfn4)
+doAssert mimports.hfn4() == 4
+doAssert bar.hfn4() == 4