Discussion/archive: use custom dedup table

core/src/main/scala/flatgraph/misc/DedupTable.scala

@@ -0,0 +1,90 @@
+package flatgraph.misc
+
+import java.util
+
+/** A basic LinkedHashSet-like structure, based on an open hashmap. Somewhat surprisingly, it actually pays to store part of the hash in the
+ * upper bits of the index into tha values array
+ *
+ * This is a fastpath to avoid some unneeded expensive isequal-checks. (expensive because they imply following an object reference)
+ *
+ * The thing is arranged such that a zero value in pos signifies an empty slot
+ */
+private[flatgraph] class DedupTable {
+ var capacity = 1024
+ var size = 0
+ var strs = new Array[String](capacity)
+ var pos = new Array[Int](capacity)
+
+ def insert(str: String): Int = {
+ val cap = capacity
+ val mask = cap - 1
+ if (str == null) return -1
+ val h = strengthenHash(str.hashCode)
+ var p = h & mask
+ val needle = h & ~mask
+ while (true) {
+ val idx0 = pos(p)
+ val idx = (idx0 & mask) - 1
+ if (idx == -1) {
+ val dst = size
+ size = size + 1
+ strs(dst) = str
+ pos(p) = (dst + 1) | needle
+ if (size + (size >> 1) > capacity) grow()
+ return dst
+ } else if ((idx0 & ~mask) == needle && str.equals(strs(idx))) return idx
+ p = p + 1
+ if (p == cap) {
+ // overflow -- rare, so better branch than branchless
+ p = 0
+ }
+ }
+ ???
+ }
+
+ def insertRehash(str: String): Unit = {
+ val cap = capacity
+ val mask = cap - 1
+ val h = strengthenHash(str.hashCode)
+ var p = h & mask
+ val needle = h & ~mask
+ while (true) {
+ val idx0 = pos(p)
+ val idx = (idx0 & mask) - 1
+ if (idx == -1) {
+ val dst = size
+ size = size + 1
+ pos(p) = (dst + 1) | needle
+ return
+ }
+ p = p + 1
+ if (p == cap) {
+ // overflow -- rare, so better branch than branchless
+ p = 0
+ }
+ }
+ ???
+ }
+
+ def grow(): Unit = {
+ val oldsize = size
+ size = 0
+ capacity = capacity * 2
+ pos = new Array[Int](capacity)
+ strs = util.Arrays.copyOf(strs, capacity)
+ for (idx <- Range(0, oldsize)) insertRehash(strs(idx))
+ }
+
+ def strengthenHash(hash0: Int): Int = {
+ // using the simple murmur 32 bit mixing to strengthen the hash
+ var hash = hash0 ^ (hash0 >>> 16)
+ hash *= 0x85ebca6b
+ hash ^= hash >>> 13
+ hash
+ // murmur does a bit more, but we don't need that.
+ // hash *= 0xc2b2ae35
+ // hash ^= hash >>> 16
+ // hash & (capacity - 1)
+ }
+
+}

core/src/main/scala/flatgraph/storage/Serialization.scala

@@ -24,7 +24,7 @@ class WriterContext(val fileChannel: FileChannel, val executor: concurrent.Execu
  val writeQueue = mutable.ArrayDeque[concurrent.Future[Any]]()
  val jobQueue = mutable.ArrayBuffer[() => (OutlineStorage, Array[Byte])]()
  val stringQueue = mutable.ArrayDeque[(OutlineStorage, Array[String])]()
- val stringpool = mutable.LinkedHashMap[String, Int]()
+ val stringpool = new flatgraph.misc.DedupTable
 
  def submitCompress(block: => (OutlineStorage, ByteBuffer)): Unit = {
  compressQueue.addOne(executor.submit((() => block)))
@@ -46,12 +46,6 @@ class WriterContext(val fileChannel: FileChannel, val executor: concurrent.Execu
  }
  }
 
- // NOT threadsafe!
- private def insertString(stringPool: mutable.LinkedHashMap[String, Int])(s: String): Int = {
- if (s == null) -1
- else stringPool.getOrElseUpdate(s, stringPool.size)
- }
-
  private[flatgraph] def encodeAny(item: Any, outlineStorage: OutlineStorage = new OutlineStorage, delta: Int = -1): OutlineStorage = {
  item match {
  case _: DefaultValue => null
@@ -161,7 +155,7 @@ class WriterContext(val fileChannel: FileChannel, val executor: concurrent.Execu
  writeItem(item, bytes)
  }
  val (item, strings) = stringQueue.removeHead()
- val indices = strings.map(insertString(stringpool))
+ val indices = strings.map(stringpool.insert)
  submitCompress {
  val bytes = new Array[Byte](4 * strings.length)
  ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).asIntBuffer().put(indices)
@@ -172,7 +166,7 @@ class WriterContext(val fileChannel: FileChannel, val executor: concurrent.Execu
  val poolLenBytes = new Array[Byte](4 * stringpool.size)
  val poolLenBuffer = ByteBuffer.wrap(poolLenBytes).order(ByteOrder.LITTLE_ENDIAN).asIntBuffer()
  val poolBytes = new ByteArrayOutputStream()
- for (s <- stringpool.keysIterator) {
+ for (s <- stringpool.strs.iterator.take(stringpool.size)) {
  val bytes = s.getBytes(StandardCharsets.UTF_8)
  poolBytes.write(bytes)
  poolLenBuffer.put(bytes.length)