WIP: Refactor bitfield to replicator & reduce bitfield memory by relying on storage

index.js

@@ -554,6 +554,7 @@ class Hypercore extends EventEmitter {
  this.core.replicator.findingPeers -= this._findingPeers
  this.core.replicator.clearRequests(this.activeRequests, error)
  this.core.replicator.updateActivity(this._active ? -1 : 0)
+ await this.core.replicator.localBitfield.flush()
 
  this._findingPeers = 0
 
@@ -818,9 +819,9 @@ class Hypercore extends EventEmitter {
  throw ASSERTION('has range is invalid', this.discoveryKey)
 
  if (this.state.isDefault()) {
- if (end === start + 1) return this.core.bitfield.get(start)
+ if (end === start + 1) return this.core.replicator.localBitfield.get(start)
 
- const i = this.core.bitfield.firstUnset(start)
+ const i = await this.core.replicator.localBitfield.firstUnset(start)
  return i === -1 || i >= end
  }
 
@@ -919,7 +920,7 @@ class Hypercore extends EventEmitter {
 
  // lets check the bitfield to see if we got it during the above async calls
  // this is the last resort before replication, so always safe.
- if (this.core.bitfield.get(index)) {
+ if (await this.core.replicator.localBitfield.get(index)) {
  const coreBlock = await readBlock(this.state.storage.read(), index)
  // TODO: this should not be needed, only needed atm in case we are doing a moveTo during this (we should fix)
  if (coreBlock !== null) return coreBlock

lib/audit.js

@@ -78,7 +78,7 @@ module.exports = async function auditCore(
  let tx = null
 
  for await (const block of core.state.storage.createBlockStream()) {
- if (!core.bitfield.get(block.index)) {
+ if (!(await core.replicator.localBitfield.get(block.index))) {
  if (!tx && !stats.corrupt) tx = core.state.storage.write()
  tx.deleteBlock(block.index)
  stats.droppedBlocks++
@@ -107,7 +107,7 @@ module.exports = async function auditCore(
  if (bitfield) {
  let tx = null
 
- for (const index of allBits(core.bitfield)) {
+ for await (const index of allBits(core.replicator.localBitfield)) {
  const rx = core.state.storage.read()
  const blockPromise = rx.getBlock(index)
 
@@ -120,10 +120,11 @@ module.exports = async function auditCore(
 
  if (!tx && !stats.corrupt) tx = core.state.storage.write()
 
- core.bitfield.set(index, false)
+ await core.replicator.localBitfield.set(index, false)
 
- const page = core.bitfield.getBitfield(index)
- if (page.bitfield) tx.putBitfieldPage(page.index, page.bitfield)
+ const page = core.replicator.localBitfield.getBitfield(index)
+ const bitfield = await page.getBuffer()
+ if (bitfield) tx.putBitfieldPage(page.index, bitfield)
  else tx.deleteBitfieldPage(page.index)
  continue
  }
@@ -150,11 +151,11 @@ function isBadTree(parent, left, right) {
  return !b4a.equals(hash, parent.hash) || parent.size !== left.size + right.size
 }
 
-function* allBits(bitfield) {
+async function* allBits(bitfield) {
  let i = 0
- if (bitfield.get(0)) yield 0
+ if (await bitfield.get(0)) yield 0
  while (true) {
- i = bitfield.findFirst(true, i + 1)
+ i = await bitfield.findFirst(true, i + 1)
  if (i === -1) break
  yield i
  }

lib/bit-interlude.js

@@ -119,7 +119,7 @@ module.exports = class BitInterlude {
  this.ranges.push({ start, end, value })
  }
 
- flush(tx, bitfield) {
+ async flush(tx, bitfield) {
  if (!this.ranges.length) return []
 
  let index = this.ranges[0].start
@@ -134,7 +134,7 @@ module.exports = class BitInterlude {
  const buf = b4a.allocUnsafe(bitfield.getPageByteLength())
 
  if (page) {
- const src = page.bitfield // Uint32Array
+ const src = await page.getBuffer() // Uint32Array
  buf.set(b4a.from(src.buffer, src.byteOffset, src.byteLength), 0)
  } else {
  b4a.fill(buf, 0)