feat: importDatabaseToOpfs, importDatabaseToIdb

Author: subframe7536

playground/index.html

@@ -10,7 +10,9 @@
  <button class="main">main</button>
  <button class="worker">worker</button>
  <button class="import">import</button>
+ <button class="importW">import worker</button>
  <button class="download">download</button>
+ <button class="downloadW">download worker</button>
  <button class="clear">clear</button>
 </body>
 <script type="module" src="/src/index.ts"></script>

playground/src/index.ts

@@ -11,12 +11,12 @@ import {
  useMemoryStorage,
  withExistDB,
 } from '../../src/index'
-// import { useIdbMemoryStorage } from '../../src/vfs/idb-memory'
-import { useIdbStorage } from '../../src/vfs/idb'
+import { useIdbMemoryStorage } from '../../src/vfs/idb-memory'
+// import { useIdbStorage } from '../../src/vfs/idb'
 import { runSQL } from './runSQL'
 import OpfsWorker from './worker?worker'
 
-let db: SQLiteDB
+let db: SQLiteDB | undefined
 
 const supportModuleWorker = isModuleWorkerSupport()
 const supportIDB = isIdbSupported()
@@ -26,8 +26,8 @@ console.log('support IDBBatchAtomicVFS:', supportIDB)
 console.log('support OPFSCoopSyncVFS:', supportOPFS)
 document.querySelector('.main')?.addEventListener('click', async () => {
  if (!db) {
- // db = await initSQLite(useIdbMemoryStorage('test.db', { url }))
- db = await initSQLite(useIdbStorage('test.db', { url }))
+ db = await initSQLite(useIdbMemoryStorage('test.db', { url }))
+ // db = await initSQLite(useIdbStorage('test.db', { url }))
  }
  await runSQL(db.run)
  await runSQL((await initSQLite(useMemoryStorage({ url: syncUrl }))).run)
@@ -43,80 +43,105 @@ document.querySelector('.import')?.addEventListener('click', async () => {
  return
  }
  db = await initSQLite(
- // useIdbMemoryStorage('test.db', withExistDB(file, { url })),
- useIdbStorage('test.db', { url }),
+ useIdbMemoryStorage('test.db', withExistDB(file, { url })),
+ // useIdbMemoryStorage('test.db', { url }),
+ // useIdbStorage('test.db', { url }),
  )
- await importDatabase(db.vfs, db.path, file)
+ // await importDatabase(db.vfs, db.path, file)
  console.log(
  await db.run(`SELECT "type", "tbl_name" AS "table", CASE WHEN "sql" LIKE '%PRIMARY KEY AUTOINCREMENT%' THEN 1 ELSE "name" END AS "name" FROM "sqlite_master"`),
  )
 })
 
 document.querySelector('.download')?.addEventListener('click', async () => {
- download(await db.dump())
+ if (db) {
+ download(await db.dump())
+ }
 })
 
+const ee = mitt<{
+ data: [any]
+ done: []
+}>()
+
+function test(): AsyncIterableIterator<any> {
+ let resolver: ((value: IteratorResult<any>) => void) | null = null
+
+ ee.on('data', (...data) => {
+ if (resolver) {
+ resolver({ value: data[0] })
+ resolver = null
+ }
+ })
+
+ ee.on('done', () => {
+ if (resolver) {
+ resolver({ value: undefined, done: true })
+ }
+ })
+
+ return {
+ [Symbol.asyncIterator]() {
+ return this
+ },
+ async next() {
+ return new Promise<IteratorResult<any>>((resolve) => {
+ resolver = resolve
+ })
+ },
+ async return() {
+ return { value: undefined, done: true }
+ },
+ } satisfies AsyncIterableIterator<any>
+}
+
 document.querySelector('.worker')?.addEventListener('click', async () => {
  const worker = new OpfsWorker()
- const ee = mitt<{
- data: [any]
- done: []
- }>()
  worker.onmessage = ({ data }) => {
  if (data === 'done') {
  ee.emit('done')
  } else {
  ee.emit('data', data)
  }
  }
- function test(): AsyncIterableIterator<any> {
- let resolver: ((value: IteratorResult<any>) => void) | null = null
-
- ee.on('data', (...data) => {
- if (resolver) {
- console.log('data')
- resolver({ value: data[0] })
- resolver = null
- }
- })
-
- ee.on('done', () => {
- if (resolver) {
- resolver({ value: undefined, done: true })
- }
- })
-
- return {
- [Symbol.asyncIterator]() {
- return this
- },
- async next() {
- return new Promise<IteratorResult<any>>((resolve) => {
- resolver = resolve
- })
- },
- async return() {
- return { value: undefined, done: true }
- },
- } satisfies AsyncIterableIterator<any>
- }
  worker.postMessage('')
  for await (const data of test()) {
  console.log('iterator', data)
  }
 })
+document.querySelector('.importW')?.addEventListener('click', async () => {
+ const worker = new OpfsWorker()
+ worker.onmessage = ({ data }) => {
+ if (data === 'done') {
+ ee.emit('done')
+ } else {
+ ee.emit('data', data)
+ }
+ }
+ let file
+ try {
+ file = await selectFile('.db,.sqlite,.sqlite3')
+ } catch (error) {
+ // eslint-disable-next-line no-alert
+ alert(`${error}`)
+ return
+ }
+ worker.postMessage(file)
+ for await (const data of test()) {
+ console.log('[import] iterator', data)
+ }
+})
 document.querySelector('.clear')?.addEventListener('click', async () => {
- await db.close()
+ await db?.close()
  const root = await navigator.storage.getDirectory()
  for await (const [name] of root.entries()) {
- console.log('clear', name)
  await root.removeEntry(name, { recursive: true })
- console.log('clear success', name)
+ console.log('delete entry:', name)
  }
  console.log('clear all OPFS')
  const dbs = await window.indexedDB.databases()
  dbs.forEach((db) => {
- console.log('clear', db.name)
+ console.log('delete idb:', db.name)
  window.indexedDB.deleteDatabase(db.name!)
  })
  console.log('clear all IndexedDB')

playground/src/worker.ts

@@ -1,23 +1,27 @@
 import { uuidv7 } from 'uuidv7'
 import url from '../../dist/wa-sqlite.wasm?url'
-import { customFunction, initSQLite, isOpfsSupported } from '../../src'
+import { customFunction, initSQLite, isOpfsSupported, withExistDB } from '../../src'
 import { useOpfsStorage } from '../../src/vfs/opfs'
 import { runSQLStream } from './runSQL'
 
-onmessage = async () => {
+onmessage = async ({ data }) => {
  if (!await isOpfsSupported()) {
  return
  }
- const { run, stream, lastInsertRowId, changes, sqlite, db } = await initSQLite(useOpfsStorage(
- 'test',
- { url },
- // 'https://cdn.jsdelivr.net/gh/rhashimoto/wa-sqlite@v0.9.9/dist/wa-sqlite.wasm',
+ const db = await initSQLite(useOpfsStorage(
+ 'test.db',
+ data ? withExistDB(data, { url }) : { url },
+ // { url },
  ))
- await runSQLStream(run, stream, data => postMessage(data))
- console.log(lastInsertRowId(), changes())
- customFunction(sqlite, db, 'uuidv7', () => uuidv7())
+ // if (data) {
+ // await db.sync(data)
+ // }
+ await runSQLStream(db.run, db.stream, data => postMessage(data))
+ console.log(db.lastInsertRowId(), db.changes())
+ customFunction(db.sqlite, db.pointer, 'uuidv7', () => uuidv7())
  console.log(
- await run('select uuidv7() as a'),
+ 'uuidv7():',
+ (await db.run('select uuidv7() as a'))[0].a,
  )
  postMessage('done')
 }

src/io/import.ts

@@ -1,3 +1,4 @@
+/* eslint-disable antfu/consistent-list-newline */
 // reference from https://github.com/rhashimoto/wa-sqlite/blob/master/demo/file/index.js
 import type { FacadeVFS, Promisable } from '../types'
 import {
@@ -15,79 +16,69 @@ import {
 } from '../constant'
 import { check, ignoredDataView } from './common'
 
-async function* pagify(stream: ReadableStream<Uint8Array>): AsyncGenerator<Uint8Array> {
- const chunks: Uint8Array[] = []
- const reader = stream.getReader()
+const SQLITE_BINARY_HEADER = new Uint8Array([
+ 0x53, 0x51, 0x4C, 0x69, 0x74, 0x65, 0x20, 0x66, // SQLite f
+ 0x6F, 0x72, 0x6D, 0x61, 0x74, 0x20, 0x33, 0x00, // ormat 3\0
+])
+
+async function parseHeaderAndVerify(
+ reader: ReadableStreamDefaultReader<Uint8Array<ArrayBufferLike>>,
+): Promise<Uint8Array<ArrayBufferLike>> {
+ const headerData = await readExactBytes(reader, 32)
+ for (let i = 0; i < SQLITE_BINARY_HEADER.length; i++) {
+ if (headerData[i] !== SQLITE_BINARY_HEADER[i]) {
+ throw new Error('Not a SQLite database file')
+ }
+ }
+ return headerData
+}
+
+async function readExactBytes(
+ reader: ReadableStreamDefaultReader<Uint8Array>,
+ size: number,
+): Promise<Uint8Array> {
+ const result = new Uint8Array(size)
+ let offset = 0
 
- // read file header
- while (chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0) < 32) {
+ while (offset < size) {
  const { done, value } = await reader.read()
  if (done) {
  throw new Error('Unexpected EOF')
  }
- chunks.push(value!)
- }
 
- let copyOffset = 0
- const header = new DataView(new ArrayBuffer(32))
- for (const chunk of chunks) {
- const src = chunk.subarray(0, header.byteLength - copyOffset)
- const dst = new Uint8Array(header.buffer, copyOffset)
- dst.set(src)
- copyOffset += src.byteLength
+ const bytesToCopy = Math.min(size - offset, value!.length)
+ result.set(value!.subarray(0, bytesToCopy), offset)
+ offset += bytesToCopy
  }
 
- if (new TextDecoder().decode(header.buffer.slice(0, 16)) !== 'SQLite format 3\x00') {
- throw new Error('Not a SQLite database file')
- }
+ return result
+}
 
- const pageSize = (field => field === 1 ? 65536 : field)(header.getUint16(16))
- const pageCount = header.getUint32(28)
- // console.log(`${pageCount} pages, ${pageSize} bytes each, ${pageCount * pageSize} bytes total`)
-
- // copy pages
- for (let i = 0; i < pageCount; ++i) {
- while (chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0) < pageSize) {
- const { done, value } = await reader.read()
- if (done) {
- throw new Error('Unexpected EOF')
- }
- chunks.push(value!)
- }
+async function* pagify(stream: ReadableStream<Uint8Array>): AsyncGenerator<Uint8Array> {
+ const reader = stream.getReader()
 
- let page: Uint8Array
- if (chunks[0]?.byteLength >= pageSize) {
- page = chunks[0].subarray(0, pageSize)
- chunks[0] = chunks[0].subarray(pageSize)
- if (!chunks[0].byteLength) {
- chunks.shift()
- }
- } else {
- let copyOffset = 0
- page = new Uint8Array(pageSize)
- while (copyOffset < pageSize) {
- const src = chunks[0].subarray(0, pageSize - copyOffset)
- const dst = new Uint8Array(page.buffer, copyOffset)
- dst.set(src)
- copyOffset += src.byteLength
-
- chunks[0] = chunks[0].subarray(src.byteLength)
- if (!chunks[0].byteLength) {
- chunks.shift()
- }
- }
- }
+ try {
+ const headerData = await parseHeaderAndVerify(reader)
 
- yield page
- }
+ const view = new DataView(headerData.buffer)
+ const rawPageSize = view.getUint16(16)
+ const pageSize = rawPageSize === 1 ? 65536 : rawPageSize
+ const pageCount = view.getUint32(28)
+
+ for (let i = 0; i < pageCount; i++) {
+ yield await readExactBytes(reader, pageSize)
+ }
 
- const { done } = await reader.read()
- if (!done) {
- throw new Error('Unexpected data after last page')
+ const { done } = await reader.read()
+ if (!done) {
+ throw new Error('Unexpected data after last page')
+ }
+ } finally {
+ reader.releaseLock()
  }
 }
 
-export async function importDatabaseStream(
+export async function importDatabaseToIdb(
  vfs: FacadeVFS,
  path: string,
  stream: ReadableStream<Uint8Array>,
@@ -129,6 +120,21 @@ export async function importDatabaseStream(
  }
 }
 
+async function importDatabaseToOpfs(
+ path: string,
+ source: ReadableStream<Uint8Array>,
+): Promise<void> {
+ const root = await navigator.storage.getDirectory()
+ const handle = await root.getFileHandle(path, { create: true })
+ const [streamForVerify, streamData] = source.tee()
+
+ await parseHeaderAndVerify(streamForVerify.getReader())
+
+ const writable = await handle.createWritable()
+ // `pipeTo()` will auto close `writable`
+ await streamData.pipeTo(writable)
+}
+
 /**
  * Import database from `File` or `ReadableStream`
  * @param vfs SQLite VFS
@@ -140,9 +146,11 @@ export async function importDatabase(
  path: string,
  data: File | ReadableStream<Uint8Array>,
 ): Promise<void> {
- return await importDatabaseStream(
- vfs,
- path,
- data instanceof File ? data.stream() : data,
- )
+ const stream = data instanceof globalThis.File ? data.stream() : data
+ // is `OPFSCoopSyncVFS`
+ if ('releaser' in vfs) {
+ await importDatabaseToOpfs(path, stream)
+ } else {
+ await importDatabaseToIdb(vfs, path, stream)
+ }
 }