Skip to content

Commit bc5ad00

Browse files
subframe7536subframe7536
committed
feat: import and export #1 
1 parent 1fe581c commit bc5ad00

File tree

13 files changed

+455
-69
lines changed

13 files changed

+455
-69
lines changed

‎playground/index.html‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@
77
<link rel="icon" href="data:,">
88
</head>
99
<body>
10+
<button class="main">main</button>
1011
<button class="worker">worker</button>
12+
<button class="import">import</button>
13+
<button class="download">download</button>
1114
<button class="clear">clear</button>
1215
</body>
1316
<script type="module" src="/src/index.ts"></script>

‎playground/src/index.ts‎

Lines changed: 124 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -7,76 +7,100 @@ import {
77
isModuleWorkerSupport,
88
isOpfsSupported,
99
useMemoryStorage,
10-
} from '../../src'
11-
import { useIdbStorage } from '../../src/vfs/idb'
10+
withExistDB,
11+
} from '../../src/index'
12+
import { useIdbMemoryStorage as useIdbStorage } from '../../src/vfs/idb'
1213
import { runSQL } from './runSQL'
1314
import OpfsWorker from './worker?worker'
1415

15-
const { run, close } = await initSQLite(useIdbStorage('test.db', {
16-
url,
17-
// url: 'https://cdn.jsdelivr.net/gh/rhashimoto/wa-sqlite@v0.9.9/dist/wa-sqlite-async.wasm',
18-
}))
16+
let db = await initSQLite(useIdbStorage('test.db', { url }))
1917

2018
const supportModuleWorker = isModuleWorkerSupport()
2119
const supportIDB = isIdbSupported()
2220
const supportOPFS = await isOpfsSupported()
2321
console.log('support module worker:', supportModuleWorker)
2422
console.log('support IDBBatchAtomicVFS:', supportIDB)
2523
console.log('support OPFSCoopSyncVFS:', supportOPFS)
26-
await runSQL(run)
27-
await runSQL((await initSQLite(useMemoryStorage({ url: syncUrl }))).run)
28-
29-
const worker = new OpfsWorker()
30-
const ee = mitt<{
31-
data: [any]
32-
done: []
33-
}>()
34-
worker.onmessage = ({ data }) => {
35-
if (data === 'done') {
36-
ee.emit('done')
37-
} else {
38-
ee.emit('data', data)
24+
document.querySelector('.main')?.addEventListener('click', async () => {
25+
if (!db) {
26+
db = await initSQLite(useIdbStorage('test.db', { url }))
3927
}
40-
}
41-
function test(): AsyncIterableIterator<any> {
42-
let resolver: ((value: IteratorResult<any>) => void) | null = null
28+
await runSQL(db.run)
29+
await runSQL((await initSQLite(useMemoryStorage({ url: syncUrl }))).run)
30+
})
31+
document.querySelector('.import')?.addEventListener('click', async () => {
32+
await db?.close()
33+
let file
34+
try {
35+
file = await selectFile('.db,.sqlite,.sqlite3')
36+
} catch (error) {
37+
// eslint-disable-next-line no-alert
38+
prompt(`${error}`)
39+
return
40+
}
41+
db = await initSQLite(
42+
useIdbStorage('test.db', withExistDB(file, { url })),
43+
)
44+
console.log(
45+
await db.run(`SELECT "type", "tbl_name" AS "table", CASE WHEN "sql" LIKE '%PRIMARY KEY AUTOINCREMENT%' THEN 1 ELSE "name" END AS "name" FROM "sqlite_master"`),
46+
)
47+
})
4348

44-
ee.on('data', (...data) => {
45-
if (resolver) {
46-
console.log('data')
47-
resolver({ value: data[0] })
48-
resolver = null
49-
}
50-
})
49+
document.querySelector('.download')?.addEventListener('click', async () => {
50+
download(await db.dump())
51+
})
5152

52-
ee.on('done', () => {
53-
if (resolver) {
54-
resolver({ value: undefined, done: true })
53+
document.querySelector('.worker')?.addEventListener('click', async () => {
54+
const worker = new OpfsWorker()
55+
const ee = mitt<{
56+
data: [any]
57+
done: []
58+
}>()
59+
worker.onmessage = ({ data }) => {
60+
if (data === 'done') {
61+
ee.emit('done')
62+
} else {
63+
ee.emit('data', data)
5564
}
56-
})
65+
}
66+
function test(): AsyncIterableIterator<any> {
67+
let resolver: ((value: IteratorResult<any>) => void) | null = null
5768

58-
return {
59-
[Symbol.asyncIterator]() {
60-
return this
61-
},
62-
async next() {
63-
return new Promise<IteratorResult<any>>((resolve) => {
64-
resolver = resolve
65-
})
66-
},
67-
async return() {
68-
return { value: undefined, done: true }
69-
},
70-
} satisfies AsyncIterableIterator<any>
71-
}
72-
document.querySelector('.worker')?.addEventListener('click', async () => {
69+
ee.on('data', (...data) => {
70+
if (resolver) {
71+
console.log('data')
72+
resolver({ value: data[0] })
73+
resolver = null
74+
}
75+
})
76+
77+
ee.on('done', () => {
78+
if (resolver) {
79+
resolver({ value: undefined, done: true })
80+
}
81+
})
82+
83+
return {
84+
[Symbol.asyncIterator]() {
85+
return this
86+
},
87+
async next() {
88+
return new Promise<IteratorResult<any>>((resolve) => {
89+
resolver = resolve
90+
})
91+
},
92+
async return() {
93+
return { value: undefined, done: true }
94+
},
95+
} satisfies AsyncIterableIterator<any>
96+
}
7397
worker.postMessage('')
7498
for await (const data of test()) {
7599
console.log('iterator', data)
76100
}
77101
})
78102
document.querySelector('.clear')?.addEventListener('click', async () => {
79-
await close()
103+
await db.close()
80104
const root = await navigator.storage.getDirectory()
81105
for await (const [name] of root.entries()) {
82106
console.log('clear', name)
@@ -91,3 +115,54 @@ document.querySelector('.clear')?.addEventListener('click', async () => {
91115
})
92116
console.log('clear all IndexedDB')
93117
})
118+
119+
function download(buffer: Uint8Array): void {
120+
const blob = new Blob([buffer])
121+
const reader = new FileReader()
122+
reader.readAsDataURL(blob)
123+
reader.onload = (e) => {
124+
const a = document.createElement('a')
125+
a.download = `test.db`
126+
a.href = e.target?.result as string
127+
document.body.appendChild(a)
128+
a.click()
129+
document.body.removeChild(a)
130+
}
131+
}
132+
133+
async function selectFile(accept?: string): Promise<File> {
134+
return await new Promise((resolve, reject) => {
135+
const input = document.createElement('input')
136+
input.type = 'file'
137+
if (accept) {
138+
input.accept = accept
139+
}
140+
141+
input.onchange = () => {
142+
const file = input.files?.[0]
143+
if (file) {
144+
resolve(file)
145+
} else {
146+
reject(new Error('No file selected'))
147+
}
148+
input.remove()
149+
}
150+
151+
input.oncancel = () => {
152+
reject(new Error('File selection cancelled'))
153+
input.remove()
154+
}
155+
156+
// 处理用户点击取消的情况
157+
window.addEventListener('focus', () => {
158+
setTimeout(() => {
159+
if (!input.files?.length) {
160+
reject(new Error('File selection cancelled'))
161+
input.remove()
162+
}
163+
}, 300)
164+
}, { once: true })
165+
166+
input.click()
167+
})
168+
}

‎src/core.ts‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
import type { Options, Promisable, SQLiteDB, SQLiteDBCore } from './types'
2-
import { Factory, SQLITE_OPEN_CREATE, SQLITE_OPEN_READONLY, SQLITE_OPEN_READWRITE, SQLITE_ROW } from 'wa-sqlite'
2+
import {
3+
Factory,
4+
SQLITE_OPEN_CREATE,
5+
SQLITE_OPEN_READONLY,
6+
SQLITE_OPEN_READWRITE,
7+
SQLITE_ROW,
8+
} from 'wa-sqlite'
9+
import { exportDatabase } from './io'
310

411
/**
512
* Load SQLite database, presets: `useMemoryStorage`, `useIdbStorage`, `useIdbMemoryStorage`, `useOpfsStorage`
@@ -12,6 +19,7 @@ export async function initSQLite(options: Promisable<Options>): Promise<SQLiteDB
1219
...core,
1320
changes: () => changes(core),
1421
close: () => close(core),
22+
dump: () => exportDatabase(core.vfs, core.path),
1523
lastInsertRowId: () => lastInsertRowId(core),
1624
run: (...args) => run(core, ...args),
1725
stream: (...args) => stream(core, ...args),
@@ -27,18 +35,20 @@ export async function initSQLite(options: Promisable<Options>): Promise<SQLiteDB
2735
export async function initSQLiteCore(
2836
options: Promisable<Options>,
2937
): Promise<SQLiteDBCore> {
30-
const { path, sqliteModule, vfsFn, vfsOptions, readonly } = await options
38+
const { path, sqliteModule, vfsFn, vfsOptions, readonly, beforeOpen } = await options
3139
const sqlite = Factory(sqliteModule)
3240
const vfs = await vfsFn(path, sqliteModule, vfsOptions)
3341
sqlite.vfs_register(vfs as unknown as SQLiteVFS, true)
34-
const db = await sqlite.open_v2(
42+
beforeOpen?.(vfs, path)
43+
const pointer = await sqlite.open_v2(
3544
path,
3645
readonly ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,
3746
)
3847
/// keep-sorted
3948
return {
40-
db,
49+
db: pointer,
4150
path,
51+
pointer,
4252
sqlite,
4353
sqliteModule,
4454
vfs,

‎src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
export * from './core'
2+
export * from './io'
23
export * from './types'
34
export * from './utils'
45
export * from './vfs/memory'

‎src/io/common.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
import type { Promisable } from '../types'
2+
import { SQLITE_OK } from '../constant'
3+
4+
export async function check(code: Promisable<number>): Promise<void> {
5+
if (await code !== SQLITE_OK) {
6+
throw new Error(`Error code: ${await code}`)
7+
}
8+
}
9+
10+
export function ignoredDataView(): DataView {
11+
return new DataView(new ArrayBuffer(4))
12+
}

‎src/io/export.ts‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// reference from https://github.com/rhashimoto/wa-sqlite/blob/master/demo/file/service-worker.js
2+
import type { FacadeVFS, Promisable, SQLiteDBCore } from '../types'
3+
import {
4+
SQLITE_LOCK_NONE,
5+
SQLITE_LOCK_SHARED,
6+
SQLITE_OPEN_MAIN_DB,
7+
SQLITE_OPEN_READONLY,
8+
} from '../constant'
9+
import { check, ignoredDataView } from './common'
10+
11+
export function dumpReadableStream(vfs: FacadeVFS, path: string): ReadableStream {
12+
const source = getExistDataSource(vfs, path)
13+
source.isDone.finally(() => {
14+
vfs.close()
15+
})
16+
17+
return new ReadableStream(source)
18+
}
19+
20+
function getExistDataSource(
21+
vfs: FacadeVFS,
22+
path: string,
23+
): UnderlyingDefaultSource & { isDone: Promise<void> } {
24+
let onDone: (() => Promisable<any>)[] = []
25+
let resolve!: () => void
26+
let reject!: (reason?: any) => void
27+
let isDone: Promise<void> = new Promise<void>((res, rej) => {
28+
resolve = res
29+
reject = rej
30+
}).finally(async () => {
31+
while (onDone.length) {
32+
await onDone.pop()!()
33+
}
34+
})
35+
let fileId = Math.floor(Math.random() * 0x100000000)
36+
let iOffset = 0
37+
let bytesRemaining = 0
38+
39+
return {
40+
isDone,
41+
async start(controller: ReadableStreamDefaultController): Promise<void> {
42+
try {
43+
const flags = SQLITE_OPEN_MAIN_DB | SQLITE_OPEN_READONLY
44+
await check(vfs.jOpen(path, fileId, flags, ignoredDataView()))
45+
onDone.push(() => vfs.jClose(fileId))
46+
47+
await check(vfs.jLock(fileId, SQLITE_LOCK_SHARED))
48+
onDone.push(() => vfs.jUnlock(fileId, SQLITE_LOCK_NONE))
49+
50+
const fileSize = new DataView(new ArrayBuffer(8))
51+
await check(vfs.jFileSize(fileId, fileSize))
52+
bytesRemaining = Number(fileSize.getBigUint64(0, true))
53+
} catch (e) {
54+
controller.error(e)
55+
reject(e)
56+
}
57+
},
58+
59+
async pull(controller: ReadableStreamDefaultController): Promise<void> {
60+
try {
61+
const buffer = new Uint8Array(Math.min(bytesRemaining, 65536))
62+
await check(vfs.jRead(fileId, buffer, iOffset))
63+
controller.enqueue(buffer)
64+
65+
iOffset += buffer.byteLength
66+
bytesRemaining -= buffer.byteLength
67+
if (bytesRemaining === 0) {
68+
controller.close()
69+
resolve()
70+
}
71+
} catch (e) {
72+
controller.error(e)
73+
reject(e)
74+
}
75+
},
76+
77+
cancel(reason: string): void {
78+
reject(new Error(reason))
79+
},
80+
}
81+
}
82+
83+
export async function streamToUint8Array(stream: ReadableStream): Promise<Uint8Array> {
84+
const chunks: Uint8Array[] = []
85+
const reader = stream.getReader()
86+
87+
while (true) {
88+
const { done, value } = await reader.read()
89+
if (done) {
90+
break
91+
}
92+
chunks.push(value)
93+
}
94+
95+
// Combine all chunks into a single Uint8Array
96+
const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0)
97+
const result = new Uint8Array(totalLength)
98+
let position = 0
99+
100+
for (const chunk of chunks) {
101+
result.set(chunk, position)
102+
position += chunk.length
103+
}
104+
105+
return result
106+
}
107+
108+
export async function exportDatabase(vfs: FacadeVFS, path: string): Promise<Uint8Array> {
109+
return await streamToUint8Array(dumpReadableStream(vfs, path))
110+
}

0 commit comments

Comments
 (0)