Javascript Binary Structured Object Notation
This library provides tools to :
- encode javascript values like
objects,Map,numbers,ArrayBuffer, etc... into a sequence of bytes - decode JBSON bytes' sequence into js values
- clone complex variables (structured clone)
It may be used to transmit complex data structure than JSON doesn't support like:
- binary data (ArrayBuffer)
- built-in types: Map, Set, RegExp, Date, BigInt, etc...
- circular references and pointers
Or you may use it to clone a complex variable.
To install:
yarn add @lifaon/jbson # or npm i @lifaon/jbson --saveEntry point: index.js. I recommend you to use rollup to import/bundle the package, but you may use an already bundled version in bundles/.
You may also use unpkg: https://unpkg.com/@lifaon/jbson
WARN: This implementation is different than mongoDB's BSON.
INFO: This implementation doesn't aim to compress data:
- in some cases the size may be strongly reduced (10~30% of the size of the JSON equivalent)
- in other cases data may be bigger than JSON (a few)
- moreover, strings are not compressed, so you'll still benefit of gziping the bytes when sending them.
INFO: suggested mime-type: application/jbson
Related:
// WARN the returned Uint8Array is shared, use .slice() to clone its values function EncodeToJBSON<T>(value: T): Uint8Array;Example:
const obj: any = {}; obj.obj = obj; console.log(EncodeToJBSON(obj)); // output Uint8Array([17, 1, 4, 3, 111, 98, 106, 127, 0])function DecodeFromJBSON<T>(buffer: Uint8Array): T;Example:
console.log(DecodeFromJBSON(new Uint8Array([17, 1, 4, 3, 111, 98, 106, 127, 0]))); // output { obj: { obj: { ... } } }function StructuredClone<T>(value: T): T;Example:
console.log(StructuredClone({ a: 1 })); // output { a: 1 }Assuming than:
writeis a function which writes a byte into a buffer and increment the write index.readis a function which reads a byte from a buffer and increment the read index.
export type WriteFunction = (value: number) => void; export type ReadFunction = () => number;Pointers:
export type Pointer = number; export type GetPointerFunction = () => Pointer;Some structures may include some circular or shared references like: const obj = {}; obj.obj = obj;
JBSON supports such conditions by creating a Pointer which is nothing more than the index where is reference as been encoded.
Notations:
0b[bit 7, bit 6, ... bit 0]=> represents a byte[0b[...], 0b[...]]or[1, 2, 3, ...]=> represents a sequence of bytes
The size is a variable length number, encoded in 7 bits every bytes, where the 8th bit is used to notify than the next byte is part of this number.
[0b[<bit 7>: 1 if more bits are required to encode the number, <bit 6-0>: number bits (little endian)], ...repeat{0,}]
Example: encoding 1234 (0b 0000 0100 1101 0010)
[0b11010010, 0b00000100]
show
export function EncodeSize(size: number, write: WriteFunction): void { let byte: number; do { byte = (size & 0b01111111); size >>= 7; byte |= ((size !== 0) as any) << 7; write(byte); } while (size !== 0); } export function DecodeSize(read: ReadFunction): number { let size: number = 0; let byte: number; let offset: number = 0; do { byte = read(); size |= (byte & 0b01111111) << offset; offset += 7; } while (byte & 0b10000000); return size; } export function EncodeBigSize(size: bigint, write: WriteFunction): void { let byte: number; do { byte = Number(size & 0b01111111n); size >>= 7n; byte |= ((size !== 0n) as any) << 7; write(byte); } while (size !== 0n); } export function DecodeBigSize(read: ReadFunction): bigint { let size: bigint = 0n; let byte: number; let offset: bigint = 0n; do { byte = read(); size |= BigInt(byte & 0b01111111) << offset; offset += 7n; } while (byte & 0b10000000); return size; }The following encoders/decoders wont specify the type byte preceding the encoded bits. The type byte values are specified into the any section.
The booleans are simply stored as 0 if false and 1 if true.
[0b[<bit 7-1>: 0, boolean ? 1 : 0]]
Example:
- true:
[0b00000001]=[1] - false:
[0b00000000]=[0]
show
export function EncodeBoolean(boolean: boolean, write: WriteFunction): void { write(boolean ? 1 : 0); } export function DecodeBoolean(read: ReadFunction): boolean { return (read() !== 0); }Numbers may have the following types:
export enum NUMBER_TYPES { INT8 = 0x00, UINT8 = 0x01, INT16 = 0x02, UINT16 = 0x03, INT32 = 0x04, UINT32 = 0x05, INT64 = 0x06, UINT64 = 0x07, FLOAT32 = 0x08, FLOAT64 = 0x09, }They are stored like that:
[0b[<bit 7-0>: NUMBER_TYPES[type of the number]], ...number bits stored as big-endian{1-8}]
Example: encoding 1234
- Inferred type:
NUMBER_TYPES.UINT16 - Bytes:
[3 /* number type (uint16) */, 4 /* high byte of the number */, 210 /* low byte of the number */]=[3, 4, 210]
show
const dataView = new DataView(new ArrayBuffer(8)); export function EncodeNumber(number: number, write: WriteFunction): void { const type: NUMBER_TYPES = InferNumberTypeOfNumber(number); write(type); SetNumberInDataView(number, type, dataView, 0, false); for (let i = 0, l = NumberTypeByteLength(type); i < l; i++) { write(dataView.getUint8(i)); } } export function DecodeNumber(read: ReadFunction): number { const type: NUMBER_TYPES = read(); for (let i = 0, l = NumberTypeByteLength(type); i < l; i++) { dataView.setUint8(i, read()); } return GetNumberInDataView(type, dataView, 0, false); }Strings are converted into an utf8 encoded Uint8Array, then the array length is encoded using EncodeSize and finally the content is written just after.
[...size of the string{1,}, ...content of the string{0,}]
Example: encoding 'abc'
[3 /* string's length */, 97 /* 'a' */, 98 /* 'b' */, 99 /* 'c' */] = [3, 97, 98, 99]
show
export function EncodeString(string: string, write: WriteFunction): void { const bytes: Uint8Array = textEncoder.encode(string); EncodeSize(bytes.length, write); for (let i = 0, l = bytes.length; i < l; i++) { write(bytes[i]); } } export function DecodeString(read: ReadFunction): string { const size: number = DecodeSize(read); const bytes: Uint8Array = (size < tempUint8Array.length) ? tempUint8Array : new Uint8Array(size); for (let i = 0; i < size; i++) { bytes[i] = read(); } return textDecoder.decode(bytes.subarray(0, size)); }BigInts are simply stored as if they where size
Example: encoding 1234n
[210, 9]
show
export function EncodeBigInt(number: bigint, write: WriteFunction): void { return EncodeBigSize(number, write); } export function DecodeBigInt(read: ReadFunction): bigint { return DecodeBigSize(read); }Dates are stored as number (timestamp in milliseconds) using EncodeNumber.
[...timestamp of the date in milliseconds encoded as number{2,9}]
Example: encoding new Date('04 Dec 1995 00:12:00 GMT')
[7 /* number type (uint64) */, 0, 0, 0, 190, 118, 189, 140, 128 /* ... number bits */] = [7, 0, 0, 0, 190, 118, 189, 140, 128]
show
export function EncodeDate(date: Date, write: WriteFunction): void { EncodeNumber(date.valueOf(), write); } export function DecodeDate(read: ReadFunction): Date { return new Date(DecodeNumber(read)); }RegExps are stored as a tuple of string composed of the source and the flags using EncodeString.
[...regexp.source encoded as string{1,}, ...regexp.flags encoded as string{1,}]
Example: encoding new RegExp(/abc/g)
[3 /* regex.source's length */, 97, 98, 99 /* ... 'abc' */, 1 /* regex.flags' length */, 103 /* 'g' */] = [3, 97, 98, 99, 1, 103]
show
export function EncodeRegExp(regexp: RegExp, write: WriteFunction): void { EncodeString(regexp.source, write); EncodeString(regexp.flags, write); } export function DecodeRegExp(read: ReadFunction): RegExp { return new RegExp(DecodeString(read), DecodeString(read)); }ArrayBuffers are stored as a tuple composed of its size and its content bytes.
[...size of the buffer{1,}, ...buffer bytes{0,}]
Example: encoding new Uint8Array([0, 1, 2]).buffer
[3 /* buffer's size */, 0, 1, 2 /* ... buffer's content */] = [3, 0, 1, 2]
show
export function EncodeArrayBuffer(buffer: ArrayBuffer | SharedArrayBuffer, write: WriteFunction, byteOffset: number = 0, byteLength: number = buffer.byteLength): void { EncodeSize(byteLength, write); const bytes: Uint8Array = new Uint8Array(buffer, byteOffset, byteLength); for (let i = 0, l = bytes.length; i < l; i++) { write(bytes[i]); } } export function DecodeArrayBuffer(read: ReadFunction): ArrayBuffer { const bytes: Uint8Array = new Uint8Array(DecodeSize(read)); for (let i = 0; i < bytes.length; i++) { bytes[i] = read(); } return bytes.buffer; }ArrayBufferView (Uint8Array, Uint16Array, ...) are stored as a tuple composed of its number type (uint8, uint16, etc... see NUMBER_TYPES) and its content encoded as an ArrayBuffer with EncodeArrayBuffer.
[buffer type {1}, ...buffer size and bytes{1,}]
Example: encoding new Uint8Array([0, 1, 2])
[1 /* buffer's type (uint8) */, 3 /* buffer's size */, 0, 1, 2 /* ... buffer's content */] = [1, 3, 0, 1, 2]
show
export function EncodeArrayBufferView(buffer: ArrayBufferView, write: WriteFunction): void { write(ArrayBufferViewToNumberType(buffer)); EncodeArrayBuffer(buffer.buffer, write, buffer.byteOffset, buffer.byteLength); } export function DecodeArrayBufferView(read: ReadFunction): ArrayBufferView { return new (NumberTypeToArrayBufferViewConstructor(read()))(DecodeArrayBuffer(read)); }Maps are stored as:
[map entries' size {1,}, ...for each entries: tuple<EncodeAny(key), EncodeAny(value)>]
Example: encoding new Map([['a', 1]])
[ 1 /* number of entries in the map */, /** entry 0: **/ 4 /* string type */, 1 /* string's length */, 97 /* 'a' */, 3 /* number type */, 1 /* (uint8) */, 1 /* value */ ]= [1, 4, 1, 97, 3, 1, 1]
show
export function EncodeMap( map: Map<any, any>, write: WriteFunction, getPointer: GetPointerFunction, memory: Map<any, Pointer> = new Map<any, Pointer>() ): void { EncodeSize(map.size, write); for (const entry of map.entries()) { EncodeAny(entry[0], write, getPointer, memory); EncodeAny(entry[1], write, getPointer, memory); } } export function DecodeMap( read: ReadFunction, getPointer: GetPointerFunction, memory: Map<Pointer, any> = new Map<Pointer, any>(), pointer: Pointer = getPointer() ): Map<any, any> { const size: number = DecodeSize(read); const map: Map<any, any> = new Map<any, any>(); memory.set(pointer, map); for (let i = 0; i < size; i++) { const key: any = DecodeAny(read, getPointer, memory); const value: any = DecodeAny(read, getPointer, memory); map.set(key, value); } return map; }Sets are stored as:
[set values' size {1,}, ...for each values: EncodeAny(value)]
Example: encoding new Set(['a', 1])
[ 2 /* number of values in the set */, /** entry 0: **/ 4 /* string type */, 1 /* string's length */, 97 /* 'a' */, /** entry 1: **/ 3 /* number type */, 1 /* (uint8) */, 1 /* value */ ]= [2, 4, 1, 97, 3, 1, 1]
show
export function EncodeSet( set: Set<any>, write: WriteFunction, getPointer: GetPointerFunction, memory: Map<any, Pointer> = new Map<any, Pointer>() ): void { EncodeSize(set.size, write); for (const value of set.values()) { EncodeAny(value, write, getPointer, memory); } } export function DecodeSet( read: ReadFunction, getPointer: GetPointerFunction, memory: Map<Pointer, any> = new Map<Pointer, any>(), pointer: Pointer = getPointer() ): Set<any> { const size: number = DecodeSize(read); const set: Set<any> = new Set<any>(); memory.set(pointer, set); for (let i = 0; i < size; i++) { set.add(DecodeAny(read, getPointer, memory)); } return set; }Arrays are stored exactly as Set
Example: encoding ['a', 1]
[ 2 /* number of values in the array */, /** entry 0: **/ 4 /* string type */, 1 /* string's length */, 97 /* 'a' */, /** entry 1: **/ 3 /* number type */, 1 /* (uint8) */, 1 /* value */ ]= [2, 4, 1, 97, 3, 1, 1]
show
export function EncodeArray( array: any[], write: WriteFunction, getPointer: GetPointerFunction, memory: Map<any, Pointer> = new Map<any, Pointer>() ): void { EncodeSize(array.length, write); for (let i = 0, l = array.length; i < l; i++) { EncodeAny(array[i], write, getPointer, memory); } } export function DecodeArray( read: ReadFunction, getPointer: GetPointerFunction, memory: Map<Pointer, any> = new Map<Pointer, any>(), pointer: Pointer = getPointer() ): any[] { const size: number = DecodeSize(read); const array: any[] = new Array<any>(size); memory.set(pointer, array); for (let i = 0; i < size; i++) { array[i] = DecodeAny(read, getPointer, memory); } return array; }Objects are stored exactly as Map
Example: encoding { a: 1 }
[ 1 /* number of properties in the object */, /** property 0: **/ /** property's key: **/ 4 /* string type */, 1 /* string's length */, 97 /* 'a' */, /** property's value: **/ 3 /* number type */, 1 /* (uint8) */, 1 /* value */ ]= [1, 4, 1, 97, 3, 1, 1]
show
export function EncodeObject( object: any, write: WriteFunction, getPointer: GetPointerFunction, memory: Map<any, Pointer> = new Map<any, Pointer>() ): void { const entries: [any, any][] = Object.entries(object); EncodeSize(entries.length, write); for (let i = 0, l = entries.length; i < l; i++) { EncodeAny(entries[i][0], write, getPointer, memory); EncodeAny(entries[i][1], write, getPointer, memory); } } export function DecodeObject( read: ReadFunction, getPointer: GetPointerFunction, memory: Map<Pointer, any> = new Map<Pointer, any>(), pointer: Pointer = getPointer() ): object { const size: number = DecodeSize(read); const object: any = {}; memory.set(pointer, object); for (let i = 0; i < size; i++) { const key: any = DecodeAny(read, getPointer, memory); object[key] = DecodeAny(read, getPointer, memory); } return object; }export type Pointer = number; export type GetPointerFunction = () => Pointer;Some structures may include circular or shared references like: const obj = {}; obj.obj = obj;
JBSON supports such conditions by creating a Pointer which is nothing more than the index where is reference as been encoded.
Pointers are stored as size
Example: full encoding of const obj = {}; obj.obj = obj;
[ 17 /* object type */, 1 /* number of properties in the object */, /** property 0: **/ /** property's key: **/ 4 /* string type */, 3/* property key's length */, 111, 98, 106, /* 'obj' */ /** property's value: **/ 127 /* pointer type */, 0 /* index where is stored the reference's value => 0 which is the index of the object */ ]= [17, 1, 4, 3, 111, 98, 106, 127, 0]
show
export function EncodePointer(pointer: Pointer, write: WriteFunction): void { return EncodeSize(pointer, write); } export function DecodePointer(read: ReadFunction): Pointer { return DecodeSize(read); }Any is the entry point for every value you want to encode / decode. The encoder will convert a value into a sequence of bytes composed of the type of the value and its encoded bytes.
export enum ANY_TYPES { UNDEFINED = 0x00, NULL = 0x01, BOOLEAN = 0x02, NUMBER = 0x03, STRING = 0x04, SYMBOL = 0x05, BOOLEAN_OBJECT = 0x06, NUMBER_OBJECT = 0x07, STRING_OBJECT = 0x08, DATE = 0x09, REGEXP = 0x0a, SHARED_ARRAY_BUFFER = 0x0b, ARRAY_BUFFER = 0x0c, ARRAY_BUFFER_VIEW = 0x0d, MAP = 0x0e, SET = 0x0f, ARRAY = 0x10, OBJECT = 0x11, BIGINT = 0x12, POINTER = 0x7f, }[value's type {1}, ...encoded value's bytes{1,}]
show encoding
export function EncodeAny( value: any, write: WriteFunction, getPointer: GetPointerFunction, memory: Map<any, Pointer> = new Map<any, Pointer>() ): void { if (memory.has(value)) { write(ANY_TYPES.POINTER); EncodePointer(memory.get(value) as Pointer, write); } else { if ((value !== null) && (value !== void 0) && (typeof value.toJBSON === 'function')) { EncodeAny(value.toJBSON(), write, getPointer, memory); } else { const type: string = typeof value; // p4 if (type === 'undefined') { write(ANY_TYPES.UNDEFINED); } else if (value === null) { write(ANY_TYPES.NULL); } else if (type === 'boolean') { write(ANY_TYPES.BOOLEAN); EncodeBoolean(value, write); } else if (type === 'number') { write(ANY_TYPES.NUMBER); EncodeNumber(value, write); } else if (type === 'string') { write(ANY_TYPES.STRING); EncodeString(value, write); } else if (type === 'symbol') { // p5 throw new Error(`Value could not be cloned: ${ value.toString() } is a Symbol`); } else if (type === 'bigint') { write(ANY_TYPES.BIGINT); EncodeBigInt(value, write); } else if (type === 'object') { memory.set(value, getPointer()); // p6 & p23 if (value instanceof Boolean) { // p7 write(ANY_TYPES.BOOLEAN_OBJECT); EncodeBoolean(value.valueOf(), write); } else if (value instanceof Number) { // p8 write(ANY_TYPES.NUMBER_OBJECT); EncodeNumber(value.valueOf(), write); } else if (value instanceof String) { // p9 write(ANY_TYPES.STRING_OBJECT); EncodeString(value.valueOf(), write); } else if (value instanceof Date) { // p10 write(ANY_TYPES.DATE); EncodeDate(value, write); } else if (value instanceof RegExp) { // p11 write(ANY_TYPES.REGEXP); EncodeRegExp(value, write); } else if ((typeof SharedArrayBuffer !== 'undefined') && (value instanceof SharedArrayBuffer)) { // p12.2 // if(forStorage) throw new DataCloneError('Value could not be cloned: is a SharedArrayBuffer'); write(ANY_TYPES.SHARED_ARRAY_BUFFER); EncodeArrayBuffer(value, write); } else if (value instanceof ArrayBuffer) { // p12.3 write(ANY_TYPES.ARRAY_BUFFER); EncodeArrayBuffer(value, write); } else if (ArrayBuffer.isView(value)) { // p13 write(ANY_TYPES.ARRAY_BUFFER_VIEW); EncodeArrayBufferView(value, write); } else if (value instanceof Map) { // p14 write(ANY_TYPES.MAP); EncodeMap(value, write, getPointer, memory); } else if (value instanceof Set) { // p15 write(ANY_TYPES.SET); EncodeSet(value, write, getPointer, memory); } else if (Array.isArray(value)) { // p16 write(ANY_TYPES.ARRAY); EncodeArray(value, write, getPointer, memory); } else if (!IsPlainObject(value)) { // p18 if (typeof value.toJSON === 'function') { EncodeAny(value.toJSON(), write, getPointer, memory); } else { // INFO super hard to implement let string: string = String(value); if (string.length > 200) { string = string.substring(0, 150) + '\n[...]\n' + string.slice(-50); } console.log(value); throw new TypeError(`Unsupported type : ${ string }`); } } else { write(ANY_TYPES.OBJECT); EncodeObject(value, write, getPointer, memory); } } else { throw new TypeError(`Unsupported type : ${ type }`); } } } }show decoding
export function DecodeAny( read: ReadFunction, getPointer: GetPointerFunction, memory: Map<Pointer, any> = new Map<Pointer, any>() ): any { const pointer: Pointer = getPointer(); const type: number = read(); let value: any; switch (type) { case ANY_TYPES.UNDEFINED: return void 0; case ANY_TYPES.NULL: return null; case ANY_TYPES.BOOLEAN: return DecodeBoolean(read); case ANY_TYPES.NUMBER: return DecodeNumber(read); case ANY_TYPES.STRING: return DecodeString(read); case ANY_TYPES.BIGINT: return DecodeBigInt(read); case ANY_TYPES.BOOLEAN_OBJECT: value = Boolean(DecodeBoolean(read)); break; case ANY_TYPES.NUMBER_OBJECT: value = Number(DecodeNumber(read)); break; case ANY_TYPES.STRING_OBJECT: value = String(DecodeString(read)); break; case ANY_TYPES.DATE: value = DecodeDate(read); break; case ANY_TYPES.REGEXP: value = DecodeRegExp(read); break; case ANY_TYPES.SHARED_ARRAY_BUFFER: value = DecodeArrayBuffer(read); break; case ANY_TYPES.ARRAY_BUFFER: value = DecodeArrayBuffer(read); break; case ANY_TYPES.ARRAY_BUFFER_VIEW: value = DecodeArrayBufferView(read); break; case ANY_TYPES.MAP: value = DecodeMap(read, getPointer, memory, pointer); break; case ANY_TYPES.SET: value = DecodeSet(read, getPointer, memory, pointer); break; case ANY_TYPES.ARRAY: value = DecodeArray(read, getPointer, memory, pointer); break; case ANY_TYPES.OBJECT: value = DecodeObject(read, getPointer, memory, pointer); break; case ANY_TYPES.POINTER: const address: Pointer = DecodePointer(read); if (memory.has(address)) { return memory.get(address); } else { throw new TypeError(`Find a pointer without valid pointed value`); } default: throw new TypeError(`Invalid type found : ${ type }`); } memory.set(pointer, value); return value; }