Dyomedea
/
dyomedea
1
1
Fork 0
Code Issues 4 Pull Requests Projects 3 Releases 1 Wiki Activity

Searching a new balance between the two web workers...

This commit is contained in:
Eric van der Vlist 2023-02-11 15:55:49 +01:00
parent 71bd820da9
commit 0e714ac361
8 changed files with 248 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
src/db-admin/compact.ts → src/db-admin/health.ts
View File

@ -2,24 +2,71 @@ import { Signal } from 'solid-js';
import PouchDB from 'pouchdb';
import indexeddb from 'pouchdb-adapter-indexeddb';
import { getSettings } from '../db/settings';
import { delay } from '../lib/delay';
import { getSettings, putSettings } from '../db/settings';
import { sleep } from '../lib/async-wait';
import {
getAccounts,
initialAccount,
putAccount,
getAccountById,
} from '../db/account';
PouchDB.plugin(indexeddb);
export const compactDb = async (p: { signal: Signal<any>; params: any }) => {
const { signal, params } = p;
const [, setStatus] = signal;
console.log({ caller: 'compactDb', p });
declare global {
var localDb: any;
var db: any;
}
const initDb = async () => {
if (globalThis.localDb === undefined) {
globalThis.localDb = new PouchDB('_local_dyomedea_', {
adapter: 'indexeddb',
auto_compaction: true,
});
}
const accounts = await getAccounts();
console.log({ caller: 'initDb', accounts });
if (accounts.length === 0) {
accounts[0] = initialAccount;
await putAccount({ id: initialAccount.id, account: initialAccount });
}
const settings = await getSettings();
let currentAccount = getAccountById(accounts, settings.currentAccountId);
if (currentAccount === undefined) {
settings.currentAccountId = accounts[0].id;
await putSettings({ settings });
currentAccount = accounts[0];
}
console.log({ caller: 'initDb', settings, currentAccount });
if (globalThis.db === undefined) {
globalThis.db = new PouchDB(currentAccount.localDb, {
adapter: 'indexeddb',
auto_compaction: true,
});
}
};
export const watchDb = async (p: { signal: Signal<any>; params: any }) => {
const { signal } = p;
const [, setStatus] = signal;
console.log({ caller: 'watchDb', p });
await initDb();
db = globalThis.db;
localDb = globalThis.localDb;
const db = new PouchDB('_dyomedea_', {
auto_compaction: true,
adapter: 'indexeddb',
});
const dbinfo = await db.info();
const localDbInfo = await localDb.info();
const tasks = PouchDB.activeTasks.list();
setStatus({ status: 'starting', db: '_dyomedea_', dbinfo, tasks });
setStatus({ opened: true, dbinfo, localDbinfo: localDbInfo, tasks });
return;
const timerId = setInterval(async () => {
const dbinfo = await db.info();
@ -39,7 +86,7 @@ export const compactDb = async (p: { signal: Signal<any>; params: any }) => {
const iDb = indexedDB.open(name);
iDb.onerror = (event: any) => {
console.error({
caller: 'compactDb',
caller: 'watchDb',
message: 'open db error',
target: event.target,
});
@ -48,7 +95,7 @@ export const compactDb = async (p: { signal: Signal<any>; params: any }) => {
iDb.onsuccess = (event: any) => {
console.log({
caller: 'compactDb',
caller: 'watchDb',
message: 'open db',
target: event.target,
});
@ -61,7 +108,7 @@ export const compactDb = async (p: { signal: Signal<any>; params: any }) => {
const request = store.getAll();
request.onerror = (event: any) => {
console.error({
caller: 'compactDb',
caller: 'watchDb',
message: 'getAll error',
target: event.target,
});
@ -70,7 +117,7 @@ export const compactDb = async (p: { signal: Signal<any>; params: any }) => {
request.onsuccess = (event: any) => {
console.log({
caller: 'compactDb',
caller: 'watchDb',
message: 'getAll',
target: event.target,
});
@ -83,7 +130,7 @@ export const compactDb = async (p: { signal: Signal<any>; params: any }) => {
const bySequence = iDb.transaction('docs', 'readonly');
const store = bySequence.objectStore('docs');
console.log({
caller: 'compactDb',
caller: 'watchDb',
message: 'transaction opened',
bySequence,
store,
@ -101,14 +148,14 @@ export const compactDb = async (p: { signal: Signal<any>; params: any }) => {
if (deleted !== 0) {
// const purge = await db.purge(id, rev);
console.log({
caller: 'compactDb',
caller: 'watchDb',
message: 'purging',
id,
rev,
doc,
// purge,
});
await delay(1000);
await sleep(1000);
}
});
};

256
src/db/index.ts
View File

@ -28,147 +28,147 @@ declare global {
var dbReady: boolean;
}
export const initDb = async (params: any) => {
export const initDb = async () => {
console.log({ caller: 'initDb' });
if (globalThis.localDb === undefined) {
globalThis.dbReady = false;
globalThis.localDb = new PouchDB('_local_dyomedea_', {
adapter: 'indexeddb',
auto_compaction: true,
revs_limit: 10,
});
}
const accounts = await getAccounts();
console.log({ caller: 'initDb', accounts });
const accounts = await getAccounts();
console.log({ caller: 'initDb', accounts });
if (accounts.length === 0) {
accounts[0] = initialAccount;
await putAccount({ id: initialAccount.id, account: initialAccount });
}
const settings = await getSettings();
let currentAccount = getAccountById(accounts, settings.currentAccountId);
if (currentAccount === undefined) {
settings.currentAccountId = accounts[0].id;
await putSettings({ settings });
currentAccount = accounts[0];
}
console.log({ caller: 'initDb', settings, currentAccount });
if (globalThis.db === undefined) {
globalThis.db = new PouchDB(currentAccount.localDb, {
adapter: 'indexeddb',
auto_compaction: true,
});
}
const db = globalThis.db;
const dbinfo = await db.info();
// db.compact();
console.log({
caller: 'initDb',
dbinfo,
activeTasks: PouchDB.activeTasks.list(),
storage: await navigator.storage.estimate(),
});
//
var previousDbDefinition = {
_id: dbDefinitionId,
type: dbDefinitionId,
def: { version: '0' },
};
try {
previousDbDefinition = await db.get(dbDefinitionId);
} catch (error: any) {
if (error.status !== 404) {
console.log(
`Unexpected error fetching db definition: ${JSON.stringify(error)}`
);
return;
if (accounts.length === 0) {
accounts[0] = initialAccount;
await putAccount({ id: initialAccount.id, account: initialAccount });
}
}
if (previousDbDefinition.def.version < currentDbDefinition.def.version) {
previousDbDefinition.def = currentDbDefinition.def;
db.put(previousDbDefinition);
// TODO: support migrations
}
//await await db.compact();
const settings = await getSettings();
let currentAccount = getAccountById(accounts, settings.currentAccountId);
if (currentAccount === undefined) {
settings.currentAccountId = accounts[0].id;
await putSettings({ settings });
currentAccount = accounts[0];
}
if (currentAccount.remoteDbServer) {
const url = `${currentAccount.remoteDbServer}/userdb-${toHex(
currentAccount.remoteDbUser
)}`;
console.log({ caller: 'initDb', settings, currentAccount });
const remoteDb = new PouchDB(url, {
auth: {
username: currentAccount.remoteDbUser,
password: currentAccount.remoteDbPassword,
},
skip_setup: true,
});
const sync = PouchDB.sync(db, remoteDb, {
live: true,
retry: true,
})
.on('change', function (info) {
// handle change
console.log({ caller: 'Sync / change', info });
})
.on('paused', function (err) {
// replication paused (e.g. replication up to date, user went offline)
console.log({ caller: 'Sync / paused', err });
})
.on('active', function () {
// replicate resumed (e.g. new changes replicating, user went back online)
console.log({ caller: 'Sync / active' });
})
.on('denied', function (err) {
// a document failed to replicate (e.g. due to permissions)
console.error({ caller: 'Sync / denied', err });
})
.on('complete', function (info) {
// handle complete
console.log({ caller: 'Sync / complete', info });
})
.on('error', function (err) {
// handle error
console.error({ caller: 'Sync / error', err });
if (globalThis.db === undefined) {
globalThis.db = new PouchDB(currentAccount.localDb, {
adapter: 'indexeddb',
auto_compaction: true,
});
}
const db = globalThis.db;
const dbinfo = await db.info();
// db.compact();
console.log({
caller: 'initDb',
dbinfo,
activeTasks: PouchDB.activeTasks.list(),
storage: await navigator.storage.estimate(),
});
//
var previousDbDefinition = {
_id: dbDefinitionId,
type: dbDefinitionId,
def: { version: '0' },
};
try {
previousDbDefinition = await db.get(dbDefinitionId);
} catch (error: any) {
if (error.status !== 404) {
console.log(
`Unexpected error fetching db definition: ${JSON.stringify(error)}`
);
return;
}
}
if (previousDbDefinition.def.version < currentDbDefinition.def.version) {
previousDbDefinition.def = currentDbDefinition.def;
db.put(previousDbDefinition);
// TODO: support migrations
}
//await await db.compact();
if (currentAccount.remoteDbServer) {
const url = `${currentAccount.remoteDbServer}/userdb-${toHex(
currentAccount.remoteDbUser
)}`;
const remoteDb = new PouchDB(url, {
auth: {
username: currentAccount.remoteDbUser,
password: currentAccount.remoteDbPassword,
},
skip_setup: true,
});
const sync = PouchDB.sync(db, remoteDb, {
live: true,
retry: true,
})
.on('change', function (info) {
// handle change
console.log({ caller: 'Sync / change', info });
})
.on('paused', function (err) {
// replication paused (e.g. replication up to date, user went offline)
console.log({ caller: 'Sync / paused', err });
})
.on('active', function () {
// replicate resumed (e.g. new changes replicating, user went back online)
console.log({ caller: 'Sync / active' });
})
.on('denied', function (err) {
// a document failed to replicate (e.g. due to permissions)
console.error({ caller: 'Sync / denied', err });
})
.on('complete', function (info) {
// handle complete
console.log({ caller: 'Sync / complete', info });
})
.on('error', function (err) {
// handle error
console.error({ caller: 'Sync / error', err });
});
}
// console.log({ caller: 'initDb / before db.changes' });
const changes = db
.changes({ since: 'now', live: true, include_docs: false })
.on('change', changeHandler)
.on('complete', (info: any) => {
console.log({ caller: 'changes / complete', info });
})
.on('error', (error: any) => {
console.log({ caller: 'changes / complete', error });
});
const localDbChanges = localDb
.changes({ since: 'now', live: true, include_docs: false })
.on('change', changeHandler)
.on('complete', (info: any) => {
console.log({ caller: 'localDb changes / complete', info });
})
.on('error', (error: any) => {
console.log({ caller: 'localDb changes / complete', error });
});
// console.log({ caller: 'initDb / back from db.changes', changes });
// changes.cancel();
console.log({
caller: 'initDb (end)',
dbinfo,
activeTasks: PouchDB.activeTasks.list(),
storage: await navigator.storage.estimate(),
});
globalThis.dbReady = true;
}
// console.log({ caller: 'initDb / before db.changes' });
const changes = db
.changes({ since: 'now', live: true, include_docs: false })
.on('change', changeHandler)
.on('complete', (info: any) => {
console.log({ caller: 'changes / complete', info });
})
.on('error', (error: any) => {
console.log({ caller: 'changes / complete', error });
});
const localDbChanges = localDb
.changes({ since: 'now', live: true, include_docs: false })
.on('change', changeHandler)
.on('complete', (info: any) => {
console.log({ caller: 'localDb changes / complete', info });
})
.on('error', (error: any) => {
console.log({ caller: 'localDb changes / complete', error });
});
// console.log({ caller: 'initDb / back from db.changes', changes });
// changes.cancel();
console.log({
caller: 'initDb (end)',
dbinfo,
activeTasks: PouchDB.activeTasks.list(),
storage: await navigator.storage.estimate(),
});
globalThis.dbReady = true;
};

13
src/index.tsx
View File

@ -58,18 +58,11 @@ const getLanguage = () => {
const i18nDict = createI18nContext(dict, getLanguage());
const settings: any = await dispatch({
action: 'getSettings',
});
const accounts: any = await dispatch({
action: 'getAccounts',
});
const [compactDbStatus] = createWorkerSignal({
provider: 'compactDb',
params: { settings, accounts },
const [watchDbStatus] = createWorkerSignal({
provider: 'watchDb',
});
createEffect(() => {
console.log({ caller: 'createEffect', compactDbStatus: compactDbStatus() });
console.log({ caller: 'createEffect', watchDbStatus: watchDbStatus() });
});
render(

2
src/lib/delay.ts
View File

@ -1,2 +0,0 @@
export const delay = (ms: number) =>
new Promise((resolve) => setTimeout(resolve, ms));

8
src/solid-workers/solid-worker-main.ts
View File

@ -49,11 +49,19 @@ export const createWorkerSignal = (parameters: {
const id = JSON.stringify({ provider, params });
if (globalThis.exportQueue.has(id)) {
globalThis.exportQueue.get(id).count++;
console.log({
caller: 'solid-worker / createWorkerSignal / existing',
queueItem: globalThis.exportQueue.get(id),
});
return globalThis.exportQueue.get(id).signal;
}
const signal = createSignal(initialValue);
globalThis.exportQueue.set(id, { signal, count: 1 });
worker.postMessage({ ...parameters, _id: id });
console.log({
caller: 'solid-worker / createWorkerSignal / new',
queueItem: globalThis.exportQueue.get(id),
});
return signal;
};

4
src/solid-workers/solid-worker.ts
View File

@ -1,5 +1,5 @@
import { createEffect, createRoot, createSignal, Signal } from 'solid-js';
import { compactDb } from '../db-admin/compact';
import { watchDb } from '../db-admin/health';
declare global {
var importQueue: Map<string, any>;
@ -8,7 +8,7 @@ declare global {
globalThis.importQueue = new Map<string, Signal<any>>();
const providers = {
compactDb,
watchDb,
};
onmessage = function (e) {

62
src/workers/dispatcher-main.ts
View File

@ -1,3 +1,6 @@
import { createEffect } from 'solid-js';
import { until } from '../lib/async-wait';
import { createWorkerSignal } from '../solid-workers/solid-worker-main';
import { getWorker } from './get-worker';
declare global {
@ -27,42 +30,55 @@ export const init = () => {
cancelDispatch(id);
}
};
};
dispatch({ action: 'initDb' });
const [watchDbStatus] = createWorkerSignal({
provider: 'watchDb',
});
const dbReady = () => {
console.log({
caller: 'dispatcher-main / waiting',
dbStatus: watchDbStatus(),
dbReady: watchDbStatus()?.opened,
});
return watchDbStatus()?.opened;
};
const dispatch = (
payload: any,
callBack?: (error: any, result: any, id?: number|undefined) => void,
callBack?: (error: any, result: any, id?: number | undefined) => void,
live?: boolean
) => {
console.log({ caller: 'dispatcher-main / dispatch', payload });
if (worker === undefined) {
init();
}
if (callBack === undefined) {
/** If a callback function is not provided, return a promise */
return new Promise((resolve, reject) => {
dispatch(payload, (error, result) => {
if (error) {
reject(error);
} else {
resolve(result);
}
until(dbReady, 1000).then(() => {
// Wait until databases have been created by health.ts
if (callBack === undefined) {
/** If a callback function is not provided, return a promise */
return new Promise((resolve, reject) => {
dispatch(payload, (error, result) => {
if (error) {
reject(error);
} else {
resolve(result);
}
});
});
}
/** Otherwise, use the callback function */
dispatcherQueue.queue.set(dispatcherQueue.index, { callBack, live });
const message = {
id: dispatcherQueue.index++,
payload: payload,
};
worker.postMessage(message);
console.log({
caller: 'dispatcher-main / message sent',
message,
dispatcherQueue,
});
}
/** Otherwise, use the callback function */
dispatcherQueue.queue.set(dispatcherQueue.index, { callBack, live });
const message = {
id: dispatcherQueue.index++,
payload: payload,
};
worker.postMessage(message);
console.log({
caller: 'dispatcher-main / message sent',
message,
dispatcherQueue,
});
};

22
src/workers/dispatcher-worker.ts
View File

@ -27,12 +27,6 @@ import { until } from '../lib/async-wait';
console.log({ caller: 'dispatcher-worker' });
declare global {
var dbReady: boolean;
}
globalThis.dbReady = false;
onmessage = async function (e) {
const actions = {
initDb,
@ -74,6 +68,9 @@ onmessage = async function (e) {
};
console.log({ caller: 'dispatcher-worker / onmessage', e });
initDb();
await until(() => globalThis.dbReady, 100); // Wait until databases have been initialized in this worker
const { id, payload } = e.data;
var returnValue: any = 'unknownAction';
if (payload.action in actions) {
@ -83,19 +80,6 @@ onmessage = async function (e) {
payload,
dbReady: globalThis.dbReady,
});
if (payload.action !== 'initDb' && !globalThis.dbReady) {
console.log({
caller: 'dispatcher-worker / waiting for dbReady',
id,
dbReady: globalThis.dbReady,
});
await until(() => globalThis.dbReady, 10);
console.log({
caller: 'dispatcher-worker / dbReady',
id,
dbReady: globalThis.dbReady,
});
}
returnValue = await actions[<keyof typeof actions>payload.action]({
...payload.params,
_dispatchId: id,