javascriptsqlitegoogle-chrome-extensionweb-workerorigin-private-file-system

Using OPFS (Origin Private FileSystem) with SQLite WASM in background.js Chrome Extention (Worker is not defined error)

I'm trying to get persistent storage working with WASM SQLite.

My code in the background.js file is this

const worker = new Worker('./worker.js');
worker.postMessage({ loadSQLite: true });

But it never gets past the Worker part. I get the following error.

enter image description here

Uncaught ReferenceError: Worker is not defined

So after doing some research I tried some possible solutions. Some people were saying that you can't spawn workers within workers, and there was a subworkers solution, but that did not work for me (if it works for you please let me know, but I could not get that to work).

The code in worker.js I assume doesn't matter, but I'll post what I'm trying to do in there.

worker.js

import sqlite3InitModule from './lib/@sqlite.org/sqlite-wasm/index.mjs';
const log = (...args) => console.log(...args);
const error = (...args) => console.error(...args);
console.log("worker loaded...");

//@JA - SQLite Database Setup
const start = function (sqlite3) {
    console.log(sqlite3);
    const capi = sqlite3.capi; // C-style API
    const oo = sqlite3.oo1; // High-level OO API
    console.log(`SQLite3 Version ${capi.sqlite3_libversion()} - ${capi.sqlite3_sourceid()}`);
    let db;
    if ('OpfsDb' in oo) {
        db = new oo.OpfsDb('/mydb.sqlite3');
        console.log('The OPFS is available.');
        console.log(`Persisted db = ${db.filename}`);
    } else {
        db = new oo.DB('/mydb.sqlite3', 'ct');
        console.log('The OPFS is not available.');
        console.log(`transient db = ${db.filename}`);
    }

    try{
        db.exec(`CREATE TABLE IF NOT EXISTS results (id INTEGER PRIMARY KEY AUTOINCREMENT, search_name TEXT, strategy_name TEXT, json_data TEXT)`);
    } finally {
        db.close();
    }
}

self.addEventListener('message', (event) => {
    if (event.data.loadSQLite) {
        // Load the SQLite library within the web worker
        sqlite3InitModule({
            print: log,
            printErr: error,
        }).then(function (sqlite3) {
            log('SQLite Done initializing.');
            try {
                start(sqlite3);
            } catch (e) {
                error('Exception:', e.message);
            }
        });
    }
});

This code works if I take it outside of worker.js BUT then it's transient, meaning the database will NOT SAVE!

My goal is to make it use OPFS for persistent storage, but this proving to be quite difficult and I've hit a road block. Any help is appreciated!

My goal is to have it say The OPFS is available.

From this resource: https://sqlite.org/wasm/doc/trunk/persistence.md

It says that it must be in a service worker to work. I assumed that background.js was a service worker but apparently it's not or this would have worked there. So that is why I'm trying to create a service worker in there.

My manifest file is here:

{
    "name" : "__MSG_appName__",
    "short_name": "__MSG_appShortName__",
    "description": "__MSG_appDescription__",
    "version" : "2.0.2",
    "version_name": "2.0.2",
    "manifest_version": 3,
    "default_locale": "en",
    "minimum_chrome_version": "88",
    "permissions": [
        "activeTab",
        "tabs",
        "storage",
        "scripting",
        "unlimitedStorage",
        "declarativeNetRequest",
        "declarativeNetRequestFeedback"
    ],
    "icons": {
        "16": "./assets/img/icon-16.png",
        "32": "./assets/img/icon-32.png",
        "48": "./assets/img/icon-48.png",
        "128": "./assets/img/icon-128.png"
    },
    "background":{
        "service_worker": "background.js",
        "type": "module"
    },
    "action":{
        "default_popup": "./popup.html",
        "default_icons": {
            "16": "./assets/img/icon-16.png",
            "32": "./assets/img/icon-32.png",
            "48": "./assets/img/icon-48.png",
            "128": "./assets/img/icon-128.png" 
        },
        "default_title": "__MSG_appName__"
    },
    "host_permissions": [
        "https://*.tradingview.com/*"
    ],
    "content_scripts": [
        {
            "all_frames": true,
            "matches": [
                "https://*.tradingview.com/*",
                "http://*.tradingtools.software/*"
            ],
            "css": [
                "./css/trading-view.css",
                "./lib/datetimepicker/jquery.datetimepicker.min.css"
            ],
            "js": [
                "./lib/jquery/jquery-3.6.0.min.js",
                "./lib/lodash/lodash.js",
                "./lib/luxon/non-es6-luxon.js",
                "./lib/datetimepicker/jquery.datetimepicker.full.min.js",
                "./js/classes/scanner.js",
                "./js/classes/dialogs.js",
                "./js/classes/queues.js",
                "./js/helpers/helpers.js",
                "./js/helpers/inputs.js",
                "./js/helpers/security.js",
                "./js/helpers/manipulators.js",
                "./js/helpers/strategy.js",
                "./js/helpers/license.js",
                "./js/globals/globals.js",
                "./js/content/tradingview-content-script.js"  
            ],
            "run_at": "document_idle"
        },
        {
            "all_frames": true,
            "matches":["https://*.tradingview.com/*","http://*.tradingtools.software/*"],
            "js":[ "./js/injectors/before.js" ],
            "run_at": "document_start"
        },
        {
            "all_frames": true,
            "matches":["https://*.tradingview.com/*","http://*.tradingtools.software/*"],
            "js":[ "./js/injectors/after.js" ],
            "run_at": "document_end"
        }
    ],
    "web_accessible_resources": [{
        "resources": ["/js/injectors/before_inject.js","/js/injectors/after_inject.js","/js/injectors/page-script.js","/assets/*","workers.js"],
        "matches": ["<all_urls>"]
    }],
    "content_security_policy": {
        "extension_pages": "script-src 'self' 'wasm-unsafe-eval'; object-src 'self'; worker-src 'self'",
        "sandbox": "sandbox allow-scripts; default-src 'self' 'wasm-unsafe-eval'; frame-src 'self'; worker-src 'self'"
      }
}
Solution

  • You can't start a worker from the extension service worker, but you can create one inside the offscreen document

    As for running SQLite using OPFS inside a chrome extension, you can do this with the offscreen document. Here is an example.

    You can also clone this repo and run the example - https://github.com/clmnin/sqlite-opfs-mv3

    manifest.json

    {
  "manifest_version": 3,
  "name": "MV3 Sqlite OPFS Example",
  "version": "1",
  "description": "MV3 Sqlite OPFS Example",
  "background": {
    "service_worker": "index.js",
    "type": "module"
  },
  "content_security_policy": {
    "extension_pages": "script-src 'self' 'wasm-unsafe-eval'"
  },
  "permissions": [
    "offscreen"
  ]
}

    Background Service Worker

    index.ts

    console.log('bg ready');

let creating: Promise<void> | null;
async function setupOffscreenDocument(path: string) {
  // Check all windows controlled by the service worker to see if one
  // of them is the offscreen document with the given path
  if (await chrome.offscreen.hasDocument?.()) return;

  // create offscreen document
  if (creating) {
    await creating;
  } else {
    creating = chrome.offscreen.createDocument({
      url: chrome.runtime.getURL(path),
      reasons: [
        chrome.offscreen.Reason.WORKERS || chrome.offscreen.Reason.BLOBS,
      ],
      justification: "To run web worker to run sqlite",
    });
    await creating;
    creating = null;
  }
}

setupOffscreenDocument("index.html");

    Offscreen Document

    index.html

    <!DOCTYPE html>
<script type="module" src="./index.ts"></script>

    index.ts

    const worker = new Worker(
  new URL("./worker.js", import.meta.url),
  { type: "module" },
);
worker.onmessage = function ({ data }) {
  switch (data.type) {
    case 'log':
      console.log(`Message: ${data.payload.args}`);
      break;
    default:
      console.log(`ERROR: Unhandled message: ${data.type}`);
  }
};

    worker.js

    import sqlite3InitModule from '@sqlite.org/sqlite-wasm';

const logHtml = function (cssClass, ...args) {
  postMessage({
    type: 'log',
    payload: { cssClass, args },
  });
};

const log = (...args) => logHtml('', ...args);
const error = (...args) => logHtml('error', ...args);

const start = function (sqlite3) {
  const capi = sqlite3.capi; // C-style API
  const oo = sqlite3.oo1; // High-level OO API
  log('SQLite3 version', capi.sqlite3_libversion(), capi.sqlite3_sourceid());
  let db;
  if ('OpfsDb' in oo) {
    db = new oo.OpfsDb('/mydb.sqlite3');
    log('The OPFS is available.');
    log('Persisted db =', db.filename);
  } else {
    db = new oo.DB('/mydb.sqlite3', 'ct');
    log('The OPFS is not available.');
    log('transient db =', db.filename);
  }

  try {
    log('Create a table...');
    db.exec('CREATE TABLE IF NOT EXISTS t(a,b)');
    log('Insert some data using exec()...');
    let i;
    for (i = 20; i <= 25; ++i) {
      db.exec({
        sql: 'INSERT INTO t(a,b) VALUES (?,?)',
        bind: [i, i * 2],
      });
    }
    log("Query data with exec() using rowMode 'array'...");
    db.exec({
      sql: 'SELECT a FROM t ORDER BY a LIMIT 3',
      rowMode: 'array', // 'array' (default), 'object', or 'stmt'
      callback: function (row) {
        log('row ', ++this.counter, '=', row);
      }.bind({ counter: 0 }),
    });
  } finally {
    db.close();
  }
};

log('Loading and initializing sqlite3 module...');
sqlite3InitModule({
  print: log,
  printErr: error,
}).then(function (sqlite3) {
  log('Done initializing. Running demo...');
  try {
    start(sqlite3);
  } catch (e) {
    error('Exception:', e.message);
  }
});