From c40bdcbbed42bb67518e67c2cab60489fb3762e1 Mon Sep 17 00:00:00 2001 From: "HRG @ SCExC" Date: Sat, 8 Jun 2024 18:29:31 -0400 Subject: [PATCH] websocket support and route fixes --- index.js | 239 +++++++++++++++++++---------- migrations/000_bootstrap_db.sql | 3 +- package-lock.json | 51 ++++++ package.json | 1 + views/workshop/new_route_modal.eta | 6 +- views/workshop/sidebar.eta | 2 +- 6 files changed, 218 insertions(+), 84 deletions(-) diff --git a/index.js b/index.js index 8ff1328..f8127a4 100644 --- a/index.js +++ b/index.js @@ -8,9 +8,11 @@ const SQLiteStore = require("better-sqlite3-session-store")(session); const betterSqlite3 = require("better-sqlite3"); const { Eta } = require("eta"); const { match } = require("path-to-regexp"); +const { LRUCache } = require("lru-cache"); const bcrypt = require("bcrypt"); const cheerio = require("cheerio"); const app = express(); +const _expressWs = require("express-ws")(app); const bodyParser = require("body-parser"); const PORT = 3000; @@ -27,6 +29,39 @@ const db = betterSqlite3("./dbs/0.sqlite"); db.pragma("journal_mode = WAL"); +const dbCache = new LRUCache({ max: 25 }); + +function getDbInstance(dbId) { + let dbInstance = dbCache.get(dbId); + + if (!dbInstance) { + dbInstance = betterSqlite3(`dbs/${dbId}.sqlite`); + dbInstance.pragma("journal_mode = WAL"); + dbCache.set(dbId, dbInstance); + } + + return dbInstance; +} + +const templateCache = new LRUCache({ max: 100 }); + +function getTemplater(structId) { + let etaInstance = templateCache.get(structId); + + if (!etaInstance) { + etaInstance = new Eta({}); + etaInstance.resolvePath = function (path, _) { + return path; + }; + etaInstance.readFile = function (templateAlias) { + return getTemplateContentByName(db, structId, templateAlias).content; + }; + templateCache.set(structId, etaInstance); + } + + return etaInstance; +} + function getAllRoutes(db) { return db .prepare( @@ -78,6 +113,61 @@ function applyMigrations() { }); } +function bootstrapContext(db, structureId, initContext) { + const allDbInstances = {}; + function getDb(alias) { + return allDbInstances[alias]; + } + + // todo wrap template in data-bliss-edit-template thing since ws can't take us to the editor on a component basis?? maybe... + // for now just designing with component approach + const eta = getTemplater(structureId); + + const libs = { eta, db: getDb }; + + let dbs = getDbsForStructure(db, structureId); + let context = vm.createContext({ + ...initContext, + require: function (str) { + return libs[str]; + }, + module: { exports: null }, + }); + for (let appDb of dbs) { + let dbInstance = getDbInstance(appDb.id); + context.sql = dbInstance; + vm.runInContext(appDb.library, context); + allDbInstances[appDb.alias] = { + library: context.module.exports, + sql: dbInstance, + }; + context.module.exports = null; + } + + return context; +} + +function routeWithPrefix(route) { + const prefix = route.route_prefix; + return prefix ? path.join(prefix, route.path) : route.path; +} + +function bootstrapWebsocketHandler(route) { + // todo only expose app when running the handler, should not be available to the handler itself + // need to move to runscript or whatever + try { + const context = bootstrapContext(db, route.structure_id, { app }); + let result = vm.runInContext( + route.handler + `\n\napp.ws("${routeWithPrefix(route)}", handler)`, + context + ); + updateRoute(db, { ...route, error: null }); + return result; + } catch (e) { + updateRoute(db, { ...route, error: e.stack }); + } +} + function buildRoutes() { let newRoutes = { GET: [], @@ -87,13 +177,16 @@ function buildRoutes() { }; for (let route of getAllRoutes(db)) { - const prefix = route.route_prefix; - const p = prefix ? path.join(route.route_prefix, route.path) : route.path; - newRoutes[route.verb].push({ - matcher: match(p, { decode: decodeURIComponent }), - id: route.id, - path: route.path, - }); + const p = routeWithPrefix(route); + if (route.verb == "WS") { + bootstrapWebsocketHandler(route); + } else { + newRoutes[route.verb].push({ + matcher: match(p, { decode: decodeURIComponent }), + id: route.id, + path: route.path, + }); + } } routes = newRoutes; } @@ -192,6 +285,7 @@ function createStructure(db, name, userId) { } function createRoute(db, verb, path, structureId, handler) { + path = encodeURI(path) // add default handler here const stmt = db.prepare( "INSERT INTO routes (verb, path, structure_id, handler) VALUES (?, ?, ?, ?)" @@ -211,7 +305,14 @@ function getRoute(db, routeId) { } function updateRoute(db, route) { - const fields = ["verb", "path", "structure_id", "handler", "updated_at"]; + const fields = [ + "verb", + "path", + "structure_id", + "handler", + "updated_at", + "error", + ]; const values = fields.map((field) => route[field]); const placeholders = fields.map((field) => `${field} = ?`).join(", "); @@ -232,7 +333,6 @@ function updateDb(db, appDb) { function updateStruct(db, struct) { const fields = ["name", "route_prefix", "head_injection"]; - console.log(struct, fields); const values = fields.map((field) => struct[field]); const placeholders = fields.map((field) => `${field} = ?`).join(", "); @@ -358,8 +458,6 @@ function getFilesForStruct(db, structureId) { .prepare("SELECT * FROM files WHERE structure_id = ? ORDER BY id DESC") .all(structureId); - console.log(db.prepare("PRAGMA table_info(files)").all()); - return test; } @@ -375,39 +473,6 @@ function createFile(db, structure_id, name, filePath, mime_type, mime_subtype) { .run(structure_id, name, filePath, mime_type, mime_subtype).lastInsertRowid; } -const { LRUCache } = require("lru-cache"); -const templateCache = new LRUCache({ max: 100 }); - -function getTemplater(structId) { - let etaInstance = templateCache.get(structId); - - if (!etaInstance) { - etaInstance = new Eta({}); - etaInstance.resolvePath = function (path, _) { - return path; - }; - etaInstance.readFile = function (templateAlias) { - return getTemplateContentByName(db, structId, templateAlias).content; - }; - templateCache.set(structId, etaInstance); - } - - return etaInstance; -} - -const dbCache = new LRUCache({ max: 25 }); - -function getDbInstance(dbId) { - let dbInstance = dbCache.get(dbId); - - if (!dbInstance) { - dbInstance = betterSqlite3(`dbs/${dbId}.sqlite`); - dbInstance.pragma("journal_mode = WAL"); - dbCache.set(dbId, dbInstance); - } - - return dbInstance; -} function cloneStructure( structId, @@ -436,8 +501,6 @@ function cloneStructure( const toClone = new Set(cloneDbs); const toAlias = new Set(); - console.log(toClone, dbIds); - for (let { db_id } of dbIds) { if (!toClone.has(db_id.toString())) { toAlias.add(db_id.toString()); @@ -504,10 +567,13 @@ function bootstrapTemplateWithHTMXetc( blissClone, blissCopy, headInjection, - htmxRequest=false + htmxRequest = false ) { - if (htmlString.toLowerCase().startsWith("") || htmlString.toLowerCase().startsWith("") || + htmlString.toLowerCase().startsWith(" + ${headInjection || ""} `); @@ -533,14 +600,13 @@ function bootstrapTemplateWithHTMXetc( } htmlString = $.html(); - console.log(htmlString) } else if (htmxRequest && blissRoute) { const $ = cheerio.load(htmlString, null, false); $.root().children().attr("data-bliss-route", blissRoute); $.root().children().attr("data-bliss-clone", blissClone); if (blissCopy) $.root().children().attr("data-bliss-copy", blissCopy); - htmlString = $.html() - + htmlString = $.html(); + if (htmxRequest) { htmlString += ``; } @@ -642,14 +708,27 @@ app.get("/workshop/:structure_id/db/:db_id", (req, res) => { }); app.post("/workshop/:structure_id/route", (req, res) => { - let path = req.body.path; + let p = req.body.path; + let dummyHandler = + "// put your handler code here\n\nfunction handler(req, res) {\n res.send('henlo world') \n}"; + + if (req.body.verb == "WS") { + dummyHandler = `// put your websocket handler code here\n\nfunction handler(ws, req) {\n ws.on('message', function(msg) {\n ws.send(msg);\n })\n}`; + } + + if (p[0] == "/") { + p = p.substring(1); + } + if (p[p.length - 1] == "/") { + p = p.substring(0, p.length - 1); + } const route = createRoute( db, req.body.verb, - path[0] == "/" ? path : "/" + path, + p[0] == "/" ? p.substring(1) : "/" + p, req.params.structure_id, - "// put your handler code here\n\nfunction handler(req, res) {\n res.send('henlo world') \n}" + dummyHandler ); // todo optimize by only adding new, don't just rebuild all buildRoutes(); @@ -782,10 +861,18 @@ app.get("/workshop/:structure_id/template/:template_id/preview", (req, res) => { app.get("/workshop/:structure_id/route/:id", (req, res) => { const route = getRoute(db, req.params.id); - const routePrefix = getStructure(db, req.params.structure_id).route_prefix; + let routePrefix = getStructure(db, req.params.structure_id).route_prefix; let previewUrl = routePrefix ? path.join(routePrefix || "", route.path) : route.path; + + console.log(previewUrl, "baba"); + // let previewUrl = prefixUrlWithHost( + // req, + // routePrefix ? path.join(routePrefix.substring(1), route.path) : route.path + // ); + + console.log(previewUrl); return res.send( bootstrapTemplateWithHTMXetc( eta.render("workshop/route", { @@ -873,12 +960,20 @@ app.get("/workshop/:structure_id/settings", (req, res) => { }); app.put("/workshop/:structure_id/settings", (req, res) => { - console.log(req.body) let structId = req.params.structure_id; let struct = getStructure(db, structId); + let routePrefix = req.body.route_prefix; + + if (routePrefix[0] != "/") { + routePrefix = "/" + routePrefix; + } + if (routePrefix[routePrefix.length - 1] == "/") { + routePrefix = routePrefix.substring(0, routePrefix.length - 1); + } + updateStruct(db, { ...struct, - route_prefix: req.body.route_prefix, + route_prefix: routePrefix, head_injection: req.body.head_injection, }); struct = getStructure(db, structId); @@ -919,7 +1014,6 @@ app.get("/workshop/:structure_id/new_db_modal", (req, res) => { app.get("/workshop/:structure_id/clone_modal", (req, res) => { const structure = getStructure(db, req.params.structure_id); const dbs = getDbsForStructure(db, req.params.structure_id); - console.log(dbs); return res.send( bootstrapTemplateWithHTMXetc( @@ -956,35 +1050,18 @@ app.all("*", (req, res) => { .prepare("SELECT * FROM routes WHERE id = ?") .get(routeMatch.id); - let dbs = getDbsForStructure(db, route.structure_id); const structure = getStructure(db, route.structure_id); - const __urlPrefix = structure.route_prefix - console.log(structure) - - const allDbInstances = {}; + const __urlPrefix = structure.route_prefix; try { - let context = vm.createContext({ + // todo: only add req res to contexst when running the handler() + // which means moving to runscript instead of runincontext for that + let context = bootstrapContext(db, route.structure_id, { req, res, eta, - getDb: function (alias) { - return allDbInstances[alias]; - }, - module: { exports: null }, }); - for (let appDb of dbs) { - let dbInstance = getDbInstance(appDb.id); - context.sql = dbInstance; - vm.runInContext(appDb.library, context); - allDbInstances[appDb.alias] = { - library: context.module.exports, - sql: dbInstance, - }; - context.module.exports = null; - } - res.render = (template, context) => { context = context || {}; context.route = function (url) { diff --git a/migrations/000_bootstrap_db.sql b/migrations/000_bootstrap_db.sql index 33bd21a..62f5dc0 100644 --- a/migrations/000_bootstrap_db.sql +++ b/migrations/000_bootstrap_db.sql @@ -46,10 +46,11 @@ CREATE TABLE templates ( CREATE TABLE routes ( id INTEGER PRIMARY KEY AUTOINCREMENT, - verb TEXT CHECK(verb IN ('POST', 'GET', 'PUT', 'DELETE')) NOT NULL, + verb TEXT CHECK(verb IN ('POST', 'GET', 'PUT', 'DELETE', 'WS')) NOT NULL, path TEXT NOT NULL, structure_id INTEGER, handler TEXT NOT NULL, + error TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY(structure_id) REFERENCES structures(id), diff --git a/package-lock.json b/package-lock.json index 2d751cf..beb4ea4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "express": "^4.18.2", "express-fileupload": "^1.5.0", "express-session": "^1.18.0", + "express-ws": "^5.0.2", "jsdom": "^24.1.0", "lru-cache": "^10.2.0", "nunjucks": "^3.2.4", @@ -883,6 +884,40 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==" }, + "node_modules/express-ws": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/express-ws/-/express-ws-5.0.2.tgz", + "integrity": "sha512-0uvmuk61O9HXgLhGl3QhNSEtRsQevtmbL94/eILaliEADZBHZOQUAiHFrGPrgsjikohyrmSG5g+sCfASTt0lkQ==", + "dependencies": { + "ws": "^7.4.6" + }, + "engines": { + "node": ">=4.5.0" + }, + "peerDependencies": { + "express": "^4.0.0 || ^5.0.0-alpha.1" + } + }, + "node_modules/express-ws/node_modules/ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/express/node_modules/body-parser": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", @@ -3210,6 +3245,22 @@ } } }, + "express-ws": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/express-ws/-/express-ws-5.0.2.tgz", + "integrity": "sha512-0uvmuk61O9HXgLhGl3QhNSEtRsQevtmbL94/eILaliEADZBHZOQUAiHFrGPrgsjikohyrmSG5g+sCfASTt0lkQ==", + "requires": { + "ws": "^7.4.6" + }, + "dependencies": { + "ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "requires": {} + } + } + }, "file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", diff --git a/package.json b/package.json index b87ffa7..0b0c31e 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "express": "^4.18.2", "express-fileupload": "^1.5.0", "express-session": "^1.18.0", + "express-ws": "^5.0.2", "jsdom": "^24.1.0", "lru-cache": "^10.2.0", "nunjucks": "^3.2.4", diff --git a/views/workshop/new_route_modal.eta b/views/workshop/new_route_modal.eta index 8df4860..116b313 100644 --- a/views/workshop/new_route_modal.eta +++ b/views/workshop/new_route_modal.eta @@ -10,7 +10,7 @@
-
<%= it.structure.route_prefix %>/
+
<%= it.structure.route_prefix || "" %>/
verb @@ -30,6 +30,10 @@ +
+ + +
diff --git a/views/workshop/sidebar.eta b/views/workshop/sidebar.eta index 6ce763e..75aad23 100644 --- a/views/workshop/sidebar.eta +++ b/views/workshop/sidebar.eta @@ -7,7 +7,7 @@
  • <%= route.verb %> - - <% if (it.structure.route_prefix) { %><%= it.structure.route_prefix %><% } %><%= route.path %> + <% if (it.structure.route_prefix) { %><%= it.structure.route_prefix %><% } %><%= route.path %><% if (route.error) { %>⚠️<% } %>
    • <% }) %>