From 03c70e3902a9618db59a6f635b44cddd3f799aa3 Mon Sep 17 00:00:00 2001 From: Jamie Curnow Date: Wed, 20 May 2026 08:16:10 +1000 Subject: [PATCH] Fixes for upgraded debian trixie - Adds request logging in debug mode for some endpoints - Moves certbot version determination to the startup scripts and removes bash script encapsulation when installing plugins - Revert loose domain validation, which was there for a specific reason addressing CVE's - Fix Cypress suite for cert generation - Adds Cypress test that iterates over the entire certbot plugins list and installs each one, ensuring at the very least that the install works - Fixed some plugins based on this - (!) Still some work to do on this, hostinger is still broken at least - Improved cypress tests for custom certs; they will generate on each run instead of being baked in. The baked ones were due to expire soon --- .gitignore | 2 + backend/certbot/dns-plugins.json | 6 +- backend/lib/certbot.js | 29 +++++---- backend/lib/express/log-request.js | 7 +++ backend/lib/utils.js | 3 - backend/package.json | 2 +- backend/routes/ci.js | 59 +++++++++++++++++++ backend/routes/main.js | 10 ++++ backend/schema/common.json | 6 +- .../schema/components/certificate-object.json | 10 +++- docker/docker-compose.ci.sqlite.yml | 7 ++- .../rootfs/etc/s6-overlay/s6-rc.d/backend/run | 4 +- docker/rootfs/usr/bin/common.sh | 4 ++ test/cypress/e2e/api/CertbotPlugins.cy.js | 28 +++++++++ test/cypress/e2e/api/Certificates.cy.js | 18 ++++-- test/cypress/e2e/api/Streams.cy.js | 24 ++++---- test/cypress/fixtures/.gitignore | 1 + test/cypress/fixtures/.gitkeep | 0 test/cypress/support/commands.mjs | 26 ++++++-- 19 files changed, 197 insertions(+), 49 deletions(-) create mode 100644 backend/lib/express/log-request.js create mode 100644 backend/routes/ci.js create mode 100644 test/cypress/e2e/api/CertbotPlugins.cy.js create mode 100644 test/cypress/fixtures/.gitignore create mode 100644 test/cypress/fixtures/.gitkeep diff --git a/.gitignore b/.gitignore index 5bf37c0d..a83a2110 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ test/node_modules */node_modules docker/dev/dnsrouter-config.json.tmp docker/dev/resolv.conf +.claude + diff --git a/backend/certbot/dns-plugins.json b/backend/certbot/dns-plugins.json index 974c7265..a84cca6b 100644 --- a/backend/certbot/dns-plugins.json +++ b/backend/certbot/dns-plugins.json @@ -341,8 +341,8 @@ "full_plugin_name": "dns-hostinger", "name": "Hostinger.com", "package_name": "certbot-dns-hostinger", - "version": "~=0.1.5" - }, + "version": "~=0.1.3" + }, "hostingnl": { "credentials": "dns_hostingnl_api_key = 0123456789abcdef0123456789abcdef", "dependencies": "", @@ -553,7 +553,7 @@ }, "powerdns": { "credentials": "dns_powerdns_api_url = https://api.mypowerdns.example.org\ndns_powerdns_api_key = AbCbASsd!@34", - "dependencies": "PyYAML==5.3.1", + "dependencies": "PyYAML>=6.0.1", "full_plugin_name": "dns-powerdns", "name": "PowerDNS", "package_name": "certbot-dns-powerdns", diff --git a/backend/lib/certbot.js b/backend/lib/certbot.js index 3a2dd072..3cafe50f 100644 --- a/backend/lib/certbot.js +++ b/backend/lib/certbot.js @@ -4,8 +4,6 @@ import { certbot as logger } from "../logger.js"; import errs from "./error.js"; import utils from "./utils.js"; -const CERTBOT_VERSION_REPLACEMENT = "$(certbot --version | grep -Eo '[0-9](\\.[0-9]+)+')"; - /** * Installs a cerbot plugin given the key for the object from * ../certbot/dns-plugins.json @@ -15,24 +13,31 @@ const CERTBOT_VERSION_REPLACEMENT = "$(certbot --version | grep -Eo '[0-9](\\.[0 */ const installPlugin = async (pluginKey) => { if (typeof dnsPlugins[pluginKey] === "undefined") { - // throw Error(`Certbot plugin ${pluginKey} not found`); throw new errs.ItemNotFoundError(pluginKey); } const plugin = dnsPlugins[pluginKey]; logger.start(`Installing ${pluginKey}...`); - plugin.version = plugin.version.replace(/{{certbot-version}}/g, CERTBOT_VERSION_REPLACEMENT); - plugin.dependencies = plugin.dependencies.replace(/{{certbot-version}}/g, CERTBOT_VERSION_REPLACEMENT); + plugin.version = plugin.version.replace(/{{certbot-version}}/g, process.env.CERTBOT_VERSION); + plugin.dependencies = plugin.dependencies.replace(/{{certbot-version}}/g, process.env.CERTBOT_VERSION); - // SETUPTOOLS_USE_DISTUTILS is required for certbot plugins to install correctly - // in new versions of Python - let env = Object.assign({}, process.env, { SETUPTOOLS_USE_DISTUTILS: "stdlib" }); + // SETUPTOOLS_USE_DISTUTILS=local uses setuptools' own bundled distutils. + // "stdlib" breaks Python 3.13+ where distutils was removed from the standard library. + let env = Object.assign({}, process.env, { SETUPTOOLS_USE_DISTUTILS: "local" }); if (typeof plugin.env === "object") { env = Object.assign(env, plugin.env); } - const cmd = `. /opt/certbot/bin/activate && pip install --no-cache-dir ${plugin.dependencies} ${plugin.package_name}${plugin.version} && deactivate`; + const quotedDeps = plugin.dependencies.trim() + ? plugin.dependencies + .trim() + .split(/\s+/) + .filter(Boolean) + .map((d) => `'${d}'`) + .join(" ") + : ""; + const cmd = `. /opt/certbot/bin/activate && pip install --no-cache-dir ${quotedDeps} '${plugin.package_name}${plugin.version}' && deactivate`; return utils .exec(cmd, { env }) .then((result) => { @@ -73,9 +78,7 @@ const installPlugins = async (pluginKeys) => { }) .end(() => { if (hasErrors) { - reject( - new errs.CommandError("Some plugins failed to install. Please check the logs above", 1), - ); + reject(new errs.CommandError("Some plugins failed to install. Please check the logs above", 1)); } else { resolve(); } @@ -83,4 +86,4 @@ const installPlugins = async (pluginKeys) => { }); }; -export { installPlugins, installPlugin }; +export { installPlugin, installPlugins }; diff --git a/backend/lib/express/log-request.js b/backend/lib/express/log-request.js new file mode 100644 index 00000000..a70ecc71 --- /dev/null +++ b/backend/lib/express/log-request.js @@ -0,0 +1,7 @@ +import chalk from "chalk"; +import { debug, express as logger } from "../../logger.js"; + +export default (req, _res, next) => { + debug(logger, `[${chalk.yellow(req.method.toUpperCase())}] ${chalk.green(req.path)}`); + next(); +}; diff --git a/backend/lib/utils.js b/backend/lib/utils.js index af7ad3c9..a19142af 100644 --- a/backend/lib/utils.js +++ b/backend/lib/utils.js @@ -3,14 +3,12 @@ import { dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { Liquid } from "liquidjs"; import _ from "lodash"; -import { debug, global as logger } from "../logger.js"; import errs from "./error.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const exec = async (cmd, options = {}) => { - debug(logger, "CMD:", cmd); const { stdout, stderr } = await new Promise((resolve, reject) => { const child = nodeExec(cmd, options, (isError, stdout, stderr) => { if (isError) { @@ -34,7 +32,6 @@ const exec = async (cmd, options = {}) => { * @returns {Promise} */ const execFile = (cmd, args, options) => { - debug(logger, `CMD: ${cmd} ${args ? args.join(" ") : ""}`); const opts = options || {}; return new Promise((resolve, reject) => { diff --git a/backend/package.json b/backend/package.json index 34c39217..4cafa403 100644 --- a/backend/package.json +++ b/backend/package.json @@ -20,6 +20,7 @@ "bcrypt": "^6.0.0", "better-sqlite3": "^12.10.0", "body-parser": "^2.2.2", + "chalk": "5.6.2", "compression": "^1.8.1", "express": "^5.2.1", "express-fileupload": "^1.5.2", @@ -43,7 +44,6 @@ "devDependencies": { "@apidevtools/swagger-parser": "^12.1.0", "@biomejs/biome": "^2.4.15", - "chalk": "5.6.2", "nodemon": "^3.1.14" }, "signale": { diff --git a/backend/routes/ci.js b/backend/routes/ci.js new file mode 100644 index 00000000..a6f8be03 --- /dev/null +++ b/backend/routes/ci.js @@ -0,0 +1,59 @@ +import express from "express"; +import dnsPlugins from "../certbot/dns-plugins.json" with { type: "json" }; +import { installPlugin } from "../lib/certbot.js"; +import { debug, express as logger } from "../logger.js"; + +const router = express.Router({ + caseSensitive: true, + strict: true, + mergeParams: true, +}); + +/** + * ONLY AVAILABLE IN CI ENVIRONMENT! + */ + +/** + * /api/ci/certbot-plugins + */ +router + .route("/certbot-plugins") + .options((_, res) => { + res.sendStatus(204); + }) + + // Return all certbot plugins + .get(async (_req, res, _next) => { + res.status(200).send(dnsPlugins); + }); + +/** + * /api/ci/certbot-plugins/{plugin} + */ +router + .route("/certbot-plugins/:plugin") + .options((_, res) => { + res.sendStatus(204); + }) + + // Install a certbot plugin + .post(async (req, res, next) => { + try { + const pluginName = req.params.plugin; + // check if plugin exists + if (!dnsPlugins[pluginName]) { + return res.status(404).send({ + error: "Plugin not found", + }); + } + + await installPlugin(pluginName); + res.status(200).send(true); + } catch (err) { + debug(logger, `${req.method.toUpperCase()} ${req.path}: ${err}`); + next(err); + } + return; + }); + +export default router; diff --git a/backend/routes/main.js b/backend/routes/main.js index 94682cfb..a308ea61 100644 --- a/backend/routes/main.js +++ b/backend/routes/main.js @@ -1,8 +1,11 @@ import express from "express"; +import { isCI } from "../lib/config.js"; import errs from "../lib/error.js"; +import logRequest from "../lib/express/log-request.js"; import pjson from "../package.json" with { type: "json" }; import { isSetup } from "../setup.js"; import auditLogRoutes from "./audit-log.js"; +import ciRoutes from "./ci.js"; import accessListsRoutes from "./nginx/access_lists.js"; import certificatesHostsRoutes from "./nginx/certificates.js"; import deadHostsRoutes from "./nginx/dead_hosts.js"; @@ -22,6 +25,8 @@ const router = express.Router({ mergeParams: true, }); +router.use(logRequest); + /** * Health Check * GET /api @@ -55,6 +60,11 @@ router.use("/nginx/streams", streamsRoutes); router.use("/nginx/access-lists", accessListsRoutes); router.use("/nginx/certificates", certificatesHostsRoutes); +// Only include CI routes if we're in a CI environment +if (isCI()) { + router.use("/ci", ciRoutes); +} + /** * API 404 for all other routes * diff --git a/backend/schema/common.json b/backend/schema/common.json index e56c9f70..00b06e00 100644 --- a/backend/schema/common.json +++ b/backend/schema/common.json @@ -77,12 +77,14 @@ "example": 3 }, "domain_names": { - "description": "Domain Names array", + "description": "Domain Names separated by a comma", "type": "array", + "minItems": 1, + "maxItems": 100, "uniqueItems": true, "items": { "type": "string", - "minLength": 1 + "pattern": "^[^&| @!#%^();:/\\\\}{=+?<>,~`'\"]+$" }, "example": ["example.com", "www.example.com"] }, diff --git a/backend/schema/components/certificate-object.json b/backend/schema/components/certificate-object.json index 4ef666b4..80cd92be 100644 --- a/backend/schema/components/certificate-object.json +++ b/backend/schema/components/certificate-object.json @@ -25,7 +25,15 @@ "example": "My Custom Cert" }, "domain_names": { - "$ref": "../common.json#/properties/domain_names" + "description": "Domain Names separated by a comma", + "type": "array", + "maxItems": 100, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[^&| @!#%^();:/\\\\}{=+?<>,~`'\"]+$" + }, + "example": ["example.com", "www.example.com"] }, "expires_on": { "description": "Date and time of expiration", diff --git a/docker/docker-compose.ci.sqlite.yml b/docker/docker-compose.ci.sqlite.yml index 1c5be48e..a8e057a8 100644 --- a/docker/docker-compose.ci.sqlite.yml +++ b/docker/docker-compose.ci.sqlite.yml @@ -1,9 +1,12 @@ # WARNING: This is a CI docker-compose file used for building and testing of the entire app, it should not be used for production. services: + cypress: + environment: + CYPRESS_stack: "sqlite" fullstack: environment: - DB_SQLITE_FILE: '/data/mydb.sqlite' + DB_SQLITE_FILE: "/data/mydb.sqlite" PUID: 1000 PGID: 1000 - DISABLE_IPV6: 'true' + DISABLE_IPV6: "true" diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/backend/run b/docker/rootfs/etc/s6-overlay/s6-rc.d/backend/run index 37a95f7d..b140257b 100755 --- a/docker/rootfs/etc/s6-overlay/s6-rc.d/backend/run +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/backend/run @@ -11,11 +11,11 @@ log_info 'Starting backend ...' if [ "${DEVELOPMENT:-}" = 'true' ]; then s6-setuidgid "$PUID:$PGID" yarn install - exec s6-setuidgid "$PUID:$PGID" bash -c "export HOME=$NPMHOME;node --max_old_space_size=250 --abort_on_uncaught_exception node_modules/nodemon/bin/nodemon.js" + exec s6-setuidgid "$PUID:$PGID" bash -c "export HOME=$NPMHOME;export CERTBOT_VERSION=$CERTBOT_VERSION;node --max_old_space_size=250 --abort_on_uncaught_exception node_modules/nodemon/bin/nodemon.js" else while : do - s6-setuidgid "$PUID:$PGID" bash -c "export HOME=$NPMHOME;node --abort_on_uncaught_exception --max_old_space_size=250 index.js" + s6-setuidgid "$PUID:$PGID" bash -c "export HOME=$NPMHOME;export CERTBOT_VERSION=$CERTBOT_VERSION;node --abort_on_uncaught_exception --max_old_space_size=250 index.js" sleep 1 done fi diff --git a/docker/rootfs/usr/bin/common.sh b/docker/rootfs/usr/bin/common.sh index 46529870..92bf19d9 100644 --- a/docker/rootfs/usr/bin/common.sh +++ b/docker/rootfs/usr/bin/common.sh @@ -21,6 +21,10 @@ NPMGROUP=npm NPMHOME=/tmp/npmuserhome export NPMUSER NPMGROUP NPMHOME +# Query the certbot version just once and use it elsewhere +CERTBOT_VERSION="$(certbot --version | grep -Eo '[0-9](\.[0-9]+)+')" +export CERTBOT_VERSION + if [[ "$PUID" -ne '0' ]] && [ "$PGID" = '0' ]; then # set group id to same as user id, # the user probably forgot to specify the group id and diff --git a/test/cypress/e2e/api/CertbotPlugins.cy.js b/test/cypress/e2e/api/CertbotPlugins.cy.js new file mode 100644 index 00000000..acf769c0 --- /dev/null +++ b/test/cypress/e2e/api/CertbotPlugins.cy.js @@ -0,0 +1,28 @@ +/// + +// Only tested once in the sqlite stack + +describe('CertbotPlugins', () => { + it('Should install all certbot plugins', () => { + cy.env(['stack']).then(({ stack }) => { + cy.log(`CertbotPlugins.cy.js - Running tests for stack: ${stack}`); + if (stack === 'sqlite') { + cy.task('backendApiGet', { + path: '/api/ci/certbot-plugins', + }).then((data) => { + expect(data).to.be.an('object'); + + // Install each plugin + for (const plugin of Object.keys(data)) { + cy.log(`Installing plugin: ${plugin}`); + cy.task('backendApiPost', { + path: `/api/ci/certbot-plugins/${plugin}`, + }).then((result) => { + expect(result).to.be.true; + }); + } + }); + } + }); + }); +}); diff --git a/test/cypress/e2e/api/Certificates.cy.js b/test/cypress/e2e/api/Certificates.cy.js index ce4200fb..25d1b8c4 100644 --- a/test/cypress/e2e/api/Certificates.cy.js +++ b/test/cypress/e2e/api/Certificates.cy.js @@ -4,8 +4,16 @@ describe('Certificates endpoints', () => { let token; let certID; + const certFile = 'test.example.com.pem'; + const keyFile = 'test.example.com-key.pem'; + before(() => { - cy.createCustomCerts(); + cy.createCustomCerts({ + domain: 'test.example.com', + certFile, + keyFile, + }) + cy.resetUsers(); cy.getToken().then((tok) => { token = tok; @@ -17,8 +25,8 @@ describe('Certificates endpoints', () => { token: token, path: '/api/nginx/certificates/validate', files: { - certificate: 'test.example.com.pem', - certificate_key: 'test.example.com-key.pem', + certificate: certFile, + certificate_key: keyFile, }, }).then((data) => { cy.validateSwaggerSchema('post', 200, '/nginx/certificates/validate', data); @@ -46,8 +54,8 @@ describe('Certificates endpoints', () => { token: token, path: `/api/nginx/certificates/${certID}/upload`, files: { - certificate: 'test.example.com.pem', - certificate_key: 'test.example.com-key.pem', + certificate: certFile, + certificate_key: keyFile, }, }).then((data) => { cy.validateSwaggerSchema('post', 200, '/nginx/certificates/{certID}/upload', data); diff --git a/test/cypress/e2e/api/Streams.cy.js b/test/cypress/e2e/api/Streams.cy.js index b2c97b44..95c6dba1 100644 --- a/test/cypress/e2e/api/Streams.cy.js +++ b/test/cypress/e2e/api/Streams.cy.js @@ -3,7 +3,16 @@ describe('Streams', () => { let token; + const certFile = 'website1.pem'; + const keyFile = 'website1.key.pem'; + before(() => { + cy.createCustomCerts({ + domain: 'website1.example.com', + certFile, + keyFile, + }) + cy.resetUsers(); cy.getToken().then((tok) => { token = tok; @@ -22,17 +31,6 @@ describe('Streams', () => { }); }); - // Create a custom cert pair - cy.task('getFixturesFolder').then((fixturesFolder) => { - cy.exec(`mkcert -cert-file=${fixturesFolder}/website1.pem -key-file=${fixturesFolder}/website1.key.pem website1.example.com`).then((result) => { - expect(result.exitCode).to.eq(0); - // Install CA - cy.exec('mkcert -install').then((result) => { - expect(result.exitCode).to.eq(0); - }); - }); - }); - cy.exec('rm -f /test/results/testssl.json'); }); @@ -136,8 +134,8 @@ describe('Streams', () => { token: token, path: `/api/nginx/certificates/${certID}/upload`, files: { - certificate: 'website1.pem', - certificate_key: 'website1.key.pem', + certificate: certFile, + certificate_key: keyFile, }, }).then((data) => { cy.validateSwaggerSchema('post', 200, '/nginx/certificates/{certID}/upload', data); diff --git a/test/cypress/fixtures/.gitignore b/test/cypress/fixtures/.gitignore new file mode 100644 index 00000000..cfaad761 --- /dev/null +++ b/test/cypress/fixtures/.gitignore @@ -0,0 +1 @@ +*.pem diff --git a/test/cypress/fixtures/.gitkeep b/test/cypress/fixtures/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/test/cypress/support/commands.mjs b/test/cypress/support/commands.mjs index 0001ded8..03c5bf22 100644 --- a/test/cypress/support/commands.mjs +++ b/test/cypress/support/commands.mjs @@ -157,10 +157,28 @@ Cypress.Commands.add('waitForCertificateStatus', (token, certID, expected, timeo // Creates CA files for testing, if they already exist they will be deleted // and recreated with the same content. This is to ensure that the files exist // for testing and are in a known state. -Cypress.Commands.add('createCustomCerts', () => { - cy.task('getFixturesFolder').then((fixturesFolder) => { - cy.exec('mkcert -install', {failOnNonZeroExit: false}).then(() => { - cy.exec(`mkcert -cert-file=${fixturesFolder}/test.example.com.pem -key-file=${fixturesFolder}/test.example.com-key.pem test.example.com`, {failOnNonZeroExit: false}); +Cypress.Commands.add('createCustomCerts', ({domain, certFile, keyFile}) => { + domain = domain || 'website1.example.com'; + certFile = certFile || 'website1.pem'; + keyFile = keyFile || 'website1.key.pem'; + + return cy.task('getFixturesFolder').then((fixturesFolder) => { + const fullCertFile = `${fixturesFolder}/${certFile}`; + const fullKeyFile = `${fixturesFolder}/${keyFile}`; + const cmd = `mkcert -cert-file="${fullCertFile}" -key-file="${fullKeyFile}" "${domain}"`; + cy.log(`Creating custom certs with command: ${cmd}`); + return cy.exec(cmd).then((result) => { + cy.log(`mkcert output:\n${JSON.stringify(result)}`); + expect(result.exitCode).to.eq(0); + return cy.exec('mkcert -install').then((result2) => { + cy.log(`mkcert install output:\n${JSON.stringify(result2)}`); + expect(result2.exitCode).to.eq(0); + }).then(() => { + return cy.wrap({ + certFile: fullCertFile, + keyFile: fullKeyFile, + }); + }); }); }); });