Fixes for upgraded debian trixie

- Adds request logging in debug mode for some endpoints
- Moves certbot version determination to the startup scripts and removes
  bash script encapsulation when installing plugins
- Revert loose domain validation, which was there for a specific reason
  addressing CVE's
- Fix Cypress suite for cert generation
- Adds Cypress test that iterates over the entire certbot plugins list
  and installs each one, ensuring at the very least that the install
  works
- Fixed some plugins based on this
- (!) Still some work to do on this, hostinger is still broken at least
- Improved cypress tests for custom certs; they will generate on each
  run instead of being baked in. The baked ones were due to expire soon

.gitignore

@@ -8,3 +8,5 @@ test/node_modules
 */node_modules
 docker/dev/dnsrouter-config.json.tmp
 docker/dev/resolv.conf
+.claude
+

backend/certbot/dns-plugins.json

@@ -341,7 +341,7 @@
 		"full_plugin_name": "dns-hostinger",
 		"name": "Hostinger.com",
 		"package_name": "certbot-dns-hostinger",
-		"version": "~=0.1.5"
+		"version": "~=0.1.3"
 	},
 	"hostingnl": {
 		"credentials": "dns_hostingnl_api_key = 0123456789abcdef0123456789abcdef",
@@ -553,7 +553,7 @@
 	},
 	"powerdns": {
 		"credentials": "dns_powerdns_api_url = https://api.mypowerdns.example.org\ndns_powerdns_api_key = AbCbASsd!@34",
-		"dependencies": "PyYAML==5.3.1",
+		"dependencies": "PyYAML>=6.0.1",
 		"full_plugin_name": "dns-powerdns",
 		"name": "PowerDNS",
 		"package_name": "certbot-dns-powerdns",

backend/lib/certbot.js

@@ -4,8 +4,6 @@ import { certbot as logger } from "../logger.js";
 import errs from "./error.js";
 import utils from "./utils.js";
 
-const CERTBOT_VERSION_REPLACEMENT = "$(certbot --version | grep -Eo '[0-9](\\.[0-9]+)+')";
-
 /**
  * Installs a cerbot plugin given the key for the object from
  * ../certbot/dns-plugins.json
@@ -15,24 +13,31 @@ const CERTBOT_VERSION_REPLACEMENT = "$(certbot --version | grep -Eo '[0-9](\\.[0
  */
 const installPlugin = async (pluginKey) => {
 	if (typeof dnsPlugins[pluginKey] === "undefined") {
-		// throw Error(`Certbot plugin ${pluginKey} not found`);
 		throw new errs.ItemNotFoundError(pluginKey);
 	}
 
 	const plugin = dnsPlugins[pluginKey];
 	logger.start(`Installing ${pluginKey}...`);
 
-	plugin.version = plugin.version.replace(/{{certbot-version}}/g, CERTBOT_VERSION_REPLACEMENT);
-	plugin.dependencies = plugin.dependencies.replace(/{{certbot-version}}/g, CERTBOT_VERSION_REPLACEMENT);
+	plugin.version = plugin.version.replace(/{{certbot-version}}/g, process.env.CERTBOT_VERSION);
+	plugin.dependencies = plugin.dependencies.replace(/{{certbot-version}}/g, process.env.CERTBOT_VERSION);
 
-	// SETUPTOOLS_USE_DISTUTILS is required for certbot plugins to install correctly
-	// in new versions of Python
-	let env = Object.assign({}, process.env, { SETUPTOOLS_USE_DISTUTILS: "stdlib" });
+	// SETUPTOOLS_USE_DISTUTILS=local uses setuptools' own bundled distutils.
+	// "stdlib" breaks Python 3.13+ where distutils was removed from the standard library.
+	let env = Object.assign({}, process.env, { SETUPTOOLS_USE_DISTUTILS: "local" });
 	if (typeof plugin.env === "object") {
 		env = Object.assign(env, plugin.env);
 	}
 
-	const cmd = `. /opt/certbot/bin/activate && pip install --no-cache-dir ${plugin.dependencies} ${plugin.package_name}${plugin.version}  && deactivate`;
+	const quotedDeps = plugin.dependencies.trim()
+		? plugin.dependencies
+				.trim()
+				.split(/\s+/)
+				.filter(Boolean)
+				.map((d) => `'${d}'`)
+				.join(" ")
+		: "";
+	const cmd = `. /opt/certbot/bin/activate && pip install --no-cache-dir ${quotedDeps} '${plugin.package_name}${plugin.version}' && deactivate`;
 	return utils
 		.exec(cmd, { env })
 		.then((result) => {
@@ -73,9 +78,7 @@ const installPlugins = async (pluginKeys) => {
 			})
 			.end(() => {
 				if (hasErrors) {
-					reject(
-						new errs.CommandError("Some plugins failed to install. Please check the logs above", 1),
-					);
+					reject(new errs.CommandError("Some plugins failed to install. Please check the logs above", 1));
 				} else {
 					resolve();
 				}
@@ -83,4 +86,4 @@ const installPlugins = async (pluginKeys) => {
 	});
 };
 
-export { installPlugins, installPlugin };
+export { installPlugin, installPlugins };

backend/lib/express/log-request.js

@@ -0,0 +1,7 @@
+import chalk from "chalk";
+import { debug, express as logger } from "../../logger.js";
+
+export default (req, _res, next) => {
+	debug(logger, `[${chalk.yellow(req.method.toUpperCase())}] ${chalk.green(req.path)}`);
+	next();
+};

backend/lib/utils.js

@@ -3,14 +3,12 @@ import { dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { Liquid } from "liquidjs";
 import _ from "lodash";
-import { debug, global as logger } from "../logger.js";
 import errs from "./error.js";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
 const exec = async (cmd, options = {}) => {
-	debug(logger, "CMD:", cmd);
 	const { stdout, stderr } = await new Promise((resolve, reject) => {
 		const child = nodeExec(cmd, options, (isError, stdout, stderr) => {
 			if (isError) {
@@ -34,7 +32,6 @@ const exec = async (cmd, options = {}) => {
  * @returns {Promise}
  */
 const execFile = (cmd, args, options) => {
-	debug(logger, `CMD: ${cmd} ${args ? args.join(" ") : ""}`);
 	const opts = options || {};
 
 	return new Promise((resolve, reject) => {

backend/package.json

@@ -20,6 +20,7 @@
 		"bcrypt": "^6.0.0",
 		"better-sqlite3": "^12.10.0",
 		"body-parser": "^2.2.2",
+		"chalk": "5.6.2",
 		"compression": "^1.8.1",
 		"express": "^5.2.1",
 		"express-fileupload": "^1.5.2",
@@ -43,7 +44,6 @@
 	"devDependencies": {
 		"@apidevtools/swagger-parser": "^12.1.0",
 		"@biomejs/biome": "^2.4.15",
-		"chalk": "5.6.2",
 		"nodemon": "^3.1.14"
 	},
 	"signale": {

backend/routes/ci.js

@@ -0,0 +1,59 @@
+import express from "express";
+import dnsPlugins from "../certbot/dns-plugins.json" with { type: "json" };
+import { installPlugin } from "../lib/certbot.js";
+import { debug, express as logger } from "../logger.js";
+
+const router = express.Router({
+	caseSensitive: true,
+	strict: true,
+	mergeParams: true,
+});
+
+/**
+ * ONLY AVAILABLE IN CI ENVIRONMENT!
+ */
+
+/**
+ * /api/ci/certbot-plugins
+ */
+router
+	.route("/certbot-plugins")
+	.options((_, res) => {
+		res.sendStatus(204);
+	})
+
+	// Return all certbot plugins
+	.get(async (_req, res, _next) => {
+		res.status(200).send(dnsPlugins);
+	});
+
+/**
+ * /api/ci/certbot-plugins/{plugin}
+ */
+router
+	.route("/certbot-plugins/:plugin")
+	.options((_, res) => {
+		res.sendStatus(204);
+	})
+
+	// Install a certbot plugin
+	.post(async (req, res, next) => {
+		try {
+			const pluginName = req.params.plugin;
+			// check if plugin exists
+			if (!dnsPlugins[pluginName]) {
+				return res.status(404).send({
+					error: "Plugin not found",
+				});
+			}
+
+			await installPlugin(pluginName);
+			res.status(200).send(true);
+		} catch (err) {
+			debug(logger, `${req.method.toUpperCase()} ${req.path}: ${err}`);
+			next(err);
+		}
+		return;
+	});
+
+export default router;

backend/routes/main.js

@@ -1,8 +1,11 @@
 import express from "express";
+import { isCI } from "../lib/config.js";
 import errs from "../lib/error.js";
+import logRequest from "../lib/express/log-request.js";
 import pjson from "../package.json" with { type: "json" };
 import { isSetup } from "../setup.js";
 import auditLogRoutes from "./audit-log.js";
+import ciRoutes from "./ci.js";
 import accessListsRoutes from "./nginx/access_lists.js";
 import certificatesHostsRoutes from "./nginx/certificates.js";
 import deadHostsRoutes from "./nginx/dead_hosts.js";
@@ -22,6 +25,8 @@ const router = express.Router({
 	mergeParams: true,
 });
 
+router.use(logRequest);
+
 /**
  * Health Check
  * GET /api
@@ -55,6 +60,11 @@ router.use("/nginx/streams", streamsRoutes);
 router.use("/nginx/access-lists", accessListsRoutes);
 router.use("/nginx/certificates", certificatesHostsRoutes);
 
+// Only include CI routes if we're in a CI environment
+if (isCI()) {
+	router.use("/ci", ciRoutes);
+}
+
 /**
  * API 404 for all other routes
  *

backend/schema/common.json

@@ -77,12 +77,14 @@
 			"example": 3
 		},
 		"domain_names": {
-			"description": "Domain Names array",
+			"description": "Domain Names separated by a comma",
 			"type": "array",
+			"minItems": 1,
+			"maxItems": 100,
 			"uniqueItems": true,
 			"items": {
 				"type": "string",
-				"minLength": 1
+				"pattern": "^[^&| @!#%^();:/\\\\}{=+?<>,~`'\"]+$"
 			},
 			"example": ["example.com", "www.example.com"]
 		},

backend/schema/components/certificate-object.json

@@ -25,7 +25,15 @@
 			"example": "My Custom Cert"
 		},
 		"domain_names": {
-			"$ref": "../common.json#/properties/domain_names"
+			"description": "Domain Names separated by a comma",
+			"type": "array",
+			"maxItems": 100,
+			"uniqueItems": true,
+			"items": {
+				"type": "string",
+				"pattern": "^[^&| @!#%^();:/\\\\}{=+?<>,~`'\"]+$"
+			},
+			"example": ["example.com", "www.example.com"]
 		},
 		"expires_on": {
 			"description": "Date and time of expiration",

docker/docker-compose.ci.sqlite.yml

@@ -1,9 +1,12 @@
 # WARNING: This is a CI docker-compose file used for building and testing of the entire app, it should not be used for production.
 services:
+  cypress:
+    environment:
+      CYPRESS_stack: "sqlite"
 
   fullstack:
     environment:
-      DB_SQLITE_FILE: '/data/mydb.sqlite'
+      DB_SQLITE_FILE: "/data/mydb.sqlite"
       PUID: 1000
       PGID: 1000
-      DISABLE_IPV6: 'true'
+      DISABLE_IPV6: "true"

docker/rootfs/etc/s6-overlay/s6-rc.d/backend/run

@@ -11,11 +11,11 @@ log_info 'Starting backend ...'
 
 if [ "${DEVELOPMENT:-}" = 'true' ]; then
 	s6-setuidgid "$PUID:$PGID" yarn install
-	exec s6-setuidgid "$PUID:$PGID" bash -c "export HOME=$NPMHOME;node --max_old_space_size=250 --abort_on_uncaught_exception node_modules/nodemon/bin/nodemon.js"
+	exec s6-setuidgid "$PUID:$PGID" bash -c "export HOME=$NPMHOME;export CERTBOT_VERSION=$CERTBOT_VERSION;node --max_old_space_size=250 --abort_on_uncaught_exception node_modules/nodemon/bin/nodemon.js"
 else
 	while :
 	do
-		s6-setuidgid "$PUID:$PGID" bash -c "export HOME=$NPMHOME;node --abort_on_uncaught_exception --max_old_space_size=250 index.js"
+		s6-setuidgid "$PUID:$PGID" bash -c "export HOME=$NPMHOME;export CERTBOT_VERSION=$CERTBOT_VERSION;node --abort_on_uncaught_exception --max_old_space_size=250 index.js"
 		sleep 1
 	done
 fi

docker/rootfs/usr/bin/common.sh

@@ -21,6 +21,10 @@ NPMGROUP=npm
 NPMHOME=/tmp/npmuserhome
 export NPMUSER NPMGROUP NPMHOME
 
+# Query the certbot version just once and use it elsewhere
+CERTBOT_VERSION="$(certbot --version | grep -Eo '[0-9](\.[0-9]+)+')"
+export CERTBOT_VERSION
+
 if [[ "$PUID" -ne '0' ]] && [ "$PGID" = '0' ]; then
 	# set group id to same as user id,
 	# the user probably forgot to specify the group id and

test/cypress/e2e/api/CertbotPlugins.cy.js

@@ -0,0 +1,28 @@
+/// <reference types="cypress" />
+
+// Only tested once in the sqlite stack
+
+describe('CertbotPlugins', () => {
+	it('Should install all certbot plugins', () => {
+		cy.env(['stack']).then(({ stack }) => {
+			cy.log(`CertbotPlugins.cy.js - Running tests for stack: ${stack}`);
+			if (stack === 'sqlite') {
+				cy.task('backendApiGet', {
+					path:  '/api/ci/certbot-plugins',
+				}).then((data) => {
+					expect(data).to.be.an('object');
+
+					// Install each plugin
+					for (const plugin of Object.keys(data)) {
+						cy.log(`Installing plugin: ${plugin}`);
+						cy.task('backendApiPost', {
+							path: `/api/ci/certbot-plugins/${plugin}`,
+						}).then((result) => {
+							expect(result).to.be.true;
+						});
+					}
+				});
+			}
+		});
+	});
+});

test/cypress/e2e/api/Certificates.cy.js

@@ -4,8 +4,16 @@ describe('Certificates endpoints', () => {
 	let token;
 	let certID;
 
+	const certFile = 'test.example.com.pem';
+	const keyFile = 'test.example.com-key.pem';
+
 	before(() => {
-		cy.createCustomCerts();
+		cy.createCustomCerts({
+			domain: 'test.example.com',
+			certFile,
+			keyFile,
+		})
+
 		cy.resetUsers();
 		cy.getToken().then((tok) => {
 			token = tok;
@@ -17,8 +25,8 @@ describe('Certificates endpoints', () => {
 			token: token,
 			path:  '/api/nginx/certificates/validate',
 			files:  {
-				certificate: 'test.example.com.pem',
-				certificate_key: 'test.example.com-key.pem',
+				certificate: certFile,
+				certificate_key: keyFile,
 			},
 		}).then((data) => {
 			cy.validateSwaggerSchema('post', 200, '/nginx/certificates/validate', data);
@@ -46,8 +54,8 @@ describe('Certificates endpoints', () => {
 				token: token,
 				path:  `/api/nginx/certificates/${certID}/upload`,
 				files:  {
-					certificate: 'test.example.com.pem',
-					certificate_key: 'test.example.com-key.pem',
+					certificate: certFile,
+					certificate_key: keyFile,
 				},
 			}).then((data) => {
 				cy.validateSwaggerSchema('post', 200, '/nginx/certificates/{certID}/upload', data);

test/cypress/e2e/api/Streams.cy.js

@@ -3,7 +3,16 @@
 describe('Streams', () => {
 	let token;
 
+	const certFile = 'website1.pem';
+	const keyFile = 'website1.key.pem';
+
 	before(() => {
+		cy.createCustomCerts({
+			domain: 'website1.example.com',
+			certFile,
+			keyFile,
+		})
+
 		cy.resetUsers();
 		cy.getToken().then((tok) => {
 			token = tok;
@@ -22,17 +31,6 @@ describe('Streams', () => {
 			});
 		});
 
-		// Create a custom cert pair
-		cy.task('getFixturesFolder').then((fixturesFolder) => {
-			cy.exec(`mkcert -cert-file=${fixturesFolder}/website1.pem -key-file=${fixturesFolder}/website1.key.pem website1.example.com`).then((result) => {
-				expect(result.exitCode).to.eq(0);
-				// Install CA
-				cy.exec('mkcert -install').then((result) => {
-					expect(result.exitCode).to.eq(0);
-				});
-			});
-		});
-
 		cy.exec('rm -f /test/results/testssl.json');
 	});
 
@@ -136,8 +134,8 @@ describe('Streams', () => {
 				token: token,
 				path:  `/api/nginx/certificates/${certID}/upload`,
 				files:  {
-					certificate: 'website1.pem',
-					certificate_key: 'website1.key.pem',
+					certificate: certFile,
+					certificate_key: keyFile,
 				},
 			}).then((data) => {
 				cy.validateSwaggerSchema('post', 200, '/nginx/certificates/{certID}/upload', data);

test/cypress/fixtures/.gitignore

@@ -0,0 +1 @@
+*.pem

test/cypress/support/commands.mjs

@@ -157,10 +157,28 @@ Cypress.Commands.add('waitForCertificateStatus', (token, certID, expected, timeo
 // Creates CA files for testing, if they already exist they will be deleted
 // and recreated with the same content. This is to ensure that the files exist
 // for testing and are in a known state.
-Cypress.Commands.add('createCustomCerts', () => {
-	cy.task('getFixturesFolder').then((fixturesFolder) => {
-		cy.exec('mkcert -install', {failOnNonZeroExit: false}).then(() => {
-			cy.exec(`mkcert -cert-file=${fixturesFolder}/test.example.com.pem -key-file=${fixturesFolder}/test.example.com-key.pem test.example.com`, {failOnNonZeroExit: false});
+Cypress.Commands.add('createCustomCerts', ({domain, certFile, keyFile}) => {
+	domain = domain || 'website1.example.com';
+	certFile = certFile || 'website1.pem';
+	keyFile = keyFile || 'website1.key.pem';
+
+	return cy.task('getFixturesFolder').then((fixturesFolder) => {
+		const fullCertFile = `${fixturesFolder}/${certFile}`;
+		const fullKeyFile = `${fixturesFolder}/${keyFile}`;
+		const cmd = `mkcert -cert-file="${fullCertFile}" -key-file="${fullKeyFile}" "${domain}"`;
+		cy.log(`Creating custom certs with command: ${cmd}`);
+		return cy.exec(cmd).then((result) => {
+			cy.log(`mkcert output:\n${JSON.stringify(result)}`);
+			expect(result.exitCode).to.eq(0);
+			return cy.exec('mkcert -install').then((result2) => {
+				cy.log(`mkcert install output:\n${JSON.stringify(result2)}`);
+				expect(result2.exitCode).to.eq(0);
+			}).then(() => {
+				return cy.wrap({
+					certFile: fullCertFile,
+					keyFile: fullKeyFile,
+				});
+			});
 		});
 	});
 });