fix networking issues in gitea action runner

* claude-plans-execute-upgrade:
  add build extensions dir script
  update gitignore
  add php error logging
  fix bootstrap2 skin for 1.39
  update LocalSettings.php
  upgrade to mediawiki 1.39 in mw dockerfile
  bump mysql dockerfile version from 5.7 to 8.0
  add mysql no root password plan
  add execution notes
  update plan
  remove plan
  add fixes for getting non-root mysql user
  fix wiki backups and canary script to check for missing trailer, not just nonzero files
  remove stupid dead file
  add plan to fix sql backups, plus implemented fixes for sql backups
  add mysql no root pw transition plan
  add mediawiki upgrade plan

.gitignore

@@ -1,20 +1,26 @@
+*.log
 *.pyc
 environment
 attic
 
 # gitea
-#d-gitea/data/
+d-gitea/data/
-d-gitea/custom/conf/app.ini
+d-gitea/custom/
-d-gitea/custom/gitea.db
-d-gitea/custom/avatars
-d-gitea/custom/log/
-d-gitea/custom/queues/
 
 # mediawiki
 charlesreid1.wiki.conf
+d-mediawiki/mediawiki/
+d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/Bootstrap2.php
+d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/navbar.php
+d-mediawiki/charlesreid1-config/mediawiki/mathjax
+
+# nginx
+d-nginx-charlesreid1/conf.d/http.DOMAIN.conf
+d-nginx-charlesreid1/conf.d/https.DOMAIN.conf
 
 # scripts dir
 scripts/git_*_www.py
+scripts/certbot/renew_charlesreid1_certs.sh
 *.timer
 *.service
 

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "mkdocs-material"]
 	path = mkdocs-material
-	url = git@github.com:charlesreid1-docker/mkdocs-material.git
+	url = https://github.com/charlesreid1/mkdocs-material

Makefile

@@ -26,7 +26,7 @@ help:
 	@echo "--------------------------------------------------"
 	@echo "                   Backups:"
 	@echo ""
-	@echo "make backups:        Create backups of every service (gitea, wiki database, wiki files) in ~/backups"
+	@echo "make backups:        Create backups of every service (wiki database, wiki files) in ~/backups"
 	@echo ""
 	@echo "make clean-backups:  Remove files from ~/backups directory older than 30 days"
 	@echo ""
@@ -53,7 +53,7 @@ help:
 	@echo ""
 	@echo "make install:        Install and start systemd service to run pod-charlesreid1."
 	@echo "                     Also install and start systemd service for pod-charlesreid1 backup services"
-	@echo "                     for each service (gitea/mediawiki/mysql) part of pod-charlesreid1."
+	@echo "                     for each service (mediawiki/mysql) part of pod-charlesreid1."
 	@echo ""
 	@echo "make uninstall:      Remove all systemd startup services and timers part of pod-charlesreid1"
 	@echo ""
@@ -61,18 +61,20 @@ help:
 # Templates
 
 templates:
-	python3 $(POD_CHARLESREID1_DIR)/scripts/apply_templates.py
+	@find * -name "*.service.j2" | xargs -I '{}' chmod 644 {}
+	@find * -name "*.timer.j2" | xargs -I '{}' chmod 644 {}
+	/home/charles/.pyenv/shims/python3 $(POD_CHARLESREID1_DIR)/scripts/apply_templates.py
 
 list-templates:
 	@find * -name "*.j2"
 
 clean-templates:
-	python3 $(POD_CHARLESREID1_DIR)/scripts/clean_templates.py
+	# sudo is required because bind-mounted gitea files end up owned by root. stupid docker.
+	sudo -E /home/charles/.pyenv/shims/python3 $(POD_CHARLESREID1_DIR)/scripts/clean_templates.py
 
 # Backups
 
-backups: templates
+backups:
-	$(POD_CHARLESREID1_DIR)/scripts/backups/gitea_dump.sh
 	$(POD_CHARLESREID1_DIR)/scripts/backups/wikidb_dump.sh
 	$(POD_CHARLESREID1_DIR)/scripts/backups/wikifiles_dump.sh
 
@@ -88,52 +90,105 @@ mw-fix-extensions: mw-build-extensions
 	$(POD_CHARLESREID1_DIR)/scripts/mw/build_extensions_dir.sh
 
 mw-fix-localsettings:
-	$(POD_CHARLESEREID1_DIR)/scripts/mw/fix_LocalSettings.sh
+	$(POD_CHARLESREID1_DIR)/scripts/mw/fix_LocalSettings.sh
 
 mw-fix-skins:
-	$(POD_CHARLESEREID1_DIR)/scripts/mw/fix_skins.sh
+	$(POD_CHARLESREID1_DIR)/scripts/mw/fix_skins.sh
 
 # /www Dir
 
-clone-www: templates
+clone-www:
-	python3 $(POD_CHARLESREID1_DIR)/scripts/git_clone_www.py
+	/home/charles/.pyenv/shims/python3 $(POD_CHARLESREID1_DIR)/scripts/git_clone_www.py
 
-pull-www: templates
+pull-www:
-	python3 $(POD_CHARLESREID1_DIR)/scripts/git_pull_www.py
+	/home/charles/.pyenv/shims/python3 $(POD_CHARLESREID1_DIR)/scripts/git_pull_www.py
 
-install: templates
+install:
 ifeq ($(shell which systemctl),)
 	$(error Please run this make command on a system with systemctl installed)
 endif
-	cp $(POD_CHARLESREID1_DIR)/scripts/pod-charlesreid1.service /etc/systemd/system/pod-charlesreid1.service
+	@/home/charles/.pyenv/shims/python3 -c 'import botocore' || (echo "Please install the botocore library using python3 or pip3 binary"; exit 1)
-	cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-gitea.{service,timer} /etc/systemd/system/.
+	@/home/charles/.pyenv/shims/python3 -c 'import boto3' || (echo "Please install the boto3 library using python3 or pip3 binary"; exit 1)
-	cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikidb.{service,timer} /etc/systemd/system/.
+
-	cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikifiles.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/pod-charlesreid1.service /etc/systemd/system/pod-charlesreid1.service
-	systemctl daemon-reload
+
-	systemctl enable pod-charlesreid1
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-aws.{service,timer} /etc/systemd/system/.
-	systemctl enable pod-charlesreid1-backups-gitea.timer
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-cleanolderthan.{service,timer} /etc/systemd/system/.
-	systemctl enable pod-charlesreid1-backups-wikidb.timer
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-gitea.{service,timer} /etc/systemd/system/.
-	systemctl enable pod-charlesreid1-backups-wikifiles.timer
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikidb.{service,timer} /etc/systemd/system/.
-	systemctl start pod-charlesreid1-backups-gitea.timer
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikifiles.{service,timer} /etc/systemd/system/.
-	systemctl start pod-charlesreid1-backups-wikidb.timer
+
-	systemctl start pod-charlesreid1-backups-wikifiles.timer
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/canary/pod-charlesreid1-canary.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/certbot/pod-charlesreid1-certbot.{service,timer} /etc/systemd/system/.
+
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/10-pod-charlesreid1-rsyslog.conf /etc/rsyslog.d/.
+
+	sudo chmod 664 /etc/systemd/system/pod-charlesreid1*
+	sudo systemctl daemon-reload
+
+	sudo systemctl restart rsyslog
+
+	sudo systemctl enable pod-charlesreid1
+	sudo systemctl enable pod-charlesreid1-backups-wikidb.timer
+	sudo systemctl enable pod-charlesreid1-backups-wikifiles.timer
+	sudo systemctl enable pod-charlesreid1-backups-gitea.timer
+	sudo systemctl enable pod-charlesreid1-backups-aws.timer
+	sudo systemctl enable pod-charlesreid1-backups-cleanolderthan.timer
+	sudo systemctl enable pod-charlesreid1-canary.timer
+	sudo systemctl enable pod-charlesreid1-certbot.timer
+
+	sudo systemctl start pod-charlesreid1-backups-wikidb.timer
+	sudo systemctl start pod-charlesreid1-backups-wikifiles.timer
+	sudo systemctl start pod-charlesreid1-backups-gitea.timer
+	sudo systemctl start pod-charlesreid1-backups-aws.timer
+	sudo systemctl start pod-charlesreid1-backups-cleanolderthan.timer
+	sudo systemctl start pod-charlesreid1-canary.timer
+	sudo systemctl start pod-charlesreid1-certbot.timer
+
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-aws.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-cleanolderthan.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-gitea.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-wikidb.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-wikifiles.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-canary.service.log
 
 uninstall:
 ifeq ($(shell which systemctl),)
 	$(error Please run this make command on a system with systemctl installed)
 endif
-	systemctl disable pod-charlesreid1
+	-sudo systemctl disable pod-charlesreid1
-	systemctl disable pod-charlesreid1-backups-gitea.timer
+	-sudo systemctl disable pod-charlesreid1-backups-aws.timer
-	systemctl disable pod-charlesreid1-backups-wikidb.timer
+	-sudo systemctl disable pod-charlesreid1-backups-cleanolderthan.timer
-	systemctl disable pod-charlesreid1-backups-wikifiles.timer
+	-sudo systemctl disable pod-charlesreid1-backups-gitea.timer
-	systemctl stop pod-charlesreid1
+	-sudo systemctl disable pod-charlesreid1-backups-wikidb.timer
-	systemctl stop pod-charlesreid1-backups-gitea.timer
+	-sudo systemctl disable pod-charlesreid1-backups-wikifiles.timer
-	systemctl stop pod-charlesreid1-backups-wikidb.timer
+	-sudo systemctl disable pod-charlesreid1-canary.timer
-	systemctl stop pod-charlesreid1-backups-wikifiles.timer
+	-sudo systemctl disable pod-charlesreid1-certbot.timer
-	rm -f /etc/systemd/system/pod-charlesreid1.service
+
-	rm -f /etc/systemd/system/pod-charlesreid1-backups-gitea.{service,timer}
+	# Leave the pod running!
-	rm -f /etc/systemd/system/pod-charlesreid1-backups-wikidb.{service,timer}
+	# -sudo systemctl stop pod-charlesreid1
-	rm -f /etc/systemd/system/pod-charlesreid1-backups-wikifiles.{service,timer}
+
-	systemctl daemon-reload
+	-sudo systemctl stop pod-charlesreid1-backups-aws.timer
+	-sudo systemctl stop pod-charlesreid1-backups-cleanolderthan.timer
+	-sudo systemctl stop pod-charlesreid1-backups-gitea.timer
+	-sudo systemctl stop pod-charlesreid1-backups-wikidb.timer
+	-sudo systemctl stop pod-charlesreid1-backups-wikifiles.timer
+	-sudo systemctl stop pod-charlesreid1-canary.timer
+	-sudo systemctl stop pod-charlesreid1-certbot.timer
+
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1.service
+
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-aws.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-cleanolderthan.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-gitea.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-wikidb.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-wikifiles.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-canary.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-certbot.{service,timer}
+
+	sudo systemctl daemon-reload
+
+	-sudo rm -f /etc/rsyslog.d/10-pod-charlesreid1-rsyslog.conf
+	-sudo systemctl restart rsyslog
 
 .PHONY: help

Troubleshooting.md

@@ -0,0 +1,19 @@
+To get a shell in a container that has been created, before it is runnning in a pod, use `docker run`:
+
+```
+docker run --rm -it --entrypoint bash <image-name-or-id>
+
+
+docker run --rm -it --entrypoint bash pod-charlesreid1_stormy_mediawiki
+```
+
+To get a shell in a container that is running in a pod, use `docker exec`:
+
+```
+docker exec -it <image-name> /bin/bash
+
+docker exec -it stormy_mw /bin/bash
+```
+
+Also, if no changes are picking up, and you've already tried rebuilding the container image, try editing the Dockerfile.
+

d-gitea/README.md

@@ -18,7 +18,7 @@ The data directory contains any instance-specific gitea data.
 
 The data directory is bind-mounted to `/app/gitea/data` in the container.
 
-## Repository Data
+## Repository Drive
 
 Gitea stores all of its repositories in a separate drive that is at
 `/gitea_repositories` on the host machine.

d-gitea/custom/conf/app.ini.j2

@@ -6,12 +6,14 @@
 ;; https://github.com/go-gitea/gitea/blob/master/conf/app.ini
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-APP_NAME = {{ gitea_app_name }}
+APP_NAME = {{ pod_charlesreid1_gitea_app_name }}
 RUN_USER = git
 RUN_MODE = prod
+WORK_PATH = /data/gitea
 
 [ui]
-DEFAULT_THEME = arc-green
+DEFAULT_THEME = gitea-dark
+THEMES = gitea-dark
 
 [database]
 DB_TYPE  = sqlite3
@@ -31,17 +33,17 @@ DISABLE_HTTP_GIT = false
 
 [server]
 PROTOCOL     = http
-DOMAIN       = git.{{ server_name_default }}
+DOMAIN       = git.{{ pod_charlesreid1_server_name }}
 #CERT_FILE    = /www/gitea/certs/cert.pem
 #KEY_FILE     = /www/gitea/certs/key.pem
-SSH_DOMAIN   = git.{{ server_name_default }}
+SSH_DOMAIN   = git.{{ pod_charlesreid1_server_name }}
 HTTP_PORT    = 3000
 HTTP_ADDR    = 0.0.0.0
-ROOT_URL     = https://git.{{ server_name_default }}
+ROOT_URL     = https://git.{{ pod_charlesreid1_server_name }}
 ;ROOT_URL     = %(PROTOCOL)s://%(DOMAIN)s:%(HTTP_PORT)s/
 DISABLE_SSH  = false
 ; port to display in clone url:
-SSH_PORT     = 222
+;SSH_PORT     = 222
 ; port for built-in ssh server to listen on:
 SSH_LISTEN_PORT = 22
 OFFLINE_MODE = false
@@ -92,9 +94,12 @@ ENABLED      = false
 
 [security]
 INSTALL_LOCK        = true
-SECRET_KEY          = {{ gitea_secret_key }}
+SECRET_KEY          = {{ pod_charlesreid1_gitea_secretkey }}
-MIN_PASSWORD_LENGTH = 6
+MIN_PASSWORD_LENGTH = 10
-INTERNAL_TOKEN      = {{ gitea_internal_token }}
+INTERNAL_TOKEN      = {{ pod_charlesreid1_gitea_internaltoken }}
+
+[actions]
+ENABLED = true
 
 [other]
 SHOW_FOOTER_BRANDING           = false

d-gitea/runner/config.yaml

@@ -0,0 +1,20 @@
+log:
+  level: info
+
+runner:
+  # Label format: <label>:<runner-type>:<image>
+  # "ubuntu-latest" is the standard GitHub Actions label.
+  # Map it (and common aliases) to a Docker image so jobs don't sit waiting.
+  labels:
+    # alpine: ~50MB, has python3+pip; install extras with: apk add --no-cache git curl
+    - "alpine:docker://python:3.12-alpine"
+    - "ubuntu-latest:docker://catthehacker/ubuntu:act-22.04"
+    - "ubuntu-22.04:docker://catthehacker/ubuntu:act-22.04"
+    - "ubuntu-20.04:docker://catthehacker/ubuntu:act-20.04"
+    - "ubuntu-24.04:docker://catthehacker/ubuntu:act-22.04"
+
+container:
+  network: "pod-charlesreid1_frontend"
+
+cache:
+  enabled: true

d-mediawiki/Dockerfile

@@ -1,22 +1,12 @@
-FROM mediawiki
+FROM mediawiki:1.39.12
 
 EXPOSE 8989
 
-VOLUME ["/var/www/html"]
 
-# Install ImageMagick
+# Install ImageMagick (used for image thumbnailing)
-# and math stuff mentioned by Math extension readme
 RUN apt-get update && \
-    apt-get install -y build-essential \
+    apt-get install -y imagemagick && \
-            dvipng \
+    rm -rf /var/lib/apt/lists/*
-            ocaml \
-            ghostscript \
-            imagemagick \
-            texlive-latex-base \
-            texlive-latex-extra \
-            texlive-fonts-recommended \
-            texlive-lang-greek \
-            texlive-latex-recommended
 
 # Copy skins, config files, and other particulars into container
 
@@ -24,15 +14,13 @@ RUN apt-get update && \
 # MediaWiki needs everything, everything, to be in one folder.
 # Docker is totally incapable of mounting a file in a volume.
 # I cannot update LocalSettings.php without clearing the cache.
-# I cannot clear the cache without reinstalling all of latex.
 # I can't bind-mount the skins dir, because then it's owned by root.
 # I can't fix the fact that all bind-mounted dirs are owned by root, 
 # because I can only add commands in THIS DOCKERFILE.
 # and when you run the commands in this dockerfile, 
 # YOU CANNOT SEE THE BIND-MOUNTED STUFF.
 
-# Extensions
+# Extensions (REL1_39 branches; EmbedVideo skipped for the 1.34 -> 1.39 upgrade)
-COPY charlesreid1-config/mediawiki/extensions/EmbedVideo              /var/www/html/extensions/EmbedVideo
 COPY charlesreid1-config/mediawiki/extensions/Math                    /var/www/html/extensions/Math
 COPY charlesreid1-config/mediawiki/extensions/ParserFunctions         /var/www/html/extensions/ParserFunctions
 COPY charlesreid1-config/mediawiki/extensions/SyntaxHighlight_GeSHi   /var/www/html/extensions/SyntaxHighlight_GeSHi
@@ -41,22 +29,27 @@ RUN chown -R www-data:www-data /var/www/html/*
 # Skins
 COPY charlesreid1-config/mediawiki/skins /var/www/html/skins
 RUN chown -R www-data:www-data /var/www/html/skins
+RUN touch /var/www/html/skins
+
+# MathJax 3.2.2 (self-hosted, served via Apache alias at /w/mathjax/*).
+# Math extension runs in 'source' mode; MathJax renders client-side, so we
+# never call out to restbase/mathoid. See LocalSettings.php.j2.
+COPY charlesreid1-config/mediawiki/mathjax /var/www/html/mathjax
+RUN chown -R www-data:www-data /var/www/html/mathjax
 
 # Settings
 COPY charlesreid1-config/mediawiki/LocalSettings.php /var/www/html/LocalSettings.php
 RUN chown -R www-data:www-data /var/www/html/LocalSettings*
 RUN chmod 600 /var/www/html/LocalSettings.php
 
-# MediaWiki Fail2ban log directory
-RUN mkdir -p /var/log/mwf2b
-RUN chown -R www-data:www-data /var/log/mwf2b
-RUN chmod 700 /var/log/mwf2b
-
 # Apache conf file
 COPY charlesreid1-config/apache/*.conf /etc/apache2/sites-enabled/
 RUN a2enmod rewrite
 RUN service apache2 restart
 
-## make texvc
+# PHP conf file
-#CMD cd /var/www/html/extensions/Math && make && apache2-foreground
+# https://hub.docker.com/_/php/
+COPY php/php.ini /usr/local/etc/php/
+
+# Start
 CMD apache2-foreground

d-mediawiki/Skins.md

@@ -5,6 +5,10 @@ To update the MediaWiki skin:
 - Rebuild the MW container while the docker pod is still running (won't effect the docker pod)
 - When finished rebuilding the MW container, restart the docker pod.
 
+The skin currently in use is in `charlesreid1-config/mediawiki/skins/Bootstrap2`
+
+To rebuild and then restart the pod:
+
 ```
 # switch to main pod directory
 cd ../

d-mediawiki/charlesreid1-config/apache/charlesreid1.wiki.conf.j2

@@ -1,4 +1,4 @@
-ServerName {{ server_name_default }}
+ServerName {{ pod_charlesreid1_server_name }}
 
 Listen 8989
 
@@ -7,10 +7,10 @@ Listen 8989
     # talks to apache via 127.0.0.1
     # on port 8989
 
-    ServerAlias www.{{ server_name_default }}
+    ServerAlias www.{{ pod_charlesreid1_server_name }}
 
     LogLevel warn
-    ServerAdmin {{ admin_email }}
+    ServerAdmin {{ pod_charlesreid1_mediawiki_admin_email }}
     DirectoryIndex index.html index.cgi index.php
 
 

d-mediawiki/charlesreid1-config/mediawiki/LocalSettings.php.j2

@@ -13,8 +13,8 @@ if ( !defined( 'MEDIAWIKI' ) ) {
 }
 
 ## The protocol and server name to use in fully-qualified URLs
-$wgServer = 'https://{{ server_name_default }}';
+$wgServer = 'https://{{ pod_charlesreid1_server_name }}';
-$wgCanonicalServer = 'https://{{ server_name_default }}';
+$wgCanonicalServer = 'https://{{ pod_charlesreid1_server_name }}';
 
 ## The URL path to static resources (images, scripts, etc.)
 $wgStylePath = "$wgScriptPath/skins";
@@ -43,10 +43,11 @@ $wgDBpassword = getenv('MYSQL_PASSWORD');
 # MySQL specific settings
 $wgDBprefix = "";
 $wgDBTableOptions = "ENGINE=InnoDB, DEFAULT CHARSET=binary";
-$wgDBmysql5 = true;
+# $wgDBmysql5 removed — deprecated in MW 1.39
 
 # Shared memory settings
 $wgMainCacheType = CACHE_ACCEL;
+$wgCacheDirectory = "$IP/cache";
 $wgMemCachedServers = [];
 
 # To enable image uploads, make sure the 'images' directory
@@ -83,16 +84,25 @@ $wgPingback = false;
 # available UTF-8 locale
 $wgShellLocale = "en_US.utf8";
 
-# If you have the appropriate support software installed
+# Math rendering: Math extension emits raw LaTeX ('source' mode), then
-# you can enable inline LaTeX equations:
+# a self-hosted MathJax 3 build at /w/mathjax/ renders it client-side.
-$wgUseTeX           = true;
+# No mathoid, no restbase, no external CDN.
-$wgTexvc = "$IP/extensions/Math/math/texvc";
+$wgDefaultUserOptions['math'] = 'source';
-#$wgTexvc = '/usr/bin/texvc';
+$wgMathValidModes = [ 'source' ];
-
+# Skip TeX validation entirely — default validator calls out to restbase,
-# Set MathML as default rendering option
+# which breaks air-gapped installs even when we only emit source HTML.
-$wgDefaultUserOptions['math'] = 'mathml';
+$wgMathDisableTexFilter = 'always';
-$wgMathFullRestbaseURL = 'https://en.wikipedia.org/api/rest_';
+$wgHooks['BeforePageDisplay'][] = function ( $out, $skin ) {
-$wgMathMathMLUrl = 'https://mathoid-beta.wmflabs.org/';
+    $out->addHeadItem( 'mathjax',
+        '<script>window.MathJax = {'
+        . 'tex: { inlineMath: [["$","$"],["\\\\(","\\\\)"]], '
+        . 'displayMath: [["$$","$$"],["\\\\[","\\\\]"]], '
+        . 'processEscapes: true }, '
+        . 'options: { processHtmlClass: "mwe-math-fallback-source-inline|mwe-math-fallback-source-display|mwe-math-element" } '
+        . '};</script>'
+        . '<script async src="/w/mathjax/tex-chtml.js"></script>'
+    );
+};
 
 # Site language code, should be one of the list in ./languages/data/Names.php
 $wgLanguageCode = "en";
@@ -104,7 +114,7 @@ $wgAuthenticationTokenVersion = "1";
 
 # Site upgrade key. Must be set to a string (default provided) to turn on the
 # web installer while LocalSettings.php is in place
-$wgUpgradeKey = "984c1d9858dabc27";
+$wgUpgradeKey = getenv('MEDIAWIKI_UPGRADEKEY');
 
 # No license info
 $wgRightsPage = "";
@@ -156,7 +166,7 @@ $wgPutIPinRC=true;
 # Getting some weird "Error creating thumbnail: Invalid thumbnail parameters" messages w/ thumbnail
 # http://www.gossamer-threads.com/lists/wiki/mediawiki/169439
 $wgMaxImageArea=64000000;
-$wgMaxShellMemory=0;
+$wgMaxShellMemory=512000;
 
 $wgFavicon="$wgScriptPath/favicon.ico";
 
@@ -197,24 +207,22 @@ $wgSyntaxHighlightDefaultLang = "text";
 wfLoadExtension( 'ParserFunctions' );
 
 ##############################################
-# Embed videos extension
+# Embed videos extension — SKIPPED for MW 1.39 upgrade (add back later)
-# https://github.com/HydraWiki/mediawiki-embedvideo/
-# require_once("$IP/extensions/EmbedVideo/EmbedVideo.php");
-
-wfLoadExtension( 'EmbedVideo' );
 
 ###########################################
 # Math extension
 # https://github.com/wikimedia/mediawiki-extensions-Math.git
 
-require_once "$IP/extensions/Math/Math.php";
+wfLoadExtension( 'Math' );
 
-#############################################
+###########################################
-# Fail2banlog extension
+# Parsoid (bundled in MW 1.39, runs in-process)
-# https://www.mediawiki.org/wiki/Extension:Fail2banlog
+# Required for REST API /v1/page/{title}/with_html endpoint
 
-require_once "$IP/extensions/Fail2banlog/Fail2banlog.php";
+wfLoadExtension( 'Parsoid', "$IP/vendor/wikimedia/parsoid/extension.json" );
-$wgFail2banlogfile = "/var/log/apache2/mwf2b.log";
+$wgParsoidSettings = [
+    'useSelser' => true,
+];
 
 #############################################
 # Fix cookies crap
@@ -224,7 +232,7 @@ session_save_path("/tmp");
 ##############################################
 # Secure login
 
-$wgServer = "https://{{ server_name_default }}";
+$wgServer = "https://{{ pod_charlesreid1_server_name }}";
 $wgSecureLogin = true;
 
 ###################################

d-mediawiki/charlesreid1-config/mediawiki/build_extensions_dir.sh

@@ -1,93 +0,0 @@
-#!/bin/bash
-# 
-# clone or download each extension
-# and build o
-
-mkdir -p extensions
-(
-cd extensions
-
-##############################
-
-Extension="SyntaxHighlight_GeSHi"
-if [ ! -d ${Extension} ]
-then
-    ## This requires mediawiki > 1.31
-    ## (so does REL1_31)
-    #git clone https://github.com/wikimedia/mediawiki-extensions-SyntaxHighlight_GeSHi.git SyntaxHighlight_GeSHi
-
-    ## This manually downloads REL1_30
-    #wget https://extdist.wmflabs.org/dist/extensions/SyntaxHighlight_GeSHi-REL1_30-87392f1.tar.gz -O SyntaxHighlight_GeSHi.tar.gz
-    #tar -xzf SyntaxHighlight_GeSHi.tar.gz -C ${PWD}
-    #rm -f SyntaxHighlight_GeSHi.tar.gz
-
-    # Best of both worlds
-    git clone https://github.com/wikimedia/mediawiki-extensions-SyntaxHighlight_GeSHi.git SyntaxHighlight_GeSHi
-    (
-    cd ${Extension}
-    git checkout --track remotes/origin/REL1_34
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="ParserFunctions"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/wikimedia/mediawiki-extensions-ParserFunctions.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout --track remotes/origin/REL1_34
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="EmbedVideo"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/HydraWiki/mediawiki-embedvideo.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout v2.7.3
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="Math"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/wikimedia/mediawiki-extensions-Math.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout REL1_34
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="Fail2banlog"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/charlesreid1-docker/mw-fail2ban.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout master
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-# fin
-)

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/Bootstrap2.php.j2

@@ -23,12 +23,12 @@ class SkinBootstrap2 extends SkinTemplate {
         // cmr 05/08/2014
         $template = 'Bootstrap2Template';
 
-    function setupSkinUserCss( OutputPage $out ) {
+    // MW 1.39: Skin::setupSkinUserCss() was removed. initPage() is the
-        global $wgHandheldStyle;
+    // per-request hook that still receives OutputPage and runs before
+    // headElement is generated, so addStyle() calls land in the <head>.
+    public function initPage( OutputPage $out ) {
+        parent::initPage( $out );
 
-        parent::setupSkinUserCss( $out );
-
-        // Append to the default screen common & print styles...
         $out->addStyle( 'Bootstrap2/IE50Fixes.css', 'screen', 'lt IE 5.5000' );
         $out->addStyle( 'Bootstrap2/IE55Fixes.css', 'screen', 'IE 5.5000' );
         $out->addStyle( 'Bootstrap2/IE60Fixes.css', 'screen', 'IE 6' );
@@ -36,7 +36,6 @@ class SkinBootstrap2 extends SkinTemplate {
 
         $out->addStyle( 'Bootstrap2/rtl.css', 'screen', '', 'rtl' );
 
-
         $out->addStyle( 'Bootstrap2/bootstrap.css' );
         $out->addStyle( 'Bootstrap2/slate.css' );
         $out->addStyle( 'Bootstrap2/main.css' );
@@ -72,7 +71,8 @@ class Bootstrap2Template extends QuickTemplate {
 
         // -------- Start ------------
         // Adding the following line makes Geshi work
-        $this->html( 'headelement' );
+        // (MW 1.39: read $this->data directly to avoid QuickTemplate::html('headelement') deprecation)
+        echo $this->data['headelement'];
         // Left this out because the [edit] buttons were becoming right-aligned
         // Got around that behavior by changing shared.css
         // -------- End ------------
@@ -106,7 +106,7 @@ include('/var/www/html/skins/Bootstrap2/navbar.php');
             <div class="container-fixed">
                 <div class="navbar-header">
                     <a href="/wiki/" class="navbar-brand">
-                    {{ top_domain }} wiki
+                    {{ pod_charlesreid1_server_name }} wiki
                     </a>
                 </div>
                 <div>
@@ -146,7 +146,7 @@ include('/var/www/html/skins/Bootstrap2/navbar.php');
                                     echo ' ';
                                     echo $tab['class'];
                                 }
-                                echo '" id="' . Sanitizer::escapeId( "ca-$key" ) . '">';
+                                echo '" id="' . Sanitizer::escapeIdForAttribute( "ca-$key" ) . '">';
                                 echo '<a href="';
                                 echo htmlspecialchars($tab['href']);
                                 echo '">';
@@ -329,7 +329,7 @@ include('/var/www/html/skins/Bootstrap2/footer.php');
 <?php   }
         if($this->data['feeds']) { ?>
             <li id="feedlinks"><?php foreach($this->data['feeds'] as $key => $feed) {
-                    ?><a id="<?php echo Sanitizer::escapeId( "feed-$key" ) ?>" href="<?php
+                    ?><a id="<?php echo Sanitizer::escapeIdForAttribute( "feed-$key" ) ?>" href="<?php
                     echo htmlspecialchars($feed['href']) ?>" rel="alternate" type="application/<?php echo $key ?>+xml" class="feedlink"<?php echo $this->skin->tooltipAndAccesskey('feed-'.$key) ?>><?php echo htmlspecialchars($feed['text'])?></a>&nbsp;
                     <?php } ?></li><?php
         }
@@ -390,7 +390,7 @@ include('/var/www/html/skins/Bootstrap2/footer.php');
         }
 
         //wfRunHooks( 'BootstrapTemplateToolboxEnd', array( &$this ) );
-        wfRunHooks( 'BootstrapTemplateToolboxEnd', array( &$this ) );
+        Hooks::run( 'BootstrapTemplateToolboxEnd', array( &$this ) );
 ?>
             </ul>
 <!--
@@ -429,7 +429,7 @@ include('/var/www/html/skins/Bootstrap2/footer.php');
 
 <?php   if ( is_array( $cont ) ) { ?>
             <ul class="nav nav-list">
-                <li class="nav-header"><?php $out = wfMsg( $bar ); if (wfEmptyMsg($bar, $out)) echo htmlspecialchars($bar); else echo htmlspecialchars($out); ?></li>
+                <li class="nav-header"><?php $msg = wfMessage( $bar ); if ($msg->isDisabled()) echo htmlspecialchars($bar); else echo htmlspecialchars($msg->text()); ?></li>
 <?php           foreach($cont as $key => $val) { ?>
                 <li id="<?php echo Sanitizer::escapeId($val['id']) ?>"<?php
                     if ( $val['active'] ) { ?> class="active" <?php }

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/footer.php

@@ -11,7 +11,7 @@
     </span>
     Made from the command line with vim by 
     <a href="http://charlesreid1.com">charlesreid1</a><br />
-    with help from <a href="https://getbootstrap.com/">Bootstrap</a> and <a href="http://getpelican.com">Pelican</a>.
+    with help from <a href="https://getbootstrap.com/">Bootstrap</a> and <a href="http://mediawiki.org">MediaWiki</a>.
 </p>
 
 <p style="text-align: center">

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/main.css

@@ -518,8 +518,18 @@ a.new:visited {
         color: #a55858;
 }
 
-span.editsection {
+.mw-editsection, .editsection {
 	font-size: small;
+	font-weight: normal;
+	margin-left: 1em;
+}
+
+.editOptions {
+    background-color: #777;
+}
+
+.mw-editsection-bracket {
+    margin-left: 0;
 }
 
 #preftoc {

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/navbar.php.j2

@@ -6,14 +6,14 @@
                 <span class="icon-bar"></span>
                 <span class="icon-bar"></span> 
             </button>
-            <a href="/" class="navbar-brand">{{ top_domain }}</a>
+            <a href="/" class="navbar-brand">{{ pod_charlesreid1_server_name }}</a>
         </div>
         <div>
             <div class="collapse navbar-collapse" id="myNavbar">
                 <ul class="nav navbar-nav">
 
                     <li>
-                        <a href="https://{{ top_domain }}/wiki">Wiki</a>
+                        <a href="https://{{ pod_charlesreid1_server_name }}/wiki">Wiki</a>
                     </li>
 
                 </ul>

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/slate.css

@@ -1086,7 +1086,8 @@ html {
 }
 body {
   font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-  font-size: 14px;
+  /*font-size: 14px;*/
+  font-size: 20px;
   line-height: 1.42857143;
   color: #c8c8c8;
   background-color: #272b30;

d-mediawiki/php/php.ini

@@ -0,0 +1,6 @@
+post_max_size = 128M
+memory_limit = 128M
+upload_max_filesize = 100M
+display_errors = Off
+log_errors = On
+error_log = /var/log/apache2/php_errors.log

d-mysql/Dockerfile

@@ -1,11 +1,7 @@
-FROM mysql:5.7
+FROM mysql:8.0
 MAINTAINER charles@charlesreid1.com
 
 # make mysql data a volume
 VOLUME ["/var/lib/mysql"]
 
-# put password in a password file
-RUN printf "[client]\nuser=root\npassword=$MYSQL_ROOT_PASSWORD" > /root/.mysql.rootpw.cnf
-RUN chmod 0600 /root/.mysql.rootpw.cnf
-
 RUN chown mysql:mysql /var/lib/mysql

d-mysql/conf.d/slow-log.cnf

@@ -0,0 +1,5 @@
+[mysqld]
+slow_query_log = 1
+slow_query_log_file = /var/log/mysql/mysql-slow.log
+long_query_time = 2
+log_queries_not_using_indexes = 0

d-nginx-charlesreid1/conf.d/_.conf

@@ -1,6 +0,0 @@
-# https://serverfault.com/a/525011
-server {
-    server_name _;
-    listen *:80 default_server deferred;
-    return 444;
-}

d-nginx-charlesreid1/conf.d/http.DOMAIN.conf.j2

@@ -1,6 +1,6 @@
 ####################
 # 
-# {{ server_name_default }}
+# {{ pod_charlesreid1_server_name }}
 # http/{{ port_default }}
 # 
 # basically, just redirects to https
@@ -10,20 +10,20 @@
 server {
     listen 80;
     listen [::]:80;
-    server_name {{ server_name_default }};
+    server_name {{ pod_charlesreid1_server_name }};
-    return 301 https://{{ server_name_default }}$request_uri;
+    return 301 https://{{ pod_charlesreid1_server_name }}$request_uri;
 }
 
 server {
     listen 80;
     listen [::]:80;
-    server_name www.{{ server_name_default }};
+    server_name www.{{ pod_charlesreid1_server_name }};
-    return 301 https://www.{{ server_name_default }}$request_uri;
+    return 301 https://www.{{ pod_charlesreid1_server_name }}$request_uri;
 }
 
 server {
     listen 80;
     listen [::]:80;
-    server_name git.{{ server_name_default }};
+    server_name git.{{ pod_charlesreid1_server_name }};
-    return 301 https://git.{{ server_name_default }}$request_uri;
+    return 301 https://git.{{ pod_charlesreid1_server_name }}$request_uri;
 }

d-nginx-charlesreid1/conf.d/https.DOMAIN.conf.j2

@@ -1,9 +1,9 @@
 ####################
 #
-# {{ server_name_default }}
+# {{ pod_charlesreid1_server_name }}
 # https/443
 # 
-# {{ server_name_default }} and www.{{ server_name_default }}
+# {{ pod_charlesreid1_server_name }} and www.{{ pod_charlesreid1_server_name }}
 # should handle the following cases:
 # - w/ and wiki/ should reverse proxy story_mw
 # - gitea subdomain should reverse proxy stormy_gitea
@@ -15,30 +15,46 @@
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
-    server_name {{ server_name_default }} default_server;
+    server_name {{ pod_charlesreid1_server_name }};
 
-    ssl_certificate /etc/letsencrypt/live/{{ server_name_default }}/fullchain.pem;
+    ssl_certificate /etc/letsencrypt/live/{{ pod_charlesreid1_server_name }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/{{ server_name_default }}/privkey.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ pod_charlesreid1_server_name }}/privkey.pem;
     include /etc/letsencrypt/options-ssl-nginx.conf;
     include /etc/nginx/conf.d/secheaders.conf;
     include /etc/nginx/conf.d/csp.conf;
 
     location / {
         try_files $uri $uri/ =404;
-        root /www/{{ server_name_default }}/htdocs;
+        root /www/{{ pod_charlesreid1_server_name }}/htdocs;
         index index.html;
     }
 
+    location = /robots.txt {
+        alias /var/www/robots/robots.txt;
+    }
+
     location /wiki/ {
+        # Apply rate limit here.
+        limit_req zone=gitealimit burst=20 nodelay;
+        # Limit download rate to 500 KB/s per connection (4 Mbps)
+        limit_rate 500k;
+
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;
         proxy_pass http://stormy_mw:8989/wiki/;
     }
 
     location /w/ {
+        # Apply rate limit here.
+        limit_req zone=gitealimit burst=20 nodelay;
+        # Limit download rate to 500 KB/s per connection (4 Mbps)
+        limit_rate 500k;
+
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;
         proxy_pass http://stormy_mw:8989/w/;
     }
@@ -55,31 +71,43 @@ server {
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
-    server_name www.{{ server_name_default }};
+    server_name www.{{ pod_charlesreid1_server_name }};
 
-    ssl_certificate /etc/letsencrypt/live/www.{{ server_name_default }}/fullchain.pem;
+    ssl_certificate /etc/letsencrypt/live/www.{{ pod_charlesreid1_server_name }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/www.{{ server_name_default }}/privkey.pem;
+    ssl_certificate_key /etc/letsencrypt/live/www.{{ pod_charlesreid1_server_name }}/privkey.pem;
     include /etc/letsencrypt/options-ssl-nginx.conf;
     include /etc/nginx/conf.d/secheaders.conf;
     include /etc/nginx/conf.d/csp.conf;
 
-    root /www/{{ server_name_default }}/htdocs;
+    root /www/{{ pod_charlesreid1_server_name }}/htdocs;
 
     location / {
         try_files $uri $uri/ =404;
         index index.html;
     }
 
+    location = /robots.txt {
+        alias /var/www/robots/robots.txt;
+    }
+
     location /wiki/ {
+        limit_req zone=gitealimit burst=20 nodelay;
+        limit_rate 500k;
+
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;
         proxy_pass http://stormy_mw:8989/wiki/;
     }
 
     location /w/ {
+        # Apply rate limit here.
+        limit_req zone=gitealimit burst=20 nodelay;
+
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;
         proxy_pass http://stormy_mw:8989/w/;
     }
@@ -94,18 +122,29 @@ server {
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
-    server_name git.{{ server_name_default }};
+    server_name git.{{ pod_charlesreid1_server_name }};
 
-    ssl_certificate /etc/letsencrypt/live/git.{{ server_name_default }}/fullchain.pem;
+    ssl_certificate /etc/letsencrypt/live/git.{{ pod_charlesreid1_server_name }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/git.{{ server_name_default }}/privkey.pem;
+    ssl_certificate_key /etc/letsencrypt/live/git.{{ pod_charlesreid1_server_name }}/privkey.pem;
     include /etc/letsencrypt/options-ssl-nginx.conf;
     include /etc/nginx/conf.d/secheaders.conf;
     include /etc/nginx/conf.d/giteacsp.conf;
 
     location / {
+        # Apply the rate limit here.
+        # Allows a burst of 20 requests, but anything beyond the max is queued.
+        limit_req zone=gitealimit burst=20 nodelay;
+        # Limit download rate to 500 KB/s per connection (4 Mbps)
+        limit_rate 500k;
+
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;
         proxy_pass http://stormy_gitea:3000/;
     }
+
+    location = /robots.txt {
+        alias /var/www/robots/gitea.txt;
+    }
 }

d-nginx-charlesreid1/nginx.conf

@@ -0,0 +1,37 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log notice;
+pid        /run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+
+    # Gitea rate limiting: 
+    # 5 requests per second rate limit
+    limit_req_zone $binary_remote_addr zone=gitealimit:10m rate=5r/s;
+
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}
+

d-nginx-charlesreid1/robots/gitea.txt

@@ -0,0 +1,16 @@
+User-agent: *
+Disallow: */commit/*
+Disallow: */src/*
+Disallow: */tree/*
+Disallow: */activity/*
+Disallow: */wiki/*
+Disallow: */releases/*
+Disallow: */pulls/*
+Disallow: */stars
+Disallow: */watchers
+Disallow: */forks
+Disallow: *?tab=activity
+Disallow: *?tab=stars
+Disallow: *?tab=following
+Disallow: *?tab=followers
+Disallow: *?lang=*

d-nginx-charlesreid1/robots/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /w/

docker-compose.yml.j2

@@ -5,7 +5,7 @@ services:
 # https://stackoverflow.com/a/39039830
 
   stormy_gitea:
-    image: gitea/gitea:latest
+    image: gitea/gitea:1.24.5
     container_name: stormy_gitea
     environment:
       - USER_UID=1000
@@ -13,6 +13,7 @@ services:
     restart: always
     volumes:
       - "stormy_gitea_data:/data"
+      - "./d-nginx-charlesreid1/robots:/var/www/robots:ro"
       - "./d-gitea/custom:/data/gitea"
       - "./d-gitea/data:/app/gitea/data"
       - "/gitea_repositories:/data/git/repositories"
@@ -23,53 +24,91 @@ services:
         max-file: "10"
     ports:
       - "22:22"
+    networks:
+      - frontend
+
+  stormy_gitea_runner:
+    image: gitea/act_runner:latest
+    container_name: stormy_gitea_runner
+    restart: always
+    volumes:
+      - "stormy_gitea_runner_data:/data"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "./d-gitea/runner/config.yaml:/etc/act_runner/config.yaml:ro"
+    environment:
+      - GITEA_INSTANCE_URL=http://stormy_gitea:3000
+      - GITEA_RUNNER_REGISTRATION_TOKEN={{ pod_charlesreid1_gitea_runner_token }}
+      - GITEA_RUNNER_NAME=stormy-runner
+      - CONFIG_FILE=/etc/act_runner/config.yaml
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 1m
+        max-file: "10"
+    depends_on:
+      - stormy_gitea
+    networks:
+      - frontend
 
   stormy_mysql:
+    restart: always
     build: d-mysql
     container_name: stormy_mysql
     volumes:
       - "stormy_mysql_data:/var/lib/mysql"
+      - "./d-mysql/conf.d:/etc/mysql/conf.d:ro"
     logging:
       driver: "json-file"
       options:
         max-size: 1m
         max-file: "10"
     environment:
-      - MYSQL_ROOT_PASSWORD={{ mysql_password }}
+      - MYSQL_ROOT_PASSWORD={{ pod_charlesreid1_mysql_password }}
+      - MYSQL_DATABASE=wikidb
+      - MYSQL_USER=wikiuser
+      - MYSQL_PASSWORD={{ pod_charlesreid1_mysql_wikiuser_password }}
+    networks:
+      - backend
 
   stormy_mw:
+    restart: always
     build: d-mediawiki
     container_name: stormy_mw
     volumes:
-      - "stormy_mw_data:/var/www/html"
+      - "stormy_mw_images:/var/www/html/images"
-      - "./mwf2b:/var/log/mwf2b"
     logging:
       driver: "json-file"
       options:
         max-size: 1m
         max-file: "10"
     environment:
-      - MEDIAWIKI_SITE_SERVER=https://{{ server_name_default }}
+      - MEDIAWIKI_SITE_SERVER=https://{{ pod_charlesreid1_server_name }}
-      - MEDIAWIKI_SECRETKEY={{ mediawiki_secretkey }}
+      - MEDIAWIKI_SECRETKEY={{ pod_charlesreid1_mediawiki_secretkey }}
+      - MEDIAWIKI_UPGRADEKEY={{ pod_charlesreid1_mediawiki_upgradekey }}
       - MYSQL_HOST=stormy_mysql
       - MYSQL_DATABASE=wikidb
-      - MYSQL_USER=root
+      - MYSQL_USER=wikiuser
-      - MYSQL_PASSWORD={{ mysql_password }}
+      - MYSQL_PASSWORD={{ pod_charlesreid1_mysql_wikiuser_password }}
     depends_on:
       - stormy_mysql
+    networks:
+      - frontend
+      - backend
 
   stormy_nginx:
     restart: always
-    image: nginx
+    image: nginx:1.27.5
     container_name: stormy_nginx
-    hostname: {{ server_name_default }}
+    hostname: {{ pod_charlesreid1_server_name }}
-    hostname: charlesreid1.com
     command: /bin/bash -c "nginx -g 'daemon off;'"
     volumes:
+      - "./d-nginx-charlesreid1/nginx.conf:/etc/nginx/nginx.conf:ro"
       - "./d-nginx-charlesreid1/conf.d:/etc/nginx/conf.d:ro"
+      - "./d-nginx-charlesreid1/robots:/var/www/robots:ro"
       - "/etc/localtime:/etc/localtime:ro"
-      - "/etc/letsencrypt:/etc/letsencrypt"
+      - "/etc/letsencrypt:/etc/letsencrypt:ro"
-      - "/www/{{ server_name_default }}/htdocs:/www/{{ server_name_default }}/htdocs:ro"
+      - "/www/{{ pod_charlesreid1_server_name }}/htdocs:/www/{{ pod_charlesreid1_server_name }}/htdocs:ro"
+      - "stormy_nginx_logs:/var/log/nginx"
     logging:
       driver: "json-file"
       options:
@@ -82,8 +121,19 @@ services:
     ports:
       - "80:80"
       - "443:443"
+    networks:
+      - frontend
+
+networks:
+  frontend:
+  backend:
 
 volumes:
   stormy_mysql_data:
+  stormy_mw_images:
+    external: true
   stormy_mw_data:
+    external: true
   stormy_gitea_data:
+  stormy_gitea_runner_data:
+  stormy_nginx_logs:

docs/BlockIps.md

@@ -0,0 +1,9 @@
+To block IP address:
+
+* Modify the nginx config file template at 
+  `d-nginx-charlesreid1/conf.d/https.DOMAIN.conf.j2`
+* Re-render the Jinja templates into config files via
+  `make clean-templates && make templates`
+* Stop and restart the pod service:
+  `sudo systemctl stop pod-charlesreid1 && 
+  sudo systemctl start pod-charlesreid1`

environment.example

@@ -2,31 +2,36 @@
 
 # multiple templates:
 # -------------------
-POD_CHARLESREID1_DIR="/path/to/pod-charlesreid1"
+export POD_CHARLESREID1_DIR="/path/to/pod-charlesreid1"
-POD_CHARLESREID1_TLD="example.com"
+export POD_CHARLESREID1_TLD="example.com"
+export POD_CHARLESREID1_USER="nonrootuser"
+export POD_CHARLESREID1_VPN_IP_ADDR="1.2.3.4"
 
 # mediawiki:
 # ----------
-POD_CHARLESREID1_MW_ADMIN_EMAIL="email@example.com"
+export POD_CHARLESREID1_MW_ADMIN_EMAIL="email@example.com"
-POD_CHARLESREID1_MW_SECRET_KEY="SecretKeyString"
+export POD_CHARLESREID1_MW_SECRET_KEY="SecretKeyString"
+export POD_CHARLESREID1_MW_UPGRADE_KEY="UpgradeKeyString"
 
 # mysql:
 # ------
-POD_CHARLESREID1_MYSQL_PASSWORD="SuperSecretPassword"
+export POD_CHARLESREID1_MYSQL_PASSWORD="SuperSecretPassword"
+export POD_CHARLESREID1_MYSQL_WIKIUSER_PASSWORD="AnotherSecretPassword"
 
 # gitea:
 # ------
-POD_CHARLESREID1_GITEA_APP_NAME=""
+export POD_CHARLESREID1_GITEA_APP_NAME=""
-POD_CHARLESREID1_GITEA_SECRET_KEY="GiteaSecretKey"
+export POD_CHARLESREID1_GITEA_SECRET_KEY="GiteaSecretKey"
-POD_CHARLESREID1_GITEA_INTERNAL_TOKEN="GiteaInternalToken"
+export POD_CHARLESREID1_GITEA_INTERNAL_TOKEN="GiteaInternalToken"
 
 # aws:
 # ----
-POD_CHARLESREID1_AWS_ACCESS_KEY="AAAAAAAAAAAAAAAAAAAA"
+export AWS_ACCESS_KEY_ID="AAAAAAA"
-POD_CHARLESREID1_AWS_ACCESS_SECRET="0000000000000000000000000000000000000000"
+export AWS_SECRET_ACCESS_KEY="BBBBBBBB"
+export AWS_DEFAULT_REGION="us-west-1"
 
 # backups and scripts:
 # --------------------
-POD_CHARLESREID1_USER="charles"
+export POD_CHARLESREID1_BACKUP_DIR="/path/to"
-POD_CHARLESREID1_BACKUP_S3BUCKET="name-of-backups-bucket"
+export POD_CHARLESREID1_BACKUP_S3BUCKET="name-of-backups-bucket"
-POD_CHARLESREID1_BACKUPCANARY_WEBHOOKURL="https://hooks.slack.com/services/000000000/AAAAAAAAA/111111111111111111111111"
+export POD_CHARLESREID1_CANARY_WEBHOOK="https://hooks.slack.com/services/000000000/AAAAAAAAA/111111111111111111111111"

environment.j2

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# multiple templates:
+# -------------------
+export POD_CHARLESREID1_DIR="{{ pod_charlesreid1_pod_install_dir }}"
+export POD_CHARLESREID1_TLD="{{ pod_charlesreid1_server_name }}"
+export POD_CHARLESREID1_USER="{{ pod_charlesreid1_username }}"
+export POD_CHARLESREID1_VPN_IP_ADDR="{{ pod_charlesreid1_vpn_ip_addr }}"
+
+# mediawiki:
+# ----------
+export POD_CHARLESREID1_MW_ADMIN_EMAIL="{{ pod_charlesreid1_mediawiki_admin_email }}"
+export POD_CHARLESREID1_MW_SECRET_KEY="{{ pod_charlesreid1_mediawiki_secretkey }}"
+export POD_CHARLESREID1_MW_UPGRADE_KEY="{{ pod_charlesreid1_mediawiki_upgradekey }}"
+
+# mysql:
+# ------
+export POD_CHARLESREID1_MYSQL_PASSWORD="{{ pod_charlesreid1_mysql_password }}"
+export POD_CHARLESREID1_MYSQL_WIKIUSER_PASSWORD="{{ pod_charlesreid1_mysql_wikiuser_password }}"
+
+# gitea:
+# ------
+export POD_CHARLESREID1_GITEA_APP_NAME="{{ pod_charlesreid1_gitea_app_name }}"
+export POD_CHARLESREID1_GITEA_SECRET_KEY="{{ pod_charlesreid1_gitea_secretkey }}"
+export POD_CHARLESREID1_GITEA_INTERNAL_TOKEN="{{ pod_charlesreid1_gitea_internaltoken }}"
+
+# aws:
+# ----
+export AWS_ACCESS_KEY_ID="{{ pod_charlesreid1_backups_aws_access_key }}"
+export AWS_SECRET_ACCESS_KEY="{{ pod_charlesreid1_backups_aws_secret_access_key }}"
+export AWS_DEFAULT_REGION="{{ pod_charlesreid1_backups_aws_region }}"
+
+# backups and scripts:
+# --------------------
+export POD_CHARLESREID1_BACKUP_DIR="{{ pod_charlesreid1_backups_dir }}"
+export POD_CHARLESREID1_BACKUP_S3BUCKET="{{ pod_charlesreid1_backups_bucket }}"
+export POD_CHARLESREID1_CANARY_WEBHOOK="{{ pod_charlesreid1_backups_canary_slack_url }}"

scripts/Readme.md

@@ -21,10 +21,12 @@ Cleans all rendered Jinja templates. Does not require environment variables.
 This script is destructive! Be careful!
 
 
-# Ansible Scripts
+# /www Directory Scripts
 
-These scripts are used by ansible when setting up a machine
+These scripts set up or pull a git repo that is set up to
-to run the charlesreid1 docker pod.
+have a pecular directory structure.
+
+The clone script is used by Ansible when setting up this pod.
 
 ## `git_clone_www.py`
 

scripts/apply_templates.py

@@ -2,36 +2,43 @@ import os
 import re
 import sys
 import glob
+import time
+import subprocess
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 
-"""
-Apply Default Values to all Jinja Templates
-"""
-
 
 # Should existing files be overwritten
 OVERWRITE = True
 
+# Map of jinja variables to environment variables
+jinja_to_env = {
+    "pod_charlesreid1_pod_install_dir": "POD_CHARLESREID1_DIR",
+    "pod_charlesreid1_server_name": "POD_CHARLESREID1_TLD",
+    "pod_charlesreid1_username": "POD_CHARLESREID1_USER",
+    "pod_charlesreid1_vpn_ip_addr": "POD_CHARLESREID1_VPN_IP_ADDR",
+    "pod_charlesreid1_mediawiki_admin_email": "POD_CHARLESREID1_MW_ADMIN_EMAIL",
+    "pod_charlesreid1_mediawiki_secretkey": "POD_CHARLESREID1_MW_SECRET_KEY",
+    "pod_charlesreid1_mediawiki_upgradekey": "POD_CHARLESREID1_MW_UPGRADE_KEY",
+    "pod_charlesreid1_mysql_password": "POD_CHARLESREID1_MYSQL_PASSWORD",
+    "pod_charlesreid1_mysql_wikiuser_password": "POD_CHARLESREID1_MYSQL_WIKIUSER_PASSWORD",
+    "pod_charlesreid1_gitea_app_name": "POD_CHARLESREID1_GITEA_APP_NAME",
+    "pod_charlesreid1_gitea_secretkey": "POD_CHARLESREID1_GITEA_SECRET_KEY",
+    "pod_charlesreid1_gitea_internaltoken": "POD_CHARLESREID1_GITEA_INTERNAL_TOKEN",
+    "pod_charlesreid1_gitea_runner_token": "POD_CHARLESREID1_GITEA_RUNNER_TOKEN",
+    "pod_charlesreid1_backups_aws_access_key": "AWS_ACCESS_KEY_ID",
+    "pod_charlesreid1_backups_aws_secret_access_key": "AWS_SECRET_ACCESS_KEY",
+    "pod_charlesreid1_backups_aws_region": "AWS_DEFAULT_REGION",
+    "pod_charlesreid1_backups_dir": "POD_CHARLESREID1_BACKUP_DIR",
+    "pod_charlesreid1_backups_bucket": "POD_CHARLESREID1_BACKUP_S3BUCKET",
+    "pod_charlesreid1_backups_canary_slack_url": "POD_CHARLESREID1_CANARY_WEBHOOK",
+}
 
 scripts_dir = os.path.dirname(os.path.abspath(__file__))
 repo_root = os.path.abspath(os.path.join(scripts_dir, '..'))
 
 
 def check_env_vars():
-    env_var_list = [
+    env_var_list = jinja_to_env.values()
-        'POD_CHARLESREID1_DIR',
-        'POD_CHARLESREID1_TLD',
-        'POD_CHARLESREID1_USER',
-        'POD_CHARLESREID1_MYSQL_PASSWORD',
-        'POD_CHARLESREID1_MW_ADMIN_EMAIL',
-        'POD_CHARLESREID1_GITEA_APP_NAME',
-        'POD_CHARLESREID1_GITEA_SECRET_KEY',
-        'POD_CHARLESREID1_GITEA_INTERNAL_TOKEN',
-        'POD_CHARLESREID1_BACKUP_S3BUCKET',
-        'POD_CHARLESREID1_AWS_ACCESS_KEY',
-        'POD_CHARLESREID1_AWS_ACCESS_SECRET',
-        'POD_CHARLESREID1_BACKUPCANARY_WEBHOOKURL',
-    ]
     nerrs = 0
     print("Checking environment variables")
     for env_var in env_var_list:
@@ -48,6 +55,8 @@ def main():
 
     check_env_vars()
 
+    ignore_list = ['environment']
+
     p = os.path.join(repo_root,'**','*.j2')
     template_files = glob.glob(p, recursive=True)
 
@@ -63,41 +72,35 @@ def main():
         rname = tname[:-3]
         rpath = os.path.join(tdir, rname)
 
+        if rname in ignore_list:
+            print(f"\nSkipping template on ignore list: {tname}\n")
+            continue
+
         env = Environment(loader=FileSystemLoader(tdir))
     
         print(f"Rendering template {tname}:")
         print(f"    Template path: {tpath}")
         print(f"    Output path: {rpath}")
-        #content = env.get_template(tpath).render({
+
-        content = env.get_template(tname).render({
+        jinja_vars = {}
-            "pod_install_dir": os.environ['POD_CHARLESREID1_DIR'],
+        for k, v in jinja_to_env.items():
-            "top_domain": os.environ['POD_CHARLESREID1_TLD'],
+            jinja_vars[k] = os.environ[v]
-            "server_name_default" : os.environ['POD_CHARLESREID1_TLD'],
+
-            "username": os.environ['POD_CHARLESREID1_USER'],
+        content = env.get_template(tname).render(jinja_vars)
-            # docker-compose:
-            "mysql_password" : os.environ['POD_CHARLESREID1_MYSQL_PASSWORD'],
-            "mediawiki_secretkey" : os.environ['POD_CHARLESREID1_MW_ADMIN_EMAIL'],
-            # mediawiki:
-            "admin_email": os.environ['POD_CHARLESREID1_MW_ADMIN_EMAIL'],
-            # gitea:
-            "gitea_app_name": os.environ['POD_CHARLESREID1_GITEA_APP_NAME'],
-            "gitea_secret_key": os.environ['POD_CHARLESREID1_GITEA_SECRET_KEY'],
-            "gitea_internal_token": os.environ['POD_CHARLESREID1_GITEA_INTERNAL_TOKEN'],
-            # aws:
-            "aws_backup_s3_bucket": os.environ['POD_CHARLESREID1_BACKUP_S3BUCKET'],
-            "aws_access_key": os.environ['POD_CHARLESREID1_AWS_ACCESS_KEY'],
-            "aws_access_secret": os.environ['POD_CHARLESREID1_AWS_ACCESS_SECRET'],
-            "backup_canary_webhook_url": os.environ['POD_CHARLESREID1_BACKUPCANARY_WEBHOOKURL'],
-        })
     
         # Write to file
         if os.path.exists(rpath) and not OVERWRITE:
-            raise Exception("Error: file %s already exists!"%(rpath))
+            msg = "\n[!!!] Warning: file %s already exists! Skipping...\n"%(rpath)
+            print(msg)
+            time.sleep(1)
         else:
             with open(rpath,'w') as f:
                 f.write(content)
             print(f"    Done!")
             print("")
 
+        if rpath[-3:] == ".sh":
+            subprocess.call(['chmod', '+x', rpath])
+    
 if __name__=="__main__":
     main()

scripts/backups/10-pod-charlesreid1-rsyslog.conf

@@ -0,0 +1,28 @@
+if ( $programname startswith "pod-charlesreid1-canary" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-canary.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-certbot" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-certbot.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-aws" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-aws.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-cleanolderthan" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-cleanolderthan.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-gitea" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-gitea.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-wikidb" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-wikidb.service.log" flushOnTXEnd="on")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-wikifiles" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-wikifiles.service.log" flushOnTXEnd="on")
+    stop
+}

scripts/backups/Readme.md

@@ -0,0 +1,52 @@
+# backup scripts
+
+This directory contains several files for several services:
+
+* Systemd .service file (Jinja template) to define a service that backs up files
+* Systemd .timer file (Jinja template) to define a timer that runs the service on a schedule
+* Shell script .sh that actually performs the backup operation and is called by the .service file
+
+Use `make templates` in the top level of this repo to render
+the Jinja templates using the environment variables in the
+evnrionment file. That fixes the locations of the scripts
+for the systemd service.
+
+Use `make install` in the top level of this repo to install
+the rendered service and timer files.
+
+## syslog filtering
+
+Due to a bug in systemd bundled with Ubuntu 18.04, we can't just use the nice easy solution of
+directing output and error to a specific file.
+
+Instead, the services all send their stderr and stdout to the system log, and then rsyslog
+filters those messages and collects them into a separate log file.
+
+First, install the services.
+
+Then, install the following rsyslog config file:
+
+`/etc/rsyslog.d/10-pod-charlesreid1-rsyslog.conf`:
+
+```
+if $programname == 'pod-charlesreid1-canary' then /var/log/pod-charlesreid1-canary.service.log
+if $programname == 'pod-charlesreid1-canary' then stop
+
+if $programname == 'pod-charlesreid1-backups-aws' then /var/log/pod-charlesreid1-backups-aws.service.log
+if $programname == 'pod-charlesreid1-backups-aws' then stop
+
+if $programname == 'pod-charlesreid1-backups-cleanolderthan' then /var/log/pod-charlesreid1-backups-cleanolderthan.service.log
+if $programname == 'pod-charlesreid1-backups-cleanolderthan' then stop
+
+if $programname == 'pod-charlesreid1-backups-gitea' then /var/log/pod-charlesreid1-backups-gitea.service.log
+if $programname == 'pod-charlesreid1-backups-gitea' then stop
+
+if $programname == 'pod-charlesreid1-backups-wikidb' then /var/log/pod-charlesreid1-backups-wikidb.service.log
+if $programname == 'pod-charlesreid1-backups-wikidb' then stop
+
+if $programname == 'pod-charlesreid1-backups-wikifiles' then /var/log/pod-charlesreid1-backups-wikifiles.service.log
+if $programname == 'pod-charlesreid1-backups-wikifiles' then stop
+```
+
+
+

scripts/backups/aws_backup.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+#
+# Find the last backup created, and copy it
+# to an S3 bucket.
+set -eux
+
+function usage {
+    set +x
+    echo ""
+    echo "aws_backup.sh script:"
+    echo ""
+    echo "Find the last backup that was created,"
+    echo "and copy it to the backups bucket."
+    echo ""
+    echo "       ./aws_backup.sh"
+    echo ""
+    exit 1;
+}
+
+if [ "$(id -u)" == "0" ]; then
+    echo ""
+    echo ""
+    echo "This script should NOT be run as root!"
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+if [ "$#" == "0" ]; then
+
+    echo ""
+    echo "pod-charlesreid1: aws_backup.sh"
+    echo "-----------------------------------"
+    echo ""
+    echo "Backup directory: ${POD_CHARLESREID1_BACKUP_DIR}"
+    echo "Backup bucket: ${POD_CHARLESREID1_BACKUP_S3BUCKET}"
+    echo ""
+
+    echo "Checking that directory exists"
+    /usr/bin/test -d "${POD_CHARLESREID1_BACKUP_DIR}"
+
+    echo "Checking that we can access the S3 bucket"
+    aws s3 ls "s3://${POD_CHARLESREID1_BACKUP_S3BUCKET}" > /dev/null
+
+    # Get name of last backup, to copy to AWS
+    LAST_BACKUP=$(/bin/ls -1 -t "${POD_CHARLESREID1_BACKUP_DIR}" | /usr/bin/head -n1)
+    echo "Last backup found: ${LAST_BACKUP}"
+    echo "Last backup directory: ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP}"
+
+    BACKUP_SIZE=$(/usr/bin/du -hs "${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP}" | cut -f 1)
+    echo "Backup directory size: ${BACKUP_SIZE}"
+
+    # Copy to AWS
+    echo "Backing up directory ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP}"
+    aws s3 cp --only-show-errors --no-progress --recursive "${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP}" "s3://${POD_CHARLESREID1_BACKUP_S3BUCKET}/backups/${LAST_BACKUP}"
+    echo "Done."
+
+else
+    usage
+fi

scripts/backups/canary/backups_canary.py

@@ -0,0 +1,124 @@
+import os
+import sys
+import json
+import requests
+import boto3
+import botocore
+import subprocess
+
+
+webhook_url = os.environ['POD_CHARLESREID1_CANARY_WEBHOOK']
+backup_dir = os.environ['POD_CHARLESREID1_BACKUP_DIR']
+backup_bucket = os.environ['POD_CHARLESREID1_BACKUP_S3BUCKET']
+
+
+# Check for backups created in the last N days
+N = 7
+
+
+def main():
+    # verify the backups directory exists
+    if not os.path.exists(backup_dir):
+        msg = "Local Backups Error:\n"
+        msg += f"The backup directory `{backup_dir}` does not exist!"
+        alert(msg)
+
+    # verify there is a backup newer than N days
+    newer_backups = subprocess.getoutput(f'find {backup_dir}/* -mtime -{N}').split('\n')
+    if len(newer_backups)==1 and newer_backups[0]=='':
+        msg = "Local Backups Error:\n"
+        msg += f"The backup directory `{backup_dir}` is missing backup files from the last {N} day(s)!"
+        alert(msg)
+
+    newest_backup_name = subprocess.getoutput(f'ls -t {backup_dir} | head -n1')
+    newest_backup_path = os.path.join(backup_dir, newest_backup_name)
+    newest_backup_files = subprocess.getoutput(f'find {newest_backup_path} -type f').split('\n')
+
+    # verify the most recent backup directory is not empty
+    if len(newest_backup_files)==1 and newest_backup_files[0]=='':
+        msg = "Local Backups Error:\n"
+        msg += f"The most recent backup directory `{newest_backup_path}` is empty!"
+        alert(msg)
+
+    # verify the most recent backup files have nonzero size
+    for backup_file in newest_backup_files:
+        if os.path.getsize(backup_file)==0:
+            msg = "Local Backups Error:\n"
+            msg += f"The most recent backup directory `{newest_backup_path}` contains an empty backup file!\n"
+            msg += f"Backup file name: {backup_file}!"
+            alert(msg)
+
+    # verify .sql dumps end with the mysqldump completion trailer.
+    # A non-empty file can still be truncated mid-row (e.g. PTY deadlock,
+    # net_write_timeout) — without this check, a 439 MB partial dump looks
+    # healthy to a size-only canary.
+    for backup_file in newest_backup_files:
+        if not backup_file.endswith('.sql'):
+            continue
+        with open(backup_file, 'rb') as f:
+            f.seek(0, os.SEEK_END)
+            f.seek(max(0, f.tell() - 512))
+            tail = f.read()
+        if b'Dump completed on' not in tail:
+            msg = "Local Backups Error:\n"
+            msg += f"SQL backup file `{backup_file}` is missing the "
+            msg += "`-- Dump completed on ...` trailer.\n"
+            msg += "mysqldump did not finish — the dump is truncated and not restorable."
+            alert(msg)
+
+    # verify the most recent backup files exist in the s3 backups bucket
+    bucket_base_path = os.path.join('backups', newest_backup_name)
+    for backup_file in newest_backup_files:
+        backup_name = os.path.basename(backup_file)
+        backup_bucket_path = os.path.join(bucket_base_path, backup_name)
+        check_exists(backup_bucket, backup_bucket_path)
+
+def check_exists(bucket_name, bucket_path):
+    s3 = boto3.resource('s3')
+    try:
+        s3.Object(bucket_name, bucket_path).load()
+    except botocore.exceptions.ClientError as e:
+        if e.response['Error']['Code'] == "404":
+            # File does not exist
+            msg = "S3 Backups Error:\n"
+            msg += f"Failed to find the file `{bucket_path}` in bucket `{bucket_name}`"
+            alert(msg)
+        else:
+            # Problem accessing backups on bucket
+            msg = "S3 Backups Error:\n"
+            msg += f"Failed to access the file `{bucket_path}` in bucket `{bucket_name}`"
+            alert(msg)
+
+
+def alert(msg):
+    title = ":bangbang: pod-charlesreid1 backups canary"
+    hostname = subprocess.getoutput('hostname')
+    msg += f"\n\nHost: {hostname}"
+    slack_data = {
+        "username": "backups_canary",
+        "channel" : "#alerts",
+        "attachments": [
+            {
+                "color": "#CC0000",
+                "fields": [
+                    {
+                        "title": title,
+                        "value": msg,
+                        "short": "false",
+                    }
+                ]
+            }
+        ]
+    }
+    byte_length = str(sys.getsizeof(slack_data))
+    headers = {'Content-Type': "application/json", 'Content-Length': byte_length}
+    response = requests.post(webhook_url, data=json.dumps(slack_data), headers=headers)
+    if response.status_code != 200:
+        raise Exception(response.status_code, response.text)
+
+    print("Goodbye.")
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

scripts/backups/canary/pod-charlesreid1-canary.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Backup canary service for pod-charlesreid1
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+StandardError=syslog
+StandardOutput=syslog
+SyslogIdentifier=pod-charlesreid1-canary
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; /home/charles/.pyenv/shims/python3 {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/canary/backups_canary.py'
+User=charles
+Group=charles

scripts/backups/canary/pod-charlesreid1-canary.timer.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=Timer to run the pod-charlesreid1 backups canary
+
+[Timer]
+OnCalendar=*-*-* 7:01:00
+
+[Install]
+WantedBy=timers.target

scripts/backups/canary/requirements.txt

@@ -0,0 +1,3 @@
+boto3
+botocore
+requests

scripts/backups/clean_olderthan.sh

@@ -2,13 +2,24 @@
 #
 # Clean any files older than N days
 # from the backup directory.
-set -eu
+set -eux
 
 # Number of days of backups to retain.
 # Everything older than this many days will be deleted
-N="30"
+N="22"
 
-BACKUP_DIR="$HOME/backups"
+function usage {
+    set +x
+    echo ""
+    echo "clean_olderthan.sh script:"
+    echo ""
+    echo "Clean files older than ${N} days from the"
+    echo "backups directory, ~/backups"
+    echo ""
+    echo "       ./clean_olderthan.sh"
+    echo ""
+    exit 1;
+}
 
 if [ "$(id -u)" == "0" ]; then
     echo ""
@@ -21,8 +32,21 @@ fi
 
 if [ "$#" == "0" ]; then
 
-    echo "Cleaning backups directory $BACKUP_DIR"
+    echo ""
-    echo "Files older than $N days will be deleted"
+    echo "pod-charlesreid1: clean_olderthan.sh"
-    find $BACKUP_DIR -mtime +${N} -delete
+    echo "------------------------------------"
+    echo ""
+    echo "Backup directory: ${POD_CHARLESREID1_BACKUP_DIR}"
+    echo ""
 
+    echo "Cleaning backups directory $POD_CHARLESREID1_BACKUP_DIR"
+    echo "The following files older than $N days will be deleted:"
+    find ${POD_CHARLESREID1_BACKUP_DIR} -mtime +${N}
+
+    echo "Deleting files"
+    find ${POD_CHARLESREID1_BACKUP_DIR} -mtime +${N} -delete
+    echo "Done"
+
+else
+    usage
 fi

scripts/backups/gitea_backup.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# Bcak up the Gitea custom/ and data/ directories.
+# These are needed to restore the site
+# (as well as repository data, which is not backed up
+# by this script, it is a separate drive).
+set -eux
+
+CONTAINER_NAME="stormy_gitea"
+STAMP="`date +"%Y%m%d"`"
+
+
+function usage {
+    set +x
+    echo ""
+    echo "gitea_backup.sh script:"
+    echo ""
+    echo "Create a tar file containing gitea"
+    echo "custom/ and data/ directories."
+    echo ""
+    echo "       ./gitea_backup.sh"
+    echo ""
+    exit 1;
+}
+
+if [ "$(id -u)" == "0" ]; then
+    echo ""
+    echo ""
+    echo "This script should NOT be run as root!"
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+if [ "$#" == "0" ]; then
+
+    CUSTOM_NAME="gitea_custom_${STAMP}.tar.gz"
+    DATA_NAME="gitea_data_${STAMP}.tar.gz"
+
+    CUSTOM_TARGET="${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}/${CUSTOM_NAME}"
+    DATA_TARGET="${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}/${DATA_NAME}"
+
+    echo ""
+    echo "pod-charlesreid1: gitea_backup.sh"
+    echo "-----------------------------------"
+    echo ""
+    echo "Backup target: custom: ${CUSTOM_TARGET}"
+    echo "Backup target: data: ${DATA_TARGET}"
+    echo ""
+
+    mkdir -p ${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}
+
+    # We don't need to use docker, since these directories
+    # are both bind-mounted into the Docker container
+    echo "Backing up custom directory"
+    tar --exclude='gitea.log' --ignore-failed-read -czf ${CUSTOM_TARGET} ${POD_CHARLESREID1_DIR}/d-gitea/custom
+    echo "Backing up data directory"
+    tar czf ${DATA_TARGET} ${POD_CHARLESREID1_DIR}/d-gitea/data
+
+    echo "Done."
+
+else
+    usage
+fi

scripts/backups/gitea_dump.sh

@@ -1,86 +0,0 @@
-#!/bin/bash
-#
-# Run the gitea dump command and send the dump file
-# to the specified backup directory.
-#
-# Backup directory:
-#       /home/user/backups/gitea
-
-BACKUP_DIR="$HOME/backups/gitea"
-CONTAINER_NAME="stormy_gitea"
-
-function usage {
-    set +x
-    echo ""
-    echo "gitea_dump.sh script:"
-    echo ""
-    echo "Run the gitea dump command inside the gitea docker container,"
-    echo "and copy the resulting zip file to the specified directory."
-    echo "The resulting gitea dump zip file will be timestamped."
-    echo ""
-    echo "       ./gitea_dump.sh"
-    echo ""
-    echo "Example:"
-    echo ""
-    echo "       ./gitea_dump.sh"
-    echo "       (creates ${BACKUP_DIR}/gitea-dump_20200101_000000.zip)"
-    echo ""
-    exit 1;
-}
-
-if [ "$(id -u)" == "0" ]; then
-    echo ""
-    echo ""
-    echo "This script should NOT be run as root!"
-    echo ""
-    echo ""
-    exit 1;
-fi
-
-if [ "$#" == "0" ]; then
-
-    STAMP="`date +"%Y-%m-%d"`"
-    TARGET="gitea-dump_${STAMP}.zip"
-
-    echo ""
-    echo "pod-charlesreid1: gitea_dump.sh"
-    echo "-------------------------------"
-    echo ""
-    echo "Backup target: ${BACKUP_DIR}/${TARGET}"
-    echo ""
-
-    mkdir -p $BACKUP_DIR
-
-    ## If this script is being run from a cron job,
-    ## don't use -i flag with docker
-    #CRON="$( pstree -s $$ | /bin/grep -c cron )"
-    #DOCKER="/usr/local/bin/docker"
-    #DOCKERX=""
-    #if [[ "$CRON" -eq 1 ]]; 
-    #then
-    #    DOCKERX="${DOCKER} exec -t"
-    #else
-    #    DOCKERX="${DOCKER} exec -it"
-    #fi
-    DOCKER="/usr/local/bin/docker"
-    DOCKERX="${DOCKER} exec -t"
-
-    echo "Step 1: Run gitea dump command inside docker machine"
-    set -x
-    ${DOCKERX} --user git ${CONTAINER_NAME} /bin/bash -c 'cd /app/gitea && /app/gitea/gitea dump --file gitea-dump.zip --skip-repository'
-    set +x
-
-    echo "Step 2: Copy gitea dump file out of docker machine"
-    set -x
-    ${DOCKER} cp ${CONTAINER_NAME}:/app/gitea/gitea-dump.zip ${BACKUP_DIR}/${TARGET}
-    set +x
-
-    echo "Step 3: Clean up gitea dump file"
-    set -x
-    ${DOCKERX} ${CONTAINER_NAME} /bin/bash -c "rm -f /app/gitea/gitea-dump.zip"
-    set +x
-
-    echo "Done."
-else
-    usage
-fi

scripts/backups/pod-charlesreid1-backups-aws.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Copy the latest pod-charlesreid1 backup to an S3 bucket
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+StandardError=syslog
+StandardOutput=syslog
+SyslogIdentifier=pod-charlesreid1-backups-aws
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/aws_backup.sh'
+User=charles
+Group=charles

scripts/backups/pod-charlesreid1-backups-aws.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Timer to copy the lastest pod-charlesreid1 backup to an S3 bucket
+
+[Timer]
+OnCalendar=Sun *-*-* 2:56:00
+#OnCalendar=*-*-* 2:56:00
+
+[Install]
+WantedBy=timers.target

scripts/backups/pod-charlesreid1-backups-cleanolderthan.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Clean pod-charlesreid1 backups older than N days
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+StandardError=syslog
+StandardOutput=syslog
+SyslogIdentifier=pod-charlesreid1-backups-cleanolderthan
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/clean_olderthan.sh'
+User=charles
+Group=charles

scripts/backups/pod-charlesreid1-backups-cleanolderthan.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Timer to clean files older than N days from the pod-charlesreid1 backups dir
+
+[Timer]
+OnCalendar=Sun *-*-* 2:28:00
+#OnCalendar=*-*-* 2:28:00
+
+[Install]
+WantedBy=timers.target

scripts/backups/pod-charlesreid1-backups-gitea.service.j2

@@ -5,8 +5,10 @@ After=docker.service
 
 [Service]
 Type=oneshot
-StandardError={{ pod_install_dir }}/.pod-charlesreid1-backups-gitea.service.error.log
+StandardError=syslog
-StandardOutput={{ pod_install_dir }}/.pod-charlesreid1-backups-gitea.service.output.log
+StandardOutput=syslog
-ExecStart={{ pod_install_dir }}/scripts/backups/gitea_dump.sh
+SyslogIdentifier=pod-charlesreid1-backups-gitea
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/gitea_backup.sh'
 User=charles
 Group=charles

scripts/backups/pod-charlesreid1-backups-gitea.timer.j2

@@ -2,8 +2,8 @@
 Description=Timer to back up pod-charlesreid1 gitea files
 
 [Timer]
-OnCalendar=*-*-* 0/2:23:00
+OnCalendar=Sun *-*-* 2:12:00
+#OnCalendar=*-*-* 2:12:00
 
 [Install]
 WantedBy=timers.target
-

scripts/backups/pod-charlesreid1-backups-wikidb.service.j2

@@ -5,9 +5,10 @@ After=docker.service
 
 [Service]
 Type=oneshot
-StandardError={{ pod_install_dir }}/.pod-charlesreid1-backups-wikidb.service.error.log
+StandardError=syslog
-StandardOutput={{ pod_install_dir }}/.pod-charlesreid1-backups-wikidb.service.output.log
+StandardOutput=syslog
-ExecStart={{ pod_install_dir }}/scripts/backups/wikidb_dump.sh
+SyslogIdentifier=pod-charlesreid1-backups-wikidb
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/wikidb_dump.sh'
 User=charles
 Group=charles
-

scripts/backups/pod-charlesreid1-backups-wikidb.timer.j2

@@ -2,7 +2,7 @@
 Description=Timer to back up the pod-charlesreid1 wiki database
 
 [Timer]
-OnCalendar=*-*-* 0/2:03:00
+OnCalendar=Sun *-*-* 2:02:00
 
 [Install]
 WantedBy=timers.target

scripts/backups/pod-charlesreid1-backups-wikifiles.service.j2

@@ -1,12 +1,14 @@
 [Unit]
-Description=Back up the pod-charlesreid1 wiki database
+Description=Back up pod-charlesreid1 wiki files
 Requires=docker.service
 After=docker.service
 
 [Service]
 Type=oneshot
-StandardError={{ pod_install_dir }}/.pod-charlesreid1-backups-wikifiles.service.error.log
+StandardError=syslog
-StandardOutput={{ pod_install_dir }}/.pod-charlesreid1-backups-wikifiles.service.output.log
+StandardOutput=syslog
-ExecStart={{ pod_install_dir }}/scripts/backups/wikifiles_dump.sh
+SyslogIdentifier=pod-charlesreid1-backups-wikifiles
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/wikifiles_dump.sh'
 User=charles
 Group=charles

scripts/backups/pod-charlesreid1-backups-wikifiles.timer.j2

@@ -1,9 +1,8 @@
 [Unit]
-Description=Timer to back up the pod-charlesreid1 wiki database
+Description=Timer to back up pod-charlesreid1 wiki files
 
 [Timer]
-OnCalendar=*-*-* 0/2:13:00
+OnCalendar=Sun *-*-* 2:08:00
 
 [Install]
 WantedBy=timers.target
-

scripts/backups/wikidb_dump.sh

@@ -2,13 +2,11 @@
 #
 # Run the mysql dump command to back up wikidb table, and send the
 # resulting SQL file to the specified backup directory.
-#
+set -eux
-# Backup directory:
-#       /home/user/backups/mysql
 
-BACKUP_DIR="$HOME/backups"
 CONTAINER_NAME="stormy_mysql"
-STAMP="`date +"%Y%m%d"`"
+DATESTAMP="`date +"%Y%m%d"`"
+TIMESTAMP="`date +"%Y%m%d_%H%M%S"`"
 
 function usage {
     set +x
@@ -23,7 +21,7 @@ function usage {
     echo "Example:"
     echo ""
     echo "       ./wikidb_dump.sh"
-    echo "       (creates ${BACKUP_DIR}/20200101/wikidb_20200101.sql)"
+    echo "       (creates ${POD_CHARLESREID1_BACKUP_DIR}/YYYYMMDD/wikidb_YYYYMMDD_HHMMSS.sql)"
     echo ""
     exit 1;
 }
@@ -39,36 +37,64 @@ fi
 
 if [ "$#" == "0" ]; then
 
-    TARGET="wikidb_${STAMP}.sql"
+    TARGET="wikidb_${TIMESTAMP}.sql"
-    BACKUP_TARGET="${BACKUP_DIR}/${STAMP}/${TARGET}"
+    BACKUP_DIR="${POD_CHARLESREID1_BACKUP_DIR}/${DATESTAMP}"
+    BACKUP_TARGET="${BACKUP_DIR}/${TARGET}"
 
     echo ""
     echo "pod-charlesreid1: wikidb_dump.sh"
     echo "--------------------------------"
     echo ""
+    echo "Backup directory: ${BACKUP_DIR}"
     echo "Backup target: ${BACKUP_TARGET}"
     echo ""
 
-    mkdir -p ${BACKUP_DIR}/${STAMP}
+    mkdir -p "${BACKUP_DIR}"
 
-    # If this script is being run from a cron job,
+    echo "Running mysqldump inside the mysql container"
-    # don't use -i flag with docker
+
-    CRON="$( pstree -s $$ | /bin/grep -c cron )"
+    # Pull the root password out of the container so we don't duplicate the
-    DOCKER=$(which docker)
+    # secret on the host, and forward it in via MYSQL_PWD (which mysqldump
-    DOCKERX=""
+    # reads automatically). No -t: a PTY corrupts --default-character-set=binary
-    if [[ "$CRON" -eq 1 ]]; 
+    # output (LF→CRLF translation on binary blobs) and its small kernel buffer
-    then
+    # can deadlock on large dumps.
-        DOCKERX="${DOCKER} exec -t"
+    set +x
-    else
+    MYSQL_PWD="$(docker exec "${CONTAINER_NAME}" printenv MYSQL_ROOT_PASSWORD)"
-        DOCKERX="${DOCKER} exec -it"
+    export MYSQL_PWD
+    set -x
+
+    docker exec -i \
+        -e MYSQL_PWD \
+        "${CONTAINER_NAME}" \
+        sh -c 'exec mysqldump \
+                  --user=root \
+                  --single-transaction \
+                  --quick \
+                  --routines \
+                  --triggers \
+                  --events \
+                  --default-character-set=binary \
+                  --databases wikidb' \
+        > "${BACKUP_TARGET}"
+
+    unset MYSQL_PWD
+
+    # A complete mysqldump always ends with "-- Dump completed on ...".
+    # Missing trailer means the dump is truncated and not restorable.
+    if ! tail -c 200 "${BACKUP_TARGET}" | grep -q 'Dump completed on'; then
+        echo "ERROR: dump file ${BACKUP_TARGET} is missing the completion trailer." >&2
+        echo "       mysqldump did not finish successfully." >&2
+        exit 2
     fi
 
-    echo "Running mysqldump"
+    size=$(stat -c %s "${BACKUP_TARGET}")
-    set -x
+    if [ "${size}" -lt $((50 * 1024 * 1024)) ]; then
-    ${DOCKERX} ${CONTAINER_NAME} sh -c 'exec mysqldump wikidb --databases -uroot -p"$MYSQL_ROOT_PASSWORD"' > ${BACKUP_TARGET}
+        echo "ERROR: dump file ${BACKUP_TARGET} is only ${size} bytes; suspicious." >&2
-    set +x
+        exit 3
+    fi
+
+    echo "Dump OK: ${BACKUP_TARGET} (${size} bytes)"
 
-    echo "Done."
 else
     usage
 fi

scripts/backups/wikidb_restore_test.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+#
+# Restore a wikidb dump into a throwaway MySQL 5.7 container and run sanity
+# queries against it. Compares row counts to live stormy_mysql. Exits non-zero
+# on any failure.
+#
+# Usage:
+#   ./wikidb_restore_test.sh <path-to-dump.sql>
+#
+# A backup is only a backup if you have actually restored from it.
+set -euo pipefail
+
+DUMP="${1:-}"
+if [ -z "${DUMP}" ] || [ ! -f "${DUMP}" ]; then
+    echo "Usage: $0 <path-to-wikidb-dump.sql>" >&2
+    exit 1
+fi
+
+LIVE_CONTAINER="stormy_mysql"
+TEST_CONTAINER="wikidb_restore_test_$$"
+TEST_PW="temp_restore_test_pw_$$"
+IMAGE="mysql:5.7"
+
+cleanup() {
+    docker stop "${TEST_CONTAINER}" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+echo "[1/5] Starting throwaway MySQL container ${TEST_CONTAINER}..."
+docker run -d --rm \
+    --name "${TEST_CONTAINER}" \
+    -e MYSQL_ROOT_PASSWORD="${TEST_PW}" \
+    "${IMAGE}" >/dev/null
+
+echo "[2/5] Waiting for MySQL to accept authenticated connections..."
+# `mysqladmin ping` returns OK before the root user is actually set up, so we
+# have to probe with a real authenticated query and accept only success.
+ready=0
+for i in $(seq 1 60); do
+    if docker exec -e MYSQL_PWD="${TEST_PW}" "${TEST_CONTAINER}" \
+        mysql -uroot -e 'SELECT 1' >/dev/null 2>&1; then
+        ready=1
+        break
+    fi
+    sleep 2
+done
+if [ "${ready}" -ne 1 ]; then
+    echo "ERROR: MySQL in ${TEST_CONTAINER} never became ready." >&2
+    docker logs "${TEST_CONTAINER}" 2>&1 | tail -20 >&2
+    exit 4
+fi
+
+echo "[3/5] Piping dump into throwaway MySQL..."
+docker exec -i -e MYSQL_PWD="${TEST_PW}" "${TEST_CONTAINER}" \
+    mysql -uroot < "${DUMP}"
+
+echo "[4/5] Querying restored DB..."
+restored=$(docker exec -e MYSQL_PWD="${TEST_PW}" "${TEST_CONTAINER}" \
+    mysql -uroot -N -B -e "
+        USE wikidb;
+        SELECT COUNT(*) FROM page;
+        SELECT COUNT(*) FROM revision;
+        SELECT COUNT(*) FROM text;
+        SELECT COALESCE(MAX(rev_timestamp), 'none') FROM revision;
+    ")
+
+echo "--- restored ---"
+echo "${restored}"
+
+echo "[5/5] Querying live ${LIVE_CONTAINER}..."
+LIVE_PW="$(docker exec "${LIVE_CONTAINER}" printenv MYSQL_ROOT_PASSWORD)"
+live=$(docker exec -e MYSQL_PWD="${LIVE_PW}" "${LIVE_CONTAINER}" \
+    mysql -uroot -N -B -e "
+        USE wikidb;
+        SELECT COUNT(*) FROM page;
+        SELECT COUNT(*) FROM revision;
+        SELECT COUNT(*) FROM text;
+        SELECT COALESCE(MAX(rev_timestamp), 'none') FROM revision;
+    ")
+
+echo "--- live ---"
+echo "${live}"
+
+r_page=$(echo "${restored}"   | sed -n '1p')
+r_rev=$(echo  "${restored}"   | sed -n '2p')
+r_text=$(echo "${restored}"   | sed -n '3p')
+l_page=$(echo "${live}"       | sed -n '1p')
+l_rev=$(echo  "${live}"       | sed -n '2p')
+l_text=$(echo "${live}"       | sed -n '3p')
+
+fail=0
+for kind in page rev text; do
+    r_var="r_${kind}"
+    l_var="l_${kind}"
+    r="${!r_var}"
+    l="${!l_var}"
+    if [ "${r}" != "${l}" ]; then
+        echo "MISMATCH: ${kind} count restored=${r} live=${l}" >&2
+        fail=1
+    else
+        echo "OK: ${kind} count = ${r}"
+    fi
+done
+
+if [ "${fail}" -ne 0 ]; then
+    echo "RESTORE TEST FAILED." >&2
+    exit 5
+fi
+
+echo "RESTORE TEST PASSED."

scripts/backups/wikifiles_dump.sh

@@ -2,13 +2,11 @@
 #
 # Create a tar file containing wiki files
 # from the mediawiki docker container.
-#
+set -eux
-# Backup directory:
-#       /home/user/backups/mediawiki
 
-BACKUP_DIR="$HOME/backups"
 CONTAINER_NAME="stormy_mw"
-STAMP="`date +"%Y%m%d"`"
+DATESTAMP="`date +"%Y%m%d"`"
+TIMESTAMP="`date +"%Y%m%d_%H%M%S"`"
 
 function usage {
     set +x
@@ -23,7 +21,7 @@ function usage {
     echo "Example:"
     echo ""
     echo "       ./wikifiles_dump.sh"
-    echo "       (creates ${BACKUP_DIR}/20200101/wikifiles_20200101.tar.gz)"
+    echo "       (creates ${POD_CHARLESREID1_BACKUP_DIR}/YYYYMMDD/wikifiles_YYYYMMDD_HHMMSS.tar.gz)"
     echo ""
     exit 1;
 }
@@ -39,48 +37,36 @@ fi
 
 if [ "$#" == "0" ]; then
 
-    TARGET="wikifiles_${STAMP}.tar.gz"
+    TARGET="wikifiles_${TIMESTAMP}.tar.gz"
-    BACKUP_TARGET="${BACKUP_DIR}/${STAMP}/${TARGET}"
+    BACKUP_DIR="${POD_CHARLESREID1_BACKUP_DIR}/${DATESTAMP}"
+    BACKUP_TARGET="${BACKUP_DIR}/${TARGET}"
 
     echo ""
     echo "pod-charlesreid1: wikifiles_dump.sh"
     echo "-----------------------------------"
     echo ""
+    echo "Backup directory: ${BACKUP_DIR}"
     echo "Backup target: ${BACKUP_TARGET}"
     echo ""
 
-    mkdir -p ${BACKUP_DIR}/${STAMP}
+    mkdir -p ${BACKUP_DIR}
 
-    # If this script is being run from a cron job,
-    # don't use -i flag with docker
-    CRON="$( pstree -s $$ | /bin/grep -c cron )"
     DOCKER=$(which docker)
-    DOCKERX=""
-    if [[ "$CRON" -eq 1 ]]; 
-    then
     DOCKERX="${DOCKER} exec -t"
-    else
-        DOCKERX="${DOCKER} exec -it"
-    fi
 
     echo "Step 1: Compress wiki files inside container"
-    set -x
     ${DOCKERX} ${CONTAINER_NAME} /bin/tar czf /tmp/${TARGET} /var/www/html/images
-    set +x
 
     echo "Step 2: Copy tar.gz file out of container"
-    mkdir -p $(dirname "$1")
+    mkdir -p $(dirname "${BACKUP_TARGET}")
-    set -x
     ${DOCKER} cp ${CONTAINER_NAME}:/tmp/${TARGET} ${BACKUP_TARGET}
-    set +x
 
     echo "Step 3: Clean up tar.gz file"
-    set -x
     ${DOCKERX} ${CONTAINER_NAME} /bin/rm -f /tmp/${TARGET}
-    set +x
 
+    echo "Successfully wrote wikifiles dump to file: ${BACKUP_TARGET}"
     echo "Done."
+
 else
     usage
 fi
-

scripts/backups/wikifiles_restore.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Restore wiki files from a tar file
+# into the stormy_mw container.
+set -eu
+
+function usage {
+    echo ""
+    echo "restore_wikifiles.sh script:"
+    echo "Restore wiki files from a tar file"
+    echo "into the stormy_mw container"
+    echo ""
+    echo "       ./restore_wikifiles.sh <tar-file>"
+    echo ""
+    echo "Example:"
+    echo ""
+    echo "       ./restore_wikifiles.sh /path/to/wikifiles.tar.gz"
+    echo ""
+    echo ""
+    exit 1;
+}
+
+# NOTE:
+# I assume images/ is the only directory to back up/restore.
+# If there are more I forgot, add them back in here.
+# (skins and extensions are static, added into image at build time.)
+
+if [[ "$#" -eq 1 ]];
+then
+
+    NAME="stormy_mw"
+    TAR=$(basename "$1")
+
+    echo "Checking that container ${NAME} exists"
+    docker ps --format '{{.Names}}' | grep ${NAME} || exit 1;
+
+    echo "Copying dir $1 into container ${NAME}"
+    set -x
+    docker cp $1 ${NAME}:/tmp/${TAR}
+    docker exec -it ${NAME} rm -rf /var/www/html/images.old
+    docker exec -it ${NAME} mv /var/www/html/images /var/www/html/images.old
+    docker exec -it ${NAME} tar -xf /tmp/${TAR} -C / && rm -f /tmp/${TAR}
+    docker exec -it ${NAME} chown -R www-data:www-data /var/www/html/images
+
+else
+    usage
+fi

scripts/certbot/pod-charlesreid1-certbot.service.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=Renew certificates for pod-charlesreid1
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+StandardError=syslog
+StandardOutput=syslog
+SyslogIdentifier=pod-charlesreid1-certbot
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/certbot/renew_charlesreid1_certs.sh'

scripts/certbot/pod-charlesreid1-certbot.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Timer to renew certificates for pod-charlesreid1
+
+[Timer]
+# Run daily
+OnCalendar=*-*-* 4:03:00
+
+[Install]
+WantedBy=timers.target

scripts/certbot/renew_charlesreid1_certs.sh.j2

@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# renew/run certbot on krash
+set -eux
+
+SERVICE="pod-charlesreid1"
+
+function usage {
+    set +x
+    echo ""
+    echo "renew_charlesreid1_certs.sh script:"
+    echo ""
+    echo "Renew all certs used in the charlesreid1.com pod"
+    echo ""
+    echo "       ./renew_charlesreid1_certs.sh"
+    echo ""
+    exit 1;
+}
+
+if [ "$(id -u)" != "0" ]; then
+    echo ""
+    echo ""
+    echo "This script should be run as root."
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+if [ "$#" == "0" ]; then
+
+    # disable system service that will re-spawn docker pod
+    echo "Disable and stop system service ${SERVICE}"
+    sudo systemctl disable ${SERVICE}
+    sudo systemctl stop ${SERVICE}
+    
+    echo "Stop pod"
+    docker-compose -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml down
+    
+    echo "Run certbot renew"
+    SUBS="git www"
+    DOMS="charlesreid1.com"
+
+    # top level domains
+    for DOM in $DOMS; do
+        certbot certonly \
+            --standalone \
+            --non-interactive \
+            --agree-tos \
+            --email charles@charlesreid1.com \
+            -d ${DOM}
+    done
+
+    # subdomains
+    for SUB in $SUBS; do
+        for DOM in $DOMS; do
+            certbot certonly \
+                --standalone \
+                --non-interactive \
+                --agree-tos \
+                --email charles@charlesreid1.com \
+                -d ${SUB}.${DOM}
+        done
+    done
+    
+    echo "Start pod"
+    docker-compose -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml up -d
+    
+    echo "Enable and start system service ${SERVICE}"
+    sudo systemctl enable ${SERVICE}
+    sudo systemctl start ${SERVICE}
+
+    echo "Done"
+
+else
+    usage
+fi

scripts/clean_templates.py

@@ -13,7 +13,9 @@ def clean():
         rname = tname[:-3]
         rpath = os.path.join(tdir, rname)
 
-        if os.path.exists(rpath):
+        ignore_list = ['environment']
+
+        if os.path.exists(rpath) and rname not in ignore_list:
             print(f"Removing file {rpath}")
             os.remove(rpath)
         else:

scripts/git_clone_www.py.j2

@@ -11,8 +11,8 @@ directory structure for charlesreid1.com
 content. (Or, charlesreid1.XYZ, whatever.)
 """
 
-SERVER_NAME_DEFAULT = '{{ server_name_default }}'
+SERVER_NAME_DEFAULT = '{{ pod_charlesreid1_server_name }}'
-USERNAME = '{{ username }}'
+USERNAME = '{{ pod_charlesreid1_username }}'
 
 
 

scripts/git_pull_www.py.j2

@@ -10,8 +10,8 @@ This script git pulls the /www directory
 for updating charlesreid1.com content.
 """
 
-SERVER_NAME_DEFAULT = '{{ server_name_default }}'
+SERVER_NAME_DEFAULT = '{{ pod_charlesreid1_server_name }}'
-USERNAME = '{{ username }}'
+USERNAME = '{{ pod_charlesreid1_username }}'
 
 
 

scripts/mw/build_extensions_dir.sh

@@ -1,37 +1,27 @@
 #!/bin/bash
 #
-# clone or download each extension, and build
+# Clone each REL1_39 extension into d-mediawiki-new for the MW 1.39 green stack.
+# EmbedVideo is intentionally skipped for now (add back later if needed).
 set -eux
 
-MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
+MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki-new"
-CONF_DIR="${MW_DIR}/charlesreid1-config"
+MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
-MW_CONF_DIR="${MW_CONF_DIR}/mediawiki"
 EXT_DIR="${MW_CONF_DIR}/extensions"
 
-mkdir -p ${EXT_DIR}/extensions
+mkdir -p ${EXT_DIR}
 
 (
-cd ${EXT_DIR}/extensions
+cd ${EXT_DIR}
 
 ##############################
 
 Extension="SyntaxHighlight_GeSHi"
 if [ ! -d ${Extension} ]
 then
-    ## This requires mediawiki > 1.31
+    git clone https://github.com/wikimedia/mediawiki-extensions-SyntaxHighlight_GeSHi.git ${Extension}
-    ## (so does REL1_31)
-    #git clone https://github.com/wikimedia/mediawiki-extensions-SyntaxHighlight_GeSHi.git SyntaxHighlight_GeSHi
-
-    ## This manually downloads REL1_30
-    #wget https://extdist.wmflabs.org/dist/extensions/SyntaxHighlight_GeSHi-REL1_30-87392f1.tar.gz -O SyntaxHighlight_GeSHi.tar.gz
-    #tar -xzf SyntaxHighlight_GeSHi.tar.gz -C ${PWD}
-    #rm -f SyntaxHighlight_GeSHi.tar.gz
-
-    # Best of both worlds
-    git clone https://github.com/wikimedia/mediawiki-extensions-SyntaxHighlight_GeSHi.git SyntaxHighlight_GeSHi
     (
     cd ${Extension}
-    git checkout --track remotes/origin/REL1_34
+    git checkout --track remotes/origin/REL1_39
     )
 else
     echo "Skipping ${Extension}"
@@ -45,21 +35,7 @@ then
     git clone https://github.com/wikimedia/mediawiki-extensions-ParserFunctions.git ${Extension}
     (
     cd ${Extension}
-    git checkout --track remotes/origin/REL1_34
+    git checkout --track remotes/origin/REL1_39
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="EmbedVideo"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/HydraWiki/mediawiki-embedvideo.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout v2.7.3
     )
 else
     echo "Skipping ${Extension}"
@@ -73,21 +49,7 @@ then
     git clone https://github.com/wikimedia/mediawiki-extensions-Math.git ${Extension}
     (
     cd ${Extension}
-    git checkout REL1_34
+    git checkout --track remotes/origin/REL1_39
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="Fail2banlog"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/charlesreid1-docker/mw-fail2ban.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout master
     )
 else
     echo "Skipping ${Extension}"

scripts/mw/fix_LocalSettings.sh

@@ -1,20 +1,12 @@
 #!/bin/bash
 # 
 # fix LocalSettings.php in the mediawiki container.
-# 
-# docker is stupid, so it doesn't let you bind mount
-# a single file into a docker volume.
-# 
-# so, rather than rebuilding the entire goddamn container
-# just to update LocalSettings.php when it changes, we just
-# use a docker cp command to copy it into the container.
 set -eux
 
 NAME="stormy_mw"
 
 MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
-CONF_DIR="${MW_DIR}/charlesreid1-config"
+MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
-MW_CONF_DIR="${MW_CONF_DIR}/mediawiki"
 
 echo "Checking that container exists"
 docker ps --format '{{.Names}}' | grep ${NAME} || exit 1;

scripts/mw/fix_extensions_dir.sh

@@ -1,12 +1,6 @@
 #!/bin/bash
 # 
 # fix extensions dir in the mediawiki container
-#
-# in theory, we should be able to update the
-# extensions folder in d-mediawiki/charlesreid1-config,
-# but in reality this falls on its face.
-# So, we have to fix the fucking extensions directory
-# ourselves.
 set -eux
 
 NAME="stormy_mw"
@@ -14,8 +8,7 @@ NAME="stormy_mw"
 EXTENSIONS="SyntaxHighlight_GeSHi ParserFunctions EmbedVideo Math Fail2banlog"
 
 MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
-CONF_DIR="${MW_DIR}/charlesreid1-config"
+MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
-MW_CONF_DIR="${MW_CONF_DIR}/mediawiki"
 EXT_DIR="${MW_CONF_DIR}/extensions"
 
 echo "Checking that container exists..."

scripts/mw/fix_skins.sh

@@ -1,20 +1,12 @@
 #!/bin/bash
 # 
 # fix skins in the mediawiki container.
-# 
-# docker is stupid, so it doesn't let you bind mount
-# a single file into a docker volume.
-#
-# so, rather than rebuilding the entire goddamn container
-# just to update the skin when it changes, we just
-# use a docker cp command to copy it into the container.
 set -eux
 
 NAME="stormy_mw"
 
 MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
-CONF_DIR="${MW_DIR}/charlesreid1-config"
+MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
-MW_CONF_DIR="${MW_CONF_DIR}/mediawiki"
 SKINS_DIR="${MW_CONF_DIR}/skins"
 
 echo "Checking that container exists"
@@ -24,8 +16,8 @@ echo "Checking that skins dir exists"
 test -d ${SKINS_DIR}
 
 echo "Installing skins into $NAME"
-docker exec -it $NAME /bin/bash -c 'rm -rf /var/www/html/skins'
+docker exec -i $NAME /bin/bash -c 'rm -rf /var/www/html/skins'
 docker cp ${SKINS_DIR} $NAME:/var/www/html/skins
-docker exec -it $NAME /bin/bash -c 'chown -R www-data:www-data /var/www/html/skins'
+docker exec -i $NAME /bin/bash -c 'chown -R www-data:www-data /var/www/html/skins'
 
 echo "Finished installing skins into $NAME"

scripts/mw/restore_wikifiles.sh

@@ -2,7 +2,7 @@
 #
 # Restore wiki files from a tar file
 # into the stormy_mw container.
-set -eux
+set -eu
 
 function usage {
     echo ""
@@ -31,16 +31,16 @@ then
     NAME="stormy_mw"
     TAR=$(basename "$1")
 
-	echo "Checking that container exists"
+	echo "Checking that container ${NAME} exists"
 	docker ps --format '{{.Names}}' | grep ${NAME} || exit 1;
 
-	echo "Copying $1 into container ${NAME}"
+	echo "Copying dir $1 into container ${NAME}"
     set -x
     docker cp $1 ${NAME}:/tmp/${TAR}
+    docker exec -it ${NAME} rm -rf /var/www/html/images.old
     docker exec -it ${NAME} mv /var/www/html/images /var/www/html/images.old
     docker exec -it ${NAME} tar -xf /tmp/${TAR} -C / && rm -f /tmp/${TAR}
     docker exec -it ${NAME} chown -R www-data:www-data /var/www/html/images
-    set +x
 
 else
     usage

scripts/mysql/dump_database.sh

@@ -1,35 +1,36 @@
 #!/bin/bash
+echo "this script is deprecated, see ../backups/wikidb_dump.sh"
+##
+## Dump a database to an .sql file
+## from the stormy_mysql container.
+#set -eu
 #
-# Dump a database to an .sql file
+#function usage {
-# from the stormy_mysql container.
+#    echo ""
-set -x
+#    echo "dump_database.sh script:"
-
+#    echo "Dump a database to an .sql file "
-function usage {
+#    echo "from the stormy_mysql container."
-    echo ""
+#    echo ""
-    echo "dump_database.sh script:"
+#    echo "       ./dump_database.sh <sql-dump-file>"
-    echo "Dump a database to an .sql file "
+#    echo ""
-    echo "from the stormy_mysql container."
+#    echo "Example:"
-    echo ""
+#    echo ""
-    echo "       ./dump_database.sh <sql-dump-file>"
+#    echo "       ./dump_database.sh /path/to/wikidb_dump.sql"
-    echo ""
+#    echo ""
-    echo "Example:"
+#    echo ""
-    echo ""
+#    exit 1;
-    echo "       ./dump_database.sh /path/to/wikidb_dump.sql"
+#}
-    echo ""
+#
-    echo ""
+#CONTAINER_NAME="stormy_mysql"
-    exit 1;
+#
-}
+#if [[ "$#" -gt 0 ]];
-
+#then
-CONTAINER_NAME="stormy_mysql"
+#
-
+#    TARGET="$1"
-if [[ "$#" -gt 0 ]];
+#    mkdir -p $(dirname $TARGET)
-then
+#	set -x
-
+#    docker exec -i ${CONTAINER_NAME} sh -c 'exec mysqldump wikidb --databases -uroot -p"$MYSQL_ROOT_PASSWORD"' > $TARGET
-    TARGET="$1"
+#
-    mkdir -p $(dirname $TARGET)
+#else
-    docker exec -i ${CONTAINER_NAME} sh -c 'exec mysqldump wikidb --databases -uroot -p"$MYSQL_ROOT_PASSWORD"' > $TARGET
+#    usage
-
+#fi
-else
-    usage
-fi
-

scripts/mysql/restore_database.sh

@@ -6,6 +6,7 @@
 # Note that this expects the .sql dump
 # to create its own databases.
 # Use the --databases flag with mysqldump.
+set -eu
 
 function usage {
     echo ""
@@ -42,31 +43,23 @@ function usage {
 # because of all these one-off 
 # "whoopsie we don't do that" problems.
 
+if [[ "$#" -eq 1 ]];
+then
+
 	CONTAINER_NAME="stormy_mysql"
 	TARGET=$(basename $1)
 	TARGET_DIR=$(dirname $1)
 
-
-if [[ "$#" -eq 1 ]];
-then
-
-    # Step 1: Copy the sql dump into the container
     set -x
+    # Step 1: Copy the sql dump into the container
     docker cp $1 ${CONTAINER_NAME}:/tmp/${TARGET}
-    set +x
 
     # Step 2: Run sqldump inside the container
-    set -x
     docker exec -i ${CONTAINER_NAME} sh -c "/usr/bin/mysql --defaults-file=/root/.mysql.rootpw.cnf < /tmp/${TARGET}"
-    set +x
 
     # Step 3: Clean up sql dump from inside container
-    set -x
+    docker exec -i ${CONTAINER_NAME} sh -c "/bin/rm -fr /tmp/${TARGET}"
-    docker exec -i ${CONTAINER_NAME} sh -c "/bin/rm -fr /tmp/${TARGET}.sql"
-    set +x
 
-
-    set +x
 else
     usage
 fi

scripts/pod-charlesreid1.service.j2

@@ -5,11 +5,11 @@ After=docker.service
 
 [Service]
 Restart=always
-StandardError=null
+StandardError=journal
-StandardOutput=null
+StandardOutput=journal
-ExecStartPre=test -f {{ pod_install_dir }}/docker-compose.yml
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml
-ExecStart=/usr/local/bin/docker-compose -f {{ pod_install_dir }}/docker-compose.yml up
+ExecStart=/usr/local/bin/docker-compose -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml up
-ExecStop=/usr/local/bin/docker-compose  -f {{ pod_install_dir }}/docker-compose.yml stop
+ExecStop=/usr/local/bin/docker-compose  -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml stop
 
 [Install]
 WantedBy=default.target