fix networking issues in gitea action runner

* claude-plans-execute-upgrade:
  add build extensions dir script
  update gitignore
  add php error logging
  fix bootstrap2 skin for 1.39
  update LocalSettings.php
  upgrade to mediawiki 1.39 in mw dockerfile
  bump mysql dockerfile version from 5.7 to 8.0
  add mysql no root password plan
  add execution notes
  update plan
  remove plan
  add fixes for getting non-root mysql user
  fix wiki backups and canary script to check for missing trailer, not just nonzero files
  remove stupid dead file
  add plan to fix sql backups, plus implemented fixes for sql backups
  add mysql no root pw transition plan
  add mediawiki upgrade plan

.gitignore

@@ -9,9 +9,10 @@ d-gitea/custom/
 
 # mediawiki
 charlesreid1.wiki.conf
+d-mediawiki/mediawiki/
 d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/Bootstrap2.php
 d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/navbar.php
-d-mediawiki/mediawiki/
+d-mediawiki/charlesreid1-config/mediawiki/mathjax
 
 # nginx
 d-nginx-charlesreid1/conf.d/http.DOMAIN.conf

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "mkdocs-material"]
 	path = mkdocs-material
-	url = git@github.com:charlesreid1-docker/mkdocs-material.git
+	url = https://github.com/charlesreid1/mkdocs-material

Makefile

@@ -63,13 +63,14 @@ help:
 templates:
 	@find * -name "*.service.j2" | xargs -I '{}' chmod 644 {}
 	@find * -name "*.timer.j2" | xargs -I '{}' chmod 644 {}
-	python3 $(POD_CHARLESREID1_DIR)/scripts/apply_templates.py
+	/home/charles/.pyenv/shims/python3 $(POD_CHARLESREID1_DIR)/scripts/apply_templates.py
 
 list-templates:
 	@find * -name "*.j2"
 
 clean-templates:
-	python3 $(POD_CHARLESREID1_DIR)/scripts/clean_templates.py
+	# sudo is required because bind-mounted gitea files end up owned by root. stupid docker.
+	sudo -E /home/charles/.pyenv/shims/python3 $(POD_CHARLESREID1_DIR)/scripts/clean_templates.py
 
 # Backups
 
@@ -97,31 +98,42 @@ mw-fix-skins:
 # /www Dir
 
 clone-www:
-	python3 $(POD_CHARLESREID1_DIR)/scripts/git_clone_www.py
+	/home/charles/.pyenv/shims/python3 $(POD_CHARLESREID1_DIR)/scripts/git_clone_www.py
 
 pull-www:
-	python3 $(POD_CHARLESREID1_DIR)/scripts/git_pull_www.py
+	/home/charles/.pyenv/shims/python3 $(POD_CHARLESREID1_DIR)/scripts/git_pull_www.py
 
 install:
 ifeq ($(shell which systemctl),)
 	$(error Please run this make command on a system with systemctl installed)
 endif
+	@/home/charles/.pyenv/shims/python3 -c 'import botocore' || (echo "Please install the botocore library using python3 or pip3 binary"; exit 1)
+	@/home/charles/.pyenv/shims/python3 -c 'import boto3' || (echo "Please install the boto3 library using python3 or pip3 binary"; exit 1)
+
 	sudo cp $(POD_CHARLESREID1_DIR)/scripts/pod-charlesreid1.service /etc/systemd/system/pod-charlesreid1.service
+
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-aws.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-cleanolderthan.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-gitea.{service,timer} /etc/systemd/system/.
 	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikidb.{service,timer} /etc/systemd/system/.
 	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikifiles.{service,timer} /etc/systemd/system/.
-	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-gitea.{service,timer} /etc/systemd/system/.
+
-	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-aws.{service,timer} /etc/systemd/system/.
 	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/canary/pod-charlesreid1-canary.{service,timer} /etc/systemd/system/.
 	sudo cp $(POD_CHARLESREID1_DIR)/scripts/certbot/pod-charlesreid1-certbot.{service,timer} /etc/systemd/system/.
 
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/10-pod-charlesreid1-rsyslog.conf /etc/rsyslog.d/.
+
 	sudo chmod 664 /etc/systemd/system/pod-charlesreid1*
 	sudo systemctl daemon-reload
 
+	sudo systemctl restart rsyslog
+
 	sudo systemctl enable pod-charlesreid1
 	sudo systemctl enable pod-charlesreid1-backups-wikidb.timer
 	sudo systemctl enable pod-charlesreid1-backups-wikifiles.timer
 	sudo systemctl enable pod-charlesreid1-backups-gitea.timer
 	sudo systemctl enable pod-charlesreid1-backups-aws.timer
+	sudo systemctl enable pod-charlesreid1-backups-cleanolderthan.timer
 	sudo systemctl enable pod-charlesreid1-canary.timer
 	sudo systemctl enable pod-charlesreid1-certbot.timer
 
@@ -129,37 +141,54 @@ endif
 	sudo systemctl start pod-charlesreid1-backups-wikifiles.timer
 	sudo systemctl start pod-charlesreid1-backups-gitea.timer
 	sudo systemctl start pod-charlesreid1-backups-aws.timer
+	sudo systemctl start pod-charlesreid1-backups-cleanolderthan.timer
 	sudo systemctl start pod-charlesreid1-canary.timer
 	sudo systemctl start pod-charlesreid1-certbot.timer
 
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-aws.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-cleanolderthan.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-gitea.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-wikidb.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-wikifiles.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-canary.service.log
+
 uninstall:
 ifeq ($(shell which systemctl),)
 	$(error Please run this make command on a system with systemctl installed)
 endif
 	-sudo systemctl disable pod-charlesreid1
+	-sudo systemctl disable pod-charlesreid1-backups-aws.timer
+	-sudo systemctl disable pod-charlesreid1-backups-cleanolderthan.timer
+	-sudo systemctl disable pod-charlesreid1-backups-gitea.timer
 	-sudo systemctl disable pod-charlesreid1-backups-wikidb.timer
 	-sudo systemctl disable pod-charlesreid1-backups-wikifiles.timer
-	-sudo systemctl disable pod-charlesreid1-backups-gitea.timer
-	-sudo systemctl disable pod-charlesreid1-backups-aws.timer
 	-sudo systemctl disable pod-charlesreid1-canary.timer
 	-sudo systemctl disable pod-charlesreid1-certbot.timer
 
 	# Leave the pod running!
 	# -sudo systemctl stop pod-charlesreid1
+
+	-sudo systemctl stop pod-charlesreid1-backups-aws.timer
+	-sudo systemctl stop pod-charlesreid1-backups-cleanolderthan.timer
+	-sudo systemctl stop pod-charlesreid1-backups-gitea.timer
 	-sudo systemctl stop pod-charlesreid1-backups-wikidb.timer
 	-sudo systemctl stop pod-charlesreid1-backups-wikifiles.timer
-	-sudo systemctl stop pod-charlesreid1-backups-gitea.timer
-	-sudo systemctl stop pod-charlesreid1-backups-aws.timer
 	-sudo systemctl stop pod-charlesreid1-canary.timer
 	-sudo systemctl stop pod-charlesreid1-certbot.timer
 
 	-sudo rm -f /etc/systemd/system/pod-charlesreid1.service
+
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-aws.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-cleanolderthan.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-gitea.{service,timer}
 	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-wikidb.{service,timer}
 	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-wikifiles.{service,timer}
-	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-gitea.{service,timer}
-	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-aws.{service,timer}
 	-sudo rm -f /etc/systemd/system/pod-charlesreid1-canary.{service,timer}
 	-sudo rm -f /etc/systemd/system/pod-charlesreid1-certbot.{service,timer}
+
 	sudo systemctl daemon-reload
 
+	-sudo rm -f /etc/rsyslog.d/10-pod-charlesreid1-rsyslog.conf
+	-sudo systemctl restart rsyslog
+
 .PHONY: help

Troubleshooting.md

@@ -0,0 +1,19 @@
+To get a shell in a container that has been created, before it is runnning in a pod, use `docker run`:
+
+```
+docker run --rm -it --entrypoint bash <image-name-or-id>
+
+
+docker run --rm -it --entrypoint bash pod-charlesreid1_stormy_mediawiki
+```
+
+To get a shell in a container that is running in a pod, use `docker exec`:
+
+```
+docker exec -it <image-name> /bin/bash
+
+docker exec -it stormy_mw /bin/bash
+```
+
+Also, if no changes are picking up, and you've already tried rebuilding the container image, try editing the Dockerfile.
+

d-gitea/custom/conf/app.ini.j2

@@ -6,12 +6,14 @@
 ;; https://github.com/go-gitea/gitea/blob/master/conf/app.ini
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-APP_NAME = {{ gitea_app_name }}
+APP_NAME = {{ pod_charlesreid1_gitea_app_name }}
 RUN_USER = git
 RUN_MODE = prod
+WORK_PATH = /data/gitea
 
 [ui]
-DEFAULT_THEME = arc-green
+DEFAULT_THEME = gitea-dark
+THEMES = gitea-dark
 
 [database]
 DB_TYPE  = sqlite3
@@ -31,17 +33,17 @@ DISABLE_HTTP_GIT = false
 
 [server]
 PROTOCOL     = http
-DOMAIN       = git.{{ server_name_default }}
+DOMAIN       = git.{{ pod_charlesreid1_server_name }}
 #CERT_FILE    = /www/gitea/certs/cert.pem
 #KEY_FILE     = /www/gitea/certs/key.pem
-SSH_DOMAIN   = git.{{ server_name_default }}
+SSH_DOMAIN   = git.{{ pod_charlesreid1_server_name }}
 HTTP_PORT    = 3000
 HTTP_ADDR    = 0.0.0.0
-ROOT_URL     = https://git.{{ server_name_default }}
+ROOT_URL     = https://git.{{ pod_charlesreid1_server_name }}
 ;ROOT_URL     = %(PROTOCOL)s://%(DOMAIN)s:%(HTTP_PORT)s/
 DISABLE_SSH  = false
 ; port to display in clone url:
-SSH_PORT     = 222
+;SSH_PORT     = 222
 ; port for built-in ssh server to listen on:
 SSH_LISTEN_PORT = 22
 OFFLINE_MODE = false
@@ -92,9 +94,12 @@ ENABLED      = false
 
 [security]
 INSTALL_LOCK        = true
-SECRET_KEY          = {{ gitea_secret_key }}
+SECRET_KEY          = {{ pod_charlesreid1_gitea_secretkey }}
-MIN_PASSWORD_LENGTH = 6
+MIN_PASSWORD_LENGTH = 10
-INTERNAL_TOKEN      = {{ gitea_internal_token }}
+INTERNAL_TOKEN      = {{ pod_charlesreid1_gitea_internaltoken }}
+
+[actions]
+ENABLED = true
 
 [other]
 SHOW_FOOTER_BRANDING           = false

d-gitea/runner/config.yaml

@@ -0,0 +1,20 @@
+log:
+  level: info
+
+runner:
+  # Label format: <label>:<runner-type>:<image>
+  # "ubuntu-latest" is the standard GitHub Actions label.
+  # Map it (and common aliases) to a Docker image so jobs don't sit waiting.
+  labels:
+    # alpine: ~50MB, has python3+pip; install extras with: apk add --no-cache git curl
+    - "alpine:docker://python:3.12-alpine"
+    - "ubuntu-latest:docker://catthehacker/ubuntu:act-22.04"
+    - "ubuntu-22.04:docker://catthehacker/ubuntu:act-22.04"
+    - "ubuntu-20.04:docker://catthehacker/ubuntu:act-20.04"
+    - "ubuntu-24.04:docker://catthehacker/ubuntu:act-22.04"
+
+container:
+  network: "pod-charlesreid1_frontend"
+
+cache:
+  enabled: true

d-mediawiki/Dockerfile

@@ -1,22 +1,12 @@
-FROM mediawiki
+FROM mediawiki:1.39.12
 
 EXPOSE 8989
 
-VOLUME ["/var/www/html"]
 
-# Install ImageMagick
+# Install ImageMagick (used for image thumbnailing)
-# and math stuff mentioned by Math extension readme
 RUN apt-get update && \
-    apt-get install -y build-essential \
+    apt-get install -y imagemagick && \
-            dvipng \
+    rm -rf /var/lib/apt/lists/*
-            ocaml \
-            ghostscript \
-            imagemagick \
-            texlive-latex-base \
-            texlive-latex-extra \
-            texlive-fonts-recommended \
-            texlive-lang-greek \
-            texlive-latex-recommended
 
 # Copy skins, config files, and other particulars into container
 
@@ -24,15 +14,13 @@ RUN apt-get update && \
 # MediaWiki needs everything, everything, to be in one folder.
 # Docker is totally incapable of mounting a file in a volume.
 # I cannot update LocalSettings.php without clearing the cache.
-# I cannot clear the cache without reinstalling all of latex.
 # I can't bind-mount the skins dir, because then it's owned by root.
 # I can't fix the fact that all bind-mounted dirs are owned by root, 
 # because I can only add commands in THIS DOCKERFILE.
 # and when you run the commands in this dockerfile, 
 # YOU CANNOT SEE THE BIND-MOUNTED STUFF.
 
-# Extensions
+# Extensions (REL1_39 branches; EmbedVideo skipped for the 1.34 -> 1.39 upgrade)
-COPY charlesreid1-config/mediawiki/extensions/EmbedVideo              /var/www/html/extensions/EmbedVideo
 COPY charlesreid1-config/mediawiki/extensions/Math                    /var/www/html/extensions/Math
 COPY charlesreid1-config/mediawiki/extensions/ParserFunctions         /var/www/html/extensions/ParserFunctions
 COPY charlesreid1-config/mediawiki/extensions/SyntaxHighlight_GeSHi   /var/www/html/extensions/SyntaxHighlight_GeSHi
@@ -41,22 +29,27 @@ RUN chown -R www-data:www-data /var/www/html/*
 # Skins
 COPY charlesreid1-config/mediawiki/skins /var/www/html/skins
 RUN chown -R www-data:www-data /var/www/html/skins
+RUN touch /var/www/html/skins
+
+# MathJax 3.2.2 (self-hosted, served via Apache alias at /w/mathjax/*).
+# Math extension runs in 'source' mode; MathJax renders client-side, so we
+# never call out to restbase/mathoid. See LocalSettings.php.j2.
+COPY charlesreid1-config/mediawiki/mathjax /var/www/html/mathjax
+RUN chown -R www-data:www-data /var/www/html/mathjax
 
 # Settings
 COPY charlesreid1-config/mediawiki/LocalSettings.php /var/www/html/LocalSettings.php
 RUN chown -R www-data:www-data /var/www/html/LocalSettings*
 RUN chmod 600 /var/www/html/LocalSettings.php
 
-# MediaWiki Fail2ban log directory
-RUN mkdir -p /var/log/mwf2b
-RUN chown -R www-data:www-data /var/log/mwf2b
-RUN chmod 700 /var/log/mwf2b
-
 # Apache conf file
 COPY charlesreid1-config/apache/*.conf /etc/apache2/sites-enabled/
 RUN a2enmod rewrite
 RUN service apache2 restart
 
-## make texvc
+# PHP conf file
-#CMD cd /var/www/html/extensions/Math && make && apache2-foreground
+# https://hub.docker.com/_/php/
+COPY php/php.ini /usr/local/etc/php/
+
+# Start
 CMD apache2-foreground

d-mediawiki/Skins.md

@@ -5,6 +5,10 @@ To update the MediaWiki skin:
 - Rebuild the MW container while the docker pod is still running (won't effect the docker pod)
 - When finished rebuilding the MW container, restart the docker pod.
 
+The skin currently in use is in `charlesreid1-config/mediawiki/skins/Bootstrap2`
+
+To rebuild and then restart the pod:
+
 ```
 # switch to main pod directory
 cd ../

d-mediawiki/charlesreid1-config/apache/charlesreid1.wiki.conf.j2

@@ -1,4 +1,4 @@
-ServerName {{ server_name_default }}
+ServerName {{ pod_charlesreid1_server_name }}
 
 Listen 8989
 
@@ -7,10 +7,10 @@ Listen 8989
     # talks to apache via 127.0.0.1
     # on port 8989
 
-    ServerAlias www.{{ server_name_default }}
+    ServerAlias www.{{ pod_charlesreid1_server_name }}
 
     LogLevel warn
-    ServerAdmin {{ admin_email }}
+    ServerAdmin {{ pod_charlesreid1_mediawiki_admin_email }}
     DirectoryIndex index.html index.cgi index.php
 
 

d-mediawiki/charlesreid1-config/mediawiki/LocalSettings.php.j2

@@ -13,8 +13,8 @@ if ( !defined( 'MEDIAWIKI' ) ) {
 }
 
 ## The protocol and server name to use in fully-qualified URLs
-$wgServer = 'https://{{ server_name_default }}';
+$wgServer = 'https://{{ pod_charlesreid1_server_name }}';
-$wgCanonicalServer = 'https://{{ server_name_default }}';
+$wgCanonicalServer = 'https://{{ pod_charlesreid1_server_name }}';
 
 ## The URL path to static resources (images, scripts, etc.)
 $wgStylePath = "$wgScriptPath/skins";
@@ -43,10 +43,11 @@ $wgDBpassword = getenv('MYSQL_PASSWORD');
 # MySQL specific settings
 $wgDBprefix = "";
 $wgDBTableOptions = "ENGINE=InnoDB, DEFAULT CHARSET=binary";
-$wgDBmysql5 = true;
+# $wgDBmysql5 removed — deprecated in MW 1.39
 
 # Shared memory settings
 $wgMainCacheType = CACHE_ACCEL;
+$wgCacheDirectory = "$IP/cache";
 $wgMemCachedServers = [];
 
 # To enable image uploads, make sure the 'images' directory
@@ -83,16 +84,25 @@ $wgPingback = false;
 # available UTF-8 locale
 $wgShellLocale = "en_US.utf8";
 
-# If you have the appropriate support software installed
+# Math rendering: Math extension emits raw LaTeX ('source' mode), then
-# you can enable inline LaTeX equations:
+# a self-hosted MathJax 3 build at /w/mathjax/ renders it client-side.
-$wgUseTeX           = true;
+# No mathoid, no restbase, no external CDN.
-$wgTexvc = "$IP/extensions/Math/math/texvc";
+$wgDefaultUserOptions['math'] = 'source';
-#$wgTexvc = '/usr/bin/texvc';
+$wgMathValidModes = [ 'source' ];
-
+# Skip TeX validation entirely — default validator calls out to restbase,
-# Set MathML as default rendering option
+# which breaks air-gapped installs even when we only emit source HTML.
-$wgDefaultUserOptions['math'] = 'mathml';
+$wgMathDisableTexFilter = 'always';
-$wgMathFullRestbaseURL = 'https://en.wikipedia.org/api/rest_';
+$wgHooks['BeforePageDisplay'][] = function ( $out, $skin ) {
-$wgMathMathMLUrl = 'https://mathoid-beta.wmflabs.org/';
+    $out->addHeadItem( 'mathjax',
+        '<script>window.MathJax = {'
+        . 'tex: { inlineMath: [["$","$"],["\\\\(","\\\\)"]], '
+        . 'displayMath: [["$$","$$"],["\\\\[","\\\\]"]], '
+        . 'processEscapes: true }, '
+        . 'options: { processHtmlClass: "mwe-math-fallback-source-inline|mwe-math-fallback-source-display|mwe-math-element" } '
+        . '};</script>'
+        . '<script async src="/w/mathjax/tex-chtml.js"></script>'
+    );
+};
 
 # Site language code, should be one of the list in ./languages/data/Names.php
 $wgLanguageCode = "en";
@@ -104,7 +114,7 @@ $wgAuthenticationTokenVersion = "1";
 
 # Site upgrade key. Must be set to a string (default provided) to turn on the
 # web installer while LocalSettings.php is in place
-$wgUpgradeKey = "984c1d9858dabc27";
+$wgUpgradeKey = getenv('MEDIAWIKI_UPGRADEKEY');
 
 # No license info
 $wgRightsPage = "";
@@ -156,7 +166,7 @@ $wgPutIPinRC=true;
 # Getting some weird "Error creating thumbnail: Invalid thumbnail parameters" messages w/ thumbnail
 # http://www.gossamer-threads.com/lists/wiki/mediawiki/169439
 $wgMaxImageArea=64000000;
-$wgMaxShellMemory=0;
+$wgMaxShellMemory=512000;
 
 $wgFavicon="$wgScriptPath/favicon.ico";
 
@@ -197,24 +207,22 @@ $wgSyntaxHighlightDefaultLang = "text";
 wfLoadExtension( 'ParserFunctions' );
 
 ##############################################
-# Embed videos extension
+# Embed videos extension — SKIPPED for MW 1.39 upgrade (add back later)
-# https://github.com/HydraWiki/mediawiki-embedvideo/
-# require_once("$IP/extensions/EmbedVideo/EmbedVideo.php");
-
-wfLoadExtension( 'EmbedVideo' );
 
 ###########################################
 # Math extension
 # https://github.com/wikimedia/mediawiki-extensions-Math.git
 
-require_once "$IP/extensions/Math/Math.php";
+wfLoadExtension( 'Math' );
 
-#############################################
+###########################################
-# Fail2banlog extension
+# Parsoid (bundled in MW 1.39, runs in-process)
-# https://www.mediawiki.org/wiki/Extension:Fail2banlog
+# Required for REST API /v1/page/{title}/with_html endpoint
 
-require_once "$IP/extensions/Fail2banlog/Fail2banlog.php";
+wfLoadExtension( 'Parsoid', "$IP/vendor/wikimedia/parsoid/extension.json" );
-$wgFail2banlogfile = "/var/log/apache2/mwf2b.log";
+$wgParsoidSettings = [
+    'useSelser' => true,
+];
 
 #############################################
 # Fix cookies crap
@@ -224,7 +232,7 @@ session_save_path("/tmp");
 ##############################################
 # Secure login
 
-$wgServer = "https://{{ server_name_default }}";
+$wgServer = "https://{{ pod_charlesreid1_server_name }}";
 $wgSecureLogin = true;
 
 ###################################

d-mediawiki/charlesreid1-config/mediawiki/build_extensions_dir.sh

@@ -1,93 +0,0 @@
-#!/bin/bash
-# 
-# clone or download each extension
-# and build o
-
-mkdir -p extensions
-(
-cd extensions
-
-##############################
-
-Extension="SyntaxHighlight_GeSHi"
-if [ ! -d ${Extension} ]
-then
-    ## This requires mediawiki > 1.31
-    ## (so does REL1_31)
-    #git clone https://github.com/wikimedia/mediawiki-extensions-SyntaxHighlight_GeSHi.git SyntaxHighlight_GeSHi
-
-    ## This manually downloads REL1_30
-    #wget https://extdist.wmflabs.org/dist/extensions/SyntaxHighlight_GeSHi-REL1_30-87392f1.tar.gz -O SyntaxHighlight_GeSHi.tar.gz
-    #tar -xzf SyntaxHighlight_GeSHi.tar.gz -C ${PWD}
-    #rm -f SyntaxHighlight_GeSHi.tar.gz
-
-    # Best of both worlds
-    git clone https://github.com/wikimedia/mediawiki-extensions-SyntaxHighlight_GeSHi.git SyntaxHighlight_GeSHi
-    (
-    cd ${Extension}
-    git checkout --track remotes/origin/REL1_34
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="ParserFunctions"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/wikimedia/mediawiki-extensions-ParserFunctions.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout --track remotes/origin/REL1_34
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="EmbedVideo"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/HydraWiki/mediawiki-embedvideo.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout v2.7.3
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="Math"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/wikimedia/mediawiki-extensions-Math.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout REL1_34
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="Fail2banlog"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/charlesreid1-docker/mw-fail2ban.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout master
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-# fin
-)

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/Bootstrap2.php.j2

@@ -23,12 +23,12 @@ class SkinBootstrap2 extends SkinTemplate {
         // cmr 05/08/2014
         $template = 'Bootstrap2Template';
 
-    function setupSkinUserCss( OutputPage $out ) {
+    // MW 1.39: Skin::setupSkinUserCss() was removed. initPage() is the
-        global $wgHandheldStyle;
+    // per-request hook that still receives OutputPage and runs before
+    // headElement is generated, so addStyle() calls land in the <head>.
+    public function initPage( OutputPage $out ) {
+        parent::initPage( $out );
 
-        parent::setupSkinUserCss( $out );
-
-        // Append to the default screen common & print styles...
         $out->addStyle( 'Bootstrap2/IE50Fixes.css', 'screen', 'lt IE 5.5000' );
         $out->addStyle( 'Bootstrap2/IE55Fixes.css', 'screen', 'IE 5.5000' );
         $out->addStyle( 'Bootstrap2/IE60Fixes.css', 'screen', 'IE 6' );
@@ -36,13 +36,12 @@ class SkinBootstrap2 extends SkinTemplate {
 
         $out->addStyle( 'Bootstrap2/rtl.css', 'screen', '', 'rtl' );
 
+        $out->addStyle( 'Bootstrap2/bootstrap.css' );
+        $out->addStyle( 'Bootstrap2/slate.css' );
+        $out->addStyle( 'Bootstrap2/main.css' );
+        $out->addStyle( 'Bootstrap2/dox.css' );
 
-        $out->addStyle('Bootstrap2/bootstrap.css' );
+        $out->addStyle( 'Bootstrap2/css/font-awesome.css' );
-        $out->addStyle('Bootstrap2/slate.css' );
-        $out->addStyle('Bootstrap2/main.css' );
-        $out->addStyle('Bootstrap2/dox.css' );
-
-        $out->addStyle('Bootstrap2/css/font-awesome.css');
         //$out->addStyle('Bootstrap2/cmr-bootstrap-cyborg.css');
         //$out->addStyle('Bootstrap2/cmr-bootstrap-cyborg-wiki.css');
         //
@@ -72,7 +71,8 @@ class Bootstrap2Template extends QuickTemplate {
 
         // -------- Start ------------
         // Adding the following line makes Geshi work
-        $this->html( 'headelement' );
+        // (MW 1.39: read $this->data directly to avoid QuickTemplate::html('headelement') deprecation)
+        echo $this->data['headelement'];
         // Left this out because the [edit] buttons were becoming right-aligned
         // Got around that behavior by changing shared.css
         // -------- End ------------
@@ -106,7 +106,7 @@ include('/var/www/html/skins/Bootstrap2/navbar.php');
             <div class="container-fixed">
                 <div class="navbar-header">
                     <a href="/wiki/" class="navbar-brand">
-                    {{ top_domain }} wiki
+                    {{ pod_charlesreid1_server_name }} wiki
                     </a>
                 </div>
                 <div>
@@ -146,7 +146,7 @@ include('/var/www/html/skins/Bootstrap2/navbar.php');
                                     echo ' ';
                                     echo $tab['class'];
                                 }
-                                echo '" id="' . Sanitizer::escapeId( "ca-$key" ) . '">';
+                                echo '" id="' . Sanitizer::escapeIdForAttribute( "ca-$key" ) . '">';
                                 echo '<a href="';
                                 echo htmlspecialchars($tab['href']);
                                 echo '">';
@@ -329,7 +329,7 @@ include('/var/www/html/skins/Bootstrap2/footer.php');
 <?php   }
         if($this->data['feeds']) { ?>
             <li id="feedlinks"><?php foreach($this->data['feeds'] as $key => $feed) {
-                    ?><a id="<?php echo Sanitizer::escapeId( "feed-$key" ) ?>" href="<?php
+                    ?><a id="<?php echo Sanitizer::escapeIdForAttribute( "feed-$key" ) ?>" href="<?php
                     echo htmlspecialchars($feed['href']) ?>" rel="alternate" type="application/<?php echo $key ?>+xml" class="feedlink"<?php echo $this->skin->tooltipAndAccesskey('feed-'.$key) ?>><?php echo htmlspecialchars($feed['text'])?></a>&nbsp;
                     <?php } ?></li><?php
         }
@@ -390,7 +390,7 @@ include('/var/www/html/skins/Bootstrap2/footer.php');
         }
 
         //wfRunHooks( 'BootstrapTemplateToolboxEnd', array( &$this ) );
-        wfRunHooks( 'BootstrapTemplateToolboxEnd', array( &$this ) );
+        Hooks::run( 'BootstrapTemplateToolboxEnd', array( &$this ) );
 ?>
             </ul>
 <!--
@@ -429,7 +429,7 @@ include('/var/www/html/skins/Bootstrap2/footer.php');
 
 <?php   if ( is_array( $cont ) ) { ?>
             <ul class="nav nav-list">
-                <li class="nav-header"><?php $out = wfMsg( $bar ); if (wfEmptyMsg($bar, $out)) echo htmlspecialchars($bar); else echo htmlspecialchars($out); ?></li>
+                <li class="nav-header"><?php $msg = wfMessage( $bar ); if ($msg->isDisabled()) echo htmlspecialchars($bar); else echo htmlspecialchars($msg->text()); ?></li>
 <?php           foreach($cont as $key => $val) { ?>
                 <li id="<?php echo Sanitizer::escapeId($val['id']) ?>"<?php
                     if ( $val['active'] ) { ?> class="active" <?php }

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/footer.php

@@ -11,7 +11,7 @@
     </span>
     Made from the command line with vim by 
     <a href="http://charlesreid1.com">charlesreid1</a><br />
-    with help from <a href="https://getbootstrap.com/">Bootstrap</a> and <a href="http://getpelican.com">Pelican</a>.
+    with help from <a href="https://getbootstrap.com/">Bootstrap</a> and <a href="http://mediawiki.org">MediaWiki</a>.
 </p>
 
 <p style="text-align: center">

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/main.css

@@ -518,8 +518,18 @@ a.new:visited {
         color: #a55858;
 }
 
-span.editsection {
+.mw-editsection, .editsection {
 	font-size: small;
+	font-weight: normal;
+	margin-left: 1em;
+}
+
+.editOptions {
+    background-color: #777;
+}
+
+.mw-editsection-bracket {
+    margin-left: 0;
 }
 
 #preftoc {

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/navbar.php.j2

@@ -6,14 +6,14 @@
                 <span class="icon-bar"></span>
                 <span class="icon-bar"></span> 
             </button>
-            <a href="/" class="navbar-brand">{{ top_domain }}</a>
+            <a href="/" class="navbar-brand">{{ pod_charlesreid1_server_name }}</a>
         </div>
         <div>
             <div class="collapse navbar-collapse" id="myNavbar">
                 <ul class="nav navbar-nav">
 
                     <li>
-                        <a href="https://{{ top_domain }}/wiki">Wiki</a>
+                        <a href="https://{{ pod_charlesreid1_server_name }}/wiki">Wiki</a>
                     </li>
 
                 </ul>

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/slate.css

@@ -1086,7 +1086,8 @@ html {
 }
 body {
   font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-  font-size: 14px;
+  /*font-size: 14px;*/
+  font-size: 20px;
   line-height: 1.42857143;
   color: #c8c8c8;
   background-color: #272b30;

d-mediawiki/php/php.ini

@@ -0,0 +1,6 @@
+post_max_size = 128M
+memory_limit = 128M
+upload_max_filesize = 100M
+display_errors = Off
+log_errors = On
+error_log = /var/log/apache2/php_errors.log

d-mysql/Dockerfile

@@ -1,11 +1,7 @@
-FROM mysql:5.7
+FROM mysql:8.0
 MAINTAINER charles@charlesreid1.com
 
 # make mysql data a volume
 VOLUME ["/var/lib/mysql"]
 
-# put password in a password file
-RUN printf "[client]\nuser=root\npassword=$MYSQL_ROOT_PASSWORD" > /root/.mysql.rootpw.cnf
-RUN chmod 0600 /root/.mysql.rootpw.cnf
-
 RUN chown mysql:mysql /var/lib/mysql

d-mysql/conf.d/slow-log.cnf

@@ -0,0 +1,5 @@
+[mysqld]
+slow_query_log = 1
+slow_query_log_file = /var/log/mysql/mysql-slow.log
+long_query_time = 2
+log_queries_not_using_indexes = 0

d-nginx-charlesreid1/conf.d/_.conf

@@ -1,6 +0,0 @@
-# https://serverfault.com/a/525011
-server {
-    server_name _;
-    listen *:80 default_server deferred;
-    return 444;
-}

d-nginx-charlesreid1/conf.d/http.DOMAIN.conf.j2

@@ -1,6 +1,6 @@
 ####################
 # 
-# {{ server_name_default }}
+# {{ pod_charlesreid1_server_name }}
 # http/{{ port_default }}
 # 
 # basically, just redirects to https
@@ -10,20 +10,20 @@
 server {
     listen 80;
     listen [::]:80;
-    server_name {{ server_name_default }};
+    server_name {{ pod_charlesreid1_server_name }};
-    return 301 https://{{ server_name_default }}$request_uri;
+    return 301 https://{{ pod_charlesreid1_server_name }}$request_uri;
 }
 
 server {
     listen 80;
     listen [::]:80;
-    server_name www.{{ server_name_default }};
+    server_name www.{{ pod_charlesreid1_server_name }};
-    return 301 https://www.{{ server_name_default }}$request_uri;
+    return 301 https://www.{{ pod_charlesreid1_server_name }}$request_uri;
 }
 
 server {
     listen 80;
     listen [::]:80;
-    server_name git.{{ server_name_default }};
+    server_name git.{{ pod_charlesreid1_server_name }};
-    return 301 https://git.{{ server_name_default }}$request_uri;
+    return 301 https://git.{{ pod_charlesreid1_server_name }}$request_uri;
 }

d-nginx-charlesreid1/conf.d/https.DOMAIN.conf.j2

@@ -1,9 +1,9 @@
 ####################
 #
-# {{ server_name_default }}
+# {{ pod_charlesreid1_server_name }}
 # https/443
 # 
-# {{ server_name_default }} and www.{{ server_name_default }}
+# {{ pod_charlesreid1_server_name }} and www.{{ pod_charlesreid1_server_name }}
 # should handle the following cases:
 # - w/ and wiki/ should reverse proxy story_mw
 # - gitea subdomain should reverse proxy stormy_gitea
@@ -15,30 +15,46 @@
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
-    server_name {{ server_name_default }} default_server;
+    server_name {{ pod_charlesreid1_server_name }};
 
-    ssl_certificate /etc/letsencrypt/live/{{ server_name_default }}/fullchain.pem;
+    ssl_certificate /etc/letsencrypt/live/{{ pod_charlesreid1_server_name }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/{{ server_name_default }}/privkey.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ pod_charlesreid1_server_name }}/privkey.pem;
     include /etc/letsencrypt/options-ssl-nginx.conf;
     include /etc/nginx/conf.d/secheaders.conf;
     include /etc/nginx/conf.d/csp.conf;
 
     location / {
         try_files $uri $uri/ =404;
-        root /www/{{ server_name_default }}/htdocs;
+        root /www/{{ pod_charlesreid1_server_name }}/htdocs;
         index index.html;
     }
 
+    location = /robots.txt {
+        alias /var/www/robots/robots.txt;
+    }
+
     location /wiki/ {
+        # Apply rate limit here.
+        limit_req zone=gitealimit burst=20 nodelay;
+        # Limit download rate to 500 KB/s per connection (4 Mbps)
+        limit_rate 500k;
+
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;
         proxy_pass http://stormy_mw:8989/wiki/;
     }
 
     location /w/ {
+        # Apply rate limit here.
+        limit_req zone=gitealimit burst=20 nodelay;
+        # Limit download rate to 500 KB/s per connection (4 Mbps)
+        limit_rate 500k;
+
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;
         proxy_pass http://stormy_mw:8989/w/;
     }
@@ -55,31 +71,43 @@ server {
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
-    server_name www.{{ server_name_default }};
+    server_name www.{{ pod_charlesreid1_server_name }};
 
-    ssl_certificate /etc/letsencrypt/live/www.{{ server_name_default }}/fullchain.pem;
+    ssl_certificate /etc/letsencrypt/live/www.{{ pod_charlesreid1_server_name }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/www.{{ server_name_default }}/privkey.pem;
+    ssl_certificate_key /etc/letsencrypt/live/www.{{ pod_charlesreid1_server_name }}/privkey.pem;
     include /etc/letsencrypt/options-ssl-nginx.conf;
     include /etc/nginx/conf.d/secheaders.conf;
     include /etc/nginx/conf.d/csp.conf;
 
-    root /www/{{ server_name_default }}/htdocs;
+    root /www/{{ pod_charlesreid1_server_name }}/htdocs;
 
     location / {
         try_files $uri $uri/ =404;
         index index.html;
     }
 
+    location = /robots.txt {
+        alias /var/www/robots/robots.txt;
+    }
+
     location /wiki/ {
+        limit_req zone=gitealimit burst=20 nodelay;
+        limit_rate 500k;
+
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;
         proxy_pass http://stormy_mw:8989/wiki/;
     }
 
     location /w/ {
+        # Apply rate limit here.
+        limit_req zone=gitealimit burst=20 nodelay;
+
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;
         proxy_pass http://stormy_mw:8989/w/;
     }
@@ -94,18 +122,29 @@ server {
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
-    server_name git.{{ server_name_default }};
+    server_name git.{{ pod_charlesreid1_server_name }};
 
-    ssl_certificate /etc/letsencrypt/live/git.{{ server_name_default }}/fullchain.pem;
+    ssl_certificate /etc/letsencrypt/live/git.{{ pod_charlesreid1_server_name }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/git.{{ server_name_default }}/privkey.pem;
+    ssl_certificate_key /etc/letsencrypt/live/git.{{ pod_charlesreid1_server_name }}/privkey.pem;
     include /etc/letsencrypt/options-ssl-nginx.conf;
     include /etc/nginx/conf.d/secheaders.conf;
     include /etc/nginx/conf.d/giteacsp.conf;
 
     location / {
+        # Apply the rate limit here.
+        # Allows a burst of 20 requests, but anything beyond the max is queued.
+        limit_req zone=gitealimit burst=20 nodelay;
+        # Limit download rate to 500 KB/s per connection (4 Mbps)
+        limit_rate 500k;
+
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;
         proxy_pass http://stormy_gitea:3000/;
     }
+
+    location = /robots.txt {
+        alias /var/www/robots/gitea.txt;
+    }
 }

d-nginx-charlesreid1/nginx.conf

@@ -0,0 +1,37 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log notice;
+pid        /run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+
+    # Gitea rate limiting: 
+    # 5 requests per second rate limit
+    limit_req_zone $binary_remote_addr zone=gitealimit:10m rate=5r/s;
+
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}
+

d-nginx-charlesreid1/robots/gitea.txt

@@ -0,0 +1,16 @@
+User-agent: *
+Disallow: */commit/*
+Disallow: */src/*
+Disallow: */tree/*
+Disallow: */activity/*
+Disallow: */wiki/*
+Disallow: */releases/*
+Disallow: */pulls/*
+Disallow: */stars
+Disallow: */watchers
+Disallow: */forks
+Disallow: *?tab=activity
+Disallow: *?tab=stars
+Disallow: *?tab=following
+Disallow: *?tab=followers
+Disallow: *?lang=*

d-nginx-charlesreid1/robots/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /w/

docker-compose.yml.j2

@@ -5,7 +5,7 @@ services:
 # https://stackoverflow.com/a/39039830
 
   stormy_gitea:
-    image: gitea/gitea:latest
+    image: gitea/gitea:1.24.5
     container_name: stormy_gitea
     environment:
       - USER_UID=1000
@@ -13,6 +13,7 @@ services:
     restart: always
     volumes:
       - "stormy_gitea_data:/data"
+      - "./d-nginx-charlesreid1/robots:/var/www/robots:ro"
       - "./d-gitea/custom:/data/gitea"
       - "./d-gitea/data:/app/gitea/data"
       - "/gitea_repositories:/data/git/repositories"
@@ -23,53 +24,91 @@ services:
         max-file: "10"
     ports:
       - "22:22"
+    networks:
+      - frontend
+
+  stormy_gitea_runner:
+    image: gitea/act_runner:latest
+    container_name: stormy_gitea_runner
+    restart: always
+    volumes:
+      - "stormy_gitea_runner_data:/data"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "./d-gitea/runner/config.yaml:/etc/act_runner/config.yaml:ro"
+    environment:
+      - GITEA_INSTANCE_URL=http://stormy_gitea:3000
+      - GITEA_RUNNER_REGISTRATION_TOKEN={{ pod_charlesreid1_gitea_runner_token }}
+      - GITEA_RUNNER_NAME=stormy-runner
+      - CONFIG_FILE=/etc/act_runner/config.yaml
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 1m
+        max-file: "10"
+    depends_on:
+      - stormy_gitea
+    networks:
+      - frontend
 
   stormy_mysql:
+    restart: always
     build: d-mysql
     container_name: stormy_mysql
     volumes:
       - "stormy_mysql_data:/var/lib/mysql"
+      - "./d-mysql/conf.d:/etc/mysql/conf.d:ro"
     logging:
       driver: "json-file"
       options:
         max-size: 1m
         max-file: "10"
     environment:
-      - MYSQL_ROOT_PASSWORD={{ mysql_password }}
+      - MYSQL_ROOT_PASSWORD={{ pod_charlesreid1_mysql_password }}
+      - MYSQL_DATABASE=wikidb
+      - MYSQL_USER=wikiuser
+      - MYSQL_PASSWORD={{ pod_charlesreid1_mysql_wikiuser_password }}
+    networks:
+      - backend
 
   stormy_mw:
+    restart: always
     build: d-mediawiki
     container_name: stormy_mw
     volumes:
-      - "stormy_mw_data:/var/www/html"
+      - "stormy_mw_images:/var/www/html/images"
-      - "./mwf2b:/var/log/mwf2b"
     logging:
       driver: "json-file"
       options:
         max-size: 1m
         max-file: "10"
     environment:
-      - MEDIAWIKI_SITE_SERVER=https://{{ server_name_default }}
+      - MEDIAWIKI_SITE_SERVER=https://{{ pod_charlesreid1_server_name }}
-      - MEDIAWIKI_SECRETKEY={{ mediawiki_secretkey }}
+      - MEDIAWIKI_SECRETKEY={{ pod_charlesreid1_mediawiki_secretkey }}
+      - MEDIAWIKI_UPGRADEKEY={{ pod_charlesreid1_mediawiki_upgradekey }}
       - MYSQL_HOST=stormy_mysql
       - MYSQL_DATABASE=wikidb
-      - MYSQL_USER=root
+      - MYSQL_USER=wikiuser
-      - MYSQL_PASSWORD={{ mysql_password }}
+      - MYSQL_PASSWORD={{ pod_charlesreid1_mysql_wikiuser_password }}
     depends_on:
       - stormy_mysql
+    networks:
+      - frontend
+      - backend
 
   stormy_nginx:
     restart: always
-    image: nginx
+    image: nginx:1.27.5
     container_name: stormy_nginx
-    hostname: {{ server_name_default }}
+    hostname: {{ pod_charlesreid1_server_name }}
-    hostname: charlesreid1.com
     command: /bin/bash -c "nginx -g 'daemon off;'"
     volumes:
+      - "./d-nginx-charlesreid1/nginx.conf:/etc/nginx/nginx.conf:ro"
       - "./d-nginx-charlesreid1/conf.d:/etc/nginx/conf.d:ro"
+      - "./d-nginx-charlesreid1/robots:/var/www/robots:ro"
       - "/etc/localtime:/etc/localtime:ro"
-      - "/etc/letsencrypt:/etc/letsencrypt"
+      - "/etc/letsencrypt:/etc/letsencrypt:ro"
-      - "/www/{{ server_name_default }}/htdocs:/www/{{ server_name_default }}/htdocs:ro"
+      - "/www/{{ pod_charlesreid1_server_name }}/htdocs:/www/{{ pod_charlesreid1_server_name }}/htdocs:ro"
+      - "stormy_nginx_logs:/var/log/nginx"
     logging:
       driver: "json-file"
       options:
@@ -82,8 +121,19 @@ services:
     ports:
       - "80:80"
       - "443:443"
+    networks:
+      - frontend
+
+networks:
+  frontend:
+  backend:
 
 volumes:
   stormy_mysql_data:
+  stormy_mw_images:
+    external: true
   stormy_mw_data:
+    external: true
   stormy_gitea_data:
+  stormy_gitea_runner_data:
+  stormy_nginx_logs:

docs/BlockIps.md

@@ -0,0 +1,9 @@
+To block IP address:
+
+* Modify the nginx config file template at 
+  `d-nginx-charlesreid1/conf.d/https.DOMAIN.conf.j2`
+* Re-render the Jinja templates into config files via
+  `make clean-templates && make templates`
+* Stop and restart the pod service:
+  `sudo systemctl stop pod-charlesreid1 && 
+  sudo systemctl start pod-charlesreid1`

environment.example

@@ -5,15 +5,18 @@
 export POD_CHARLESREID1_DIR="/path/to/pod-charlesreid1"
 export POD_CHARLESREID1_TLD="example.com"
 export POD_CHARLESREID1_USER="nonrootuser"
+export POD_CHARLESREID1_VPN_IP_ADDR="1.2.3.4"
 
 # mediawiki:
 # ----------
 export POD_CHARLESREID1_MW_ADMIN_EMAIL="email@example.com"
 export POD_CHARLESREID1_MW_SECRET_KEY="SecretKeyString"
+export POD_CHARLESREID1_MW_UPGRADE_KEY="UpgradeKeyString"
 
 # mysql:
 # ------
 export POD_CHARLESREID1_MYSQL_PASSWORD="SuperSecretPassword"
+export POD_CHARLESREID1_MYSQL_WIKIUSER_PASSWORD="AnotherSecretPassword"
 
 # gitea:
 # ------
@@ -29,6 +32,6 @@ export AWS_DEFAULT_REGION="us-west-1"
 
 # backups and scripts:
 # --------------------
-export POD_CHARLESREID1_USER="charles"
+export POD_CHARLESREID1_BACKUP_DIR="/path/to"
 export POD_CHARLESREID1_BACKUP_S3BUCKET="name-of-backups-bucket"
 export POD_CHARLESREID1_CANARY_WEBHOOK="https://hooks.slack.com/services/000000000/AAAAAAAAA/111111111111111111111111"

environment.j2

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# multiple templates:
+# -------------------
+export POD_CHARLESREID1_DIR="{{ pod_charlesreid1_pod_install_dir }}"
+export POD_CHARLESREID1_TLD="{{ pod_charlesreid1_server_name }}"
+export POD_CHARLESREID1_USER="{{ pod_charlesreid1_username }}"
+export POD_CHARLESREID1_VPN_IP_ADDR="{{ pod_charlesreid1_vpn_ip_addr }}"
+
+# mediawiki:
+# ----------
+export POD_CHARLESREID1_MW_ADMIN_EMAIL="{{ pod_charlesreid1_mediawiki_admin_email }}"
+export POD_CHARLESREID1_MW_SECRET_KEY="{{ pod_charlesreid1_mediawiki_secretkey }}"
+export POD_CHARLESREID1_MW_UPGRADE_KEY="{{ pod_charlesreid1_mediawiki_upgradekey }}"
+
+# mysql:
+# ------
+export POD_CHARLESREID1_MYSQL_PASSWORD="{{ pod_charlesreid1_mysql_password }}"
+export POD_CHARLESREID1_MYSQL_WIKIUSER_PASSWORD="{{ pod_charlesreid1_mysql_wikiuser_password }}"
+
+# gitea:
+# ------
+export POD_CHARLESREID1_GITEA_APP_NAME="{{ pod_charlesreid1_gitea_app_name }}"
+export POD_CHARLESREID1_GITEA_SECRET_KEY="{{ pod_charlesreid1_gitea_secretkey }}"
+export POD_CHARLESREID1_GITEA_INTERNAL_TOKEN="{{ pod_charlesreid1_gitea_internaltoken }}"
+
+# aws:
+# ----
+export AWS_ACCESS_KEY_ID="{{ pod_charlesreid1_backups_aws_access_key }}"
+export AWS_SECRET_ACCESS_KEY="{{ pod_charlesreid1_backups_aws_secret_access_key }}"
+export AWS_DEFAULT_REGION="{{ pod_charlesreid1_backups_aws_region }}"
+
+# backups and scripts:
+# --------------------
+export POD_CHARLESREID1_BACKUP_DIR="{{ pod_charlesreid1_backups_dir }}"
+export POD_CHARLESREID1_BACKUP_S3BUCKET="{{ pod_charlesreid1_backups_bucket }}"
+export POD_CHARLESREID1_CANARY_WEBHOOK="{{ pod_charlesreid1_backups_canary_slack_url }}"

scripts/apply_templates.py

@@ -8,25 +8,29 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape
 
 
 # Should existing files be overwritten
-OVERWRITE = False
+OVERWRITE = True
 
 # Map of jinja variables to environment variables
 jinja_to_env = {
-    "pod_install_dir": "POD_CHARLESREID1_DIR",
+    "pod_charlesreid1_pod_install_dir": "POD_CHARLESREID1_DIR",
-    "top_domain": "POD_CHARLESREID1_TLD",
+    "pod_charlesreid1_server_name": "POD_CHARLESREID1_TLD",
-    "server_name_default" : "POD_CHARLESREID1_TLD",
+    "pod_charlesreid1_username": "POD_CHARLESREID1_USER",
-    "username": "POD_CHARLESREID1_USER",
+    "pod_charlesreid1_vpn_ip_addr": "POD_CHARLESREID1_VPN_IP_ADDR",
-    # docker-compose:
+    "pod_charlesreid1_mediawiki_admin_email": "POD_CHARLESREID1_MW_ADMIN_EMAIL",
-    "mysql_password" : "POD_CHARLESREID1_MYSQL_PASSWORD",
+    "pod_charlesreid1_mediawiki_secretkey": "POD_CHARLESREID1_MW_SECRET_KEY",
-    "mediawiki_secretkey" : "POD_CHARLESREID1_MW_SECRET_KEY",
+    "pod_charlesreid1_mediawiki_upgradekey": "POD_CHARLESREID1_MW_UPGRADE_KEY",
-    # mediawiki:
+    "pod_charlesreid1_mysql_password": "POD_CHARLESREID1_MYSQL_PASSWORD",
-    "admin_email": "POD_CHARLESREID1_MW_ADMIN_EMAIL",
+    "pod_charlesreid1_mysql_wikiuser_password": "POD_CHARLESREID1_MYSQL_WIKIUSER_PASSWORD",
-    # gitea:
+    "pod_charlesreid1_gitea_app_name": "POD_CHARLESREID1_GITEA_APP_NAME",
-    "gitea_app_name": "POD_CHARLESREID1_GITEA_APP_NAME",
+    "pod_charlesreid1_gitea_secretkey": "POD_CHARLESREID1_GITEA_SECRET_KEY",
-    "gitea_secret_key": "POD_CHARLESREID1_GITEA_SECRET_KEY",
+    "pod_charlesreid1_gitea_internaltoken": "POD_CHARLESREID1_GITEA_INTERNAL_TOKEN",
-    "gitea_internal_token": "POD_CHARLESREID1_GITEA_INTERNAL_TOKEN",
+    "pod_charlesreid1_gitea_runner_token": "POD_CHARLESREID1_GITEA_RUNNER_TOKEN",
-    # aws:
+    "pod_charlesreid1_backups_aws_access_key": "AWS_ACCESS_KEY_ID",
-    "backup_canary_webhook_url": "POD_CHARLESREID1_CANARY_WEBHOOK",
+    "pod_charlesreid1_backups_aws_secret_access_key": "AWS_SECRET_ACCESS_KEY",
+    "pod_charlesreid1_backups_aws_region": "AWS_DEFAULT_REGION",
+    "pod_charlesreid1_backups_dir": "POD_CHARLESREID1_BACKUP_DIR",
+    "pod_charlesreid1_backups_bucket": "POD_CHARLESREID1_BACKUP_S3BUCKET",
+    "pod_charlesreid1_backups_canary_slack_url": "POD_CHARLESREID1_CANARY_WEBHOOK",
 }
 
 scripts_dir = os.path.dirname(os.path.abspath(__file__))

scripts/backups/10-pod-charlesreid1-rsyslog.conf

@@ -0,0 +1,28 @@
+if ( $programname startswith "pod-charlesreid1-canary" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-canary.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-certbot" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-certbot.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-aws" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-aws.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-cleanolderthan" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-cleanolderthan.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-gitea" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-gitea.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-wikidb" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-wikidb.service.log" flushOnTXEnd="on")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-wikifiles" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-wikifiles.service.log" flushOnTXEnd="on")
+    stop
+}

scripts/backups/Readme.md

@@ -13,3 +13,40 @@ for the systemd service.
 
 Use `make install` in the top level of this repo to install
 the rendered service and timer files.
+
+## syslog filtering
+
+Due to a bug in systemd bundled with Ubuntu 18.04, we can't just use the nice easy solution of
+directing output and error to a specific file.
+
+Instead, the services all send their stderr and stdout to the system log, and then rsyslog
+filters those messages and collects them into a separate log file.
+
+First, install the services.
+
+Then, install the following rsyslog config file:
+
+`/etc/rsyslog.d/10-pod-charlesreid1-rsyslog.conf`:
+
+```
+if $programname == 'pod-charlesreid1-canary' then /var/log/pod-charlesreid1-canary.service.log
+if $programname == 'pod-charlesreid1-canary' then stop
+
+if $programname == 'pod-charlesreid1-backups-aws' then /var/log/pod-charlesreid1-backups-aws.service.log
+if $programname == 'pod-charlesreid1-backups-aws' then stop
+
+if $programname == 'pod-charlesreid1-backups-cleanolderthan' then /var/log/pod-charlesreid1-backups-cleanolderthan.service.log
+if $programname == 'pod-charlesreid1-backups-cleanolderthan' then stop
+
+if $programname == 'pod-charlesreid1-backups-gitea' then /var/log/pod-charlesreid1-backups-gitea.service.log
+if $programname == 'pod-charlesreid1-backups-gitea' then stop
+
+if $programname == 'pod-charlesreid1-backups-wikidb' then /var/log/pod-charlesreid1-backups-wikidb.service.log
+if $programname == 'pod-charlesreid1-backups-wikidb' then stop
+
+if $programname == 'pod-charlesreid1-backups-wikifiles' then /var/log/pod-charlesreid1-backups-wikifiles.service.log
+if $programname == 'pod-charlesreid1-backups-wikifiles' then stop
+```
+
+
+

scripts/backups/aws_backup.sh

@@ -37,22 +37,22 @@ if [ "$#" == "0" ]; then
     echo ""
 
     echo "Checking that directory exists"
-    /usr/bin/test -d ${POD_CHARLESREID1_BACKUP_DIR}
+    /usr/bin/test -d "${POD_CHARLESREID1_BACKUP_DIR}"
 
     echo "Checking that we can access the S3 bucket"
-    aws s3 ls s3://${POD_CHARLESREID1_BACKUP_S3BUCKET} > /dev/null
+    aws s3 ls "s3://${POD_CHARLESREID1_BACKUP_S3BUCKET}" > /dev/null
-    
+
     # Get name of last backup, to copy to AWS
-    LAST_BACKUP=$(/bin/ls -1 -t ${POD_CHARLESREID1_BACKUP_DIR} | /usr/bin/head -n1)
+    LAST_BACKUP=$(/bin/ls -1 -t "${POD_CHARLESREID1_BACKUP_DIR}" | /usr/bin/head -n1)
     echo "Last backup found: ${LAST_BACKUP}"
     echo "Last backup directory: ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP}"
 
-    BACKUP_SIZE=$(du -hs ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP})
+    BACKUP_SIZE=$(/usr/bin/du -hs "${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP}" | cut -f 1)
     echo "Backup directory size: ${BACKUP_SIZE}"
 
     # Copy to AWS
     echo "Backing up directory ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP}"
-    aws s3 cp --only-show-errors --recursive ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP} s3://${POD_CHARLESREID1_BACKUP_S3BUCKET}/backups/${LAST_BACKUP}
+    aws s3 cp --only-show-errors --no-progress --recursive "${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP}" "s3://${POD_CHARLESREID1_BACKUP_S3BUCKET}/backups/${LAST_BACKUP}"
     echo "Done."
 
 else

scripts/backups/canary/backups_canary.py

@@ -24,7 +24,7 @@ def main():
         alert(msg)
 
     # verify there is a backup newer than N days
-    newer_backups = subprocess.getoutput(f'find {backup_dir} -mtime -{N}').split('\n')
+    newer_backups = subprocess.getoutput(f'find {backup_dir}/* -mtime -{N}').split('\n')
     if len(newer_backups)==1 and newer_backups[0]=='':
         msg = "Local Backups Error:\n"
         msg += f"The backup directory `{backup_dir}` is missing backup files from the last {N} day(s)!"
@@ -35,7 +35,7 @@ def main():
     newest_backup_files = subprocess.getoutput(f'find {newest_backup_path} -type f').split('\n')
 
     # verify the most recent backup directory is not empty
-    if len(newest_backup_files)==1 and newer_backups[0]=='':
+    if len(newest_backup_files)==1 and newest_backup_files[0]=='':
         msg = "Local Backups Error:\n"
         msg += f"The most recent backup directory `{newest_backup_path}` is empty!"
         alert(msg)
@@ -48,6 +48,24 @@ def main():
             msg += f"Backup file name: {backup_file}!"
             alert(msg)
 
+    # verify .sql dumps end with the mysqldump completion trailer.
+    # A non-empty file can still be truncated mid-row (e.g. PTY deadlock,
+    # net_write_timeout) — without this check, a 439 MB partial dump looks
+    # healthy to a size-only canary.
+    for backup_file in newest_backup_files:
+        if not backup_file.endswith('.sql'):
+            continue
+        with open(backup_file, 'rb') as f:
+            f.seek(0, os.SEEK_END)
+            f.seek(max(0, f.tell() - 512))
+            tail = f.read()
+        if b'Dump completed on' not in tail:
+            msg = "Local Backups Error:\n"
+            msg += f"SQL backup file `{backup_file}` is missing the "
+            msg += "`-- Dump completed on ...` trailer.\n"
+            msg += "mysqldump did not finish — the dump is truncated and not restorable."
+            alert(msg)
+
     # verify the most recent backup files exist in the s3 backups bucket
     bucket_base_path = os.path.join('backups', newest_backup_name)
     for backup_file in newest_backup_files:
@@ -64,10 +82,12 @@ def check_exists(bucket_name, bucket_path):
             # File does not exist
             msg = "S3 Backups Error:\n"
             msg += f"Failed to find the file `{bucket_path}` in bucket `{bucket_name}`"
+            alert(msg)
         else:
             # Problem accessing backups on bucket
             msg = "S3 Backups Error:\n"
             msg += f"Failed to access the file `{bucket_path}` in bucket `{bucket_name}`"
+            alert(msg)
 
 
 def alert(msg):
@@ -97,7 +117,7 @@ def alert(msg):
         raise Exception(response.status_code, response.text)
 
     print("Goodbye.")
-    sys.exit(1)
+    sys.exit(0)
 
 
 if __name__ == '__main__':

scripts/backups/canary/pod-charlesreid1-canary.service.j2

@@ -5,9 +5,10 @@ After=docker.service
 
 [Service]
 Type=oneshot
-StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-canary.service.error.log
+StandardError=syslog
-StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-canary.service.output.log
+StandardOutput=syslog
-ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/backups/canary/vp/bin/python3 {{ pod_install_dir }}/scripts/backups/canary/backups_canary.py'
+SyslogIdentifier=pod-charlesreid1-canary
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; /home/charles/.pyenv/shims/python3 {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/canary/backups_canary.py'
 User=charles
 Group=charles
-

scripts/backups/canary/pod-charlesreid1-canary.timer.j2

@@ -2,7 +2,7 @@
 Description=Timer to run the pod-charlesreid1 backups canary
 
 [Timer]
-OnCalendar=Sun *-*-* 9:03:00
+OnCalendar=*-*-* 7:01:00
 
 [Install]
 WantedBy=timers.target

scripts/backups/clean_olderthan.sh

@@ -6,7 +6,7 @@ set -eux
 
 # Number of days of backups to retain.
 # Everything older than this many days will be deleted
-N="45"
+N="22"
 
 function usage {
     set +x
@@ -39,7 +39,7 @@ if [ "$#" == "0" ]; then
     echo "Backup directory: ${POD_CHARLESREID1_BACKUP_DIR}"
     echo ""
 
-    echo "Cleaning backups directory $BACKUP_DIR"
+    echo "Cleaning backups directory $POD_CHARLESREID1_BACKUP_DIR"
     echo "The following files older than $N days will be deleted:"
     find ${POD_CHARLESREID1_BACKUP_DIR} -mtime +${N}
 

scripts/backups/gitea_backup.sh

@@ -53,7 +53,7 @@ if [ "$#" == "0" ]; then
     # We don't need to use docker, since these directories
     # are both bind-mounted into the Docker container
     echo "Backing up custom directory"
-    tar czf ${CUSTOM_TARGET} ${POD_CHARLESREID1_DIR}/d-gitea/custom
+    tar --exclude='gitea.log' --ignore-failed-read -czf ${CUSTOM_TARGET} ${POD_CHARLESREID1_DIR}/d-gitea/custom
     echo "Backing up data directory"
     tar czf ${DATA_TARGET} ${POD_CHARLESREID1_DIR}/d-gitea/data
 

scripts/backups/pod-charlesreid1-backups-aws.service.j2

@@ -5,10 +5,10 @@ After=docker.service
 
 [Service]
 Type=oneshot
-StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-aws.service.error.log
+StandardError=syslog
-StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-aws.service.output.log
+StandardOutput=syslog
-ExecStartPre=/usr/bin/test -f {{ pod_install_dir }}/environment
+SyslogIdentifier=pod-charlesreid1-backups-aws
-ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/backups/aws_backup.sh'
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/aws_backup.sh'
 User=charles
 Group=charles
-

scripts/backups/pod-charlesreid1-backups-aws.timer.j2

@@ -3,6 +3,7 @@ Description=Timer to copy the lastest pod-charlesreid1 backup to an S3 bucket
 
 [Timer]
 OnCalendar=Sun *-*-* 2:56:00
+#OnCalendar=*-*-* 2:56:00
 
 [Install]
 WantedBy=timers.target

scripts/backups/pod-charlesreid1-backups-cleanolderthan.service.j2

@@ -1,12 +1,14 @@
 [Unit]
-Description=Copy the latest pod-charlesreid1 backup to an S3 bucket
+Description=Clean pod-charlesreid1 backups older than N days
 Requires=docker.service
 After=docker.service
 
 [Service]
 Type=oneshot
-StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-cleanolderthan.service.error.log
+StandardError=syslog
-StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-cleanolderthan.service.output.log
+StandardOutput=syslog
-ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/backups/clean_olderthan.sh'
+SyslogIdentifier=pod-charlesreid1-backups-cleanolderthan
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/clean_olderthan.sh'
 User=charles
 Group=charles

scripts/backups/pod-charlesreid1-backups-cleanolderthan.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Timer to clean files older than N days from the pod-charlesreid1 backups dir
+
+[Timer]
+OnCalendar=Sun *-*-* 2:28:00
+#OnCalendar=*-*-* 2:28:00
+
+[Install]
+WantedBy=timers.target

scripts/backups/pod-charlesreid1-backups-gitea.service.j2

@@ -5,10 +5,10 @@ After=docker.service
 
 [Service]
 Type=oneshot
-StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-gitea.service.error.log
+StandardError=syslog
-StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-gitea.service.output.log
+StandardOutput=syslog
-ExecStartPre=/usr/bin/test -f {{ pod_install_dir }}/environment
+SyslogIdentifier=pod-charlesreid1-backups-gitea
-ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/backups/gitea_backup.sh'
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/gitea_backup.sh'
 User=charles
 Group=charles
-

scripts/backups/pod-charlesreid1-backups-gitea.timer.j2

@@ -3,6 +3,7 @@ Description=Timer to back up pod-charlesreid1 gitea files
 
 [Timer]
 OnCalendar=Sun *-*-* 2:12:00
+#OnCalendar=*-*-* 2:12:00
 
 [Install]
 WantedBy=timers.target

scripts/backups/pod-charlesreid1-backups-wikidb.service.j2

@@ -5,10 +5,10 @@ After=docker.service
 
 [Service]
 Type=oneshot
-StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-wikidb.service.error.log
+StandardError=syslog
-StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-wikidb.service.output.log
+StandardOutput=syslog
-ExecStartPre=/usr/bin/test -f {{ pod_install_dir }}/environment
+SyslogIdentifier=pod-charlesreid1-backups-wikidb
-ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/backups/wikidb_dump.sh'
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/wikidb_dump.sh'
 User=charles
 Group=charles
-

scripts/backups/pod-charlesreid1-backups-wikifiles.service.j2

@@ -1,13 +1,14 @@
 [Unit]
-Description=Back up the pod-charlesreid1 wiki files
+Description=Back up pod-charlesreid1 wiki files
 Requires=docker.service
 After=docker.service
 
 [Service]
 Type=oneshot
-StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-wikifiles.service.error.log
+StandardError=syslog
-StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-wikifiles.service.output.log
+StandardOutput=syslog
-ExecStartPre=/usr/bin/test -f {{ pod_install_dir }}/environment
+SyslogIdentifier=pod-charlesreid1-backups-wikifiles
-ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/backups/wikifiles_dump.sh'
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/wikifiles_dump.sh'
 User=charles
 Group=charles

scripts/backups/pod-charlesreid1-backups-wikifiles.timer.j2

@@ -1,5 +1,5 @@
 [Unit]
-Description=Timer to back up the pod-charlesreid1 wiki files
+Description=Timer to back up pod-charlesreid1 wiki files
 
 [Timer]
 OnCalendar=Sun *-*-* 2:08:00

scripts/backups/wikidb_dump.sh

@@ -5,7 +5,8 @@
 set -eux
 
 CONTAINER_NAME="stormy_mysql"
-STAMP="`date +"%Y%m%d"`"
+DATESTAMP="`date +"%Y%m%d"`"
+TIMESTAMP="`date +"%Y%m%d_%H%M%S"`"
 
 function usage {
     set +x
@@ -20,7 +21,7 @@ function usage {
     echo "Example:"
     echo ""
     echo "       ./wikidb_dump.sh"
-    echo "       (creates ${POD_CHARLESREID1_BACKUP_DIR}/20200101/wikidb_20200101.sql)"
+    echo "       (creates ${POD_CHARLESREID1_BACKUP_DIR}/YYYYMMDD/wikidb_YYYYMMDD_HHMMSS.sql)"
     echo ""
     exit 1;
 }
@@ -36,26 +37,63 @@ fi
 
 if [ "$#" == "0" ]; then
 
-    TARGET="wikidb_${STAMP}.sql"
+    TARGET="wikidb_${TIMESTAMP}.sql"
-    BACKUP_TARGET="${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}/${TARGET}"
+    BACKUP_DIR="${POD_CHARLESREID1_BACKUP_DIR}/${DATESTAMP}"
+    BACKUP_TARGET="${BACKUP_DIR}/${TARGET}"
 
     echo ""
     echo "pod-charlesreid1: wikidb_dump.sh"
     echo "--------------------------------"
     echo ""
-    echo "Backup directory: ${POD_CHARLESREID1_BACKUP_DIR}"
+    echo "Backup directory: ${BACKUP_DIR}"
     echo "Backup target: ${BACKUP_TARGET}"
     echo ""
 
-    mkdir -p ${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}
+    mkdir -p "${BACKUP_DIR}"
-
-    DOCKER=$(which docker)
-    DOCKERX="${DOCKER} exec -t"
 
     echo "Running mysqldump inside the mysql container"
-    ${DOCKERX} ${CONTAINER_NAME} sh -c 'exec mysqldump wikidb --databases -uroot -p"$MYSQL_ROOT_PASSWORD"' 2>&1 | grep -v "Using a password" > ${BACKUP_TARGET}
 
-    echo "Done."
+    # Pull the root password out of the container so we don't duplicate the
+    # secret on the host, and forward it in via MYSQL_PWD (which mysqldump
+    # reads automatically). No -t: a PTY corrupts --default-character-set=binary
+    # output (LF→CRLF translation on binary blobs) and its small kernel buffer
+    # can deadlock on large dumps.
+    set +x
+    MYSQL_PWD="$(docker exec "${CONTAINER_NAME}" printenv MYSQL_ROOT_PASSWORD)"
+    export MYSQL_PWD
+    set -x
+
+    docker exec -i \
+        -e MYSQL_PWD \
+        "${CONTAINER_NAME}" \
+        sh -c 'exec mysqldump \
+                  --user=root \
+                  --single-transaction \
+                  --quick \
+                  --routines \
+                  --triggers \
+                  --events \
+                  --default-character-set=binary \
+                  --databases wikidb' \
+        > "${BACKUP_TARGET}"
+
+    unset MYSQL_PWD
+
+    # A complete mysqldump always ends with "-- Dump completed on ...".
+    # Missing trailer means the dump is truncated and not restorable.
+    if ! tail -c 200 "${BACKUP_TARGET}" | grep -q 'Dump completed on'; then
+        echo "ERROR: dump file ${BACKUP_TARGET} is missing the completion trailer." >&2
+        echo "       mysqldump did not finish successfully." >&2
+        exit 2
+    fi
+
+    size=$(stat -c %s "${BACKUP_TARGET}")
+    if [ "${size}" -lt $((50 * 1024 * 1024)) ]; then
+        echo "ERROR: dump file ${BACKUP_TARGET} is only ${size} bytes; suspicious." >&2
+        exit 3
+    fi
+
+    echo "Dump OK: ${BACKUP_TARGET} (${size} bytes)"
 
 else
     usage

scripts/backups/wikidb_restore_test.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+#
+# Restore a wikidb dump into a throwaway MySQL 5.7 container and run sanity
+# queries against it. Compares row counts to live stormy_mysql. Exits non-zero
+# on any failure.
+#
+# Usage:
+#   ./wikidb_restore_test.sh <path-to-dump.sql>
+#
+# A backup is only a backup if you have actually restored from it.
+set -euo pipefail
+
+DUMP="${1:-}"
+if [ -z "${DUMP}" ] || [ ! -f "${DUMP}" ]; then
+    echo "Usage: $0 <path-to-wikidb-dump.sql>" >&2
+    exit 1
+fi
+
+LIVE_CONTAINER="stormy_mysql"
+TEST_CONTAINER="wikidb_restore_test_$$"
+TEST_PW="temp_restore_test_pw_$$"
+IMAGE="mysql:5.7"
+
+cleanup() {
+    docker stop "${TEST_CONTAINER}" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+echo "[1/5] Starting throwaway MySQL container ${TEST_CONTAINER}..."
+docker run -d --rm \
+    --name "${TEST_CONTAINER}" \
+    -e MYSQL_ROOT_PASSWORD="${TEST_PW}" \
+    "${IMAGE}" >/dev/null
+
+echo "[2/5] Waiting for MySQL to accept authenticated connections..."
+# `mysqladmin ping` returns OK before the root user is actually set up, so we
+# have to probe with a real authenticated query and accept only success.
+ready=0
+for i in $(seq 1 60); do
+    if docker exec -e MYSQL_PWD="${TEST_PW}" "${TEST_CONTAINER}" \
+        mysql -uroot -e 'SELECT 1' >/dev/null 2>&1; then
+        ready=1
+        break
+    fi
+    sleep 2
+done
+if [ "${ready}" -ne 1 ]; then
+    echo "ERROR: MySQL in ${TEST_CONTAINER} never became ready." >&2
+    docker logs "${TEST_CONTAINER}" 2>&1 | tail -20 >&2
+    exit 4
+fi
+
+echo "[3/5] Piping dump into throwaway MySQL..."
+docker exec -i -e MYSQL_PWD="${TEST_PW}" "${TEST_CONTAINER}" \
+    mysql -uroot < "${DUMP}"
+
+echo "[4/5] Querying restored DB..."
+restored=$(docker exec -e MYSQL_PWD="${TEST_PW}" "${TEST_CONTAINER}" \
+    mysql -uroot -N -B -e "
+        USE wikidb;
+        SELECT COUNT(*) FROM page;
+        SELECT COUNT(*) FROM revision;
+        SELECT COUNT(*) FROM text;
+        SELECT COALESCE(MAX(rev_timestamp), 'none') FROM revision;
+    ")
+
+echo "--- restored ---"
+echo "${restored}"
+
+echo "[5/5] Querying live ${LIVE_CONTAINER}..."
+LIVE_PW="$(docker exec "${LIVE_CONTAINER}" printenv MYSQL_ROOT_PASSWORD)"
+live=$(docker exec -e MYSQL_PWD="${LIVE_PW}" "${LIVE_CONTAINER}" \
+    mysql -uroot -N -B -e "
+        USE wikidb;
+        SELECT COUNT(*) FROM page;
+        SELECT COUNT(*) FROM revision;
+        SELECT COUNT(*) FROM text;
+        SELECT COALESCE(MAX(rev_timestamp), 'none') FROM revision;
+    ")
+
+echo "--- live ---"
+echo "${live}"
+
+r_page=$(echo "${restored}"   | sed -n '1p')
+r_rev=$(echo  "${restored}"   | sed -n '2p')
+r_text=$(echo "${restored}"   | sed -n '3p')
+l_page=$(echo "${live}"       | sed -n '1p')
+l_rev=$(echo  "${live}"       | sed -n '2p')
+l_text=$(echo "${live}"       | sed -n '3p')
+
+fail=0
+for kind in page rev text; do
+    r_var="r_${kind}"
+    l_var="l_${kind}"
+    r="${!r_var}"
+    l="${!l_var}"
+    if [ "${r}" != "${l}" ]; then
+        echo "MISMATCH: ${kind} count restored=${r} live=${l}" >&2
+        fail=1
+    else
+        echo "OK: ${kind} count = ${r}"
+    fi
+done
+
+if [ "${fail}" -ne 0 ]; then
+    echo "RESTORE TEST FAILED." >&2
+    exit 5
+fi
+
+echo "RESTORE TEST PASSED."

scripts/backups/wikifiles_dump.sh

@@ -5,7 +5,8 @@
 set -eux
 
 CONTAINER_NAME="stormy_mw"
-STAMP="`date +"%Y%m%d"`"
+DATESTAMP="`date +"%Y%m%d"`"
+TIMESTAMP="`date +"%Y%m%d_%H%M%S"`"
 
 function usage {
     set +x
@@ -20,7 +21,7 @@ function usage {
     echo "Example:"
     echo ""
     echo "       ./wikifiles_dump.sh"
-    echo "       (creates ${POD_CHARLESREID1_BACKUP_DIR}/20200101/wikifiles_20200101.tar.gz)"
+    echo "       (creates ${POD_CHARLESREID1_BACKUP_DIR}/YYYYMMDD/wikifiles_YYYYMMDD_HHMMSS.tar.gz)"
     echo ""
     exit 1;
 }
@@ -36,18 +37,19 @@ fi
 
 if [ "$#" == "0" ]; then
 
-    TARGET="wikifiles_${STAMP}.tar.gz"
+    TARGET="wikifiles_${TIMESTAMP}.tar.gz"
-    BACKUP_TARGET="${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}/${TARGET}"
+    BACKUP_DIR="${POD_CHARLESREID1_BACKUP_DIR}/${DATESTAMP}"
+    BACKUP_TARGET="${BACKUP_DIR}/${TARGET}"
 
     echo ""
     echo "pod-charlesreid1: wikifiles_dump.sh"
     echo "-----------------------------------"
     echo ""
-    echo "Backup directory: ${POD_CHARLESREID1_BACKUP_DIR}"
+    echo "Backup directory: ${BACKUP_DIR}"
     echo "Backup target: ${BACKUP_TARGET}"
     echo ""
 
-    mkdir -p ${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}
+    mkdir -p ${BACKUP_DIR}
 
     DOCKER=$(which docker)
     DOCKERX="${DOCKER} exec -t"
@@ -62,6 +64,7 @@ if [ "$#" == "0" ]; then
     echo "Step 3: Clean up tar.gz file"
     ${DOCKERX} ${CONTAINER_NAME} /bin/rm -f /tmp/${TARGET}
 
+    echo "Successfully wrote wikifiles dump to file: ${BACKUP_TARGET}"
     echo "Done."
 
 else

scripts/backups/wikifiles_restore.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Restore wiki files from a tar file
+# into the stormy_mw container.
+set -eu
+
+function usage {
+    echo ""
+    echo "restore_wikifiles.sh script:"
+    echo "Restore wiki files from a tar file"
+    echo "into the stormy_mw container"
+    echo ""
+    echo "       ./restore_wikifiles.sh <tar-file>"
+    echo ""
+    echo "Example:"
+    echo ""
+    echo "       ./restore_wikifiles.sh /path/to/wikifiles.tar.gz"
+    echo ""
+    echo ""
+    exit 1;
+}
+
+# NOTE:
+# I assume images/ is the only directory to back up/restore.
+# If there are more I forgot, add them back in here.
+# (skins and extensions are static, added into image at build time.)
+
+if [[ "$#" -eq 1 ]];
+then
+
+    NAME="stormy_mw"
+    TAR=$(basename "$1")
+
+    echo "Checking that container ${NAME} exists"
+    docker ps --format '{{.Names}}' | grep ${NAME} || exit 1;
+
+    echo "Copying dir $1 into container ${NAME}"
+    set -x
+    docker cp $1 ${NAME}:/tmp/${TAR}
+    docker exec -it ${NAME} rm -rf /var/www/html/images.old
+    docker exec -it ${NAME} mv /var/www/html/images /var/www/html/images.old
+    docker exec -it ${NAME} tar -xf /tmp/${TAR} -C / && rm -f /tmp/${TAR}
+    docker exec -it ${NAME} chown -R www-data:www-data /var/www/html/images
+
+else
+    usage
+fi

scripts/certbot/pod-charlesreid1-certbot.service.j2

@@ -5,6 +5,8 @@ After=docker.service
 
 [Service]
 Type=oneshot
-StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-certbot.service.error.log
+StandardError=syslog
-StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-certbot.service.output.log
+StandardOutput=syslog
-ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/certbot/renew_charlesreid1_certs.sh'
+SyslogIdentifier=pod-charlesreid1-certbot
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/certbot/renew_charlesreid1_certs.sh'

scripts/certbot/pod-charlesreid1-certbot.timer.j2

@@ -2,8 +2,8 @@
 Description=Timer to renew certificates for pod-charlesreid1
 
 [Timer]
-# Run on the first Sunday of every month
+# Run daily
-OnCalendar=Sun *-*-01..07 4:03:00
+OnCalendar=*-*-* 4:03:00
 
 [Install]
 WantedBy=timers.target

scripts/certbot/renew_charlesreid1_certs.sh.j2

@@ -34,7 +34,7 @@ if [ "$#" == "0" ]; then
     sudo systemctl stop ${SERVICE}
     
     echo "Stop pod"
-    docker-compose -f {{ pod_install_dir }}/docker-compose.yml down
+    docker-compose -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml down
     
     echo "Run certbot renew"
     SUBS="git www"
@@ -63,7 +63,7 @@ if [ "$#" == "0" ]; then
     done
     
     echo "Start pod"
-    docker-compose -f {{ pod_install_dir }}/docker-compose.yml up -d
+    docker-compose -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml up -d
     
     echo "Enable and start system service ${SERVICE}"
     sudo systemctl enable ${SERVICE}

scripts/clean_templates.py

@@ -13,7 +13,9 @@ def clean():
         rname = tname[:-3]
         rpath = os.path.join(tdir, rname)
 
-        if os.path.exists(rpath):
+        ignore_list = ['environment']
+
+        if os.path.exists(rpath) and rname not in ignore_list:
             print(f"Removing file {rpath}")
             os.remove(rpath)
         else:

scripts/git_clone_www.py.j2

@@ -11,8 +11,8 @@ directory structure for charlesreid1.com
 content. (Or, charlesreid1.XYZ, whatever.)
 """
 
-SERVER_NAME_DEFAULT = '{{ server_name_default }}'
+SERVER_NAME_DEFAULT = '{{ pod_charlesreid1_server_name }}'
-USERNAME = '{{ username }}'
+USERNAME = '{{ pod_charlesreid1_username }}'
 
 
 

scripts/git_pull_www.py.j2

@@ -10,8 +10,8 @@ This script git pulls the /www directory
 for updating charlesreid1.com content.
 """
 
-SERVER_NAME_DEFAULT = '{{ server_name_default }}'
+SERVER_NAME_DEFAULT = '{{ pod_charlesreid1_server_name }}'
-USERNAME = '{{ username }}'
+USERNAME = '{{ pod_charlesreid1_username }}'
 
 
 

scripts/mw/build_extensions_dir.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
-# 
+#
-# clone or download each extension, and build
+# Clone each REL1_39 extension into d-mediawiki-new for the MW 1.39 green stack.
+# EmbedVideo is intentionally skipped for now (add back later if needed).
 set -eux
 
-MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
+MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki-new"
 MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
 EXT_DIR="${MW_CONF_DIR}/extensions"
 
@@ -17,20 +18,10 @@ cd ${EXT_DIR}
 Extension="SyntaxHighlight_GeSHi"
 if [ ! -d ${Extension} ]
 then
-    ## This requires mediawiki > 1.31
+    git clone https://github.com/wikimedia/mediawiki-extensions-SyntaxHighlight_GeSHi.git ${Extension}
-    ## (so does REL1_31)
-    #git clone https://github.com/wikimedia/mediawiki-extensions-SyntaxHighlight_GeSHi.git SyntaxHighlight_GeSHi
-
-    ## This manually downloads REL1_30
-    #wget https://extdist.wmflabs.org/dist/extensions/SyntaxHighlight_GeSHi-REL1_30-87392f1.tar.gz -O SyntaxHighlight_GeSHi.tar.gz
-    #tar -xzf SyntaxHighlight_GeSHi.tar.gz -C ${PWD}
-    #rm -f SyntaxHighlight_GeSHi.tar.gz
-
-    # Best of both worlds
-    git clone https://github.com/wikimedia/mediawiki-extensions-SyntaxHighlight_GeSHi.git SyntaxHighlight_GeSHi
     (
     cd ${Extension}
-    git checkout --track remotes/origin/REL1_34
+    git checkout --track remotes/origin/REL1_39
     )
 else
     echo "Skipping ${Extension}"
@@ -44,21 +35,7 @@ then
     git clone https://github.com/wikimedia/mediawiki-extensions-ParserFunctions.git ${Extension}
     (
     cd ${Extension}
-    git checkout --track remotes/origin/REL1_34
+    git checkout --track remotes/origin/REL1_39
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="EmbedVideo"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/HydraWiki/mediawiki-embedvideo.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout v2.7.3
     )
 else
     echo "Skipping ${Extension}"
@@ -72,21 +49,7 @@ then
     git clone https://github.com/wikimedia/mediawiki-extensions-Math.git ${Extension}
     (
     cd ${Extension}
-    git checkout REL1_34
+    git checkout --track remotes/origin/REL1_39
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="Fail2banlog"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/charlesreid1-docker/mw-fail2ban.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout master
     )
 else
     echo "Skipping ${Extension}"

scripts/mw/fix_LocalSettings.sh

@@ -1,13 +1,6 @@
 #!/bin/bash
 # 
 # fix LocalSettings.php in the mediawiki container.
-# 
-# docker is stupid, so it doesn't let you bind mount
-# a single file into a docker volume.
-# 
-# so, rather than rebuilding the entire goddamn container
-# just to update LocalSettings.php when it changes, we just
-# use a docker cp command to copy it into the container.
 set -eux
 
 NAME="stormy_mw"

scripts/mw/fix_extensions_dir.sh

@@ -1,12 +1,6 @@
 #!/bin/bash
 # 
 # fix extensions dir in the mediawiki container
-#
-# in theory, we should be able to update the
-# extensions folder in d-mediawiki/charlesreid1-config,
-# but in reality this falls on its face.
-# So, we have to fix the fucking extensions directory
-# ourselves.
 set -eux
 
 NAME="stormy_mw"

scripts/mw/fix_skins.sh

@@ -1,13 +1,6 @@
 #!/bin/bash
 # 
 # fix skins in the mediawiki container.
-# 
-# docker is stupid, so it doesn't let you bind mount
-# a single file into a docker volume.
-#
-# so, rather than rebuilding the entire goddamn container
-# just to update the skin when it changes, we just
-# use a docker cp command to copy it into the container.
 set -eux
 
 NAME="stormy_mw"
@@ -23,8 +16,8 @@ echo "Checking that skins dir exists"
 test -d ${SKINS_DIR}
 
 echo "Installing skins into $NAME"
-docker exec -it $NAME /bin/bash -c 'rm -rf /var/www/html/skins'
+docker exec -i $NAME /bin/bash -c 'rm -rf /var/www/html/skins'
 docker cp ${SKINS_DIR} $NAME:/var/www/html/skins
-docker exec -it $NAME /bin/bash -c 'chown -R www-data:www-data /var/www/html/skins'
+docker exec -i $NAME /bin/bash -c 'chown -R www-data:www-data /var/www/html/skins'
 
 echo "Finished installing skins into $NAME"

scripts/mw/restore_wikifiles.sh

@@ -2,7 +2,7 @@
 #
 # Restore wiki files from a tar file
 # into the stormy_mw container.
-set -eux
+set -eu
 
 function usage {
     echo ""
@@ -31,16 +31,16 @@ then
     NAME="stormy_mw"
     TAR=$(basename "$1")
 
-	echo "Checking that container exists"
+	echo "Checking that container ${NAME} exists"
 	docker ps --format '{{.Names}}' | grep ${NAME} || exit 1;
 
-	echo "Copying $1 into container ${NAME}"
+	echo "Copying dir $1 into container ${NAME}"
     set -x
     docker cp $1 ${NAME}:/tmp/${TAR}
+    docker exec -it ${NAME} rm -rf /var/www/html/images.old
     docker exec -it ${NAME} mv /var/www/html/images /var/www/html/images.old
     docker exec -it ${NAME} tar -xf /tmp/${TAR} -C / && rm -f /tmp/${TAR}
     docker exec -it ${NAME} chown -R www-data:www-data /var/www/html/images
-    set +x
 
 else
     usage

scripts/mysql/dump_database.sh

@@ -1,35 +1,36 @@
 #!/bin/bash
+echo "this script is deprecated, see ../backups/wikidb_dump.sh"
+##
+## Dump a database to an .sql file
+## from the stormy_mysql container.
+#set -eu
 #
-# Dump a database to an .sql file
+#function usage {
-# from the stormy_mysql container.
+#    echo ""
-set -x
+#    echo "dump_database.sh script:"
-
+#    echo "Dump a database to an .sql file "
-function usage {
+#    echo "from the stormy_mysql container."
-    echo ""
+#    echo ""
-    echo "dump_database.sh script:"
+#    echo "       ./dump_database.sh <sql-dump-file>"
-    echo "Dump a database to an .sql file "
+#    echo ""
-    echo "from the stormy_mysql container."
+#    echo "Example:"
-    echo ""
+#    echo ""
-    echo "       ./dump_database.sh <sql-dump-file>"
+#    echo "       ./dump_database.sh /path/to/wikidb_dump.sql"
-    echo ""
+#    echo ""
-    echo "Example:"
+#    echo ""
-    echo ""
+#    exit 1;
-    echo "       ./dump_database.sh /path/to/wikidb_dump.sql"
+#}
-    echo ""
+#
-    echo ""
+#CONTAINER_NAME="stormy_mysql"
-    exit 1;
+#
-}
+#if [[ "$#" -gt 0 ]];
-
+#then
-CONTAINER_NAME="stormy_mysql"
+#
-
+#    TARGET="$1"
-if [[ "$#" -gt 0 ]];
+#    mkdir -p $(dirname $TARGET)
-then
+#	set -x
-
+#    docker exec -i ${CONTAINER_NAME} sh -c 'exec mysqldump wikidb --databases -uroot -p"$MYSQL_ROOT_PASSWORD"' > $TARGET
-    TARGET="$1"
+#
-    mkdir -p $(dirname $TARGET)
+#else
-    docker exec -i ${CONTAINER_NAME} sh -c 'exec mysqldump wikidb --databases -uroot -p"$MYSQL_ROOT_PASSWORD"' > $TARGET
+#    usage
-
+#fi
-else
-    usage
-fi
-

scripts/mysql/restore_database.sh

@@ -6,6 +6,7 @@
 # Note that this expects the .sql dump
 # to create its own databases.
 # Use the --databases flag with mysqldump.
+set -eu
 
 function usage {
     echo ""
@@ -42,31 +43,23 @@ function usage {
 # because of all these one-off 
 # "whoopsie we don't do that" problems.
 
-CONTAINER_NAME="stormy_mysql"
-TARGET=$(basename $1)
-TARGET_DIR=$(dirname $1)
-
-
 if [[ "$#" -eq 1 ]];
 then
 
-    # Step 1: Copy the sql dump into the container
+	CONTAINER_NAME="stormy_mysql"
+	TARGET=$(basename $1)
+	TARGET_DIR=$(dirname $1)
+
     set -x
+    # Step 1: Copy the sql dump into the container
     docker cp $1 ${CONTAINER_NAME}:/tmp/${TARGET}
-    set +x
 
     # Step 2: Run sqldump inside the container
-    set -x
     docker exec -i ${CONTAINER_NAME} sh -c "/usr/bin/mysql --defaults-file=/root/.mysql.rootpw.cnf < /tmp/${TARGET}"
-    set +x
 
     # Step 3: Clean up sql dump from inside container
-    set -x
+    docker exec -i ${CONTAINER_NAME} sh -c "/bin/rm -fr /tmp/${TARGET}"
-    docker exec -i ${CONTAINER_NAME} sh -c "/bin/rm -fr /tmp/${TARGET}.sql"
-    set +x
 
-
-    set +x
 else
     usage
 fi

scripts/pod-charlesreid1.service.j2

@@ -5,11 +5,11 @@ After=docker.service
 
 [Service]
 Restart=always
-StandardError=null
+StandardError=journal
-StandardOutput=null
+StandardOutput=journal
-ExecStartPre=/usr/bin/test -f {{ pod_install_dir }}/docker-compose.yml
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml
-ExecStart=/usr/local/bin/docker-compose -f {{ pod_install_dir }}/docker-compose.yml up
+ExecStart=/usr/local/bin/docker-compose -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml up
-ExecStop=/usr/local/bin/docker-compose  -f {{ pod_install_dir }}/docker-compose.yml stop
+ExecStop=/usr/local/bin/docker-compose  -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml stop
 
 [Install]
 WantedBy=default.target