add service to clean files older than N days

* cert-renewal:
  add certbot renewal script, plus service, plus timer

.gitignore

@@ -1,20 +1,25 @@
+*.log
 *.pyc
 environment
 attic
 
 # gitea
-#d-gitea/data/
-d-gitea/custom/conf/app.ini
-d-gitea/custom/gitea.db
-d-gitea/custom/avatars
-d-gitea/custom/log/
-d-gitea/custom/queues/
+d-gitea/data/
+d-gitea/custom/
 
 # mediawiki
 charlesreid1.wiki.conf
+d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/Bootstrap2.php
+d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/navbar.php
+d-mediawiki/mediawiki/
+
+# nginx
+d-nginx-charlesreid1/conf.d/http.DOMAIN.conf
+d-nginx-charlesreid1/conf.d/https.DOMAIN.conf
 
 # scripts dir
 scripts/git_*_www.py
+scripts/certbot/renew_charlesreid1_certs.sh
 *.timer
 *.service
 

Makefile

@@ -26,7 +26,7 @@ help:
 	@echo "--------------------------------------------------"
 	@echo "                   Backups:"
 	@echo ""
-	@echo "make backups:        Create backups of every service (gitea, wiki database, wiki files) in ~/backups"
+	@echo "make backups:        Create backups of every service (wiki database, wiki files) in ~/backups"
 	@echo ""
 	@echo "make clean-backups:  Remove files from ~/backups directory older than 30 days"
 	@echo ""
@@ -53,7 +53,7 @@ help:
 	@echo ""
 	@echo "make install:        Install and start systemd service to run pod-charlesreid1."
 	@echo "                     Also install and start systemd service for pod-charlesreid1 backup services"
-	@echo "                     for each service (gitea/mediawiki/mysql) part of pod-charlesreid1."
+	@echo "                     for each service (mediawiki/mysql) part of pod-charlesreid1."
 	@echo ""
 	@echo "make uninstall:      Remove all systemd startup services and timers part of pod-charlesreid1"
 	@echo ""
@@ -61,6 +61,8 @@ help:
 # Templates
 
 templates:
+	@find * -name "*.service.j2" | xargs -I '{}' chmod 644 {}
+	@find * -name "*.timer.j2" | xargs -I '{}' chmod 644 {}
 	python3 $(POD_CHARLESREID1_DIR)/scripts/apply_templates.py
 
 list-templates:
@@ -71,8 +73,7 @@ clean-templates:
 
 # Backups
 
-backups: templates
-	$(POD_CHARLESREID1_DIR)/scripts/backups/gitea_dump.sh
+backups:
 	$(POD_CHARLESREID1_DIR)/scripts/backups/wikidb_dump.sh
 	$(POD_CHARLESREID1_DIR)/scripts/backups/wikifiles_dump.sh
 
@@ -88,52 +89,77 @@ mw-fix-extensions: mw-build-extensions
 	$(POD_CHARLESREID1_DIR)/scripts/mw/build_extensions_dir.sh
 
 mw-fix-localsettings:
-	$(POD_CHARLESEREID1_DIR)/scripts/mw/fix_LocalSettings.sh
+	$(POD_CHARLESREID1_DIR)/scripts/mw/fix_LocalSettings.sh
 
 mw-fix-skins:
-	$(POD_CHARLESEREID1_DIR)/scripts/mw/fix_skins.sh
+	$(POD_CHARLESREID1_DIR)/scripts/mw/fix_skins.sh
 
 # /www Dir
 
-clone-www: templates
+clone-www:
 	python3 $(POD_CHARLESREID1_DIR)/scripts/git_clone_www.py
 
-pull-www: templates
+pull-www:
 	python3 $(POD_CHARLESREID1_DIR)/scripts/git_pull_www.py
 
-install: templates
+install:
 ifeq ($(shell which systemctl),)
 	$(error Please run this make command on a system with systemctl installed)
 endif
-	cp $(POD_CHARLESREID1_DIR)/scripts/pod-charlesreid1.service /etc/systemd/system/pod-charlesreid1.service
-	cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-gitea.{service,timer} /etc/systemd/system/.
-	cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikidb.{service,timer} /etc/systemd/system/.
-	cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikifiles.{service,timer} /etc/systemd/system/.
-	systemctl daemon-reload
-	systemctl enable pod-charlesreid1
-	systemctl enable pod-charlesreid1-backups-gitea.timer
-	systemctl enable pod-charlesreid1-backups-wikidb.timer
-	systemctl enable pod-charlesreid1-backups-wikifiles.timer
-	systemctl start pod-charlesreid1-backups-gitea.timer
-	systemctl start pod-charlesreid1-backups-wikidb.timer
-	systemctl start pod-charlesreid1-backups-wikifiles.timer
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/pod-charlesreid1.service /etc/systemd/system/pod-charlesreid1.service
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikidb.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikifiles.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-gitea.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-aws.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/canary/pod-charlesreid1-canary.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/certbot/pod-charlesreid1-certbot.{service,timer} /etc/systemd/system/.
+
+	sudo chmod 664 /etc/systemd/system/pod-charlesreid1*
+	sudo systemctl daemon-reload
+
+	sudo systemctl enable pod-charlesreid1
+	sudo systemctl enable pod-charlesreid1-backups-wikidb.timer
+	sudo systemctl enable pod-charlesreid1-backups-wikifiles.timer
+	sudo systemctl enable pod-charlesreid1-backups-gitea.timer
+	sudo systemctl enable pod-charlesreid1-backups-aws.timer
+	sudo systemctl enable pod-charlesreid1-canary.timer
+	sudo systemctl enable pod-charlesreid1-certbot.timer
+
+	sudo systemctl start pod-charlesreid1-backups-wikidb.timer
+	sudo systemctl start pod-charlesreid1-backups-wikifiles.timer
+	sudo systemctl start pod-charlesreid1-backups-gitea.timer
+	sudo systemctl start pod-charlesreid1-backups-aws.timer
+	sudo systemctl start pod-charlesreid1-canary.timer
+	sudo systemctl start pod-charlesreid1-certbot.timer
 
 uninstall:
 ifeq ($(shell which systemctl),)
 	$(error Please run this make command on a system with systemctl installed)
 endif
-	systemctl disable pod-charlesreid1
-	systemctl disable pod-charlesreid1-backups-gitea.timer
-	systemctl disable pod-charlesreid1-backups-wikidb.timer
-	systemctl disable pod-charlesreid1-backups-wikifiles.timer
-	systemctl stop pod-charlesreid1
-	systemctl stop pod-charlesreid1-backups-gitea.timer
-	systemctl stop pod-charlesreid1-backups-wikidb.timer
-	systemctl stop pod-charlesreid1-backups-wikifiles.timer
-	rm -f /etc/systemd/system/pod-charlesreid1.service
-	rm -f /etc/systemd/system/pod-charlesreid1-backups-gitea.{service,timer}
-	rm -f /etc/systemd/system/pod-charlesreid1-backups-wikidb.{service,timer}
-	rm -f /etc/systemd/system/pod-charlesreid1-backups-wikifiles.{service,timer}
-	systemctl daemon-reload
+	-sudo systemctl disable pod-charlesreid1
+	-sudo systemctl disable pod-charlesreid1-backups-wikidb.timer
+	-sudo systemctl disable pod-charlesreid1-backups-wikifiles.timer
+	-sudo systemctl disable pod-charlesreid1-backups-gitea.timer
+	-sudo systemctl disable pod-charlesreid1-backups-aws.timer
+	-sudo systemctl disable pod-charlesreid1-canary.timer
+	-sudo systemctl disable pod-charlesreid1-certbot.timer
+
+	# Leave the pod running!
+	# -sudo systemctl stop pod-charlesreid1
+	-sudo systemctl stop pod-charlesreid1-backups-wikidb.timer
+	-sudo systemctl stop pod-charlesreid1-backups-wikifiles.timer
+	-sudo systemctl stop pod-charlesreid1-backups-gitea.timer
+	-sudo systemctl stop pod-charlesreid1-backups-aws.timer
+	-sudo systemctl stop pod-charlesreid1-canary.timer
+	-sudo systemctl stop pod-charlesreid1-certbot.timer
+
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1.service
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-wikidb.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-wikifiles.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-gitea.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-aws.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-canary.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-certbot.{service,timer}
+	sudo systemctl daemon-reload
 
 .PHONY: help

d-gitea/README.md

@@ -18,7 +18,7 @@ The data directory contains any instance-specific gitea data.
 
 The data directory is bind-mounted to `/app/gitea/data` in the container.
 
-## Repository Data
+## Repository Drive
 
 Gitea stores all of its repositories in a separate drive that is at
 `/gitea_repositories` on the host machine.

environment.example

@@ -2,31 +2,33 @@
 
 # multiple templates:
 # -------------------
-POD_CHARLESREID1_DIR="/path/to/pod-charlesreid1"
-POD_CHARLESREID1_TLD="example.com"
+export POD_CHARLESREID1_DIR="/path/to/pod-charlesreid1"
+export POD_CHARLESREID1_TLD="example.com"
+export POD_CHARLESREID1_USER="nonrootuser"
 
 # mediawiki:
 # ----------
-POD_CHARLESREID1_MW_ADMIN_EMAIL="email@example.com"
-POD_CHARLESREID1_MW_SECRET_KEY="SecretKeyString"
+export POD_CHARLESREID1_MW_ADMIN_EMAIL="email@example.com"
+export POD_CHARLESREID1_MW_SECRET_KEY="SecretKeyString"
 
 # mysql:
 # ------
-POD_CHARLESREID1_MYSQL_PASSWORD="SuperSecretPassword"
+export POD_CHARLESREID1_MYSQL_PASSWORD="SuperSecretPassword"
 
 # gitea:
 # ------
-POD_CHARLESREID1_GITEA_APP_NAME=""
-POD_CHARLESREID1_GITEA_SECRET_KEY="GiteaSecretKey"
-POD_CHARLESREID1_GITEA_INTERNAL_TOKEN="GiteaInternalToken"
+export POD_CHARLESREID1_GITEA_APP_NAME=""
+export POD_CHARLESREID1_GITEA_SECRET_KEY="GiteaSecretKey"
+export POD_CHARLESREID1_GITEA_INTERNAL_TOKEN="GiteaInternalToken"
 
 # aws:
 # ----
-POD_CHARLESREID1_AWS_ACCESS_KEY="AAAAAAAAAAAAAAAAAAAA"
-POD_CHARLESREID1_AWS_ACCESS_SECRET="0000000000000000000000000000000000000000"
+export AWS_ACCESS_KEY_ID="AAAAAAA"
+export AWS_SECRET_ACCESS_KEY="BBBBBBBB"
+export AWS_DEFAULT_REGION="us-west-1"
 
 # backups and scripts:
 # --------------------
-POD_CHARLESREID1_USER="charles"
-POD_CHARLESREID1_BACKUP_S3BUCKET="name-of-backups-bucket"
-POD_CHARLESREID1_BACKUPCANARY_WEBHOOKURL="https://hooks.slack.com/services/000000000/AAAAAAAAA/111111111111111111111111"
+export POD_CHARLESREID1_USER="charles"
+export POD_CHARLESREID1_BACKUP_S3BUCKET="name-of-backups-bucket"
+export POD_CHARLESREID1_CANARY_WEBHOOK="https://hooks.slack.com/services/000000000/AAAAAAAAA/111111111111111111111111"

scripts/Readme.md

@@ -21,10 +21,12 @@ Cleans all rendered Jinja templates. Does not require environment variables.
 This script is destructive! Be careful!
 
 
-# Ansible Scripts
+# /www Directory Scripts
 
-These scripts are used by ansible when setting up a machine
-to run the charlesreid1 docker pod.
+These scripts set up or pull a git repo that is set up to
+have a pecular directory structure.
+
+The clone script is used by Ansible when setting up this pod.
 
 ## `git_clone_www.py`
 

scripts/apply_templates.py

@@ -2,36 +2,39 @@ import os
 import re
 import sys
 import glob
+import time
+import subprocess
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 
-"""
-Apply Default Values to all Jinja Templates
-"""
-
 
 # Should existing files be overwritten
-OVERWRITE = True 
+OVERWRITE = False
 
+# Map of jinja variables to environment variables
+jinja_to_env = {
+    "pod_install_dir": "POD_CHARLESREID1_DIR",
+    "top_domain": "POD_CHARLESREID1_TLD",
+    "server_name_default" : "POD_CHARLESREID1_TLD",
+    "username": "POD_CHARLESREID1_USER",
+    # docker-compose:
+    "mysql_password" : "POD_CHARLESREID1_MYSQL_PASSWORD",
+    "mediawiki_secretkey" : "POD_CHARLESREID1_MW_SECRET_KEY",
+    # mediawiki:
+    "admin_email": "POD_CHARLESREID1_MW_ADMIN_EMAIL",
+    # gitea:
+    "gitea_app_name": "POD_CHARLESREID1_GITEA_APP_NAME",
+    "gitea_secret_key": "POD_CHARLESREID1_GITEA_SECRET_KEY",
+    "gitea_internal_token": "POD_CHARLESREID1_GITEA_INTERNAL_TOKEN",
+    # aws:
+    "backup_canary_webhook_url": "POD_CHARLESREID1_CANARY_WEBHOOK",
+}
 
 scripts_dir = os.path.dirname(os.path.abspath(__file__))
 repo_root = os.path.abspath(os.path.join(scripts_dir, '..'))
 
 
 def check_env_vars():
-    env_var_list = [
-        'POD_CHARLESREID1_DIR',
-        'POD_CHARLESREID1_TLD',
-        'POD_CHARLESREID1_USER',
-        'POD_CHARLESREID1_MYSQL_PASSWORD',
-        'POD_CHARLESREID1_MW_ADMIN_EMAIL',
-        'POD_CHARLESREID1_GITEA_APP_NAME',
-        'POD_CHARLESREID1_GITEA_SECRET_KEY',
-        'POD_CHARLESREID1_GITEA_INTERNAL_TOKEN',
-        'POD_CHARLESREID1_BACKUP_S3BUCKET',
-        'POD_CHARLESREID1_AWS_ACCESS_KEY',
-        'POD_CHARLESREID1_AWS_ACCESS_SECRET',
-        'POD_CHARLESREID1_BACKUPCANARY_WEBHOOKURL',
-    ]
+    env_var_list = jinja_to_env.values()
     nerrs = 0
     print("Checking environment variables")
     for env_var in env_var_list:
@@ -48,6 +51,8 @@ def main():
 
     check_env_vars()
 
+    ignore_list = ['environment']
+
     p = os.path.join(repo_root,'**','*.j2')
     template_files = glob.glob(p, recursive=True)
 
@@ -63,41 +68,35 @@ def main():
         rname = tname[:-3]
         rpath = os.path.join(tdir, rname)
 
+        if rname in ignore_list:
+            print(f"\nSkipping template on ignore list: {tname}\n")
+            continue
+
         env = Environment(loader=FileSystemLoader(tdir))
     
         print(f"Rendering template {tname}:")
         print(f"    Template path: {tpath}")
         print(f"    Output path: {rpath}")
-        #content = env.get_template(tpath).render({
-        content = env.get_template(tname).render({
-            "pod_install_dir": os.environ['POD_CHARLESREID1_DIR'],
-            "top_domain": os.environ['POD_CHARLESREID1_TLD'],
-            "server_name_default" : os.environ['POD_CHARLESREID1_TLD'],
-            "username": os.environ['POD_CHARLESREID1_USER'],
-            # docker-compose:
-            "mysql_password" : os.environ['POD_CHARLESREID1_MYSQL_PASSWORD'],
-            "mediawiki_secretkey" : os.environ['POD_CHARLESREID1_MW_ADMIN_EMAIL'],
-            # mediawiki:
-            "admin_email": os.environ['POD_CHARLESREID1_MW_ADMIN_EMAIL'],
-            # gitea:
-            "gitea_app_name": os.environ['POD_CHARLESREID1_GITEA_APP_NAME'],
-            "gitea_secret_key": os.environ['POD_CHARLESREID1_GITEA_SECRET_KEY'],
-            "gitea_internal_token": os.environ['POD_CHARLESREID1_GITEA_INTERNAL_TOKEN'],
-            # aws:
-            "aws_backup_s3_bucket": os.environ['POD_CHARLESREID1_BACKUP_S3BUCKET'],
-            "aws_access_key": os.environ['POD_CHARLESREID1_AWS_ACCESS_KEY'],
-            "aws_access_secret": os.environ['POD_CHARLESREID1_AWS_ACCESS_SECRET'],
-            "backup_canary_webhook_url": os.environ['POD_CHARLESREID1_BACKUPCANARY_WEBHOOKURL'],
-        })
+
+        jinja_vars = {}
+        for k, v in jinja_to_env.items():
+            jinja_vars[k] = os.environ[v]
+
+        content = env.get_template(tname).render(jinja_vars)
     
         # Write to file
         if os.path.exists(rpath) and not OVERWRITE:
-            raise Exception("Error: file %s already exists!"%(rpath))
+            msg = "\n[!!!] Warning: file %s already exists! Skipping...\n"%(rpath)
+            print(msg)
+            time.sleep(1)
         else:
             with open(rpath,'w') as f:
                 f.write(content)
             print(f"    Done!")
             print("")
+
+        if rpath[-3:] == ".sh":
+            subprocess.call(['chmod', '+x', rpath])
     
 if __name__=="__main__":
     main()

scripts/backups/Readme.md

@@ -0,0 +1,15 @@
+# backup scripts
+
+This directory contains several files for several services:
+
+* Systemd .service file (Jinja template) to define a service that backs up files
+* Systemd .timer file (Jinja template) to define a timer that runs the service on a schedule
+* Shell script .sh that actually performs the backup operation and is called by the .service file
+
+Use `make templates` in the top level of this repo to render
+the Jinja templates using the environment variables in the
+evnrionment file. That fixes the locations of the scripts
+for the systemd service.
+
+Use `make install` in the top level of this repo to install
+the rendered service and timer files.

scripts/backups/aws_backup.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+#
+# Find the last backup created, and copy it
+# to an S3 bucket.
+set -eux
+
+function usage {
+    set +x
+    echo ""
+    echo "aws_backup.sh script:"
+    echo ""
+    echo "Find the last backup that was created,"
+    echo "and copy it to the backups bucket."
+    echo ""
+    echo "       ./aws_backup.sh"
+    echo ""
+    exit 1;
+}
+
+if [ "$(id -u)" == "0" ]; then
+    echo ""
+    echo ""
+    echo "This script should NOT be run as root!"
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+if [ "$#" == "0" ]; then
+
+    echo ""
+    echo "pod-charlesreid1: aws_backup.sh"
+    echo "-----------------------------------"
+    echo ""
+    echo "Backup directory: ${POD_CHARLESREID1_BACKUP_DIR}"
+    echo "Backup bucket: ${POD_CHARLESREID1_BACKUP_S3BUCKET}"
+    echo ""
+
+    echo "Checking that directory exists"
+    /usr/bin/test -d ${POD_CHARLESREID1_BACKUP_DIR}
+
+    echo "Checking that we can access the S3 bucket"
+    aws s3 ls s3://${POD_CHARLESREID1_BACKUP_S3BUCKET} > /dev/null
+    
+    # Get name of last backup, to copy to AWS
+    LAST_BACKUP=$(/bin/ls -1 -t ${POD_CHARLESREID1_BACKUP_DIR} | /usr/bin/head -n1)
+    echo "Last backup found: ${LAST_BACKUP}"
+    echo "Last backup directory: ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP}"
+
+    BACKUP_SIZE=$(du -hs ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP})
+    echo "Backup directory size: ${BACKUP_SIZE}"
+
+    # Copy to AWS
+    echo "Backing up directory ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP}"
+    aws s3 cp --only-show-errors --recursive ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP} s3://${POD_CHARLESREID1_BACKUP_S3BUCKET}/backups/${LAST_BACKUP}
+    echo "Done."
+
+else
+    usage
+fi

scripts/backups/canary/backups_canary.py

@@ -0,0 +1,104 @@
+import os
+import sys
+import json
+import requests
+import boto3
+import botocore
+import subprocess
+
+
+webhook_url = os.environ['POD_CHARLESREID1_CANARY_WEBHOOK']
+backup_dir = os.environ['POD_CHARLESREID1_BACKUP_DIR']
+backup_bucket = os.environ['POD_CHARLESREID1_BACKUP_S3BUCKET']
+
+
+# Check for backups created in the last N days
+N = 7
+
+
+def main():
+    # verify the backups directory exists
+    if not os.path.exists(backup_dir):
+        msg = "Local Backups Error:\n"
+        msg += f"The backup directory `{backup_dir}` does not exist!"
+        alert(msg)
+
+    # verify there is a backup newer than N days
+    newer_backups = subprocess.getoutput(f'find {backup_dir} -mtime -{N}').split('\n')
+    if len(newer_backups)==1 and newer_backups[0]=='':
+        msg = "Local Backups Error:\n"
+        msg += f"The backup directory `{backup_dir}` is missing backup files from the last {N} day(s)!"
+        alert(msg)
+
+    newest_backup_name = subprocess.getoutput(f'ls -t {backup_dir} | head -n1')
+    newest_backup_path = os.path.join(backup_dir, newest_backup_name)
+    newest_backup_files = subprocess.getoutput(f'find {newest_backup_path} -type f').split('\n')
+
+    # verify the most recent backup directory is not empty
+    if len(newest_backup_files)==1 and newer_backups[0]=='':
+        msg = "Local Backups Error:\n"
+        msg += f"The most recent backup directory `{newest_backup_path}` is empty!"
+        alert(msg)
+
+    # verify the most recent backup files have nonzero size
+    for backup_file in newest_backup_files:
+        if os.path.getsize(backup_file)==0:
+            msg = "Local Backups Error:\n"
+            msg += f"The most recent backup directory `{newest_backup_path}` contains an empty backup file!\n"
+            msg += f"Backup file name: {backup_file}!"
+            alert(msg)
+
+    # verify the most recent backup files exist in the s3 backups bucket
+    bucket_base_path = os.path.join('backups', newest_backup_name)
+    for backup_file in newest_backup_files:
+        backup_name = os.path.basename(backup_file)
+        backup_bucket_path = os.path.join(bucket_base_path, backup_name)
+        check_exists(backup_bucket, backup_bucket_path)
+
+def check_exists(bucket_name, bucket_path):
+    s3 = boto3.resource('s3')
+    try:
+        s3.Object(bucket_name, bucket_path).load()
+    except botocore.exceptions.ClientError as e:
+        if e.response['Error']['Code'] == "404":
+            # File does not exist
+            msg = "S3 Backups Error:\n"
+            msg += f"Failed to find the file `{bucket_path}` in bucket `{bucket_name}`"
+        else:
+            # Problem accessing backups on bucket
+            msg = "S3 Backups Error:\n"
+            msg += f"Failed to access the file `{bucket_path}` in bucket `{bucket_name}`"
+
+
+def alert(msg):
+    title = ":bangbang: pod-charlesreid1 backups canary"
+    hostname = subprocess.getoutput('hostname')
+    msg += f"\n\nHost: {hostname}"
+    slack_data = {
+        "username": "backups_canary",
+        "channel" : "#alerts",
+        "attachments": [
+            {
+                "color": "#CC0000",
+                "fields": [
+                    {
+                        "title": title,
+                        "value": msg,
+                        "short": "false",
+                    }
+                ]
+            }
+        ]
+    }
+    byte_length = str(sys.getsizeof(slack_data))
+    headers = {'Content-Type': "application/json", 'Content-Length': byte_length}
+    response = requests.post(webhook_url, data=json.dumps(slack_data), headers=headers)
+    if response.status_code != 200:
+        raise Exception(response.status_code, response.text)
+
+    print("Goodbye.")
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

scripts/backups/canary/pod-charlesreid1-canary.service.j2

@@ -0,0 +1,13 @@
+[Unit]
+Description=Backup canary service for pod-charlesreid1
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-canary.service.error.log
+StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-canary.service.output.log
+ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/backups/canary/vp/bin/python3 {{ pod_install_dir }}/scripts/backups/canary/backups_canary.py'
+User=charles
+Group=charles
+

scripts/backups/canary/pod-charlesreid1-canary.timer.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=Timer to run the pod-charlesreid1 backups canary
+
+[Timer]
+OnCalendar=Sun *-*-* 9:03:00
+
+[Install]
+WantedBy=timers.target

scripts/backups/canary/requirements.txt

@@ -0,0 +1,3 @@
+boto3
+botocore
+requests

scripts/backups/clean_olderthan.sh

@@ -2,13 +2,24 @@
 #
 # Clean any files older than N days
 # from the backup directory.
-set -eu
+set -eux
 
 # Number of days of backups to retain.
 # Everything older than this many days will be deleted
-N="30"
+N="45"
 
-BACKUP_DIR="$HOME/backups"
+function usage {
+    set +x
+    echo ""
+    echo "clean_olderthan.sh script:"
+    echo ""
+    echo "Clean files older than ${N} days from the"
+    echo "backups directory, ~/backups"
+    echo ""
+    echo "       ./clean_olderthan.sh"
+    echo ""
+    exit 1;
+}
 
 if [ "$(id -u)" == "0" ]; then
     echo ""
@@ -21,8 +32,21 @@ fi
 
 if [ "$#" == "0" ]; then
 
-    echo "Cleaning backups directory $BACKUP_DIR"
-    echo "Files older than $N days will be deleted"
-    find $BACKUP_DIR -mtime +${N} -delete
+    echo ""
+    echo "pod-charlesreid1: clean_olderthan.sh"
+    echo "------------------------------------"
+    echo ""
+    echo "Backup directory: ${POD_CHARLESREID1_BACKUP_DIR}"
+    echo ""
 
+    echo "Cleaning backups directory $BACKUP_DIR"
+    echo "The following files older than $N days will be deleted:"
+    find ${POD_CHARLESREID1_BACKUP_DIR} -mtime +${N}
+
+    echo "Deleting files"
+    find ${POD_CHARLESREID1_BACKUP_DIR} -mtime +${N} -delete
+    echo "Done"
+
+else
+    usage
 fi

scripts/backups/gitea_backup.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# Bcak up the Gitea custom/ and data/ directories.
+# These are needed to restore the site
+# (as well as repository data, which is not backed up
+# by this script, it is a separate drive).
+set -eux
+
+CONTAINER_NAME="stormy_gitea"
+STAMP="`date +"%Y%m%d"`"
+
+
+function usage {
+    set +x
+    echo ""
+    echo "gitea_backup.sh script:"
+    echo ""
+    echo "Create a tar file containing gitea"
+    echo "custom/ and data/ directories."
+    echo ""
+    echo "       ./gitea_backup.sh"
+    echo ""
+    exit 1;
+}
+
+if [ "$(id -u)" == "0" ]; then
+    echo ""
+    echo ""
+    echo "This script should NOT be run as root!"
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+if [ "$#" == "0" ]; then
+
+    CUSTOM_NAME="gitea_custom_${STAMP}.tar.gz"
+    DATA_NAME="gitea_data_${STAMP}.tar.gz"
+
+    CUSTOM_TARGET="${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}/${CUSTOM_NAME}"
+    DATA_TARGET="${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}/${DATA_NAME}"
+
+    echo ""
+    echo "pod-charlesreid1: gitea_backup.sh"
+    echo "-----------------------------------"
+    echo ""
+    echo "Backup target: custom: ${CUSTOM_TARGET}"
+    echo "Backup target: data: ${DATA_TARGET}"
+    echo ""
+
+    mkdir -p ${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}
+
+    # We don't need to use docker, since these directories
+    # are both bind-mounted into the Docker container
+    echo "Backing up custom directory"
+    tar czf ${CUSTOM_TARGET} ${POD_CHARLESREID1_DIR}/d-gitea/custom
+    echo "Backing up data directory"
+    tar czf ${DATA_TARGET} ${POD_CHARLESREID1_DIR}/d-gitea/data
+
+    echo "Done."
+
+else
+    usage
+fi

scripts/backups/gitea_dump.sh

@@ -1,86 +0,0 @@
-#!/bin/bash
-#
-# Run the gitea dump command and send the dump file
-# to the specified backup directory.
-#
-# Backup directory:
-#       /home/user/backups/gitea
-
-BACKUP_DIR="$HOME/backups/gitea"
-CONTAINER_NAME="stormy_gitea"
-
-function usage {
-    set +x
-    echo ""
-    echo "gitea_dump.sh script:"
-    echo ""
-    echo "Run the gitea dump command inside the gitea docker container,"
-    echo "and copy the resulting zip file to the specified directory."
-    echo "The resulting gitea dump zip file will be timestamped."
-    echo ""
-    echo "       ./gitea_dump.sh"
-    echo ""
-    echo "Example:"
-    echo ""
-    echo "       ./gitea_dump.sh"
-    echo "       (creates ${BACKUP_DIR}/gitea-dump_20200101_000000.zip)"
-    echo ""
-    exit 1;
-}
-
-if [ "$(id -u)" == "0" ]; then
-    echo ""
-    echo ""
-    echo "This script should NOT be run as root!"
-    echo ""
-    echo ""
-    exit 1;
-fi
-
-if [ "$#" == "0" ]; then
-
-    STAMP="`date +"%Y-%m-%d"`"
-    TARGET="gitea-dump_${STAMP}.zip"
-
-    echo ""
-    echo "pod-charlesreid1: gitea_dump.sh"
-    echo "-------------------------------"
-    echo ""
-    echo "Backup target: ${BACKUP_DIR}/${TARGET}"
-    echo ""
-
-    mkdir -p $BACKUP_DIR
-
-    ## If this script is being run from a cron job,
-    ## don't use -i flag with docker
-    #CRON="$( pstree -s $$ | /bin/grep -c cron )"
-    #DOCKER="/usr/local/bin/docker"
-    #DOCKERX=""
-    #if [[ "$CRON" -eq 1 ]]; 
-    #then
-    #    DOCKERX="${DOCKER} exec -t"
-    #else
-    #    DOCKERX="${DOCKER} exec -it"
-    #fi
-    DOCKER="/usr/local/bin/docker"
-    DOCKERX="${DOCKER} exec -t"
-
-    echo "Step 1: Run gitea dump command inside docker machine"
-    set -x
-    ${DOCKERX} --user git ${CONTAINER_NAME} /bin/bash -c 'cd /app/gitea && /app/gitea/gitea dump --file gitea-dump.zip --skip-repository'
-    set +x
-
-    echo "Step 2: Copy gitea dump file out of docker machine"
-    set -x
-    ${DOCKER} cp ${CONTAINER_NAME}:/app/gitea/gitea-dump.zip ${BACKUP_DIR}/${TARGET}
-    set +x
-
-    echo "Step 3: Clean up gitea dump file"
-    set -x
-    ${DOCKERX} ${CONTAINER_NAME} /bin/bash -c "rm -f /app/gitea/gitea-dump.zip"
-    set +x
-
-    echo "Done."
-else
-    usage
-fi

scripts/backups/pod-charlesreid1-backups-aws.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Copy the latest pod-charlesreid1 backup to an S3 bucket
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-aws.service.error.log
+StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-aws.service.output.log
+ExecStartPre=/usr/bin/test -f {{ pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/backups/aws_backup.sh'
+User=charles
+Group=charles
+

scripts/backups/pod-charlesreid1-backups-aws.timer.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=Timer to copy the lastest pod-charlesreid1 backup to an S3 bucket
+
+[Timer]
+OnCalendar=Sun *-*-* 2:56:00
+
+[Install]
+WantedBy=timers.target

scripts/backups/pod-charlesreid1-backups-cleanolderthan.service.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=Copy the latest pod-charlesreid1 backup to an S3 bucket
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-cleanolderthan.service.error.log
+StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-cleanolderthan.service.output.log
+ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/backups/clean_olderthan.sh'
+User=charles
+Group=charles

scripts/backups/pod-charlesreid1-backups-gitea.service.j2

@@ -5,8 +5,10 @@ After=docker.service
 
 [Service]
 Type=oneshot
-StandardError={{ pod_install_dir }}/.pod-charlesreid1-backups-gitea.service.error.log
-StandardOutput={{ pod_install_dir }}/.pod-charlesreid1-backups-gitea.service.output.log
-ExecStart={{ pod_install_dir }}/scripts/backups/gitea_dump.sh
+StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-gitea.service.error.log
+StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-gitea.service.output.log
+ExecStartPre=/usr/bin/test -f {{ pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/backups/gitea_backup.sh'
 User=charles
 Group=charles
+

scripts/backups/pod-charlesreid1-backups-gitea.timer.j2

@@ -2,8 +2,7 @@
 Description=Timer to back up pod-charlesreid1 gitea files
 
 [Timer]
-OnCalendar=*-*-* 0/2:23:00
+OnCalendar=Sun *-*-* 2:12:00
 
 [Install]
 WantedBy=timers.target
-

scripts/backups/pod-charlesreid1-backups-wikidb.service.j2

@@ -5,9 +5,10 @@ After=docker.service
 
 [Service]
 Type=oneshot
-StandardError={{ pod_install_dir }}/.pod-charlesreid1-backups-wikidb.service.error.log
-StandardOutput={{ pod_install_dir }}/.pod-charlesreid1-backups-wikidb.service.output.log
-ExecStart={{ pod_install_dir }}/scripts/backups/wikidb_dump.sh
+StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-wikidb.service.error.log
+StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-wikidb.service.output.log
+ExecStartPre=/usr/bin/test -f {{ pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/backups/wikidb_dump.sh'
 User=charles
 Group=charles
 

scripts/backups/pod-charlesreid1-backups-wikidb.timer.j2

@@ -2,7 +2,7 @@
 Description=Timer to back up the pod-charlesreid1 wiki database
 
 [Timer]
-OnCalendar=*-*-* 0/2:03:00
+OnCalendar=Sun *-*-* 2:02:00
 
 [Install]
 WantedBy=timers.target

scripts/backups/pod-charlesreid1-backups-wikifiles.service.j2

@@ -1,12 +1,13 @@
 [Unit]
-Description=Back up the pod-charlesreid1 wiki database
+Description=Back up the pod-charlesreid1 wiki files
 Requires=docker.service
 After=docker.service
 
 [Service]
 Type=oneshot
-StandardError={{ pod_install_dir }}/.pod-charlesreid1-backups-wikifiles.service.error.log
-StandardOutput={{ pod_install_dir }}/.pod-charlesreid1-backups-wikifiles.service.output.log
-ExecStart={{ pod_install_dir }}/scripts/backups/wikifiles_dump.sh
+StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-wikifiles.service.error.log
+StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-backups-wikifiles.service.output.log
+ExecStartPre=/usr/bin/test -f {{ pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/backups/wikifiles_dump.sh'
 User=charles
 Group=charles

scripts/backups/pod-charlesreid1-backups-wikifiles.timer.j2

@@ -1,9 +1,8 @@
 [Unit]
-Description=Timer to back up the pod-charlesreid1 wiki database
+Description=Timer to back up the pod-charlesreid1 wiki files
 
 [Timer]
-OnCalendar=*-*-* 0/2:13:00
+OnCalendar=Sun *-*-* 2:08:00
 
 [Install]
 WantedBy=timers.target
-

scripts/backups/wikidb_dump.sh

@@ -2,11 +2,8 @@
 #
 # Run the mysql dump command to back up wikidb table, and send the
 # resulting SQL file to the specified backup directory.
-#
-# Backup directory:
-#       /home/user/backups/mysql
+set -eux
 
-BACKUP_DIR="$HOME/backups"
 CONTAINER_NAME="stormy_mysql"
 STAMP="`date +"%Y%m%d"`"
 
@@ -23,7 +20,7 @@ function usage {
     echo "Example:"
     echo ""
     echo "       ./wikidb_dump.sh"
-    echo "       (creates ${BACKUP_DIR}/20200101/wikidb_20200101.sql)"
+    echo "       (creates ${POD_CHARLESREID1_BACKUP_DIR}/20200101/wikidb_20200101.sql)"
     echo ""
     exit 1;
 }
@@ -40,35 +37,26 @@ fi
 if [ "$#" == "0" ]; then
 
     TARGET="wikidb_${STAMP}.sql"
-    BACKUP_TARGET="${BACKUP_DIR}/${STAMP}/${TARGET}"
+    BACKUP_TARGET="${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}/${TARGET}"
 
     echo ""
     echo "pod-charlesreid1: wikidb_dump.sh"
     echo "--------------------------------"
     echo ""
+    echo "Backup directory: ${POD_CHARLESREID1_BACKUP_DIR}"
     echo "Backup target: ${BACKUP_TARGET}"
     echo ""
 
-    mkdir -p ${BACKUP_DIR}/${STAMP}
+    mkdir -p ${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}
 
-    # If this script is being run from a cron job,
-    # don't use -i flag with docker
-    CRON="$( pstree -s $$ | /bin/grep -c cron )"
     DOCKER=$(which docker)
-    DOCKERX=""
-    if [[ "$CRON" -eq 1 ]]; 
-    then
-        DOCKERX="${DOCKER} exec -t"
-    else
-        DOCKERX="${DOCKER} exec -it"
-    fi
+    DOCKERX="${DOCKER} exec -t"
 
-    echo "Running mysqldump"
-    set -x
-    ${DOCKERX} ${CONTAINER_NAME} sh -c 'exec mysqldump wikidb --databases -uroot -p"$MYSQL_ROOT_PASSWORD"' > ${BACKUP_TARGET}
-    set +x
+    echo "Running mysqldump inside the mysql container"
+    ${DOCKERX} ${CONTAINER_NAME} sh -c 'exec mysqldump wikidb --databases -uroot -p"$MYSQL_ROOT_PASSWORD"' 2>&1 | grep -v "Using a password" > ${BACKUP_TARGET}
 
     echo "Done."
+
 else
     usage
 fi

scripts/backups/wikifiles_dump.sh

@@ -2,11 +2,8 @@
 #
 # Create a tar file containing wiki files
 # from the mediawiki docker container.
-#
-# Backup directory:
-#       /home/user/backups/mediawiki
+set -eux
 
-BACKUP_DIR="$HOME/backups"
 CONTAINER_NAME="stormy_mw"
 STAMP="`date +"%Y%m%d"`"
 
@@ -23,7 +20,7 @@ function usage {
     echo "Example:"
     echo ""
     echo "       ./wikifiles_dump.sh"
-    echo "       (creates ${BACKUP_DIR}/20200101/wikifiles_20200101.tar.gz)"
+    echo "       (creates ${POD_CHARLESREID1_BACKUP_DIR}/20200101/wikifiles_20200101.tar.gz)"
     echo ""
     exit 1;
 }
@@ -40,47 +37,33 @@ fi
 if [ "$#" == "0" ]; then
 
     TARGET="wikifiles_${STAMP}.tar.gz"
-    BACKUP_TARGET="${BACKUP_DIR}/${STAMP}/${TARGET}"
+    BACKUP_TARGET="${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}/${TARGET}"
 
     echo ""
     echo "pod-charlesreid1: wikifiles_dump.sh"
     echo "-----------------------------------"
     echo ""
+    echo "Backup directory: ${POD_CHARLESREID1_BACKUP_DIR}"
     echo "Backup target: ${BACKUP_TARGET}"
     echo ""
 
-    mkdir -p ${BACKUP_DIR}/${STAMP}
+    mkdir -p ${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}
 
-    # If this script is being run from a cron job,
-    # don't use -i flag with docker
-    CRON="$( pstree -s $$ | /bin/grep -c cron )"
     DOCKER=$(which docker)
-    DOCKERX=""
-    if [[ "$CRON" -eq 1 ]]; 
-    then
-        DOCKERX="${DOCKER} exec -t"
-    else
-        DOCKERX="${DOCKER} exec -it"
-    fi
+    DOCKERX="${DOCKER} exec -t"
 
     echo "Step 1: Compress wiki files inside container"
-    set -x
     ${DOCKERX} ${CONTAINER_NAME} /bin/tar czf /tmp/${TARGET} /var/www/html/images
-    set +x
 
     echo "Step 2: Copy tar.gz file out of container"
-    mkdir -p $(dirname "$1")
-    set -x
+    mkdir -p $(dirname "${BACKUP_TARGET}")
     ${DOCKER} cp ${CONTAINER_NAME}:/tmp/${TARGET} ${BACKUP_TARGET}
-    set +x
 
     echo "Step 3: Clean up tar.gz file"
-    set -x
     ${DOCKERX} ${CONTAINER_NAME} /bin/rm -f /tmp/${TARGET}
-    set +x
 
     echo "Done."
+
 else
     usage
 fi
-

scripts/certbot/pod-charlesreid1-certbot.service.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Renew certificates for pod-charlesreid1
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+StandardError=file:{{ pod_install_dir }}/.pod-charlesreid1-certbot.service.error.log
+StandardOutput=file:{{ pod_install_dir }}/.pod-charlesreid1-certbot.service.output.log
+ExecStart=/bin/bash -ac '. {{ pod_install_dir }}/environment; {{ pod_install_dir }}/scripts/certbot/renew_charlesreid1_certs.sh'

scripts/certbot/pod-charlesreid1-certbot.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Timer to renew certificates for pod-charlesreid1
+
+[Timer]
+# Run on the first Sunday of every month
+OnCalendar=Sun *-*-01..07 4:03:00
+
+[Install]
+WantedBy=timers.target

scripts/certbot/renew_charlesreid1_certs.sh.j2

@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# renew/run certbot on krash
+set -eux
+
+SERVICE="pod-charlesreid1"
+
+function usage {
+    set +x
+    echo ""
+    echo "renew_charlesreid1_certs.sh script:"
+    echo ""
+    echo "Renew all certs used in the charlesreid1.com pod"
+    echo ""
+    echo "       ./renew_charlesreid1_certs.sh"
+    echo ""
+    exit 1;
+}
+
+if [ "$(id -u)" != "0" ]; then
+    echo ""
+    echo ""
+    echo "This script should be run as root."
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+if [ "$#" == "0" ]; then
+
+    # disable system service that will re-spawn docker pod
+    echo "Disable and stop system service ${SERVICE}"
+    sudo systemctl disable ${SERVICE}
+    sudo systemctl stop ${SERVICE}
+    
+    echo "Stop pod"
+    docker-compose -f {{ pod_install_dir }}/docker-compose.yml down
+    
+    echo "Run certbot renew"
+    SUBS="git www"
+    DOMS="charlesreid1.com"
+
+    # top level domains
+    for DOM in $DOMS; do
+        certbot certonly \
+            --standalone \
+            --non-interactive \
+            --agree-tos \
+            --email charles@charlesreid1.com \
+            -d ${DOM}
+    done
+
+    # subdomains
+    for SUB in $SUBS; do
+        for DOM in $DOMS; do
+            certbot certonly \
+                --standalone \
+                --non-interactive \
+                --agree-tos \
+                --email charles@charlesreid1.com \
+                -d ${SUB}.${DOM}
+        done
+    done
+    
+    echo "Start pod"
+    docker-compose -f {{ pod_install_dir }}/docker-compose.yml up -d
+    
+    echo "Enable and start system service ${SERVICE}"
+    sudo systemctl enable ${SERVICE}
+    sudo systemctl start ${SERVICE}
+
+    echo "Done"
+
+else
+    usage
+fi

scripts/mw/build_extensions_dir.sh

@@ -4,14 +4,13 @@
 set -eux
 
 MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
-CONF_DIR="${MW_DIR}/charlesreid1-config"
-MW_CONF_DIR="${MW_CONF_DIR}/mediawiki"
+MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
 EXT_DIR="${MW_CONF_DIR}/extensions"
 
-mkdir -p ${EXT_DIR}/extensions
+mkdir -p ${EXT_DIR}
 
 (
-cd ${EXT_DIR}/extensions
+cd ${EXT_DIR}
 
 ##############################
 

scripts/mw/fix_LocalSettings.sh

@@ -13,8 +13,7 @@ set -eux
 NAME="stormy_mw"
 
 MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
-CONF_DIR="${MW_DIR}/charlesreid1-config"
-MW_CONF_DIR="${MW_CONF_DIR}/mediawiki"
+MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
 
 echo "Checking that container exists"
 docker ps --format '{{.Names}}' | grep ${NAME} || exit 1;

scripts/mw/fix_extensions_dir.sh

@@ -14,8 +14,7 @@ NAME="stormy_mw"
 EXTENSIONS="SyntaxHighlight_GeSHi ParserFunctions EmbedVideo Math Fail2banlog"
 
 MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
-CONF_DIR="${MW_DIR}/charlesreid1-config"
-MW_CONF_DIR="${MW_CONF_DIR}/mediawiki"
+MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
 EXT_DIR="${MW_CONF_DIR}/extensions"
 
 echo "Checking that container exists..."

scripts/mw/fix_skins.sh

@@ -13,8 +13,7 @@ set -eux
 NAME="stormy_mw"
 
 MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
-CONF_DIR="${MW_DIR}/charlesreid1-config"
-MW_CONF_DIR="${MW_CONF_DIR}/mediawiki"
+MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
 SKINS_DIR="${MW_CONF_DIR}/skins"
 
 echo "Checking that container exists"

scripts/pod-charlesreid1.service.j2

@@ -7,7 +7,7 @@ After=docker.service
 Restart=always
 StandardError=null
 StandardOutput=null
-ExecStartPre=test -f {{ pod_install_dir }}/docker-compose.yml
+ExecStartPre=/usr/bin/test -f {{ pod_install_dir }}/docker-compose.yml
 ExecStart=/usr/local/bin/docker-compose -f {{ pod_install_dir }}/docker-compose.yml up
 ExecStop=/usr/local/bin/docker-compose  -f {{ pod_install_dir }}/docker-compose.yml stop