add Troubleshooting.md

* feature/environment-template:
  massive rename of all ansible variables
  prep apply templates script for ansible variable rename
  fix missing var name in environment.j2

.gitignore

@@ -1,20 +1,25 @@
+*.log
 *.pyc
 environment
 attic
 
 # gitea
-#d-gitea/data/
-d-gitea/custom/conf/app.ini
-d-gitea/custom/gitea.db
-d-gitea/custom/avatars
-d-gitea/custom/log/
-d-gitea/custom/queues/
+d-gitea/data/
+d-gitea/custom/
 
 # mediawiki
 charlesreid1.wiki.conf
+d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/Bootstrap2.php
+d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/navbar.php
+d-mediawiki/mediawiki/
+
+# nginx
+d-nginx-charlesreid1/conf.d/http.DOMAIN.conf
+d-nginx-charlesreid1/conf.d/https.DOMAIN.conf
 
 # scripts dir
 scripts/git_*_www.py
+scripts/certbot/renew_charlesreid1_certs.sh
 *.timer
 *.service
 

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "mkdocs-material"]
 	path = mkdocs-material
-	url = git@github.com:charlesreid1-docker/mkdocs-material.git
+	url = https://github.com/charlesreid1/mkdocs-material

Makefile

@@ -26,7 +26,7 @@ help:
 	@echo "--------------------------------------------------"
 	@echo "                   Backups:"
 	@echo ""
-	@echo "make backups:        Create backups of every service (gitea, wiki database, wiki files) in ~/backups"
+	@echo "make backups:        Create backups of every service (wiki database, wiki files) in ~/backups"
 	@echo ""
 	@echo "make clean-backups:  Remove files from ~/backups directory older than 30 days"
 	@echo ""
@@ -53,7 +53,7 @@ help:
 	@echo ""
 	@echo "make install:        Install and start systemd service to run pod-charlesreid1."
 	@echo "                     Also install and start systemd service for pod-charlesreid1 backup services"
-	@echo "                     for each service (gitea/mediawiki/mysql) part of pod-charlesreid1."
+	@echo "                     for each service (mediawiki/mysql) part of pod-charlesreid1."
 	@echo ""
 	@echo "make uninstall:      Remove all systemd startup services and timers part of pod-charlesreid1"
 	@echo ""
@@ -61,18 +61,20 @@ help:
 # Templates
 
 templates:
-	python3 $(POD_CHARLESREID1_DIR)/scripts/apply_templates.py
+	@find * -name "*.service.j2" | xargs -I '{}' chmod 644 {}
+	@find * -name "*.timer.j2" | xargs -I '{}' chmod 644 {}
+	/home/charles/.pyenv/shims/python3 $(POD_CHARLESREID1_DIR)/scripts/apply_templates.py
 
 list-templates:
 	@find * -name "*.j2"
 
 clean-templates:
-	python3 $(POD_CHARLESREID1_DIR)/scripts/clean_templates.py
+	# sudo is required because bind-mounted gitea files end up owned by root. stupid docker.
+	sudo -E /home/charles/.pyenv/shims/python3 $(POD_CHARLESREID1_DIR)/scripts/clean_templates.py
 
 # Backups
 
-backups: templates
-	$(POD_CHARLESREID1_DIR)/scripts/backups/gitea_dump.sh
+backups:
 	$(POD_CHARLESREID1_DIR)/scripts/backups/wikidb_dump.sh
 	$(POD_CHARLESREID1_DIR)/scripts/backups/wikifiles_dump.sh
 
@@ -88,52 +90,105 @@ mw-fix-extensions: mw-build-extensions
 	$(POD_CHARLESREID1_DIR)/scripts/mw/build_extensions_dir.sh
 
 mw-fix-localsettings:
-	$(POD_CHARLESEREID1_DIR)/scripts/mw/fix_LocalSettings.sh
+	$(POD_CHARLESREID1_DIR)/scripts/mw/fix_LocalSettings.sh
 
 mw-fix-skins:
-	$(POD_CHARLESEREID1_DIR)/scripts/mw/fix_skins.sh
+	$(POD_CHARLESREID1_DIR)/scripts/mw/fix_skins.sh
 
 # /www Dir
 
-clone-www: templates
-	python3 $(POD_CHARLESREID1_DIR)/scripts/git_clone_www.py
+clone-www:
+	/home/charles/.pyenv/shims/python3 $(POD_CHARLESREID1_DIR)/scripts/git_clone_www.py
 
-pull-www: templates
-	python3 $(POD_CHARLESREID1_DIR)/scripts/git_pull_www.py
+pull-www:
+	/home/charles/.pyenv/shims/python3 $(POD_CHARLESREID1_DIR)/scripts/git_pull_www.py
 
-install: templates
+install:
 ifeq ($(shell which systemctl),)
 	$(error Please run this make command on a system with systemctl installed)
 endif
-	cp $(POD_CHARLESREID1_DIR)/scripts/pod-charlesreid1.service /etc/systemd/system/pod-charlesreid1.service
-	cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-gitea.{service,timer} /etc/systemd/system/.
-	cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikidb.{service,timer} /etc/systemd/system/.
-	cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikifiles.{service,timer} /etc/systemd/system/.
-	systemctl daemon-reload
-	systemctl enable pod-charlesreid1
-	systemctl enable pod-charlesreid1-backups-gitea.timer
-	systemctl enable pod-charlesreid1-backups-wikidb.timer
-	systemctl enable pod-charlesreid1-backups-wikifiles.timer
-	systemctl start pod-charlesreid1-backups-gitea.timer
-	systemctl start pod-charlesreid1-backups-wikidb.timer
-	systemctl start pod-charlesreid1-backups-wikifiles.timer
+	@/home/charles/.pyenv/shims/python3 -c 'import botocore' || (echo "Please install the botocore library using python3 or pip3 binary"; exit 1)
+	@/home/charles/.pyenv/shims/python3 -c 'import boto3' || (echo "Please install the boto3 library using python3 or pip3 binary"; exit 1)
+
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/pod-charlesreid1.service /etc/systemd/system/pod-charlesreid1.service
+
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-aws.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-cleanolderthan.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-gitea.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikidb.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/pod-charlesreid1-backups-wikifiles.{service,timer} /etc/systemd/system/.
+
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/canary/pod-charlesreid1-canary.{service,timer} /etc/systemd/system/.
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/certbot/pod-charlesreid1-certbot.{service,timer} /etc/systemd/system/.
+
+	sudo cp $(POD_CHARLESREID1_DIR)/scripts/backups/10-pod-charlesreid1-rsyslog.conf /etc/rsyslog.d/.
+
+	sudo chmod 664 /etc/systemd/system/pod-charlesreid1*
+	sudo systemctl daemon-reload
+
+	sudo systemctl restart rsyslog
+
+	sudo systemctl enable pod-charlesreid1
+	sudo systemctl enable pod-charlesreid1-backups-wikidb.timer
+	sudo systemctl enable pod-charlesreid1-backups-wikifiles.timer
+	sudo systemctl enable pod-charlesreid1-backups-gitea.timer
+	sudo systemctl enable pod-charlesreid1-backups-aws.timer
+	sudo systemctl enable pod-charlesreid1-backups-cleanolderthan.timer
+	sudo systemctl enable pod-charlesreid1-canary.timer
+	sudo systemctl enable pod-charlesreid1-certbot.timer
+
+	sudo systemctl start pod-charlesreid1-backups-wikidb.timer
+	sudo systemctl start pod-charlesreid1-backups-wikifiles.timer
+	sudo systemctl start pod-charlesreid1-backups-gitea.timer
+	sudo systemctl start pod-charlesreid1-backups-aws.timer
+	sudo systemctl start pod-charlesreid1-backups-cleanolderthan.timer
+	sudo systemctl start pod-charlesreid1-canary.timer
+	sudo systemctl start pod-charlesreid1-certbot.timer
+
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-aws.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-cleanolderthan.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-gitea.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-wikidb.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-backups-wikifiles.service.log
+	sudo chown syslog:syslog /var/log/pod-charlesreid1-canary.service.log
 
 uninstall:
 ifeq ($(shell which systemctl),)
 	$(error Please run this make command on a system with systemctl installed)
 endif
-	systemctl disable pod-charlesreid1
-	systemctl disable pod-charlesreid1-backups-gitea.timer
-	systemctl disable pod-charlesreid1-backups-wikidb.timer
-	systemctl disable pod-charlesreid1-backups-wikifiles.timer
-	systemctl stop pod-charlesreid1
-	systemctl stop pod-charlesreid1-backups-gitea.timer
-	systemctl stop pod-charlesreid1-backups-wikidb.timer
-	systemctl stop pod-charlesreid1-backups-wikifiles.timer
-	rm -f /etc/systemd/system/pod-charlesreid1.service
-	rm -f /etc/systemd/system/pod-charlesreid1-backups-gitea.{service,timer}
-	rm -f /etc/systemd/system/pod-charlesreid1-backups-wikidb.{service,timer}
-	rm -f /etc/systemd/system/pod-charlesreid1-backups-wikifiles.{service,timer}
-	systemctl daemon-reload
+	-sudo systemctl disable pod-charlesreid1
+	-sudo systemctl disable pod-charlesreid1-backups-aws.timer
+	-sudo systemctl disable pod-charlesreid1-backups-cleanolderthan.timer
+	-sudo systemctl disable pod-charlesreid1-backups-gitea.timer
+	-sudo systemctl disable pod-charlesreid1-backups-wikidb.timer
+	-sudo systemctl disable pod-charlesreid1-backups-wikifiles.timer
+	-sudo systemctl disable pod-charlesreid1-canary.timer
+	-sudo systemctl disable pod-charlesreid1-certbot.timer
+
+	# Leave the pod running!
+	# -sudo systemctl stop pod-charlesreid1
+
+	-sudo systemctl stop pod-charlesreid1-backups-aws.timer
+	-sudo systemctl stop pod-charlesreid1-backups-cleanolderthan.timer
+	-sudo systemctl stop pod-charlesreid1-backups-gitea.timer
+	-sudo systemctl stop pod-charlesreid1-backups-wikidb.timer
+	-sudo systemctl stop pod-charlesreid1-backups-wikifiles.timer
+	-sudo systemctl stop pod-charlesreid1-canary.timer
+	-sudo systemctl stop pod-charlesreid1-certbot.timer
+
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1.service
+
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-aws.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-cleanolderthan.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-gitea.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-wikidb.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-backups-wikifiles.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-canary.{service,timer}
+	-sudo rm -f /etc/systemd/system/pod-charlesreid1-certbot.{service,timer}
+
+	sudo systemctl daemon-reload
+
+	-sudo rm -f /etc/rsyslog.d/10-pod-charlesreid1-rsyslog.conf
+	-sudo systemctl restart rsyslog
 
 .PHONY: help

Troubleshooting.md

@@ -0,0 +1,19 @@
+To get a shell in a container that has been created, before it is runnning in a pod, use `docker run`:
+
+```
+docker run --rm -it --entrypoint bash <image-name-or-id>
+
+
+docker run --rm -it --entrypoint bash pod-charlesreid1_stormy_mediawiki
+```
+
+To get a shell in a container that is running in a pod, use `docker exec`:
+
+```
+docker exec -it <image-name> /bin/bash
+
+docker exec -it stormy_mw /bin/bash
+```
+
+Also, if no changes are picking up, and you've already tried rebuilding the container image, try editing the Dockerfile.
+

d-gitea/README.md

@@ -18,7 +18,7 @@ The data directory contains any instance-specific gitea data.
 
 The data directory is bind-mounted to `/app/gitea/data` in the container.
 
-## Repository Data
+## Repository Drive
 
 Gitea stores all of its repositories in a separate drive that is at
 `/gitea_repositories` on the host machine.

d-gitea/custom/conf/app.ini.j2

@@ -6,12 +6,14 @@
 ;; https://github.com/go-gitea/gitea/blob/master/conf/app.ini
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-APP_NAME = {{ gitea_app_name }}
+APP_NAME = {{ pod_charlesreid1_gitea_app_name }}
 RUN_USER = git
 RUN_MODE = prod
+WORK_PATH = /data/gitea
 
 [ui]
-DEFAULT_THEME = arc-green
+DEFAULT_THEME = gitea-dark
+THEMES = gitea-dark
 
 [database]
 DB_TYPE  = sqlite3
@@ -31,17 +33,17 @@ DISABLE_HTTP_GIT = false
 
 [server]
 PROTOCOL     = http
-DOMAIN       = git.{{ server_name_default }}
+DOMAIN       = git.{{ pod_charlesreid1_server_name }}
 #CERT_FILE    = /www/gitea/certs/cert.pem
 #KEY_FILE     = /www/gitea/certs/key.pem
-SSH_DOMAIN   = git.{{ server_name_default }}
+SSH_DOMAIN   = git.{{ pod_charlesreid1_server_name }}
 HTTP_PORT    = 3000
 HTTP_ADDR    = 0.0.0.0
-ROOT_URL     = https://git.{{ server_name_default }}
+ROOT_URL     = https://git.{{ pod_charlesreid1_server_name }}
 ;ROOT_URL     = %(PROTOCOL)s://%(DOMAIN)s:%(HTTP_PORT)s/
 DISABLE_SSH  = false
 ; port to display in clone url:
-SSH_PORT     = 222
+;SSH_PORT     = 222
 ; port for built-in ssh server to listen on:
 SSH_LISTEN_PORT = 22
 OFFLINE_MODE = false
@@ -92,9 +94,9 @@ ENABLED      = false
 
 [security]
 INSTALL_LOCK        = true
-SECRET_KEY          = {{ gitea_secret_key }}
+SECRET_KEY          = {{ pod_charlesreid1_gitea_secretkey }}
 MIN_PASSWORD_LENGTH = 6
-INTERNAL_TOKEN      = {{ gitea_internal_token }}
+INTERNAL_TOKEN      = {{ pod_charlesreid1_gitea_internaltoken }}
 
 [other]
 SHOW_FOOTER_BRANDING           = false

d-mediawiki/Dockerfile

@@ -1,4 +1,4 @@
-FROM mediawiki
+FROM mediawiki:1.34
 
 EXPOSE 8989
 
@@ -41,17 +41,13 @@ RUN chown -R www-data:www-data /var/www/html/*
 # Skins
 COPY charlesreid1-config/mediawiki/skins /var/www/html/skins
 RUN chown -R www-data:www-data /var/www/html/skins
+RUN touch /var/www/html/skins
 
 # Settings
 COPY charlesreid1-config/mediawiki/LocalSettings.php /var/www/html/LocalSettings.php
 RUN chown -R www-data:www-data /var/www/html/LocalSettings*
 RUN chmod 600 /var/www/html/LocalSettings.php
 
-# MediaWiki Fail2ban log directory
-RUN mkdir -p /var/log/mwf2b
-RUN chown -R www-data:www-data /var/log/mwf2b
-RUN chmod 700 /var/log/mwf2b
-
 # Apache conf file
 COPY charlesreid1-config/apache/*.conf /etc/apache2/sites-enabled/
 RUN a2enmod rewrite
@@ -59,4 +55,10 @@ RUN service apache2 restart
 
 ## make texvc
 #CMD cd /var/www/html/extensions/Math && make && apache2-foreground
+
+# PHP conf file
+# https://hub.docker.com/_/php/
+COPY php/php.ini /usr/local/etc/php/
+
+# Start
 CMD apache2-foreground

d-mediawiki/Skins.md

@@ -5,6 +5,10 @@ To update the MediaWiki skin:
 - Rebuild the MW container while the docker pod is still running (won't effect the docker pod)
 - When finished rebuilding the MW container, restart the docker pod.
 
+The skin currently in use is in `charlesreid1-config/mediawiki/skins/Bootstrap2`
+
+To rebuild and then restart the pod:
+
 ```
 # switch to main pod directory
 cd ../

d-mediawiki/charlesreid1-config/apache/charlesreid1.wiki.conf.j2

@@ -1,4 +1,4 @@
-ServerName {{ server_name_default }}
+ServerName {{ pod_charlesreid1_server_name }}
 
 Listen 8989
 
@@ -7,10 +7,10 @@ Listen 8989
     # talks to apache via 127.0.0.1
     # on port 8989
 
-    ServerAlias www.{{ server_name_default }}
+    ServerAlias www.{{ pod_charlesreid1_server_name }}
 
     LogLevel warn
-    ServerAdmin {{ admin_email }}
+    ServerAdmin {{ pod_charlesreid1_mediawiki_admin_email }}
     DirectoryIndex index.html index.cgi index.php
 
 

d-mediawiki/charlesreid1-config/mediawiki/LocalSettings.php.j2

@@ -13,8 +13,8 @@ if ( !defined( 'MEDIAWIKI' ) ) {
 }
 
 ## The protocol and server name to use in fully-qualified URLs
-$wgServer = 'https://{{ server_name_default }}';
-$wgCanonicalServer = 'https://{{ server_name_default }}';
+$wgServer = 'https://{{ pod_charlesreid1_server_name }}';
+$wgCanonicalServer = 'https://{{ pod_charlesreid1_server_name }}';
 
 ## The URL path to static resources (images, scripts, etc.)
 $wgStylePath = "$wgScriptPath/skins";
@@ -209,13 +209,6 @@ wfLoadExtension( 'EmbedVideo' );
 
 require_once "$IP/extensions/Math/Math.php";
 
-#############################################
-# Fail2banlog extension
-# https://www.mediawiki.org/wiki/Extension:Fail2banlog
-
-require_once "$IP/extensions/Fail2banlog/Fail2banlog.php";
-$wgFail2banlogfile = "/var/log/apache2/mwf2b.log";
-
 #############################################
 # Fix cookies crap
 
@@ -224,7 +217,7 @@ session_save_path("/tmp");
 ##############################################
 # Secure login
 
-$wgServer = "https://{{ server_name_default }}";
+$wgServer = "https://{{ pod_charlesreid1_server_name }}";
 $wgSecureLogin = true;
 
 ###################################

d-mediawiki/charlesreid1-config/mediawiki/build_extensions_dir.sh

@@ -1,93 +0,0 @@
-#!/bin/bash
-# 
-# clone or download each extension
-# and build o
-
-mkdir -p extensions
-(
-cd extensions
-
-##############################
-
-Extension="SyntaxHighlight_GeSHi"
-if [ ! -d ${Extension} ]
-then
-    ## This requires mediawiki > 1.31
-    ## (so does REL1_31)
-    #git clone https://github.com/wikimedia/mediawiki-extensions-SyntaxHighlight_GeSHi.git SyntaxHighlight_GeSHi
-
-    ## This manually downloads REL1_30
-    #wget https://extdist.wmflabs.org/dist/extensions/SyntaxHighlight_GeSHi-REL1_30-87392f1.tar.gz -O SyntaxHighlight_GeSHi.tar.gz
-    #tar -xzf SyntaxHighlight_GeSHi.tar.gz -C ${PWD}
-    #rm -f SyntaxHighlight_GeSHi.tar.gz
-
-    # Best of both worlds
-    git clone https://github.com/wikimedia/mediawiki-extensions-SyntaxHighlight_GeSHi.git SyntaxHighlight_GeSHi
-    (
-    cd ${Extension}
-    git checkout --track remotes/origin/REL1_34
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="ParserFunctions"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/wikimedia/mediawiki-extensions-ParserFunctions.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout --track remotes/origin/REL1_34
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="EmbedVideo"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/HydraWiki/mediawiki-embedvideo.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout v2.7.3
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="Math"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/wikimedia/mediawiki-extensions-Math.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout REL1_34
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-Extension="Fail2banlog"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/charlesreid1-docker/mw-fail2ban.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout master
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
-# fin
-)

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/Bootstrap2.php.j2

@@ -106,7 +106,7 @@ include('/var/www/html/skins/Bootstrap2/navbar.php');
             <div class="container-fixed">
                 <div class="navbar-header">
                     <a href="/wiki/" class="navbar-brand">
-                    {{ top_domain }} wiki
+                    {{ pod_charlesreid1_server_name }} wiki
                     </a>
                 </div>
                 <div>

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/footer.php

@@ -11,7 +11,7 @@
     </span>
     Made from the command line with vim by 
     <a href="http://charlesreid1.com">charlesreid1</a><br />
-    with help from <a href="https://getbootstrap.com/">Bootstrap</a> and <a href="http://getpelican.com">Pelican</a>.
+    with help from <a href="https://getbootstrap.com/">Bootstrap</a> and <a href="http://mediawiki.org">MediaWiki</a>.
 </p>
 
 <p style="text-align: center">

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/navbar.php.j2

@@ -6,14 +6,14 @@
                 <span class="icon-bar"></span>
                 <span class="icon-bar"></span> 
             </button>
-            <a href="/" class="navbar-brand">{{ top_domain }}</a>
+            <a href="/" class="navbar-brand">{{ pod_charlesreid1_server_name }}</a>
         </div>
         <div>
             <div class="collapse navbar-collapse" id="myNavbar">
                 <ul class="nav navbar-nav">
 
                     <li>
-                        <a href="https://{{ top_domain }}/wiki">Wiki</a>
+                        <a href="https://{{ pod_charlesreid1_server_name }}/wiki">Wiki</a>
                     </li>
 
                 </ul>

d-mediawiki/charlesreid1-config/mediawiki/skins/Bootstrap2/slate.css

@@ -1086,7 +1086,8 @@ html {
 }
 body {
   font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-  font-size: 14px;
+  /*font-size: 14px;*/
+  font-size: 20px;
   line-height: 1.42857143;
   color: #c8c8c8;
   background-color: #272b30;

d-mediawiki/php/php.ini

@@ -0,0 +1,3 @@
+post_max_size = 128M
+memory_limit = 128M
+upload_max_filesize = 100M

d-nginx-charlesreid1/conf.d/http.DOMAIN.conf.j2

@@ -1,6 +1,6 @@
 ####################
 # 
-# {{ server_name_default }}
+# {{ pod_charlesreid1_server_name }}
 # http/{{ port_default }}
 # 
 # basically, just redirects to https
@@ -10,20 +10,20 @@
 server {
     listen 80;
     listen [::]:80;
-    server_name {{ server_name_default }};
-    return 301 https://{{ server_name_default }}$request_uri;
+    server_name {{ pod_charlesreid1_server_name }};
+    return 301 https://{{ pod_charlesreid1_server_name }}$request_uri;
 }
 
 server {
     listen 80;
     listen [::]:80;
-    server_name www.{{ server_name_default }};
-    return 301 https://www.{{ server_name_default }}$request_uri;
+    server_name www.{{ pod_charlesreid1_server_name }};
+    return 301 https://www.{{ pod_charlesreid1_server_name }}$request_uri;
 }
 
 server {
     listen 80;
     listen [::]:80;
-    server_name git.{{ server_name_default }};
-    return 301 https://git.{{ server_name_default }}$request_uri;
+    server_name git.{{ pod_charlesreid1_server_name }};
+    return 301 https://git.{{ pod_charlesreid1_server_name }}$request_uri;
 }

d-nginx-charlesreid1/conf.d/https.DOMAIN.conf.j2

@@ -1,9 +1,9 @@
 ####################
 #
-# {{ server_name_default }}
+# {{ pod_charlesreid1_server_name }}
 # https/443
 # 
-# {{ server_name_default }} and www.{{ server_name_default }}
+# {{ pod_charlesreid1_server_name }} and www.{{ pod_charlesreid1_server_name }}
 # should handle the following cases:
 # - w/ and wiki/ should reverse proxy story_mw
 # - gitea subdomain should reverse proxy stormy_gitea
@@ -15,20 +15,24 @@
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
-    server_name {{ server_name_default }} default_server;
+    server_name {{ pod_charlesreid1_server_name }} default_server;
 
-    ssl_certificate /etc/letsencrypt/live/{{ server_name_default }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/{{ server_name_default }}/privkey.pem;
+    ssl_certificate /etc/letsencrypt/live/{{ pod_charlesreid1_server_name }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ pod_charlesreid1_server_name }}/privkey.pem;
     include /etc/letsencrypt/options-ssl-nginx.conf;
     include /etc/nginx/conf.d/secheaders.conf;
     include /etc/nginx/conf.d/csp.conf;
 
     location / {
         try_files $uri $uri/ =404;
-        root /www/{{ server_name_default }}/htdocs;
+        root /www/{{ pod_charlesreid1_server_name }}/htdocs;
         index index.html;
     }
 
+    location = /robots.txt {
+        alias /var/www/robots/robots.txt;
+    }
+
     location /wiki/ {
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
@@ -55,21 +59,25 @@ server {
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
-    server_name www.{{ server_name_default }};
+    server_name www.{{ pod_charlesreid1_server_name }};
 
-    ssl_certificate /etc/letsencrypt/live/www.{{ server_name_default }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/www.{{ server_name_default }}/privkey.pem;
+    ssl_certificate /etc/letsencrypt/live/www.{{ pod_charlesreid1_server_name }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/www.{{ pod_charlesreid1_server_name }}/privkey.pem;
     include /etc/letsencrypt/options-ssl-nginx.conf;
     include /etc/nginx/conf.d/secheaders.conf;
     include /etc/nginx/conf.d/csp.conf;
 
-    root /www/{{ server_name_default }}/htdocs;
+    root /www/{{ pod_charlesreid1_server_name }}/htdocs;
 
     location / {
         try_files $uri $uri/ =404;
         index index.html;
     }
 
+    location = /robots.txt {
+        alias /var/www/robots/robots.txt;
+    }
+
     location /wiki/ {
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
@@ -94,18 +102,50 @@ server {
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
-    server_name git.{{ server_name_default }};
+    server_name git.{{ pod_charlesreid1_server_name }};
 
-    ssl_certificate /etc/letsencrypt/live/git.{{ server_name_default }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/git.{{ server_name_default }}/privkey.pem;
+    ssl_certificate /etc/letsencrypt/live/git.{{ pod_charlesreid1_server_name }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/git.{{ pod_charlesreid1_server_name }}/privkey.pem;
     include /etc/letsencrypt/options-ssl-nginx.conf;
     include /etc/nginx/conf.d/secheaders.conf;
     include /etc/nginx/conf.d/giteacsp.conf;
 
     location / {
+        # Ban jerks
+        deny 52.39.237.48;
+        deny 52.70.240.171;
+        deny 54.36.148.135;
+        deny 34.215.160.160;
+        deny 217.113.194.226;
+        deny 189.84.38.222;
+
+        deny 47.76.0.0/16;
+        deny 47.79.0.0/16;
+        # Fuck you in particular
+        deny 47.76.209.138;
+        deny 47.76.99.127;
+        deny 47.76.220.119;
+        deny 47.79.118.97;
+        deny 84.33.26.105;
+
+        deny 8.210.0.0/16;
+        deny 8.218.0.0/16;
+        # Fuck you in particular
+        deny 8.210.187.5;
+        deny 8.210.164.94;
+        deny 168.90.209.163;
+        deny 168.90.209.127;
+
+        deny 89.116.78.169;
+        allow all;
+
         proxy_set_header X-Real-IP  $remote_addr;
         proxy_set_header X-Forwarded-For $remote_addr;
         proxy_set_header Host $host;
         proxy_pass http://stormy_gitea:3000/;
     }
+
+    location = /robots.txt {
+        alias /var/www/robots/gitea.txt;
+    }
 }

d-nginx-charlesreid1/robots/gitea.txt

@@ -0,0 +1,16 @@
+User-agent: *
+Disallow: */commit/*
+Disallow: */src/*
+Disallow: */tree/*
+Disallow: */activity/*
+Disallow: */wiki/*
+Disallow: */releases/*
+Disallow: */pulls/*
+Disallow: */stars
+Disallow: */watchers
+Disallow: */forks
+Disallow: *?tab=activity
+Disallow: *?tab=stars
+Disallow: *?tab=following
+Disallow: *?tab=followers
+Disallow: *?lang=*

d-nginx-charlesreid1/robots/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /w/

docker-compose.yml.j2

@@ -13,6 +13,7 @@ services:
     restart: always
     volumes:
       - "stormy_gitea_data:/data"
+      - "./d-nginx-charlesreid1/robots:/var/www/robots:ro"
       - "./d-gitea/custom:/data/gitea"
       - "./d-gitea/data:/app/gitea/data"
       - "/gitea_repositories:/data/git/repositories"
@@ -35,26 +36,25 @@ services:
         max-size: 1m
         max-file: "10"
     environment:
-      - MYSQL_ROOT_PASSWORD={{ mysql_password }}
+      - MYSQL_ROOT_PASSWORD={{ pod_charlesreid1_mysql_password }}
 
   stormy_mw:
     build: d-mediawiki
     container_name: stormy_mw
     volumes:
       - "stormy_mw_data:/var/www/html"
-      - "./mwf2b:/var/log/mwf2b"
     logging:
       driver: "json-file"
       options:
         max-size: 1m
         max-file: "10"
     environment:
-      - MEDIAWIKI_SITE_SERVER=https://{{ server_name_default }}
-      - MEDIAWIKI_SECRETKEY={{ mediawiki_secretkey }}
+      - MEDIAWIKI_SITE_SERVER=https://{{ pod_charlesreid1_server_name }}
+      - MEDIAWIKI_SECRETKEY={{ pod_charlesreid1_mediawiki_secretkey }}
       - MYSQL_HOST=stormy_mysql
       - MYSQL_DATABASE=wikidb
       - MYSQL_USER=root
-      - MYSQL_PASSWORD={{ mysql_password }}
+      - MYSQL_PASSWORD={{ pod_charlesreid1_mysql_password }}
     depends_on:
       - stormy_mysql
 
@@ -62,14 +62,15 @@ services:
     restart: always
     image: nginx
     container_name: stormy_nginx
-    hostname: {{ server_name_default }}
+    hostname: {{ pod_charlesreid1_server_name }}
     hostname: charlesreid1.com
     command: /bin/bash -c "nginx -g 'daemon off;'"
     volumes:
       - "./d-nginx-charlesreid1/conf.d:/etc/nginx/conf.d:ro"
+      - "./d-nginx-charlesreid1/robots:/var/www/robots:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "/etc/letsencrypt:/etc/letsencrypt"
-      - "/www/{{ server_name_default }}/htdocs:/www/{{ server_name_default }}/htdocs:ro"
+      - "/www/{{ pod_charlesreid1_server_name }}/htdocs:/www/{{ pod_charlesreid1_server_name }}/htdocs:ro"
     logging:
       driver: "json-file"
       options:

docs/BlockIps.md

@@ -0,0 +1,9 @@
+To block IP address:
+
+* Modify the nginx config file template at 
+  `d-nginx-charlesreid1/conf.d/https.DOMAIN.conf.j2`
+* Re-render the Jinja templates into config files via
+  `make clean-templates && make templates`
+* Stop and restart the pod service:
+  `sudo systemctl stop pod-charlesreid1 && 
+  sudo systemctl start pod-charlesreid1`

environment.example

@@ -2,31 +2,33 @@
 
 # multiple templates:
 # -------------------
-POD_CHARLESREID1_DIR="/path/to/pod-charlesreid1"
-POD_CHARLESREID1_TLD="example.com"
+export POD_CHARLESREID1_DIR="/path/to/pod-charlesreid1"
+export POD_CHARLESREID1_TLD="example.com"
+export POD_CHARLESREID1_USER="nonrootuser"
 
 # mediawiki:
 # ----------
-POD_CHARLESREID1_MW_ADMIN_EMAIL="email@example.com"
-POD_CHARLESREID1_MW_SECRET_KEY="SecretKeyString"
+export POD_CHARLESREID1_MW_ADMIN_EMAIL="email@example.com"
+export POD_CHARLESREID1_MW_SECRET_KEY="SecretKeyString"
 
 # mysql:
 # ------
-POD_CHARLESREID1_MYSQL_PASSWORD="SuperSecretPassword"
+export POD_CHARLESREID1_MYSQL_PASSWORD="SuperSecretPassword"
 
 # gitea:
 # ------
-POD_CHARLESREID1_GITEA_APP_NAME=""
-POD_CHARLESREID1_GITEA_SECRET_KEY="GiteaSecretKey"
-POD_CHARLESREID1_GITEA_INTERNAL_TOKEN="GiteaInternalToken"
+export POD_CHARLESREID1_GITEA_APP_NAME=""
+export POD_CHARLESREID1_GITEA_SECRET_KEY="GiteaSecretKey"
+export POD_CHARLESREID1_GITEA_INTERNAL_TOKEN="GiteaInternalToken"
 
 # aws:
 # ----
-POD_CHARLESREID1_AWS_ACCESS_KEY="AAAAAAAAAAAAAAAAAAAA"
-POD_CHARLESREID1_AWS_ACCESS_SECRET="0000000000000000000000000000000000000000"
+export AWS_ACCESS_KEY_ID="AAAAAAA"
+export AWS_SECRET_ACCESS_KEY="BBBBBBBB"
+export AWS_DEFAULT_REGION="us-west-1"
 
 # backups and scripts:
 # --------------------
-POD_CHARLESREID1_USER="charles"
-POD_CHARLESREID1_BACKUP_S3BUCKET="name-of-backups-bucket"
-POD_CHARLESREID1_BACKUPCANARY_WEBHOOKURL="https://hooks.slack.com/services/000000000/AAAAAAAAA/111111111111111111111111"
+export POD_CHARLESREID1_USER="charles"
+export POD_CHARLESREID1_BACKUP_S3BUCKET="name-of-backups-bucket"
+export POD_CHARLESREID1_CANARY_WEBHOOK="https://hooks.slack.com/services/000000000/AAAAAAAAA/111111111111111111111111"

environment.j2

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# multiple templates:
+# -------------------
+export POD_CHARLESREID1_DIR="{{ pod_charlesreid1_pod_install_dir }}"
+export POD_CHARLESREID1_TLD="{{ pod_charlesreid1_server_name }}"
+export POD_CHARLESREID1_USER="{{ pod_charlesreid1_username }}"
+export POD_CHARLESREID1_VPN_IP_ADDR="{{ pod_charlesreid1_vpn_ip_addr }}"
+
+# mediawiki:
+# ----------
+export POD_CHARLESREID1_MW_ADMIN_EMAIL="{{ pod_charlesreid1_mediawiki_admin_email }}"
+export POD_CHARLESREID1_MW_SECRET_KEY="{{ pod_charlesreid1_mediawiki_secretkey }}"
+
+# mysql:
+# ------
+export POD_CHARLESREID1_MYSQL_PASSWORD="{{ pod_charlesreid1_mysql_password }}"
+
+# gitea:
+# ------
+export POD_CHARLESREID1_GITEA_APP_NAME="{{ pod_charlesreid1_gitea_app_name }}"
+export POD_CHARLESREID1_GITEA_SECRET_KEY="{{ pod_charlesreid1_gitea_secretkey }}"
+export POD_CHARLESREID1_GITEA_INTERNAL_TOKEN="{{ pod_charlesreid1_gitea_internaltoken }}"
+
+# aws:
+# ----
+export AWS_ACCESS_KEY_ID="{{ pod_charlesreid1_backups_aws_access_key }}"
+export AWS_SECRET_ACCESS_KEY="{{ pod_charlesreid1_backups_aws_secret_access_key }}"
+export AWS_DEFAULT_REGION="{{ pod_charlesreid1_backups_aws_region }}"
+
+# backups and scripts:
+# --------------------
+export POD_CHARLESREID1_BACKUP_DIR="{{ pod_charlesreid1_backups_dir }}"
+export POD_CHARLESREID1_BACKUP_S3BUCKET="{{ pod_charlesreid1_backups_bucket }}"
+export POD_CHARLESREID1_CANARY_WEBHOOK="{{ pod_charlesreid1_backups_canary_slack_url }}"

scripts/Readme.md

@@ -21,10 +21,12 @@ Cleans all rendered Jinja templates. Does not require environment variables.
 This script is destructive! Be careful!
 
 
-# Ansible Scripts
+# /www Directory Scripts
 
-These scripts are used by ansible when setting up a machine
-to run the charlesreid1 docker pod.
+These scripts set up or pull a git repo that is set up to
+have a pecular directory structure.
+
+The clone script is used by Ansible when setting up this pod.
 
 ## `git_clone_www.py`
 

scripts/apply_templates.py

@@ -2,36 +2,40 @@ import os
 import re
 import sys
 import glob
+import time
+import subprocess
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 
-"""
-Apply Default Values to all Jinja Templates
-"""
-
 
 # Should existing files be overwritten
-OVERWRITE = True 
+OVERWRITE = False
 
+# Map of jinja variables to environment variables
+jinja_to_env = {
+    "pod_charlesreid1_pod_install_dir": "POD_CHARLESREID1_DIR",
+    "pod_charlesreid1_server_name": "POD_CHARLESREID1_TLD",
+    "pod_charlesreid1_username": "POD_CHARLESREID1_USER",
+    "pod_charlesreid1_vpn_ip_addr": "POD_CHARLESREID1_VPN_IP_ADDR",
+    "pod_charlesreid1_mediawiki_admin_email": "POD_CHARLESREID1_MW_ADMIN_EMAIL",
+    "pod_charlesreid1_mediawiki_secretkey": "POD_CHARLESREID1_MW_SECRET_KEY",
+    "pod_charlesreid1_mysql_password": "POD_CHARLESREID1_MYSQL_PASSWORD",
+    "pod_charlesreid1_gitea_app_name": "POD_CHARLESREID1_GITEA_APP_NAME",
+    "pod_charlesreid1_gitea_secretkey": "POD_CHARLESREID1_GITEA_SECRET_KEY",
+    "pod_charlesreid1_gitea_internaltoken": "POD_CHARLESREID1_GITEA_INTERNAL_TOKEN",
+    "pod_charlesreid1_backups_aws_access_key": "AWS_ACCESS_KEY_ID",
+    "pod_charlesreid1_backups_aws_secret_access_key": "AWS_SECRET_ACCESS_KEY",
+    "pod_charlesreid1_backups_aws_region": "AWS_DEFAULT_REGION",
+    "pod_charlesreid1_backups_dir": "POD_CHARLESREID1_BACKUP_DIR",
+    "pod_charlesreid1_backups_bucket": "POD_CHARLESREID1_BACKUP_S3BUCKET",
+    "pod_charlesreid1_backups_canary_slack_url": "POD_CHARLESREID1_CANARY_WEBHOOK",
+}
 
 scripts_dir = os.path.dirname(os.path.abspath(__file__))
 repo_root = os.path.abspath(os.path.join(scripts_dir, '..'))
 
 
 def check_env_vars():
-    env_var_list = [
-        'POD_CHARLESREID1_DIR',
-        'POD_CHARLESREID1_TLD',
-        'POD_CHARLESREID1_USER',
-        'POD_CHARLESREID1_MYSQL_PASSWORD',
-        'POD_CHARLESREID1_MW_ADMIN_EMAIL',
-        'POD_CHARLESREID1_GITEA_APP_NAME',
-        'POD_CHARLESREID1_GITEA_SECRET_KEY',
-        'POD_CHARLESREID1_GITEA_INTERNAL_TOKEN',
-        'POD_CHARLESREID1_BACKUP_S3BUCKET',
-        'POD_CHARLESREID1_AWS_ACCESS_KEY',
-        'POD_CHARLESREID1_AWS_ACCESS_SECRET',
-        'POD_CHARLESREID1_BACKUPCANARY_WEBHOOKURL',
-    ]
+    env_var_list = jinja_to_env.values()
     nerrs = 0
     print("Checking environment variables")
     for env_var in env_var_list:
@@ -48,6 +52,8 @@ def main():
 
     check_env_vars()
 
+    ignore_list = ['environment']
+
     p = os.path.join(repo_root,'**','*.j2')
     template_files = glob.glob(p, recursive=True)
 
@@ -63,41 +69,35 @@ def main():
         rname = tname[:-3]
         rpath = os.path.join(tdir, rname)
 
+        if rname in ignore_list:
+            print(f"\nSkipping template on ignore list: {tname}\n")
+            continue
+
         env = Environment(loader=FileSystemLoader(tdir))
     
         print(f"Rendering template {tname}:")
         print(f"    Template path: {tpath}")
         print(f"    Output path: {rpath}")
-        #content = env.get_template(tpath).render({
-        content = env.get_template(tname).render({
-            "pod_install_dir": os.environ['POD_CHARLESREID1_DIR'],
-            "top_domain": os.environ['POD_CHARLESREID1_TLD'],
-            "server_name_default" : os.environ['POD_CHARLESREID1_TLD'],
-            "username": os.environ['POD_CHARLESREID1_USER'],
-            # docker-compose:
-            "mysql_password" : os.environ['POD_CHARLESREID1_MYSQL_PASSWORD'],
-            "mediawiki_secretkey" : os.environ['POD_CHARLESREID1_MW_ADMIN_EMAIL'],
-            # mediawiki:
-            "admin_email": os.environ['POD_CHARLESREID1_MW_ADMIN_EMAIL'],
-            # gitea:
-            "gitea_app_name": os.environ['POD_CHARLESREID1_GITEA_APP_NAME'],
-            "gitea_secret_key": os.environ['POD_CHARLESREID1_GITEA_SECRET_KEY'],
-            "gitea_internal_token": os.environ['POD_CHARLESREID1_GITEA_INTERNAL_TOKEN'],
-            # aws:
-            "aws_backup_s3_bucket": os.environ['POD_CHARLESREID1_BACKUP_S3BUCKET'],
-            "aws_access_key": os.environ['POD_CHARLESREID1_AWS_ACCESS_KEY'],
-            "aws_access_secret": os.environ['POD_CHARLESREID1_AWS_ACCESS_SECRET'],
-            "backup_canary_webhook_url": os.environ['POD_CHARLESREID1_BACKUPCANARY_WEBHOOKURL'],
-        })
+
+        jinja_vars = {}
+        for k, v in jinja_to_env.items():
+            jinja_vars[k] = os.environ[v]
+
+        content = env.get_template(tname).render(jinja_vars)
     
         # Write to file
         if os.path.exists(rpath) and not OVERWRITE:
-            raise Exception("Error: file %s already exists!"%(rpath))
+            msg = "\n[!!!] Warning: file %s already exists! Skipping...\n"%(rpath)
+            print(msg)
+            time.sleep(1)
         else:
             with open(rpath,'w') as f:
                 f.write(content)
             print(f"    Done!")
             print("")
 
+        if rpath[-3:] == ".sh":
+            subprocess.call(['chmod', '+x', rpath])
+    
 if __name__=="__main__":
     main()

scripts/backups/10-pod-charlesreid1-rsyslog.conf

@@ -0,0 +1,28 @@
+if ( $programname startswith "pod-charlesreid1-canary" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-canary.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-certbot" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-certbot.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-aws" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-aws.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-cleanolderthan" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-cleanolderthan.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-gitea" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-gitea.service.log" flushOnTXEnd="off")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-wikidb" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-wikidb.service.log" flushOnTXEnd="on")
+    stop
+}
+if ( $programname startswith "pod-charlesreid1-backups-wikifiles" ) then {
+    action(type="omfile" file="/var/log/pod-charlesreid1-backups-wikifiles.service.log" flushOnTXEnd="on")
+    stop
+}

scripts/backups/Readme.md

@@ -0,0 +1,52 @@
+# backup scripts
+
+This directory contains several files for several services:
+
+* Systemd .service file (Jinja template) to define a service that backs up files
+* Systemd .timer file (Jinja template) to define a timer that runs the service on a schedule
+* Shell script .sh that actually performs the backup operation and is called by the .service file
+
+Use `make templates` in the top level of this repo to render
+the Jinja templates using the environment variables in the
+evnrionment file. That fixes the locations of the scripts
+for the systemd service.
+
+Use `make install` in the top level of this repo to install
+the rendered service and timer files.
+
+## syslog filtering
+
+Due to a bug in systemd bundled with Ubuntu 18.04, we can't just use the nice easy solution of
+directing output and error to a specific file.
+
+Instead, the services all send their stderr and stdout to the system log, and then rsyslog
+filters those messages and collects them into a separate log file.
+
+First, install the services.
+
+Then, install the following rsyslog config file:
+
+`/etc/rsyslog.d/10-pod-charlesreid1-rsyslog.conf`:
+
+```
+if $programname == 'pod-charlesreid1-canary' then /var/log/pod-charlesreid1-canary.service.log
+if $programname == 'pod-charlesreid1-canary' then stop
+
+if $programname == 'pod-charlesreid1-backups-aws' then /var/log/pod-charlesreid1-backups-aws.service.log
+if $programname == 'pod-charlesreid1-backups-aws' then stop
+
+if $programname == 'pod-charlesreid1-backups-cleanolderthan' then /var/log/pod-charlesreid1-backups-cleanolderthan.service.log
+if $programname == 'pod-charlesreid1-backups-cleanolderthan' then stop
+
+if $programname == 'pod-charlesreid1-backups-gitea' then /var/log/pod-charlesreid1-backups-gitea.service.log
+if $programname == 'pod-charlesreid1-backups-gitea' then stop
+
+if $programname == 'pod-charlesreid1-backups-wikidb' then /var/log/pod-charlesreid1-backups-wikidb.service.log
+if $programname == 'pod-charlesreid1-backups-wikidb' then stop
+
+if $programname == 'pod-charlesreid1-backups-wikifiles' then /var/log/pod-charlesreid1-backups-wikifiles.service.log
+if $programname == 'pod-charlesreid1-backups-wikifiles' then stop
+```
+
+
+

scripts/backups/aws_backup.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+#
+# Find the last backup created, and copy it
+# to an S3 bucket.
+set -eux
+
+function usage {
+    set +x
+    echo ""
+    echo "aws_backup.sh script:"
+    echo ""
+    echo "Find the last backup that was created,"
+    echo "and copy it to the backups bucket."
+    echo ""
+    echo "       ./aws_backup.sh"
+    echo ""
+    exit 1;
+}
+
+if [ "$(id -u)" == "0" ]; then
+    echo ""
+    echo ""
+    echo "This script should NOT be run as root!"
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+if [ "$#" == "0" ]; then
+
+    echo ""
+    echo "pod-charlesreid1: aws_backup.sh"
+    echo "-----------------------------------"
+    echo ""
+    echo "Backup directory: ${POD_CHARLESREID1_BACKUP_DIR}"
+    echo "Backup bucket: ${POD_CHARLESREID1_BACKUP_S3BUCKET}"
+    echo ""
+
+    echo "Checking that directory exists"
+    /usr/bin/test -d ${POD_CHARLESREID1_BACKUP_DIR}
+
+    echo "Checking that we can access the S3 bucket"
+    aws s3 ls s3://${POD_CHARLESREID1_BACKUP_S3BUCKET} > /dev/null
+    
+    # Get name of last backup, to copy to AWS
+    LAST_BACKUP=$(/bin/ls -1 -t ${POD_CHARLESREID1_BACKUP_DIR} | /usr/bin/head -n1)
+    echo "Last backup found: ${LAST_BACKUP}"
+    echo "Last backup directory: ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP}"
+
+    BACKUP_SIZE=$(/usr/bin/du -hs ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP} | cut -f 1)
+    echo "Backup directory size: ${BACKUP_SIZE}"
+
+    # Copy to AWS
+    echo "Backing up directory ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP}"
+    aws s3 cp --only-show-errors --no-progress --recursive ${POD_CHARLESREID1_BACKUP_DIR}/${LAST_BACKUP} s3://${POD_CHARLESREID1_BACKUP_S3BUCKET}/backups/${LAST_BACKUP}
+    echo "Done."
+
+else
+    usage
+fi

scripts/backups/canary/backups_canary.py

@@ -0,0 +1,104 @@
+import os
+import sys
+import json
+import requests
+import boto3
+import botocore
+import subprocess
+
+
+webhook_url = os.environ['POD_CHARLESREID1_CANARY_WEBHOOK']
+backup_dir = os.environ['POD_CHARLESREID1_BACKUP_DIR']
+backup_bucket = os.environ['POD_CHARLESREID1_BACKUP_S3BUCKET']
+
+
+# Check for backups created in the last N days
+N = 7
+
+
+def main():
+    # verify the backups directory exists
+    if not os.path.exists(backup_dir):
+        msg = "Local Backups Error:\n"
+        msg += f"The backup directory `{backup_dir}` does not exist!"
+        alert(msg)
+
+    # verify there is a backup newer than N days
+    newer_backups = subprocess.getoutput(f'find {backup_dir}/* -mtime -{N}').split('\n')
+    if len(newer_backups)==1 and newer_backups[0]=='':
+        msg = "Local Backups Error:\n"
+        msg += f"The backup directory `{backup_dir}` is missing backup files from the last {N} day(s)!"
+        alert(msg)
+
+    newest_backup_name = subprocess.getoutput(f'ls -t {backup_dir} | head -n1')
+    newest_backup_path = os.path.join(backup_dir, newest_backup_name)
+    newest_backup_files = subprocess.getoutput(f'find {newest_backup_path} -type f').split('\n')
+
+    # verify the most recent backup directory is not empty
+    if len(newest_backup_files)==1 and newer_backups[0]=='':
+        msg = "Local Backups Error:\n"
+        msg += f"The most recent backup directory `{newest_backup_path}` is empty!"
+        alert(msg)
+
+    # verify the most recent backup files have nonzero size
+    for backup_file in newest_backup_files:
+        if os.path.getsize(backup_file)==0:
+            msg = "Local Backups Error:\n"
+            msg += f"The most recent backup directory `{newest_backup_path}` contains an empty backup file!\n"
+            msg += f"Backup file name: {backup_file}!"
+            alert(msg)
+
+    # verify the most recent backup files exist in the s3 backups bucket
+    bucket_base_path = os.path.join('backups', newest_backup_name)
+    for backup_file in newest_backup_files:
+        backup_name = os.path.basename(backup_file)
+        backup_bucket_path = os.path.join(bucket_base_path, backup_name)
+        check_exists(backup_bucket, backup_bucket_path)
+
+def check_exists(bucket_name, bucket_path):
+    s3 = boto3.resource('s3')
+    try:
+        s3.Object(bucket_name, bucket_path).load()
+    except botocore.exceptions.ClientError as e:
+        if e.response['Error']['Code'] == "404":
+            # File does not exist
+            msg = "S3 Backups Error:\n"
+            msg += f"Failed to find the file `{bucket_path}` in bucket `{bucket_name}`"
+        else:
+            # Problem accessing backups on bucket
+            msg = "S3 Backups Error:\n"
+            msg += f"Failed to access the file `{bucket_path}` in bucket `{bucket_name}`"
+
+
+def alert(msg):
+    title = ":bangbang: pod-charlesreid1 backups canary"
+    hostname = subprocess.getoutput('hostname')
+    msg += f"\n\nHost: {hostname}"
+    slack_data = {
+        "username": "backups_canary",
+        "channel" : "#alerts",
+        "attachments": [
+            {
+                "color": "#CC0000",
+                "fields": [
+                    {
+                        "title": title,
+                        "value": msg,
+                        "short": "false",
+                    }
+                ]
+            }
+        ]
+    }
+    byte_length = str(sys.getsizeof(slack_data))
+    headers = {'Content-Type': "application/json", 'Content-Length': byte_length}
+    response = requests.post(webhook_url, data=json.dumps(slack_data), headers=headers)
+    if response.status_code != 200:
+        raise Exception(response.status_code, response.text)
+
+    print("Goodbye.")
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

scripts/backups/canary/pod-charlesreid1-canary.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Backup canary service for pod-charlesreid1
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+StandardError=syslog
+StandardOutput=syslog
+SyslogIdentifier=pod-charlesreid1-canary
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; /home/charles/.pyenv/shims/python3 {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/canary/backups_canary.py'
+User=charles
+Group=charles

scripts/backups/canary/pod-charlesreid1-canary.timer.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=Timer to run the pod-charlesreid1 backups canary
+
+[Timer]
+OnCalendar=*-*-* 7:01:00
+
+[Install]
+WantedBy=timers.target

scripts/backups/canary/requirements.txt

@@ -0,0 +1,3 @@
+boto3
+botocore
+requests

scripts/backups/clean_olderthan.sh

@@ -2,13 +2,24 @@
 #
 # Clean any files older than N days
 # from the backup directory.
-set -eu
+set -eux
 
 # Number of days of backups to retain.
 # Everything older than this many days will be deleted
-N="30"
+N="22"
 
-BACKUP_DIR="$HOME/backups"
+function usage {
+    set +x
+    echo ""
+    echo "clean_olderthan.sh script:"
+    echo ""
+    echo "Clean files older than ${N} days from the"
+    echo "backups directory, ~/backups"
+    echo ""
+    echo "       ./clean_olderthan.sh"
+    echo ""
+    exit 1;
+}
 
 if [ "$(id -u)" == "0" ]; then
     echo ""
@@ -21,8 +32,21 @@ fi
 
 if [ "$#" == "0" ]; then
 
-    echo "Cleaning backups directory $BACKUP_DIR"
-    echo "Files older than $N days will be deleted"
-    find $BACKUP_DIR -mtime +${N} -delete
+    echo ""
+    echo "pod-charlesreid1: clean_olderthan.sh"
+    echo "------------------------------------"
+    echo ""
+    echo "Backup directory: ${POD_CHARLESREID1_BACKUP_DIR}"
+    echo ""
 
+    echo "Cleaning backups directory $POD_CHARLESREID1_BACKUP_DIR"
+    echo "The following files older than $N days will be deleted:"
+    find ${POD_CHARLESREID1_BACKUP_DIR} -mtime +${N}
+
+    echo "Deleting files"
+    find ${POD_CHARLESREID1_BACKUP_DIR} -mtime +${N} -delete
+    echo "Done"
+
+else
+    usage
 fi

scripts/backups/gitea_backup.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# Bcak up the Gitea custom/ and data/ directories.
+# These are needed to restore the site
+# (as well as repository data, which is not backed up
+# by this script, it is a separate drive).
+set -eux
+
+CONTAINER_NAME="stormy_gitea"
+STAMP="`date +"%Y%m%d"`"
+
+
+function usage {
+    set +x
+    echo ""
+    echo "gitea_backup.sh script:"
+    echo ""
+    echo "Create a tar file containing gitea"
+    echo "custom/ and data/ directories."
+    echo ""
+    echo "       ./gitea_backup.sh"
+    echo ""
+    exit 1;
+}
+
+if [ "$(id -u)" == "0" ]; then
+    echo ""
+    echo ""
+    echo "This script should NOT be run as root!"
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+if [ "$#" == "0" ]; then
+
+    CUSTOM_NAME="gitea_custom_${STAMP}.tar.gz"
+    DATA_NAME="gitea_data_${STAMP}.tar.gz"
+
+    CUSTOM_TARGET="${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}/${CUSTOM_NAME}"
+    DATA_TARGET="${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}/${DATA_NAME}"
+
+    echo ""
+    echo "pod-charlesreid1: gitea_backup.sh"
+    echo "-----------------------------------"
+    echo ""
+    echo "Backup target: custom: ${CUSTOM_TARGET}"
+    echo "Backup target: data: ${DATA_TARGET}"
+    echo ""
+
+    mkdir -p ${POD_CHARLESREID1_BACKUP_DIR}/${STAMP}
+
+    # We don't need to use docker, since these directories
+    # are both bind-mounted into the Docker container
+    echo "Backing up custom directory"
+    tar --exclude='gitea.log' --ignore-failed-read -czf ${CUSTOM_TARGET} ${POD_CHARLESREID1_DIR}/d-gitea/custom
+    echo "Backing up data directory"
+    tar czf ${DATA_TARGET} ${POD_CHARLESREID1_DIR}/d-gitea/data
+
+    echo "Done."
+
+else
+    usage
+fi

scripts/backups/gitea_dump.sh

@@ -1,86 +0,0 @@
-#!/bin/bash
-#
-# Run the gitea dump command and send the dump file
-# to the specified backup directory.
-#
-# Backup directory:
-#       /home/user/backups/gitea
-
-BACKUP_DIR="$HOME/backups/gitea"
-CONTAINER_NAME="stormy_gitea"
-
-function usage {
-    set +x
-    echo ""
-    echo "gitea_dump.sh script:"
-    echo ""
-    echo "Run the gitea dump command inside the gitea docker container,"
-    echo "and copy the resulting zip file to the specified directory."
-    echo "The resulting gitea dump zip file will be timestamped."
-    echo ""
-    echo "       ./gitea_dump.sh"
-    echo ""
-    echo "Example:"
-    echo ""
-    echo "       ./gitea_dump.sh"
-    echo "       (creates ${BACKUP_DIR}/gitea-dump_20200101_000000.zip)"
-    echo ""
-    exit 1;
-}
-
-if [ "$(id -u)" == "0" ]; then
-    echo ""
-    echo ""
-    echo "This script should NOT be run as root!"
-    echo ""
-    echo ""
-    exit 1;
-fi
-
-if [ "$#" == "0" ]; then
-
-    STAMP="`date +"%Y-%m-%d"`"
-    TARGET="gitea-dump_${STAMP}.zip"
-
-    echo ""
-    echo "pod-charlesreid1: gitea_dump.sh"
-    echo "-------------------------------"
-    echo ""
-    echo "Backup target: ${BACKUP_DIR}/${TARGET}"
-    echo ""
-
-    mkdir -p $BACKUP_DIR
-
-    ## If this script is being run from a cron job,
-    ## don't use -i flag with docker
-    #CRON="$( pstree -s $$ | /bin/grep -c cron )"
-    #DOCKER="/usr/local/bin/docker"
-    #DOCKERX=""
-    #if [[ "$CRON" -eq 1 ]]; 
-    #then
-    #    DOCKERX="${DOCKER} exec -t"
-    #else
-    #    DOCKERX="${DOCKER} exec -it"
-    #fi
-    DOCKER="/usr/local/bin/docker"
-    DOCKERX="${DOCKER} exec -t"
-
-    echo "Step 1: Run gitea dump command inside docker machine"
-    set -x
-    ${DOCKERX} --user git ${CONTAINER_NAME} /bin/bash -c 'cd /app/gitea && /app/gitea/gitea dump --file gitea-dump.zip --skip-repository'
-    set +x
-
-    echo "Step 2: Copy gitea dump file out of docker machine"
-    set -x
-    ${DOCKER} cp ${CONTAINER_NAME}:/app/gitea/gitea-dump.zip ${BACKUP_DIR}/${TARGET}
-    set +x
-
-    echo "Step 3: Clean up gitea dump file"
-    set -x
-    ${DOCKERX} ${CONTAINER_NAME} /bin/bash -c "rm -f /app/gitea/gitea-dump.zip"
-    set +x
-
-    echo "Done."
-else
-    usage
-fi

scripts/backups/pod-charlesreid1-backups-aws.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Copy the latest pod-charlesreid1 backup to an S3 bucket
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+StandardError=syslog
+StandardOutput=syslog
+SyslogIdentifier=pod-charlesreid1-backups-aws
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/aws_backup.sh'
+User=charles
+Group=charles

scripts/backups/pod-charlesreid1-backups-aws.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Timer to copy the lastest pod-charlesreid1 backup to an S3 bucket
+
+[Timer]
+OnCalendar=Sun *-*-* 2:56:00
+#OnCalendar=*-*-* 2:56:00
+
+[Install]
+WantedBy=timers.target

scripts/backups/pod-charlesreid1-backups-cleanolderthan.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Clean pod-charlesreid1 backups older than N days
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+StandardError=syslog
+StandardOutput=syslog
+SyslogIdentifier=pod-charlesreid1-backups-cleanolderthan
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/clean_olderthan.sh'
+User=charles
+Group=charles

scripts/backups/pod-charlesreid1-backups-cleanolderthan.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Timer to clean files older than N days from the pod-charlesreid1 backups dir
+
+[Timer]
+OnCalendar=Sun *-*-* 2:28:00
+#OnCalendar=*-*-* 2:28:00
+
+[Install]
+WantedBy=timers.target

scripts/backups/pod-charlesreid1-backups-gitea.service.j2

@@ -5,8 +5,10 @@ After=docker.service
 
 [Service]
 Type=oneshot
-StandardError={{ pod_install_dir }}/.pod-charlesreid1-backups-gitea.service.error.log
-StandardOutput={{ pod_install_dir }}/.pod-charlesreid1-backups-gitea.service.output.log
-ExecStart={{ pod_install_dir }}/scripts/backups/gitea_dump.sh
+StandardError=syslog
+StandardOutput=syslog
+SyslogIdentifier=pod-charlesreid1-backups-gitea
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/gitea_backup.sh'
 User=charles
 Group=charles

scripts/backups/pod-charlesreid1-backups-gitea.timer.j2

@@ -2,8 +2,8 @@
 Description=Timer to back up pod-charlesreid1 gitea files
 
 [Timer]
-OnCalendar=*-*-* 0/2:23:00
+OnCalendar=Sun *-*-* 2:12:00
+#OnCalendar=*-*-* 2:12:00
 
 [Install]
 WantedBy=timers.target
-

scripts/backups/pod-charlesreid1-backups-wikidb.service.j2

@@ -5,9 +5,10 @@ After=docker.service
 
 [Service]
 Type=oneshot
-StandardError={{ pod_install_dir }}/.pod-charlesreid1-backups-wikidb.service.error.log
-StandardOutput={{ pod_install_dir }}/.pod-charlesreid1-backups-wikidb.service.output.log
-ExecStart={{ pod_install_dir }}/scripts/backups/wikidb_dump.sh
+StandardError=syslog
+StandardOutput=syslog
+SyslogIdentifier=pod-charlesreid1-backups-wikidb
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/wikidb_dump.sh'
 User=charles
 Group=charles
-

scripts/backups/pod-charlesreid1-backups-wikidb.timer.j2

@@ -2,7 +2,7 @@
 Description=Timer to back up the pod-charlesreid1 wiki database
 
 [Timer]
-OnCalendar=*-*-* 0/2:03:00
+OnCalendar=Sun *-*-* 2:02:00
 
 [Install]
 WantedBy=timers.target

scripts/backups/pod-charlesreid1-backups-wikifiles.service.j2

@@ -1,12 +1,14 @@
 [Unit]
-Description=Back up the pod-charlesreid1 wiki database
+Description=Back up pod-charlesreid1 wiki files
 Requires=docker.service
 After=docker.service
 
 [Service]
 Type=oneshot
-StandardError={{ pod_install_dir }}/.pod-charlesreid1-backups-wikifiles.service.error.log
-StandardOutput={{ pod_install_dir }}/.pod-charlesreid1-backups-wikifiles.service.output.log
-ExecStart={{ pod_install_dir }}/scripts/backups/wikifiles_dump.sh
+StandardError=syslog
+StandardOutput=syslog
+SyslogIdentifier=pod-charlesreid1-backups-wikifiles
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/backups/wikifiles_dump.sh'
 User=charles
 Group=charles

scripts/backups/pod-charlesreid1-backups-wikifiles.timer.j2

@@ -1,9 +1,8 @@
 [Unit]
-Description=Timer to back up the pod-charlesreid1 wiki database
+Description=Timer to back up pod-charlesreid1 wiki files
 
 [Timer]
-OnCalendar=*-*-* 0/2:13:00
+OnCalendar=Sun *-*-* 2:08:00
 
 [Install]
 WantedBy=timers.target
-

scripts/backups/wikidb_dump.sh

@@ -2,13 +2,11 @@
 #
 # Run the mysql dump command to back up wikidb table, and send the
 # resulting SQL file to the specified backup directory.
-#
-# Backup directory:
-#       /home/user/backups/mysql
+set -eux
 
-BACKUP_DIR="$HOME/backups"
 CONTAINER_NAME="stormy_mysql"
-STAMP="`date +"%Y%m%d"`"
+DATESTAMP="`date +"%Y%m%d"`"
+TIMESTAMP="`date +"%Y%m%d_%H%M%S"`"
 
 function usage {
     set +x
@@ -23,7 +21,7 @@ function usage {
     echo "Example:"
     echo ""
     echo "       ./wikidb_dump.sh"
-    echo "       (creates ${BACKUP_DIR}/20200101/wikidb_20200101.sql)"
+    echo "       (creates ${POD_CHARLESREID1_BACKUP_DIR}/YYYYMMDD/wikidb_YYYYMMDD_HHMMSS.sql)"
     echo ""
     exit 1;
 }
@@ -39,36 +37,35 @@ fi
 
 if [ "$#" == "0" ]; then
 
-    TARGET="wikidb_${STAMP}.sql"
-    BACKUP_TARGET="${BACKUP_DIR}/${STAMP}/${TARGET}"
+    TARGET="wikidb_${TIMESTAMP}.sql"
+    BACKUP_DIR="${POD_CHARLESREID1_BACKUP_DIR}/${DATESTAMP}"
+    BACKUP_TARGET="${BACKUP_DIR}/${TARGET}"
 
     echo ""
     echo "pod-charlesreid1: wikidb_dump.sh"
     echo "--------------------------------"
     echo ""
+    echo "Backup directory: ${BACKUP_DIR}"
     echo "Backup target: ${BACKUP_TARGET}"
     echo ""
 
-    mkdir -p ${BACKUP_DIR}/${STAMP}
+    mkdir -p ${BACKUP_DIR}
 
-    # If this script is being run from a cron job,
-    # don't use -i flag with docker
-    CRON="$( pstree -s $$ | /bin/grep -c cron )"
     DOCKER=$(which docker)
-    DOCKERX=""
-    if [[ "$CRON" -eq 1 ]]; 
-    then
     DOCKERX="${DOCKER} exec -t"
-    else
-        DOCKERX="${DOCKER} exec -it"
-    fi
 
-    echo "Running mysqldump"
-    set -x
-    ${DOCKERX} ${CONTAINER_NAME} sh -c 'exec mysqldump wikidb --databases -uroot -p"$MYSQL_ROOT_PASSWORD"' > ${BACKUP_TARGET}
-    set +x
+    echo "Running mysqldump inside the mysql container"
 
+    # this works, except the first line is a stupid warning about passwords
+    ${DOCKERX} ${CONTAINER_NAME} sh -c 'exec mysqldump wikidb --databases -uroot -p"$MYSQL_ROOT_PASSWORD" --default-character-set=binary' > ${BACKUP_TARGET}
+
+    # trim stupid first line warning
+    tail -n +2 ${BACKUP_TARGET} > ${BACKUP_TARGET}.tmp
+    mv ${BACKUP_TARGET}.tmp ${BACKUP_TARGET}
+
+    echo "Successfully wrote SQL dump to file: ${BACKUP_TARGET}"
     echo "Done."
+
 else
     usage
 fi

scripts/backups/wikifiles_dump.sh

@@ -2,13 +2,11 @@
 #
 # Create a tar file containing wiki files
 # from the mediawiki docker container.
-#
-# Backup directory:
-#       /home/user/backups/mediawiki
+set -eux
 
-BACKUP_DIR="$HOME/backups"
 CONTAINER_NAME="stormy_mw"
-STAMP="`date +"%Y%m%d"`"
+DATESTAMP="`date +"%Y%m%d"`"
+TIMESTAMP="`date +"%Y%m%d_%H%M%S"`"
 
 function usage {
     set +x
@@ -23,7 +21,7 @@ function usage {
     echo "Example:"
     echo ""
     echo "       ./wikifiles_dump.sh"
-    echo "       (creates ${BACKUP_DIR}/20200101/wikifiles_20200101.tar.gz)"
+    echo "       (creates ${POD_CHARLESREID1_BACKUP_DIR}/YYYYMMDD/wikifiles_YYYYMMDD_HHMMSS.tar.gz)"
     echo ""
     exit 1;
 }
@@ -39,48 +37,36 @@ fi
 
 if [ "$#" == "0" ]; then
 
-    TARGET="wikifiles_${STAMP}.tar.gz"
-    BACKUP_TARGET="${BACKUP_DIR}/${STAMP}/${TARGET}"
+    TARGET="wikifiles_${TIMESTAMP}.tar.gz"
+    BACKUP_DIR="${POD_CHARLESREID1_BACKUP_DIR}/${DATESTAMP}"
+    BACKUP_TARGET="${BACKUP_DIR}/${TARGET}"
 
     echo ""
     echo "pod-charlesreid1: wikifiles_dump.sh"
     echo "-----------------------------------"
     echo ""
+    echo "Backup directory: ${BACKUP_DIR}"
     echo "Backup target: ${BACKUP_TARGET}"
     echo ""
 
-    mkdir -p ${BACKUP_DIR}/${STAMP}
+    mkdir -p ${BACKUP_DIR}
 
-    # If this script is being run from a cron job,
-    # don't use -i flag with docker
-    CRON="$( pstree -s $$ | /bin/grep -c cron )"
     DOCKER=$(which docker)
-    DOCKERX=""
-    if [[ "$CRON" -eq 1 ]]; 
-    then
     DOCKERX="${DOCKER} exec -t"
-    else
-        DOCKERX="${DOCKER} exec -it"
-    fi
 
     echo "Step 1: Compress wiki files inside container"
-    set -x
     ${DOCKERX} ${CONTAINER_NAME} /bin/tar czf /tmp/${TARGET} /var/www/html/images
-    set +x
 
     echo "Step 2: Copy tar.gz file out of container"
-    mkdir -p $(dirname "$1")
-    set -x
+    mkdir -p $(dirname "${BACKUP_TARGET}")
     ${DOCKER} cp ${CONTAINER_NAME}:/tmp/${TARGET} ${BACKUP_TARGET}
-    set +x
 
     echo "Step 3: Clean up tar.gz file"
-    set -x
     ${DOCKERX} ${CONTAINER_NAME} /bin/rm -f /tmp/${TARGET}
-    set +x
 
+    echo "Successfully wrote wikifiles dump to file: ${BACKUP_TARGET}"
     echo "Done."
+
 else
     usage
 fi
-

scripts/backups/wikifiles_restore.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Restore wiki files from a tar file
+# into the stormy_mw container.
+set -eu
+
+function usage {
+    echo ""
+    echo "restore_wikifiles.sh script:"
+    echo "Restore wiki files from a tar file"
+    echo "into the stormy_mw container"
+    echo ""
+    echo "       ./restore_wikifiles.sh <tar-file>"
+    echo ""
+    echo "Example:"
+    echo ""
+    echo "       ./restore_wikifiles.sh /path/to/wikifiles.tar.gz"
+    echo ""
+    echo ""
+    exit 1;
+}
+
+# NOTE:
+# I assume images/ is the only directory to back up/restore.
+# If there are more I forgot, add them back in here.
+# (skins and extensions are static, added into image at build time.)
+
+if [[ "$#" -eq 1 ]];
+then
+
+    NAME="stormy_mw"
+    TAR=$(basename "$1")
+
+    echo "Checking that container ${NAME} exists"
+    docker ps --format '{{.Names}}' | grep ${NAME} || exit 1;
+
+    echo "Copying dir $1 into container ${NAME}"
+    set -x
+    docker cp $1 ${NAME}:/tmp/${TAR}
+    docker exec -it ${NAME} rm -rf /var/www/html/images.old
+    docker exec -it ${NAME} mv /var/www/html/images /var/www/html/images.old
+    docker exec -it ${NAME} tar -xf /tmp/${TAR} -C / && rm -f /tmp/${TAR}
+    docker exec -it ${NAME} chown -R www-data:www-data /var/www/html/images
+
+else
+    usage
+fi

scripts/certbot/pod-charlesreid1-certbot.service.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=Renew certificates for pod-charlesreid1
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+StandardError=syslog
+StandardOutput=syslog
+SyslogIdentifier=pod-charlesreid1-certbot
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/environment
+ExecStart=/bin/bash -ac '. {{ pod_charlesreid1_pod_install_dir }}/environment; {{ pod_charlesreid1_pod_install_dir }}/scripts/certbot/renew_charlesreid1_certs.sh'

scripts/certbot/pod-charlesreid1-certbot.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Timer to renew certificates for pod-charlesreid1
+
+[Timer]
+# Run on the first Sunday of every month
+OnCalendar=Sun *-*-01..07 4:03:00
+
+[Install]
+WantedBy=timers.target

scripts/certbot/renew_charlesreid1_certs.sh.j2

@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# renew/run certbot on krash
+set -eux
+
+SERVICE="pod-charlesreid1"
+
+function usage {
+    set +x
+    echo ""
+    echo "renew_charlesreid1_certs.sh script:"
+    echo ""
+    echo "Renew all certs used in the charlesreid1.com pod"
+    echo ""
+    echo "       ./renew_charlesreid1_certs.sh"
+    echo ""
+    exit 1;
+}
+
+if [ "$(id -u)" != "0" ]; then
+    echo ""
+    echo ""
+    echo "This script should be run as root."
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+if [ "$#" == "0" ]; then
+
+    # disable system service that will re-spawn docker pod
+    echo "Disable and stop system service ${SERVICE}"
+    sudo systemctl disable ${SERVICE}
+    sudo systemctl stop ${SERVICE}
+    
+    echo "Stop pod"
+    docker-compose -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml down
+    
+    echo "Run certbot renew"
+    SUBS="git www"
+    DOMS="charlesreid1.com"
+
+    # top level domains
+    for DOM in $DOMS; do
+        certbot certonly \
+            --standalone \
+            --non-interactive \
+            --agree-tos \
+            --email charles@charlesreid1.com \
+            -d ${DOM}
+    done
+
+    # subdomains
+    for SUB in $SUBS; do
+        for DOM in $DOMS; do
+            certbot certonly \
+                --standalone \
+                --non-interactive \
+                --agree-tos \
+                --email charles@charlesreid1.com \
+                -d ${SUB}.${DOM}
+        done
+    done
+    
+    echo "Start pod"
+    docker-compose -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml up -d
+    
+    echo "Enable and start system service ${SERVICE}"
+    sudo systemctl enable ${SERVICE}
+    sudo systemctl start ${SERVICE}
+
+    echo "Done"
+
+else
+    usage
+fi

scripts/clean_templates.py

@@ -13,7 +13,9 @@ def clean():
         rname = tname[:-3]
         rpath = os.path.join(tdir, rname)
 
-        if os.path.exists(rpath):
+        ignore_list = ['environment']
+
+        if os.path.exists(rpath) and rname not in ignore_list:
             print(f"Removing file {rpath}")
             os.remove(rpath)
         else:

scripts/git_clone_www.py.j2

@@ -11,8 +11,8 @@ directory structure for charlesreid1.com
 content. (Or, charlesreid1.XYZ, whatever.)
 """
 
-SERVER_NAME_DEFAULT = '{{ server_name_default }}'
-USERNAME = '{{ username }}'
+SERVER_NAME_DEFAULT = '{{ pod_charlesreid1_server_name }}'
+USERNAME = '{{ pod_charlesreid1_username }}'
 
 
 

scripts/git_pull_www.py.j2

@@ -10,8 +10,8 @@ This script git pulls the /www directory
 for updating charlesreid1.com content.
 """
 
-SERVER_NAME_DEFAULT = '{{ server_name_default }}'
-USERNAME = '{{ username }}'
+SERVER_NAME_DEFAULT = '{{ pod_charlesreid1_server_name }}'
+USERNAME = '{{ pod_charlesreid1_username }}'
 
 
 

scripts/mw/build_extensions_dir.sh

@@ -4,14 +4,13 @@
 set -eux
 
 MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
-CONF_DIR="${MW_DIR}/charlesreid1-config"
-MW_CONF_DIR="${MW_CONF_DIR}/mediawiki"
+MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
 EXT_DIR="${MW_CONF_DIR}/extensions"
 
-mkdir -p ${EXT_DIR}/extensions
+mkdir -p ${EXT_DIR}
 
 (
-cd ${EXT_DIR}/extensions
+cd ${EXT_DIR}
 
 ##############################
 
@@ -81,19 +80,5 @@ fi
 
 ##############################
 
-Extension="Fail2banlog"
-if [ ! -d ${Extension} ]
-then
-    git clone https://github.com/charlesreid1-docker/mw-fail2ban.git ${Extension}
-    (
-    cd ${Extension}
-    git checkout master
-    )
-else
-    echo "Skipping ${Extension}"
-fi
-
-##############################
-
 # fin
 )

scripts/mw/fix_LocalSettings.sh

@@ -1,20 +1,12 @@
 #!/bin/bash
 # 
 # fix LocalSettings.php in the mediawiki container.
-# 
-# docker is stupid, so it doesn't let you bind mount
-# a single file into a docker volume.
-# 
-# so, rather than rebuilding the entire goddamn container
-# just to update LocalSettings.php when it changes, we just
-# use a docker cp command to copy it into the container.
 set -eux
 
 NAME="stormy_mw"
 
 MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
-CONF_DIR="${MW_DIR}/charlesreid1-config"
-MW_CONF_DIR="${MW_CONF_DIR}/mediawiki"
+MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
 
 echo "Checking that container exists"
 docker ps --format '{{.Names}}' | grep ${NAME} || exit 1;

scripts/mw/fix_extensions_dir.sh

@@ -1,12 +1,6 @@
 #!/bin/bash
 # 
 # fix extensions dir in the mediawiki container
-#
-# in theory, we should be able to update the
-# extensions folder in d-mediawiki/charlesreid1-config,
-# but in reality this falls on its face.
-# So, we have to fix the fucking extensions directory
-# ourselves.
 set -eux
 
 NAME="stormy_mw"
@@ -14,8 +8,7 @@ NAME="stormy_mw"
 EXTENSIONS="SyntaxHighlight_GeSHi ParserFunctions EmbedVideo Math Fail2banlog"
 
 MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
-CONF_DIR="${MW_DIR}/charlesreid1-config"
-MW_CONF_DIR="${MW_CONF_DIR}/mediawiki"
+MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
 EXT_DIR="${MW_CONF_DIR}/extensions"
 
 echo "Checking that container exists..."

scripts/mw/fix_skins.sh

@@ -1,20 +1,12 @@
 #!/bin/bash
 # 
 # fix skins in the mediawiki container.
-# 
-# docker is stupid, so it doesn't let you bind mount
-# a single file into a docker volume.
-#
-# so, rather than rebuilding the entire goddamn container
-# just to update the skin when it changes, we just
-# use a docker cp command to copy it into the container.
 set -eux
 
 NAME="stormy_mw"
 
 MW_DIR="${POD_CHARLESREID1_DIR}/d-mediawiki"
-CONF_DIR="${MW_DIR}/charlesreid1-config"
-MW_CONF_DIR="${MW_CONF_DIR}/mediawiki"
+MW_CONF_DIR="${MW_DIR}/charlesreid1-config/mediawiki"
 SKINS_DIR="${MW_CONF_DIR}/skins"
 
 echo "Checking that container exists"

scripts/mw/restore_wikifiles.sh

@@ -2,7 +2,7 @@
 #
 # Restore wiki files from a tar file
 # into the stormy_mw container.
-set -eux
+set -eu
 
 function usage {
     echo ""
@@ -31,16 +31,16 @@ then
     NAME="stormy_mw"
     TAR=$(basename "$1")
 
-	echo "Checking that container exists"
+	echo "Checking that container ${NAME} exists"
 	docker ps --format '{{.Names}}' | grep ${NAME} || exit 1;
 
-	echo "Copying $1 into container ${NAME}"
+	echo "Copying dir $1 into container ${NAME}"
     set -x
     docker cp $1 ${NAME}:/tmp/${TAR}
+    docker exec -it ${NAME} rm -rf /var/www/html/images.old
     docker exec -it ${NAME} mv /var/www/html/images /var/www/html/images.old
     docker exec -it ${NAME} tar -xf /tmp/${TAR} -C / && rm -f /tmp/${TAR}
     docker exec -it ${NAME} chown -R www-data:www-data /var/www/html/images
-    set +x
 
 else
     usage

scripts/mysql/dump_database.sh

@@ -1,35 +1,36 @@
 #!/bin/bash
+echo "this script is deprecated, see ../backups/wikidb_dump.sh"
+##
+## Dump a database to an .sql file
+## from the stormy_mysql container.
+#set -eu
 #
-# Dump a database to an .sql file
-# from the stormy_mysql container.
-set -x
-
-function usage {
-    echo ""
-    echo "dump_database.sh script:"
-    echo "Dump a database to an .sql file "
-    echo "from the stormy_mysql container."
-    echo ""
-    echo "       ./dump_database.sh <sql-dump-file>"
-    echo ""
-    echo "Example:"
-    echo ""
-    echo "       ./dump_database.sh /path/to/wikidb_dump.sql"
-    echo ""
-    echo ""
-    exit 1;
-}
-
-CONTAINER_NAME="stormy_mysql"
-
-if [[ "$#" -gt 0 ]];
-then
-
-    TARGET="$1"
-    mkdir -p $(dirname $TARGET)
-    docker exec -i ${CONTAINER_NAME} sh -c 'exec mysqldump wikidb --databases -uroot -p"$MYSQL_ROOT_PASSWORD"' > $TARGET
-
-else
-    usage
-fi
-
+#function usage {
+#    echo ""
+#    echo "dump_database.sh script:"
+#    echo "Dump a database to an .sql file "
+#    echo "from the stormy_mysql container."
+#    echo ""
+#    echo "       ./dump_database.sh <sql-dump-file>"
+#    echo ""
+#    echo "Example:"
+#    echo ""
+#    echo "       ./dump_database.sh /path/to/wikidb_dump.sql"
+#    echo ""
+#    echo ""
+#    exit 1;
+#}
+#
+#CONTAINER_NAME="stormy_mysql"
+#
+#if [[ "$#" -gt 0 ]];
+#then
+#
+#    TARGET="$1"
+#    mkdir -p $(dirname $TARGET)
+#	set -x
+#    docker exec -i ${CONTAINER_NAME} sh -c 'exec mysqldump wikidb --databases -uroot -p"$MYSQL_ROOT_PASSWORD"' > $TARGET
+#
+#else
+#    usage
+#fi

scripts/mysql/restore_database.sh

@@ -6,6 +6,7 @@
 # Note that this expects the .sql dump
 # to create its own databases.
 # Use the --databases flag with mysqldump.
+set -eu
 
 function usage {
     echo ""
@@ -42,31 +43,23 @@ function usage {
 # because of all these one-off 
 # "whoopsie we don't do that" problems.
 
+if [[ "$#" -eq 1 ]];
+then
+
 	CONTAINER_NAME="stormy_mysql"
 	TARGET=$(basename $1)
 	TARGET_DIR=$(dirname $1)
 
-
-if [[ "$#" -eq 1 ]];
-then
-
-    # Step 1: Copy the sql dump into the container
     set -x
+    # Step 1: Copy the sql dump into the container
     docker cp $1 ${CONTAINER_NAME}:/tmp/${TARGET}
-    set +x
 
     # Step 2: Run sqldump inside the container
-    set -x
     docker exec -i ${CONTAINER_NAME} sh -c "/usr/bin/mysql --defaults-file=/root/.mysql.rootpw.cnf < /tmp/${TARGET}"
-    set +x
 
     # Step 3: Clean up sql dump from inside container
-    set -x
-    docker exec -i ${CONTAINER_NAME} sh -c "/bin/rm -fr /tmp/${TARGET}.sql"
-    set +x
+    docker exec -i ${CONTAINER_NAME} sh -c "/bin/rm -fr /tmp/${TARGET}"
 
-
-    set +x
 else
     usage
 fi

scripts/pod-charlesreid1.service.j2

@@ -7,9 +7,9 @@ After=docker.service
 Restart=always
 StandardError=null
 StandardOutput=null
-ExecStartPre=test -f {{ pod_install_dir }}/docker-compose.yml
-ExecStart=/usr/local/bin/docker-compose -f {{ pod_install_dir }}/docker-compose.yml up
-ExecStop=/usr/local/bin/docker-compose  -f {{ pod_install_dir }}/docker-compose.yml stop
+ExecStartPre=/usr/bin/test -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml
+ExecStart=/usr/local/bin/docker-compose -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml up
+ExecStop=/usr/local/bin/docker-compose  -f {{ pod_charlesreid1_pod_install_dir }}/docker-compose.yml stop
 
 [Install]
 WantedBy=default.target