remove solarized color scheme from gvimrc

* 'master' of github.com:charlesreid1/debian-dotfiles:
  change path to be absolute
  rename and update backup gitea docker volume script

attic/cloud_init/README.md

@@ -15,6 +15,18 @@ in as the root user.
 [digital ocean guide to cloud-init](https://www.digitalocean.com/community/tutorials/how-to-use-cloud-config-for-your-initial-server-setup)
 
 
+## What to copy and paste in
+
+Copy and paste this into the box:
+
+```
+#!/bin/bash
+bash <( curl https://git.charlesreid1.com/dotfiles/debian/raw/branch/master/cloud_init/cloud_init_XXXXX.sh )
+```
+
+where you have checked in a cloud init script called `cloud_init_XXXXX.sh`
+that sets the machine name and non-root user that you want.
+
 ## Notes
 
 problems with pasting bash script with shebang

attic/cloud_init/cloud_init_fardaa.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -x
+
+# this is hard-coded into the system tasks:
+# sudo_make_user_charles.sh
+USER="charles"
+
+# first things first
+apt-get update
+apt-get install -y git
+
+# check out root user dotfiles at /root/dotfiles
+DOTFILES="$HOME/dotfiles"
+git clone https://git.charlesreid1.com/dotfiles/debian $DOTFILES
+
+# run root init script
+THE_HOSTNAME="fardaa"
+$DOTFILES/tasks/sudo_all.sh $THE_HOSTNAME
+
+# above script creates user charles with pw zeno135
+# change it
+
+# copy the user init script
+cp $DOTFILES/tasks/charles_init.sh /home/$USER/.
+chown $USER:$USER /home/$USER/charles_init.sh
+
+# run user init script as user
+sudo -H -i -u $USER /home/$USER/charles_init.sh
+

crontab/bear.crontab

@@ -0,0 +1,11 @@
+# MIN: 0-59
+# HOUR: 0-23
+# DOM (day of month): 1-31
+# MONTH: 1-12 or Jan-Dec
+# DOW (day of week): 0-6 or Sun-Sat
+#
+# MIN   HR  DOM   MONTH   DOW   CMD
+  3     2   *     *       Sun   /home/charles/pod-charlesreid1/utils-backup/gitea_dump.sh
+  5     2   *     *       Sun   /home/charles/pod-charlesreid1/utils-backup/wikidb_dump.sh
+  7     2   *     *       Sun   /home/charles/pod-charlesreid1/utils-backup/wikifiles_dump.sh
+  9     2   *     *       Sun   /home/charles/pod-charlesreid1/utils-backup/clean.sh

crontab/bespin.crontab

@@ -0,0 +1,10 @@
+# MIN: 0-59
+# HOUR: 0-23
+# DOM (day of month): 1-31
+# MONTH: 1-12 or Jan-Dec
+# DOW (day of week): 0-6 or Sun-Sat
+#
+# MIN   HR  DOM   MONTH   DOW   CMD
+  5     3   *     *       *     /home/charles/pod-private-wiki/utils-backup/wikidb_dump.sh
+  6     3   *     *       *     /home/charles/pod-private-wiki/utils-backup/wikifiles_dump.sh
+

crontab/dorky.crontab

@@ -0,0 +1,8 @@
+# MIN: 0-59
+# HOUR: 0-23
+# DOM (day of month): 1-31
+# MONTH: 1-12 or Jan-Dec
+# DOW (day of week): 0-6 or Sun-Sat
+#
+# MIN   HR  DOM   MONTH   DOW   CMD
+  *     *   *     *       *     /home/charles/dotfiles/dotfiles/scripts/fail.sh

crontab/krash.crontab

@@ -3,13 +3,17 @@
 #
 #
 #
-# Description:  Daily charlesreid1.com backup
+# Description:  charlesreid1.com backup
-# Frequency:    All day every day, 2:00 am
+# 
+# Bimonthly
 #
 #   MIN      HR         DOM            MONTH            DOW          CMD
     0        2          *              *                *            /home/charles/codes/docker/pod-charlesreid1/utils-backup/daily_mysql.sh
     5        2          *              *                *            /home/charles/codes/docker/pod-charlesreid1/utils-backup/daily_wikifiles.sh
-    10       2          *              *                *            /home/charles/codes/docker/pod-charlesreid1/utils-backup/daily_gitea.sh
+#
+    10       2          1              *                *            /home/charles/codes/docker/pod-charlesreid1/utils-backup/monthly_gitea.sh
+    10       2          15             *                *            /home/charles/codes/docker/pod-charlesreid1/utils-backup/monthly_gitea.sh
+#
 #
     10       *          *              *                *            /home/charles/codes/docker/pod-charlesreid1/utils-gitea/assemble_gitea_counts.sh
     11       *          *              *                *            /home/charles/scripts/git_pull_data.sh

crontab/krash.sudo.crontab

@@ -3,6 +3,12 @@
 #
 # Description:  XXXX
 # Frequency:    YYYY
+# 
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+# 
 #   MIN      HR         DOM            MONTH            DOW          CMD
 #    0        2          *              *                *            /usr/bin/env python /home/charles/script.py
      13       *          *              *                *            /home/charles/codes/docker/pod-charlesreid1/utils-gitea/scrape_gitea_as_sudo.sh
+     0        4          25             *                *            /home/charles/codes/certbot/krash_renew.sh
+     5        3          *              *                *            /home/charles/scripts/iptables_ban_jerks.sh
+     0        0          *              *                0            service dockerpod-charesreid1 restart

dotfiles/.aliases

@@ -11,13 +11,20 @@ git() {
 
 
 # lazy af
+# -------
 alias mk='make'
 alias smk='snakemake'
 alias smake='snakemake'
 alias py='python3'
 alias ipy='ipython --pylab'
-alias ip="dig +short myip.opendns.com @resolver1.opendns.com"
+alias ip="curl -4 icanhazip.com"
-alias ipaddr="dig +short myip.opendns.com @resolver1.opendns.com"
+alias ipaddr="curl -4 icanhazip.com"
+
+
+# python stuff
+# ------------
+# https://unix.stackexchange.com/a/459586
+alias vp="[ -d vp ] || virtualenv -p python3 vp && source vp/bin/activate"
 
 
 # one-letter aliases
@@ -37,6 +44,7 @@ alias sha1='openssl sha1'
 alias du='du -kh'
 alias dh='dh -kTh'
 alias pc='pelican content'
+alias lsop='lsof -nP +c 15 | /usr/bin/grep LISTEN'
 
 
 
@@ -52,40 +60,42 @@ alias mv='mv -i'
 
 
 # ls aliases
-# -------------------- 
+# --------------------
 # Detect which `ls` flavor is in use
-if ls --color > /dev/null 2>&1; then # GNU `ls`
+
+LS=$(which ls)
+if $LS --color > /dev/null 2>&1; then # GNU `ls`
     colorflag="--color"
     export LS_COLORS='no=00:fi=00:di=01;31:ln=01;36:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arj=01;31:*.taz=01;31:*.lzh=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.gz=01;31:*.bz2=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.jpg=01;35:*.jpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.avi=01;35:*.fli=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.ogg=01;35:*.mp3=01;35:*.wav=01;35:'
     groupflag="--group-directories-first"
 else # macOS `ls`
     colorflag="-G"
     export LSCOLORS='BxBxhxDxfxhxhxhxhxcxcx'
-    groupflag=""
+    groupflag="--group-directories-first"
 fi
 
 
 
-alias ls="/bin/ls -1 ${colorflag} ${groupflag}"
+alias ls="$LS -1 ${colorflag} ${groupflag}"
 alias sl='ls'
 
 
 
 # List all files colorized in long format
-alias l="/bin/ls -lhAF ${colorflag}"
+alias l="$LS -lhAF ${colorflag}"
 alias ll="l"
 
 # List all files colorized in long format, including dot files
 alias la="l -a"
 
 # List only directories
-alias lsd="/bin/ls -lF ${colorflag} | grep --color=never '^d'"
+alias lsd="$LS -lF ${colorflag} | grep --color=never '^d'"
 
 
 alias lr='ls -r' # reverse
 alias llr='ll -r' # reverse
 
-alias lt='/bin/ls -lhAFt -G' # time
+alias lt='$LS -lhAFt -G' # time
 
 alias lt='ls -t' # time
 alias llt='ll -t' # time
@@ -98,7 +108,7 @@ alias lk='ls -lSr' # by size (small to big)
 
 
 
-# clear 
+# clear
 # ---------------
 alias clc='clear; pwd'
 alias cl='clc; ll'
@@ -124,77 +134,44 @@ alias socks='ssh -fN -D 8080 charles@charlesreid1.com'
 
 alias charles="charles"
 function charles() {
-  ssh-agent > ~/ssh.file # env vars in ssh.file 
+  ssh-agent > ~/ssh.file # env vars in ssh.file
   chmod +x ~/ssh.file # execute file
   ~/ssh.file > /dev/null
   rm -f ~/ssh.file
   export IP="charlesreid1.com"
-  ssh -Y charles@${IP} # the actual ssh call
+  ssh -p 5778 -Y charles@${IP} # the actual ssh call
 }
 
-alias blackbeard="blackbeard"
+alias bear="bear"
-function blackbeard() {
+function bear() {
-  ssh-agent > ~/ssh.file # env vars in ssh.file 
+  ssh-agent > ~/ssh.file # env vars in ssh.file
   chmod +x ~/ssh.file # execute file
   ~/ssh.file > /dev/null
   rm -f ~/ssh.file
-  export IP="54.191.178.27"
+  export IP="173.255.249.79"
   ssh -Y charles@${IP} # the actual ssh call
 }
 
-alias jupiter="jupiter"
+alias bespin="bespin"
-function jupiter() {
+function bespin() {
-  ssh-agent > ~/ssh.file # env vars in ssh.file 
+  ssh-agent > ~/ssh.file # env vars in ssh.file
   chmod +x ~/ssh.file # execute file
   ~/ssh.file > /dev/null
   rm -f ~/ssh.file
-  export IP="10.6.0.2"
+  export IP="bespin"
   ssh -Y charles@${IP} # the actual ssh call
 }
 
-alias maya="maya"
+alias dorky="dorky"
-function maya() {
+function dorky() {
-  ssh-agent > ~/ssh.file # env vars in ssh.file 
+  ssh-agent > ~/ssh.file # env vars in ssh.file
   chmod +x ~/ssh.file # execute file
   ~/ssh.file > /dev/null
   rm -f ~/ssh.file
-  export IP="10.6.0.10"
+  export IP="charlesreid1.party"
   ssh -Y charles@${IP} # the actual ssh call
 }
 
-alias cronus="cronus"
-function cronus() {
-  ssh-agent > ~/ssh.file # env vars in ssh.file 
-  chmod +x ~/ssh.file # execute file
-  ~/ssh.file > /dev/null
-  rm -f ~/ssh.file
-  export IP="10.6.0.20"
-  ssh -Y charles@${IP} # the actual ssh call
-}
-
-alias krash="krash"
-function krash() {
-  ssh-agent > ~/ssh.file # env vars in ssh.file 
-  chmod +x ~/ssh.file # execute file
-  ~/ssh.file > /dev/null
-  rm -f ~/ssh.file
-  export IP="45.56.87.232"
-  ssh -Y charles@${IP} # the actual ssh call
-}
-
-alias wisko="wisko"
-function wisko() {
-  ssh-agent > ~/ssh.file # env vars in ssh.file 
-  chmod +x ~/ssh.file # execute file
-  ~/ssh.file > /dev/null
-  rm -f ~/ssh.file
-  export IP="159.65.75.41"
-  ssh -Y melo@${IP} # the actual ssh call
-}
-
-
-
-
 
 # tmux shortcuts
 # constructed with help from https://robots.thoughtbot.com/a-tmux-crash-course
@@ -251,7 +228,7 @@ alias cealr='clear'
 
 
 
-# cd 
+# cd
 # ---------------
 alias back='cd $OLDPWD'
 
@@ -268,20 +245,22 @@ alias back='cd $OLDPWD'
 # Easier navigation: .., ..., ...., ....., ~ and -
 alias ..="cd .."
 alias ...="cd ../.."
-### alias ....="cd ../../.."
+alias ....="cd ../../.."
-### alias .....="cd ../../../.."
+alias .....="cd ../../../.."
-### alias ~="cd ~" # `cd` is probably faster to type though
+alias ~="cd ~" # `cd` is probably faster to type though
-### alias -- -="cd -"
+alias -- -="cd -"
 
 # Shortcuts
 alias d="cd ~/Desktop"
 alias g="git"
+alias gs="git status"
 alias h="history"
 
+#alias s="screen"
 
 # Always enable colored `grep` output
 # Note: `GREP_OPTIONS="--color=auto"` is deprecated, hence the alias usage.
-alias grep='grep --color=auto'
+alias grep='grep -i --color=auto --exclude-dir=vp'
 alias fgrep='fgrep --color=auto'
 alias egrep='egrep --color=auto'
 
@@ -291,6 +270,12 @@ alias week='date +%V'
 # Stopwatch
 alias timer='echo "Timer started. Stop with Ctrl-D." && date && time cat && date'
 
+# IP addresses
+alias localip="ipconfig getifaddr en0"
+alias ips="ifconfig -a | grep -o 'inet6\? \(addr:\)\?\s\?\(\(\([0-9]\+\.\)\{3\}[0-9]\+\)\|[a-fA-F0-9:]\+\)' | awk '{ sub(/inet6? (addr:)? ?/, \"\"); print }'"
+
+# Reload the shell (i.e. invoke as a login shell)
+alias reload="exec ${SHELL} -l"
 
 # Print each PATH entry on a separate line
 alias path='echo -e ${PATH//:/\\n}'

dotfiles/.bash_profile

@@ -1,45 +1,71 @@
 # This is the bash profile.
-# 
+#
 # This file sets PATH and bash options.
 #
 # to add your own non-committed machine-specific settings,
-# use ~/.extra 
+# use ~/.extra
 
 # Must
 EDITOR="vim"
+GIT_EDITOR="vim"
+
+# Better man pages
+PAGER="most"
+
+# Go stuff
+GOROOT=$HOME/go
+GOPATH=$HOME/go
 
 # Set $PATH here
-PATH="/usr/local/bin:$PATH"
 PATH="${HOME}/scripts:${PATH}"
-PATH="/sbin:${PATH}"
+PATH="/usr/local/bin:$PATH"
-PATH="/opt/collectd/bin:${PATH}"
+PATH="/usr/local/sbin:${PATH}" # homebrew admin tools
-PATH="/opt/collectd/sbin:${PATH}"
+PATH="${PATH}:${GOROOT}/bin"
-#PATH="/usr/local/bro/bin:${PATH}"
+PATH="/usr/local/opt/coreutils/libexec/gnubin:${PATH}"
+PATH="${HOME}/bin:${PATH}"
+if [[ ("$HOSTNAME" == "seawater") || ("$HOSTNAME" == "bascom") ]]; then
+    PATH="${HOME}/bin/elasticsearch-5.4.2/bin:${PATH}"
 
+    # assume-role cli util:
+    # Add homebrew-installed ruby to path:
+    # (WARNING: THIS CAN CAUSE PROBLEMS!)
+    export PATH="/usr/local/opt/ruby/bin:$PATH"
 
+    # Add homebrew-ruby-gem-installed packages to path:
+    export PATH="/usr/local/lib/ruby/gems/2.6.0/bin:$PATH"
 
-# Python:
+    # aws - load config file when using assume-role
-# Try not to have to deal with PYTHONPATH...
+    export AWS_SDK_LOAD_CONFIG="1"
+fi
 
-# go 
+# Tell git not to look for getext.sh
-export GOPATH="${HOME}/gocode"
+# since pyenv has trouble with that
-export PATH="$GOPATH/bin:$PATH"
+export GIT_INTERNAL_GETTEXT_TEST_FALLBACKS=1
 
-# goenv
+if [[ "$HOSTNAME" == "bascom" ]]; then
+    # git tab completion
+    source ${HOME}/.git-completion.bash
+fi
+
+# goenv installer
 export GOENV_ROOT="$HOME/.goenv"
 export PATH="$GOENV_ROOT/bin:$PATH"
 
-# pyenv
+# Only enable this if you are using go.
-export PATH="${HOME}/.pyenv/bin:${PATH}"
+# This will add half a second every time you
+# open a new shell.
+#eval "$(goenv init -)"
 
-# activate pyenv
+# pyenv installer
-# you should also install a pyenv version!
+# https://github.com/pyenv/pyenv-installer
-# pyenv install --list
+export PYENV_ROOT="$HOME/.pyenv"
-# pyenv install -s miniconda3-4.3.30
+export PATH="$PYENV_ROOT/bin:$PATH"
 eval "$(pyenv init -)"
-#eval "$(pyenv virtualenv-init -)"
 
+export PATH
 
+# Just let homebrew take care of PYTHONPATH, yeah?
+# But if you really needed to, you could set it here.
 
 
 # Bash history
@@ -50,13 +76,48 @@ HISTIGNORE="ls:cls:clc:clear:pwd:l:ll:[ ]*"
 HISTSIZE=1000000
 HISTTIMEFORMAT=': %Y-%m-%d_%H:%M:%S; '
 
+# Save Bash history
+shopt -s cmdhist;
 # Append to the Bash history file, rather than overwriting it
 shopt -s histappend;
-# Save Bash history 
+# Write history to .bash_history immediately.
-shopt -s cmdhist;
+# -a writes current/new lines to history file
+# -n reloads only new commands
+# https://askubuntu.com/a/673283
+PROMPT_COMMAND='history -a;history -n'
+
+# don't try to autocomplete commands when tab is pressed and line is empty
+shopt -s no_empty_cmd_completion
+
+if [[ "$HOSTNAME" == "bascom" ]]; then
+    # aws cli tab-completion
+    # https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-completion.html
+    complete -C "$(pyenv which aws_completer)" aws
+fi
 
 
 
+#############################
+# ssh-agent setup
+SSH_ENV="$HOME/.ssh/agent-environment"
+
+function start_agent {
+    /usr/bin/ssh-agent | sed 's/^echo/#echo/' > "${SSH_ENV}"
+    chmod 600 "${SSH_ENV}"
+    . "${SSH_ENV}" > /dev/null
+    /usr/bin/ssh-add;
+}
+
+# Source SSH settings, if applicable
+if [ -f "${SSH_ENV}" ]; then
+    . "${SSH_ENV}" > /dev/null
+    ps -ef | grep ${SSH_AGENT_PID} | grep ssh-agent$ > /dev/null || {
+        start_agent;
+    }
+else
+    start_agent;
+fi
+
 
 #############################
 # modified mathias
@@ -75,22 +136,13 @@ shopt -s nocaseglob;
 # Autocorrect typos in path names when using `cd`
 shopt -s cdspell;
 
-# Enable some Bash 4 features when possible:
-# * `autocd`, e.g. `**/qux` will enter `./foo/bar/baz/qux`
-# * Recursive globbing, e.g. `echo **/*.txt`
-for option in autocd globstar; do
-	shopt -s "$option" 2> /dev/null;
-done;
-
 if [ -f /etc/bash_completion ]; then
 	source /etc/bash_completion;
 fi;
 
-# Enable tab completion for `g` by marking it as an alias for `git`
+if [[ "$HOSTNAME" == "bascom" ]]; then
-if type _git &> /dev/null && [ -f /usr/local/etc/bash_completion.d/git-completion.bash ]; then
+    # Enable tab completion for `g` by marking it as an alias for `git`
-	complete -o default -o nospace -F _git g;
+    if type _git &> /dev/null && [ -f /usr/local/etc/bash_completion.d/git-completion.bash ]; then
-fi;
+    	complete -o default -o nospace -F _git g;
-
+    fi;
-# Add tab completion for SSH hostnames based on ~/.ssh/config, ignoring wildcards
+fi
-[ -e "$HOME/.ssh/config" ] && complete -o "default" -o "nospace" -W "$(grep "^Host" ~/.ssh/config | grep -v "[?*]" | cut -d " " -f2- | tr ' ' '\n')" scp sftp ssh;
-

dotfiles/.bash_prompt

@@ -41,14 +41,23 @@ else
 	white="\e[1;37m";
 fi;
 
-# User color depends on machine
 if [[ "${HOSTNAME}" == "rojo" ]]; then
     userStyle="${red}"
 elif [[ "${HOSTNAME}" == "jupiter" ]]; then
     userStyle="${green}"
 elif [[ "${HOSTNAME}" == "krash" ]]; then
     userStyle="${cyan}"
-elif [[ "${HOSTNAME}" == "blackbeard" ]]; then
+elif [[ "${HOSTNAME}" == "bear" ]]; then
+    userStyle="${red}"
+elif [[ "${HOSTNAME}" == "bespin" ]]; then
+    userStyle="${purple}"
+elif [[ "${HOSTNAME}" == "dorky" ]]; then
+    userStyle="${yellow}"
+elif [[ "${HOSTNAME}" == "emmett" ]]; then
+    userStyle="${green}"
+elif [[ "${HOSTNAME}" == "dudley" ]]; then
+    userStyle="${cyan}"
+elif [[ "${HOSTNAME}" == "lars" ]]; then
     userStyle="${white}"
 else
     userStyle="${yellow}"

dotfiles/.exports

@@ -9,13 +9,6 @@
 # Make vim the default editor.
 export EDITOR='vim';
 
-#### Enable persistent REPL history for `node`.
-###export NODE_REPL_HISTORY=~/.node_history;
-#### Allow 32³ entries; the default is 1000.
-###export NODE_REPL_HISTORY_SIZE='32768';
-#### Use sloppy mode by default, matching web browsers.
-###export NODE_REPL_MODE='sloppy';
-
 # Make Python use UTF-8 encoding for output to stdin, stdout, and stderr.
 export PYTHONIOENCODING='UTF-8';
 
@@ -34,13 +27,3 @@ HISTTIMEFORMAT=': %Y-%m-%d %H:%M:%S; '
 export LANG='en_US.UTF-8';
 export LC_ALL='en_US.UTF-8';
 
-# Highlight section titles in manual pages.
-export LESS_TERMCAP_md="${yellow}";
-
-# Don’t clear the screen after quitting a manual page.
-export MANPAGER='less -X';
-
-# Avoid issues with `gpg` as installed via Homebrew.
-# https://stackoverflow.com/a/42265848/96656
-export GPG_TTY=$(tty);
-

dotfiles/.gitconfig

@@ -1,3 +1,10 @@
+######################
+## see bottom for url shorthands,
+## but worth mentioning here:
+#
+## replace any git@github stuff with gh:
+## replace any ssh://git@g.... with cmr:
+
 [user]
     email = charlesreid1@gmail.com
     name = Charles Reid
@@ -20,14 +27,71 @@
     lg2-specific = log --graph --abbrev-commit --decorate --format=format:'%C(bold blue)%h%C(reset) - %C(bold cyan)%aD%C(reset) %C(bold green)(%ar)%C(reset)%C(auto)%d%C(reset)%n''          %C(white)%s%C(reset) %C(dim white)- %an%C(reset)'
     lg3-specific = log --graph --abbrev-commit --decorate --format=format:'%C(bold blue)%h%C(reset) - %C(bold cyan)%aD%C(reset) %C(bold green)(%ar)%C(reset) %C(bold cyan)(committed: %cD)%C(reset) %C(auto)%d%C(reset)%n''          %C(white)%s%C(reset)%n''          %C(dim white)- %an <%ae> %C(reset) %C(dim white)(committer: %cn <%ce>)%C(reset)'
 
+    # typo stuff
+    co = checkout
+    cehckout = checkout
+    ceckout = checkout
+    chekout = checkout
+    commti = commit
+    comimt = commit
+    comit = commit
+
+    # https://victorzhou.com/blog/git-aliases/
+    a = add
+    cm = commit -m
+    rh = reset --hard
+    lo = log --oneline -n 10
+    lolo = log --oneline -n 50
 
     # View abbreviated SHA, description, and history graph of the latest 20 commits
     l = log --pretty=oneline -n 20 --graph --abbrev-commit
 
-    # View the current working tree status using the short format
+    # View the current working tree status: short format and long format
     s = status -s
+    ss = status
 
+    # Just print the name of the current branch
+    cb = branch --show-current
+
+    # Show branches
+    b = branch -v
+    bv = branch -v
+    bb = branch -v
+
+    ba = branch -a
+    bb = branch -v -a
+
+    # Show remotes
+    r = remote -v
+    rv = remote -v
+    ra = remote -v
+
+    f = fetch --all
+
+    # Show log history of commits on this branch
+    # that are different from master
+    # (these aliases are useful for rebases)
+    #
+    # dm = *Different from *Master (log)
+    # https://stackoverflow.com/a/59273667/463213
+    dm = "log --boundary --right-only --graph --oneline master..HEAD"
+
+    # dms = *Different from *Master, *Status
+    dms = "diff --stat master..HEAD"
+
+    # dmf = *Different from *Master, *Files
+    dmf = "diff --name-only master..HEAD"
+
+    # show diff of staged changes
     cdiff = diff --cached
+    cd = diff --cached
+
+    # Interactive rebase with the given number of latest commits
+    # reb N = interactive rebase with last N commits (inclusive)
+    reb = "!r() { git rebase -i HEAD~$1; }; r"
+
+    # continue the rebase
+    rebc = "rebase --continue"
 
     # Show the diff between the latest commit and the current state
     d = !"git diff-index --quiet HEAD -- || clear; git --no-pager diff --patch-with-stat"
@@ -35,16 +99,25 @@
     # `git di $number` shows the diff between the state `$number` revisions ago and the current state
     di = !"d() { git diff --patch-with-stat HEAD~$1; }; git diff-index --quiet HEAD -- || clear; d"
 
+    p = "!f() { git pull $1 $2; }; f"
+
     # Pull in remote changes for the current repository and all its submodules
-    p = !"git pull; git submodule foreach git pull origin master"
+    pp = !"git pull; git submodule foreach git pull origin master"
 
     # Clone a repository including all submodules
     c = clone --recursive
 
+    # Amend the currently staged files to the latest commit
+    amend = commit --amend --reuse-message=HEAD
+
+    # Oops
+    fix = commit --amend --reuse-message=HEAD --edit
+
     # Commit all changes
     ca = !git add -A && git commit -av
 
-    # Switch to a branch, creating it if necessary
+    # Switch to a branch, creating it 
+    # from the current branch if necessary
     go = "!f() { git checkout -b \"$1\" 2> /dev/null || git checkout \"$1\"; }; f"
 
     # Show verbose output about tags, branches or remotes
@@ -55,49 +128,24 @@
     # List aliases
     aliases = config --get-regexp alias
 
-    # Amend the currently staged files to the latest commit
-    amend = commit --amend --reuse-message=HEAD
-
-    # Credit an author on the latest commit
-    credit = "!f() { git commit --amend --author \"$1 <$2>\" -C HEAD; }; f"
-
-    # Interactive rebase with the given number of latest commits
-    reb = "!r() { git rebase -i HEAD~$1; }; r"
-
     # Remove the old tag with this name and tag the latest commit with it.
     retag = "!r() { git tag -d $1 && git push origin :refs/tags/$1 && git tag $1; }; r"
 
-    # Find branches containing commit
+    ### # Find branches containing commit
-    fb = "!f() { git branch -a --contains $1; }; f"
+    ### fb = "!f() { git branch -a --contains $1; }; f"
 
-    # Find tags containing commit
+    ### # Find tags containing commit
-    ft = "!f() { git describe --always --contains $1; }; f"
+    ### ft = "!f() { git describe --always --contains $1; }; f"
 
-    # Find commits by source code
+    ### # Find commits by source code
-    fc = "!f() { git log --pretty=format:'%C(yellow)%h  %Cblue%ad  %Creset%s%Cgreen  [%cn] %Cred%d' --decorate --date=short -S$1; }; f"
+    ### fc = "!f() { git log --pretty=format:'%C(yellow)%h  %Cblue%ad  %Creset%s%Cgreen  [%cn] %Cred%d' --decorate --date=short -S$1; }; f"
 
-    # Find commits by commit message
+    ### # Find commits by commit message
-    fm = "!f() { git log --pretty=format:'%C(yellow)%h  %Cblue%ad  %Creset%s%Cgreen  [%cn] %Cred%d' --decorate --date=short --grep=$1; }; f"
+    ### fm = "!f() { git log --pretty=format:'%C(yellow)%h  %Cblue%ad  %Creset%s%Cgreen  [%cn] %Cred%d' --decorate --date=short --grep=$1; }; f"
 
     # List contributors with number of commits
     contributors = shortlog --summary --numbered
 
-    # Merge GitHub pull request on top of the current branch or,
-    # if a branch name is specified, on top of the specified branch
-    mpr = "!f() { \
-        declare currentBranch=\"$(git symbolic-ref --short HEAD)\"; \
-        declare branch=\"${2:-$currentBranch}\"; \
-        if [ $(printf \"%s\" \"$1\" | grep '^[0-9]\\+$' > /dev/null; printf $?) -eq 0 ]; then \
-            git fetch origin refs/pull/$1/head:pr/$1 && \
-            git checkout -B $branch && \
-            git rebase $branch pr/$1 && \
-            git checkout -B $branch && \
-            git merge pr/$1 && \
-            git branch -D pr/$1 && \
-            git commit --amend -m \"$(git log -1 --pretty=%B)\n\nCloses #$1.\"; \
-        fi \
-    }; f"
-
 [apply]
 
     # Detect whitespace errors when applying a patch
@@ -105,6 +153,8 @@
 
 [core]
 
+    editor = vim
+
     # Use custom `.gitignore` and `.gitattributes`
     excludesfile = ~/.gitignore
     attributesfile = ~/.gitattributes
@@ -177,6 +227,12 @@
 
 # URL shorthands
 
+
+[url "ssh://git@git.charlesreid1.com:222/"]
+
+    insteadOf = "cmr:"
+
+
 [url "git@github.com:"]
 
     insteadOf = "gh:"
@@ -196,3 +252,8 @@
 [url "git://gist.github.com/"]
 
     insteadOf = "gist:"
+
+[rerere]
+
+    enabled = true
+    autoupdate = true

dotfiles/.gvimrc

@@ -1,6 +1,3 @@
-" Use the Solarized Dark theme
-set background=dark
-colorscheme solarized
 " Use 14pt Monaco
 set guifont=Monaco:h14
 " Don’t blink cursor in normal mode

dotfiles/.tmux.conf

@@ -1,81 +1,118 @@
 ###################
 # charles
+#
+# Useful links:
+# https://github.com/mathiasbynens/dotfiles/blob/master/.tmux.conf
+# https://github.com/samoshkin/tmux-config/blob/master/tmux/tmux.conf
+# https://github.com/tony/tmux-config/blob/master/scripts/resize-adaptable.sh
+# https://stackoverflow.com/a/40009032
+# 
+# Basic layout:
+# - .tmux.conf contains tmux configuration variables
+# - .tmux.sessionX contains tmux session setup (keyboard shortcut)
+# - scripts/devtmux contains tmux session setup (named sessions)
 
 set -g default-terminal "screen-256color"
 set-option -g default-terminal screen-256color
 
-
+# Use C-a, since it's on the home row and easier to hit than C-b
-# use C-a, since it's on the home row and easier to hit than C-b
 set-option -g prefix C-a
-unbind-key C-a
+unbind C-a
-bind-key C-a send-prefix
+bind C-a send-prefix
+
+# Set parent terminal title to reflect current window in tmux session
+set -g set-titles on
+set -g set-titles-string "#I:#W"
+
+# Start indexing of windows at 1, not 0
 set -g base-index 1
+setw -g pane-base-index 1
 
 # Easy config reload
 unbind r
-bind-key R source-file ~/.tmux.conf \; display-message "tmux.conf reloaded."
+bind R source-file ~/.tmux.conf \; display-message "tmux.conf reloaded."
 
 # vi is good
 setw -g mode-keys vi
 
-bind-key : command-prompt
+bind : command-prompt
-bind-key r refresh-client
+bind r refresh-client
-bind-key L clear-history
 
-bind-key space next-window
+bind space next-window
-bind-key bspace previous-window
+bind bspace previous-window
-bind-key enter next-layout
+bind enter next-layout
 
 # use vim-like keys for splits and windows
-bind-key v split-window -h -c "#{pane_current_path}"
+# 
-bind-key s split-window -v -c "#{pane_current_path}"
+#   Control-A plus:
-bind-key h select-pane -L
+# 
-bind-key j select-pane -D
+#   s = horizontal split
-bind-key k select-pane -U
+#   v = vertical split
-bind-key l select-pane -R
+#   h = pick left pane
+#   j = pick down pane
+#   k = pick up pane
+#   l = pick right pane
+# 
+bind v split-window -h -c "#{pane_current_path}"
+bind s split-window -v -c "#{pane_current_path}"
+bind h select-pane -L
+bind j select-pane -D
+bind k select-pane -U
+bind l select-pane -R
+bind C-h select-pane -L
+bind C-l select-pane -R
 
 # smart pane switching with awareness of vim splits
+# 
+# Control plus:
+# 
+#   h = pick left pane
+#   j = pick left pane
+#   k = pick left pane
+#   l = pick left pane
+#   \ = pick another pane (?)
+#
 bind -n C-h run "(tmux display-message -p '#{pane_current_command}' | grep -iqE '(^|\/)vim$' && tmux send-keys C-h) || tmux select-pane -L"
 bind -n C-j run "(tmux display-message -p '#{pane_current_command}' | grep -iqE '(^|\/)vim$' && tmux send-keys C-j) || tmux select-pane -D"
 bind -n C-k run "(tmux display-message -p '#{pane_current_command}' | grep -iqE '(^|\/)vim$' && tmux send-keys C-k) || tmux select-pane -U"
 bind -n C-l run "(tmux display-message -p '#{pane_current_command}' | grep -iqE '(^|\/)vim$' && tmux send-keys C-l) || tmux select-pane -R"
 bind -n C-\ run "(tmux display-message -p '#{pane_current_command}' | grep -iqE '(^|\/)vim$' && tmux send-keys 'C-\\') || tmux select-pane -l"
+
+# Control-A then Control-L clears the screen
 bind C-l send-keys 'C-l'
 
-bind-key C-o rotate-window
+# Control-A then Control-O swaps out windows in their respective positions
+bind C-o rotate-window
 
-bind-key + select-layout main-horizontal
+# Control-A plus + makes existing windows have horizontal layout
-bind-key = select-layout main-vertical
+bind + select-layout main-horizontal
-set-window-option -g other-pane-height 25
-set-window-option -g other-pane-width 80
 
-bind-key a last-pane
+# Control-A plus = makes existing windows have vertical layout
-bind-key q display-panes
+bind = select-layout main-vertical
-bind-key c new-window
-bind-key t next-window
-bind-key T previous-window
 
-bind-key [ copy-mode
+## (no idea what this does)
-bind-key ] paste-buffer
+#set-window-option -g other-pane-height 25
+#set-window-option -g other-pane-width 80
 
+# Control-A plus [ or ] moves into copy mode or paste mode
+bind [ copy-mode
+bind ] paste-buffer
+
+# (no idea)
 set-window-option -g display-panes-time 1500
 
+# Control-A plus | or _ splits panes vertically (|) or horizontally (_)
+bind | split-window -h -c "#{pane_current_path}"
+bind _ split-window -v -c "#{pane_current_path}"
+
 # Status Bar
 set-option -g status-interval 1
 set-option -g status-left ''
 set-option -g status-right '%l:%M%p'
-set-window-option -g window-status-current-fg magenta
 set-option -g status-fg default
 
 # Status Bar solarized-dark (default)
 set-option -g status-bg black
-set-option -g pane-active-border-fg black
-set-option -g pane-border-fg black
-
-# Status Bar solarized-light
-if-shell "[ \"$COLORFGBG\" = \"11;15\" ]" "set-option -g status-bg white"
-if-shell "[ \"$COLORFGBG\" = \"11;15\" ]" "set-option -g pane-active-border-fg white"
-if-shell "[ \"$COLORFGBG\" = \"11;15\" ]" "set-option -g pane-border-fg white"
 
 # Set window notifications
 setw -g monitor-activity on
@@ -83,5 +120,10 @@ set -g visual-activity on
 
 # Allow the arrow key to be used immediately after changing windows
 set-option -g repeat-time 0
+# Set up easy go-to sessions with shortcuts 
+bind s source-file ~/.tmux.session1
 
-
+# When in copy mode, v starts a selection and y finishes it
+#bind-key -T copy-mode-vi 'v' send -X begin-selection
+#bind-key -T copy-mode-vi 'y' send -X copy-selection-and-cancel
+bind-key -T copy-mode-vi 'y' send -X copy-pipe

dotfiles/.tmux.session1

@@ -0,0 +1,5 @@
+# https://stackoverflow.com/a/5753059
+new-session -A -s dev -n dev
+send-keys 'vim' C-m
+split-window -v -p 20
+select-pane -t 1

dotfiles/.vimrc

@@ -1,11 +1,7 @@
 " ==============================================
 "               cmr vimrc
 " ==============================================
-"
+set nocompatible              " iMproved
-
-
-" don't bother with vi compatibility
-set nocompatible              " be iMproved, required
 filetype off                  " required
 filetype plugin indent on
 "setlocal spell spelllang=en_us
@@ -13,47 +9,73 @@ set nofoldenable    " disable folding
 
 
 
-
+" ----------------------------------
+" Extremely Important Fix
+" ----------------------------------
 " Turn off Esc+O key sequences (eliminates one-second delay when pressing O):
 " https://superuser.com/questions/161178/why-does-vim-delay-for-a-second-whenever-i-use-the-o-command-open-a-new-line#161216
 " if this is re-enabled i may stab someone.
 set noesckeys
 set ttimeoutlen=5
-" timeout of 5 ms
-" http://cscope.sourceforge.net/cscope_maps.vim
 
-
+" now you have to do this a second time
-" you have to do this a second time
+" (after the above lines)
 set nocompatible
 
 
+" ------------------------
+"  Lazy stuffs
+"  -----------------------
+"  https://spf13.com/post/perfect-vimrc-vim-config-file/
+"
+" make ; work like : to save us from Shift
+" nnoremap --> extra n means normal mode only,
+" nore means no recursive
+nnoremap ; :
+"
+"""" Remove trailing whitespaces and \^M chars
+"""autocmd FileType c,cpp,java,php,js,python,twig,xml,yml autocmd
+"""BufWritePre <buffer> :call
+"""setline(1,map(getline(1,"$"),'substitute(v:val,"\\\\s\\\\+$","","")'))
 
-" ----------------------------------------
-" fix latex highlighting in markdown
-" ----------------------------------------
 
-function! MathAndLiquid()
-    "" Define certain regions
-    " Block math. Look for "$$[anything]$$"
-    syn region math start=/\$\$/ end=/\$\$/
-    " inline math. Look for "$[not $][anything]$"
-    syn match math_block '\$[^$].\{-}\$'
 
-    " Fenced code blocks, used in GitHub Flavored Markdown (GFM)
+" -----------------------------------
-    syn region highlight_block start='```' end='```'
+" TwiddleCase function
-
+" -----------------------------------
-    "" Actually highlight those regions.
+" TwiddleCase cycles a visual selection through
-    hi link math Statement
+" lower case/UPPER CASE/Capital Case
-    hi link math_block Function
+"
-    hi link highlight_block Function
+" Originallly mapped to ~
+" Switched to Control+P
+"   b/c it was not doing anything useful
+function! TwiddleCase(str)
+  if a:str ==# toupper(a:str)
+    let result = tolower(a:str)
+  elseif a:str ==# tolower(a:str)
+    let result = substitute(a:str,'\(\<\w\+\>\)', '\u\1', 'g')
+  else
+    let result = toupper(a:str)
+  endif
+  return result
 endfunction
-
+"" Map twiddle to ~
-" Call everytime we open a Markdown file
+"vnoremap ~ y:call setreg('', TwiddleCase(@"), getregtype(''))<CR>gv""Pgv
-autocmd BufRead,BufNewFile,BufEnter *.md,*.markdown call MathAndLiquid()
+" Map twiddle to C-p
-
+" (You have to have a visual selection first!!!)
+" (This is complete black magic, no idea where this comes from)
+vnoremap <C-p> y:call setreg('', TwiddleCase(@"), getregtype(''))<CR>gv""Pgv
 
 
 
+" ----------------------------------------
+" insert new line without leaving normal mode
+" by literally typing ,o or ,O
+" (this still sucks.)
+" https://vi.stackexchange.com/a/3877
+" ----------------------------------------
+nnoremap <Leader>o o<Esc>
+nnoremap <Leader>O O<Esc>
 
 
 
@@ -72,12 +94,9 @@ nnoremap } }zz
 nnoremap <F1> <nop>
 nnoremap Q <nop>
 nnoremap K <nop>
-
 " use j and k to move among display lines, not just file lines
 noremap j gj
 noremap k gk
-
-
 " if compiled with autocmd, jump to last cursor position
 if has("autocmd")
   " When editing a file, always jump to the last known cursor position.
@@ -87,26 +106,13 @@ if has("autocmd")
     \ if line("'\"") > 0 && line("'\"") <= line("$") |
     \   exe "normal g`\"" |
     \ endif
-
 endif " has("autocmd")
-
-" in Python, don't move comment hashtag to first column.
-" smartindent unnecessary for python anyway.
-" http://stackoverflow.com/questions/2063175/vim-insert-mode-comments-go-to-start-of-line
-au! FileType python setl nosmartindent
-
 " allow visual mode to go to blank space at end of lines
 set virtualedit=block
-
 " put horz./vert. splits in the right place
 set splitbelow
 set splitright
 
-" ------------------------
-"    End Vim Annoyances
-" -----------------------
-
-
 
 
 " ------------------------
@@ -115,72 +121,105 @@ set splitright
 
 " this turns on syntax highlighting
 syntax on
+set synmaxcol=200 " don't syntax highlight past this many chars
 set ic
-
 " this highlights search items
 set hls
-
 " this highlights search items as they are typed
 set incsearch
 hi IncSearch cterm=none ctermfg=blue ctermbg=green
 " make searches case-insensitive, unless they contain upper-case letters:
 set ignorecase
 set smartcase
-
 " toggle search highlighting:
-" press Space to turn off highlighting and clear any message
+" space after search turns off highlights and clears messages
-" already displayed.
 :nnoremap <silent> <Space> :nohlsearch<Bar>:echo<CR>
-
 " turn off splash message
 set shortmess+=I
-
 " allow backspacing after first insert spot
 set backspace=indent,eol,start
-
-
 " murder that cursed blinking cursor
 let &guicursor = &guicursor . ",a:blinkon0"
 
-" ------------------------
+
-"   End For Sanity
+" --------------------------
-" -----------------------
+" Pathogen
-
+" --------------------------
+"
+"  to install vim pathogen plugin:
+"  mkdir -p ~/.vim/autoload ~/.vim/bundle && \
+"  curl -LSso ~/.vim/autoload/pathogen.vim https://tpo.pe/pathogen.vim
+"
+"  now you need to "infect" yourself
+"  (questionable word choices...)
+execute pathogen#infect()
 
 
 
+" --------------------------
+" Go settings
+" --------------------------
+"
+" https://github.com/paulswanson/congo/blob/master/congo.sh
+"
+filetype indent plugin on
+"set number
+"set mouse=a
+"
+" to install vim-go plugin:
+" git clone https://github.com/fatih/vim-go.git ~/.vim/bundle/vim-go
+"
+let g:go_highlight_functions = 1
+let g:go_highlight_methods = 1
+let g:go_highlight_structs = 1
+let g:go_fmt_command = "goimports"
 
 
 " ------------------------
 "   Filetype Settings
 " -----------------------
 
-" Do special stuff for Makefiles:
+" python
-" don't expand tabs to spaces, since actual tab characters are
+" ------------------------
-" needed, and have indentation at 8 chars to be sure that all indents are tabs
+" don't move comment hashtag to the first column.
-" (despite the mappings later):
+" smartindent is unnecessary for python anyway.
+" http://stackoverflow.com/questions/2063175/vim-insert-mode-comments-go-to-start-of-line
+au! FileType python setl nosmartindent
+
+" golang
+" see https://github.com/paulswanson/congo/blob/master/congo.sh
+" ------------------------
+au BufRead,BufNewFile *.go set noexpandtab
+
+" Makefiles
+" ------------------------
 au BufRead,BufNewFile Makefile*,*.make,*.mk set noexpandtab
 
-
+" C++
-" C++ files: 4 or even 8 (the 8 looks nice)
+" ------------------------
 au BufRead,BufNewFile *.cpp,*.cxx,*.cc,*.c,*.h,*.hpp,*.hxx,*.hh set tabstop=4 shiftwidth=4 softtabstop=4 nowrap
 
-
+" Snakemake files: Snakefile, .rule, .snake, .settings, .smk
-" Snakemake files are named Snakefile, or .rule, or .snake, or .smk
-" https://snakemake.readthedocs.io/en/stable/project_info/faq.html#how-do-i-enable-syntax-highlighting-in-vim-for-snakefiles
-au BufNewFile,BufRead Snakefile set syntax=snakemake
-au BufNewFile,BufRead *rule set syntax=snakemake
-au BufNewFile,BufRead *.smk set syntax=snakemake
-
-
 " ------------------------
-"     Yaml Files
+au BufNewFile,BufRead set syntax=snakemake
-" -----------------------
+au BufNewFile,BufRead Snakefile*,*.rule,*.snake,*.smk set syntax=snakemake
 
+" Yaml
+" ------------------------
 autocmd FileType yaml setlocal ts=2 sts=2 sw=2 expandtab
+" don't autoindent yaml files
+filetype plugin indent on
+au filetype yaml call DisableIndentY()
+function! DisableIndentY()
+        set autoindent&
+        set cindent&
+        set smartindent&
+        set indentexpr&
+endfunction
 
-
+" Javascript
-
+" -------------------------
+au BufRead,BufNewFile *.js,*.javascript set tabstop=2 shiftwidth=2 softtabstop=2 nowrap
 
 
 
@@ -198,27 +237,26 @@ set nowrap
 set tabstop=4 shiftwidth=4 expandtab
 set nosmartindent   " die die die
 
-" color scheme stuff
+" set the text width at
-"colorscheme desert
+" 80 or 88, whatever
-"colorscheme solarized
+set textwidth=115
+" IMPORTANT -
+" above directive will auto-wrap
+" your text as you type it, and may
+" end up driving you mad.
+
+" > Bugbear's documentation explains 88 vs 80:
+" > "it's like highway speed limits, we won't bother
+" > you if you overdo it by a few km/h".
 "
-" Turn character 80 red
+" Turn character 80/88 red
 " (for visibility conforming to
 "  coding standards)
 " https://stackoverflow.com/questions/23246962/vim-highlight-a-single-character-at-column-80#23247938
 hi Bang ctermfg=red guifg=red
+"match Bang /\%>87v.*\%<89v/
 match Bang /\%>79v.*\%<81v/
 
-" ------------------------
-"     End Lines & Tabs
-" ------------------------
-
-
-
-
-
-
-
 
 
 " ---------------------------
@@ -227,46 +265,33 @@ match Bang /\%>79v.*\%<81v/
 " correct my common typos without me even noticing them:
 abbreviate teh the
 abbreviate hte the
-
 abbreviate reccommend recommend
 abbreviate reccomend recommend
 abbreviate recomend recommend
-
 abbreviate slef self
-
+abbreviate paramters parameters
-" ------------------------
+abbreviate exmaple example
-"   End Lines & Tabs
+abbreviate improt import
-" -----------------------
+abbreviate impot import
-
+abbreviate imrpot import
-
+abbreviate surpress suppress
-
+abbreviate supress suppress
-
 
 
 
 " ------------------------
 "     Tab Wild Mode
 " -----------------------
-
 " from http://dotfiles.org/~brendano/.vimrc:
-"=================================================
-"
 " :e <tab> brings up longest; <tab> again shows list
 set wildmode=longest,list
-
 if exists('+autochdir')
   " so :e is relative to current file
   set autochdir
 endif
-
 " Running command :CD will change to current file's directory
 com! CD cd %:p:h
 
-" ------------------------
-"   End Tab Wild Mode
-" -----------------------
-
-
 
 
 " ------------------------
@@ -294,15 +319,6 @@ if has("user_commands")
     command! -bang Qa qa<bang>
 endif
 
-" ------------------------
-"    End Fat Fingers
-" -----------------------
-
-
-
-
-
-
 
 
 " ------------------------
@@ -312,7 +328,6 @@ endif
 " don't autoindent markdown files
 filetype plugin indent on
 au filetype mkd call DisableIndent()
-
 function! DisableIndent()
         set autoindent&
         set cindent&
@@ -320,11 +335,25 @@ function! DisableIndent()
         set indentexpr&
 endfunction
 
-" ------------------------
+" fix latex highlighting in markdown
-"       End Markdown
+function! MathAndLiquid()
-" -----------------------
+    "" Define certain regions
+    " Block math. Look for "$$[anything]$$"
+    syn region math start=/\$\$/ end=/\$\$/
+    " inline math. Look for "$[not $][anything]$"
+    syn match math_block '\$[^$].\{-}\$'
 
+    " Fenced code blocks, used in GitHub Flavored Markdown (GFM)
+    syn region highlight_block start='```' end='```'
 
+    "" Actually highlight those regions.
+    hi link math Statement
+    hi link math_block Function
+    hi link highlight_block Function
+endfunction
+
+" Call everytime we open a Markdown file
+autocmd BufRead,BufNewFile,BufEnter *.md,*.markdown call MathAndLiquid()
 
 
 
@@ -333,29 +362,27 @@ endfunction
 " -----------------------
 "
 " Persistent Undo
-" https://www.reddit.com/r/vim/comments/kz84u/what_are_some_simple_yet_mindblowing_tweaks_to/c2onmqe
 if has("persistent_undo")
     set undodir=~/.vim/undodir
     set undofile
 endif
 
-" ------------------------
-"       End Reddit
-" -----------------------
-
-
-
-
 
 
 " ------------------------
 "  Github Maximum Awesome
 " -----------------------
 "
+" By default, <Leader> is \
+" but that's hard to reach,
+" and no one uses , anyway
+let mapleader = ','
+" now shortcuts are as easy as
+" ,A ,B ,C
+
 " Shift+Tab should de-indent
 " Control + D is de-indent
 inoremap <S-Tab> <C-d>
-
 set autoindent
 set backspace=2                                              " Fix broken backspace in some setups
 set clipboard=unnamed                                        " yank and paste with the system clipboard
@@ -364,20 +391,14 @@ set ruler                                                    " show where you ar
 set showcmd
 set smartcase                                                " case-sensitive search if any caps
 set wildignore=log/**,node_modules/**,target/**,tmp/**,*.pyc
-
-" keyboard shortcuts
-let mapleader = ','
 noremap <C-h> <C-w>h
 noremap <C-j> <C-w>j
 noremap <C-k> <C-w>k
 noremap <C-l> <C-w>l
-
 " in case you forgot to sudo
 cnoremap w!! %!sudo tee > /dev/null %
-
 " automatically rebalance windows on vim resize
 autocmd VimResized * :wincmd =
-
 " Fix Cursor in TMUX
 if exists('$TMUX')
   let &t_SI = "\<Esc>Ptmux;\<Esc>\<Esc>]50;CursorShape=1\x7\<Esc>\\"
@@ -386,18 +407,9 @@ else
   let &t_SI = "\<Esc>]50;CursorShape=1\x7"
   let &t_EI = "\<Esc>]50;CursorShape=0\x7"
 endif
-
 " Don't copy the contents of an overwritten selection.
 vnoremap p "_dP
 
-" -----------------------------
-"  End Github Maximum Awesome
-" -----------------------------
-"
-
-
-
-
 
 
 " ====================
@@ -405,39 +417,25 @@ vnoremap p "_dP
 " ====================
 " " (woah.)
 
-" " Use the Solarized Dark theme
-" set background=dark
-" colorscheme solarized
-" let g:solarized_termtrans=1
-"
-"
 " Enhance command-line completion
 set wildmenu
-
 " Allow backspace in insert mode
 set backspace=indent,eol,start
-
 " Optimize for fast terminal connections
 set ttyfast
-
 """"""""""""""""""""""""
 "" EVIL
 "set binary
 """"""""""""""""""""""""
-
 " Don’t add empty newlines at the end of files
 set noeol
-
+" Set swap file dir
-" Centralize backups, swapfiles and undo history
-set backupdir=~/.vim/backups
 set directory=~/.vim/swap
 if exists("&undodir")
 	set undodir=~/.vim/undo
 endif
-
 " Don’t create backups when editing files in certain directories
 set backupskip=/tmp/*,/private/tmp/*
-
 " Respect modeline in files
 set modeline
 set modelines=4
@@ -446,9 +444,8 @@ set modelines=4
 """set secure
 " Enable line numbers
 set number
-" Enable syntax highlighting
+"""""""""""""" Enable syntax highlighting
-syntax on
+"""""""""""""syntax on
-
 " Highlight current line
 set cursorline
 " Highlight searches
@@ -476,24 +473,12 @@ set title
 " Show the (partial) command as it’s being typed
 set showcmd
 " Use relative line numbers
-if exists("&relativenumber")
+""" if exists("&relativenumber")
-	set relativenumber
+""" 	set relativenumber
-	au BufReadPost * set relativenumber
+""" 	au BufReadPost * set relativenumber
-endif
+""" endif
-" Start scrolling three lines before the horizontal window border
+" Start scrolling N lines before the horizontal window border
-set scrolloff=7
+set scrolloff=5
-
-" Strip trailing whitespace (,ss)
-function! StripWhitespace()
-	let save_cursor = getpos(".")
-	let old_query = getreg('/')
-	:%s/\s\+$//e
-	call setpos('.', save_cursor)
-	call setreg('/', old_query)
-endfunction
-noremap <leader>ss :call StripWhitespace()<CR>
-" Save a file as root (,W)
-noremap <leader>W :w !sudo tee % > /dev/null<CR>
 
 " Automatic commands
 if has("autocmd")
@@ -504,8 +489,113 @@ if has("autocmd")
     " Treat .md files as Markdown
     autocmd BufNewFile,BufRead *.md setlocal filetype=markdown
 endif
-
+" Mark special characters
-
+"set listchars=nbsp:☠,tab:▸␣
-set listchars=nbsp:☠,tab:▸␣
 "set listchars=tab:▸␣
-set list
+"set list
+
+
+" ---------------------------
+"  <leader> is set to , above
+"  and gives us a whole namespace
+"  of shortcuts to work with.
+"
+"  Can map things to:
+"  - custom functions
+"  - system comands
+
+" Show leader in bottom right
+set showcmd
+
+" Strip whitespace - trailing whitespace - with (,ss)
+function! StripWhitespace()
+	let save_cursor = getpos(".")
+	let old_query = getreg('/')
+	:%s/\s\+$//e
+	call setpos('.', save_cursor)
+	call setreg('/', old_query)
+endfunction
+noremap <Leader>ss :call StripWhitespace()<cr>
+
+" Strip annoying windows newline characters ^M
+function! StripWinLineBreaks()
+    let save_cursor = getpos(".")
+    let old_query = getreg('/')
+    :%s/
+//g
+    call setpos('.', save_cursor)
+    call setreg('/', old_query)
+endfunction
+noremap <Leader>sn :call StripWinLineBreaks()<cr>
+
+" Save a file as root (,W)
+noremap <Leader>W :w !sudo tee % > /dev/null<cr>
+
+
+
+
+" ----------------------------
+" colorrrzzzzz
+" ----------------------------
+
+set background=dark
+
+set t_Co=256
+
+function! BgToggle()
+  if &background == "light"
+    execute ":set background=dark"
+  else
+    execute ":set background=light"
+  endif
+endfunction
+nnoremap <F5> :call BgToggle()<cr>
+
+
+" more color schemes:
+"colorscheme blue
+"colorscheme darkblue
+"colorscheme default
+"colorscheme delek
+colorscheme desert " <-- old standby
+"colorscheme elflord
+"colorscheme evening
+"colorscheme industry
+"colorscheme koehler
+"colorscheme macvim
+"colorscheme morning
+"colorscheme murphy
+"colorscheme pablo
+"colorscheme peachpuff " <-- not bad
+"colorscheme ron
+"colorscheme shine
+"colorscheme slate
+"colorscheme solarized
+"colorscheme torte
+"colorscheme zellner
+
+" ------------------------
+" Move Faster
+" ------------------------
+" (these MUST go at the end)
+"
+" default shift + j (combines lines)
+" moves to shift + L
+nnoremap <S-L> :join<CR>
+
+" shift + j and shift + k
+" move up and down n lines
+map <S-j> 7j
+map <S-k> 7k
+
+" Bubble text up/down
+" Single line:
+nmap <C-k> ddkP
+nmap <C-j> ddp
+" Multiple lines:
+vmap <C-k> xkP`[V`]
+vmap <C-j> xp`[V`]
+
+" -----------------------------
+" Jedi Autocomplete Plugin
+" -----------------------------

dotfiles/bluebear_scripts/captain_hook_canary.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+: '
+Captain Hook Canary Script
+
+
+Note: this needs an associated systemd service.
+See the services directory of the dotfiles repo.
+
+This is a canary script for connecting
+the Captain Hook container to the host 
+machine, and triggering tasks on the 
+host machine with webhooks.
+
+The Captain Hook container mounts the 
+following host directory inside the 
+container (same location for host/container):
+
+/tmp/triggers/
+
+When a webhook in Captain Hook wants to 
+trigger an event on the host (blackbeard),
+it puts a file in /tmp/triggers/.
+
+Meanwhile, on the host, this script checks
+every 10 seconds for trigger files.
+
+Each webhook can create its own trigger file,
+and this script processes each trigger differently.
+'
+
+while true
+do
+    # bootstrap-pull captain hook
+    if [ -f "/tmp/triggers/push-b-captain-hook-master" ]; then
+        echo "CAPTAIN HOOK'S CANARY:"
+        echo "Running trigger to update Captain Hook on the host machine (user charles)"
+        sudo -H -u charles python /home/charles/scripts/captain_hook_pull_host.py
+        echo "All done."
+        rm -f "/tmp/triggers/push-b-captain-hook-master"
+        touch /tmp/canary-yup-host-works
+    fi
+
+    sleep 10;
+done
+

dotfiles/bluebear_scripts/captain_hook_pull_host.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+import subprocess
+import os
+import time
+
+"""
+Captain Hook: Pull Captain Hook on the Host 
+
+This script is called by the host machine 
+(blackbeard) running the Captain Hook container.
+
+This is triggered by push actions to the 
+master branch of b-captain-hook.
+
+The action is to update (git pull) the copy 
+of Captain Hook running on the host, and
+restart the container pod.
+"""
+
+work_dir = os.path.join('/home','charles','codes','docker','pod-webhooks','b-captain-hook')
+pod_dir = os.path.join('/home','charles','codes','docker','pod-webhooks')
+
+from datetime import datetime
+d = datetime.now().strftime('%Y-m-%d')
+with open('/tmp/captain_hook_pull_host_%s.log'%(d),'w') as f:
+
+    # Step 1:
+    # Update Captain Hook
+    co_cmd = ['git','checkout','master']
+    subprocess.call(co_cmd, cwd=work_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    stdout, stderr = proc.communicate()
+    o = stdout.decode('utf-8')
+    e = stderr.decode('utf-8')
+    f.write(" ".join(co_cmd))
+    f.write("\n")
+    f.write("-"*40)
+    f.write("\n")
+    f.write(o)
+    f.write("\n")
+    f.write(e)
+    f.write("\n\n")
+    
+    f_cmd = ['git','fetch','--all']
+    subprocess.call(f_cmd, cwd=work_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    
+    stdout, stderr = proc.communicate()
+    o = stdout.decode('utf-8')
+    e = stderr.decode('utf-8')
+    f.write(" ".join(f_cmd))
+    f.write("\n")
+    f.write("-"*40)
+    f.write("\n")
+    f.write(o)
+    f.write("\n")
+    f.write(e)
+    f.write("\n\n")
+    time.sleep(5)
+    
+    pull_cmd = ['git','pull','cmr','master']
+    subprocess.call(pull_cmd, cwd=work_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    
+    stdout, stderr = proc.communicate()
+    o = stdout.decode('utf-8')
+    e = stderr.decode('utf-8')
+    f.write(" ".join(pull_cmd))
+    f.write("\n")
+    f.write("-"*40)
+    f.write("\n")
+    f.write(o)
+    f.write("\n")
+    f.write(e)
+    f.write("\n\n")
+    time.sleep(10)
+    
+    # Step 2:
+    # Restart Captain Hook pod
+    pod_restart = ['docker-compose','restart']
+    subprocess.call(pod_restart, cwd=pod_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    stdout, stderr = proc.communicate()
+    o = stdout.decode('utf-8')
+    e = stderr.decode('utf-8')
+    f.write(" ".join(pod_restart))
+    f.write("\n")
+    f.write("-"*40)
+    f.write("\n")
+    f.write(o)
+    f.write("\n")
+    f.write(e)
+    f.write("\n\n")
+
+

dotfiles/bluebear_scripts/git_init_www.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -x
+
+if [ "$(id -u)" != "0" ]; then
+    echo ""
+    echo ""
+    echo "This script should be run as root."
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+for SUB in pages bots hooks; do
+    REPOURL="https://git.charlesreid1.com/charlesreid1/${SUB}.charlesreid1.com.git"
+
+    mkdir -p /www/${SUB}.charlesreid1.com
+    sudo chown -R charles:charles /www/${SUB}.charlesreid1.com
+    sudo -H -u charles git -C /www/${SUB}.charlesreid1.com \
+            clone \
+            --recursive \
+            --separate-git-dir=git \
+            -b gh-pages \
+            $REPOURL htdocs
+done
+
+set +x
+

dotfiles/bluebear_scripts/git_pull_www.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -x
+
+if [ "$(id -u)" != "0" ]; then
+    echo ""
+    echo ""
+    echo "This script should be run as root."
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+for SUB in pages bots hooks; do
+    REPOURL="https://git.charlesreid1.com/charlesreid1/${SUB}.charlesreid1.com.git"
+
+    mkdir -p /www/${SUB}.charlesreid1.com
+    sudo chown -R charles:charles /www/${SUB}.charlesreid1.com
+
+    sudo -H -u charles git -C /www/${SUB}.charlesreid1com \
+            --git-dir=git --work-tree=htdocs \
+            pull origin gh-pages
+
+done
+
+set +x
+
+

dotfiles/bluebear_scripts/init_pages.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+
+repos="bots/b-apollo
+bots/b-captain-hook
+bots/b-ginsberg
+bots/b-milton
+bots/boring-mind-machine
+bots/b-rainbow-mind-machine
+docker/d-gitea
+docker/d-mediawiki
+docker/d-mysql
+docker/d-nginx-charlesreid1
+docker/d-nginx-subdomains
+charlesreid1/dont-sudo-pip
+docker/d-phpmyadmin
+docker/d-python-files
+docker/d-python-helium
+bots/embarcadero-mind-machine
+charlesreid1/git-commit-ectomy
+charlesreid1/github-heroku-attack-rabbits
+charlesreid1/git-subway-maps
+charlesreid1/how-do-i-heroku
+charlesreid1/how-do-i-pandoc
+charlesreid1/how-do-i-pelican
+charlesreid1/how-do-i-pyenv
+charlesreid1/how-do-i-snakemake
+bots/papyrus-mind-machine
+docker/pod-bots
+docker/pod-charlesreid1
+docker/pod-webhooks
+bots/rainbow-mind-machine
+bots/russian-rainbow-mind-machine
+charlesreid1/scurvy-knave-theme
+charlesreid1/search-demo-mkdocs-material
+charlesreid1/translate-yer-docs
+bots/uncle-archie
+charlesreid1/wisko-manual
+"
+
+for i in $repos; do
+
+    repourl="https://git.charlesreid1.com/${i}"
+
+    r=`echo ${i} | sed 's+.*\/\(.*\)$+\1+'`
+
+    echo "Now cloning repo ${r} = ${i}"
+
+    sudo -H -u charles git -C /www/pages.charlesreid1.com/htdocs \
+        clone \
+        --recursive \
+        --separate-git-dir=git.${r} \
+        -b gh-pages \
+        ${repourl} ${r}
+done
+
+#git.b-apollo
+#git.b-captain-hook
+#git.b-ginsberg
+#git.b-milton
+#git.boring-mind-machine
+#git.b-rainbow-mind-machine
+#git.d-gitea
+#git.d-mediawiki
+#git.d-mysql
+#git.d-nginx-charlesreid1
+#git.d-nginx-subdomains
+#git.dont-sudo-pip
+#git.d-phpmyadmin
+#git.d-python-files
+#git.d-python-helium
+#git.embarcadero-mind-machine
+#git.git-commit-ectomy
+#git.github-heroku-attack-rabbits
+#git.git-subway-maps
+#git.how-do-i-heroku
+#git.how-do-i-pandoc
+#git.how-do-i-pelican
+#git.how-do-i-pyenv
+#git.how-do-i-snakemake
+#git.papyrus-mind-machine
+#git.pod-bots
+#git.pod-charlesreid1
+#git.pod-webhooks
+#git.rainbow-mind-machine
+#git.russian-rainbow-mind-machine
+#git.scurvy-knave-theme
+#git.search-demo-mkdocs-material
+#git.translate-yer-docs
+#git.uncle-archie
+#git.waxing-gibbous-mind-machine
+#git.wisko-manual

dotfiles/bootstrap.sh

@@ -15,14 +15,18 @@ EXTRA_EXCLUDE=''
 
 function doIt() {
     rsync \
+        --exclude "bootstrap.sh" \
+        --exclude "bluebeard_scripts" \
+        --exclude "redbeard_scripts" \
+        --exclude "blackbeard_scripts" \
+        --exclude "bluebear_scripts" \
+        --exclude "jupiter_scripts" \
+        --exclude "krash_scripts" \
+        --exclude "rojo_scripts" \
+        --exclude "scripts" \
+        --exclude "diff_dotfiles.sh" \
         --exclude ".git" \
         --exclude ".gitignore" \
-        --exclude "bootstrap.sh" \
-        --exclude "scripts" \
-        --exclude "rojo_scripts" \
-        --exclude "jupiter_scripts" \
-        --exclude "blackbeard_scripts" \
-        --exclude "krash_scripts" \
         -avh --no-perms . ~;
     source ~/.bash_profile;
 }
@@ -30,6 +34,7 @@ function doIt() {
 if [ "$1" == "--force" -o "$1" == "-f" ]; then
     doIt;
 else
+    ./diff_dotfiles.sh
     read -p "This may overwrite existing files in your home directory. Are you sure? (y/n) " -n 1;
     echo "";
     if [[ $REPLY =~ ^[Yy]$ ]]; then
@@ -41,7 +46,8 @@ unset doIt;
 if [ -d "${PWD}/${HOSTNAME}_scripts" ]; then
     echo "Creating scripts link:"
     set -x
-    ln -fs ${PWD}/${HOSTNAME}_scripts ${HOME}/scripts
+    rm -fr ${HOME}/scripts
+    ln -fs ${PWD}/${HOSTNAME}_scripts/ ${HOME}/scripts
     set +x
 fi
 

dotfiles/diff_dotfiles.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+## The short version:
+#find . -type f -maxdepth 1 -name ".*" | xargs -t -n1 -I% diff % ~/%
+
+# The (more careful) loop version:
+for file in $(find . -type f -maxdepth 1 -name ".*"); do
+    if [[ -f ~/$file ]]; then
+        echo "------------"
+        echo "$file exists in home directory, calculating diff..."
+        $(which git) diff --no-index --exit-code ~/$file $file && echo "No differences in $file!"
+        # order is important - home file first, shows changes to be deleted in red
+    fi
+done
+

dotfiles/jupiter_scripts/Readme.md

@@ -0,0 +1,37 @@
+# Jupiter Scripts
+
+The main task run on Jupiter is scraping the wiki.
+
+Main driver:
+
+* `push_wiki.py` - This is the main entrypoint. This script will
+  first scrape pages for links and edits and populate 
+  that information in MongoDB. It will then compile a CSV file
+  for data visualization and commit that to the data repo at
+  <https://git.charlesreid1.com/data/charlesreid1-data>
+
+Main functions:
+
+* `wiki_history.py` - Create a database containing page history
+  data for the charlesreid1.com wiki
+
+* `wiki_graph.py` - Create a graph database with link data
+  for the charlesreid1.com wiki
+
+MediaWiki:
+
+* `user-config.py` - configuration for pywikibot
+
+* (notes on [family file](https://www.mediawiki.org/wiki/Manual:Pywikibot/Use_on_third-party_wikis))
+
+Graphs:
+
+* `graph.py` - Graph object, useful for dealing with page graphs.
+
+* `mongo_graph.py` - extended Graph object that has additional
+  methods for serializing the graph into something that can be
+  stored in a MongoDB database table.
+
+* `graph_algorithms.py` - Graph algorithms useful for analyzing the
+  wiki page graph.
+

dotfiles/jupiter_scripts/push_wiki.py

@@ -5,10 +5,14 @@ import tempfile
 import socket
 import subprocess
 from datetime import datetime
-import pandas as pd
 
-from wiki_history import page_history_database, page_history_to_csv
+from os.path import join
-from wiki_graph import graphdb, graphdb_to_json
+
+from wiki_history import edit_history_database
+from wiki_history import edit_history_to_csv
+
+from wiki_graph import graphdb
+from wiki_graph import graphdb_to_json
 
 
 """
@@ -40,19 +44,19 @@ def push_wiki():
 
     # Update the page history database
     dbg("- updating page history database")
-    page_history_database()
+    edit_history_database()
 
     # Extract page history data to CSV
     dbg("- extracting page history data")
-    page_history_to_csv(tmpdir)
+    edit_history_to_csv(tmpdir)
 
-    # Update the page graph database
+    ### # Update the page graph database
-    dbg("- updating page graph database")
+    ### dbg("- updating page graph database")
-    graphdb()
+    ### graphdb()
 
-    # Extract page graph to JSON
+    ### # Extract page graph to JSON
-    dbg("- extracting page graph json")
+    ### dbg("- extracting page graph json")
-    graphdb_to_json(tmpdir)
+    ### graphdb_to_json(tmpdir)
 
     # Git add/commit/push changes
     dbg("- push changes")
@@ -63,57 +67,33 @@ def push_changes(tmpdir):
     """
     Commit changes to data/wiki repo
     """
-    # check out the repo
+    # clone the charlesreid1 data repo
-    reponame = "wiki"
+    dbg("    - cloning charlesreid1 data repo")
+    reponame = "charlesreid1-data"
     repodir = tmpdir + "/" + reponame
     clonecmd =  ["git","clone"]
+    clonecmd += ["--recursive"]
     clonecmd += ["git@git.charlesreid1.com:data/%s.git"%(reponame)]
     clonecmd += [repodir]
     subprocess.call(clonecmd, cwd=tmpdir)
 
     # copy the page_edits.csv file to the repo
     edits_repopath = "page_edits.csv"
-    edits_cpcmd = ["/bin/cp","page_edits.csv",reponame+"/"+edits_repopath]
+    edits_cpcmd = ["/bin/cp","page_edits.csv", join(reponame,edits_repopath)]
     subprocess.call(edits_cpcmd, cwd=tmpdir)
 
-    # copy the page_graph.json file to the repo
+    ### # copy the page_graph.json file to the repo
-    graph_repopath = "page_graph.json"
+    ### graph_repopath = "page_graph.json"
-    graph_cpcmd = ["/bin/cp","page_graph.json",reponame+"/"+graph_repopath]
+    ### graph_cpcmd = ["/bin/cp","page_graph.json", join(reponame,graph_repopath)]
-    subprocess.call(graph_cpcmd, cwd=tmpdir)
+    ### subprocess.call(graph_cpcmd, cwd=tmpdir)
-
-    # add/commit/push
-    addcmd = ["git","add",edits_repopath,graph_repopath]
-    subprocess.call(addcmd, cwd=tmpdir+"/"+reponame)
-
-    commitcmd = ["git","commit",edits_repopath,graph_repopath,"-m","[SCRIPT] updating wiki page edit and page graph data."]
-    subprocess.call(commitcmd, cwd=tmpdir+"/"+reponame)
-
-    pushcmd = ["git","push","origin","master"]
-    subprocess.call(pushcmd, cwd=tmpdir+"/"+reponame)
-
-
-
-    # clone the data master repo
-    dbg("    - cloning data master repo")
-    reponame = "data"
-    repodir = tmpdir + "/" + reponame
-    clonecmd =  ["git","clone"]
-    clonecmd += ["--recursive"]
-    clonecmd += ["git@git.charlesreid1.com:data/data-master.git"]
-    clonecmd += [repodir]
-    subprocess.call(clonecmd, cwd=tmpdir)
-
-    # update the data
-    wikidatadir = repodir + "/wiki"
-    pullcmd = ["git","pull","origin","master"]
-    subprocess.call(pullcmd, cwd=wikidatadir)
 
     # add commit push
     dbg("    - git add")
     addcmd = ["git","add","git"]
     subprocess.call(addcmd, cwd=repodir)
 
-    commitcmd = ["git","commit","wiki","-m","[SCRIPT] updating to latest git data"]
+    ### commitcmd = ["git","commit",edits_repopath,graph_repopath,"-m","[push_wiki.py] updating charlesreid1 wiki edit data"]
+    commitcmd = ["git","commit",edits_repopath,"-m","[push_wiki.py] updating charlesreid1 wiki edit data"]
     dbg("    - git commit")
     subprocess.call(commitcmd, cwd=repodir)
 

dotfiles/jupiter_scripts/wiki_history.py

@@ -43,18 +43,23 @@ Database schema:
 """
 
 
-JUPITER_IP = '10.6.0.2'
+JUPITER_IP = '192.168.1.153'
 JUPITER_PORT = 27017
 
 
-def page_history_to_csv(tmpdir):
+def edit_history_to_csv(tmpdir):
+    """
+    Step 2:
+    Load edit history from database,
+    do a pandas groupby, and dump to csv.
+    """
 
     # Make connection to database
-    # Requires page_history database to be populated already
+    # Requires edit_history database to be populated already
-    # See https://charlesreid1.com:3000/wiki/charlesreid1-wiki-data
+    # Also see https://git.charlesreid1.com/wiki/charlesreid1-wiki-data
     client = MongoClient(JUPITER_IP,JUPITER_PORT)
     db = client['charlesreid1wiki']
-    collection = db['page_history']
+    collection = db['edit_history']
     
     # Extract timestamp and character count for revision 
     df = pd.DataFrame()
@@ -65,7 +70,7 @@ def page_history_to_csv(tmpdir):
             print(i+1)
     
         # If you want to stop early
-        if(i>300 and False):
+        if(i>300 and True):
             break
     
         # Very simple csv: timestamp and count
@@ -80,17 +85,18 @@ def page_history_to_csv(tmpdir):
 
 
 
-def page_history_database():
+def edit_history_database():
-    """Run the algorithm that iterates through
+    """
-    each page and each revision, creating a document
+    Step 1:
-    for each revision.
+    Iterate over every edit of every page on the wiki.
+    Create a MongoDB document for each edit.
     """
     N = 0
     sleepytime = 0.1
 
     # Get connection/database/collections objects
-    prefix = 'page_history'
+    prefix = 'edit_history'
-    client, db, page_history_collection = get_collection(prefix)
+    client, db, edit_history_collection = get_collection(prefix)
 
     # Get the site
     site = get_site()
@@ -120,10 +126,10 @@ def page_history_database():
             doc['count'] = len(rev.text)
 
             # Remove the old document
-            page_history_collection.delete_one({"_id": rev.sha1})
+            edit_history_collection.delete_one({"_id": rev.sha1})
 
             # Insert the new document
-            page_history_collection.insert_one(doc)
+            edit_history_collection.insert_one(doc)
 
         time.sleep(sleepytime)
 
@@ -150,8 +156,8 @@ def get_collection(collections_label):
     db = client['charlesreid1wiki']
 
     # Collections:
-    #   page_history
+    #   edit_history
-    #   page_history_meta
+    #   edit_history_meta
     col = db[collections_label]
 
     return client, db, col
@@ -176,8 +182,8 @@ def get_page_generator(s,max_items=0):
 def nuke():
     """Nuke everybody"""
     # Get connection/database/collections objects
-    client, db, page_history_collection = get_collections()
+    client, db, edit_history_collection = get_collections()
-    page_history_collection.drop()
+    edit_history_collection.drop()
     client.close()
 
 

dotfiles/krash_scripts/backup_gitea_docker_volume.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+# 
+# this container contains a handy volume-backup script:
+# https://github.com/loomchild/volume-backup
+#
+# usage:
+# docker run --rm -v some_volume:/volume -v /tmp:/backup loomchild/volume-backup backup some_archive
+# docker run --rm -v some_volume:/volume -v /tmp:/backup loomchild/volume-backup restore some_archive
+set -x
+set -e
+
+
+###########
+# Note:
+# unfortunately the path you specify
+# is always relative to /tmp, so
+# even when you specify an absolute path
+# it just ends up in /tmp.
+#
+# :massive_eye_roll: 
+#################
+
+GITEAVOL="podcharlesreid1_stormy_gitea_data"
+TS=$(date +"%Y-%m-%d")
+BACKUPDIR=/junkinthetrunk/backups/monthly/gitea_dockervolume_${TS}
+
+mkdir -p ${BACKUPDIR}
+
+# backup:
+docker run --rm -v ${GITEAVOL}:/volume -v /tmp:/backup loomchild/volume-backup backup ${BACKUPDIR}/gitea_snapshot
+
+set +x
+echo "Gitea volume ${GITEAVOL} backed up to file /tmp/${BACKUPDIR}/"
+
+# restore:
+#docker run --rm -v ${GITEAVOL}:/volume -v /tmp:/backup loomchild/volume-backup restore ${BACKUPDIR}/gitea_snapshot

dotfiles/krash_scripts/git_clone_www.sh

@@ -2,7 +2,7 @@
 
 REPOURL="https://git.charlesreid1.com/charlesreid1/charlesreid1.com.git"
 
-git -C /www/example.com \
+git -C /www/charlesreid1.com \
     clone \
     --separate-git-dir=git \
     -b gh-pages \

dotfiles/krash_scripts/iptables_ban_jerks.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# use iptables to ban jerks
+# from accessing the server
+
+if [ "$(id -u)" != "0" ]; then
+    echo ""
+    echo ""
+    echo "This script should be run as root."
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+# some incredibly obnoxious marketing spammer that was (is?) DDOSing me
+CIDR_IP="46.229.168.0/24"
+/sbin/iptables -A INPUT -s ${CIDR_IP} -j DROP
+
+# fail2ban
+for IPADDR in $(cat /var/log/fail2ban.log | /bin/grep "Ban " | sed 's/^.*Ban \(.*\)$/\1/g' | sort | uniq); do
+
+    /sbin/iptables -A INPUT -s ${IPADDR} -j DROP
+
+done
+

dotfiles/krash_scripts/s3_copy_dir.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+aws s3 --region us-west-2 cp --recursive /path/to/file s3://mah-bukkit/.

dotfiles/scripts/devtmux

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+if [ -z "$TMUX" ] && [ ${UID} != 0 ]
+then
+    # Adding -A and -s 'dev' means that
+    # we will try and reconnect to session
+    # named "dev" if it exists, and will
+    # only start a new "dev" session if 
+    # an existing one is not found.
+    tmux new-session -A -s 'dev' \; \
+      send-keys 'vim' C-m \; \
+      \
+      split-window -v -p 20 \; \
+      \
+      select-pane -t 1 \;
+
+    ## Example: ipython (80%) + top (20%)
+    #tmux new-session -A -s 'dev' \; \
+    #  send-keys 'ipython' C-m \; \
+    #  \
+    #  split-window -v -p 20 \; \
+    #  send-keys 'top' C-m \; \
+    #  \
+    #  select-pane -t 1 \;
+fi
+

dotfiles/scripts/fail.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+echo "o noes!!!"
+
+exit 1;

firewall/Readme.md

@@ -0,0 +1,10 @@
+# firewall
+
+Contains scripts for setting up a firewall using `ufw`.
+
+[initial setup of ubuntu server with ufw](https://www.digitalocean.com/community/tutorials/initial-server-setup-with-ubuntu-16-04)
+
+[ufw essentials](https://www.digitalocean.com/community/tutorials/ufw-essentials-common-firewall-rules-and-commands)
+
+[(older) how to set up a ufw firewall](https://www.digitalocean.com/community/tutorials/how-to-set-up-a-firewall-with-ufw-on-ubuntu-14-04)
+

firewall/bear_ufw.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Set up the ufw firewall for bear.
+# 22 - ssh
+# 80 - http
+# 443 - https
+# 8080, 8888, 8000 - for something to use
+
+if [ "$(id -u)" != "0" ]; then
+    echo ""
+    echo ""
+    echo "This script must be run as root!"
+    echo ""
+    echo ""
+    exit 1;
+fi
+
+set -x
+
+# Start by setting defaults on ufw
+ufw --force reset
+ufw default deny incoming
+ufw default allow outgoing
+ufw allow 22
+ufw allow 80
+ufw allow 443
+ufw --force enable
+
+# allow ufw to nat connections from
+# external interface to internal 
+# (docker) interfaces
+sed -i -e 's/DEFAULT_FORWARD_POLICY="DROP"/DEFAULT_FORWARD_POLICY="ACCEPT"/g' /etc/default/ufw
+ufw reload
+
+# enable packet masquerading so we can
+# get the real IP of clients inside
+# docker containers.
+iptables -t nat -A POSTROUTING ! -o docker0 -s 172.17.0.0/16 -j MASQUERADE
+

firewall/bespin_iptables_permissive.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -e
+
+ipt="sudo /sbin/iptables"
+
+# start by flushing all rules and setting defaults
+$ipt -F
+# should we do this?
+#$ipt -P INPUT DROP
+#$ipt -P FORWARD DROP
+$ipt -P INPUT ACCEPT
+$ipt -P FORWARD ACCEPT
+$ipt -P OUTPUT ACCEPT
+$ipt -t nat -F
+$ipt -t mangle -F
+$ipt -F
+$ipt -X
+
+$ipt -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
+
+##################################
+# PIA VPN Tunnels
+
+# These are PIA tunnels that handle traffic from APs
+PIA_AP_TUNNELS="tun1"
+for TUN in TUNNELS; do
+    # Accept all traffic coming in from tunnel
+    $ipt -A INPUT -i ${TUN} -j ACCEPT
+    # Masquaerade outgoing traffic leaving via the tunnel
+    $ipt -t nat -A POSTROUTING -o ${TUN} -j MASQUERADE
+done
+
+##################################
+# AP-PIA Tunneling
+
+# Forward outgoing traffic for APs through tunnel
+AP="wlan1"
+TUN="tun1"
+$ipt -A FORWARD -i ${AP} -o ${TUN} -j ACCEPT
+$ipt -A FORWARD -i ${TUN} -o ${AP} -m state --state ESTABLISHED,RELATED -j ACCEPT
+
+##################################
+# DNS Tunneling
+
+# Forward outgoing DNS traffic from lo:1 (PiHole) through PIA tunnel
+DNS="lo:1"
+TUN="tun1"
+PROTOCOLS="udp tcp"
+for PROTOCOL in $PROTOCOLS; do
+    # PiHole can always send DNS queries out through tunnel
+    $ipt -A FORWARD -p ${PROTOCOL} -i ${DNS} -o ${TUN} --dport 53 -j ACCEPT
+    # Responses to PiHole can always return via tunnel
+    $ipt -A FORWARD -p ${PROTOCOL} -i ${TUN} -o ${DNS} --dport 53 -m state --state ESTABLISHED,RELATED -j ACCEPT
+done
+
+# Enable logging
+$ipt -N LOGGING
+$ipt -A INPUT -j LOGGING
+$ipt -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "iptables dropped: " --log-level 4
+$ipt -A LOGGING -j DROP
+
+# Make rules persistent
+sudo netfilter-persistent save
+
+# Restore docker iptables rules
+sudo service docker restart

firewall/bespin_iptables_strict.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+set -e
+
+ipt="sudo /sbin/iptables"
+
+# Set default policies
+$ipt -P INPUT DROP
+$ipt -P FORWARD DROP
+$ipt -P OUTPUT ACCEPT
+
+# Flush and clear everything
+$ipt -t nat -F
+$ipt -t mangle -F
+$ipt -F
+$ipt -X
+
+# Name of PIA VPN tunnel device
+PIATUN="tun1"
+# Name of loopback interface for PiHole DNS server
+PHDNS="lo:1"
+# Name of loopback interface for dnsmasq DNS server
+DDNS="lo"
+# Name of hostapd AP device
+AP="wlan1"
+
+########### LOOPBACK ##########
+$ipt -A INPUT -i lo -j ACCEPT
+$ipt -A INPUT -i lo:1 -j ACCEPT
+
+########### INCOMING ##########
+# Allow any established connection to come in or out
+$ipt -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+$ipt -A OUTPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+
+########### PING ##############
+# Allow incoming ping requests
+$ipt -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
+
+########### SSH ###############
+# Allow incoming SSH sessions, new or established
+$ipt -A INPUT -p tcp --dport 22 -m conntrack --ctstate NEW,ESTABLISHED -j ACCEPT
+# Allow incoming SSH traffic, if part of established conversation
+$ipt -A INPUT -p tcp --sport 22 -m conntrack --ctstate ESTABLISHED -j ACCEPT
+
+########### VPN ###############
+# Allow incoming VPN sessions destined for 1194, new or established
+$ipt -A INPUT -p udp --dport 1194 -m conntrack --ctstate NEW,ESTABLISHED -j ACCEPT
+# Allow incoming VPN traffic coming from 1194, part of established conversation
+$ipt -A INPUT -p udp --sport 1194 -m conntrack --ctstate ESTABLISHED -j ACCEPT
+
+########### HTTP/HTTPS ########
+# Allow incoming HTTP/HTTPS traffic, part of established conversation
+$ipt -A INPUT -p tcp --sport 80 -m conntrack --ctstate ESTABLISHED -j ACCEPT
+$ipt -A INPUT -p tcp --sport 443 -m conntrack --ctstate ESTABLISHED -j ACCEPT
+
+########### DHCP ##############
+# Allow any DHCP traffic to come in or out
+$ipt -A INPUT  -p udp --dport 67:68 --sport 67:68 -j ACCEPT
+$ipt -A OUTPUT -p udp --dport 67:68 --sport 67:68 -j ACCEPT
+
+########### PIA VPN ##############
+# This is a PIA VPN tunnel that handles traffic from APs
+# Accept all traffic coming in from tunnel
+$ipt -A INPUT -i ${PIATUN} -j ACCEPT
+# Masquaerade outgoing traffic leaving via the tunnel
+$ipt -t nat -A POSTROUTING -o ${PIATUN} -j MASQUERADE
+
+########### DNS ###############
+PROTOCOLS="tcp udp"
+for prot in $PROTOCOLS; do
+    # General DNS Traffic:
+    # Allow incoming DNS traffic coming from 53, part of established conversation
+    $ipt -A INPUT  -p $prot --sport 53 --dport 1024:65535 -m state --state ESTABLISHED -j ACCEPT
+
+    # PiHole self-accept traffic from port 53
+    $ipt -A INPUT -p $prot -i ${PHDNS} --dport 53 -j ACCEPT
+
+    # PiHole DNS (lo:1) <-> PIA VPN Tunnel (tun0):
+    # PiHole can always send DNS queries out through tunnel
+    $ipt -A FORWARD -p $prot -i ${PHDNS} -o ${PIATUN} --dport 53 -j ACCEPT
+    # Responses to PiHole can always return via tunnel
+    $ipt -A FORWARD -p $prot -i ${PIATUN} -o ${PHDNS} --dport 53 -m state --state ESTABLISHED,RELATED -j ACCEPT
+
+    # dnsmasq DNS (lo) <-> PiHole DNS (lo:1)
+    # Allow all DNS traffic from local dnsmasq DNS server to local PiHole DNS server
+    $ipt -A FORWARD -p $prot -i ${DDNS} -o ${PHDNS} --dport 53 -j ACCEPT
+    # Allow responses to dnsmasq to return via the PiHole DNS server
+    $ipt -A FORWARD -p $prot -i ${PHDNS} -o ${DDNS} --dport 53 -m state --state ESTABLISHED,RELATED -j ACCEPT
+
+    # hostapd AP (wlan1) <-> dnsmasq DNS (lo)
+    # Allow DNS traffic to travel both ways between AP and dnsmasq
+    $ipt -A FORWARD -p $prot -i ${AP} -o ${DDNS} --dport 53 -j ACCEPT
+    $ipt -A FORWARD -p $prot -o ${AP} -i ${DDNS} --sport 53 -j ACCEPT
+done
+
+########### PIHOLE UI #########
+# 8080/8443
+
+## Enable logging
+#$ipt -N LOGGING
+#$ipt -A INPUT -j LOGGING
+#$ipt -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "iptables dropped: " --log-level 4
+#$ipt -A LOGGING -j DROP
+
+# Make rules persistent
+sudo netfilter-persistent save
+
+# Restore docker iptables rules
+sudo service docker restart

tasks/install/sudo_install_docker.sh

@@ -13,3 +13,4 @@ sudo curl -L https://github.com/docker/compose/releases/download/1.20.1/docker-c
 sudo chmod +x /usr/local/bin/docker-compose
 docker-compose --version
 
+sudo gpasswd -a charles docker