update .gitignore file to ignore custom/ entires

* 'master' of github.com:charlesreid1-docker/d-gitea:
  modify app.ini to give https clone addresses

.gitignore

@@ -1,3 +1,9 @@
 site
 *.secret
+data/
 custom/conf/app.ini
+custom/gitea.db
+custom/avatars
+custom/log/
+custom/public/
+custom/queues/

README.md

@@ -5,13 +5,65 @@ This is the gitea docker container used to run gitea on charlesreid1.com.
 You should not run this container by itself (see 
 [pod-charlesreid1](https://git.charlesreid1.com/docker/pod-charlesreid1.git)). 
 
+See the documentation here: <https://pages.charlesreid1.com/d-gitea/>
+
+## TODO
+
+Organize the documentation better, there's a lot going on
+
+* Setup
+* Repo organization
+* Jinja/ansible
+* Gitea files
+* New instance
+* Restoring instance
+* Backing up instance
+
+## Quick Start
+
+### Before you begin
+
+Check `docker-compose.yml` and ensure the bind mounting of folders is
+set up to match what you want. Currently:
+
+* `d-gitea/custom/` directory maps to `/data/gitea` in the container
+* `d-gitea/data/` directory maps to `/app/gitea/data`
+
+To change this, modify the `docker-compose.yml` file in this repo
+or the `docker-compose.yml.j2` file in the pod-charlesreid1 repo.
+
+### Create app.ini
+
+The `app.ini` file is not stored directly in this repo, only a template file
+`app.ini.j2` is stored. You need to create an `app.ini` file from the template
+before beginning.
+
+To create an `app.ini` file from the template, populate the secret files at:
+
+* `internal_token.secret`
+* `secret_key.secret`
+
+Then run the `make_app_ini.sh` script:
+
+```
+./make_app_ini.sh
+```
+
+This will create an `app.ini` file from the template at `custom/conf/app.ini.j2`,
+and will put the new file in `custom/conf/app.ini`.
+
+### Running
+
+Start the container with `docker-compose up` if running standalone, or by starting
+the docker pod if running in a pod.
+
 ## Links
 
-[documentation: d-gitea container](https://pages.charlesreid1.com/d-gitea/) (you are here)
+[documentation: d-gitea container](https://pages.charlesreid1.com/d-gitea/)
 
 [source code on git.charlesreid1.com: docker/d-gitea](https://git.charlesreid1.com/docker/d-gitea)
 
-[source code on github.com: charlesreid1-docker/d-gitea](https://github.com/charlesreid1-docker/d-gitea)
+[source code on github.com: charlesreid1-docker/d-gitea](https://github.com/charlesreid1-docker/d-gitea) (you are here)
 
 [gitea documentation - running gitea with docker](https://docs.gitea.io/en-us/install-with-docker/)
 

custom/conf/app.ini.j2

@@ -5,6 +5,7 @@
 ;; march 2017
 ;; https://github.com/go-gitea/gitea/blob/master/conf/app.ini
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
 APP_NAME = {{ gitea_app_name }}
 RUN_USER = git
 RUN_MODE = prod
@@ -23,18 +24,18 @@ ROOT               = /data/git/repositories
 PREFERRED_LICENSES = MIT License
 
 ; This gets rid of the HTTP option to check out repos...
-DISABLE_HTTP_GIT = true
+DISABLE_HTTP_GIT = false
 
 [server]
 PROTOCOL     = http
-DOMAIN       = {{ gitea_domain }}
+DOMAIN       = git.{{ server_name_default }}
 #CERT_FILE    = /www/gitea/certs/cert.pem
 #KEY_FILE     = /www/gitea/certs/key.pem
-SSH_DOMAIN   = {{ gitea_domain }}
+SSH_DOMAIN   = git.{{ server_name_default }}
 HTTP_PORT    = 3000
 HTTP_ADDR    = 0.0.0.0
-;ROOT_URL     = https://{{ gitea_domain }}
+ROOT_URL     = https://git.{{ server_name_default }}
-ROOT_URL     = %(PROTOCOL)s://%(DOMAIN)s:%(HTTP_PORT)s/
+;ROOT_URL     = %(PROTOCOL)s://%(DOMAIN)s:%(HTTP_PORT)s/
 DISABLE_SSH  = false
 ; port to display in clone url:
 SSH_PORT     = 222

custom/conf/apply_templates.py

@@ -0,0 +1,87 @@
+import os, re, sys
+import glob
+import subprocess
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+
+"""
+Apply Default Values to Jinja Templates
+
+
+This script applies default values to 
+templates in this folder.
+
+The templates are used by Ansible,
+but this script uses the same template
+engine as Ansible to apply template
+variable values to the template files
+and make real files.
+
+variables are:
+- `gitea_app_name` - name of gitea app (?)
+- `server_name_default` - name of server (e.g., charlesreid1.com)
+- `gitea_secret_key` - secret key for gitea
+- `gitea_internal_token` - internal token for gitea
+"""
+
+
+# Where templates live
+TEMPLATEDIR = '.'
+
+# Where rendered templates will go
+OUTDIR = '.'
+
+# Should existing (destination) files 
+# be overwritten if they exist?
+OVERWRITE = True
+
+# Template variables
+TV = {
+        'gitea_app_name':       'charlesreid1.red',
+        'server_name_default':  'charlesreid1.red',
+        'gitea_secret_key':     'abc123',
+        'gitea_internal_token': '123abc'
+}
+
+
+
+def apply_templates(template_dir, output_dir, template_vars, overwrite=False):
+    """Apply the template variables 
+    to the template files.
+    """
+
+    if not os.path.exists(output_dir):
+        msg = "Error: output dir %s does not exist!"%(output_dir)
+        raise Exception(msg)
+
+    if not os.path.exists(template_dir):
+        msg = "Error: template dir %s does not exist!"%(output_dir)
+        raise Exception(msg)
+
+    # Jinja env
+    env = Environment(loader=FileSystemLoader('.'))
+
+    # Render templates
+    tfile = 'app.ini.j2'
+    rfile = 'app.ini'
+
+    # Get rendered template content
+    content = env.get_template(tfile).render(**template_vars)
+
+    # Write to file
+    dest = os.path.join(output_dir,rfile)
+    if os.path.exists(dest) and overwrite is False:
+        msg = "Error: template rendering destination %s already exists!"%(dest)
+        raise Exception(msg)
+
+    with open(dest,'w') as f:
+        f.write(content)
+
+    print("Rendered the following templates:%s\nOutput files:%s\n"%(
+            "\n- "+os.path.join(output_dir,tfile),
+            "\n- "+os.path.join(output_dir,rfile)
+    ))
+
+
+if __name__=="__main__":
+    apply_templates(TEMPLATEDIR,OUTDIR,TV,OVERWRITE)
+

custom/templates/home.tmpl

@@ -40,7 +40,7 @@ var formatStuff = d3.format(",");
 /*
 TEH COLORRRZZZZ
 */
-var realBackgroundColor = "#fff";//"#272b30";
+var realBackgroundColor = "#383c4a";//"#272b30";
 var tileBackgroundColor = realBackgroundColor;//"#3a3a3a";
 var tileStrokeColor     = "#ccc";
 var monthStrokeColor    = "#aaa";

docker-compose.yml

@@ -15,7 +15,8 @@ services:
       - gitea
     volumes:
       - "stormy_gitea_data:/data"
-       - "./custom:/data/gitea"
+      - "./custom:/data/gitea"
+      - "./data:/app/gitea/data"
     ports:
       - "3000:3000"
       - "222:22"

docs/index.md

@@ -14,30 +14,53 @@ Docker files for running gitea
 ## Table of Contents
 
 * secrets
+* app.ini file
+* jinja templates
 * container directory structure
+* container data volume
+* git repositories
 * files mounted into container
 * using the `docker-compose.yml` file
+* notes on custom files
 * configuring gitea with `app.ini`
 * customizing gitea with custom files
 * backing up and restoring gitea
 
 ## Secrets
 
-There are two secrets to set in the `app.ini` before running gitea:
+There are two secrets that must be set in the Gitea configuration file:
 the internal token and the secret key.
 
-These can be set in `*.secret` files:
+The two secret values can be placed in two `*.secret` files:
 
 ```plain
 internal_token.secret
 secret_key.secret
 ```
 
-The contents should be the value of the variable 
+The contents should be the value of the secret variable 
 you wish to use in `custom/conf/app.ini`.
 
 These files are not tracked by git.
 
+## Jinja Templates
+
+The `app.ini` script is not stored directly in this repo, since it contains
+sensitive information. Instead, we store a Jinja template, `app.ini.j2`, which
+is rendered into a real `app.ini` file after variable substitutions, etc.
+
+Normally, we use this repo with Ansible, so we don't deal with the Jinja template
+ourselves.
+
+If you want to render the Jinja template into a real config file without using
+Ansible, use the `make_app_ini.sh` script:
+
+```
+$ ./make_app_ini.sh
+```
+
+(This requires the two secret files mentioned above to be present.)
+
 ## Container Directory Structure
 
 The `custom/` dir in this folder maps to the `/data/gitea` volume
@@ -56,6 +79,8 @@ On the host machine, you can access named data volumes at
 `/var/lib/docker/volumes/gitea_gitea/_data`
 or copy files in and out of the container using `docker cp`.
 
+## Git Repositories
+
 ### directory structure before adding repos to gitea
 
 Directory structure for host-mounted gitea directory
@@ -149,54 +174,38 @@ gitea
 25 directories, 29 files
 ```
 
+### Backing up git repositories
+
+Backing up git repositories associated with the gitea instance
+is a separate step from gitea dump (see above).
+
+Before backing up repositories, perform a gitea dump without the
+repositories included.
+
+All git repositories are stored in `/data/git/repositories/`. They are stored
+in the subdirectory `org-name/repo-name`.
+
+To back up all repositories, copy the folder `/data/git/repositories` from the
+gitea container.
+
+To restore all repositories, copy the folder `/data/git/repositories` into the
+gitea container.
+
 ## Files Mounted Into Container
 
-### `custom/conf` configuration file
+The `d-gitea/custom/` directory is mounted into the container at `/data/gitea`.
 
-The conf dir contains configuration files to configure how gitea works.
+The `d-gitea/data/` directory is mounted into the container at `/app/gitea/data`.
 
-The `app.ini` file needs to contain two secrets,
+To make the custom configuration file, follow the instructions mentioned in
-which are scrubbed in `app.ini.sample`.
+the "Secrets" section.
 
-The two secrets that are needed are:
 
-* an "internal token" secret, contained in `internal_token.secret`
-* a "secret key" secret, contained in `secret_key.secret`
-
-Use the `make_app_ini.sh` script to add the two secrets to the document.
-This will use sed to find/replace instances of the scrubbed secret,
-and will output the result to `custom/conf/app.ini`.
-
-```plain
-./make_app_ini.sh
-```
-
-This generates `custom/conf/app.ini`.
-
-When the container is run, this file will be at `/data/gitea/conf/app.ini`.
-
-## `custom/templates` template files
-
-The templates directory contains template files. These are gitea templates that 
-control how particular kinds of gitea pages look. For example, a template can
-be used to modify how the user page looks, or modify the layout of repository
-main pages.
-
-In the container, this will be at `/data/gitea/templates/`.
-
-## `custom/pages` gitea pages
-
-The pages directory contains one-off pages or static content that is 
-hosted by gitea at the same domain (git.charlesreid1.com) but 
-not necessarily incorporated into the gitea site.
-
-For example, a custom "about me" page could be added as a static .html file,
-and it would be hosted at `git.charlesreid1.com/about`.
-
-In the container, this will be at `/data/gitea/pages/`.
 
 ## Using the `docker-compose.yml` File
 
+### Standalone
+
 This directory contains a docker-compose file that can be used to run 
 a gitea server on port 3000.
 
@@ -212,43 +221,172 @@ structure. See below for more info.
 
 Use this as a project seed to add gitea containers to other docker pods.
 
+### pod-charlesreid1
+
+The main use of this repo is as a submodule in
+[pod-charlesreid1](https://github.com/charlesreid1-docker/pod-charlesreid1.git).
+
+This pod is set up by Ansible, which integrates well with the Jinja template approach.
+
 ## Notes on Custom Files
 
-The settings in the app.ini file are documented [here](https://docs.gitea.io/en-us/config-cheat-sheet/).
+The settings in the `app.ini` file are documented [here](https://docs.gitea.io/en-us/config-cheat-sheet/).
 
-An extensive sample app.ini file is [here](https://github.com/go-gitea/gitea/blob/master/custom/conf/app.ini.sample) (WARNING: this broke gitea).
+An extensive sample `app.ini` file is [here](https://github.com/go-gitea/gitea/blob/master/custom/conf/app.ini.sample) (WARNING: this broke gitea).
 
 The existing gitea templates are in the gitea repo under [templates/](https://github.com/go-gitea/gitea/tree/master/templates).
 These can be modified as needed and placed in the `custom/templates/` directory.
 
-## Backing Up and Restoring Gitea
+### custom templates directory
 
-Fortunately, gitea provides dump functionality.
+The templates directory `d-gitea/custom/templates` contains template files. 
+These are gitea templates that control how particular kinds of gitea pages look.
+For example, a template can be used to modify how the user page looks, or modify
+the layout of repository main pages.
 
-Unfortunately, there is no restore functionality.
+In the container, this will be at `/data/gitea/templates/`.
 
-See [pod-charlesreid1/utils-gitea](https://git.charlesreid1.com/docker/pod-charlesreid1/src/branch/master/utils-gitea)
+### custom gitea pages
-for proper backup/restore scripts.
 
-### Executive Summary
+The custom pages directory `d-gitea/custom/pages` contains one-off pages or static
+content that is hosted by the gitea instance, but not necessarily incorporated into
+the gitea site.
 
-Backup:
+For example, a custom "about me" page could be added as a static .html file,
+and it would be hosted at `git.charlesreid1.com/about`.
 
-* create a backup target directory in the container
+In the container, this will be at `/data/gitea/pages/`.
-* create a gitea dump zip file using `gitea dump` command
-* create a gitea avatars zip file
-* copy everything in the backup target directory out of the container
-* remove the backup target directory
 
-Restore:
 
-* create a restore target directory in the container
+## Backing up and restoring gitea
-* copy gitea dump and gitea avatars zip files into restore target dir
+
-* unpack dump zip, unpack avatars zip
+Running `gitea dump` command will dump files required for restoring an existing
-* unzip repositories zip (contained in dump zip)
+Gitea instance. Unfortunately, gitea's backup and restore functionality is an
-* restore `repositories/` folder to `/data/git/repositories/`
+absolute dumpster fire.
-* (skip restoring `custom/` files, version control takes care of that)
+
-* restore sqlite database using sql database dump
+We do our best to walk through the process, but here is a summary:
-* restore avatars
+* On the old system:
-* remove the restore target directory
+    * Create a backup file `gitea-dump.zip` using the very specific `gitea dump` incantation that works
+    * Move the backup file out of the gitea container
+* On the new system:
+    * Unzip the backup file `gitea-dump.zip`
+    * Put appropriate files in appropriate location
+    * Use the table below to determine where in the repo the gitea dump files should go,
+      and where they will be available inside the gitea container
+
+### Creating a Backup
+
+To create a gitea dump, connect to the docker container and get a bash shell as the user
+`git` via the docker exec command:
+
+```
+# connect to gitea container
+docker exec -it --user git name_of_gitea_container /bin/bash
+```
+
+This will give you a bash shell as the user `git`. Now create a gitea dump file
+(the gitea dump command requires you to be in `/app/gitea`, this assumes that the
+gitea executable is at the default location of `/app/gitea/gitea`):
+
+```
+# necessary
+cd /app/gitea
+# create gitea dump
+/app/gitea/gitea dump --file gitea-dump.zip --skip-repository
+```
+
+**IMPORTANT: The `--skip-repositories` flag means we are making the gitea dump
+way, way, way smaller, but it also means we need to back up and restore the
+repositories folder ourselves! (See below for instructions.)**
+
+Now copy the file out of the container, then remove it from the container:
+
+```
+# copy gitea dump out of container
+docker cp name_of_gitea_container:/app/gitea/gitea-dump.zip .
+
+# remove gitea dump
+docker exec -it name_of_gitea_container rm /app/gitea/gitea-dump.zip
+```
+
+### Contents of Dump File
+
+When the gitea dump file is unzipped, it will create the following files:
+
+* `app.ini`
+* `custom/` directory
+* `log/` directory (useless duplicate, already contained in `custom/` dir)
+* `data/` directory
+
+These files should map to the following locations in the docker container
+running the live gitea instance:
+
+```
+gitea dump file:        gitea live instance:
+----------------        --------------------
+app.ini                 /data/gitea/conf/app.ini
+custom/                 /data/gitea/
+log/                    (useless duplicate of custom/log/)
+data                    /app/gitea/data
+```
+
+### Restoring a Backup
+
+To restore a backup, copy the following files from the gitea dump
+to the following locations inside this repository:
+
+```
+gitea dump file:        d-gitea repo location:
+----------------        ----------------------
+app.ini                 d-gitea/custom/conf/app.ini
+custom/*                d-gitea/custom/*
+data                    d-gitea/data
+```
+
+(If you're running pod-charlesreid1, put these files in the
+specified location in the `d-gitea` submodule.)
+
+### Restoring repositories directory
+
+Note that when we created the gitea dump, we excluded the repositories themselves.
+This is because these will greatly inflate the size of our gitea dump and will make
+it much more difficult to store our backup files.
+
+Repository contents can be backed up separately as follows:
+
+* Log in to the old server
+* Back up the `/data/git/repositories` directory (copy and compress)
+* Copy the backup to the new server
+* Log in to the new server
+* Mount the `/data/git/repositories` folder
+
+Optionally, if you want to keep the repositories folder in its own location,
+modify `docker-compose.yml` to add the following line to the `gitea` container's
+`volumes` configuration:
+
+```
+services:
+  server:
+    image: gitea/gitea:latest
+
+    ...
+
+    volumes:
+      - "/path/to/repositories:/data/git/repositories"
+    
+    ...
+```
+
+This should make it easier to manage, back up, and restore the repositories folder.
+
+### Database backups
+
+We opt for the SQLite backend for gitea, which means the database
+is kept in a flat file on disk called `/data/gitea/gitea.db`.
+
+The location of this file and the format of the database are specified
+in the config file in `d-gitea/custom/conf/app.ini`.
+
+This file should not be edited, instead change the Jinja template
+`d-gitea/custom/conf/app.ini.j2` and remake `app.ini` from the template.
 

make_app_ini.sh

@@ -4,6 +4,6 @@ set -x
 # secret_key and internal_token should be SIMPLE or this 
 # gets screwy, and I don't want to do any more debugging.
 # 
-cat app.ini.sample \
+cat custom/conf/app.ini.sample \
   | sed 's/REPLACEME_INTERNALTOKEN_SECRET/'$(cat internal_token.secret)'/g' \
   | sed 's/REPLACEME_SECRETKEY_SECRET/'$(cat secret_key.secret)'/g'  > custom/conf/app.ini