Prepare release v2 - more fixes and refactoring + prepare release notes

.env.sample

@@ -1,3 +1,6 @@
+# Internal settings (they will not be passed to running services)
+CHECK_FOR_OUTDATED_CONFIG=true
+
 # General Traefik (reverse proxy) settings
 TRAEFIK_DOMAIN=mydomain.com
 ACME_MAIL=my-email@my-provider.com

.gitignore

@@ -10,4 +10,5 @@ samples/custom*/*.yaml
 
 config.yaml
 services/custom/*.yaml
-services/custom/*.yml
+services/custom/*.yml
+services/generated/*.yaml

UPGRADE_V2.md

@@ -0,0 +1,73 @@
+# Upgrade to V2
+
+Seedbox version 2 is here!
+Since there are some breaking changes and a configuration structure migration, a major version was mandatory.
+
+> These releases notes are still a work-in-progress as V2 is not fully tested and finalized yet.
+
+## What's new?
+
+* Configuration change to new YAML format
+  * Run ``config-updater.sh`` to migrate your old services.conf to the new config.yaml format.
+  * jq (v1.5+) and yq (v4+) are now requirements
+  * Easier feature switches
+  * If a service is missing, it won't be enabled by default like before. The config is now more declarative.
+  * Traefik routing rules are now dynamically generated in a file in Traefik config directory, so no more Docker labels. They became hard to maintain due to all possibilities caused by VPN support or custom files for example.
+* VPN support
+  * With ``gluetun`` service, you can now place any service behind a VPN.
+  * Default gluetun configuration is Wireguard "custom" mode, but see below...
+* Support custom services and docker-compose yaml files
+  * Place a docker-compose.yaml file in ``services/custom/`` directory, add a service in your config.yaml specifying a ``customFile``, and you are set.
+  * Support Plex hardware transcoding using a custom-file, already available in the ``services`` directory (just specify a customFile on plex service - see [config.sample.yaml](config.sample.yaml)).
+* Support arbitrary Traefik rules
+  * Place a Traefik YAML in ``samples/custom-traefik/`` directory, it will be copied in the Traefik configuration folder.
+  * Ideal to forward traffic to other services which do not belong to this seedbox.
+* New services:
+  * ``Gluetun``: [VPN client (see above)](https://github.com/qdm12/gluetun)
+  * ``Heimdall``: [Dashboard](https://github.com/linuxserver/Heimdall)
+  * ``Readarr``: [Ebook and comic monitor](https://github.com/Readarr/Readarr)
+  * ``Komga``: [Comic Book Manager](https://github.com/gotson/komga)
+  * ``Kavita``: [Comic / Book Manager](https://github.com/Kareadita/Kavita)
+
+And also:
+
+* ``update-all.sh`` is now called ``run-seedbox.sh`` but its purpose is the same.
+* More checks in ``run-seedbox.sh``. For example, throws an error if Flood is enabled but not Deluge, or if VPN is enabled on a service but the VPN client is not.
+* You can now specify where your data lives on your host through new environments variables (see [.env.sample](.env.sample)).
+  * This change is backward-compatible as the run-seedbox.sh script will default to the old "/data/torrents" and "/data/config" paths if these variables are not set.
+* ``networks:`` section is now aligned with the new docker compose syntax
+* Nextcloud-db has moved. It is now in /data/config (see below how to mitigate the errors).
+* Disable Traefik access logs
+
+## How to migrate
+
+```sh
+./config-updater.sh
+# Check the content of your .env file (in comparison with .env.sample which brings new variables)
+./run-seedbox.sh
+```
+
+When everything runs smoothly, you can delete your old configuration file:
+
+```sh
+rm -f services.conf
+```
+
+> Also, please make sure you have read the next section about Nextcloud Database location.
+
+## Nextcloud-db has moved
+
+Since commit e4ede925a8ce09b177206f30487a889da9e10334, nextcloud-db directory (mapped on /var/lib/mysql) has moved from
+``/data/nextcloud-db`` to ``$HOST_CONFIG_PATH/nextcloud-db`` (*/data/config/nextcloud-db by default*).
+
+To ensure a smooth transition, you will have to move the directory nextcloud-db into the correct new location, then run some commands to fix the schema:
+
+```sh
+mv /data/nextcloud-db/ /data/config/
+./run-seedbox.sh
+source .env
+docker exec -it nextcloud-db mysql_upgrade -u root -p${MYSQL_ROOT_PASSWORD}
+docker restart nextcloud nextcloud-db
+```
+
+Ensure everything runs nicely by looking at nextcloud-db and nextcloud logs, and by accessing your Nextcloud web UI.

config-updater.sh

@@ -11,17 +11,14 @@ cleanup_on_exit() {
 }
 trap cleanup_on_exit EXIT
 
-# Check that jq is installed
+# Load common functions
-if ! which jq >/dev/null; then
+source config/tools.sh
-  echo "[$0] jq does not exist. Install it from here: https://stedolan.github.io/jq/download/"
-  echo "[$0] Also, please make sure it is in the PATH."
-  exit 1
-fi
 
-# Check that yq is installed
+# Check that required tools are installed
-if ! which yq >/dev/null; then
+check_utilities
-  echo "[$0] yq does not exist. Install it from here: https://github.com/mikefarah/yq/releases"
+
-  echo "[$0] Also, please make sure it is in the PATH."
+if [[ ! -f services.conf ]]; then
+  echo "[$0] ERROR. Could nof find services.conf. Exiting."
   exit 1
 fi
 
@@ -87,6 +84,10 @@ cat services.conf | while read line || [[ -n $line ]]; do
           defaultHttpAuth="true"
           ;;
       esac
+      # Define scheme // For nextcloud, scheme must be https
+      scheme="http"
+      [[ $key == "nextcloud" ]] && scheme="https"
+      
       # Define service default port from bundled config file
       internalPort=$(cat config/ports | { grep $key || true; } | sed -r "s/^${key}: (.*)$/\1/")
       rules=$(jq -n '[
@@ -94,6 +95,7 @@ cat services.conf | while read line || [[ -n $line ]]; do
             "host": "'"$key"'.'$(echo '${TRAEFIK_DOMAIN}')'",
             "httpAuth": '"${defaultHttpAuth}"',
             "internalPort": '"${internalPort}"',
+            "scheme": '"${scheme}"'
           }
         ]')
       ;;

config.sample.yaml

@@ -16,7 +16,10 @@ services:
         - host: deluge.${TRAEFIK_DOMAIN}
           httpAuth: true
           internalPort: 8112
-          scheme: http
+          # Uncomment to specify custom schme (http by default)
+          # scheme: http
+          # Uncomment to *NOT* generate LetsEncrypt certificate (useful for local domains)
+          # disableCertificateGeneration: true
   - name: flood
     enabled: true
     vpn: false
@@ -31,6 +34,8 @@ services:
     vpn: false
     # uncomment to use custom file with specific parameters for hardware transcoding
     # customFile: plex-hardware-transcoding.yaml
+    # You can also place you own file in services/custom/ and call it here like this:
+    # customFile: custom/my_own_file.yaml
     traefik:
       enabled: true
       rules:
@@ -89,7 +94,7 @@ services:
           httpAuth: true
           internalPort: 6767
   - name: lidarr
-    enabled: true
+    enabled: false
     vpn: false
     traefik:
       enabled: true
@@ -197,7 +202,7 @@ services:
           httpAuth: true
           internalPort: 19999
   - name: duplicati
-    enabled: true
+    enabled: false
     vpn: false
     traefik:
       enabled: true
@@ -220,7 +225,7 @@ services:
     traefik:
       enabled: true
       rules:
-        - host: heimdall.${TRAEFIK_DOMAIN}
+        - host: ${TRAEFIK_DOMAIN}
           httpAuth: true
           internalPort: 80
   - name: gluetun

config/tools.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+##############################################################################
+############################### UTIL FUNCTIONS ###############################
+##############################################################################
+
+check_utilities () {
+  # Check that jq is installed
+  if ! which jq >/dev/null; then
+    echo "[$0] jq does not exist. Install it from here: https://stedolan.github.io/jq/download/"
+    echo "[$0] Please install jq version 1.5 or above."
+    echo "[$0] Also, please make sure it is in the PATH."
+    exit 1
+  fi
+
+  # Check that yq is installed
+  if ! which yq >/dev/null; then
+    echo "[$0] yq does not exist. Install it from here: https://github.com/mikefarah/yq/releases"
+    echo "[$0] Please install yq version 4 or above."
+    echo "[$0] Also, please make sure it is in the PATH."
+    exit 1
+  fi
+}

notes.md

@@ -1,16 +0,0 @@
-# Nextcloud-db has moved
-
-Since commit e4ede925a8ce09b177206f30487a889da9e10334, nextcloud-db directory (mapped on /var/lib/mysql) has moved from
-``/data/nextcloud-db`` to ``$HOST_CONFIG_PATH/nextcloud-db`` (*/data/config/nextcloud-db by default*).
-
-To ensure a smooth transition, you will have to move the directory nextcloud-db into the correct new location, then run some commands to fix the schema:
-
-```sh
-mv /data/nextcloud-db/ /data/config/
-./update-all.sh
-source .env
-docker exec -it nextcloud-db mysql_upgrade -u root -p${MYSQL_ROOT_PASSWORD}
-docker restart nextcloud nextcloud-db
-```
-
-Ensure everything runs nicely by looking at nextcloud-db and nextcloud logs, and by accessing your Nextcloud web UI.

run-seedbox.sh

@@ -2,6 +2,12 @@
 
 set -e
 
+# Load common functions
+source config/tools.sh
+
+# Check that required tools are installed
+check_utilities
+
 SKIP_PULL=0
 DEBUG=0
 
@@ -26,11 +32,19 @@ cleanup_on_exit() {
 trap cleanup_on_exit EXIT
 
 echo-debug() {
-  [[ ${DEBUG} == "1" ]] && echo "$@"
+  if [[ ${DEBUG} == "1" ]]; then echo "$@"; fi
 }
 
-# Cleanup files before start, in case there was a change we start from scratch at every script execution
+###############################################################################################
-rm -f config/*-vpn.yaml
+####################################### Load variables ########################################
+###############################################################################################
+
+echo "[$0] ***** Checking environment variables and files... *****"
+
+if [[ ! -f .env ]]; then
+  echo "[$0] ERROR. \".env\" file not found. Please copy \".env.sample\" and edit its values. Then, re-run this script."
+  exit 1
+fi
 
 # Create/update http_auth file according to values in .env file
 source .env
@@ -49,21 +63,24 @@ if [[ ! -f config.yaml ]]; then
   cp config.sample.yaml config.yaml
 fi
 
-# Alert in case new services have been added (or removed) in sample but active file has not changed
-# TODO: adapt to new config structure
-# NB_SERVICES_ACTIVE=$(cat services.conf | wc -l)
-# NB_SERVICES_ORIG=$(cat services.conf.sample | wc -l)
-# if [[ ${NB_SERVICES_ACTIVE} != ${NB_SERVICES_ORIG} ]]; then
-#   echo "[$0] Your services.conf file seems outdated. It appears there are new services available, or services that have been removed."
-#   diff -yt services.conf services.conf.sample || true
-#   echo ""
-# fi
-
 ###############################################################################################
 ###################################### Pre-flight checks ######################################
 ###############################################################################################
+
+echo "[$0] ***** Checking configuration... *****"
+
 yq eval -o json config.yaml > config.json
 
+if [[ ${CHECK_FOR_OUTDATED_CONFIG} == true ]]; then
+  nb_services=$(cat config.json | jq '.services | length')
+  nb_services_sample=$(yq eval -o json config.sample.yaml | jq '.services | length')
+  if [[ $nb_services_sample -gt $nb_services ]]; then
+    echo "[$0] There are more services in the config.sample.yaml than in your config.yaml"
+    echo "[$0] You should check config.sample.yaml because it seems there are new services available for you:"
+    diff -u config.yaml config.sample.yaml | grep "name:" | grep -E "^\+" || true
+  fi
+fi
+
 # Check if some services have vpn enabled, that gluetun itself is enabled
 nb_vpn=$(cat config.json | jq '[.services[] | select(.enabled==true and .vpn==true)] | length')
 gluetun_enabled=$(cat config.json | jq '[.services[] | select(.name=="gluetun" and .enabled==true)] | length')
@@ -83,6 +100,7 @@ if [[ $(cat config.json | jq '[.services[] | select(.name=="flood" and .enabled=
     echo "[$0] ******* Exiting *******"
     exit 1
   fi
+  # Determine deluge hostname (for flood) based on the VPN status (enabled or not) of deluge
   if [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==true)] | length') -eq 1 ]]; then
     export DELUGE_HOST="gluetun"
   elif [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==false)] | length') -eq 1 ]]; then
@@ -99,11 +117,10 @@ if [[ $(cat config.json | jq '[.services[] | select(.name=="flood" and .enabled=
       echo "[$0] Consider setting FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON variable to false in .env file."
     fi
   fi
-
 fi
 
 # Apply other arbitrary custom Traefik config files
-rm -f $f traefik/custom/dynamic-*
+# rm -f $f traefik/custom/dynamic-*
 for f in `find samples/custom-traefik -maxdepth 1 -mindepth 1 -type f | grep -E "\.yml$|\.yaml$" | sort`; do
   echo "[$0] Applying custom Traefik config $f..."
   cp $f traefik/custom/dynamic-$(basename $f)
@@ -121,6 +138,12 @@ fi
 ###############################################################################################
 ####################################### SERVICES PARSING ######################################
 ###############################################################################################
+
+echo "[$0] ***** Generating configuration... *****"
+
+# Cleanup files before start, in case there was a change we start from scratch at every script execution
+rm -f services/generated/*-vpn.yaml
+
 ALL_SERVICES="-f docker-compose.yaml"
 
 # Parse the config.yaml master configuration file
@@ -153,10 +176,10 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do
   # go through gluetun (main vpn client service).
   if [[ ${vpn} == "true" ]]; then
     echo "services.${name}.network_mode: service:gluetun" > ${name}-vpn.props
-    yq -p=props ${name}-vpn.props > config/${name}-vpn.yaml
+    yq -p=props ${name}-vpn.props > services/generated/${name}-vpn.yaml
     rm -f ${name}-vpn.props
     # Append config/${name}-vpn.yaml to global list of files which will be passed to docker commands
-    ALL_SERVICES="${ALL_SERVICES} -f config/${name}-vpn.yaml"
+    ALL_SERVICES="${ALL_SERVICES} -f services/generated/${name}-vpn.yaml"
   fi
 
   ###################################### TRAEFIK RULES ######################################
@@ -176,9 +199,9 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do
     host=$(echo $rule | jq -r .host)
     internalPort=$(echo $rule | jq -r .internalPort)
     httpAuth=$(echo $rule | jq -r .httpAuth)
-    echo-debug "[$0]      Host ${host}"
+    echo-debug "[$0]      Host => ${host}"
-    echo-debug "[$0]      Internal Port ${internalPort}"
+    echo-debug "[$0]      Internal Port => ${internalPort}"
-    echo-debug "[$0]      Http Authentication ${httpAuth}"
+    echo-debug "[$0]      Http Authentication => ${httpAuth}"
 
     # If VPN => Traefik rule should redirect to gluetun container
     backendHost=${name}
@@ -205,6 +228,11 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do
       echo "http.routers.${ruleId}.service: ${ruleId}" >> rules.props
     fi
 
+    disableCertificateGeneration=$(echo $rule | jq -r .disableCertificateGeneration)
+    if [[ ${disableCertificateGeneration} == true ]]; then
+      echo "http.routers.${ruleId}.tls: EMPTYMAP" >> rules.props
+    fi
+
     # If the specified service does not contain a "@" => we create it
     # If the service has a @, it means it is defined elsewhere so we do not create it (custom file, @internal...)
     if echo ${traefikService} | grep -vq "@"; then
@@ -215,10 +243,22 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do
 done
 
 # Convert properties files into Traefik-ready YAML and place it in the correct folder loaded by Traefik
+mv traefik/custom/dynamic-rules.yaml traefik/custom/dynamic-rules-old.yaml
 yq -p=props rules.props > traefik/custom/dynamic-rules.yaml
 rm -f rules.props
 
-# echo ${ALL_SERVICES}
+# Post-transformations on the rules file
+sed -i "s/EMPTYMAP/{}/g" traefik/custom/dynamic-rules.yaml
+sed -i --regexp-extended "s/^(.*: )(Host.*$)/\1'\2'/g" traefik/custom/dynamic-rules.yaml
+sed -i --regexp-extended "s/^(.*url: )(.*$)/\1\"\2\"/g" traefik/custom/dynamic-rules.yaml
+
+rm -f traefik/custom/dynamic-rules-old.yaml
+
+echo-debug "[$0] Here is the list of all files which are going to be processed: ${ALL_SERVICES}"
+
+echo "[$0] ***** Config OK. Launching services... *****"
+
+exit 0
 
 if [[ "${SKIP_PULL}" != "1" ]]; then
   echo "[$0] ***** Pulling all images... *****"

services.conf.sample

@@ -9,8 +9,8 @@ prowlarr: enable
 sonarr: enable
 radarr: enable
 bazarr: enable
-lidarr: enable
+lidarr: disable
-readarr: false
+readarr: disable
 komga: disable
 kavita: disable
 ombi: disable
@@ -18,11 +18,11 @@ overseerr: enable
 tautulli: enable
 jdownloader: enable
 jdownloader-vpn: disable
-tdarr: enable
+tdarr: disable
 nextcloud: enable
 portainer: enable
 netdata: enable
-duplicati: enable
+duplicati: disable
 syncthing: disable
-heimdall: disable
+heimdall: enable
 gluetun: disable

services/generated/README.md

@@ -0,0 +1,3 @@
+# Generated files
+
+This directory contains all generated overrides files. They are in .gitignore.

traefik/traefik.yaml

@@ -2,10 +2,10 @@ api:
   dashboard: true
 
 # Set Access logs timezone
-accessLog:
+# accessLog:
-  fields:
+#   fields:
-    names:
+#     names:
-      StartUTC: drop
+#       StartUTC: drop
 
 providers:
   docker: