Prepare release v2 - more fixes and refactoring + prepare release notes

2022-03-06 18:07:36 +01:00 · 2022-03-06 18:07:36 +01:00 · 32da0ba9ac
parent e8c69b0e3a
commit 32da0ba9ac
12 changed files with 195 additions and 61 deletions
--- a/.env.sample
+++ b/.env.sample
@ -1,3 +1,6 @@
+# Internal settings (they will not be passed to running services)
+CHECK_FOR_OUTDATED_CONFIG=true
+
 # General Traefik (reverse proxy) settings
 TRAEFIK_DOMAIN=mydomain.com
 ACME_MAIL=my-email@my-provider.com
--- a/.gitignore
+++ b/.gitignore
@ -11,3 +11,4 @@ samples/custom*/*.yaml
 config.yaml
 services/custom/*.yaml
 services/custom/*.yml
+services/generated/*.yaml
--- a/UPGRADE_V2.md
+++ b/UPGRADE_V2.md
@ -0,0 +1,73 @@
+# Upgrade to V2
+
+Seedbox version 2 is here!
+Since there are some breaking changes and a configuration structure migration, a major version was mandatory.
+
+> These releases notes are still a work-in-progress as V2 is not fully tested and finalized yet.
+
+## What's new?
+
+* Configuration change to new YAML format
+  * Run ``config-updater.sh`` to migrate your old services.conf to the new config.yaml format.
+  * jq (v1.5+) and yq (v4+) are now requirements
+  * Easier feature switches
+  * If a service is missing, it won't be enabled by default like before. The config is now more declarative.
+  * Traefik routing rules are now dynamically generated in a file in Traefik config directory, so no more Docker labels. They became hard to maintain due to all possibilities caused by VPN support or custom files for example.
+* VPN support
+  * With ``gluetun`` service, you can now place any service behind a VPN.
+  * Default gluetun configuration is Wireguard "custom" mode, but see below...
+* Support custom services and docker-compose yaml files
+  * Place a docker-compose.yaml file in ``services/custom/`` directory, add a service in your config.yaml specifying a ``customFile``, and you are set.
+  * Support Plex hardware transcoding using a custom-file, already available in the ``services`` directory (just specify a customFile on plex service - see [config.sample.yaml](config.sample.yaml)).
+* Support arbitrary Traefik rules
+  * Place a Traefik YAML in ``samples/custom-traefik/`` directory, it will be copied in the Traefik configuration folder.
+  * Ideal to forward traffic to other services which do not belong to this seedbox.
+* New services:
+  * ``Gluetun``: [VPN client (see above)](https://github.com/qdm12/gluetun)
+  * ``Heimdall``: [Dashboard](https://github.com/linuxserver/Heimdall)
+  * ``Readarr``: [Ebook and comic monitor](https://github.com/Readarr/Readarr)
+  * ``Komga``: [Comic Book Manager](https://github.com/gotson/komga)
+  * ``Kavita``: [Comic / Book Manager](https://github.com/Kareadita/Kavita)
+
+And also:
+
+* ``update-all.sh`` is now called ``run-seedbox.sh`` but its purpose is the same.
+* More checks in ``run-seedbox.sh``. For example, throws an error if Flood is enabled but not Deluge, or if VPN is enabled on a service but the VPN client is not.
+* You can now specify where your data lives on your host through new environments variables (see [.env.sample](.env.sample)).
+  * This change is backward-compatible as the run-seedbox.sh script will default to the old "/data/torrents" and "/data/config" paths if these variables are not set.
+* ``networks:`` section is now aligned with the new docker compose syntax
+* Nextcloud-db has moved. It is now in /data/config (see below how to mitigate the errors).
+* Disable Traefik access logs
+
+## How to migrate
+
+```sh
+./config-updater.sh
+# Check the content of your .env file (in comparison with .env.sample which brings new variables)
+./run-seedbox.sh
+```
+
+When everything runs smoothly, you can delete your old configuration file:
+
+```sh
+rm -f services.conf
+```
+
+> Also, please make sure you have read the next section about Nextcloud Database location.
+
+## Nextcloud-db has moved
+
+Since commit e4ede925a8ce09b177206f30487a889da9e10334, nextcloud-db directory (mapped on /var/lib/mysql) has moved from
+``/data/nextcloud-db`` to ``$HOST_CONFIG_PATH/nextcloud-db`` (*/data/config/nextcloud-db by default*).
+
+To ensure a smooth transition, you will have to move the directory nextcloud-db into the correct new location, then run some commands to fix the schema:
+
+```sh
+mv /data/nextcloud-db/ /data/config/
+./run-seedbox.sh
+source .env
+docker exec -it nextcloud-db mysql_upgrade -u root -p${MYSQL_ROOT_PASSWORD}
+docker restart nextcloud nextcloud-db
+```
+
+Ensure everything runs nicely by looking at nextcloud-db and nextcloud logs, and by accessing your Nextcloud web UI.
--- a/config-updater.sh
+++ b/config-updater.sh
@ -11,17 +11,14 @@ cleanup_on_exit() {
 }
 trap cleanup_on_exit EXIT

-# Check that jq is installed
-if ! which jq >/dev/null; then
-  echo "[$0] jq does not exist. Install it from here: https://stedolan.github.io/jq/download/"
-  echo "[$0] Also, please make sure it is in the PATH."
-  exit 1
-fi
+# Load common functions
+source config/tools.sh

-# Check that yq is installed
-if ! which yq >/dev/null; then
-  echo "[$0] yq does not exist. Install it from here: https://github.com/mikefarah/yq/releases"
-  echo "[$0] Also, please make sure it is in the PATH."
+# Check that required tools are installed
+check_utilities
+
+if [[ ! -f services.conf ]]; then
+  echo "[$0] ERROR. Could nof find services.conf. Exiting."
  exit 1
 fi

@ -87,6 +84,10 @@ cat services.conf | while read line || [[ -n $line ]]; do
          defaultHttpAuth="true"
          ;;
      esac
+      # Define scheme // For nextcloud, scheme must be https
+      scheme="http"
+      [[ $key == "nextcloud" ]] && scheme="https"
+      
      # Define service default port from bundled config file
      internalPort=$(cat config/ports | { grep $key || true; } | sed -r "s/^${key}: (.*)$/\1/")
      rules=$(jq -n '[
@ -94,6 +95,7 @@ cat services.conf | while read line || [[ -n $line ]]; do
            "host": "'"$key"'.'$(echo '${TRAEFIK_DOMAIN}')'",
            "httpAuth": '"${defaultHttpAuth}"',
            "internalPort": '"${internalPort}"',
+            "scheme": '"${scheme}"'
          }
        ]')
      ;;
--- a/config.sample.yaml
+++ b/config.sample.yaml
@ -16,7 +16,10 @@ services:
        - host: deluge.${TRAEFIK_DOMAIN}
          httpAuth: true
          internalPort: 8112
-          scheme: http
+          # Uncomment to specify custom schme (http by default)
+          # scheme: http
+          # Uncomment to *NOT* generate LetsEncrypt certificate (useful for local domains)
+          # disableCertificateGeneration: true
  - name: flood
    enabled: true
    vpn: false
@ -31,6 +34,8 @@ services:
    vpn: false
    # uncomment to use custom file with specific parameters for hardware transcoding
    # customFile: plex-hardware-transcoding.yaml
+    # You can also place you own file in services/custom/ and call it here like this:
+    # customFile: custom/my_own_file.yaml
    traefik:
      enabled: true
      rules:
@ -89,7 +94,7 @@ services:
          httpAuth: true
          internalPort: 6767
  - name: lidarr
-    enabled: true
+    enabled: false
    vpn: false
    traefik:
      enabled: true
@ -197,7 +202,7 @@ services:
          httpAuth: true
          internalPort: 19999
  - name: duplicati
-    enabled: true
+    enabled: false
    vpn: false
    traefik:
      enabled: true
@ -220,7 +225,7 @@ services:
    traefik:
      enabled: true
      rules:
-        - host: heimdall.${TRAEFIK_DOMAIN}
+        - host: ${TRAEFIK_DOMAIN}
          httpAuth: true
          internalPort: 80
  - name: gluetun
--- a/config/tools.sh
+++ b/config/tools.sh
@ -0,0 +1,23 @@
+#!/bin/bash
+
+##############################################################################
+############################### UTIL FUNCTIONS ###############################
+##############################################################################
+
+check_utilities () {
+  # Check that jq is installed
+  if ! which jq >/dev/null; then
+    echo "[$0] jq does not exist. Install it from here: https://stedolan.github.io/jq/download/"
+    echo "[$0] Please install jq version 1.5 or above."
+    echo "[$0] Also, please make sure it is in the PATH."
+    exit 1
+  fi
+
+  # Check that yq is installed
+  if ! which yq >/dev/null; then
+    echo "[$0] yq does not exist. Install it from here: https://github.com/mikefarah/yq/releases"
+    echo "[$0] Please install yq version 4 or above."
+    echo "[$0] Also, please make sure it is in the PATH."
+    exit 1
+  fi
+}
--- a/notes.md
+++ b/notes.md
@ -1,16 +0,0 @@
-# Nextcloud-db has moved
-
-Since commit e4ede925a8ce09b177206f30487a889da9e10334, nextcloud-db directory (mapped on /var/lib/mysql) has moved from
-``/data/nextcloud-db`` to ``$HOST_CONFIG_PATH/nextcloud-db`` (*/data/config/nextcloud-db by default*).
-
-To ensure a smooth transition, you will have to move the directory nextcloud-db into the correct new location, then run some commands to fix the schema:
-
-```sh
-mv /data/nextcloud-db/ /data/config/
-./update-all.sh
-source .env
-docker exec -it nextcloud-db mysql_upgrade -u root -p${MYSQL_ROOT_PASSWORD}
-docker restart nextcloud nextcloud-db
-```
-
-Ensure everything runs nicely by looking at nextcloud-db and nextcloud logs, and by accessing your Nextcloud web UI.
--- a/old-update-all.sh
+++ b/old-update-all.sh
--- a/update-all-v2.sh
+++ b/update-all-v2.sh
@ -2,6 +2,12 @@

 set -e

+# Load common functions
+source config/tools.sh
+
+# Check that required tools are installed
+check_utilities
+
 SKIP_PULL=0
 DEBUG=0

@ -26,11 +32,19 @@ cleanup_on_exit() {
 trap cleanup_on_exit EXIT

 echo-debug() {
-  [[ ${DEBUG} == "1" ]] && echo "$@"
+  if [[ ${DEBUG} == "1" ]]; then echo "$@"; fi
 }

-# Cleanup files before start, in case there was a change we start from scratch at every script execution
-rm -f config/*-vpn.yaml
+###############################################################################################
+####################################### Load variables ########################################
+###############################################################################################
+
+echo "[$0] ***** Checking environment variables and files... *****"
+
+if [[ ! -f .env ]]; then
+  echo "[$0] ERROR. \".env\" file not found. Please copy \".env.sample\" and edit its values. Then, re-run this script."
+  exit 1
+fi

 # Create/update http_auth file according to values in .env file
 source .env
@ -49,21 +63,24 @@ if [[ ! -f config.yaml ]]; then
  cp config.sample.yaml config.yaml
 fi

-# Alert in case new services have been added (or removed) in sample but active file has not changed
-# TODO: adapt to new config structure
-# NB_SERVICES_ACTIVE=$(cat services.conf | wc -l)
-# NB_SERVICES_ORIG=$(cat services.conf.sample | wc -l)
-# if [[ ${NB_SERVICES_ACTIVE} != ${NB_SERVICES_ORIG} ]]; then
-#   echo "[$0] Your services.conf file seems outdated. It appears there are new services available, or services that have been removed."
-#   diff -yt services.conf services.conf.sample || true
-#   echo ""
-# fi
-
 ###############################################################################################
 ###################################### Pre-flight checks ######################################
 ###############################################################################################
+
+echo "[$0] ***** Checking configuration... *****"
+
 yq eval -o json config.yaml > config.json

+if [[ ${CHECK_FOR_OUTDATED_CONFIG} == true ]]; then
+  nb_services=$(cat config.json | jq '.services | length')
+  nb_services_sample=$(yq eval -o json config.sample.yaml | jq '.services | length')
+  if [[ $nb_services_sample -gt $nb_services ]]; then
+    echo "[$0] There are more services in the config.sample.yaml than in your config.yaml"
+    echo "[$0] You should check config.sample.yaml because it seems there are new services available for you:"
+    diff -u config.yaml config.sample.yaml | grep "name:" | grep -E "^\+" || true
+  fi
+fi
+
 # Check if some services have vpn enabled, that gluetun itself is enabled
 nb_vpn=$(cat config.json | jq '[.services[] | select(.enabled==true and .vpn==true)] | length')
 gluetun_enabled=$(cat config.json | jq '[.services[] | select(.name=="gluetun" and .enabled==true)] | length')
@ -83,6 +100,7 @@ if [[ $(cat config.json | jq '[.services[] | select(.name=="flood" and .enabled=
    echo "[$0] ******* Exiting *******"
    exit 1
  fi
+  # Determine deluge hostname (for flood) based on the VPN status (enabled or not) of deluge
  if [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==true)] | length') -eq 1 ]]; then
    export DELUGE_HOST="gluetun"
  elif [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==false)] | length') -eq 1 ]]; then
@ -99,11 +117,10 @@ if [[ $(cat config.json | jq '[.services[] | select(.name=="flood" and .enabled=
      echo "[$0] Consider setting FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON variable to false in .env file."
    fi
  fi
-
 fi

 # Apply other arbitrary custom Traefik config files
-rm -f $f traefik/custom/dynamic-*
+# rm -f $f traefik/custom/dynamic-*
 for f in `find samples/custom-traefik -maxdepth 1 -mindepth 1 -type f | grep -E "\.yml$|\.yaml$" | sort`; do
  echo "[$0] Applying custom Traefik config $f..."
  cp $f traefik/custom/dynamic-$(basename $f)
@ -121,6 +138,12 @@ fi
 ###############################################################################################
 ####################################### SERVICES PARSING ######################################
 ###############################################################################################
+
+echo "[$0] ***** Generating configuration... *****"
+
+# Cleanup files before start, in case there was a change we start from scratch at every script execution
+rm -f services/generated/*-vpn.yaml
+
 ALL_SERVICES="-f docker-compose.yaml"

 # Parse the config.yaml master configuration file
@ -153,10 +176,10 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do
  # go through gluetun (main vpn client service).
  if [[ ${vpn} == "true" ]]; then
    echo "services.${name}.network_mode: service:gluetun" > ${name}-vpn.props
-    yq -p=props ${name}-vpn.props > config/${name}-vpn.yaml
+    yq -p=props ${name}-vpn.props > services/generated/${name}-vpn.yaml
    rm -f ${name}-vpn.props
    # Append config/${name}-vpn.yaml to global list of files which will be passed to docker commands
-    ALL_SERVICES="${ALL_SERVICES} -f config/${name}-vpn.yaml"
+    ALL_SERVICES="${ALL_SERVICES} -f services/generated/${name}-vpn.yaml"
  fi

  ###################################### TRAEFIK RULES ######################################
@ -176,9 +199,9 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do
    host=$(echo $rule | jq -r .host)
    internalPort=$(echo $rule | jq -r .internalPort)
    httpAuth=$(echo $rule | jq -r .httpAuth)
-    echo-debug "[$0]      Host ${host}"
-    echo-debug "[$0]      Internal Port ${internalPort}"
-    echo-debug "[$0]      Http Authentication ${httpAuth}"
+    echo-debug "[$0]      Host => ${host}"
+    echo-debug "[$0]      Internal Port => ${internalPort}"
+    echo-debug "[$0]      Http Authentication => ${httpAuth}"

    # If VPN => Traefik rule should redirect to gluetun container
    backendHost=${name}
@ -205,6 +228,11 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do
      echo "http.routers.${ruleId}.service: ${ruleId}" >> rules.props
    fi

+    disableCertificateGeneration=$(echo $rule | jq -r .disableCertificateGeneration)
+    if [[ ${disableCertificateGeneration} == true ]]; then
+      echo "http.routers.${ruleId}.tls: EMPTYMAP" >> rules.props
+    fi
+
    # If the specified service does not contain a "@" => we create it
    # If the service has a @, it means it is defined elsewhere so we do not create it (custom file, @internal...)
    if echo ${traefikService} | grep -vq "@"; then
@ -215,10 +243,22 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do
 done

 # Convert properties files into Traefik-ready YAML and place it in the correct folder loaded by Traefik
+mv traefik/custom/dynamic-rules.yaml traefik/custom/dynamic-rules-old.yaml
 yq -p=props rules.props > traefik/custom/dynamic-rules.yaml
 rm -f rules.props

-# echo ${ALL_SERVICES}
+# Post-transformations on the rules file
+sed -i "s/EMPTYMAP/{}/g" traefik/custom/dynamic-rules.yaml
+sed -i --regexp-extended "s/^(.*: )(Host.*$)/\1'\2'/g" traefik/custom/dynamic-rules.yaml
+sed -i --regexp-extended "s/^(.*url: )(.*$)/\1\"\2\"/g" traefik/custom/dynamic-rules.yaml
+
+rm -f traefik/custom/dynamic-rules-old.yaml
+
+echo-debug "[$0] Here is the list of all files which are going to be processed: ${ALL_SERVICES}"
+
+echo "[$0] ***** Config OK. Launching services... *****"
+
+exit 0

 if [[ "${SKIP_PULL}" != "1" ]]; then
  echo "[$0] ***** Pulling all images... *****"
--- a/services.conf.sample
+++ b/services.conf.sample
@ -9,8 +9,8 @@ prowlarr: enable
 sonarr: enable
 radarr: enable
 bazarr: enable
-lidarr: enable
-readarr: false
+lidarr: disable
+readarr: disable
 komga: disable
 kavita: disable
 ombi: disable
@ -18,11 +18,11 @@ overseerr: enable
 tautulli: enable
 jdownloader: enable
 jdownloader-vpn: disable
-tdarr: enable
+tdarr: disable
 nextcloud: enable
 portainer: enable
 netdata: enable
-duplicati: enable
+duplicati: disable
 syncthing: disable
-heimdall: disable
+heimdall: enable
 gluetun: disable
--- a/services/generated/README.md
+++ b/services/generated/README.md
@ -0,0 +1,3 @@
+# Generated files
+
+This directory contains all generated overrides files. They are in .gitignore.
--- a/traefik/traefik.yaml
+++ b/traefik/traefik.yaml
@ -2,10 +2,10 @@ api:
  dashboard: true

 # Set Access logs timezone
-accessLog:
-  fields:
-    names:
-      StartUTC: drop
+# accessLog:
+#   fields:
+#     names:
+#       StartUTC: drop

 providers:
  docker: