From 32da0ba9acd1e491ae19ffdd7859bf183ec4a312 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 18:07:36 +0100 Subject: [PATCH] Prepare release v2 - more fixes and refactoring + prepare release notes --- .env.sample | 3 ++ .gitignore | 3 +- UPGRADE_V2.md | 73 ++++++++++++++++++++++++++ config-updater.sh | 22 ++++---- config.sample.yaml | 13 +++-- config/tools.sh | 23 +++++++++ notes.md | 16 ------ update-all.sh => old-update-all.sh | 0 update-all-v2.sh => run-seedbox.sh | 82 ++++++++++++++++++++++-------- services.conf.sample | 10 ++-- services/generated/README.md | 3 ++ traefik/traefik.yaml | 8 +-- 12 files changed, 195 insertions(+), 61 deletions(-) create mode 100644 UPGRADE_V2.md create mode 100755 config/tools.sh delete mode 100644 notes.md rename update-all.sh => old-update-all.sh (100%) rename update-all-v2.sh => run-seedbox.sh (76%) create mode 100644 services/generated/README.md diff --git a/.env.sample b/.env.sample index b4696b1..d108c91 100644 --- a/.env.sample +++ b/.env.sample @@ -1,3 +1,6 @@ +# Internal settings (they will not be passed to running services) +CHECK_FOR_OUTDATED_CONFIG=true + # General Traefik (reverse proxy) settings TRAEFIK_DOMAIN=mydomain.com ACME_MAIL=my-email@my-provider.com diff --git a/.gitignore b/.gitignore index a47ffe6..0dc44b5 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ samples/custom*/*.yaml config.yaml services/custom/*.yaml -services/custom/*.yml \ No newline at end of file +services/custom/*.yml +services/generated/*.yaml \ No newline at end of file diff --git a/UPGRADE_V2.md b/UPGRADE_V2.md new file mode 100644 index 0000000..f17fba1 --- /dev/null +++ b/UPGRADE_V2.md @@ -0,0 +1,73 @@ +# Upgrade to V2 + +Seedbox version 2 is here! +Since there are some breaking changes and a configuration structure migration, a major version was mandatory. + +> These releases notes are still a work-in-progress as V2 is not fully tested and finalized yet. + +## What's new? + +* Configuration change to new YAML format + * Run ``config-updater.sh`` to migrate your old services.conf to the new config.yaml format. + * jq (v1.5+) and yq (v4+) are now requirements + * Easier feature switches + * If a service is missing, it won't be enabled by default like before. The config is now more declarative. + * Traefik routing rules are now dynamically generated in a file in Traefik config directory, so no more Docker labels. They became hard to maintain due to all possibilities caused by VPN support or custom files for example. +* VPN support + * With ``gluetun`` service, you can now place any service behind a VPN. + * Default gluetun configuration is Wireguard "custom" mode, but see below... +* Support custom services and docker-compose yaml files + * Place a docker-compose.yaml file in ``services/custom/`` directory, add a service in your config.yaml specifying a ``customFile``, and you are set. + * Support Plex hardware transcoding using a custom-file, already available in the ``services`` directory (just specify a customFile on plex service - see [config.sample.yaml](config.sample.yaml)). +* Support arbitrary Traefik rules + * Place a Traefik YAML in ``samples/custom-traefik/`` directory, it will be copied in the Traefik configuration folder. + * Ideal to forward traffic to other services which do not belong to this seedbox. +* New services: + * ``Gluetun``: [VPN client (see above)](https://github.com/qdm12/gluetun) + * ``Heimdall``: [Dashboard](https://github.com/linuxserver/Heimdall) + * ``Readarr``: [Ebook and comic monitor](https://github.com/Readarr/Readarr) + * ``Komga``: [Comic Book Manager](https://github.com/gotson/komga) + * ``Kavita``: [Comic / Book Manager](https://github.com/Kareadita/Kavita) + +And also: + +* ``update-all.sh`` is now called ``run-seedbox.sh`` but its purpose is the same. +* More checks in ``run-seedbox.sh``. For example, throws an error if Flood is enabled but not Deluge, or if VPN is enabled on a service but the VPN client is not. +* You can now specify where your data lives on your host through new environments variables (see [.env.sample](.env.sample)). + * This change is backward-compatible as the run-seedbox.sh script will default to the old "/data/torrents" and "/data/config" paths if these variables are not set. +* ``networks:`` section is now aligned with the new docker compose syntax +* Nextcloud-db has moved. It is now in /data/config (see below how to mitigate the errors). +* Disable Traefik access logs + +## How to migrate + +```sh +./config-updater.sh +# Check the content of your .env file (in comparison with .env.sample which brings new variables) +./run-seedbox.sh +``` + +When everything runs smoothly, you can delete your old configuration file: + +```sh +rm -f services.conf +``` + +> Also, please make sure you have read the next section about Nextcloud Database location. + +## Nextcloud-db has moved + +Since commit e4ede925a8ce09b177206f30487a889da9e10334, nextcloud-db directory (mapped on /var/lib/mysql) has moved from +``/data/nextcloud-db`` to ``$HOST_CONFIG_PATH/nextcloud-db`` (*/data/config/nextcloud-db by default*). + +To ensure a smooth transition, you will have to move the directory nextcloud-db into the correct new location, then run some commands to fix the schema: + +```sh +mv /data/nextcloud-db/ /data/config/ +./run-seedbox.sh +source .env +docker exec -it nextcloud-db mysql_upgrade -u root -p${MYSQL_ROOT_PASSWORD} +docker restart nextcloud nextcloud-db +``` + +Ensure everything runs nicely by looking at nextcloud-db and nextcloud logs, and by accessing your Nextcloud web UI. diff --git a/config-updater.sh b/config-updater.sh index 9d1ecf9..7ee7d00 100755 --- a/config-updater.sh +++ b/config-updater.sh @@ -11,17 +11,14 @@ cleanup_on_exit() { } trap cleanup_on_exit EXIT -# Check that jq is installed -if ! which jq >/dev/null; then - echo "[$0] jq does not exist. Install it from here: https://stedolan.github.io/jq/download/" - echo "[$0] Also, please make sure it is in the PATH." - exit 1 -fi +# Load common functions +source config/tools.sh -# Check that yq is installed -if ! which yq >/dev/null; then - echo "[$0] yq does not exist. Install it from here: https://github.com/mikefarah/yq/releases" - echo "[$0] Also, please make sure it is in the PATH." +# Check that required tools are installed +check_utilities + +if [[ ! -f services.conf ]]; then + echo "[$0] ERROR. Could nof find services.conf. Exiting." exit 1 fi @@ -87,6 +84,10 @@ cat services.conf | while read line || [[ -n $line ]]; do defaultHttpAuth="true" ;; esac + # Define scheme // For nextcloud, scheme must be https + scheme="http" + [[ $key == "nextcloud" ]] && scheme="https" + # Define service default port from bundled config file internalPort=$(cat config/ports | { grep $key || true; } | sed -r "s/^${key}: (.*)$/\1/") rules=$(jq -n '[ @@ -94,6 +95,7 @@ cat services.conf | while read line || [[ -n $line ]]; do "host": "'"$key"'.'$(echo '${TRAEFIK_DOMAIN}')'", "httpAuth": '"${defaultHttpAuth}"', "internalPort": '"${internalPort}"', + "scheme": '"${scheme}"' } ]') ;; diff --git a/config.sample.yaml b/config.sample.yaml index 1b0eb4d..6c34a39 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -16,7 +16,10 @@ services: - host: deluge.${TRAEFIK_DOMAIN} httpAuth: true internalPort: 8112 - scheme: http + # Uncomment to specify custom schme (http by default) + # scheme: http + # Uncomment to *NOT* generate LetsEncrypt certificate (useful for local domains) + # disableCertificateGeneration: true - name: flood enabled: true vpn: false @@ -31,6 +34,8 @@ services: vpn: false # uncomment to use custom file with specific parameters for hardware transcoding # customFile: plex-hardware-transcoding.yaml + # You can also place you own file in services/custom/ and call it here like this: + # customFile: custom/my_own_file.yaml traefik: enabled: true rules: @@ -89,7 +94,7 @@ services: httpAuth: true internalPort: 6767 - name: lidarr - enabled: true + enabled: false vpn: false traefik: enabled: true @@ -197,7 +202,7 @@ services: httpAuth: true internalPort: 19999 - name: duplicati - enabled: true + enabled: false vpn: false traefik: enabled: true @@ -220,7 +225,7 @@ services: traefik: enabled: true rules: - - host: heimdall.${TRAEFIK_DOMAIN} + - host: ${TRAEFIK_DOMAIN} httpAuth: true internalPort: 80 - name: gluetun diff --git a/config/tools.sh b/config/tools.sh new file mode 100755 index 0000000..a87b794 --- /dev/null +++ b/config/tools.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +############################################################################## +############################### UTIL FUNCTIONS ############################### +############################################################################## + +check_utilities () { + # Check that jq is installed + if ! which jq >/dev/null; then + echo "[$0] jq does not exist. Install it from here: https://stedolan.github.io/jq/download/" + echo "[$0] Please install jq version 1.5 or above." + echo "[$0] Also, please make sure it is in the PATH." + exit 1 + fi + + # Check that yq is installed + if ! which yq >/dev/null; then + echo "[$0] yq does not exist. Install it from here: https://github.com/mikefarah/yq/releases" + echo "[$0] Please install yq version 4 or above." + echo "[$0] Also, please make sure it is in the PATH." + exit 1 + fi +} \ No newline at end of file diff --git a/notes.md b/notes.md deleted file mode 100644 index 6533e24..0000000 --- a/notes.md +++ /dev/null @@ -1,16 +0,0 @@ -# Nextcloud-db has moved - -Since commit e4ede925a8ce09b177206f30487a889da9e10334, nextcloud-db directory (mapped on /var/lib/mysql) has moved from -``/data/nextcloud-db`` to ``$HOST_CONFIG_PATH/nextcloud-db`` (*/data/config/nextcloud-db by default*). - -To ensure a smooth transition, you will have to move the directory nextcloud-db into the correct new location, then run some commands to fix the schema: - -```sh -mv /data/nextcloud-db/ /data/config/ -./update-all.sh -source .env -docker exec -it nextcloud-db mysql_upgrade -u root -p${MYSQL_ROOT_PASSWORD} -docker restart nextcloud nextcloud-db -``` - -Ensure everything runs nicely by looking at nextcloud-db and nextcloud logs, and by accessing your Nextcloud web UI. diff --git a/update-all.sh b/old-update-all.sh similarity index 100% rename from update-all.sh rename to old-update-all.sh diff --git a/update-all-v2.sh b/run-seedbox.sh similarity index 76% rename from update-all-v2.sh rename to run-seedbox.sh index a784d04..cfe6c5a 100755 --- a/update-all-v2.sh +++ b/run-seedbox.sh @@ -2,6 +2,12 @@ set -e +# Load common functions +source config/tools.sh + +# Check that required tools are installed +check_utilities + SKIP_PULL=0 DEBUG=0 @@ -26,11 +32,19 @@ cleanup_on_exit() { trap cleanup_on_exit EXIT echo-debug() { - [[ ${DEBUG} == "1" ]] && echo "$@" + if [[ ${DEBUG} == "1" ]]; then echo "$@"; fi } -# Cleanup files before start, in case there was a change we start from scratch at every script execution -rm -f config/*-vpn.yaml +############################################################################################### +####################################### Load variables ######################################## +############################################################################################### + +echo "[$0] ***** Checking environment variables and files... *****" + +if [[ ! -f .env ]]; then + echo "[$0] ERROR. \".env\" file not found. Please copy \".env.sample\" and edit its values. Then, re-run this script." + exit 1 +fi # Create/update http_auth file according to values in .env file source .env @@ -49,21 +63,24 @@ if [[ ! -f config.yaml ]]; then cp config.sample.yaml config.yaml fi -# Alert in case new services have been added (or removed) in sample but active file has not changed -# TODO: adapt to new config structure -# NB_SERVICES_ACTIVE=$(cat services.conf | wc -l) -# NB_SERVICES_ORIG=$(cat services.conf.sample | wc -l) -# if [[ ${NB_SERVICES_ACTIVE} != ${NB_SERVICES_ORIG} ]]; then -# echo "[$0] Your services.conf file seems outdated. It appears there are new services available, or services that have been removed." -# diff -yt services.conf services.conf.sample || true -# echo "" -# fi - ############################################################################################### ###################################### Pre-flight checks ###################################### ############################################################################################### + +echo "[$0] ***** Checking configuration... *****" + yq eval -o json config.yaml > config.json +if [[ ${CHECK_FOR_OUTDATED_CONFIG} == true ]]; then + nb_services=$(cat config.json | jq '.services | length') + nb_services_sample=$(yq eval -o json config.sample.yaml | jq '.services | length') + if [[ $nb_services_sample -gt $nb_services ]]; then + echo "[$0] There are more services in the config.sample.yaml than in your config.yaml" + echo "[$0] You should check config.sample.yaml because it seems there are new services available for you:" + diff -u config.yaml config.sample.yaml | grep "name:" | grep -E "^\+" || true + fi +fi + # Check if some services have vpn enabled, that gluetun itself is enabled nb_vpn=$(cat config.json | jq '[.services[] | select(.enabled==true and .vpn==true)] | length') gluetun_enabled=$(cat config.json | jq '[.services[] | select(.name=="gluetun" and .enabled==true)] | length') @@ -83,6 +100,7 @@ if [[ $(cat config.json | jq '[.services[] | select(.name=="flood" and .enabled= echo "[$0] ******* Exiting *******" exit 1 fi + # Determine deluge hostname (for flood) based on the VPN status (enabled or not) of deluge if [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==true)] | length') -eq 1 ]]; then export DELUGE_HOST="gluetun" elif [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==false)] | length') -eq 1 ]]; then @@ -99,11 +117,10 @@ if [[ $(cat config.json | jq '[.services[] | select(.name=="flood" and .enabled= echo "[$0] Consider setting FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON variable to false in .env file." fi fi - fi # Apply other arbitrary custom Traefik config files -rm -f $f traefik/custom/dynamic-* +# rm -f $f traefik/custom/dynamic-* for f in `find samples/custom-traefik -maxdepth 1 -mindepth 1 -type f | grep -E "\.yml$|\.yaml$" | sort`; do echo "[$0] Applying custom Traefik config $f..." cp $f traefik/custom/dynamic-$(basename $f) @@ -121,6 +138,12 @@ fi ############################################################################################### ####################################### SERVICES PARSING ###################################### ############################################################################################### + +echo "[$0] ***** Generating configuration... *****" + +# Cleanup files before start, in case there was a change we start from scratch at every script execution +rm -f services/generated/*-vpn.yaml + ALL_SERVICES="-f docker-compose.yaml" # Parse the config.yaml master configuration file @@ -153,10 +176,10 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do # go through gluetun (main vpn client service). if [[ ${vpn} == "true" ]]; then echo "services.${name}.network_mode: service:gluetun" > ${name}-vpn.props - yq -p=props ${name}-vpn.props > config/${name}-vpn.yaml + yq -p=props ${name}-vpn.props > services/generated/${name}-vpn.yaml rm -f ${name}-vpn.props # Append config/${name}-vpn.yaml to global list of files which will be passed to docker commands - ALL_SERVICES="${ALL_SERVICES} -f config/${name}-vpn.yaml" + ALL_SERVICES="${ALL_SERVICES} -f services/generated/${name}-vpn.yaml" fi ###################################### TRAEFIK RULES ###################################### @@ -176,9 +199,9 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do host=$(echo $rule | jq -r .host) internalPort=$(echo $rule | jq -r .internalPort) httpAuth=$(echo $rule | jq -r .httpAuth) - echo-debug "[$0] Host ${host}" - echo-debug "[$0] Internal Port ${internalPort}" - echo-debug "[$0] Http Authentication ${httpAuth}" + echo-debug "[$0] Host => ${host}" + echo-debug "[$0] Internal Port => ${internalPort}" + echo-debug "[$0] Http Authentication => ${httpAuth}" # If VPN => Traefik rule should redirect to gluetun container backendHost=${name} @@ -205,6 +228,11 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do echo "http.routers.${ruleId}.service: ${ruleId}" >> rules.props fi + disableCertificateGeneration=$(echo $rule | jq -r .disableCertificateGeneration) + if [[ ${disableCertificateGeneration} == true ]]; then + echo "http.routers.${ruleId}.tls: EMPTYMAP" >> rules.props + fi + # If the specified service does not contain a "@" => we create it # If the service has a @, it means it is defined elsewhere so we do not create it (custom file, @internal...) if echo ${traefikService} | grep -vq "@"; then @@ -215,10 +243,22 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do done # Convert properties files into Traefik-ready YAML and place it in the correct folder loaded by Traefik +mv traefik/custom/dynamic-rules.yaml traefik/custom/dynamic-rules-old.yaml yq -p=props rules.props > traefik/custom/dynamic-rules.yaml rm -f rules.props -# echo ${ALL_SERVICES} +# Post-transformations on the rules file +sed -i "s/EMPTYMAP/{}/g" traefik/custom/dynamic-rules.yaml +sed -i --regexp-extended "s/^(.*: )(Host.*$)/\1'\2'/g" traefik/custom/dynamic-rules.yaml +sed -i --regexp-extended "s/^(.*url: )(.*$)/\1\"\2\"/g" traefik/custom/dynamic-rules.yaml + +rm -f traefik/custom/dynamic-rules-old.yaml + +echo-debug "[$0] Here is the list of all files which are going to be processed: ${ALL_SERVICES}" + +echo "[$0] ***** Config OK. Launching services... *****" + +exit 0 if [[ "${SKIP_PULL}" != "1" ]]; then echo "[$0] ***** Pulling all images... *****" diff --git a/services.conf.sample b/services.conf.sample index a0983a9..3369ae3 100644 --- a/services.conf.sample +++ b/services.conf.sample @@ -9,8 +9,8 @@ prowlarr: enable sonarr: enable radarr: enable bazarr: enable -lidarr: enable -readarr: false +lidarr: disable +readarr: disable komga: disable kavita: disable ombi: disable @@ -18,11 +18,11 @@ overseerr: enable tautulli: enable jdownloader: enable jdownloader-vpn: disable -tdarr: enable +tdarr: disable nextcloud: enable portainer: enable netdata: enable -duplicati: enable +duplicati: disable syncthing: disable -heimdall: disable +heimdall: enable gluetun: disable \ No newline at end of file diff --git a/services/generated/README.md b/services/generated/README.md new file mode 100644 index 0000000..d178d48 --- /dev/null +++ b/services/generated/README.md @@ -0,0 +1,3 @@ +# Generated files + +This directory contains all generated overrides files. They are in .gitignore. diff --git a/traefik/traefik.yaml b/traefik/traefik.yaml index a2dfe1a..1f28c6d 100644 --- a/traefik/traefik.yaml +++ b/traefik/traefik.yaml @@ -2,10 +2,10 @@ api: dashboard: true # Set Access logs timezone -accessLog: - fields: - names: - StartUTC: drop +# accessLog: +# fields: +# names: +# StartUTC: drop providers: docker: