diff --git a/.gitignore b/.gitignore index fcd2873..a47ffe6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ .DS_Store -/config **/tunnel-options.sh **/.env **/traefik/http_auth @@ -9,4 +8,6 @@ services.conf traefik/custom/dynamic*.yaml samples/custom*/*.yaml -config.yaml \ No newline at end of file +config.yaml +services/custom/*.yaml +services/custom/*.yml \ No newline at end of file diff --git a/config-updater.sh b/config-updater.sh index 228acab..9d1ecf9 100755 --- a/config-updater.sh +++ b/config-updater.sh @@ -27,7 +27,28 @@ fi jq -n '{"services": []}' > config.json -while read -r line ; do +# First, add Traefik as it was not explicitely set by default in old config file (services.conf) +if ! grep -q "traefik" services.conf; then + jq -r '.services[.services| length] |= . + + { + "name": "traefik", + "enabled": true, + "traefik": { + "enabled": true, + "rules": [ + { + "host": "traefik.'$(echo '${TRAEFIK_DOMAIN}')'", + "service": "api@internal", + "httpAuth": true, + } + ] + } + }' config.json > tmp.json + rm -f config.json + mv tmp.json config.json +fi + +cat services.conf | while read line || [[ -n $line ]]; do key=$(echo $line | sed -r "s/^(.*):.*$/\1/") enabled="true" if grep -q "disable" <<< $line; then @@ -41,8 +62,9 @@ while read -r line ; do # If this service is disabled AND another one in the file is enabled with VPN mode, keep that information if grep -q "$key-vpn: enable" services.conf; then if [[ enabled="false" ]]; then - echo "[$0] $key => another service detected enabled with vpn..." + #echo "[$0] $key => another service detected enabled with vpn..." enableVpn="true" + enabled="true" fi fi @@ -50,7 +72,7 @@ while read -r line ; do # Define if Traefik should be enabled on the service case $key in - flaresolverr) + flaresolverr|gluetun) enableTraefik="false" rules=$(jq -n '[]') ;; @@ -58,7 +80,7 @@ while read -r line ; do enableTraefik="true" # If Traefik enabled => define if http auth Traefik middleware must be set by default case $key in - gluetun|kavita|komga|nextcloud|ombi|overseerr|plex|portainer|tautulli) + kavita|komga|nextcloud|ombi|overseerr|plex|portainer|tautulli) defaultHttpAuth="false" ;; *) @@ -69,7 +91,7 @@ while read -r line ; do internalPort=$(cat config/ports | { grep $key || true; } | sed -r "s/^${key}: (.*)$/\1/") rules=$(jq -n '[ { - "host": "'"$key"'", + "host": "'"$key"'.'$(echo '${TRAEFIK_DOMAIN}')'", "httpAuth": '"${defaultHttpAuth}"', "internalPort": '"${internalPort}"', } @@ -90,18 +112,7 @@ while read -r line ; do rm -f config.json mv tmp.json config.json -done < services.conf - -# If we should enable Plex with hardware transcoding -# if grep -q -E "plex.*transcoding: enable" services.conf; then -# if grep -q "plex: disable" services.conf; then -# cat config.json | jq -r 'select(.services[].name=="plex") += {"plexHardwareTranscode":"enable"}' > tmp.json -# rm -f config.json -# mv tmp.json config.json -# fi -# fi - -#mv config.json config.bak.json +done # Transform json into yaml, easier to manipulate for the user cat config.json | yq e -P - > config.yaml \ No newline at end of file diff --git a/config.sample.yaml b/config.sample.yaml new file mode 100644 index 0000000..1b0eb4d --- /dev/null +++ b/config.sample.yaml @@ -0,0 +1,231 @@ +services: + - name: traefik + enabled: true + traefik: + enabled: true + rules: + - host: traefik.${TRAEFIK_DOMAIN} + service: api@internal + httpAuth: true + - name: deluge + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: deluge.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8112 + scheme: http + - name: flood + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: flood.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 3000 + - name: plex + enabled: true + vpn: false + # uncomment to use custom file with specific parameters for hardware transcoding + # customFile: plex-hardware-transcoding.yaml + traefik: + enabled: true + rules: + - host: plex.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 32400 + - name: flaresolverr + enabled: true + vpn: false + traefik: + enabled: false + rules: [] + - name: jackett + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: jackett.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 9117 + - name: prowlarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: prowlarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 9696 + - name: sonarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: sonarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8989 + - name: radarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: radarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 7878 + - name: bazarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: bazarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 6767 + - name: lidarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: lidarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8686 + - name: readarr + enabled: false + vpn: false + traefik: + enabled: true + rules: + - host: readarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8787 + - name: komga + enabled: false + vpn: false + traefik: + enabled: true + rules: + - host: komga.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 8080 + - name: kavita + enabled: false + vpn: false + traefik: + enabled: true + rules: + - host: kavita.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 5000 + - name: ombi + enabled: false + vpn: false + traefik: + enabled: true + rules: + - host: ombi.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 3579 + - name: overseerr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: overseerr.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 5055 + - name: tautulli + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: tautulli.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 8181 + - name: jdownloader + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: jdownloader.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 5800 + - name: tdarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: tdarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8265 + - name: nextcloud + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: nextcloud.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 443 + - name: portainer + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: portainer.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 9443 + - name: netdata + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: netdata.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 19999 + - name: duplicati + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: duplicati.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8200 + - name: syncthing + enabled: false + vpn: false + traefik: + enabled: true + rules: + - host: syncthing.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8384 + - name: heimdall + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: heimdall.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 80 + - name: gluetun + enabled: false + vpn: false + traefik: + enabled: false + rules: [] diff --git a/config/ports b/config/ports new file mode 100644 index 0000000..54f4b74 --- /dev/null +++ b/config/ports @@ -0,0 +1,24 @@ +bazarr: 6767 +deluge: 8112 +flood: 3000 +heimdall: 80 +jackett: 9117 +jdownloader: 5800 +ombi: 3579 +overseerr: 5055 +plex: 32400 +prowlarr: 9696 +radarr: 7878 +sonarr: 8989 +lidarr: 8686 +readarr: 8787 +komga: 8080 +kavita: 5000 +nextcloud: 443 +portainer: 9443 +syncthing: 8384 +tdarr: 8265 +netdata: 19999 +duplicati: 8200 +tautulli: 8181 +traefik: 80 diff --git a/services/custom/README.md b/services/custom/README.md new file mode 100644 index 0000000..c1252f2 --- /dev/null +++ b/services/custom/README.md @@ -0,0 +1,4 @@ +# Services custom folder + +Here, you can place all custom docker-compose files (yaml format) which will be specified in your ``config.yaml``. +All YAML files placed here are ignored in .gitignore. diff --git a/update-all-v2.sh b/update-all-v2.sh new file mode 100755 index 0000000..a784d04 --- /dev/null +++ b/update-all-v2.sh @@ -0,0 +1,237 @@ +#!/bin/bash + +set -e + +SKIP_PULL=0 +DEBUG=0 + +for i in "$@"; do + case $i in + --no-pull) + SKIP_PULL=1 + ;; + --debug) + DEBUG=1 + ;; + *) + echo "[$0] ❌ ERROR: unknown parameter \"$i\"" + exit 1 + ;; + esac +done + +cleanup_on_exit() { + rm -f rules.props *-vpn.props config.json +} +trap cleanup_on_exit EXIT + +echo-debug() { + [[ ${DEBUG} == "1" ]] && echo "$@" +} + +# Cleanup files before start, in case there was a change we start from scratch at every script execution +rm -f config/*-vpn.yaml + +# Create/update http_auth file according to values in .env file +source .env +echo "${HTTP_USER}:${HTTP_PASSWORD}" > traefik/http_auth + +# Docker-compose settings +export COMPOSE_HTTP_TIMEOUT=240 + +# Retro-compatibility +[[ -z $HOST_CONFIG_PATH ]] && export HOST_CONFIG_PATH="/data/config" +[[ -z $HOST_MEDIA_PATH ]] && export HOST_MEDIA_PATH="/data/torrents" +[[ -z $DOWNLOAD_SUBFOLDER ]] && export DOWNLOAD_SUBFOLDER="deluge" + +if [[ ! -f config.yaml ]]; then + echo "[$0] No config.yaml file found. Copying from sample file..." + cp config.sample.yaml config.yaml +fi + +# Alert in case new services have been added (or removed) in sample but active file has not changed +# TODO: adapt to new config structure +# NB_SERVICES_ACTIVE=$(cat services.conf | wc -l) +# NB_SERVICES_ORIG=$(cat services.conf.sample | wc -l) +# if [[ ${NB_SERVICES_ACTIVE} != ${NB_SERVICES_ORIG} ]]; then +# echo "[$0] Your services.conf file seems outdated. It appears there are new services available, or services that have been removed." +# diff -yt services.conf services.conf.sample || true +# echo "" +# fi + +############################################################################################### +###################################### Pre-flight checks ###################################### +############################################################################################### +yq eval -o json config.yaml > config.json + +# Check if some services have vpn enabled, that gluetun itself is enabled +nb_vpn=$(cat config.json | jq '[.services[] | select(.enabled==true and .vpn==true)] | length') +gluetun_enabled=$(cat config.json | jq '[.services[] | select(.name=="gluetun" and .enabled==true)] | length') +if [[ ${nb_vpn} -gt 0 && ${gluetun_enabled} == 0 ]]; then + echo "[$0] ERROR. ${nb_vpn} VPN-enabled services have been enabled BUT gluetun has not been enabled. Please check your config.yaml file." + echo "[$0] ******* Exiting *******" + exit 1 +fi + +# Determine what host Flood should connect to +# => If deluge vpn is enabled => gluetun +# => If deluge vpn is disabled => deluge +if [[ $(cat config.json | jq '[.services[] | select(.name=="flood" and .enabled==true)] | length') -eq 1 ]]; then + # Check that if flood is enabled, deluge should also be enabled + if [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==false)] | length') -eq 1 ]]; then + echo "[$0] ERROR. Flood is enabled but Deluge is not. Please either enable Deluge or disable Flood as Flood depends on Deluge." + echo "[$0] ******* Exiting *******" + exit 1 + fi + if [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==true)] | length') -eq 1 ]]; then + export DELUGE_HOST="gluetun" + elif [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==false)] | length') -eq 1 ]]; then + export DELUGE_HOST="deluge" + fi + + # Specific instructions for Flood + # User for Deluge daemon RPC has to be created in deluge auth config file + if [[ ! -z ${FLOOD_PASSWORD} && ${FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON} == true ]]; then + if ! grep -q "flood" $HOST_CONFIG_PATH/deluge/auth; then + echo "flood:${FLOOD_PASSWORD}:10" >> $HOST_CONFIG_PATH/deluge/auth + else + echo "[$0] No need to add user/password for flood as it has already been created." + echo "[$0] Consider setting FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON variable to false in .env file." + fi + fi + +fi + +# Apply other arbitrary custom Traefik config files +rm -f $f traefik/custom/dynamic-* +for f in `find samples/custom-traefik -maxdepth 1 -mindepth 1 -type f | grep -E "\.yml$|\.yaml$" | sort`; do + echo "[$0] Applying custom Traefik config $f..." + cp $f traefik/custom/dynamic-$(basename $f) +done + +# Detect Synology devices for Netdata compatibility +if [[ $(cat config.json | jq '[.services[] | select(.name=="netdata" and .enabled==true)] | length') -eq 1 ]]; then + if [[ $(uname -a | { grep synology || true; } | wc -l) -eq 1 ]]; then + export OS_RELEASE_FILEPATH="/etc/VERSION" + else + export OS_RELEASE_FILEPATH="/etc/os-release" + fi +fi + +############################################################################################### +####################################### SERVICES PARSING ###################################### +############################################################################################### +ALL_SERVICES="-f docker-compose.yaml" + +# Parse the config.yaml master configuration file +for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do + name=$(echo $json | jq -r .name) + enabled=$(echo $json | jq -r .enabled) + vpn=$(echo $json | jq -r .vpn) + + # Skip disabled services + if [[ ${enabled} == "false" ]]; then + echo-debug "[$0] Service $name is disabled. Skipping it." + continue + fi + + echo-debug "[$0] ➡️ Parsing service: \"$name\"..." + + # Default docker-compose filename is the service name + .yaml. + # Take into account explicit filename if specified in config + customFile=$(echo $json | jq -r .customFile) + file="$name.yaml" + if [[ ${customFile} != "null" ]]; then + file=${customFile} + fi + echo-debug "[$0] File: \"$file\"..." + + # Append $file to global list of files which will be passed to docker commands + ALL_SERVICES="${ALL_SERVICES} -f services/${file}" + + # For services with VPN enabled, add a docker-compose "override" file specifying that the service network should + # go through gluetun (main vpn client service). + if [[ ${vpn} == "true" ]]; then + echo "services.${name}.network_mode: service:gluetun" > ${name}-vpn.props + yq -p=props ${name}-vpn.props > config/${name}-vpn.yaml + rm -f ${name}-vpn.props + # Append config/${name}-vpn.yaml to global list of files which will be passed to docker commands + ALL_SERVICES="${ALL_SERVICES} -f config/${name}-vpn.yaml" + fi + + ###################################### TRAEFIK RULES ###################################### + + # Skip this part for services which have Traefik rules disabled in config + traefikEnabled=$(echo $json | jq -r .traefik.enabled) + if [[ ${traefikEnabled} == "false" ]]; then + echo-debug "[$0] Traefik is disabled. Skipping rules creation..." + continue + fi + + # Loop over all Traefik rules and create the corresponding entries in the generated rules.yaml + echo-debug "[$0] Generating Traefik rules..." + i=0 + for rule in $(echo $json | jq -c .traefik.rules[]); do + ((i=i+1)) + host=$(echo $rule | jq -r .host) + internalPort=$(echo $rule | jq -r .internalPort) + httpAuth=$(echo $rule | jq -r .httpAuth) + echo-debug "[$0] Host ${host}" + echo-debug "[$0] Internal Port ${internalPort}" + echo-debug "[$0] Http Authentication ${httpAuth}" + + # If VPN => Traefik rule should redirect to gluetun container + backendHost=${name} + [[ ${vpn} == "true" ]] && backendHost="gluetun" + + # Handle custom scheme (default if non-specified is http) + scheme="http" + customScheme=$(echo $rule | jq -r .scheme) + [[ ${customScheme} != "null" ]] && scheme=${customScheme} + + # Transform the bash syntax into Traefik/go one => anything.${TRAEFIK_DOMAIN} to anything.{{ env "TRAEFIK_DOMAIN" }} + hostTraefik=$(echo ${host} | sed --regexp-extended 's/^(.*)(\$\{(.*)\})/\1\{\{ env "\3" \}\}/') + + ruleId="${name}-${i}" + echo 'http.routers.'"${ruleId}"'.rule: Host(`'${hostTraefik}'`)' >> rules.props + if [[ ${httpAuth} == "true" ]]; then + echo "http.routers.${ruleId}.middlewares.0: common-auth@file" >> rules.props + fi + + traefikService=$(echo $rule | jq -r .service) + if [[ ${traefikService} != "null" ]]; then + echo "http.routers.${ruleId}.service: ${traefikService}" >> rules.props + else + echo "http.routers.${ruleId}.service: ${ruleId}" >> rules.props + fi + + # If the specified service does not contain a "@" => we create it + # If the service has a @, it means it is defined elsewhere so we do not create it (custom file, @internal...) + if echo ${traefikService} | grep -vq "@"; then + echo "http.services.${ruleId}.loadBalancer.servers.0.url: ${scheme}://${backendHost}:${internalPort}" >> rules.props + fi + + done +done + +# Convert properties files into Traefik-ready YAML and place it in the correct folder loaded by Traefik +yq -p=props rules.props > traefik/custom/dynamic-rules.yaml +rm -f rules.props + +# echo ${ALL_SERVICES} + +if [[ "${SKIP_PULL}" != "1" ]]; then + echo "[$0] ***** Pulling all images... *****" + docker-compose ${ALL_SERVICES} pull +fi + +echo "[$0] ***** Recreating containers if required... *****" +docker-compose ${ALL_SERVICES} up -d --remove-orphans +echo "[$0] ***** Done updating containers *****" + +echo "[$0] ***** Clean unused images and volumes... *****" +docker image prune -af +docker volume prune -f + +echo "[$0] ***** Done! *****" +exit 0 \ No newline at end of file