freedombone/src/freedombone-backup-remote

#!/bin/bash
#
# .---.                  .              .
# |                      |              |
# |--- .--. .-.  .-.  .-.|  .-. .--.--. |.-.  .-. .--.  .-.
# |    |   (.-' (.-' (   | (   )|  |  | |   )(   )|  | (.-'
# '    '     --'  --'  -' -  -' '  '   -' -'   -' '   -  --'
#
#                    Freedom in the Cloud
#
# Backup to remote servers - the web of backups

# License
# =======
#
# Copyright (C) 2015-2018 Bob Mottram <bob@freedombone.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

PROJECT_NAME='freedombone'
COMPLETION_FILE=$HOME/${PROJECT_NAME}-completed.txt
CONFIGURATION_FILE=$HOME/${PROJECT_NAME}.cfg
MONGODB_APPS_FILE=$HOME/.mongodbapps
BACKUP_EXTRA_DIRECTORIES=/root/backup-extra-dirs.csv
ENABLE_VERIFICATION="no"

export TEXTDOMAIN=${PROJECT_NAME}-backup-remote
export TEXTDOMAINDIR="/usr/share/locale"

PROJECT_INSTALL_DIR=/usr/local/bin
if [ -f /usr/bin/${PROJECT_NAME} ]; then
    PROJECT_INSTALL_DIR=/usr/bin
fi

source "$PROJECT_INSTALL_DIR/${PROJECT_NAME}-vars"

# utilities needed for backup commands
UTILS_FILES="/usr/share/${PROJECT_NAME}/utils/${PROJECT_NAME}-utils-*"
for f in $UTILS_FILES
do
    source "$f"
done

# Temporary location for data to be backed up to other servers
SERVER_DIRECTORY=/root/remotebackup

# get the version of Go being used
GO_VERSION=$(grep 'GO_VERSION=' "/usr/share/${PROJECT_NAME}/utils/${PROJECT_NAME}-utils-go" | head -n 1 | awk -F '=' '{print $2}')

ADMIN_USERNAME=$(get_completion_param "Admin user")
ADMIN_NAME=$(getent passwd "$ADMIN_USERNAME" | cut -d: -f5 | cut -d, -f1)
ADMIN_EMAIL_ADDRESS=${ADMIN_USERNAME}@${HOSTNAME}
if [ ! -f /etc/ssl/private/backup.key ]; then
    echo $"Creating backup key"
    ${PROJECT_NAME}-addcert -h backup --dhkey 2048
fi

if [ ! -f "/home/${ADMIN_USERNAME}/backup.list" ]; then
    exit 1
fi

# MariaDB password
DATABASE_PASSWORD=$(${PROJECT_NAME}-pass -u root -a mariadb)

# local directory where the backup will be made
if [ ! -d $SERVER_DIRECTORY ]; then
    mkdir $SERVER_DIRECTORY
fi

if [ ! -d $SERVER_DIRECTORY/backup ]; then
    mkdir -p $SERVER_DIRECTORY/backup
fi

# The name of a currently suspended site
# Sites are suspended so that verification should work
SUSPENDED_SITE=

function suspend_site {
    # suspends a given website
    if [[ $ENABLE_VERIFICATION != "yes" ]]; then
        return
    fi
    SUSPENDED_SITE="$1"
    nginx_dissite "$SUSPENDED_SITE"
    systemctl reload nginx
}

function restart_site {
    # restarts a given website
    if [ ! "$SUSPENDED_SITE" ]; then
        return
    fi
    nginx_ensite "$SUSPENDED_SITE"
    systemctl reload nginx
    SUSPENDED_SITE=
}

function backup_blocklist {
    if [ ! -f /root/${PROJECT_NAME}-firewall-domains.cfg ]; then
        return
    fi
    echo $"Backing up ${PROJECT_NAME} blocklist"
    temp_backup_dir=/root/tempbackupblocklist
    if [ ! -d $temp_backup_dir ]; then
        mkdir -p $temp_backup_dir
    fi
    if [ -f "$NODEJS_INSTALLED_APPS_FILE" ]; then
        cp -f /root/${PROJECT_NAME}-firewall-domains.cfg $temp_backup_dir
    fi
    backup_directory_to_friend $temp_backup_dir blocklist
    rm -rf $temp_backup_dir
}

function backup_configfiles {
    echo $"Backing up ${PROJECT_NAME} configuration files"
    temp_backup_dir=/root/tempbackupconfig
    if [ ! -d $temp_backup_dir ]; then
        mkdir -p $temp_backup_dir
    fi
    if [ -f "$NODEJS_INSTALLED_APPS_FILE" ]; then
        cp -f "$NODEJS_INSTALLED_APPS_FILE" $temp_backup_dir
    fi
    if [ -f /root/.nostore ]; then
        cp -f /root/.nostore $temp_backup_dir
    else
        if [ -f $temp_backup_dir/.nostore ]; then
            rm $temp_backup_dir/.nostore
        fi
    fi
    cp -f "$CONFIGURATION_FILE" $temp_backup_dir
    cp -f "$COMPLETION_FILE" $temp_backup_dir
    if [ -f $BACKUP_EXTRA_DIRECTORIES ]; then
        cp -f $BACKUP_EXTRA_DIRECTORIES $temp_backup_dir
    fi
    if [ -f "$MONGODB_APPS_FILE" ]; then
        cp -f "$MONGODB_APPS_FILE" $temp_backup_dir
    fi
    # nginx password hashes
    if [ -f /etc/nginx/.htpasswd ]; then
        cp -f /etc/nginx/.htpasswd $temp_backup_dir/htpasswd
    fi
    backup_directory_to_friend $temp_backup_dir configfiles
}

function backup_users {
    for d in /home/*/ ; do
        USERNAME=$(echo "$d" | awk -F '/' '{print $3}')
        if [[ $(is_valid_user "$USERNAME") == "1" ]]; then
            # personal settings
            if [ -d "/home/$USERNAME/personal" ]; then
                echo $"Backing up personal settings for $USERNAME"
                backup_directory_to_friend "/home/$USERNAME/personal" "personal/$USERNAME"
            fi

            # gpg keys
            if [ -d "/home/$USERNAME/.gnupg" ]; then
                echo $"Backing up gpg keys for $USERNAME"
                backup_directory_to_friend "/home/$USERNAME/.gnupg" "gnupg/$USERNAME"
            fi

            # ssh keys
            if [ -d "/home/$USERNAME/.ssh" ]; then
                echo $"Backing up ssh keys for $USERNAME"
                backup_directory_to_friend "/home/$USERNAME/.ssh" "ssh/$USERNAME"
            fi

            # syncthing files
            if [ -d "/home/$USERNAME/Sync" ]; then
                echo $"Backing up syncthing files for $USERNAME"
                backup_directory_to_friend "/home/$USERNAME/Sync" "syncthing/$USERNAME"
                # ensure that device IDs will be backed up as part of user config settings
                if [ ! -d "/home/$USERNAME/.config/syncthing" ]; then
                    mkdir -p "/home/$USERNAME/.config/syncthing"
                    chown -R "$USERNAME":"$USERNAME" "/home/$USERNAME/.config"
                fi
                if [ -f "/home/$USERNAME/.syncthing-server-id" ]; then
                    cp "/home/$USERNAME/.syncthing-server-id" "/home/$USERNAME/.config/syncthing"
                    chown -R "$USERNAME":"$USERNAME" "/home/$USERNAME/.config"
                fi
                if [ -f "/home/$USERNAME/.syncthingids" ]; then
                    cp "/home/$USERNAME/.syncthingids" "/home/$USERNAME/.config/syncthing"
                    chown -R "$USERNAME":"$USERNAME" "/home/$USERNAME/.config"
                fi
            fi

            # config files
            if [ -d "/home/$USERNAME/.config" ]; then
                echo $"Backing up config files for $USERNAME"
                backup_directory_to_friend "/home/$USERNAME/.config" "config/$USERNAME"
            fi

            # monkeysphere files
            if [ -d "/home/$USERNAME/.monkeysphere" ]; then
                echo $"Backing up monkeysphere files for $USERNAME"
                backup_directory_to_friend "/home/$USERNAME/.monkeysphere" "monkeysphere/$USERNAME"
            fi

            # fin files
            if [ -d "/home/$USERNAME/.fin" ]; then
                echo $"Backing up fin files for $USERNAME"
                backup_directory_to_friend "/home/$USERNAME/.fin" "fin/$USERNAME"
            fi

            # local files
            if [ -d "/home/$USERNAME/.local" ]; then
                echo $"Backing up local files for $USERNAME"
                backup_directory_to_friend "/home/$USERNAME/.local" "local/$USERNAME"
            fi

            # mutt settings
            if [ -f "/home/$USERNAME/.muttrc" ]; then
                echo $"Backing up Mutt settings for $USERNAME"
                if [ ! -d "/home/$USERNAME/tempbackup" ]; then
                    mkdir -p "/home/$USERNAME/tempbackup"
                fi
                cp "/home/$USERNAME/.muttrc" "/home/$USERNAME/tempbackup"
                if [ -f /etc/Muttrc ]; then
                    cp /etc/Muttrc "/home/$USERNAME/tempbackup"
                fi
                backup_directory_to_friend "/home/$USERNAME/tempbackup" "mutt/$USERNAME"
            fi
            if [ -d "/home/$USERNAME/.mutt" ]; then
                echo $"Backing up Mutt configurations for $USERNAME"
                backup_directory_to_friend "/home/$USERNAME/.mutt" "mutt/${USERNAME}configs"
            fi

            # procmail settings
            if [ -f "/home/$USERNAME/.procmailrc" ]; then
                echo $"Backing up procmail settings for $USERNAME"
                if [ ! -d "/home/$USERNAME/tempbackup" ]; then
                    mkdir -p "/home/$USERNAME/tempbackup"
                fi
                cp "/home/$USERNAME/.procmailrc" "/home/$USERNAME/tempbackup"
                backup_directory_to_friend "/home/$USERNAME/tempbackup" "procmail/$USERNAME"
            fi

            # spamassassin settings
            if [ -d "/home/$USERNAME/.spamassassin" ]; then
                echo $"Backing up spamassassin settings for $USERNAME"
                backup_directory_to_friend "/home/$USERNAME/.spamassassin" "spamassassin/$USERNAME"
            fi

            # email
            if [ -d "/home/$USERNAME/Maildir" ]; then
                echo $"Stopping mail server"
                systemctl stop exim4
                echo $"Creating an email archive"
                if [ ! -d "/root/backupemail/$USERNAME" ]; then
                    mkdir -p "/root/backupemail/$USERNAME"
                fi
                tar -czvf "/root/backupemail/$USERNAME/maildir.tar.gz" "/home/$USERNAME/Maildir"
                echo $"Restarting mail server"
                systemctl start exim4
                echo $"Backing up emails for $USERNAME"
                backup_directory_to_friend "/root/backupemail/$USERNAME" "mail/$USERNAME"
            fi
            gpg_agent_enable "$USERNAME"
        fi
    done
}

function backup_letsencrypt {
    if [ -d /etc/letsencrypt ]; then
        echo $"Backing up Lets Encrypt settings"
        backup_directory_to_friend /etc/letsencrypt letsencrypt
    fi
}

function backup_passwordstore {
    if [ -d /root/.passwords ]; then
        echo $"Backing up password store"
        backup_directory_to_friend /root/.passwords passwordstore
    fi
}

function backup_tor {
    if [ -d /etc/letsencrypt ]; then
        echo $"Backing up Tor settings"
        backup_directory_to_friend /var/lib/tor tor
    fi
}

function backup_certs {
    if [ -d /etc/ssl ]; then
        echo $"Backing up certificates"
        backup_directory_to_friend /etc/ssl ssl
    fi
}

function backup_mailing_list {
    if [ -d /var/spool/mlmmj ]; then
        echo $"Backing up the public mailing list"
        backup_directory_to_friend /var/spool/mlmmj mailinglist
    fi
}

function backup_web_server {
    if [ -d /etc/nginx ]; then
        echo $"Backing up web settings"
        backup_directory_to_friend /etc/nginx/sites-available web
    fi
}

function backup_admin_readme {
    if [ -f "/home/$ADMIN_USERNAME/README" ]; then
        echo $"Backing up README"
        if [ ! -d "/home/$ADMIN_USERNAME/tempbackup" ]; then
            mkdir -p "/home/$ADMIN_USERNAME/tempbackup"
        fi
        cp -f "/home/$ADMIN_USERNAME/README" "/home/$ADMIN_USERNAME/tempbackup"
        backup_directory_to_friend "/home/$ADMIN_USERNAME/tempbackup" readme
    fi
}

function backup_mariadb {
    if [ ${#DATABASE_PASSWORD} -gt 1 ]; then
        temp_backup_dir=/root/tempmariadb
        if [ ! -d $temp_backup_dir ]; then
            mkdir $temp_backup_dir
        fi
        keep_database_running
        mysqldump --password="$DATABASE_PASSWORD" mysql user > "$temp_backup_dir/mysql.sql"
        if [ ! -s $temp_backup_dir/mysql.sql ]; then
            echo $"Unable to backup MariaDB settings"
            rm -rf $temp_backup_dir
            # Send a warning email
            echo $"Unable to export database settings" | mail -s "${PROJECT_NAME} backup to friends" "$ADMIN_EMAIL_ADDRESS"
            exit 653
        fi
        echo "$DATABASE_PASSWORD" > $temp_backup_dir/db
        chmod 400 $temp_backup_dir/db
        backup_directory_to_friend $temp_backup_dir mariadb
    fi
}

function backup_postgresql {
    if [ ! -d /etc/postgresql ]; then
        return
    fi

    temp_backup_dir=/root/temppostgresql
    if [ ! -d $temp_backup_dir ]; then
        mkdir $temp_backup_dir
    fi
    # shellcheck disable=SC2024
    sudo -u postgres pg_dumpall --roles-only > "$temp_backup_dir/postgresql.sql"
    if [ ! -s $temp_backup_dir/postgresql.sql ]; then
        echo $"Unable to backup postgresql settings"
        rm -rf $temp_backup_dir
        umount "$USB_MOUNT"
        rm -rf "$USB_MOUNT"
        exit 684365
    fi
    echo "$DATABASE_PASSWORD" > $temp_backup_dir/db
    chmod 400 $temp_backup_dir/db
    backup_directory_to_friend $temp_backup_dir postgresql
}

# Returns the filename of a key share
function get_key_share {
    no_of_shares=$1
    USERNAME="$2"
    REMOTE_DOMAIN="$3"

    # Get a share index based on the supplied domain name
    # This ensures that the same share is always given to the same domain
    sharenumstr=$(md5sum <<< "$REMOTE_DOMAIN")
    share_index=$(echo $((0x${sharenumstr%% *} % no_of_shares)) | tr -d -)

    # get the filename
    share_files=("/home/$USERNAME/.gnupg_fragments/keyshare.asc.*")
    share_filename=${share_files[share_index]}

    echo "$share_filename"
}

function disperse_key_shares {
    USERNAME=$1
    REMOTE_DOMAIN=$2
    REMOTE_SSH_PORT=$3
    REMOTE_PASSWORD=$4
    REMOTE_SERVER=$5

    if [ -d "/home/$USERNAME/.gnupg_fragments" ]; then
        if [ "$REMOTE_DOMAIN" ]; then
            cd "/home/$USERNAME/.gnupg_fragments" || exit 34687246462
            # shellcheck disable=SC2012
            no_of_shares=$(ls -afq keyshare.asc.* | wc -l)
            if (( no_of_shares > 1 )); then
                share_filename=$(get_key_share "$no_of_shares" "$USERNAME" "$REMOTE_DOMAIN")

                # create a temp directory containing the share
                temp_key_share_dir=/home/$USERNAME/tempkey
                temp_key_share_fragments=$temp_key_share_dir/.gnupg_fragments_${USERNAME}
                mkdir -p "$temp_key_share_fragments"
                cp "$share_filename" "$temp_key_share_fragments/"

                # copy the fragments directory to the remote server
                /usr/bin/sshpass -p "$REMOTE_PASSWORD" \
                                 scp -r -P "$REMOTE_SSH_PORT" "$temp_key_share_fragments" "$REMOTE_SERVER"
                # shellcheck disable=SC2181
                if [ ! "$?" = "0" ]; then
                    # Send a warning email
                    echo "Key share to $REMOTE_SERVER failed" | \
                        mail -s "${PROJECT_NAME} social key management" "$MY_EMAIL_ADDRESS"
                else
                    # Send a confirmation email
                    echo "Key ${share_filename} shared to $REMOTE_SERVER" | \
                        mail -s "${PROJECT_NAME} social key management" "$MY_EMAIL_ADDRESS"
                fi

                # remove the temp file/directory
                shred -zu "$temp_key_share_fragments/"*
                rm -rf "$temp_key_share_dir"
            fi
        fi
    fi
}

TEST_MODE="no"
if [[ "$1" == "test" ]]; then
    TEST_MODE="yes"
fi

gpg_agent_setup root
backup_configfiles
if [[ $TEST_MODE == "no" ]]; then
    backup_blocklist
    backup_users
    backup_letsencrypt
    backup_passwordstore
    backup_tor
    backup_web_server
    backup_admin_readme
    backup_mariadb
    backup_postgresql
    backup_certs
    backup_mailing_list
    backup_apps remote
    backup_extra_directories remote
fi

# For each remote server
while read -r remote_server
do
    # Get the server and its password
    # Format is:
    #   username@domain <port number> /home/username <ssh password>
    REMOTE_SERVER=$(echo "${remote_server}" | awk -F ' ' '{print $1}')
    if [ "$REMOTE_SERVER" ]; then
        REMOTE_DOMAIN=$(echo "${remote_server}" | awk -F ' ' '{print $1}' | awk -F '@' '{print $2}')
        REMOTE_SSH_PORT=$(echo "${remote_server}" | awk -F ' ' '{print $2}')
        REMOTE_DIRECTORY=$(echo "${remote_server}" | awk -F ' ' '{print $3}')
        REMOTE_PASSWORD=$(echo "${remote_server}" | awk -F ' ' '{print $4}')
        NOW=$(date +"%Y-%m-%d %H:%M:%S")
        REMOTE_SERVER=$REMOTE_SERVER:$REMOTE_DIRECTORY

        echo "$NOW Starting backup to $REMOTE_SERVER" >> /var/log/remotebackups.log

        # Social key management
        for d in /home/*/ ; do
            USERNAME=$(echo "$d" | awk -F '/' '{print $3}')
            if [[ $(is_valid_user "$USERNAME") == "1" ]]; then
                disperse_key_shares "$USERNAME" "$REMOTE_DOMAIN" "$REMOTE_SSH_PORT" "$REMOTE_PASSWORD" "$REMOTE_SERVER"
            fi
        done

        if [[ $TEST_MODE == "yes" ]]; then
            echo "rsync -ratlzv --rsh=\"/usr/bin/sshpass -p '$REMOTE_PASSWORD' ssh -p $REMOTE_SSH_PORT -o StrictHostKeyChecking=no\" $SERVER_DIRECTORY/backup $REMOTE_SERVER"
        fi
        if ! rsync -ratlzv --rsh="/usr/bin/sshpass -p \"$REMOTE_PASSWORD\" ssh -p $REMOTE_SSH_PORT -o StrictHostKeyChecking=no" "$SERVER_DIRECTORY/backup" "$REMOTE_SERVER"; then
            echo "$NOW Backup to $REMOTE_SERVER failed" >> /var/log/remotebackups.log
            # Send a warning email
            echo "Backup to $REMOTE_SERVER failed" | mail -s "${PROJECT_NAME} backup to friends" "$ADMIN_EMAIL_ADDRESS"
        else
            echo "$NOW Backed up to $REMOTE_SERVER" >> /var/log/remotebackups.log
        fi
    fi

done < "/home/${ADMIN_USERNAME}/backup.list"

exit 0