freedombone/src/freedombone-utils-database

#!/bin/bash
#
# .---.                  .              .
# |                      |              |
# |--- .--. .-.  .-.  .-.|  .-. .--.--. |.-.  .-. .--.  .-.
# |    |   (.-' (.-' (   | (   )|  |  | |   )(   )|  | (.-'
# '    '     --'  --'  -' -  -' '  '   -' -'   -' '   -  --'
#
#                    Freedom in the Cloud
#
# Database functions
#
# License
# =======
#
# Copyright (C) 2014-2017 Bob Mottram <bob@freedombone.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# default MariaDB password
MARIADB_PASSWORD=

# Used to indicate whether the backup contains MariaDB databases or not
BACKUP_INCLUDES_DATABASES="no"

function store_original_mariadb_password {
    if [ ! -f /root/.mariadboriginal ]; then
        echo $'Storing original mariadb password'
        ORIGINAL_MARIADB_PASSWORD=$(${PROJECT_NAME}-pass -u root -a mariadb)
        # We can store this in plaintext because it will soon be of historical interest only
        echo -n "$ORIGINAL_MARIADB_PASSWORD" > /root/.mariadboriginal
    fi
}

function keep_database_running {
    if [ ! $(daemon_is_running mariadb) ]; then
        systemctl start mariadb
    fi
}

function remove_backup_database_local {
    database_name=$1

    sed -i "/# Backup the ${database_name} database/,/# End of ${database_name} database backup/d" /usr/bin/backupdatabases
    sed -i "/# Backup ${database_name}/,/# End of backup for ${database_name}/d" /etc/cron.weekly/backupdatabasesweekly
    sed -i "/# Backup ${database_name}/,/# End of backup for ${database_name}/d" /etc/cron.monthly/backupdatabasesmonthly
    sed -i "/${database_name}/d" /etc/cron.hourly/repair
}

function backup_database_local {
    # Makes local backups of databases which can then be automatically rolled
    # back if corruption is detected
    database_name=$1

    backup_databases_script=/usr/bin/backupdatabases
    if ! grep -q "# Check database daemon" /usr/bin/backupdatabases; then
        echo '' >> /usr/bin/backupdatabases
        echo '# Check database daemon is running' >> /usr/bin/backupdatabases
        echo 'if [ ! $(systemctl is-active mariadb >/dev/null 2>&1 && echo Running) ]; then' >> /usr/bin/backupdatabases
        echo '    systemctl start mariadb' >> /usr/bin/backupdatabases
        echo 'fi' >> /usr/bin/backupdatabases
        echo '' >> /usr/bin/backupdatabases
    fi

    if ! grep -q "# Backup the ${database_name} database" $backup_databases_script; then
        echo "# Backup the ${database_name} database" >> $backup_databases_script
        echo "TEMPFILE=/root/${database_name}.sql" >> $backup_databases_script
        echo "DAILYFILE=/var/backups/${database_name}_daily.sql" >> $backup_databases_script
        echo "mysqldump --password=\"\$MYSQL_PASSWORD\" ${database_name} > \$TEMPFILE" >> $backup_databases_script
        echo 'FILESIZE=$(stat -c%s $TEMPFILE)' >> $backup_databases_script
        echo 'if [ "$FILESIZE" -eq "0" ]; then' >> $backup_databases_script
        echo '    if [ -f $DAILYFILE ]; then' >> $backup_databases_script
        echo '        cp $DAILYFILE $TEMPFILE' >> $backup_databases_script
        echo '' >> $backup_databases_script
        echo '        # try to restore yesterdays database' >> $backup_databases_script
        echo "        mysql -u root --password=\"\$MYSQL_PASSWORD\" ${database_name} -o < \$DAILYFILE" >> $backup_databases_script
        echo '' >> $backup_databases_script
        echo '        # Send a warning email' >> $backup_databases_script
        echo "        echo \"Unable to create a backup of the ${database_name} database. Attempted to restore from yesterdays backup\" | mail -s \"${database_name} backup\" \$EMAIL" >> $backup_databases_script
        echo '    else' >> $backup_databases_script
        echo '        # Send a warning email' >> $backup_databases_script
        echo "        echo \"Unable to create a backup of the ${database_name} database.\" | mail -s \"${database_name} backup\" \$EMAIL" >> $backup_databases_script
        echo '    fi' >> $backup_databases_script
        echo 'else' >> $backup_databases_script
        echo '    chmod 600 $TEMPFILE' >> $backup_databases_script
        echo '    mv $TEMPFILE $DAILYFILE' >> $backup_databases_script
        echo '' >> $backup_databases_script
        echo '    # Make the backup readable only by root' >> $backup_databases_script
        echo '    chmod 600 $DAILYFILE' >> $backup_databases_script
        echo 'fi' >> $backup_databases_script
        echo "# End of ${database_name} database backup" >> $backup_databases_script
    fi

    weekly_backup_script=/etc/cron.weekly/backupdatabasesweekly
    if ! grep -q "Backup ${database_name}" ${weekly_backup_script}; then
        echo "# Backup ${database_name}" >> ${weekly_backup_script}
        echo "if [ -f /var/backups/${database_name}_weekly.sql ]; then" >> ${weekly_backup_script}
        echo "  cp -f /var/backups/${database_name}_weekly.sql /var/backups/${database_name}_2weekly.sql" >> ${weekly_backup_script}
        echo 'fi' >> ${weekly_backup_script}
        echo "if [ -f /var/backups/${database_name}_daily.sql ]; then" >> ${weekly_backup_script}
        echo "  cp -f /var/backups/${database_name}_daily.sql /var/backups/${database_name}_weekly.sql" >> ${weekly_backup_script}
        echo 'fi' >> ${weekly_backup_script}
        echo "# End of backup for ${database_name}" >> ${weekly_backup_script}
    fi

    monthly_backup_script=/etc/cron.monthly/backupdatabasesmonthly
    if ! grep -q "Backup ${database_name}" ${monthly_backup_script}; then
        echo "# Backup ${database_name}" >> ${monthly_backup_script}
        echo "if [ -f /var/backups/${database_name}_monthly.sql ]; then" >> ${monthly_backup_script}
        echo "  cp -f /var/backups/${database_name}_monthly.sql /var/backups/${database_name}_2monthly.sql" >> ${monthly_backup_script}
        echo 'fi' >> ${monthly_backup_script}
        echo "if [ -f /var/backups/${database_name}_weekly.sql ]; then" >> ${monthly_backup_script}
        echo "  cp -f /var/backups/${database_name}_weekly.sql /var/backups/${database_name}_monthly.sql" >> ${monthly_backup_script}
        echo 'fi' >> ${monthly_backup_script}
        echo "# End of backup for ${database_name}" >> ${monthly_backup_script}
    fi

    if ! grep -q "${database_name}" /etc/cron.hourly/repair; then
        echo "${PROJECT_NAME}-repair-database ${database_name}" >> /etc/cron.hourly/repair
        # remove legacy stuff
        sed -i 's|/usr/bin/repairdatabase redmatrix||g' /etc/cron.hourly/repair
    fi
}

function get_mariadb_password {
    # migrate from database password file to using the password store
    DATABASE_PASSWORD_FILE=/root/dbpass
    if [ -f $DATABASE_PASSWORD_FILE ]; then
        MARIADB_PASSWORD=$(cat $DATABASE_PASSWORD_FILE)
        ${PROJECT_NAME}-pass -u root -a mariadb -p "$MARIADB_PASSWORD"
        stored_password=$(${PROJECT_NAME}-pass -u root -a mariadb)
        if [[ "$stored_password" == "$MARIADB_PASSWORD" ]]; then
            shred -zu $DATABASE_PASSWORD_FILE
            echo $'MariaDB password moved into password store'
            return
        fi
    fi
    MARIADB_PASSWORD=$(${PROJECT_NAME}-pass -u root -a mariadb)
    if [[ "$MARIADB_PASSWORD" == *'failed'* ]]; then
        echo $'Could not obtain mariadb password'
        exit 835225
    fi
}

function mariadb_kill_stone_dead {
    systemctl stop mariadb
    kill_pid=$(ps aux | grep mysqld_safe | awk -F ' ' '{print $2}' | head -n 1)
    kill -9 $kill_pid
    kill_pid=$(ps aux | grep mysqld | awk -F ' ' '{print $2}' | head -n 1)
    kill -9 $kill_pid
    kill_pid=$(ps aux | grep mysqld | awk -F ' ' '{print $2}' | head -n 1)
    kill -9 $kill_pid
}

function mariadb_fix_authentication {
    # See http://www.pontikis.net/blog/debian-9-stretch-rc3-web-server-setup-php7-mariadb
    # https://mariadb.com/kb/en/mariadb/unix_socket-authentication-plugin
    remove_watchdog_daemon mariadb
    mariadb_kill_stone_dead
    mysqld_safe --skip-grant-tables &
    sleep 5
    mysql -u root --password="$MARIADB_PASSWORD" << EOF
use mysql;
update mysql.user set plugin = '' where User='root';
UPDATE user SET Password=PASSWORD('$MARIADB_PASSWORD') where USER='root';
flush privileges;
EOF
    mariadb_kill_stone_dead

    sed -i 's| --skip-grant-tables||g' /lib/systemd/system/mariadb.service
    systemctl daemon-reload
    systemctl start mariadb
    add_watchdog_daemon mariadb
}

function mariadb_create_root_user {
    run_query mysql "CREATE USER 'root@localhost' IDENTIFIED BY '${MARIADB_PASSWORD}'; flush privileges;"
    run_query mysql "update mysql.user set plugin = '' where User='root@localhost'; flush privileges;"
    run_query mysql "GRANT ALL PRIVILEGES ON * TO 'root@localhost'; flush privileges;"
}

function install_mariadb {
    if [[ $(is_completed $FUNCNAME) == "1" ]]; then
        return
    fi
    apt-get -yq install software-properties-common debconf-utils
    apt-get -yq update

    remove_watchdog_daemon mariadb

    function_check get_mariadb_password
    get_mariadb_password
    if [ ! $MARIADB_PASSWORD ]; then
        if [ -f $IMAGE_PASSWORD_FILE ]; then
            MARIADB_PASSWORD="$(printf `cat $IMAGE_PASSWORD_FILE`)"
        else
            MARIADB_PASSWORD="$(openssl rand -base64 32 | cut -c1-${MINIMUM_PASSWORD_LENGTH})"
        fi
    fi
    ${PROJECT_NAME}-pass -u root -a mariadb -p "$MARIADB_PASSWORD"

    debconf-set-selections <<< "mariadb-server mariadb-server/root_password password $MARIADB_PASSWORD"
    debconf-set-selections <<< "mariadb-server mariadb-server/root_password_again password $MARIADB_PASSWORD"
    apt-get -yq install mariadb-server mariadb-client
    apt-get -yq remove --purge apache2-bin*
    if [ -d /etc/apache2 ]; then
        rm -rf /etc/apache2
        echo $'Removed Apache installation after MariaDB install'
    fi

    if [ ! -d /etc/mysql ]; then
        echo $"ERROR: mariadb-server does not appear to have installed. $CHECK_MESSAGE"
        exit 54
    fi

    if [ ! -f /usr/bin/mysql ]; then
        echo $"ERROR: mariadb-server does not appear to have installed. $CHECK_MESSAGE"
        exit 34672
    fi

    sed -i 's|ExecStart=/usr/sbin/mysqld|ExecStart=/usr/sbin/mysqld --skip-grant-tables|g' /lib/systemd/system/mariadb.service
    systemctl daemon-reload
    systemctl restart mariadb

    mariadb_fix_authentication

    mariadb_create_root_user

    mark_completed $FUNCNAME
}

function backup_databases_script_header {
    if [ ! -f /usr/bin/backupdatabases ]; then
        # daily
        echo '#!/bin/sh' > /usr/bin/backupdatabases
        echo '' >> /usr/bin/backupdatabases
        echo "EMAIL='$MY_EMAIL_ADDRESS'" >> /usr/bin/backupdatabases
        echo '' >> /usr/bin/backupdatabases
        echo "MYSQL_PASSWORD=\$(${PROJECT_NAME}-pass -u root -a mariadb)" >> /usr/bin/backupdatabases
        echo 'umask 0077' >> /usr/bin/backupdatabases
        echo '' >> /usr/bin/backupdatabases
        echo '# exit if we are backing up to friends servers' >> /usr/bin/backupdatabases
        echo "if [ -f $FRIENDS_SERVERS_LIST ]; then" >> /usr/bin/backupdatabases
        echo '  exit 1' >> /usr/bin/backupdatabases
        echo 'fi' >> /usr/bin/backupdatabases
        chmod 600 /usr/bin/backupdatabases
        chmod +x /usr/bin/backupdatabases

        echo '#!/bin/sh' > /etc/cron.daily/backupdatabasesdaily
        echo '/usr/bin/backupdatabases' >> /etc/cron.daily/backupdatabasesdaily
        chmod 600 /etc/cron.daily/backupdatabasesdaily
        chmod +x /etc/cron.daily/backupdatabasesdaily

        # weekly
        echo '#!/bin/sh' > /etc/cron.weekly/backupdatabasesweekly
        echo '' >> /etc/cron.weekly/backupdatabasesweekly
        echo 'umask 0077' >> /etc/cron.weekly/backupdatabasesweekly

        chmod 600 /etc/cron.weekly/backupdatabasesweekly
        chmod +x /etc/cron.weekly/backupdatabasesweekly

        # monthly
        echo '#!/bin/sh' > /etc/cron.monthly/backupdatabasesmonthly
        echo '' >> /etc/cron.monthly/backupdatabasesmonthly
        echo 'umask 0077' >> /etc/cron.monthly/backupdatabasesmonthly

        chmod 600 /etc/cron.monthly/backupdatabasesmonthly
        chmod +x /etc/cron.monthly/backupdatabasesmonthly
    fi
}

function repair_databases_script {
    if [ -f /etc/cron.hourly/repair ]; then
        sed -i "s|/usr/bin/repairdatabase|${PROJECT_NAME}-repair-database|g" /etc/cron.hourly/repair
    fi

    if [[ $(is_completed $FUNCNAME) == "1" ]]; then
        return
    fi

    db_pass=$(${PROJECT_NAME}-pass -u root -p mariadb)
    if [[ "$db_pass" == 'Error:'* ]]; then
        return
    fi

    echo '#!/bin/bash' > /etc/cron.hourly/repair
    echo '' >> /etc/cron.hourly/repair
    chmod 600 /etc/cron.hourly/repair
    chmod +x /etc/cron.hourly/repair

    mark_completed $FUNCNAME
}

function remove_database {
    app_name="$1"
    if [ ! -d $INSTALL_DIR ]; then
        mkdir $INSTALL_DIR
    fi
    echo "drop database ${app_name};
quit" > $INSTALL_DIR/batch.sql
    chmod 600 $INSTALL_DIR/batch.sql
    keep_database_running
    mysql -u root --password="$MARIADB_PASSWORD" < $INSTALL_DIR/batch.sql
    shred -zu $INSTALL_DIR/batch.sql
}

function initialise_database {
    database_name=$1
    database_file=$2
    keep_database_running
    mysql -u root --password="$MARIADB_PASSWORD" -D $database_name < $database_file
    if [ ! "$?" = "0" ]; then
        exit 62952
    fi
}

function run_query {
    database_name=$1
    database_query=$2
    keep_database_running
    mysql -u root --password="$MARIADB_PASSWORD" -e "$database_query" $database_name
}

function run_query_root {
    database_name=$1
    database_query=$2
    keep_database_running
    mysql -e "$database_query" $database_name
}

function create_database {
    app_name="$1"
    app_admin_password="$2"
    app_admin_username=$3
    if [ ! -d $INSTALL_DIR ]; then
        mkdir $INSTALL_DIR
    fi
    if [ ! $app_admin_username ]; then
        app_admin_username=${app_name}admin
    fi

    echo "create database ${app_name};
CREATE USER '$app_admin_username@localhost' IDENTIFIED BY '${app_admin_password}';
update mysql.user set plugin = '' where User='$app_admin_username@localhost';
GRANT ALL PRIVILEGES ON ${app_name}.* TO '$app_admin_username@localhost';
flush privileges;
quit" > $INSTALL_DIR/batch.sql
    chmod 600 $INSTALL_DIR/batch.sql
    keep_database_running
    mysql -u root --password="$MARIADB_PASSWORD" < $INSTALL_DIR/batch.sql
    shred -zu $INSTALL_DIR/batch.sql
}

function run_query_with_output {
    database_name=$1
    database_query=$2
    keep_database_running
    output=$(mysql -u root --password="$MARIADB_PASSWORD" << EOF
use $database_name;
$database_query
EOF
)
    echo "$output"
}

function drop_database {
    database_name="$1"

    get_mariadb_password

    echo "drop database ${app_name};
flush privileges;
quit" > $INSTALL_DIR/batch.sql
    chmod 600 $INSTALL_DIR/batch.sql
    keep_database_running
    mysql -u root --password="$MARIADB_PASSWORD" < $INSTALL_DIR/batch.sql
    shred -zu $INSTALL_DIR/batch.sql
}


function database_reinstall {
    apt-get -yq purge mariadb*
    rm -rf /var/lib/mysql
    rm -rf /etc/mysql
    sed -i '/mariadb/d' ~/${PROJECT_NAME}-completed.txt
    install_mariadb
}

function install_rethinkdb {
    if [[ "$(arch)" == "arm"* ]]; then
        echo $'rethinkdb does not currently support ARM debian packages'
        echo $"See http://download.rethinkdb.com/apt/dists/${DEBIAN_VERSION}/main"
        exit 723723452
    fi

    if [ ! -d $INSTALL_DIR ]; then
        mkdir -p $INSTALL_DIR
    fi

    cd $INSTALL_DIR

    echo "deb http://download.rethinkdb.com/apt $DEBIAN_VERSION main" | tee /etc/apt/sources.list.d/rethinkdb.list

    wget -qO- https://download.rethinkdb.com/apt/pubkey.gpg | apt-key add -
    apt-get update
    apt-get -yq install rethinkdb

    echo 'runuser=rethinkdb' > /etc/rethinkdb/instances.d/default.conf
    echo 'rungroup=rethinkdb' >> /etc/rethinkdb/instances.d/default.conf
    echo '# pid-file=/var/run/rethinkdb/rethinkdb.pid' >> /etc/rethinkdb/instances.d/default.conf
    echo '# directory=/var/lib/rethinkdb/default' >> /etc/rethinkdb/instances.d/default.conf
    echo '# log-file=/var/log/rethinkdb' >> /etc/rethinkdb/instances.d/default.conf
    echo 'bind=127.0.0.1' >> /etc/rethinkdb/instances.d/default.conf
    echo '# canonical-address=' >> /etc/rethinkdb/instances.d/default.conf
    echo '# driver-port=28015' >> /etc/rethinkdb/instances.d/default.conf
    echo '# cluster-port=29015' >> /etc/rethinkdb/instances.d/default.conf
    echo '# join=example.com:29015' >> /etc/rethinkdb/instances.d/default.conf
    echo '# port-offset=0' >> /etc/rethinkdb/instances.d/default.conf
    echo '# reql-http-proxy=socks5://example.com:1080' >> /etc/rethinkdb/instances.d/default.conf
    echo '# http-port=8091' >> /etc/rethinkdb/instances.d/default.conf
    echo '# no-http-admin' >> /etc/rethinkdb/instances.d/default.conf
    echo '# cores=2' >> /etc/rethinkdb/instances.d/default.conf
    echo '# cache-size=1024' >> /etc/rethinkdb/instances.d/default.conf
    echo '# io-threads=64' >> /etc/rethinkdb/instances.d/default.conf
    echo '# direct-io' >> /etc/rethinkdb/instances.d/default.conf
    echo '# server-name=server1' >> /etc/rethinkdb/instances.d/default.conf

    systemctl restart rethinkdb
}

function remove_rethinkdb {
    if [ ! -d /etc/rethinkdb ]; then
        return
    fi
    apt-get -yq remove rethinkdb
    if [ -d /etc/rethinkdb ]; then
        rm -rf /etc/rethinkdb
    fi
    if [ -f /etc/apt/sources.list.d/rethinkdb.list ]; then
        rm /etc/apt/sources.list.d/rethinkdb.list
        apt-get update
    fi
}

# NOTE: deliberately there is no "exit 0"