Skip to content

Instantly share code, notes, and snippets.

@erikw

erikw/restic-backup.service

Last active February 24, 2022 11:43
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save erikw/fea70af495e136572b970a52bb0a4c4d to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save erikw/fea70af495e136572b970a52bb0a4c4d to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. erikw revised this gist Feb 24, 2022. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion restic-backup.service
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1 @@
    MOVED TO ⮕ https://github.com/erikw/restic-systemd-automatic-backup
    MOVED TO ⮕ https://github.com/erikw/restic-automatic-backup-scheduler
  2. erikw revised this gist Apr 13, 2018. 8 changed files with 1 addition and 232 deletions.
    11 changes: 1 addition & 10 deletions restic-backup.service
    View file
    Original file line number Diff line number Diff line change
    @@ -1,10 +1 @@
    [Unit]
    Description=Backup with restic to Backblaze B2
    OnFailure=status-email-user@%n.service

    [Service]
    Type=simple
    Nice=10
    ExecStart=/usr/local/sbin/restic_backup.sh
    # $HOME or $XDG_CACHE_HOME must be set for restic to find /root/.cache/restic/
    Environment="HOME=/root"
    MOVED TO ⮕ https://github.com/erikw/restic-systemd-automatic-backup
    9 changes: 0 additions & 9 deletions restic-backup.timer
    View file
    Original file line number Diff line number Diff line change
    @@ -1,9 +0,0 @@
    [Unit]
    Description=Backup with restic on schedule

    [Timer]
    OnCalendar=daily
    Persistent=true

    [Install]
    WantedBy=timers.target
    9 changes: 0 additions & 9 deletions restic-check.service
    View file
    Original file line number Diff line number Diff line change
    @@ -1,9 +0,0 @@
    [Unit]
    Description=Check restic backup Backblaze B2 for errors
    OnFailure=status-email-user@%n.service
    Conflicts=restic.service

    [Service]
    Type=simple
    Nice=10
    ExecStart=/usr/local/sbin/restic_check.sh
    9 changes: 0 additions & 9 deletions restic-check.timer
    View file
    Original file line number Diff line number Diff line change
    @@ -1,9 +0,0 @@
    [Unit]
    Description=Check restic backup Backblaze B2 for errors on a schedule

    [Timer]
    OnCalendar=monthly
    Persistent=true

    [Install]
    WantedBy=timers.target
    85 changes: 0 additions & 85 deletions restic_backup.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -1,85 +0,0 @@
    #!/usr/bin/env bash
    # Make backup my system with restic to Backblaze B2.
    # This script is typically run by: /etc/systemd/system/restic-backup.{service,timer}

    # Exit on failure, pipe failure
    set -e -o pipefail

    # Redirect stdout ( > ) into a named pipe ( >() ) running "tee" to a file, so we can observe the status by simply tailing the log file.
    me=$(basename "$0")
    now=$(date +%F_%R)
    log_dir=/var/local/log/restic
    log_file="${log_dir}/${now}_${me}.$$.log"
    test -d $log_dir || mkdir -p $log_dir
    exec > >(tee -i $log_file)
    exec 2>&1

    # Clean up lock if we are killed.
    # If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses.
    # However if we kill this script ourselves, we need this trap that kills all subprocesses manually.
    exit_hook() {
    echo "In exit_hook(), being killed" >&2
    jobs -p | xargs kill
    restic unlock
    }
    trap exit_hook INT TERM

    RETENTION_DAYS=7
    RETENTION_WEEKS=12
    RETENTION_MONTHS=18
    RETENTION_YEARS=4

    BACKUP_PATHS="/ /boot /home /mnt/media"
    BACKUP_EXCLUDES="--exclude-file /.rsync_exclude --exclude-file /mnt/media/.rsync_exclude --exclude-file /home/erikw/.rsync_exclude"
    BACKUP_TAG=systemd.timer

    # Set all environment variables like
    # B2_ACCOUNT_ID, B2_ACCOUNT_KEY, RESTIC_REPOSITORY etc.
    source /etc/restic/b2_env.sh


    # NOTE start all commands in background and wait for them to finish.
    # Reason: bash ignores any signals while child process is executing and thus my trap exit hook is not triggered.
    # However if put in subprocesses, wait(1) waits until the process finishes OR signal is received.
    # Reference: https://unix.stackexchange.com/questions/146756/forward-sigterm-to-child-in-bash

    # Remove locks from other stale processes to keep the automated backup running.
    restic unlock &
    wait $!

    # See restic-backup(1) or http://restic.readthedocs.io/en/latest/040_backup.html
    #restic backup --tag $BACKUP_TAG --one-file-system $BACKUP_EXCLUDES $BACKUP_PATHS &
    #wait $!

    # Until
    # https://github.com/restic/restic/issues/1557
    # is fixed with the PR
    # https://github.com/restic/restic/pull/1494
    # we have to use a work-around and skip the --one-file-system and explicitly black-list the paths we don't want, as described here
    # https://forum.restic.net/t/full-system-restore/126/8?u=fd0
    restic backup \
    --tag $BACKUP_TAG \
    --exclude-file /.restic-excludes \
    $BACKUP_EXCLUDES \
    / &
    wait $!

    # See restic-forget(1) or http://restic.readthedocs.io/en/latest/060_forget.html
    restic forget \
    --tag $BACKUP_TAG \
    --keep-daily $RETENTION_DAYS \
    --keep-weekly $RETENTION_WEEKS \
    --keep-monthly $RETENTION_MONTHS \
    --keep-yearly $RETENTION_YEARS &
    wait $!

    # Remove old data not linked anymore.
    # See restic-prune(1) or http://restic.readthedocs.io/en/latest/060_forget.html
    restic prune &
    wait $!


    # Check repository for errors.
    # NOTE this takes much time (and data transfer from remote repo?), do this in a separate systemd.timer which is run less often.
    #restic check &
    #wait $!
    38 changes: 0 additions & 38 deletions restic_check.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -1,38 +0,0 @@
    #!/usr/bin/env bash
    # Check my backup with restic to Backblaze B2 for errors.
    # This script is typically run by: /etc/systemd/system/restic-check.{service,timer}

    # Exit on failure, pipe failure
    set -e -o pipefail

    # Redirect stdout ( > ) into a named pipe ( >() ) running "tee" to a file, so we can observe the status by simply tailing the log file.
    me=$(basename "$0")
    now=$(date +%F_%R)
    log_dir=/var/local/log/restic
    log_file="${log_dir}/${now}_${me}.$$.log"
    test -d $log_dir || mkdir -p $log_dir
    exec > >(tee -i $log_file)
    exec 2>&1

    # Clean up lock if we are killed.
    # If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses.
    # However if we kill this script ourselves, we need this trap that kills all subprocesses manually.
    exit_hook() {
    echo "In exit_hook(), being killed" >&2
    jobs -p | xargs kill
    restic unlock
    }
    trap exit_hook INT TERM



    source /etc/restic/b2_env.sh

    # Remove locks from other stale processes to keep the automated backup running.
    # NOTE nope, dont' unlock liek restic_backup.sh. restic_backup.sh should take preceedance over this script.
    #restic unlock &
    #wait $!

    # Check repository for errors.
    restic check &
    wait $!
    11 changes: 0 additions & 11 deletions [email protected]
    View file
    Original file line number Diff line number Diff line change
    @@ -1,11 +0,0 @@
    # Source: https://serverfault.com/questions/876233/how-to-send-an-email-if-a-systemd-service-is-restarted
    # Source: https://wiki.archlinux.org/index.php/Systemd/Timers#MAILTO

    [Unit]
    Description=Send status email for %i to user

    [Service]
    Type=oneshot
    ExecStart=/usr/local/sbin/systemd-email [email protected] %i
    User=root
    Group=systemd-journal
    61 changes: 0 additions & 61 deletions systemd-email
    View file
    Original file line number Diff line number Diff line change
    @@ -1,61 +0,0 @@
    #!/usr/bin/env sh
    # Send email notification from systemd.
    # Source: https://serverfault.com/questions/876233/how-to-send-an-email-if-a-systemd-service-is-restarted
    # Source: https://wiki.archlinux.org/index.php/Systemd/Timers#MAILTO
    # Usage: systemd-email <recipinent-email> <failed-systemd-unit-name>


    # According to
    # http://www.flashissue.com/blog/gmail-sending-limits/
    # Gmail blocks your account if you send more than 500 emails per day, which is one email every
    # (24 * 60 * 60) / 500 = 172.8 second => choose a min wait time which is significantly longer than this to be on the safe time to not exceed 500 emails per day.
    # However this source
    # https://group-mail.com/sending-email/email-send-limits-and-options/
    # says the limit when not using the Gmail webinterface but going directly to the SMTP server is 100-150 per day, which yelds maximum one email every
    # (24 * 60 * 60) / 100 = 864 second
    # One option that I used with my old Axis cameras it to use my gmx.com accunt for sending emails instead, as there are (no?) higher limits there.
    MIN_WAIT_TIME_S=900
    SCRIPT_NAME=$(basename $0)
    LAST_RUN_FILE="/tmp/${SCRIPT_NAME}_last_run.txt"

    last_touch() {
    stat -c %Y $1
    }

    waited_long_enough() {
    retval=1
    if [ -e $LAST_RUN_FILE ]; then
    now=$(date +%s)
    last=$(last_touch $LAST_RUN_FILE)
    wait_s=$(expr $now - $last)
    if [ "$wait_s" -gt "$MIN_WAIT_TIME_S" ]; then
    retval=0
    fi
    else
    retval=0
    fi

    [ $retval -eq 0 ] && touch $LAST_RUN_FILE
    return $retval
    }


    # Make sure that my Gmail account dont' get shut down because of sending too many emails!
    if ! waited_long_enough; then
    echo "Systemd email was not sent, as it's less than ${MIN_WAIT_TIME_S} seconds since the last one was sent."
    exit 1
    fi


    recipinent=$1
    system_unit=$2

    sendmail -t <<ERRMAIL
    To: $recipinent
    From: systemd <root@$HOSTNAME>
    Subject: [systemd-email] ${system_unit}
    Content-Transfer-Encoding: 8bit
    Content-Type: text/plain; charset=UTF-8
    $(systemctl status --full "$system_unit")
    ERRMAIL
  3. erikw revised this gist Feb 14, 2018. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion restic_backup.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -75,7 +75,7 @@ wait $!

    # Remove old data not linked anymore.
    # See restic-prune(1) or http://restic.readthedocs.io/en/latest/060_forget.html
    restic forget &
    restic prune &
    wait $!


  4. erikw revised this gist Feb 13, 2018. 1 changed file with 6 additions and 1 deletion.
    7 changes: 6 additions & 1 deletion restic_backup.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -67,13 +67,18 @@ wait $!
    # See restic-forget(1) or http://restic.readthedocs.io/en/latest/060_forget.html
    restic forget \
    --tag $BACKUP_TAG \
    --prune \
    --keep-daily $RETENTION_DAYS \
    --keep-weekly $RETENTION_WEEKS \
    --keep-monthly $RETENTION_MONTHS \
    --keep-yearly $RETENTION_YEARS &
    wait $!

    # Remove old data not linked anymore.
    # See restic-prune(1) or http://restic.readthedocs.io/en/latest/060_forget.html
    restic forget &
    wait $!


    # Check repository for errors.
    # NOTE this takes much time (and data transfer from remote repo?), do this in a separate systemd.timer which is run less often.
    #restic check &
  5. erikw revised this gist Feb 13, 2018. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion systemd-email
    View file
    Original file line number Diff line number Diff line change
    @@ -12,7 +12,7 @@
    # However this source
    # https://group-mail.com/sending-email/email-send-limits-and-options/
    # says the limit when not using the Gmail webinterface but going directly to the SMTP server is 100-150 per day, which yelds maximum one email every
    # (24 * 60 * 60) / 500 = 864 second
    # (24 * 60 * 60) / 100 = 864 second
    # One option that I used with my old Axis cameras it to use my gmx.com accunt for sending emails instead, as there are (no?) higher limits there.
    MIN_WAIT_TIME_S=900
    SCRIPT_NAME=$(basename $0)
  6. erikw revised this gist Feb 13, 2018. 1 changed file with 5 additions and 1 deletion.
    6 changes: 5 additions & 1 deletion restic_backup.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -57,7 +57,11 @@ wait $!
    # https://github.com/restic/restic/pull/1494
    # we have to use a work-around and skip the --one-file-system and explicitly black-list the paths we don't want, as described here
    # https://forum.restic.net/t/full-system-restore/126/8?u=fd0
    restic backup --tag $BACKUP_TAG --exclude-file /.restic-excludes $BACKUP_EXCLUDES / &
    restic backup \
    --tag $BACKUP_TAG \
    --exclude-file /.restic-excludes \
    $BACKUP_EXCLUDES \
    / &
    wait $!

    # See restic-forget(1) or http://restic.readthedocs.io/en/latest/060_forget.html
  7. erikw revised this gist Feb 13, 2018. 1 changed file with 2 additions and 1 deletion.
    3 changes: 2 additions & 1 deletion restic_backup.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -33,7 +33,8 @@ BACKUP_PATHS="/ /boot /home /mnt/media"
    BACKUP_EXCLUDES="--exclude-file /.rsync_exclude --exclude-file /mnt/media/.rsync_exclude --exclude-file /home/erikw/.rsync_exclude"
    BACKUP_TAG=systemd.timer


    # Set all environment variables like
    # B2_ACCOUNT_ID, B2_ACCOUNT_KEY, RESTIC_REPOSITORY etc.
    source /etc/restic/b2_env.sh


  8. erikw revised this gist Feb 13, 2018. No changes.
  9. erikw revised this gist Jan 27, 2018. No changes.
  10. erikw revised this gist Jan 27, 2018. No changes.
  11. erikw revised this gist Jan 27, 2018. 6 changed files with 109 additions and 0 deletions.
    10 changes: 10 additions & 0 deletions restic-backup.service
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,10 @@
    [Unit]
    Description=Backup with restic to Backblaze B2
    OnFailure=status-email-user@%n.service

    [Service]
    Type=simple
    Nice=10
    ExecStart=/usr/local/sbin/restic_backup.sh
    # $HOME or $XDG_CACHE_HOME must be set for restic to find /root/.cache/restic/
    Environment="HOME=/root"
    9 changes: 9 additions & 0 deletions restic-backup.timer
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,9 @@
    [Unit]
    Description=Backup with restic on schedule

    [Timer]
    OnCalendar=daily
    Persistent=true

    [Install]
    WantedBy=timers.target
    9 changes: 9 additions & 0 deletions restic-check.service
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,9 @@
    [Unit]
    Description=Check restic backup Backblaze B2 for errors
    OnFailure=status-email-user@%n.service
    Conflicts=restic.service

    [Service]
    Type=simple
    Nice=10
    ExecStart=/usr/local/sbin/restic_check.sh
    9 changes: 9 additions & 0 deletions restic-check.timer
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,9 @@
    [Unit]
    Description=Check restic backup Backblaze B2 for errors on a schedule

    [Timer]
    OnCalendar=monthly
    Persistent=true

    [Install]
    WantedBy=timers.target
    11 changes: 11 additions & 0 deletions [email protected]
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,11 @@
    # Source: https://serverfault.com/questions/876233/how-to-send-an-email-if-a-systemd-service-is-restarted
    # Source: https://wiki.archlinux.org/index.php/Systemd/Timers#MAILTO

    [Unit]
    Description=Send status email for %i to user

    [Service]
    Type=oneshot
    ExecStart=/usr/local/sbin/systemd-email [email protected] %i
    User=root
    Group=systemd-journal
    61 changes: 61 additions & 0 deletions systemd-email
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,61 @@
    #!/usr/bin/env sh
    # Send email notification from systemd.
    # Source: https://serverfault.com/questions/876233/how-to-send-an-email-if-a-systemd-service-is-restarted
    # Source: https://wiki.archlinux.org/index.php/Systemd/Timers#MAILTO
    # Usage: systemd-email <recipinent-email> <failed-systemd-unit-name>


    # According to
    # http://www.flashissue.com/blog/gmail-sending-limits/
    # Gmail blocks your account if you send more than 500 emails per day, which is one email every
    # (24 * 60 * 60) / 500 = 172.8 second => choose a min wait time which is significantly longer than this to be on the safe time to not exceed 500 emails per day.
    # However this source
    # https://group-mail.com/sending-email/email-send-limits-and-options/
    # says the limit when not using the Gmail webinterface but going directly to the SMTP server is 100-150 per day, which yelds maximum one email every
    # (24 * 60 * 60) / 500 = 864 second
    # One option that I used with my old Axis cameras it to use my gmx.com accunt for sending emails instead, as there are (no?) higher limits there.
    MIN_WAIT_TIME_S=900
    SCRIPT_NAME=$(basename $0)
    LAST_RUN_FILE="/tmp/${SCRIPT_NAME}_last_run.txt"

    last_touch() {
    stat -c %Y $1
    }

    waited_long_enough() {
    retval=1
    if [ -e $LAST_RUN_FILE ]; then
    now=$(date +%s)
    last=$(last_touch $LAST_RUN_FILE)
    wait_s=$(expr $now - $last)
    if [ "$wait_s" -gt "$MIN_WAIT_TIME_S" ]; then
    retval=0
    fi
    else
    retval=0
    fi

    [ $retval -eq 0 ] && touch $LAST_RUN_FILE
    return $retval
    }


    # Make sure that my Gmail account dont' get shut down because of sending too many emails!
    if ! waited_long_enough; then
    echo "Systemd email was not sent, as it's less than ${MIN_WAIT_TIME_S} seconds since the last one was sent."
    exit 1
    fi


    recipinent=$1
    system_unit=$2

    sendmail -t <<ERRMAIL
    To: $recipinent
    From: systemd <root@$HOSTNAME>
    Subject: [systemd-email] ${system_unit}
    Content-Transfer-Encoding: 8bit
    Content-Type: text/plain; charset=UTF-8
    $(systemctl status --full "$system_unit")
    ERRMAIL
  12. erikw revised this gist Jan 27, 2018. 1 changed file with 38 additions and 0 deletions.
    38 changes: 38 additions & 0 deletions restic_check.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,38 @@
    #!/usr/bin/env bash
    # Check my backup with restic to Backblaze B2 for errors.
    # This script is typically run by: /etc/systemd/system/restic-check.{service,timer}

    # Exit on failure, pipe failure
    set -e -o pipefail

    # Redirect stdout ( > ) into a named pipe ( >() ) running "tee" to a file, so we can observe the status by simply tailing the log file.
    me=$(basename "$0")
    now=$(date +%F_%R)
    log_dir=/var/local/log/restic
    log_file="${log_dir}/${now}_${me}.$$.log"
    test -d $log_dir || mkdir -p $log_dir
    exec > >(tee -i $log_file)
    exec 2>&1

    # Clean up lock if we are killed.
    # If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses.
    # However if we kill this script ourselves, we need this trap that kills all subprocesses manually.
    exit_hook() {
    echo "In exit_hook(), being killed" >&2
    jobs -p | xargs kill
    restic unlock
    }
    trap exit_hook INT TERM



    source /etc/restic/b2_env.sh

    # Remove locks from other stale processes to keep the automated backup running.
    # NOTE nope, dont' unlock liek restic_backup.sh. restic_backup.sh should take preceedance over this script.
    #restic unlock &
    #wait $!

    # Check repository for errors.
    restic check &
    wait $!
  13. erikw revised this gist Jan 27, 2018. No changes.
  14. erikw created this gist Jan 27, 2018.
    75 changes: 75 additions & 0 deletions restic_backup.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,75 @@
    #!/usr/bin/env bash
    # Make backup my system with restic to Backblaze B2.
    # This script is typically run by: /etc/systemd/system/restic-backup.{service,timer}

    # Exit on failure, pipe failure
    set -e -o pipefail

    # Redirect stdout ( > ) into a named pipe ( >() ) running "tee" to a file, so we can observe the status by simply tailing the log file.
    me=$(basename "$0")
    now=$(date +%F_%R)
    log_dir=/var/local/log/restic
    log_file="${log_dir}/${now}_${me}.$$.log"
    test -d $log_dir || mkdir -p $log_dir
    exec > >(tee -i $log_file)
    exec 2>&1

    # Clean up lock if we are killed.
    # If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses.
    # However if we kill this script ourselves, we need this trap that kills all subprocesses manually.
    exit_hook() {
    echo "In exit_hook(), being killed" >&2
    jobs -p | xargs kill
    restic unlock
    }
    trap exit_hook INT TERM

    RETENTION_DAYS=7
    RETENTION_WEEKS=12
    RETENTION_MONTHS=18
    RETENTION_YEARS=4

    BACKUP_PATHS="/ /boot /home /mnt/media"
    BACKUP_EXCLUDES="--exclude-file /.rsync_exclude --exclude-file /mnt/media/.rsync_exclude --exclude-file /home/erikw/.rsync_exclude"
    BACKUP_TAG=systemd.timer


    source /etc/restic/b2_env.sh


    # NOTE start all commands in background and wait for them to finish.
    # Reason: bash ignores any signals while child process is executing and thus my trap exit hook is not triggered.
    # However if put in subprocesses, wait(1) waits until the process finishes OR signal is received.
    # Reference: https://unix.stackexchange.com/questions/146756/forward-sigterm-to-child-in-bash

    # Remove locks from other stale processes to keep the automated backup running.
    restic unlock &
    wait $!

    # See restic-backup(1) or http://restic.readthedocs.io/en/latest/040_backup.html
    #restic backup --tag $BACKUP_TAG --one-file-system $BACKUP_EXCLUDES $BACKUP_PATHS &
    #wait $!

    # Until
    # https://github.com/restic/restic/issues/1557
    # is fixed with the PR
    # https://github.com/restic/restic/pull/1494
    # we have to use a work-around and skip the --one-file-system and explicitly black-list the paths we don't want, as described here
    # https://forum.restic.net/t/full-system-restore/126/8?u=fd0
    restic backup --tag $BACKUP_TAG --exclude-file /.restic-excludes $BACKUP_EXCLUDES / &
    wait $!

    # See restic-forget(1) or http://restic.readthedocs.io/en/latest/060_forget.html
    restic forget \
    --tag $BACKUP_TAG \
    --prune \
    --keep-daily $RETENTION_DAYS \
    --keep-weekly $RETENTION_WEEKS \
    --keep-monthly $RETENTION_MONTHS \
    --keep-yearly $RETENTION_YEARS &
    wait $!

    # Check repository for errors.
    # NOTE this takes much time (and data transfer from remote repo?), do this in a separate systemd.timer which is run less often.
    #restic check &
    #wait $!