sysadminjohn · September 13, 2022 13:34 · May 12, 2022 · Sep 24, 2021 · Sep 24, 2021 · Oct 15, 2019
diff --git a/btrfs-guide.md b/btrfs-guide.md
@@ -65,7 +65,7 @@ Documentation=man:btrfs-scrub
 
 [Timer]
 # first saturday each month
-OnCalendar=Sun *-*-1..7 3:00:00
+OnCalendar=Sat *-*-1..7 3:00:00
 RandomizedDelaySec=10min
 
 [Install]

diff --git a/btrfs-guide.md b/btrfs-guide.md
@@ -1,4 +1,4 @@
-# Btrfs guide
+# Encrypted Btrfs storage setup and maintenance guide
 
 ## Initial setup with LUKS/dm-crypt
 
@@ -51,6 +51,41 @@ Automatically mount btrfs filesystem on boot by editing `/etc/fstab`:
 # Add option 'autodefrag' to allow automatic defragmentation: useful for files with lot of random writes like databases or virtual machine images
 ```
 
+## Maintenance
+
+In a btrfs raid setup it is necessary to frequently run a `btrfs scrub` to check for corrupted blocks/flipped bits and repair them using a healthy copy from one of the mirror disks.
+
+In the example below a systemd-timer is used to run an automatic `btrfs scrub` job each month.
+
+/etc/systemd/system/btrfs-scrub.timer:
+```
+[Unit]
+Description=Monthly scrub btrfs filesystem, verify block checksums
+Documentation=man:btrfs-scrub
+
+[Timer]
+# first saturday each month
+OnCalendar=Sun *-*-1..7 3:00:00
+RandomizedDelaySec=10min
+
+[Install]
+WantedBy=timers.target
+```
+
+/etc/systemd/system/btrfs-scrub.service:
+```
+[Unit]
+Description=Scrub btrfs filesystem, verify block checksums
+Documentation=man:btrfs-scrub
+
+[Service]
+Type=simple
+ExecStart=/bin/btrfs scrub start -Bd /mnt/data
+KillSignal=SIGINT
+IOSchedulingClass=idle
+CPUSchedulingPolicy=idle
+```
+
 ## Recovery from device failure
 
 Example with one failed device:

diff --git a/btrfs-guide.md b/btrfs-guide.md
@@ -0,0 +1,85 @@
+# Btrfs guide
+
+## Initial setup with LUKS/dm-crypt
+
+This exemplary initial setup uses two devices `/dev/sdb` and `/dev/sdc` but can be applied to any amount of devices by following the steps with additional devices.
+
+Create keyfile:
+```sh
+dd bs=64 count=1 if=/dev/urandom of=/etc/cryptkey iflag=fullblock
+chmod 600 /etc/cryptkey
+```
+
+Encrypt devices:
+```sh
+cryptsetup -v -c aes-xts-plain64 -h sha512 -s 512 luksFormat /dev/sdb /etc/cryptkey
+cryptsetup -v -c aes-xts-plain64 -h sha512 -s 512 luksFormat /dev/sdc /etc/cryptkey
+```
+
+Backup LUKS header:
+```sh
+cryptsetup luksHeaderBackup --header-backup-file ~/sdb.header.bak /dev/sdb
+cryptsetup luksHeaderBackup --header-backup-file ~/sdc.header.bak /dev/sdc
+```
+
+Automatically unlock LUKS devices on boot by editing `/etc/crypttab`:
+```sh
+data1 UUID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx /etc/cryptkey luks,noearly #,discard (for SSDs)
+data2 UUID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx /etc/cryptkey luks,noearly #,discard (for SSDs)
+# Use 'blkid /dev/sdb' to get the UUID
+```
+
+Unlock encrypted devices now to create the filesystem in next step:
+```sh
+cryptsetup open --key-file=/etc/cryptkey --type luks /dev/sdb data1
+cryptsetup open --key-file=/etc/cryptkey --type luks /dev/sdc data2
+```
+
+Create filesystem:
+```sh
+mkfs.btrfs -m raid1 -d raid1 /dev/mapper/data1 /dev/mapper/data2
+```
+
+Mount filesystem:
+```sh
+mount -t btrfs -o defaults,noatime,compress=zstd /dev/mapper/data1 /mnt/data
+```
+
+Automatically mount btrfs filesystem on boot by editing `/etc/fstab`:
+```sh
+/dev/mapper/data1 /mnt/data btrfs defaults,noatime,compress=zstd 0 2
+# Add option 'autodefrag' to allow automatic defragmentation: useful for files with lot of random writes like databases or virtual machine images
+```
+
+## Recovery from device failure
+
+Example with one failed device:
+- `/dev/mapper/data1` working device
+- `/dev/mapper/data2` failed device
+- `/dev/mapper/data3` new device
+- `/mnt/data` mountpoint
+
+In case of failing/failed device, mount in degraded mode with the working device:
+```sh
+mount -t btrfs -o defaults,noatime,compress=zstd,degraded /dev/mapper/data1 /mnt/data
+```
+
+NOTE: Encrypt the new device before using it in the btrfs raid by following the steps above. Add new device to btrfs raid:
+```sh
+btrfs device add /dev/mapper/data3 /mnt/data
+```
+
+Remove the missing device (NOTE: `missing` is a special device name and not a placeholder):
+```sh
+btrfs device delete missing /mnt/data
+```
+
+Re-balance data of btrfs raid:
+```sh
+btrfs balance start /mnt/data
+```
+
+The fstab entry is left unmodified with one of the working devices:
+```
+/dev/mapper/data1 /mnt/data btrfs defaults,noatime,compress=zstd 0 2
+```