Last active
February 22, 2019 12:50
-
-
Save jasonrm/5b887d78b0b58dc06f1809830b358562 to your computer and use it in GitHub Desktop.
ceph encrypted bluestore osd
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| mkdir -p /var/lib/ceph/mds/ceph-$(hostname -s)/ | |
| ceph-authtool --create-keyring /var/lib/ceph/mds/ceph-$(hostname -s)/keyring --gen-key -n mds.$(hostname -s) | |
| ceph auth add mds.$(hostname -s) osd "allow rwx" mds "allow" mon "allow profile mds" -i /var/lib/ceph/mds/ceph-$(hostname -s)/keyring | |
| sudo chown -R ceph:ceph /var/lib/ceph | |
| systemctl enable ceph-mds@$(hostname -s) | |
| systemctl start ceph-mds@$(hostname -s) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| mkdir -p /etc/luks/ | |
| DRIVES=(sdc sdd sde sdf) | |
| for DRIVE in $DRIVES; do | |
| sgdisk --zap-all /dev/${DRIVE} | |
| sleep 5 | |
| sgdisk --new 1:0:+128M --typecode 1:FD00 /dev/${DRIVE} | |
| sgdisk --new 2:0:0 --typecode 1:FD00 /dev/${DRIVE} | |
| sleep 5 | |
| partprobe | |
| sleep 5 | |
| OSD_NUM=$(ceph osd create) | |
| for PART_NUM in 1 2; do | |
| PART_UUID=$(blkid /dev/${DRIVE}${PART_NUM} -s PARTUUID -o value) | |
| dd bs=512 count=4 if=/dev/urandom of=/etc/luks/${PART_UUID}.key | |
| cryptsetup luksFormat --batch-mode --key-file=/etc/luks/${PART_UUID}.key /dev/disk/by-partuuid/${PART_UUID} | |
| cryptsetup luksOpen --batch-mode --key-file=/etc/luks/${PART_UUID}.key /dev/disk/by-partuuid/${PART_UUID} osd-${OSD_NUM}-${PART_NUM} | |
| echo "osd-${OSD_NUM}-${PART_NUM} PARTUUID=${PART_UUID} /etc/luks/${PART_UUID}.key luks,timeout=180" >> /etc/crypttab | |
| if [ $PART_NUM -eq 1 ]; then | |
| mkfs.xfs /dev/mapper/osd-${OSD_NUM}-${PART_NUM} | |
| mkdir -p /var/lib/ceph/osd/ceph-${OSD_NUM} | |
| echo "/dev/mapper/osd-${OSD_NUM}-${PART_NUM} /var/lib/ceph/osd/ceph-${OSD_NUM} xfs defaults 0 2" >> /etc/fstab | |
| fi | |
| done | |
| cat >> /etc/ceph/ceph.conf <<DELIM | |
| [osd.${OSD_NUM}] | |
| host = $(hostname -s) | |
| osd data = /var/lib/ceph/osd/ceph-${OSD_NUM} | |
| bluestore block path = /dev/mapper/osd-${OSD_NUM}-2 | |
| DELIM | |
| done |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| mkdir -p /etc/luks/ | |
| DRIVES=(sdd sde) | |
| for DRIVE in $DRIVES; do | |
| sgdisk --zap-all /dev/${DRIVE} | |
| sgdisk --new 1:0:+128M --typecode 1:FD00 /dev/${DRIVE} | |
| sgdisk --new 2:0:+20G --typecode 1:FD00 /dev/${DRIVE} | |
| sgdisk --new 3:0:+1G --typecode 1:FD00 /dev/${DRIVE} | |
| sgdisk --new 4:0:0 --typecode 1:FD00 /dev/${DRIVE} | |
| sleep 1 | |
| partprobe | |
| OSD_NUM=$(ceph osd create) | |
| for PART_NUM in 1 2 3 4; do | |
| PART_UUID=$(blkid /dev/${DRIVE}${PART_NUM} -s PARTUUID -o value) | |
| dd bs=512 count=4 if=/dev/urandom of=/etc/luks/${PART_UUID}.key | |
| cryptsetup luksFormat --batch-mode --key-file=/etc/luks/${PART_UUID}.key /dev/disk/by-partuuid/${PART_UUID} | |
| cryptsetup luksOpen --batch-mode --key-file=/etc/luks/${PART_UUID}.key /dev/disk/by-partuuid/${PART_UUID} osd-${OSD_NUM}-${PART_NUM} | |
| echo "osd-${OSD_NUM}-${PART_NUM} PARTUUID=${PART_UUID} /etc/luks/${PART_UUID}.key luks,timeout=180" >> /etc/crypttab | |
| if [ $PART_NUM -eq 1 ]; then | |
| mkfs.xfs /dev/mapper/osd-${OSD_NUM}-${PART_NUM} | |
| mkdir -p /var/lib/ceph/osd/ceph-${OSD_NUM} | |
| echo "/dev/mapper/osd-${OSD_NUM}-${PART_NUM} /var/lib/ceph/osd/ceph-${OSD_NUM} xfs defaults 0 2" >> /etc/fstab | |
| fi | |
| done | |
| cat >> /etc/ceph/ceph.conf <<DELIM | |
| [osd.${OSD_NUM}] | |
| host = $(hostname -s) | |
| osd data = /var/lib/ceph/osd/ceph-${OSD_NUM} | |
| bluestore block path = /dev/mapper/osd-${OSD_NUM}-4 | |
| bluestore block db path = /dev/mapper/osd-${OSD_NUM}-2 | |
| bluestore block wal path = /dev/mapper/osd-${OSD_NUM}-3 | |
| DELIM | |
| done |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| DRIVES=(sde) | |
| for DRIVE in $DRIVES; do | |
| sgdisk --zap-all /dev/${DRIVE} | |
| sgdisk \ | |
| --new 1:0:+128M --typecode 1:8300 \ | |
| --new 2:0:+20G --typecode 2:fd00 \ | |
| --new 3:0:+1G --typecode 3:fd00 \ | |
| --new 4:0:0 --typecode 4:fd00 \ | |
| /dev/${DRIVE} | |
| sleep 5 | |
| # OSD_NUM=13 | |
| OSD_NUM=$(ceph osd create) | |
| for PART_NUM in 1 2 3 4; do | |
| PART_UUID=$(blkid /dev/${DRIVE}${PART_NUM} -s PARTUUID -o value) | |
| if [ $PART_NUM -eq 1 ]; then | |
| mkfs.xfs -f /dev/disk/by-partuuid/${PART_UUID} | |
| mkdir -p /var/lib/ceph/osd/ceph-${OSD_NUM} | |
| echo "PARTUUID=${PART_UUID} /var/lib/ceph/osd/ceph-${OSD_NUM} xfs defaults 0 2" >> /etc/fstab | |
| fi | |
| done | |
| PART_UUID_BLOCK=$(blkid /dev/${DRIVE}4 -s PARTUUID -o value) | |
| PART_UUID_DB=$(blkid /dev/${DRIVE}2 -s PARTUUID -o value) | |
| PART_UUID_WAL=$(blkid /dev/${DRIVE}3 -s PARTUUID -o value) | |
| cat >> /etc/ceph/ceph.conf <<DELIM | |
| [osd.${OSD_NUM}] | |
| host = $(hostname -s) | |
| osd data = /var/lib/ceph/osd/ceph-${OSD_NUM} | |
| osd objectstore = bluestore | |
| bluestore block path = /dev/disk/by-partuuid/${PART_UUID_BLOCK} | |
| bluestore block db path = /dev/disk/by-partuuid/${PART_UUID_DB} | |
| bluestore block wal path = /dev/disk/by-partuuid/${PART_UUID_WAL} | |
| DELIM | |
| #ceph-osd -i ${OSD_NUM} --mkfs --mkkey | |
| #ceph auth add osd.${OSD_NUM} osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-${OSD_NUM}/keyring | |
| #ceph osd crush add osd.${OSD_NUM} 1.0 host=$(hostname -s) | |
| done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment