Deploying servers with full disk encryption + TPM2 key storage (with fips support)

Tested on MAAS 2.9 snap based install.

The target hosts contains a TPM 2.0 chip, and has UEFI SecureBoot enforced and enabled with default Microsoft keys.

The target install is ubuntu 20.04 on a 960gb disk presented to linux as /dev/sda. You will need to modify the curtin disk config if you want something different.

fips can be enabled after maas is finished using your pro subscription, for fips validation on encrypt/decrypt operations of application data written to the /root volume.

Step one:

Prepare a late_commands curtin command by base64 encoding the below script. This can be done using base64 the same way you would encode userdata. Adjust the /dev/sda3 if your encrypted root block device is on a different partition.

#!/bin/bash

# Generate random key file
cat /dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 64 > /root/.luks.key

# Initialize TPM2 chip
apt install -y tpm2-tools
tpm2_dictionarylockout --setup-parameters --max-tries=4294967295 --clear-lockout ; # ; to ignore errors intentionally
tpm2_nvundefine 0x1500016 ; # ; to ignore errors intentionally
tpm2_nvdefine -s 64 0x1500016

# Add random key file to TPM2 chip
tpm2_nvwrite -i /root/.luks.key 0x1500016

# Add random key file to luks volume
echo -n TempSetupPass | cryptsetup luksAddKey /dev/sda3 /root/.luks.key

# Remove setup password from luks volume. Comment this out if you want a failsafe password to remain.. the password is also in the storage: block of the curtin config.
echo -n TempSetupPass | cryptsetup luksRemoveKey /dev/sda3

# Setup initrd and crypttab to pull key file from TPM2 chip
cat << EOF > /usr/local/sbin/tpm2-getkey
#!/bin/sh
if [ -f ".tpm2-getkey.tmp" ]; then
/lib/cryptsetup/askpass "Automatic disk unlock via TPM failed for  () Enter passphrase: "
exit
fi
touch .tpm2-getkey.tmp
tpm2_nvread 0x1500016
EOF

cat << EOF > /etc/initramfs-tools/hooks/tpm2-decryptkey
#!/bin/sh
PREREQ=""
prereqs()
{
echo ""
}
case \$1 in
prereqs)
prereqs
exit 0
;;
esac
. /usr/share/initramfs-tools/hook-functions
copy_exec `which tpm2_nvread`
copy_exec /usr/lib/x86_64-linux-gnu/libtss2-tcti-device.so.0.0.0
copy_exec /usr/lib/x86_64-linux-gnu/libtss2-tcti-device.so.0
copy_exec /lib/cryptsetup/askpass
exit 0
EOF

chown root: /usr/local/sbin/tpm2-getkey
chmod 750 /usr/local/sbin/tpm2-getkey
chown root: /etc/initramfs-tools/hooks/tpm2-decryptkey
chmod 755 /etc/initramfs-tools/hooks/tpm2-decryptkey
sed -i 's%$%,keyscript=/usr/local/sbin/tpm2-getkey%' /etc/crypttab
update-initramfs -u

Step two:

Create the curtin configuration file:
/var/snap/maas/current/preseeds/curtin_userdata_ubuntu_amd64_generic_focal

With this contents (example includes the above script in base64 encoding):

#cloud-config
debconf_selections:
 maas: |
  {{for line in str(curtin_preseed).splitlines()}}
  {{line}}
  {{endfor}}

late_commands:
  maas: [wget, '--no-proxy', '{{node_disable_pxe_url}}', '--post-data', '{{node_disable_pxe_data}}', '-O', '/dev/null']
  ## setup_tpm contains luks tpm setup commands. Contents viewable with base64 --decode. Expects encrypted /root block device to be /dev/sda3
  setup_tpm: ["curtin", "in-target", "--", "sh", "-c", "echo -n 'CiMhL2Jpbi9iYXNoCgojIEdlbmVyYXRlIHJhbmRvbSBrZXkgZmlsZQpjYXQgL2Rldi91cmFuZG9tIHwgdHIgLWRjICdhLXpBLVowLTknIHwgaGVhZCAtYyA2NCA+IC9yb290Ly5sdWtzLmtleQoKIyBJbml0aWFsaXplIFRQTTIgY2hpcAphcHQgaW5zdGFsbCAteSB0cG0yLXRvb2xzCnRwbTJfZGljdGlvbmFyeWxvY2tvdXQgLS1zZXR1cC1wYXJhbWV0ZXJzIC0tbWF4LXRyaWVzPTQyOTQ5NjcyOTUgLS1jbGVhci1sb2Nrb3V0IDsgIyA7IHRvIGlnbm9yZSBlcnJvcnMgaW50ZW50aW9uYWxseQp0cG0yX252dW5kZWZpbmUgMHgxNTAwMDE2IDsgIyA7IHRvIGlnbm9yZSBlcnJvcnMgaW50ZW50aW9uYWxseQp0cG0yX252ZGVmaW5lIC1zIDY0IDB4MTUwMDAxNgoKIyBBZGQgcmFuZG9tIGtleSBmaWxlIHRvIFRQTTIgY2hpcAp0cG0yX252d3JpdGUgLWkgL3Jvb3QvLmx1a3Mua2V5IDB4MTUwMDAxNgoKIyBBZGQgcmFuZG9tIGtleSBmaWxlIHRvIGx1a3Mgdm9sdW1lCmVjaG8gLW4gcGVyY2lwaWVudHNldHVwIHwgY3J5cHRzZXR1cCBsdWtzQWRkS2V5IC9kZXYvc2RhMyAvcm9vdC8ubHVrcy5rZXkKCiMgUmVtb3ZlIHNldHVwIHBhc3N3b3JkIGZyb20gbHVrcyB2b2x1bWUKZWNobyAtbiBwZXJjaXBpZW50c2V0dXAgfCBjcnlwdHNldHVwIGx1a3NSZW1vdmVLZXkgL2Rldi9zZGEzCgojIFNldHVwIGluaXRyZCBhbmQgY3J5cHR0YWIgdG8gcHVsbCBrZXkgZmlsZSBmcm9tIFRQTTIgY2hpcApjYXQgPDwgRU9GID4gL3Vzci9sb2NhbC9zYmluL3RwbTItZ2V0a2V5CiMhL2Jpbi9zaAppZiBbIC1mICIudHBtMi1nZXRrZXkudG1wIiBdOyB0aGVuCi9saWIvY3J5cHRzZXR1cC9hc2twYXNzICJBdXRvbWF0aWMgZGlzayB1bmxvY2sgdmlhIFRQTSBmYWlsZWQgZm9yICAoKSBFbnRlciBwYXNzcGhyYXNlOiAiCmV4aXQKZmkKdG91Y2ggLnRwbTItZ2V0a2V5LnRtcAp0cG0yX252cmVhZCAweDE1MDAwMTYKRU9GCgpjYXQgPDwgRU9GID4gL2V0Yy9pbml0cmFtZnMtdG9vbHMvaG9va3MvdHBtMi1kZWNyeXB0a2V5CiMhL2Jpbi9zaApQUkVSRVE9IiIKcHJlcmVxcygpCnsKZWNobyAiIgp9CmNhc2UgXCQxIGluCnByZXJlcXMpCnByZXJlcXMKZXhpdCAwCjs7CmVzYWMKLiAvdXNyL3NoYXJlL2luaXRyYW1mcy10b29scy9ob29rLWZ1bmN0aW9ucwpjb3B5X2V4ZWMgYHdoaWNoIHRwbTJfbnZyZWFkYApjb3B5X2V4ZWMgL3Vzci9saWIveDg2XzY0LWxpbnV4LWdudS9saWJ0c3MyLXRjdGktZGV2aWNlLnNvLjAuMC4wCmNvcHlfZXhlYyAvdXNyL2xpYi94ODZfNjQtbGludXgtZ251L2xpYnRzczItdGN0aS1kZXZpY2Uuc28uMApjb3B5X2V4ZWMgL2xpYi9jcnlwdHNldHVwL2Fza3Bhc3MKZXhpdCAwCkVPRgoKY2hvd24gcm9vdDogL3Vzci9sb2NhbC9zYmluL3RwbTItZ2V0a2V5CmNobW9kIDc1MCAvdXNyL2xvY2FsL3NiaW4vdHBtMi1nZXRrZXkKY2hvd24gcm9vdDogL2V0Yy9pbml0cmFtZnMtdG9vbHMvaG9va3MvdHBtMi1kZWNyeXB0a2V5CmNobW9kIDc1NSAvZXRjL2luaXRyYW1mcy10b29scy9ob29rcy90cG0yLWRlY3J5cHRrZXkKc2VkIC1pICdzJSQlLGtleXNjcmlwdD0vdXNyL2xvY2FsL3NiaW4vdHBtMi1nZXRrZXklJyAvZXRjL2NyeXB0dGFiCnVwZGF0ZS1pbml0cmFtZnMgLXUKCg==' | base64 --decode > /tmp/luks.sh && chmod +x /tmp/luks.sh && sh -c /tmp/luks.sh && rm /tmp/luks.sh"]

## 960GB /dev/sda - /dev/sda1 1gb /boot/efi, /dev/sda2 1gb /boot, /dev/sda3 encrypted /root for the rest.
storage:
  config:
  - grub_device: true
    id: sda
    name: sda
    ptable: gpt
    path: /dev/sda
    type: disk
    wipe: superblock
  - device: sda
    id: sda-part1
    name: sda-part1
    number: 1
    offset: 4194304B
    size: 998244352B
    type: partition
    uuid: 6c50ebce-c6eb-4408-98fa-0099f4b29427
    wipe: superblock
  - device: sda
    id: sda-part2
    name: sda-part2
    number: 2
    size: 998244352B
    type: partition
    uuid: 98831932-ab49-46a9-a544-9b6fdbe0dd81
    wipe: superblock
  - device: sda
    id: sda-part3
    name: sda-part3
    number: 3
    size: 958192943104B
    type: partition
    uuid: b387caa6-a9f4-4325-a155-601aa0e47932
    wipe: superblock
  - fstype: fat32
    id: sda-part1_format
    label: ''
    type: format
    uuid: ac86d441-7ca4-4d5a-bd3f-9faa0f0d85db
    volume: sda-part1
  - fstype: ext4
    id: sda-part2_format
    label: ''
    type: format
    uuid: 3a1bfe64-e156-445b-bf82-f7ae965047e9
    volume: sda-part2
  - id: sda-part3_crypt
    type: dm_crypt
    dm_name: sda3_crypt
    volume: sda-part3
    key: TempSetupPass
    keysize: '512'
  - fstype: ext4
    id: sda-part3_format
    label: ''
    type: format
    uuid: 8d523602-b9d2-4d93-88cc-fa38902dbf71
    volume: sda-part3_crypt
  - device: sda-part3_format
    id: sda-part3_mount
    options: ''
    path: /
    type: mount
  - device: sda-part2_format
    id: sda-part2_mount
    options: ''
    path: /boot
    type: mount
  - device: sda-part1_format
    id: sda-part1_mount
    options: ''
    path: /boot/efi
    type: mount
  version: 1

Step Three:

Deploy an amd64 host with ubuntu 20.04 as the os version. The above preseed will override the disk configuration on the host automatically.

Notes:

maas 2.9 uses luks2 automatically, the original reference about needing to do a luks1->luks2 upgrade are no longer needed.

The UUIDs in the curtin config can be changed to anything you want, the only part that matters for templating is using path: instead of serial: on the disk device.

You can confirm everything worked after reboots are done by ssh’ing into the machine and running:

~# cryptsetup status sda3_crypt
/dev/mapper/sda3_crypt is active and is in use.
  type:    LUKS2
  cipher:  aes-xts-plain64
  keysize: 512 bits
  key location: keyring
  device:  /dev/sda3
  sector size:  512
  offset:  32768 sectors
  size:    1871437824 sectors
  mode:    read/write

Reference Material:

TPM2.0 on ubuntu 20.04 guide: https://run.tournament.org.il/ubuntu-20-04-and-tpm2-encrypted-system-disk/

Original Thread about MaaS disk encryption: https://discourse.maas.io/t/deploying-servers-with-full-disk-encryption-luks2/3286

Thanks so much for the guide!

We’ve tried the following configuration on the curtin script on the maas server for a system with LVM and encrypted root volume.
The resulting system doesn’t detect a bootable device and goes to grub shell.
Any ideas why it’s not working?

debconf_selections:
 maas: |
  {{for line in str(curtin_preseed).splitlines()}}
  {{line}}
  {{endfor}}

late_commands:
  maas: [wget, '--no-proxy', {{node_disable_pxe_url|escape.json}}, '--post-data', {{node_disable_pxe_data|escape.json}}, '-O', '/dev/null']

storage:
  config:
    - {type: disk, id: sdc-disk, name: sdc, path: /dev/sdc, ptable: gpt, wipe: superblock}
   
    - {type: partition, id: sdc-efi, device: sdc-disk, number: 1, size: 1024M, wipe: superblock, flag: boot, grub_device: true }
    - {type: partition, id: sdc-boot, device: sdc-disk, number: 2, size: 1024M, wipe: superblock}
    - {type: partition, id: sdc-root, device: sdc-disk, number: 3, size: 32000M, wipe: superblock}
    - {type: dm_crypt, id: sdc-root-crypt, dm_name: sdc-root-crypt, volume: sdc-root, key: setup, keysize: '512'}
    - {type: lvm_volgroup, id: vg-hdd, name: vg-hdd, devices: [sdc-root-crypt]}
    - {type: lvm_partition, id: lv-root-part, name: lv-root, volgroup: vg-hdd, size: 30000M}
    - {type: format, id: efi-format, volume: sdc-efi, fstype: fat32, label: efi}
    - {type: format, id: boot-format, volume: sdc-boot, fstype: ext4}
    - {type: format, id: lv-root-format, volume: lv-root-part, fstype: ext4}
    - {type: mount, id: efi-mount, device: efi-format, path: /boot/efi}
    - {type: mount, id: boot-mount, device: boot-format, path: /boot, options: 'defaults,noatime,errors=remount-ro'}
    - {type: mount, id: root-mount, device: lv-root-format, path: /, options: 'defaults,noatime,errors=remount-ro'}

Troubleshooting:

As the above (straightforward) approach didn’t work, we tried experimenting…
Our current server deployments contain a mirrored (unencrypted) LVM.
As troubleshooting we tried to start with a linear LVM installed on sdc and create an encrypted root mirror on sdd.

In this case the system boots, as the unencrypted volume works as expected, but the encrypted root volume is not mounted/decrypted on time (the lvm mounts 1 out of 2 mirrors). We get a prompt to decrypt the volume right afterwards.

Below is a screenshot from the failed mounting and decryption promt.

/etc/crypttab file was updated manually with a line similar to:

sdd3_crypt UUID=${sdd_uuid} none luks,initramfs

added the following modules on /etc/initramfs-tools/modules

dm_crypt
dm_mod
sha256
aes
aes_x86_64
crc32c
algif_skcipher
md_mod
raid1
raid456
dm_raid
dm_log
dm_region_hash
dm_mirror
lvm

(Ran update-initramfs -u afterwards)

Also tried to add something like the following on /etc/default/grub

GRUB_CMDLINE_LINUX="cryptdevice=UUID={uuid}:sdd3_crypt root=/dev/vg-hdd/lv-root"
GRUB_PRELOAD_MODULES="luks cryptodisk lvm ext4"

(run update-grub afterwards)

Any hints/advice are welcome :slight_smile:

I’m not sure if setting “grub-device” on type: partition sdc-efi in your configuration will work. you probably need to move that to the type: disk sdc above it.

1 Like

You’re right, the grub-device needs to be at the disk type. It was a typo, i copy pasted the wrong file.
I tried again today with this file and ended up on grub again :confused:
If i remove the dm_crypt type and adjust the format type to use the root partition, then the machine boots up ok.

debconf_selections:
 maas: |
  {{for line in str(curtin_preseed).splitlines()}}
  {{line}}
  {{endfor}}

late_commands:
  maas: [wget, '--no-proxy', {{node_disable_pxe_url|escape.json}}, '--post-data', {{node_disable_pxe_data|escape.json}}, '-O', '/dev/null']

storage:
  config:
    - {type: disk, id: sda-disk, name: sda, path: /dev/sda, ptable: gpt, wipe: superblock, grub_device: false}
    - {type: disk, id: sdb-disk, name: sdb, path: /dev/sdb, ptable: gpt, wipe: superblock, grub_device: false}
    - {type: disk, id: sdc-disk, name: sdc, path: /dev/sdc, ptable: gpt, wipe: superblock, grub_device: true}
    - {type: disk, id: sdd-disk, name: sdd, path: /dev/sdd, ptable: gpt, wipe: superblock, grub_device: false}
    - {type: partition, id: sdc-efi, device: sdc-disk, number: 1, size: 1G, wipe: superblock, flag: boot }
    - {type: partition, id: sdc-boot, device: sdc-disk, number: 2, size: 1G, wipe: superblock}
    - {type: partition, id: sdc-root, device: sdc-disk, number: 3, size: 32G, wipe: superblock}
    - {type: dm_crypt, id: sdc-root-crypt, dm_name: sdc-root-crypt, volume: sdc-root, key: setup, keysize: '512'}
    - {type: format, id: efi-format, volume: sdc-efi, fstype: fat32, label: efi}
    - {type: format, id: boot-format, volume: sdc-boot, fstype: ext4, label: boot}
    - {type: format, id: root-format, volume: sdc-root-crypt, fstype: ext4, label: root}
    - {type: mount, id: efi-mount, device: efi-format, path: /boot/efi}
    - {type: mount, id: boot-mount, device: boot-format, path: /boot, options: 'auto,noatime,errors=remount-ro'}
    - {type: mount, id: root-mount, device: root-format, path: /, options: 'auto,noatime,errors=remount-ro'}

I revisted this while getting mdadm raid 1 working, and found out a few things are very important.

You need to set a uuid on partitions, mdadm, luks, and filesystems (except fat32 which doesn’t use uuid)

The order you list mounts is also very important. list your / first, then /boot, then /boot/efi, else the chroot inside the installer won’t work right.

sample working mdadm raid 1 setup (and yes, these uuid’s work fine):

storage:
  config:
  - grub_device: true
    id: sda
    name: sda
    ptable: gpt
    type: disk
    wipe: superblock
    path: /dev/sda
  - grub_device: true
    id: sdb
    name: sdb
    ptable: gpt
    type: disk
    wipe: superblock
    path: /dev/sdb
  - device: sda
    flag: boot
    id: sda-part1
    name: sda-part1
    number: 1
    offset: 4194304B
    size: 998244352B
    type: partition
    wipe: superblock
    uuid: 11111111-1111-1111-0000-000000000101
  - device: sda
    id: sda-part2
    name: sda-part2
    number: 2
    size: 998244352B
    type: partition
    wipe: superblock
    uuid: 11111111-1111-1111-0000-000000000102
  - device: sda
    id: sda-part3
    name: sda-part3
    number: 3
    size: 100G
    type: partition
    wipe: superblock
    uuid: 11111111-1111-1111-0000-000000000103
  - device: sdb
    flag: boot
    id: sdb-part1
    name: sdb-part1
    number: 1
    offset: 4194304B
    size: 998244352B
    type: partition
    wipe: superblock
    uuid: 11111111-1111-1111-0000-000000000201
  - device: sdb
    id: sdb-part2
    name: sdb-part2
    number: 2
    size: 998244352B
    type: partition
    wipe: superblock
    uuid: 11111111-1111-1111-0000-000000000202
  - device: sdb
    id: sdb-part3
    name: sdb-part3
    number: 3
    size: 100G
    type: partition
    wipe: superblock
    uuid: 11111111-1111-1111-0000-000000000203
  - devices:
    - sda-part3
    - sdb-part3
    id: md1
    name: md1
    raidlevel: 1
    spare_devices: []
    type: raid
    uuid: 11111111-1111-1111-0001-000000000001
  - id: md1_crypt
    type: dm_crypt
    dm_name: md1_crypt
    volume: md1
    key: TemporarySetupKey
    keysize: '512'
    uuid: 11111111-1111-1111-0002-000000000001
  - fstype: ext4
    id: md1_format
    label: ''
    type: format
    volume: md1_crypt
    uuid: 11111111-1111-1111-0003-000000000001
  - device: md1_format
    id: md1_mount
    options: ''
    path: /
    type: mount
  - devices:
    - sda-part2
    - sdb-part2
    id: md0
    name: md0
    raidlevel: 1
    spare_devices: []
    type: raid
    uuid: 11111111-1111-1111-0001-000000000000
  - fstype: ext4
    id: md0_format
    label: ''
    type: format
    volume: md0
    uuid: 11111111-1111-1111-0003-000000000000
  - device: md0_format
    id: md0_mount
    options: ''
    path: /boot
    type: mount 
  - fstype: fat32
    id: sda-part1_format
    label: efi
    type: format
    volume: sda-part1
  - fstype: fat32
    id: sdb-part1_format
    label: efi
    type: format
    volume: sdb-part1
  - device: sda-part1_format
    id: sda-part1_mount
    options: ''
    path: /boot/efi
    type: mount