use again bcachefs, but with multiple disks

3 files changed, 47 insertions, 93 deletions

diff --git a/mini/configuration.nix b/mini/configuration.nix
index e835ef6..aaa1f17 100644
--- a/mini/configuration.nix
+++ b/mini/configuration.nix
@@ -11,7 +11,7 @@
       ./hardware-configuration.nix
 
       # Shared config of all machines
-      /data/nixos/share/common.nix
+      /nix/data/nixos/share/common.nix
     ];
 
   # our hostname and an ID for ZFS
diff --git a/mini/hardware-configuration.nix b/mini/hardware-configuration.nix
index e02d525..9adbd15 100644
--- a/mini/hardware-configuration.nix
+++ b/mini/hardware-configuration.nix
@@ -10,18 +10,18 @@
   boot.initrd.kernelModules = [ "amdgpu" ];
   boot.kernelModules = [ "kvm-amd" ];
 
-  # efi partition
+  # /boot efi partition to boot in UEFI mode
   fileSystems."/boot" =
     { device = "/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746-part1";
       fsType = "vfat";
       neededForBoot = true;
     };
 
-  # vms
-  fileSystems."/home/cullmann/vms" =
-    { device = "vpool/vms";
-      fsType = "zfs";
-      depends = [ "/home" ];
+  # /nix encrypted bcachefs for the remaining space
+  fileSystems."/nix" =
+    { device = "/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746-part2:/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F";
+      fsType = "bcachefs";
+      neededForBoot = true;
     };
 
   nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
diff --git a/mini/install.txt b/mini/install.txt
index 8e59841..e456d83 100644
--- a/mini/install.txt
+++ b/mini/install.txt
@@ -2,11 +2,19 @@
 # enable ssh for root
 #
 
-systemctl start sshd
 sudo bash
+systemctl start sshd
 passwd
 
 #
+# install script below
+#
+
+# fix unlocking https://nixos.wiki/wiki/Bcachefs
+nix-env -iA nixos.keyutils
+keyctl link @u @s
+
+#
 # kill old efi boot stuff
 #
 
@@ -18,14 +26,11 @@ efibootmgr -b 3 -B
 efibootmgr -b 4 -B
 efibootmgr
 
-#
-# install script below
-#
-
 # Defining some helper variables (these will be used in later code
 # blocks as well, so make sure to use the same terminal session or
 # redefine them later)
 DISK=/dev/disk/by-id/nvme-CT4000P3PSSD8_2325E6E63746
+DISK2=/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F
 HOST=mini
 
 # ensure 4k sector size
@@ -40,6 +45,13 @@ blkdiscard -v $DISK
 wipefs -a $DISK
 gdisk -l $DISK
 
+# wipe second disk
+sgdisk --zap-all $DISK2
+blkdiscard -v $DISK2
+wipefs -a $DISK2
+
+sleep 5
+
 # create partitions
 parted $DISK -- mklabel gpt
 sgdisk -n 1:0:+1024M -c 1:"EFI System Partition" -t 1:EF00 $DISK
@@ -56,28 +68,8 @@ mkfs.fat -F 32 -n EFIBOOT $DISK-part1
 
 sleep 5
 
-# ZFS zpool creation with encryption
-zpool create \
-    -o ashift=12 \
-    -o autotrim=on \
-    -O acltype=posixacl \
-    -O atime=off \
-    -O canmount=off \
-    -O compression=on \
-    -O dnodesize=auto \
-    -O normalization=formD \
-    -O xattr=sa \
-    -O mountpoint=none \
-    -O encryption=on \
-    -O keylocation=prompt \
-    -O keyformat=passphrase \
-    zpool $DISK-part2
-
-sleep 5
-
-# create all the volumes
-zfs create -o mountpoint=legacy zpool/data
-zfs create -o mountpoint=legacy zpool/nix
+# create encrypted bcachefs over all disks, use only fast lz4 compression
+bcachefs format --block_size=4096 --errors=ro --compression=lz4 --wide_macs --acl --encrypted --fs_label=nix --discard -f $DISK-part2 $DISK2
 
 sleep 5
 
@@ -85,24 +77,27 @@ sleep 5
 mount -t tmpfs none /mnt
 
 # Create directories to mount file systems on
-mkdir -p /mnt/{data,nix,home,boot,root,etc/nixos}
+mkdir -p /mnt/{nix,home,boot,root,etc/nixos}
 
 # mount the ESP
 mount $DISK-part1 /mnt/boot
 
-# mount volumes
-mount -t zfs zpool/data /mnt/data
-mount -t zfs zpool/nix /mnt/nix
+# mount large bcachefs
+mount -t bcachefs $DISK-part2:$DISK2 /mnt/nix
+
+# ensure tmp fills not the RAM
+mkdir -p /mnt/tmp /mnt/nix/tmp
+mount --bind /mnt/nix/tmp /mnt/tmp
 
 # bind mount persistent stuff to data
-mkdir -p /mnt/{data/home,data/root,data/nixos/$HOST}
-mount --bind /mnt/data/home /mnt/home
-mount --bind /mnt/data/root /mnt/root
-mount --bind /mnt/data/nixos/$HOST /mnt/etc/nixos
+mkdir -p /mnt/nix/data/{home,root,nixos/$HOST}
+mount --bind /mnt/nix/data/home /mnt/home
+mount --bind /mnt/nix/data/root /mnt/root
+mount --bind /mnt/nix/data/nixos/$HOST /mnt/etc/nixos
 
-# create fake /data to have the right paths
-mkdir -p /data
-mount --bind /mnt/data /data
+# create fake /nix/data to have the right paths
+mkdir -p /nix/data
+mount --bind /mnt/nix/data /nix/data
 
 # take a look
 mount
@@ -110,62 +105,21 @@ mount
 # configure
 nixos-generate-config --root /mnt
 
-# save /mnt/etc/nixos/hardware-configuration.nix /mnt/etc/nixos/configuration.nix
+# check /mnt/etc/nixos/hardware-configuration.nix /mnt/etc/nixos/configuration.nix
 
-cp /mnt/etc/nixos/hardware-configuration.nix /tmp
-cp /mnt/etc/nixos/configuration.nix /tmp
+# copy config data from another machine including secrets
 
-# copy config data
-
-sudo scp -r /data/nixos root@192.168.13.100:/mnt/data
+sudo scp -r /nix/data/nixos root@192.168.13.100:/mnt/nix/data
 
 # install
 
 nixos-install --option experimental-features 'nix-command flakes' --no-root-passwd --root /mnt
 
-# unmount all stuff
+# unmount all stuff and sync
 
-umount -Rl /data /mnt
-zpool export -a
+umount -Rl /nix/data /mnt
+sync
 
 # sync all /data after the install
 
-sudo -E rsync -va --delete --one-file-system /data root@192.168.13.100:/
-
-#
-# after install tasks for extra file systems
-#
-
-# create vms disk
-
-DD=/dev/disk/by-id/ata-CT2000MX500SSD1_2138E5D5061F
-sgdisk --zap-all $DD
-blkdiscard -v $DD
-wipefs -a $DD
-
-sleep 5
-
-# ZFS zpool creation with encryption
-zpool create \
-    -o ashift=12 \
-    -o autotrim=on \
-    -O acltype=posixacl \
-    -O atime=off \
-    -O canmount=off \
-    -O compression=on \
-    -O dnodesize=auto \
-    -O normalization=formD \
-    -O xattr=sa \
-    -O mountpoint=none \
-    -O encryption=on \
-    -O keylocation=file:///data/nixos/key-vms.secret \
-    -O keyformat=passphrase \
-    vpool $DD
-
-sleep 5
-
-# create all the volumes
-zfs create -o mountpoint=legacy vpool/vms
-
-# update passphrase later
-# zfs change-key -o keylocation=file:///data/nixos/key-vms.secret vpool
+sudo -E rsync -va --delete --one-file-system /nix/data/ root@192.168.13.100:/nix/data/