From aaf28cd1ea443056ba1c78aed6cde6bd0ac887aa Mon Sep 17 00:00:00 2001 From: "Ricardo (XenGi) Band" Date: Sat, 15 Mar 2025 00:49:39 +0100 Subject: [PATCH] draft k8s-on-nixos-part1 --- content/software/k8s-on-nixos-part1.md | 257 +++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 content/software/k8s-on-nixos-part1.md diff --git a/content/software/k8s-on-nixos-part1.md b/content/software/k8s-on-nixos-part1.md new file mode 100644 index 0000000..c29d13b --- /dev/null +++ b/content/software/k8s-on-nixos-part1.md @@ -0,0 +1,257 @@ +--- +Title: K8s on NixOS - Chapter 1: Getting the nodes ready +Date: 2025-03-14 +Category: software +Tags: nix, nixos, flakes, server, qemu, libvirt +Slug: k8s-on-nixos-chapter1-nodes +Summary: In this chapter we will setup 3 nodes running NixOS. They will host our future Kubernetes cluster. +Status: Draft +--- + +# Getting the nodes ready + +For simplicity all nodes are QEMU VMs. In reality these can be anything, like hardware servers or Raspberry Pis. Just +adapt the setup to your needs. + +Here is the updated `flake.nix` we will use: + +```nix +{ + inputs = { + nixpkgs.url = github:NixOS/nixpkgs/nixos-24.11; + agenix = { + url = github:ryantm/agenix; + inputs = { + nixpkgs.follows = "nixpkgs"; + }; + }; + }; + + outputs = { self, nixpkgs, agenix }: + let + system = "x86_64-linux"; + pkgs = import nixpkgs { inherit system; }; + in + { + formatter.x86_64-linux = pkgs.nixpkgs-fmt; + devShells.x86_64-linux.default = pkgs.mkShell { + packages = with pkgs; [ + agenix.packages.x86_64-linux.default + age # secrets debugging + gnumake + ]; + }; + nixosConfigurations = { + "node-01" = nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + agenix.nixosModules.default + # install agenix system-wide + { environment.systemPackages = [ agenix.packages.x86_64-linux.default ]; } + { + age.secrets = {} + } + ./common.nix + ./node-01.nix + ]; + }; + "node-02" = nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + agenix.nixosModules.default + # install agenix system-wide + { environment.systemPackages = [ (agenix.packages.x86_64-linux.default.override { ageBin = "${pkgs.age}/bin/age"; }) ]; } + { + age.secrets = {} + } + ./common.nix + ./node-02.nix + ]; + }; + "node-03" = nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + agenix.nixosModules.default + # install agenix system-wide + { environment.systemPackages = [ (agenix.packages.x86_64-linux.default.override { ageBin = "${pkgs.age}/bin/age"; }) ]; } + { + age.secrets = {} + } + ./common.nix + ./node-03.nix + ]; + }; + }; + }; +} +``` + +Update the `Makefile` like this: + +```make +.DEFAULT_GOAL := help + +.PHONY: all +all: update-flake k8s ## Update flake inputs and deploy k8s cluster + +.PHONY: update-flake +update-flake: ## Update nix flake + nix flake update + +.PHONY: k8s +k8s: node-01 node-02 node-03 ## Deploy k8s cluster + +.PHONY: node-01 +node-01: ## Deploy node-01 + nixos-rebuild switch --build-host node-01 --target-host node-01 --flake ".#node-01" + +.PHONY: node-02 +node-02: ## Deploy node-02 + nixos-rebuild switch --build-host node-02 --target-host node-02 --flake ".#node-02" + +.PHONY: node-03 +node-03: ## Deploy node-03 + nixos-rebuild switch --build-host node-03 --target-host node-03 --flake ".#node-03" + +.PHONY: help +help: ## Display this help + @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +``` + +The hostnames are configured in SSH (`~/.ssh/config`) so it's easier to connect to them. + +```ssh +Host node-01 + User root + HostName 2001:db8::1 + Port 22 + IdentitiesOnly yes + IdentityFile /home/user/.ssh/id_ed25519 +``` + +Before we can use all this we have to run through the NixOS installation of these nodes. + +Download the NixOS ISO from [nixos.org/download](https://nixos.org/download/) and setup a QEMU VM with +[virt-manager](https://virt-manager.org/). + +Create a new VM with the wizard like this: + +- Step 1: Choose `Local install media (ISO image or CDROM)` +- Step 2: Use the ISO and choose `NixOS Unstable` as OS +- Step 3: Use at least 4GB of memory and 4 cores +- Step 4: Create a 64GB disk to have enough storage for the Nix store and some container images +- Step 5: Call it `k8s-node-01` and hit `Finish` or check the `Customize configuration before install` if you're + adventurous + +I usually customize the VM like this: + +- Overview: + - Use `UEFI Firmware` +- CPUs: + - Topology: + - Set 1 Socket and 4 Cores +- Boot options: + - Start VM on host boot up + +Boot the NixOS installer and run the installation. Setup partitions like this: + +```shell +sudo cfdisk /dev/vda +# Select label type: gpt +# 1GB type EFI System +# ~63GB type Linux filesystem +# Write and Quit +sudo mkdosfs -F32 -n ESP /dev/vda1 +sudo mkfs.ext4 -L NIXOS /dev/vda2 +sudo mount /dev/vda2 /mnt +sudo mkdir /mnt/boot +sudo mount /dev/vda1 /mnt/boot +``` + +Now generate the NixOS files: + +```shell +sudo nixos-generate-config --root /mnt +$EDITOR /mnt/etc/nixos/configuration.nix +``` + +Create a minimal nix configuration like this: + +```nix +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = [ + (modulesPath + "/profiles/qemu-guest.nix") + ./hardware-configuration.nix + ]; + + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + + boot.loader = { + systemd-boot.enable = true; + efi.canTouchEfiVariables = true; + }; + + networking = { + hostName = "node-01"; + firewall.enable = true; + useDHCP = true; + useNetworkd = true; + dhcpcd.enable = false; + nftables.enable = true; + tempAddresses = "disabled"; + }; + services.resolved.enable = true; + + time.timeZone = "Europe/Berlin"; + + i18n.defaultLocale = "en_US.UTF-8"; + console = { + font = "Lat2-Terminus16"; + useXkbConfig = true; + }; + + users.users.root.openssh.authorizedKeys.keys = [ + "ssh-ed25519 Put-your-ssh-keys-in-here" + ]; + + environment.systemPackages = with pkgs; [ + vim # or any other editor you like + git + ]; + + programs.vim = { # or any other editor you like + enable = true; + defaultEditor = true; + }; + + services.openssh.enable = true; + + system.stateVersion = "24.11"; # never touch that unless you know what you're doing +} +``` + +Tip: Get your SSH keys from github with: + +```shell +curl -sL github.com/your-username.keys +``` + +Start installation: + +```shell +cd /mnt +sudo nixos-install +``` + +The installer will ask you for a root password. Choose a strong one. Then `reboot` and run the installation on the +remaining VMs. + +# Update nodes from local flake + +_tba_ + + +*[SSH]: Secure Shell +