draft k8s-on-nixos-part1
This commit is contained in:
Ricardo (XenGi) Band
parent
dd2bf9df2f
commit
aaf28cd1ea
1 changed files with 257 additions and 0 deletions
257
content/software/k8s-on-nixos-part1.md
Normal file
257
content/software/k8s-on-nixos-part1.md
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
---
|
||||
Title: K8s on NixOS - Chapter 1: Getting the nodes ready
|
||||
Date: 2025-03-14
|
||||
Category: software
|
||||
Tags: nix, nixos, flakes, server, qemu, libvirt
|
||||
Slug: k8s-on-nixos-chapter1-nodes
|
||||
Summary: In this chapter we will setup 3 nodes running NixOS. They will host our future Kubernetes cluster.
|
||||
Status: Draft
|
||||
---
|
||||
|
||||
# Getting the nodes ready
|
||||
|
||||
For simplicity all nodes are QEMU VMs. In reality these can be anything, like hardware servers or Raspberry Pis. Just
|
||||
adapt the setup to your needs.
|
||||
|
||||
Here is the updated `flake.nix` we will use:
|
||||
|
||||
```nix
|
||||
{
|
||||
inputs = {
|
||||
nixpkgs.url = github:NixOS/nixpkgs/nixos-24.11;
|
||||
agenix = {
|
||||
url = github:ryantm/agenix;
|
||||
inputs = {
|
||||
nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs, agenix }:
|
||||
let
|
||||
system = "x86_64-linux";
|
||||
pkgs = import nixpkgs { inherit system; };
|
||||
in
|
||||
{
|
||||
formatter.x86_64-linux = pkgs.nixpkgs-fmt;
|
||||
devShells.x86_64-linux.default = pkgs.mkShell {
|
||||
packages = with pkgs; [
|
||||
agenix.packages.x86_64-linux.default
|
||||
age # secrets debugging
|
||||
gnumake
|
||||
];
|
||||
};
|
||||
nixosConfigurations = {
|
||||
"node-01" = nixpkgs.lib.nixosSystem {
|
||||
inherit system;
|
||||
modules = [
|
||||
agenix.nixosModules.default
|
||||
# install agenix system-wide
|
||||
{ environment.systemPackages = [ agenix.packages.x86_64-linux.default ]; }
|
||||
{
|
||||
age.secrets = {}
|
||||
}
|
||||
./common.nix
|
||||
./node-01.nix
|
||||
];
|
||||
};
|
||||
"node-02" = nixpkgs.lib.nixosSystem {
|
||||
inherit system;
|
||||
modules = [
|
||||
agenix.nixosModules.default
|
||||
# install agenix system-wide
|
||||
{ environment.systemPackages = [ (agenix.packages.x86_64-linux.default.override { ageBin = "${pkgs.age}/bin/age"; }) ]; }
|
||||
{
|
||||
age.secrets = {}
|
||||
}
|
||||
./common.nix
|
||||
./node-02.nix
|
||||
];
|
||||
};
|
||||
"node-03" = nixpkgs.lib.nixosSystem {
|
||||
inherit system;
|
||||
modules = [
|
||||
agenix.nixosModules.default
|
||||
# install agenix system-wide
|
||||
{ environment.systemPackages = [ (agenix.packages.x86_64-linux.default.override { ageBin = "${pkgs.age}/bin/age"; }) ]; }
|
||||
{
|
||||
age.secrets = {}
|
||||
}
|
||||
./common.nix
|
||||
./node-03.nix
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
Update the `Makefile` like this:
|
||||
|
||||
```make
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
.PHONY: all
|
||||
all: update-flake k8s ## Update flake inputs and deploy k8s cluster
|
||||
|
||||
.PHONY: update-flake
|
||||
update-flake: ## Update nix flake
|
||||
nix flake update
|
||||
|
||||
.PHONY: k8s
|
||||
k8s: node-01 node-02 node-03 ## Deploy k8s cluster
|
||||
|
||||
.PHONY: node-01
|
||||
node-01: ## Deploy node-01
|
||||
nixos-rebuild switch --build-host node-01 --target-host node-01 --flake ".#node-01"
|
||||
|
||||
.PHONY: node-02
|
||||
node-02: ## Deploy node-02
|
||||
nixos-rebuild switch --build-host node-02 --target-host node-02 --flake ".#node-02"
|
||||
|
||||
.PHONY: node-03
|
||||
node-03: ## Deploy node-03
|
||||
nixos-rebuild switch --build-host node-03 --target-host node-03 --flake ".#node-03"
|
||||
|
||||
.PHONY: help
|
||||
help: ## Display this help
|
||||
@grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
|
||||
```
|
||||
|
||||
The hostnames are configured in SSH (`~/.ssh/config`) so it's easier to connect to them.
|
||||
|
||||
```ssh
|
||||
Host node-01
|
||||
User root
|
||||
HostName 2001:db8::1
|
||||
Port 22
|
||||
IdentitiesOnly yes
|
||||
IdentityFile /home/user/.ssh/id_ed25519
|
||||
```
|
||||
|
||||
Before we can use all this we have to run through the NixOS installation of these nodes.
|
||||
|
||||
Download the NixOS ISO from [nixos.org/download](https://nixos.org/download/) and setup a QEMU VM with
|
||||
[virt-manager](https://virt-manager.org/).
|
||||
|
||||
Create a new VM with the wizard like this:
|
||||
|
||||
- Step 1: Choose `Local install media (ISO image or CDROM)`
|
||||
- Step 2: Use the ISO and choose `NixOS Unstable` as OS
|
||||
- Step 3: Use at least 4GB of memory and 4 cores
|
||||
- Step 4: Create a 64GB disk to have enough storage for the Nix store and some container images
|
||||
- Step 5: Call it `k8s-node-01` and hit `Finish` or check the `Customize configuration before install` if you're
|
||||
adventurous
|
||||
|
||||
I usually customize the VM like this:
|
||||
|
||||
- Overview:
|
||||
- Use `UEFI Firmware`
|
||||
- CPUs:
|
||||
- Topology:
|
||||
- Set 1 Socket and 4 Cores
|
||||
- Boot options:
|
||||
- Start VM on host boot up
|
||||
|
||||
Boot the NixOS installer and run the installation. Setup partitions like this:
|
||||
|
||||
```shell
|
||||
sudo cfdisk /dev/vda
|
||||
# Select label type: gpt
|
||||
# 1GB type EFI System
|
||||
# ~63GB type Linux filesystem
|
||||
# Write and Quit
|
||||
sudo mkdosfs -F32 -n ESP /dev/vda1
|
||||
sudo mkfs.ext4 -L NIXOS /dev/vda2
|
||||
sudo mount /dev/vda2 /mnt
|
||||
sudo mkdir /mnt/boot
|
||||
sudo mount /dev/vda1 /mnt/boot
|
||||
```
|
||||
|
||||
Now generate the NixOS files:
|
||||
|
||||
```shell
|
||||
sudo nixos-generate-config --root /mnt
|
||||
$EDITOR /mnt/etc/nixos/configuration.nix
|
||||
```
|
||||
|
||||
Create a minimal nix configuration like this:
|
||||
|
||||
```nix
|
||||
{ config, lib, pkgs, modulesPath, ... }:
|
||||
|
||||
{
|
||||
imports = [
|
||||
(modulesPath + "/profiles/qemu-guest.nix")
|
||||
./hardware-configuration.nix
|
||||
];
|
||||
|
||||
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
||||
|
||||
boot.loader = {
|
||||
systemd-boot.enable = true;
|
||||
efi.canTouchEfiVariables = true;
|
||||
};
|
||||
|
||||
networking = {
|
||||
hostName = "node-01";
|
||||
firewall.enable = true;
|
||||
useDHCP = true;
|
||||
useNetworkd = true;
|
||||
dhcpcd.enable = false;
|
||||
nftables.enable = true;
|
||||
tempAddresses = "disabled";
|
||||
};
|
||||
services.resolved.enable = true;
|
||||
|
||||
time.timeZone = "Europe/Berlin";
|
||||
|
||||
i18n.defaultLocale = "en_US.UTF-8";
|
||||
console = {
|
||||
font = "Lat2-Terminus16";
|
||||
useXkbConfig = true;
|
||||
};
|
||||
|
||||
users.users.root.openssh.authorizedKeys.keys = [
|
||||
"ssh-ed25519 Put-your-ssh-keys-in-here"
|
||||
];
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
vim # or any other editor you like
|
||||
git
|
||||
];
|
||||
|
||||
programs.vim = { # or any other editor you like
|
||||
enable = true;
|
||||
defaultEditor = true;
|
||||
};
|
||||
|
||||
services.openssh.enable = true;
|
||||
|
||||
system.stateVersion = "24.11"; # never touch that unless you know what you're doing
|
||||
}
|
||||
```
|
||||
|
||||
Tip: Get your SSH keys from github with:
|
||||
|
||||
```shell
|
||||
curl -sL github.com/your-username.keys
|
||||
```
|
||||
|
||||
Start installation:
|
||||
|
||||
```shell
|
||||
cd /mnt
|
||||
sudo nixos-install
|
||||
```
|
||||
|
||||
The installer will ask you for a root password. Choose a strong one. Then `reboot` and run the installation on the
|
||||
remaining VMs.
|
||||
|
||||
# Update nodes from local flake
|
||||
|
||||
_tba_
|
||||
|
||||
|
||||
*[SSH]: Secure Shell
|
||||
|
||||
Loading…
Reference in a new issue