diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..ba0ebdb --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,189 @@ + +# Kubernetes - Helm tips + +- [Install microk8s and helm on Debian 11 - RaspiBlitz](#install-microk8s-and-helm-on-debian-11---raspiblitz) +- [Using the Galoy Helm charts](#using-the-galoy-helm-charts) + - [Inspect chart without installing](#inspect-chart-without-installing) + - [Install](#install) +- [Bitcoind in kubernetes helm](#bitcoind-in-kubernetes-helm) + - [install](#install-1) + - [logs](#logs) + - [copy chain](#copy-chain) + - [get bitcoind password](#get-bitcoind-password) +- [Secrets](#secrets) +- [Debug](#debug) + - [Troubleshooting](#troubleshooting) + - [Check pods](#check-pods) + - [Status](#status) +- [Dashboard](#dashboard) +- [OS level tweaks](#os-level-tweaks) + - [Increase open file limits](#increase-open-file-limits) + - [Free space without restart](#free-space-without-restart) + - [Directories taking space](#directories-taking-space) + - [Change microk8s default-storage path in config](#change-microk8s-default-storage-path-in-config) + +# Install microk8s and helm on Debian 11 - RaspiBlitz + +[install.microk8s.sh](install.microk8s.sh) + +# Using the Galoy Helm charts + +## Inspect chart without installing +``` +helm pull galoy-repo/galoy +helm show chart galoy-0.2.52.tgz +helm show values galoy-0.2.52.tgz +``` +## Install + +``` +helm repo add galoy-repo https://github.com/GaloyMoney/charts +helm repo update +# microk8s.kubectl create namespace galoy +# helm install galoy -n galoy --set global.persistence.storageClass=microk8s-hostpath galoy-repo/galoy +# helm uninstall galoy -n galoy +helm install galoy --set global.persistence.storageClass=microk8s-hostpath galoy-repo/galoy --debug --timeout 10m + +helm install galoy \ + --set needFirebaseServiceAccount=false \ + --set global.persistence.storageClass=microk8s-hostpath \ + galoy-repo/galoy --debug --timeout 10m + +# needFirebaseServiceAccount: true +needFirebaseServiceAccount=false + +helm install bitcoind galoy-repo/bitcoind +helm install lnd galoy-repo/lnd + +helm install bitcoin galoy-repo/bitcoin + +# monitor +microk8s kubectl get pod -n galoy -w + +microk8s kubectl get service -n galoy +``` + +# Bitcoind in kubernetes helm +## install +``` +helm install bitcoind galoy-repo/bitcoind +# monitor +kubectl describe pod bitcoind +``` + +## logs +``` +sudo tail -f /var/snap/microk8s/common/default-storage/default-bitcoind-pvc-*/debug.log +``` + +## copy chain +``` +# check storage +ls -la /var/snap/microk8s/common/default-storage +# stop with helm +helm uninstall bitcoind + +# copy from clone / host (must not have bitcoind running) +# cd to the source bitcoin directory +cd /mnt/hdd/*/bitcoin + +# copy ./chainstate ./blocks ./indexes recursively and verbose +sudo rsync -rv ./chainstate ./blocks ./indexes \ +/var/snap/microk8s/common/default-storage/container-registry-registry-claim-pvc-*/ + +# restart with helm +helm install bitcoind galoy-repo/bitcoind +``` + +## get bitcoind password +``` +microk8s kubectl get secret bitcoind-rpcpassword -o jsonpath='{.data.password}' +``` + +# Secrets +* https://kubernetes.io/docs/tasks/configmap-secret/managing-secret-using-kubectl/ +``` +microk8s kubectl edit secrets +``` + +# Debug +* https://devopscube.com/troubleshoot-kubernetes-pods/ +## Troubleshooting +``` +microk8s.inspect +``` + +## Check pods +``` +# all pods +microk8s.kubectl get pod --all-namespaces + +# watch +microk8s.kubectl get pod -Aw +``` + +## Status +``` +microk8s.kubectl describe no +``` +# Dashboard +``` +microk8s dashboard-proxy + +# to just get the token: +token=$(microk8s kubectl -n kube-system get secret | grep default-token | cut -d " " -f1) +microk8s kubectl -n kube-system describe secret $token +``` +# OS level tweaks + +## Increase open file limits +The error: +``` +Failed to allocate directory watch: Too many open files +``` +Check: +``` +sysctl fs.inotify +``` +Fix: +``` +sudo sysctl fs.inotify.max_user_instances=512 +``` + +## Free space without restart + +``` +sudo docker system prune -a +``` + +https://serverfault.com/questions/501963/how-do-i-recover-free-space-on-deleted-files-without-restarting-the-referencing + +``` +# check free space +df -h +# Find all opened file descriptors, grep deleted, StdError to /dev/null +sudo find /proc/*/fd -ls 2> /dev/null | grep '(deleted)' +# Find and truncate all deleted files, -p prompt before execute truncate +sudo find /proc/*/fd -ls 2> /dev/null | awk '/deleted/ {print $11}' | xargs -p -n 1 sudo truncate -s 0 +df -h +``` + +## Directories taking space +``` +/var/snap/microk8s/common/default-storage +https://github.com/canonical/microk8s/issues/463#issuecomment-491285745 +sudo lsof +D /var/snap | awk '!/COMMAND/{print $1 | "sort -u"}' +``` + +## Change microk8s default-storage path in config +``` +microk8s.kubectl -n kube-system edit deploy hostpath-provisioner +``` +Change in: +``` + volumes: + - hostPath: + path: /mnt/ext/microk8s/common/default-storage + type: "" + name: pv-volume +``` diff --git a/k8s/install.microk8s.sh b/k8s/install.microk8s.sh new file mode 100644 index 0000000..f9bc746 --- /dev/null +++ b/k8s/install.microk8s.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +# install microk8s and helm on Debian 11 - RaspiBlitz + +if [ "$1" = on ]; then + sudo adduser --disabled-password --gecos "" k8s + echo '/usr/share/doc/fzf/examples/key-bindings.bash' >> ~/.bashrc + echo '/usr/share/doc/fzf/examples/completion.bash' >> ~/.bashrc + + sudo usermod -a -G sudo,bitcoin,debian-tor k8s + + sudo su - k8s + + # https://www.server-world.info/en/note?os=Debian_11&p=microk8s&f=1 + + sudo apt update + + SSDmount="/mnt/ext" + sudo mkdir -p /var/snap + sudo mv -f /var/snap ${SSDmount}/ + sudo ln -s ${SSDmount}/snap /var/snap + + sudo apt install -y snapd + sudo snap install microk8s --classic + + echo 'export PATH=/snap/bin:$PATH' >> ~/.bashrc + source ~/.bashrc + + sudo usermod -a -G microk8s k8s + sudo chown -f -R k8s ~/.kube + newgrp microk8s + + # microk8s.inspect + # troubleshooting steps on Debian + # https://microk8s.io/docs/troubleshooting + sudo iptables -P FORWARD ACCEPT + sudo apt-get install -y iptables-persistent + echo '{ + "insecure-registries" : ["localhost:32000"] + } + ' | sudo tee -a /etc/docker/daemon.json + + sudo ufw allow in on vxlan.calico && sudo ufw allow out on vxlan.calico + sudo ufw allow in on cali+ && sudo ufw allow out on cali+ + sudo ufw allow 16443 comment "microk8s" + sudo ufw allow 10443 comment "kubernetes-dashboard" + + SSDmount="/mnt/ext" + ## part of the docker install script + # echo "### 3) Symlink the working directory to the SSD" + sudo systemctl stop docker + sudo systemctl stop docker.socket + sudo mkdir -p /var/lib/docker + sudo mv -f /var/lib/docker ${SSDmount}/ + sudo ln -s ${SSDmount}/docker /var/lib/docker + sudo systemctl start docker + sudo systemctl start docker.socket + + microk8s stop + + ## symlink the microk8s containerd and default-storage to the SSD + SSDmount="/mnt/ext" + + sudo mkdir -p ${SSDmount}/microk8s/common/var/lib/containerd + sudo mkdir -p ${SSDmount}/microk8s/common/run/containerd + + echo "--config \${SNAP_DATA}/args/containerd.toml +--root ${SSDmount}/microk8s/common/var/lib/containerd +--state ${SSDmount}/microk8s/common/run/containerd +--address \${SNAP_COMMON}/run/containerd.sock +" | sudo tee /var/snap/microk8s/current/args/containerd + + microk8s start + + microk8s enable helm + microk8s enable dns + microk8s enable dashboard + microk8s enable storage + microk8s enable ingress + microk8s enable registry + + # make the config permanent + microk8s config > ~/.kube/config + sudo chmod 0600 /home/k8s/.kube/config + + # helm + sudo snap install helm --classic +fi + +if [ "$1" = off ]; then + + helm uninstall galoy + sudo snap remove helm + microk8s reset [--destroy-storage] + microk8s stop + sudo snap remove microk8s + sudo apt remove -y snapd --purge + +fi \ No newline at end of file