Difference between revisions of "Kubeadm upgrade nodes"

From UVOO Tech Wiki
Jump to navigation Jump to search
 
(6 intermediate revisions by the same user not shown)
Line 1: Line 1:
 
# New
 
# New
 +
 +
- https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/
 +
 +
- Drain node
 +
- On primary update_kubeadm_version then kubeadm_upgrade
 +
- uncordon node
 +
- update_kubeadm_version on all other nodes starting with controllers first
 +
 +
# Alternate version WIP
 
```
 
```
 +
#!/usr/bin/env bash
 +
set -euo pipefail
 +
 +
# ====== Config via ENV ======
 +
K8S_MINOR="${K8S_MINOR:-1.32}"                # e.g. 1.31 or 1.32
 +
DRAIN="${DRAIN:-false}"                      # true to drain before upgrade (default: false)
 +
ALLOW_FORCE="${ALLOW_FORCE:-false}"          # true to allow drain of bare pods
 +
DRAIN_TIMEOUT="${DRAIN_TIMEOUT:-15m}"
 +
 +
KERNEL_UPGRADE="${KERNEL_UPGRADE:-false}"    # true to apt upgrade kernel/system
 +
KERNEL_REBOOT="${KERNEL_REBOOT:-auto}"        # auto|true|false
 +
# auto = reboot only if /var/run/reboot-required exists after upgrade
 +
 +
PAUSE_IMAGE="${PAUSE_IMAGE:-registry.k8s.io/pause:3.10}"
 +
BUMP_PAUSE_IMAGE="${DRAIN:-false}"                      # true to drain before upgrade (default: false)
 +
 +
# ====== Helpers ======
 +
node_name() { hostname -s; }
 +
 +
wait_apiserver_ready() {
 +
  local timeout="${1:-300}"; local end=$(( $(date +%s) + timeout ))
 +
  echo "[health] Waiting for apiserver readyz..."
 +
  until kubectl get --raw='/readyz' 2>/dev/null | grep -q '^ok$'; do
 +
    [[ $(date +%s) -ge $end ]] && { echo "[health] apiserver not ready"; return 1; }
 +
    sleep 5
 +
  done
 +
}
 +
 +
retry() {  # retry <attempts> <sleep_initial> <cmd...>
 +
  local attempts="$1"; shift; local sleep_s="$1"; shift; local i=1
 +
  while :; do
 +
    "$@" && return 0
 +
    [[ $i -ge $attempts ]] && { echo "Retry failed: $*"; return 1; }
 +
    echo "Retry $i/$attempts -> sleeping ${sleep_s}s"
 +
    sleep "$sleep_s"; sleep_s=$(( sleep_s * 2 )); i=$(( i + 1 ))
 +
  done
 +
}
 +
 +
drain_node() {
 +
  local n; n="$(node_name)"
 +
  if [[ "$DRAIN" != "true" ]]; then
 +
    echo "[drain] Skipping drain for $n (DRAIN=false)"
 +
    return 0
 +
  fi
 +
  echo "[drain] Draining $n"
 +
  local args=(--ignore-daemonsets --delete-emptydir-data --grace-period=60 "--timeout=${DRAIN_TIMEOUT}")
 +
  [[ "$ALLOW_FORCE" == "true" ]] && args+=(--force)
 +
  kubectl drain "$n" "${args[@]}"
 +
}
 +
 +
uncordon_node() {
 +
  local n; n="$(node_name)"
 +
  if [[ "$DRAIN" != "true" ]]; then
 +
    echo "[drain] Skipping uncordon for $n (DRAIN=false)"
 +
    return 0
 +
  fi
 +
  echo "[uncordon] Uncordoning $n"
 +
  kubectl uncordon "$n" || true
 +
}
 +
 +
bump_pause_image() {
 +
  if [[ "$BUMP_PAUSE_IMAGE" != "true" ]]; then
 +
    echo "[bump] Skipping bump_pause_iamge (BUMP_PAUSE_IMAGE=false)"
 +
    return 0
 +
  fi
 +
  echo "[pause] Setting sandbox image to ${PAUSE_IMAGE}"
 +
  if command -v containerd >/dev/null 2>&1; then
 +
    local cfg=/etc/containerd/config.toml
 +
    sudo mkdir -p "$(dirname "$cfg")"
 +
    [[ -f "$cfg" ]] || sudo containerd config default | sudo tee "$cfg" >/dev/null
 +
    sudo sed -i "s|^\(\s*sandbox_image\s*=\s*\).*$|\1\"${PAUSE_IMAGE}\"|" "$cfg"
 +
    sudo systemctl restart containerd
 +
  elif command -v crio >/dev/null 2>&1; then
 +
    local cfg=/etc/crio/crio.conf
 +
    sudo mkdir -p "$(dirname "$cfg")"
 +
    sudo sed -i "s|^\(pause_image\s*=\s*\).*$|\1\"${PAUSE_IMAGE}\"|" "$cfg" || true
 +
    sudo systemctl restart crio
 +
  else
 +
    echo "[pause] No containerd or CRI-O found; skipping pause image bump" >&2
 +
  fi
 +
}
 +
 +
add_k8s_repo_and_update_pkgs() {
 +
  echo "[k8s] Configuring repo for ${K8S_MINOR} and installing kubeadm/kubelet/kubectl"
 +
  sudo mkdir -p /etc/apt/keyrings
 +
  local keyr=/etc/apt/keyrings/kubernetes-apt-keyring.gpg
 +
  if [[ ! -f "$keyr" ]]; then
 +
    curl -fsSL "https://pkgs.k8s.io/core:/stable:/v${K8S_MINOR}/deb/Release.key" | sudo gpg --dearmor -o "$keyr"
 +
  fi
 +
  echo "deb [signed-by=${keyr}] https://pkgs.k8s.io/core:/stable:/v${K8S_MINOR}/deb/ /" | \
 +
    sudo tee /etc/apt/sources.list.d/kubernetes.list >/dev/null
 +
 +
  sudo apt-get update
 +
  sudo apt-mark unhold kubeadm kubectl kubelet || true
 +
  sudo apt-get install -y "kubeadm=${K8S_MINOR}."* "kubelet=${K8S_MINOR}."* "kubectl=${K8S_MINOR}."*
 +
  sudo apt-mark hold kubeadm kubectl kubelet
 +
 +
  # Optional: pre-pull control-plane images for this version (harmless on workers)
 +
  # ver="$(kubeadm version -o short)"; sudo kubeadm config images pull --kubernetes-version "$ver" || true
 +
}
 +
 +
apply_kubeadm_node_upgrade() {
 +
  echo "[k8s] Running 'kubeadm upgrade node'"
 +
  sudo kubeadm upgrade node
 +
  # wait_apiserver_ready 600
 +
  # echo "[k8s] Running 'kubeadm upgrade node' with retries"
 +
  # retry 6 2 sudo kubeadm upgrade node
 +
  retry 3 300 sudo kubeadm upgrade node # retries 3 times every 5 minutes as kubectl is not configured for node access
 +
  echo "[k8s] Restarting kubelet"
 +
  sudo systemctl daemon-reload
 +
  sudo systemctl restart kubelet
 +
}
 +
 +
kernel_and_system_upgrade() {
 +
  if [[ "$KERNEL_UPGRADE" != "true" ]]; then
 +
    echo "[kernel] Skipping kernel/system upgrade (KERNEL_UPGRADE=false)"
 +
    return 0
 +
  fi
 +
 +
  echo "[kernel] Upgrading kernel & system packages"
 +
  # Generic + safe: full dist-upgrade (includes kernel/hwe if enabled)
 +
  sudo DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
 +
 +
  # If you want to force the meta-packages explicitly, uncomment:
 +
  # sudo apt-get install -y linux-image-generic linux-headers-generic || true
 +
  # On HWE stacks (Ubuntu LTS), you might prefer:
 +
  # sudo apt-get install -y linux-generic-hwe-24.04 || true
 +
 +
  if [[ "$KERNEL_REBOOT" == "true" ]]; then
 +
    echo "[kernel] Reboot requested (KERNEL_REBOOT=true)"
 +
    sudo reboot
 +
  elif [[ "$KERNEL_REBOOT" == "auto" ]]; then
 +
    if [[ -f /var/run/reboot-required ]]; then
 +
      echo "[kernel] Reboot required and auto-approved (KERNEL_REBOOT=auto)"
 +
      sudo reboot
 +
    else
 +
      echo "[kernel] No reboot required (KERNEL_REBOOT=auto)"
 +
    fi
 +
  else
 +
    echo "[kernel] Reboot suppressed (KERNEL_REBOOT=false)"
 +
  fi
 +
}
 +
 +
print_versions() {
 +
  echo "[versions]"
 +
  echo -n "  kubeadm: "; kubeadm version -o short
 +
  echo -n "  kubelet: "; kubelet --version
 +
  echo -n "  kubectl (client): "; kubectl version --client
 +
}
 +
 +
# ====== Main ======
 +
NODE="$(node_name)"
 +
echo "[start] Worker upgrade on ${NODE} (K8S_MINOR=${K8S_MINOR}, DRAIN=${DRAIN}, KERNEL_UPGRADE=${KERNEL_UPGRADE}, KERNEL_REBOOT=${KERNEL_REBOOT})"
 +
 +
drain_node
 +
add_k8s_repo_and_update_pkgs
 +
bump_pause_image
 +
apply_kubeadm_node_upgrade
 +
uncordon_node
 +
print_versions
 +
 +
kernel_and_system_upgrade  # may reboot here if configured
 +
 +
echo "[done] Node ${NODE} upgrade complete"
 +
```
 +
 +
 +
 +
 +
# Almost but doesn't do the kubeadm upgrade
 +
 +
update.sh
 +
```
 +
#!/bin/bash
 +
set -eu
 +
 
update_kubeadm_version(){
 
update_kubeadm_version(){
 
   k8s_minor_version=1.31
 
   k8s_minor_version=1.31
Line 14: Line 199:
 
   sudo apt-mark hold kubeadm kubectl kubelet && \
 
   sudo apt-mark hold kubeadm kubectl kubelet && \
 
   sudo systemctl restart kubelet
 
   sudo systemctl restart kubelet
 +
  sudo kubeadm upgrade node
 
   sudo apt-get -y dist-upgrade
 
   sudo apt-get -y dist-upgrade
 +
  kubeadm version -o short
 +
  echo "Reboot in 10 seconds"; sleep 10
 
   sudo reboot
 
   sudo reboot
 
}
 
}
Line 23: Line 211:
 
   echo "Updating to version ${k8s_patch_version} in 10 seconds."; sleep 10
 
   echo "Updating to version ${k8s_patch_version} in 10 seconds."; sleep 10
 
   sudo kubeadm upgrade plan
 
   sudo kubeadm upgrade plan
 +
  sudo killall -s SIGTERM kube-apiserver # trigger a graceful kube-apiserver shutdown
 +
  sleep 20 # wait a little bit to permit completing in-flight requests
 
   kubeadm upgrade apply -y ${k8s_patch_version}
 
   kubeadm upgrade apply -y ${k8s_patch_version}
 
}
 
}

Latest revision as of 20:59, 8 August 2025

New

Alternate version WIP

#!/usr/bin/env bash
set -euo pipefail

# ====== Config via ENV ======
K8S_MINOR="${K8S_MINOR:-1.32}"                # e.g. 1.31 or 1.32
DRAIN="${DRAIN:-false}"                       # true to drain before upgrade (default: false)
ALLOW_FORCE="${ALLOW_FORCE:-false}"           # true to allow drain of bare pods
DRAIN_TIMEOUT="${DRAIN_TIMEOUT:-15m}"

KERNEL_UPGRADE="${KERNEL_UPGRADE:-false}"     # true to apt upgrade kernel/system
KERNEL_REBOOT="${KERNEL_REBOOT:-auto}"        # auto|true|false
# auto = reboot only if /var/run/reboot-required exists after upgrade

PAUSE_IMAGE="${PAUSE_IMAGE:-registry.k8s.io/pause:3.10}"
BUMP_PAUSE_IMAGE="${DRAIN:-false}"                       # true to drain before upgrade (default: false)

# ====== Helpers ======
node_name() { hostname -s; }

wait_apiserver_ready() {
  local timeout="${1:-300}"; local end=$(( $(date +%s) + timeout ))
  echo "[health] Waiting for apiserver readyz..."
  until kubectl get --raw='/readyz' 2>/dev/null | grep -q '^ok$'; do
    [[ $(date +%s) -ge $end ]] && { echo "[health] apiserver not ready"; return 1; }
    sleep 5
  done
}

retry() {  # retry <attempts> <sleep_initial> <cmd...>
  local attempts="$1"; shift; local sleep_s="$1"; shift; local i=1
  while :; do
    "$@" && return 0
    [[ $i -ge $attempts ]] && { echo "Retry failed: $*"; return 1; }
    echo "Retry $i/$attempts -> sleeping ${sleep_s}s"
    sleep "$sleep_s"; sleep_s=$(( sleep_s * 2 )); i=$(( i + 1 ))
  done
}

drain_node() {
  local n; n="$(node_name)"
  if [[ "$DRAIN" != "true" ]]; then
    echo "[drain] Skipping drain for $n (DRAIN=false)"
    return 0
  fi
  echo "[drain] Draining $n"
  local args=(--ignore-daemonsets --delete-emptydir-data --grace-period=60 "--timeout=${DRAIN_TIMEOUT}")
  [[ "$ALLOW_FORCE" == "true" ]] && args+=(--force)
  kubectl drain "$n" "${args[@]}"
}

uncordon_node() {
  local n; n="$(node_name)"
  if [[ "$DRAIN" != "true" ]]; then
    echo "[drain] Skipping uncordon for $n (DRAIN=false)"
    return 0
  fi
  echo "[uncordon] Uncordoning $n"
  kubectl uncordon "$n" || true
}

bump_pause_image() {
  if [[ "$BUMP_PAUSE_IMAGE" != "true" ]]; then
    echo "[bump] Skipping bump_pause_iamge (BUMP_PAUSE_IMAGE=false)"
    return 0
  fi
  echo "[pause] Setting sandbox image to ${PAUSE_IMAGE}"
  if command -v containerd >/dev/null 2>&1; then
    local cfg=/etc/containerd/config.toml
    sudo mkdir -p "$(dirname "$cfg")"
    [[ -f "$cfg" ]] || sudo containerd config default | sudo tee "$cfg" >/dev/null
    sudo sed -i "s|^\(\s*sandbox_image\s*=\s*\).*$|\1\"${PAUSE_IMAGE}\"|" "$cfg"
    sudo systemctl restart containerd
  elif command -v crio >/dev/null 2>&1; then
    local cfg=/etc/crio/crio.conf
    sudo mkdir -p "$(dirname "$cfg")"
    sudo sed -i "s|^\(pause_image\s*=\s*\).*$|\1\"${PAUSE_IMAGE}\"|" "$cfg" || true
    sudo systemctl restart crio
  else
    echo "[pause] No containerd or CRI-O found; skipping pause image bump" >&2
  fi
}

add_k8s_repo_and_update_pkgs() {
  echo "[k8s] Configuring repo for ${K8S_MINOR} and installing kubeadm/kubelet/kubectl"
  sudo mkdir -p /etc/apt/keyrings
  local keyr=/etc/apt/keyrings/kubernetes-apt-keyring.gpg
  if [[ ! -f "$keyr" ]]; then
    curl -fsSL "https://pkgs.k8s.io/core:/stable:/v${K8S_MINOR}/deb/Release.key" | sudo gpg --dearmor -o "$keyr"
  fi
  echo "deb [signed-by=${keyr}] https://pkgs.k8s.io/core:/stable:/v${K8S_MINOR}/deb/ /" | \
    sudo tee /etc/apt/sources.list.d/kubernetes.list >/dev/null

  sudo apt-get update
  sudo apt-mark unhold kubeadm kubectl kubelet || true
  sudo apt-get install -y "kubeadm=${K8S_MINOR}."* "kubelet=${K8S_MINOR}."* "kubectl=${K8S_MINOR}."*
  sudo apt-mark hold kubeadm kubectl kubelet

  # Optional: pre-pull control-plane images for this version (harmless on workers)
  # ver="$(kubeadm version -o short)"; sudo kubeadm config images pull --kubernetes-version "$ver" || true
}

apply_kubeadm_node_upgrade() {
  echo "[k8s] Running 'kubeadm upgrade node'"
  sudo kubeadm upgrade node
  # wait_apiserver_ready 600
  # echo "[k8s] Running 'kubeadm upgrade node' with retries"
  # retry 6 2 sudo kubeadm upgrade node
  retry 3 300 sudo kubeadm upgrade node # retries 3 times every 5 minutes as kubectl is not configured for node access
  echo "[k8s] Restarting kubelet"
  sudo systemctl daemon-reload
  sudo systemctl restart kubelet
}

kernel_and_system_upgrade() {
  if [[ "$KERNEL_UPGRADE" != "true" ]]; then
    echo "[kernel] Skipping kernel/system upgrade (KERNEL_UPGRADE=false)"
    return 0
  fi

  echo "[kernel] Upgrading kernel & system packages"
  # Generic + safe: full dist-upgrade (includes kernel/hwe if enabled)
  sudo DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade

  # If you want to force the meta-packages explicitly, uncomment:
  # sudo apt-get install -y linux-image-generic linux-headers-generic || true
  # On HWE stacks (Ubuntu LTS), you might prefer:
  # sudo apt-get install -y linux-generic-hwe-24.04 || true

  if [[ "$KERNEL_REBOOT" == "true" ]]; then
    echo "[kernel] Reboot requested (KERNEL_REBOOT=true)"
    sudo reboot
  elif [[ "$KERNEL_REBOOT" == "auto" ]]; then
    if [[ -f /var/run/reboot-required ]]; then
      echo "[kernel] Reboot required and auto-approved (KERNEL_REBOOT=auto)"
      sudo reboot
    else
      echo "[kernel] No reboot required (KERNEL_REBOOT=auto)"
    fi
  else
    echo "[kernel] Reboot suppressed (KERNEL_REBOOT=false)"
  fi
}

print_versions() {
  echo "[versions]"
  echo -n "  kubeadm: "; kubeadm version -o short
  echo -n "  kubelet: "; kubelet --version
  echo -n "  kubectl (client): "; kubectl version --client
}

# ====== Main ======
NODE="$(node_name)"
echo "[start] Worker upgrade on ${NODE} (K8S_MINOR=${K8S_MINOR}, DRAIN=${DRAIN}, KERNEL_UPGRADE=${KERNEL_UPGRADE}, KERNEL_REBOOT=${KERNEL_REBOOT})"

drain_node
add_k8s_repo_and_update_pkgs
bump_pause_image
apply_kubeadm_node_upgrade
uncordon_node
print_versions

kernel_and_system_upgrade  # may reboot here if configured

echo "[done] Node ${NODE} upgrade complete"

Almost but doesn't do the kubeadm upgrade

update.sh 

#!/bin/bash
set -eu

update_kubeadm_version(){
  k8s_minor_version=1.31
  k8s_keyring_file="/etc/apt/keyrings/kubernetes-apt-keyring.gpg"
  if [ ! -f "${k8s_keyring_file}" ]; then
    curl -fsSL https://pkgs.k8s.io/core:/stable:/v${k8s_minor_version}/deb/Release.key 2>/dev/null | sudo gpg --dearmor -o ${k8s_keyring_file}
  fi
  echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${k8s_minor_version}/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
  sudo apt-get update

  sudo apt-mark unhold kubeadm kubectl kubelet && \
  sudo apt-get install -y kubelet=${k8s_minor_version}.* kubeadm=${k8s_minor_version}.* kubectl=${k8s_minor_version}.* && \
  sudo apt-mark hold kubeadm kubectl kubelet && \
  sudo systemctl restart kubelet
  sudo kubeadm upgrade node
  sudo apt-get -y dist-upgrade
  kubeadm version -o short
  echo "Reboot in 10 seconds"; sleep 10
  sudo reboot
}


kubeadm_upgrade_primary_controller(){
  k8s_patch_version=$(kubeadm version -o short)
  echo "Updating to version ${k8s_patch_version} in 10 seconds."; sleep 10
  sudo kubeadm upgrade plan
  sudo killall -s SIGTERM kube-apiserver # trigger a graceful kube-apiserver shutdown
  sleep 20 # wait a little bit to permit completing in-flight requests
  kubeadm upgrade apply -y ${k8s_patch_version}
}

Old

#!/bin/bash

# Update the package lists
sudo apt update

# Install the Kubernetes packages, specifying the desired version
sudo apt install -y kubelet=1.31.* kubeadm=1.31.* kubectl=1.31.* 

# Restart kubelet service
sudo systemctl restart kubelet


apt-get upgrade
# Verify node status
kubectl get nodes -o wide

# Optional: Drain the node before upgrading to avoid disruptions
# kubectl drain <node-name> --ignore-daemon-sets --force --delete-local-data 

# Optional: Upgrade other node components (e.g., containerd, Docker)
# - Follow the specific upgrade instructions for your container runtime

# Note:
# - This script assumes you have the Kubernetes repository configured in your system.
# - This script uses wildcards to install the latest patch version within the 1.31 series.
# - This script performs a rolling update. Consider draining nodes for zero-downtime upgrades.
# - Always back up critical data before performing any upgrades.
# - Refer to the official Kubernetes documentation for the most up-to-date instructions.
Retrieved from "https://tech.uvoo.io/index.php?title=Kubeadm_upgrade_nodes&oldid=5618"