Configuring higher VM workload density

apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
  labels:
    machineconfiguration.openshift.io/role: worker
  name: 90-worker-swap
spec:
  config:
    ignition:
      version: 3.4.0
    systemd:
      units:
        - contents: |
            [Unit]
            Description=Provision and enable swap
            ConditionFirstBoot=no

            [Service]
            Type=oneshot
            Environment=SWAP_SIZE_MB=5000
            ExecStart=/bin/sh -c "sudo dd if=/dev/zero of=/var/tmp/swapfile count=${SWAP_SIZE_MB} bs=1M && \
            sudo chmod 600 /var/tmp/swapfile && \
            sudo mkswap /var/tmp/swapfile && \
            sudo swapon /var/tmp/swapfile && \
            free -h && \
            sudo systemctl set-property --runtime system.slice MemorySwapMax=0 IODeviceLatencyTargetSec=\"/ 50ms\""

            [Install]
            RequiredBy=kubelet-dependencies.target
          enabled: true
          name: swap-provision.service

NODE_SWAP_SPACE = NODE_RAM * (MEMORY_OVER_COMMIT_PERCENT / 100% - 1)

NODE_SWAP_SPACE = 16 GB * (150% / 100% - 1)
               = 16 GB * (1.5 - 1)
               = 16 GB * (0.5)
               =  8 GB

$ oc adm new-project wasp

$ oc create sa -n wasp wasp

$ oc create clusterrolebinding wasp --clusterrole=cluster-admin --serviceaccount=wasp:wasp

$ oc adm policy add-scc-to-user -n wasp privileged -z wasp

$ oc wait mcp worker --for condition=Updated=True --timeout=-1s

$ oc get csv -n openshift-cnv -l=operators.coreos.com/kubevirt-hyperconverged.openshift-cnv -ojson | jq '.items[0].spec.relatedImages[] | select(.name|test(".*wasp-agent.*")) | .image'

kind: DaemonSet
apiVersion: apps/v1
metadata:
  name: wasp-agent
  namespace: wasp
  labels:
    app: wasp
    tier: node
spec:
  selector:
    matchLabels:
      name: wasp
  template:
    metadata:
      annotations:
        description: >-
          Configures swap for workloads
      labels:
        name: wasp
    spec:
      containers:
        - env:
            - name: SWAP_UTILIZATION_THRESHOLD_FACTOR
              value: "0.8"
            - name: MAX_AVERAGE_SWAP_IN_PAGES_PER_SECOND
              value: "1000"
            - name: MAX_AVERAGE_SWAP_OUT_PAGES_PER_SECOND
              value: "1000"
            - name: AVERAGE_WINDOW_SIZE_SECONDS
              value: "30"
            - name: VERBOSITY
              value: "1"
            - name: FSROOT
              value: /host
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          image: >-
            quay.io/openshift-virtualization/wasp-agent:v4.17 (1)
          imagePullPolicy: Always
          name: wasp-agent
          resources:
            requests:
              cpu: 100m
              memory: 50M
          securityContext:
            privileged: true
          volumeMounts:
            - mountPath: /host
              name: host
            - mountPath: /rootfs
              name: rootfs
      hostPID: true
      hostUsers: true
      priorityClassName: system-node-critical
      serviceAccountName: wasp
      terminationGracePeriodSeconds: 5
      volumes:
        - hostPath:
            path: /
          name: host
        - hostPath:
            path: /
          name: rootfs
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 10%
      maxSurge: 0

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    tier: node
    wasp.io: ""
  name: wasp-rules
  namespace: wasp
spec:
  groups:
    - name: alerts.rules
      rules:
        - alert: NodeHighSwapActivity
          annotations:
            description: High swap activity detected at {{ $labels.instance }}. The rate
              of swap out and swap in exceeds 200 in both operations in the last minute.
              This could indicate memory pressure and may affect system performance.
            runbook_url: https://github.com/openshift-virtualization/wasp-agent/tree/main/docs/runbooks/NodeHighSwapActivity.md
            summary: High swap activity detected at {{ $labels.instance }}.
          expr: rate(node_vmstat_pswpout[1m]) > 200 and rate(node_vmstat_pswpin[1m]) >
            200
          for: 1m
          labels:
            kubernetes_operator_component: kubevirt
            kubernetes_operator_part_of: kubevirt
            operator_health_impact: warning
            severity: warning

$ oc label namespace wasp openshift.io/cluster-monitoring="true"

$ oc rollout status ds wasp-agent -n wasp

daemon set "wasp-agent" successfully rolled out

$ oc get -n openshift-cnv HyperConverged kubevirt-hyperconverged -o jsonpath="{.spec.higherWorkloadDensity.memoryOvercommitPercentage}"

150