2 alerts for alertname=ToolforgeKubernetesWorkerTooManyDProcesses project=tools
View in Alertmanager
[2] Firing
Labels
alertname = ToolforgeKubernetesWorkerTooManyDProcesses
instance = tools-k8s-worker-nfs-22
job = node-exporter
project = tools
severity = warning
source = prometheus
state = D
team = wmcs
Annotations
dashboard = https://grafana.wmcloud.org/d/3jhWxB8Vk/toolforge-general-overview
description = Kubernetes worker tools-k8s-worker-nfs-22 had at least 12 procs in D state for 1 hour, may be stuck on IO (probably NFS)
runbook = https://wikitech.wikimedia.org/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesWorkerTooManyDProcesses
summary = Node tools-k8s-worker-nfs-22 has at least 12 procs in D state, and may be having NFS/IO issues
Source
Labels
alertname = ToolforgeKubernetesWorkerTooManyDProcesses
instance = tools-k8s-worker-nfs-9
job = node-exporter
project = tools
severity = warning
source = prometheus
state = D
team = wmcs
Annotations
dashboard = https://grafana.wmcloud.org/d/3jhWxB8Vk/toolforge-general-overview
description = Kubernetes worker tools-k8s-worker-nfs-9 had at least 12 procs in D state for 1 hour, may be stuck on IO (probably NFS)
runbook = https://wikitech.wikimedia.org/wiki/Portal:Toolforge/Admin/Runbooks/ToolforgeKubernetesWorkerTooManyDProcesses
summary = Node tools-k8s-worker-nfs-9 has at least 12 procs in D state, and may be having NFS/IO issues
Source