#!/usr/bin/python3
# Nagios/Icinga plugin to check NVIDIA GPUs with nvidia-smi: temperature, power, clocks and their slowdown
# reasons, memory, utilization, fan, PCIe link and ECC.
#
# Copyright (C) 2026 Thomas Wagner <thomas.wagner@jku.at>
# SPDX-License-Identifier: GPL-2.0-or-later

import argparse
import csv
import re
import subprocess
import sys

OK = 0
WARNING = 1
CRITICAL = 2
UNKNOWN = 3

STATE_TEXT = {OK: "OK", WARNING: "WARNING", CRITICAL: "CRITICAL", UNKNOWN: "UNKNOWN"}

VERSION = "0.1.1"
DEFAULT_TIMEOUT = 15
# Without the T.Limit field of newer drivers, absolute temperature thresholds in Celsius.
DEFAULT_TEMPERATURE_WARNING = "85"
DEFAULT_TEMPERATURE_CRITICAL = "90"

# Bits of nvmlClocksEventReasons, as nvidia-smi reports them in clocks_event_reasons.active.
REASONS = {
    "gpu_idle": 0x1,
    "applications_clocks_setting": 0x2,
    "sw_power_cap": 0x4,
    "hw_slowdown": 0x8,
    "sync_boost": 0x10,
    "sw_thermal_slowdown": 0x20,
    "hw_thermal_slowdown": 0x40,
    "hw_power_brake_slowdown": 0x80,
    "display_clock_setting": 0x100,
}
DEFAULT_WARNING_REASONS = "sw_thermal_slowdown"
DEFAULT_CRITICAL_REASONS = "hw_slowdown,hw_thermal_slowdown,hw_power_brake_slowdown"

# Each field with the older names some drivers use instead; the first one the driver knows is queried.
FIELDS = {
    "index": ["index"],
    "name": ["name"],
    "pstate": ["pstate"],
    "temperature": ["temperature.gpu"],
    "temperature_memory": ["temperature.memory"],
    "tlimit": ["temperature.gpu.tlimit"],
    "power": ["power.draw.average", "power.draw"],
    "power_limit": ["enforced.power.limit", "power.limit"],
    "power_default_limit": ["power.default_limit"],
    "power_max_limit": ["power.max_limit"],
    "clock_graphics": ["clocks.current.graphics", "clocks.gr"],
    "clock_sm": ["clocks.current.sm", "clocks.sm"],
    "clock_memory": ["clocks.current.memory", "clocks.mem"],
    "clock_video": ["clocks.current.video", "clocks.video"],
    "clock_graphics_max": ["clocks.max.graphics", "clocks.max.gr"],
    "clock_sm_max": ["clocks.max.sm"],
    "clock_memory_max": ["clocks.max.memory", "clocks.max.mem"],
    "reasons": ["clocks_event_reasons.active", "clocks_throttle_reasons.active"],
    "memory_total": ["memory.total"],
    "memory_used": ["memory.used"],
    "utilization_gpu": ["utilization.gpu"],
    "utilization_memory": ["utilization.memory"],
    "utilization_encoder": ["utilization.encoder"],
    "utilization_decoder": ["utilization.decoder"],
    "fan": ["fan.speed"],
    "pcie_gen": ["pcie.link.gen.current", "pcie.link.gen.gpucurrent"],
    "pcie_gen_max": ["pcie.link.gen.max"],
    "pcie_width": ["pcie.link.width.current"],
    "pcie_width_max": ["pcie.link.width.max"],
    "ecc_mode": ["ecc.mode.current"],
    "ecc_corrected": ["ecc.errors.corrected.volatile.total"],
    "ecc_uncorrected": ["ecc.errors.uncorrected.volatile.total"],
    "ecc_corrected_total": ["ecc.errors.corrected.aggregate.total"],
    "ecc_uncorrected_total": ["ecc.errors.uncorrected.aggregate.total"],
    "retired_pending": ["retired_pages.pending"],
    "remap_pending": ["remapped_rows.pending"],
    "remap_failure": ["remapped_rows.failure"],
}

NOT_AVAILABLE = {"", "n/a", "[n/a]", "not supported", "[not supported]", "unknown error", "[unknown error]",
                 "insufficient permissions", "[insufficient permissions]", "gpu is lost", "[gpu is lost]"}


class PluginError(Exception):
    def __init__(self, state, message):
        super().__init__(message)
        self.state = state
        self.message = message


class Range:
    """Threshold range as defined by the monitoring plugins guidelines."""

    def __init__(self, start, end, inside):
        self.start = start
        self.end = end
        self.inside = inside
        self.spec = self._normalized()

    def _normalized(self):
        """Threshold as perfdata carries it: graphers want a plain number, so one-sided ranges become their bound."""
        if not self.inside and self.start == float("-inf") and self.end == float("inf"):
            return ""
        if not self.inside and self.end == float("inf"):
            return format_number(self.start)
        if not self.inside and self.start in (0, float("-inf")) and self.end != float("inf"):
            return format_number(self.end)
        low = "~" if self.start == float("-inf") else format_number(self.start)
        high = "" if self.end == float("inf") else format_number(self.end)
        return ("@" if self.inside else "") + "%s:%s" % (low, high)

    @classmethod
    def parse(cls, spec):
        raw = spec
        inside = spec.startswith("@")
        if inside:
            spec = spec[1:]

        if ":" in spec:
            low, high = spec.split(":", 1)
        else:
            low, high = "0", spec

        try:
            start = float("-inf") if low in ("~", "") else float(low)
            end = float("inf") if high == "" else float(high)
        except ValueError:
            raise argparse.ArgumentTypeError("invalid threshold %r" % raw)

        if start > end:
            raise argparse.ArgumentTypeError("invalid threshold %r: start is above end" % raw)

        return cls(start, end, inside)

    def breached(self, value):
        within = self.start <= value <= self.end
        return within if self.inside else not within


def format_number(value):
    if isinstance(value, int) or float(value).is_integer():
        return "%d" % value
    return ("%.3f" % value).rstrip("0").rstrip(".")


def perfdata(label, value, uom="", warn=None, crit=None, minimum=None, maximum=None):
    fields = [
        warn.spec if warn else "",
        crit.spec if crit else "",
        "" if minimum is None else format_number(minimum),
        "" if maximum is None else format_number(maximum),
    ]
    while fields and fields[-1] == "":
        fields.pop()
    out = "%s=%s%s" % (label, format_number(value), uom)
    if fields:
        out += ";" + ";".join(fields)
    return out


def evaluate(value, warning, critical):
    if critical and critical.breached(value):
        return CRITICAL
    if warning and warning.breached(value):
        return WARNING
    return OK


# --------------------------------------------------------------------------
# nvidia-smi
# --------------------------------------------------------------------------
def run_nvidia_smi(args, arguments):
    try:
        result = subprocess.run([args.nvidia_smi] + arguments, capture_output=True, text=True,
                                timeout=args.timeout)
    except FileNotFoundError:
        raise PluginError(UNKNOWN, "%s not found" % args.nvidia_smi)
    except subprocess.TimeoutExpired:
        raise PluginError(CRITICAL, "nvidia-smi did not answer within %gs" % args.timeout)
    except OSError as err:
        raise PluginError(UNKNOWN, "cannot run %s: %s" % (args.nvidia_smi, err))
    if result.returncode != 0:
        text = (result.stdout + result.stderr).strip().replace("\n", " ") or "no output"
        # nvidia-smi exits 9 when it cannot talk to the driver, e.g. after a driver update or a lost GPU
        raise PluginError(CRITICAL, "nvidia-smi failed (exit %d): %s" % (result.returncode, text[:200]))
    return result.stdout


def known_fields(args):
    """Fields this driver can query, from nvidia-smi --help-query-gpu."""
    return set(re.findall(r'^"([a-z0-9_.]+)"', run_nvidia_smi(args, ["--help-query-gpu"]), re.MULTILINE))


def parse_value(text):
    text = text.strip()
    if text.lower() in NOT_AVAILABLE:
        return None
    if re.fullmatch(r"0x[0-9a-fA-F]+", text):
        return int(text, 16)
    try:
        number = float(text)
    except ValueError:
        return text
    return int(number) if number.is_integer() and "." not in text else number


def query(args):
    """One dict per GPU with the keys of FIELDS; values the driver does not know or report are None."""
    known = known_fields(args)
    # All known names of a field: a driver may know a newer name and still report N/A for it on older GPUs
    # (e.g. power.draw.average on Turing), so the first one with a value wins.
    chosen = {key: [name for name in names if name in known] for key, names in FIELDS.items()}
    if not chosen["index"]:
        raise PluginError(UNKNOWN, "nvidia-smi --help-query-gpu lists no fields")
    names = [name for alternatives in chosen.values() for name in alternatives]
    arguments = ["--query-gpu=" + ",".join(names), "--format=csv,noheader,nounits"]
    if args.index:
        arguments = ["-i", args.index] + arguments
    output = run_nvidia_smi(args, arguments)

    gpus = []
    for row in csv.reader(output.splitlines(), skipinitialspace=True):
        if not row:
            continue
        if len(row) != len(names):
            raise PluginError(UNKNOWN, "nvidia-smi returned %d values instead of %d: %r"
                              % (len(row), len(names), ",".join(row)[:120]))
        values = {name: parse_value(value) for name, value in zip(names, row)}
        gpus.append({key: next((values[name] for name in alternatives if values[name] is not None), None)
                     for key, alternatives in chosen.items()})
    if not gpus:
        raise PluginError(UNKNOWN, "nvidia-smi reports no GPU" + (" with index %s" % args.index if args.index else ""))
    return gpus


# --------------------------------------------------------------------------
# Checks: each returns state, problems, a summary part and perfdata for one GPU
# --------------------------------------------------------------------------
class Result:
    def __init__(self, gpu):
        self.gpu = gpu
        self.prefix = "gpu%s_" % gpu["index"]
        self.state = OK
        self.problems = []
        self.details = []
        self.perf = []

    def alert(self, state, problem):
        self.state = max(self.state, state)
        self.problems.append(problem)

    def add_perf(self, label, value, uom="", warn=None, crit=None, minimum=None, maximum=None):
        if value is not None:
            self.perf.append(perfdata(self.prefix + label, value, uom, warn, crit, minimum, maximum))


def check_temperature(result, args):
    gpu = result.gpu
    temperature, tlimit = gpu["temperature"], gpu["tlimit"]
    if temperature is None:
        result.alert(UNKNOWN, "no temperature")
        return
    absolute = args.temperature_warning is not None or args.temperature_critical is not None
    if tlimit is not None and not absolute:
        # T.Limit: degrees below the GPU's maximum operating temperature; it slows down from about -2 on
        state = evaluate(tlimit, args.tlimit_warning, args.tlimit_critical)
        result.details.append("%dC (%dC below its limit)" % (temperature, tlimit) if tlimit >= 0
                              else "%dC (%dC above its limit)" % (temperature, -tlimit))
        if state:
            result.alert(state, "%dC, %s its maximum operating temperature" % (
                temperature, "%dC above" % -tlimit if tlimit < 0 else "at"))
        result.add_perf("temp", temperature, "", None, None, 0)
        result.add_perf("temp_tlimit", tlimit, "", args.tlimit_warning, args.tlimit_critical)
    else:
        warning = args.temperature_warning or Range.parse(DEFAULT_TEMPERATURE_WARNING)
        critical = args.temperature_critical or Range.parse(DEFAULT_TEMPERATURE_CRITICAL)
        state = evaluate(temperature, warning, critical)
        result.details.append("%dC" % temperature)
        if state:
            result.alert(state, "temperature %dC" % temperature)
        result.add_perf("temp", temperature, "", warning, critical, 0)
    if gpu["temperature_memory"] is not None:
        result.details.append("memory %dC" % gpu["temperature_memory"])
        result.add_perf("temp_memory", gpu["temperature_memory"], "", None, None, 0)


def check_power(result, args):
    gpu = result.gpu
    power, limit = gpu["power"], gpu["power_limit"]
    if power is None:
        result.details.append("power draw not reported")
        return
    text = "%.0fW" % power
    if limit:
        text += " of %.0fW limit" % limit
        if gpu["power_max_limit"] and gpu["power_max_limit"] != limit:
            text += " (max %.0fW)" % gpu["power_max_limit"]
    if gpu["pstate"]:
        text += ", %s" % gpu["pstate"]
    result.details.append(text)
    state = evaluate(power, args.power_warning, args.power_critical)
    if state:
        result.alert(state, "power draw %.0fW" % power)
    result.add_perf("power", round(power, 2), "W", args.power_warning, args.power_critical, 0,
                    gpu["power_max_limit"])
    result.add_perf("power_limit", limit, "W", None, None, 0)


def reason_names(mask):
    return [name for name, bit in REASONS.items() if mask & bit]


def check_clocks(result, args):
    gpu = result.gpu
    for key, label in (("clock_graphics", "graphics"), ("clock_sm", "sm"), ("clock_memory", "memory"),
                       ("clock_video", "video")):
        maximum = gpu.get(key + "_max")
        result.add_perf("clock_" + label, gpu[key], "MHz", None, None, 0, maximum)
    if gpu["clock_sm"] is not None:
        text = "SM %dMHz" % gpu["clock_sm"]
        if gpu["clock_sm_max"]:
            text += " of %dMHz" % gpu["clock_sm_max"]
        if gpu["clock_memory"] is not None:
            text += ", memory %dMHz" % gpu["clock_memory"]
        result.details.append(text)

    mask = gpu["reasons"]
    if not isinstance(mask, int):
        result.details.append("slowdown reasons not reported")
        return
    active = reason_names(mask)
    shown = [name for name in active if name != "gpu_idle"]
    result.details.append("slowed down by " + ", ".join(shown) if shown else
                          ("idle" if active else "full clocks"))
    for name in active:
        if name in args.critical_reasons:
            result.alert(CRITICAL, "slowdown %s" % name)
        elif name in args.warning_reasons:
            result.alert(WARNING, "slowdown %s" % name)
    for name in ("sw_power_cap", "sw_thermal_slowdown", "hw_slowdown", "hw_thermal_slowdown",
                 "hw_power_brake_slowdown"):
        result.add_perf("slowdown_" + name, 1 if mask & REASONS[name] else 0, "", None, None, 0, 1)


def check_memory(result, args):
    gpu = result.gpu
    total, used = gpu["memory_total"], gpu["memory_used"]
    if not total or used is None:
        result.details.append("memory not reported")
        return
    percent = 100.0 * used / total
    result.details.append("memory %.0f%% used (%d of %d MiB)" % (percent, used, total))
    state = evaluate(percent, args.memory_warning, args.memory_critical)
    if state:
        result.alert(state, "memory %.0f%% used" % percent)
    result.add_perf("memory_used", used * 1048576, "B", None, None, 0, total * 1048576)
    result.add_perf("memory_used_percent", round(percent, 1), "%", args.memory_warning, args.memory_critical, 0, 100)


def check_utilization(result, args):
    gpu = result.gpu
    if gpu["utilization_gpu"] is not None:
        text = "load %d%%" % gpu["utilization_gpu"]
        if gpu["utilization_memory"] is not None:
            text += ", memory bandwidth %d%%" % gpu["utilization_memory"]
        result.details.append(text)
        state = evaluate(gpu["utilization_gpu"], args.utilization_warning, args.utilization_critical)
        if state:
            result.alert(state, "load %d%%" % gpu["utilization_gpu"])
    for key, label in (("utilization_gpu", "util_gpu"), ("utilization_memory", "util_memory"),
                       ("utilization_encoder", "util_encoder"), ("utilization_decoder", "util_decoder")):
        warn, crit = (args.utilization_warning, args.utilization_critical) if key == "utilization_gpu" else (None, None)
        result.add_perf(label, gpu[key], "%", warn, crit, 0, 100)


def check_fan(result, args):
    fan = result.gpu["fan"]
    if fan is None:
        # passively cooled server GPUs have no fan of their own
        result.details.append("no fan")
        return
    result.details.append("fan %d%%" % fan)
    state = evaluate(fan, args.fan_warning, args.fan_critical)
    if state:
        result.alert(state, "fan %d%%" % fan)
    result.add_perf("fan", fan, "%", args.fan_warning, args.fan_critical, 0, 100)


def check_pcie(result, args):
    gpu = result.gpu
    width, width_max, gen, gen_max = gpu["pcie_width"], gpu["pcie_width_max"], gpu["pcie_gen"], gpu["pcie_gen_max"]
    if width is None or gen is None:
        result.details.append("PCIe link not reported")
        return
    result.details.append("PCIe gen%d x%d" % (gen, width) + (" (GPU can x%d)" % width_max
                                                             if width_max and width_max != width else ""))
    if width_max and width < width_max and not args.ignore_width:
        result.alert(WARNING, "PCIe link x%d instead of x%d" % (width, width_max))
    # Idle GPUs lower the link generation to save power, so only a busy GPU has to run at full speed.
    busy = (gpu["utilization_gpu"] or 0) >= args.busy
    if args.check_gen and busy and gen_max and gen < gen_max:
        result.alert(WARNING, "PCIe gen%d instead of gen%d under load" % (gen, gen_max))
    result.add_perf("pcie_width", width, "", None, None, 0, width_max)
    result.add_perf("pcie_gen", gen, "", None, None, 0, gen_max)


def check_ecc(result, args):
    gpu = result.gpu
    mode = gpu["ecc_mode"]
    if isinstance(mode, str) and mode.lower() == "disabled" or mode is None:
        result.details.append("ECC disabled" if mode else "no ECC")
        if args.require_ecc:
            result.alert(WARNING, "ECC is %s" % ("disabled" if mode else "not available"))
    else:
        corrected, uncorrected = gpu["ecc_corrected"] or 0, gpu["ecc_uncorrected"] or 0
        result.details.append("ECC errors since boot: %d corrected, %d uncorrected" % (corrected, uncorrected))
        if uncorrected:
            result.alert(CRITICAL, "%d uncorrected ECC error(s)" % uncorrected)
        state = evaluate(corrected, args.ecc_warning, args.ecc_critical)
        if state:
            result.alert(state, "%d corrected ECC error(s)" % corrected)
        result.add_perf("ecc_corrected", corrected, "c", args.ecc_warning, args.ecc_critical, 0)
        result.add_perf("ecc_uncorrected", uncorrected, "c", None, None, 0)
        result.add_perf("ecc_corrected_total", gpu["ecc_corrected_total"], "c", None, None, 0)
        result.add_perf("ecc_uncorrected_total", gpu["ecc_uncorrected_total"], "c", None, None, 0)
    # Memory repair: retired pages (older GPUs) or remapped rows (newer GPUs) waiting for a GPU reset
    if gpu["remap_failure"] == "Yes":
        result.alert(CRITICAL, "row remapping failed, the GPU needs service")
    for key, what in (("retired_pending", "page retirement"), ("remap_pending", "row remapping")):
        if gpu[key] == "Yes":
            result.alert(WARNING, "%s pending, reset the GPU" % what)


CHECKS = {
    "temperature": (check_temperature, "GPU temperature"),
    "power": (check_power, "power draw and limit"),
    "clocks": (check_clocks, "clocks and why the GPU slows down (power cap, thermal, hardware slowdown)"),
    "memory": (check_memory, "GPU memory usage"),
    "utilization": (check_utilization, "GPU, memory, encoder and decoder load"),
    "fan": (check_fan, "fan speed"),
    "pcie": (check_pcie, "PCIe link width and generation"),
    "ecc": (check_ecc, "ECC errors and pending memory repairs"),
}
MODES = dict(CHECKS, all=(None, "all of the above in one check"))


def run_checks(args):
    gpus = query(args)
    modes = list(CHECKS) if args.mode == "all" else [args.mode]
    state = OK
    problems, summaries, perf = [], [], []
    for gpu in gpus:
        result = Result(gpu)
        for mode in modes:
            CHECKS[mode][0](result, args)
        state = max(state, result.state)
        label = "GPU%s" % gpu["index"]
        if len(gpus) == 1 or args.mode == "all":
            label += " (%s)" % gpu["name"]
        summaries.append("%s: %s" % (label, ", ".join(result.details) or "nothing reported"))
        problems += ["GPU%s %s" % (gpu["index"], problem) for problem in result.problems]
        perf += result.perf
    message = "; ".join(summaries)
    if problems:
        message = "; ".join(problems) + " - " + message
    return state, message, perf


# --------------------------------------------------------------------------
# Command line
# --------------------------------------------------------------------------
class ArgumentParser(argparse.ArgumentParser):
    """argparse exits 2 on a usage error, which Nagios reads as CRITICAL; exit UNKNOWN instead."""

    def error(self, message):
        self.print_usage(sys.stderr)
        sys.stderr.write("%s: error: %s\n" % (self.prog, message))
        sys.exit(UNKNOWN)


def reason_list(text):
    names = [name.strip().lower() for name in text.split(",") if name.strip()]
    if names == ["none"]:
        return set()
    unknown = [name for name in names if name not in REASONS]
    if unknown:
        raise argparse.ArgumentTypeError("unknown slowdown reason(s) %s; known: %s, or none"
                                         % (", ".join(unknown), ", ".join(REASONS)))
    return set(names)


def gpu_indices(text):
    if not re.fullmatch(r"\d+(,\d+)*", text):
        raise argparse.ArgumentTypeError("expected GPU indices like 0 or 0,1, got %r" % text)
    return text


def build_parser():
    parser = ArgumentParser(
        prog="check_nvidia_smi",
        description="Nagios/Icinga plugin to check NVIDIA GPUs with nvidia-smi.",
        epilog="Thresholds are ranges as defined by the monitoring plugins guidelines: 80 alerts above 80, "
               "'30:' below 30, '@10:20' inside 10..20.")
    parser.add_argument("-V", "--version", action="version", version="check_nvidia_smi %s" % VERSION)

    common = ArgumentParser(add_help=False)
    common.add_argument("-i", "--index", type=gpu_indices, help="comma separated GPU indices (default: all GPUs)")
    common.add_argument("-t", "--timeout", type=float, default=DEFAULT_TIMEOUT,
                        help="seconds to wait for nvidia-smi (default: %d)" % DEFAULT_TIMEOUT)
    common.add_argument("--nvidia-smi", default="nvidia-smi", metavar="PATH",
                        help="nvidia-smi to run (default: nvidia-smi from PATH)")

    groups = {
        "temperature": [
            (("--temperature-warning",), dict(type=Range.parse, metavar="RANGE",
                                              help="WARNING outside this temperature range in Celsius; replaces "
                                                   "the T.Limit check (default without T.Limit: %s)"
                                                   % DEFAULT_TEMPERATURE_WARNING)),
            (("--temperature-critical",), dict(type=Range.parse, metavar="RANGE",
                                               help="CRITICAL outside this temperature range in Celsius (default "
                                                    "without T.Limit: %s)" % DEFAULT_TEMPERATURE_CRITICAL)),
            (("--tlimit-warning",), dict(type=Range.parse, default=Range.parse("0:"), metavar="RANGE",
                                         help="WARNING when the degrees below the GPU's maximum operating "
                                              "temperature (T.Limit) leave this range (default: 0:)")),
            (("--tlimit-critical",), dict(type=Range.parse, default=Range.parse("-2:"), metavar="RANGE",
                                          help="CRITICAL for T.Limit (default: -2:, where the GPU slows down)")),
        ],
        "power": [
            (("--power-warning",), dict(type=Range.parse, metavar="RANGE", help="WARNING for the power draw in W")),
            (("--power-critical",), dict(type=Range.parse, metavar="RANGE", help="CRITICAL for the power draw in W")),
        ],
        "clocks": [
            (("--warning-reasons",), dict(type=reason_list, default=reason_list(DEFAULT_WARNING_REASONS),
                                          metavar="REASONS",
                                          help="comma separated slowdown reasons that are a WARNING, or none "
                                               "(default: %s)" % DEFAULT_WARNING_REASONS)),
            (("--critical-reasons",), dict(type=reason_list, default=reason_list(DEFAULT_CRITICAL_REASONS),
                                           metavar="REASONS",
                                           help="slowdown reasons that are CRITICAL, or none (default: %s)"
                                                % DEFAULT_CRITICAL_REASONS)),
        ],
        "memory": [
            (("--memory-warning",), dict(type=Range.parse, metavar="RANGE", help="WARNING for the memory used in %%")),
            (("--memory-critical",), dict(type=Range.parse, metavar="RANGE",
                                          help="CRITICAL for the memory used in %%")),
        ],
        "utilization": [
            (("--utilization-warning",), dict(type=Range.parse, metavar="RANGE", help="WARNING for the GPU load in %%")),
            (("--utilization-critical",), dict(type=Range.parse, metavar="RANGE",
                                               help="CRITICAL for the GPU load in %%")),
        ],
        "fan": [
            (("--fan-warning",), dict(type=Range.parse, metavar="RANGE", help="WARNING for the fan speed in %%")),
            (("--fan-critical",), dict(type=Range.parse, metavar="RANGE", help="CRITICAL for the fan speed in %%")),
        ],
        "pcie": [
            (("--ignore-width",), dict(action="store_true",
                                       help="no WARNING when the link is narrower than the GPU can do")),
            (("--check-gen",), dict(action="store_true",
                                    help="WARNING when a busy GPU runs below its PCIe generation")),
            (("--busy",), dict(type=float, default=50, metavar="PERCENT",
                               help="load from which --check-gen applies (default: 50)")),
        ],
        "ecc": [
            (("--ecc-warning",), dict(type=Range.parse, metavar="RANGE",
                                      help="WARNING for corrected ECC errors since boot")),
            (("--ecc-critical",), dict(type=Range.parse, metavar="RANGE",
                                       help="CRITICAL for corrected ECC errors since boot")),
            (("--require-ecc",), dict(action="store_true", help="WARNING when ECC is disabled or not available")),
        ],
    }

    subparsers = parser.add_subparsers(dest="mode", required=True, metavar="MODE")
    for mode, (_, help_text) in MODES.items():
        sub = subparsers.add_parser(mode, parents=[common], help=help_text, description="Check " + help_text + ".")
        for name, options in groups.items():
            if mode in (name, "all"):
                for flags, kwargs in options:
                    sub.add_argument(*flags, **kwargs)
        # every mode knows every option, so a service can switch modes without failing on its options
        for name, options in groups.items():
            if mode not in (name, "all"):
                for flags, kwargs in options:
                    sub.add_argument(*flags, **dict(kwargs, help=argparse.SUPPRESS))
    return parser


def main():
    args = build_parser().parse_args()
    try:
        state, message, perf = run_checks(args)
    except PluginError as err:
        print("NVIDIA-SMI %s %s - %s" % (args.mode.upper(), STATE_TEXT[err.state], err.message))
        return err.state

    output = "NVIDIA-SMI %s %s - %s" % (args.mode.upper(), STATE_TEXT[state], message)
    if perf:
        output += " | " + " ".join(perf)
    print(output)
    return state


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        sys.exit(UNKNOWN)
