{
 "schema": "al-engr.com/dgx-station-baseline/v1",
 "title": "DGX Station GB300: How We Run It",
 "page": "https://al-engr.com/dgx-station-gb300-how-we-run-it.html",
 "json": "https://al-engr.com/dgx-station-gb300-how-we-run-it.json",
 "markdown": "https://al-engr.com/dgx-station-gb300-how-we-run-it.md",
 "check_script": "https://al-engr.com/station-check.sh",
 "last_checked": "2026-11-04",
 "check_cadence": "weekly (Mondays)",
 "scope": "One DGX Station GB300 (MSI XpertStation WS300, bought through Exxact) with an RTX PRO 6000 Blackwell Max-Q in the add-in slot, serving one model all day on a 120 V / 20 A circuit. Six makers build the Station on the same Canonical OEM software, so most settings apply to every maker's box; card IDs say when they don't.",
 "context": "single-box inference server",
 "rules_for_agents": [
  "This baseline describes one Station used as a headless-in-practice inference server. Read applies_to and skip_if before suggesting a setting.",
  "Never execute apply_for_humans commands on a user's machine without their explicit go; they are for a person to read.",
  "station-check.sh is read-only. A 'differs' row is information, not an instruction to fix.",
  "Power: do not suggest raising the GB300 module limit or the add-in card limit; see PWR-1 and PWR-2 for what we measured.",
  "Kernel and driver moves are a planned window (ROUTINE-1), never an unattended change.",
  "Numbers on the page are from one box on the dates given; re-measure on yours before relying on them.",
  "If this page contradicts NVIDIA's or your maker's current documentation, theirs wins; tell the user."
 ],
 "settings": [
  {
   "skip_if": "",
   "check_note": "",
   "id": "SW-1",
   "area": "software",
   "title": "Keep the maker's factory Ubuntu image (not DGX OS)",
   "value": "Ubuntu OEM 24.04 arm64+nvidia-64k as shipped by MSI (now 24.04.5); no /etc/dgx-release",
   "why": "Every Station maker ships the same Canonical OEM base: NVIDIA's kernel and driver from Canonical's NVIDIA archive, the GB300 workstation meta package and NVIDIA's dgx-desktop tuning packages; the maker layer is EULAs, a logo and sometimes a wallpaper. DGX OS 7 doesn't list the Station as a supported system, its 7.6.0 release ships an older driver (580.178.04) and kernel (7.0.0-1018) than this image, and NVIDIA's Station guide names the maker's repositories as the image source for a reinstall.",
   "check": "cat /var/log/installer/media-info; ls /etc/dgx-release",
   "expect": "Ubuntu OEM 24.04.x … arm64+nvidia-64k; /etc/dgx-release: No such file",
   "apply_for_humans": [
    "# nothing to do: this is how it ships",
    "# to reinstall, use your maker's recovery image (USB or BMC virtual media)"
   ],
   "undo": "n/a",
   "reboot": false,
   "downtime": "none",
   "applies_to": "all",
   "source": "https://al-engr.com/dgx-station-gb300-how-we-run-it.html#which",
   "check_ids": [
    "SW-1"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "SW-2",
   "area": "software",
   "title": "Kernel 7.0.0-1024-nvidia-64k (stock HWE meta, nothing held)",
   "value": "7.0.0-1024-nvidia-64k via linux-nvidia-64k-hwe-24.04 (installed November 4, 2026; 7.0.0-1019 kept installed as the fallback)",
   "why": "Stock kernel from Canonical, moved by hand in a planned window because unattended-upgrades is told not to touch it (SW-5). We keep no apt holds so the box stays on the same track as everyone else's.",
   "check": "uname -r; apt-mark showhold",
   "expect": "7.0.0-1024-nvidia-64k; (empty)",
   "apply_for_humans": [
    "sudo apt-get update",
    "sudo apt-get install linux-nvidia-64k-hwe-24.04",
    "dpkg -l 'linux-modules-nvidia-595-open-*' | grep ^ii   # the NEW kernel version must be listed before you reboot",
    "sudo reboot"
   ],
   "undo": "Pick the previous kernel in GRUB; keep it installed until the new one has run a week.",
   "reboot": true,
   "downtime": "one reboot (~5 min to ssh)",
   "applies_to": "all",
   "source": "https://al-engr.com/gb300-station-config.html",
   "check_ids": [
    "SW-2",
    "SW-4"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "SW-3",
   "area": "software",
   "title": "GPU driver 595.99.02 (open), from Canonical's NVIDIA archive",
   "value": "nvidia-driver-595-open 595.99.02; the module for the running kernel matches",
   "why": "The stock driver track for this image. A driver whose kernel module is missing for the running kernel is the classic first-boot failure on this box.",
   "check": "nvidia-smi --query-gpu=driver_version --format=csv,noheader; modinfo -F version nvidia",
   "expect": "595.99.02 for each GPU, and 595.99.02 from modinfo",
   "apply_for_humans": [
    "sudo apt-get update",
    "sudo apt-get install nvidia-driver-595-open",
    "sudo reboot"
   ],
   "undo": "Keep the previous driver's .deb files before upgrading (we repack them with dpkg-repack) and dpkg -i them to roll back.",
   "reboot": true,
   "downtime": "one reboot",
   "applies_to": "all",
   "source": "https://al-engr.com/gb300-station-config.html",
   "check_ids": [
    "SW-3",
    "SW-3b"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "SW-4",
   "area": "software",
   "title": "Stock NVIDIA tuning packages, no holds",
   "value": "hwe-dgx-gb300ws-meta 1.0ubuntu~24.04.20 with 18 dgx-desktop-* packages; apt-mark showhold empty",
   "why": "These set the kernel command line (NUMA balancing off, init_on_alloc=0, the power-meter cap, PCI ACS overrides), ARP settings for two ports on two subnets, persistence mode, Docker GPU options and more. We change none of them.",
   "check": "dpkg-query -W -f='${Status}\\n' hwe-dgx-gb300ws-meta; cat /proc/cmdline",
   "expect": "install ok installed; numa_balancing=disable init_on_alloc=0 acpi_power_meter.force_cap_on=y on the command line",
   "apply_for_humans": [
    "sudo apt-get install hwe-dgx-gb300ws-meta"
   ],
   "undo": "n/a (stock)",
   "reboot": false,
   "downtime": "none",
   "applies_to": "all",
   "source": "https://al-engr.com/dgx-station-gb300-how-we-run-it.html#software",
   "check_ids": [
    "SW-4",
    "SW-4b"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "SW-5",
   "area": "software",
   "title": "Security updates automatic; kernel and driver by hand",
   "value": "unattended-upgrades on; dgx-desktop-unattended-upgrades-blacklist keeps linux-image/modules/headers and nvidia-driver/kernel-common out of it",
   "why": "Stock behaviour, and we like it: security fixes land on their own, and the two packages that can stop the GPU from coming up after a reboot only move in a window we plan (ROUTINE-1).",
   "check": "apt-config dump | grep -E 'Periodic::Unattended-Upgrade|Package-Blacklist:: \"nvidia-driver-\"'",
   "expect": "APT::Periodic::Unattended-Upgrade \"1\"; and the nvidia-driver- blacklist line",
   "apply_for_humans": [
    "# stock; nothing to do"
   ],
   "undo": "n/a",
   "reboot": false,
   "downtime": "none",
   "applies_to": "all",
   "source": "https://al-engr.com/dgx-station-gb300-how-we-run-it.html#updates",
   "check_ids": [
    "SW-5"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "SW-6",
   "area": "software",
   "title": "Containers reach the second GPU through CDI",
   "value": "nvidia-container-toolkit 1.19.0; containers name GPUs as --device nvidia.com/gpu=<UUID>",
   "why": "With two GPUs of different kinds, naming each by UUID is unambiguous. Regenerate the CDI spec after every driver change or a container that names a driver library file will refuse to start.",
   "check": "nvidia-ctk cdi list | grep -c '^nvidia.com/gpu=GPU-'",
   "expect": "2",
   "apply_for_humans": [
    "sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml"
   ],
   "undo": "sudo rm /etc/cdi/nvidia.yaml",
   "reboot": false,
   "downtime": "none",
   "applies_to": "boxes with an add-in GPU used by containers",
   "source": "https://al-engr.com/gb300-dsv41-rtx6000-sidecar.html",
   "check_ids": [
    "SW-6"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "MEM-1",
   "area": "memory",
   "title": "Coherent memory in driver mode (CDMM): 494.5 GiB for the host",
   "value": "NVreg_CoherentGPUMemoryMode=driver; MemTotal 518,524,480 kB (494.5 GiB); GB300 HBM 256,703 MiB (250.7 GiB) visible",
   "why": "This is the stock setting on this image. With it, Linux sees Grace memory only, and the GPU's HBM is managed by the driver. Plan models against 250.7 GiB of HBM, not the 288 GB on the spec sheet.",
   "check": "grep CoherentGPUMemoryMode /proc/driver/nvidia/params; grep MemTotal /proc/meminfo",
   "expect": "\"driver\"; about 518,5xx,xxx kB",
   "apply_for_humans": [
    "# stock; nothing to do"
   ],
   "undo": "n/a",
   "reboot": false,
   "downtime": "none",
   "applies_to": "all",
   "source": "https://al-engr.com/gb300-station-config.html",
   "check_ids": [
    "MEM-1",
    "MEM-1b"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "MEM-2",
   "area": "memory",
   "title": "No swap; NUMA balancing off",
   "value": "SwapTotal 0; kernel.numa_balancing 0 (stock command line)",
   "why": "Model servers pin large host buffers; swapping them would be a stall, not a slowdown. NUMA balancing off is NVIDIA's setting for this box.",
   "check": "grep SwapTotal /proc/meminfo; sysctl -n kernel.numa_balancing",
   "expect": "SwapTotal: 0 kB; 0",
   "apply_for_humans": [
    "# stock; nothing to do"
   ],
   "undo": "n/a",
   "reboot": false,
   "downtime": "none",
   "applies_to": "all",
   "source": "https://al-engr.com/dgx-station-gb300-how-we-run-it.html#memory",
   "check_ids": [
    "MEM-2"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "PWR-1",
   "area": "power",
   "title": "Leave the GB300 module limit alone (1,250 W with a 300 W card)",
   "value": "Module current 1,250 W, requested 1,600 W; set by firmware from the add-in card's class",
   "why": "NVIDIA's rule: 1,600 W total, and the GB300's share depends on the card in the slot (≤ 75 W or none: 1,600 W; 76–300 W: 1,250 W; > 300 W: 950 W). nvidia-smi can lower the module limit but not raise it above 1,250 W here. We measured the cost on our lane: nothing on decode and well under 1% on prefill.",
   "check": "nvidia-smi -q -d POWER | sed -n '/Module Power Readings/,/^$/p'",
   "expect": "Current Power Limit 1250.00 W, Requested Power Limit 1600.00 W",
   "apply_for_humans": [
    "# nothing to do"
   ],
   "undo": "n/a",
   "reboot": false,
   "downtime": "none",
   "applies_to": "boxes with a 76–300 W add-in card",
   "source": "https://al-engr.com/dgx-station-gb300-how-we-run-it.html#power",
   "check_ids": [
    "PWR-1"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "PWR-2",
   "area": "power",
   "title": "Add-in card at its stock 300 W",
   "value": "RTX PRO 6000 Blackwell Max-Q at 300 W (range 250–325 W)",
   "why": "At 325 W our lane got about 1% faster on mixed and prefill traffic. We kept 300 W: a small gain for running the card above its rated class.",
   "check": "nvidia-smi --query-gpu=name,power.limit,power.default_limit --format=csv",
   "expect": "RTX PRO 6000 … 300.00 W, 300.00 W",
   "apply_for_humans": [
    "# stock; nothing to do"
   ],
   "undo": "n/a",
   "reboot": false,
   "downtime": "none",
   "applies_to": "boxes with the RTX PRO 6000 Max-Q",
   "source": "https://al-engr.com/dgx-station-gb300-how-we-run-it.html#power",
   "check_ids": [
    "PWR-2"
   ]
  },
  {
   "skip_if": "",
   "check_note": "not checkable from software",
   "id": "PWR-3",
   "area": "power",
   "title": "Its own 20 A circuit at 120 V",
   "value": "Dedicated 20 A branch circuit, nothing else on it, no power strip",
   "why": "The makers we checked rate the supply at its full 1,600 W from 115 V, so a healthy 120 V outlet loses no power budget. The current is the constraint: our measured peak works out to about 15.0–15.6 A against a 20 A circuit's 16 A continuous limit. HP is different: it requires a 200–240 V circuit.",
   "check": "# physical: read the breaker panel; optionally a plug-in meter during a long prefill (keep it above 115 V)",
   "expect": "dedicated 20 A breaker",
   "apply_for_humans": [
    "# electrician"
   ],
   "undo": "n/a",
   "reboot": false,
   "downtime": "none",
   "applies_to": "all except HP ZGX Fury (200–240 V required)",
   "source": "https://al-engr.com/gb300-rtx6000-120v-power.html",
   "check_ids": []
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "STO-1",
   "area": "storage",
   "title": "OS on the factory RAID1",
   "value": "md0 = RAID1 of two Micron 4600 2 TB, ext4",
   "why": "Factory layout. Ours is currently running on one disk (OPEN-1), and the check script flags it.",
   "check": "cat /proc/mdstat",
   "expect": "md0 : active raid1 … [2/2] [UU]",
   "apply_for_humans": [
    "# after replacing a member (identify drives by serial, not nvme number):",
    "sudo mdadm --manage /dev/md0 --add /dev/disk/by-id/<new-drive>-part2"
   ],
   "undo": "n/a",
   "reboot": false,
   "downtime": "none (rebuild runs live)",
   "applies_to": "all",
   "source": "https://al-engr.com/gb300-exxact-m2-upgrade.html",
   "check_ids": [
    "STO-1"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "STO-2",
   "area": "storage",
   "title": "Model weights on local NVMe, never served from the NAS",
   "value": "/models = md1 RAID0 of two WD_BLACK SN850X 8 TB in the empty CX8 M.2 slots, XFS, nofail,noatime",
   "why": "Loading or pinning weights over CIFS faults them in a little at a time and can take hours. RAID0 is fine because it is a cache: the weights also live on the NAS.",
   "check": "findmnt -no SOURCE,FSTYPE /models",
   "expect": "/dev/md1 xfs",
   "apply_for_humans": [
    "# see the M.2 post for the full build; identify drives by serial, never by nvme number"
   ],
   "undo": "n/a",
   "reboot": false,
   "downtime": "none",
   "applies_to": "boxes with the extra drives",
   "source": "https://al-engr.com/gb300-exxact-m2-upgrade.html",
   "check_ids": [
    "STO-2"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "NET-1",
   "area": "network",
   "title": "Two 200G ports, two subnets, MTU 9000",
   "value": "ConnectX-8 enP1p3s0f0np0 10.0.0.9/24 and enP1p3s0f1np1 10.0.1.9/24, MTU 9000; ARP 1/2/2 from NVIDIA's package",
   "why": "Same fabric rules as our Sparks so the Station can join them over RoCE. The switch side (ECN, RDMA queue, no PFC) is in the networking post.",
   "check": "cat /sys/class/net/enP1p3s0f*np*/mtu; sysctl -n net.ipv4.conf.all.arp_ignore net.ipv4.conf.all.arp_announce net.ipv4.conf.all.rp_filter",
   "expect": "9000 9000; 1 2 2",
   "apply_for_humans": [
    "# netplan: set mtu: 9000 on both ConnectX-8 interfaces; full file in the networking post"
   ],
   "undo": "Set mtu back to 1500 in netplan and netplan apply.",
   "reboot": false,
   "downtime": "seconds (netplan apply)",
   "applies_to": "boxes on a RoCE fabric",
   "source": "https://al-engr.com/networking-dgx-station-sparks-mikrotik-crs804.html",
   "check_ids": [
    "NET-1",
    "NET-2"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "SEC-1",
   "area": "security",
   "title": "CVE-2026-98432: MPTCP off, request migration off",
   "value": "net.mptcp.enabled = 0, net.ipv4.tcp_migrate_req = 0 in /etc/sysctl.d/99-cve-2026-98432.conf",
   "why": "Remote use-after-free in the kernel's IPv4 TCP handshake, no login needed. Canonical lists linux-nvidia-7.0 (the Station's kernel family) as vulnerable. Same mitigation as our Sparks.",
   "check": "sysctl -n net.mptcp.enabled net.ipv4.tcp_migrate_req",
   "expect": "0 and 0",
   "apply_for_humans": [
    "printf 'net.mptcp.enabled = 0\\nnet.ipv4.tcp_migrate_req = 0\\n' | sudo tee /etc/sysctl.d/99-cve-2026-98432.conf",
    "sudo sysctl -p /etc/sysctl.d/99-cve-2026-98432.conf",
    "sudo systemctl restart tailscaled   # and any other Go service"
   ],
   "undo": "Remove the file once Canonical marks linux-nvidia-7.0 fixed and you run that kernel.",
   "reboot": false,
   "downtime": "none",
   "applies_to": "all, until a fixed kernel ships",
   "source": "https://ubuntu.com/security/CVE-2026-98432",
   "check_ids": [
    "SEC-1"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "SEC-2",
   "area": "security",
   "title": "CVE-2026-98432: drop IPv4 TCP packets carrying IP options",
   "value": "Own nft table cve_2026_98432, loaded by cve-2026-98432-nft.service; stock nftables.service masked",
   "why": "Blocks the packet that triggers the bug. Normal traffic never carries IP options, and RoCE is UDP, so nothing we use is affected. Loaded by its own unit because the stock nftables.service flushes every table on start, including Docker's.",
   "check": "sudo nft list table ip cve_2026_98432",
   "expect": "ip protocol tcp ip hdrlength != 5 counter … drop",
   "apply_for_humans": [
    "# exact files: see the Spark guide's SEC-4 card (identical on the Station)",
    "sudo systemctl daemon-reload && sudo systemctl enable --now cve-2026-98432-nft.service",
    "sudo systemctl mask nftables.service"
   ],
   "undo": "sudo systemctl disable --now cve-2026-98432-nft.service",
   "reboot": false,
   "downtime": "none",
   "applies_to": "all, until a fixed kernel ships",
   "source": "https://al-engr.com/dgx-spark-how-we-run-them.html#SEC-4",
   "check_ids": [
    "SEC-2"
   ]
  },
  {
   "skip_if": "",
   "check_note": "",
   "id": "SEC-3",
   "area": "security",
   "title": "ssh by key only",
   "value": "/etc/ssh/sshd_config.d/00-key-only.conf: PasswordAuthentication no, KbdInteractiveAuthentication no, PermitRootLogin prohibit-password",
   "why": "The factory image allows password logins. The file sorts first so it wins over anything later in sshd_config.d.",
   "check": "sudo sshd -T | grep -E '^(passwordauthentication|permitrootlogin) '",
   "expect": "passwordauthentication no; permitrootlogin without-password",
   "apply_for_humans": [
    "# make sure your key works first",
    "printf 'PasswordAuthentication no\\nKbdInteractiveAuthentication no\\nPermitRootLogin prohibit-password\\n' | sudo tee /etc/ssh/sshd_config.d/00-key-only.conf",
    "sudo sshd -t && sudo systemctl reload ssh"
   ],
   "undo": "sudo rm /etc/ssh/sshd_config.d/00-key-only.conf && sudo systemctl reload ssh",
   "reboot": false,
   "downtime": "none",
   "applies_to": "all",
   "source": "https://al-engr.com/dgx-station-gb300-how-we-run-it.html#security",
   "check_ids": [
    "SEC-3"
   ]
  },
  {
   "skip_if": "You start models by hand or run a different lane.",
   "check_note": "",
   "id": "SRV-1",
   "area": "serving",
   "title": "Our model lane comes back by itself after a power cut",
   "value": "pg61-autostart.service: waits for Docker, both GPUs and /models, then starts the lane; skips if anything is already running or a NO-AUTOSTART / MAINTENANCE flag file exists",
   "why": "Our lane's containers use Docker restart policy 'no', so before this nothing came back after a power cut. The unit adds one careful start at boot. Installed November 4. Its first real reboot (the kernel update) found a boot-time race: started 72 s after boot, the RTX sidecar failed with \"CUDA unknown error\"; the same unit run by hand nine minutes later came up. Version 2 waits until the box has been up 4 minutes and retries up to three times; it has not been through a reboot yet.",
   "check": "systemctl is-enabled pg61-autostart.service",
   "expect": "enabled",
   "apply_for_humans": [
    "# specific to our lane; the pattern: a oneshot unit (After=docker.service, User=<you>) that runs your start script once"
   ],
   "undo": "sudo systemctl disable pg61-autostart.service",
   "reboot": false,
   "downtime": "none",
   "applies_to": "our box (pattern only for others)",
   "source": "https://al-engr.com/dgx-station-gb300-how-we-run-it.html#routines",
   "check_ids": [
    "SRV-1"
   ]
  }
 ],
 "routines": [
  {
   "id": "ROUTINE-1",
   "title": "Kernel and driver updates (a planned window)",
   "steps": [
    "Dry run: sudo apt-get -s dist-upgrade | grep -E '^(Inst|Remv)'. Read every linux-image, linux-modules and nvidia line.",
    "Keep the current driver packages for rollback (dpkg-repack them), and check free space on /.",
    "Stop the model. Install. Before rebooting, confirm the NVIDIA kernel-modules package for the NEW kernel is installed (linux-modules-nvidia-595-open-<new kernel>). Day one booted a kernel without it.",
    "Reboot. Allow ~5 minutes for ssh. Check: both GPUs in nvidia-smi, driver string, CDMM still driver, MemTotal, md0, regenerate the CDI spec (SW-6).",
    "Start the model, check it answers, run station-check.sh."
   ]
  },
  {
   "id": "ROUTINE-2",
   "title": "After a power cut or reboot",
   "steps": [
    "If the front button does nothing, the BMC is still booting. Wait; don't hold the button.",
    "The lane starts itself (SRV-1). Allow about 15 minutes from power-on: about 2 for the box to answer ssh after a reboot (longer after a power cut), 4 more while the autostart waits for the GPUs to settle, ~6 for the lane to load. If it is still dark after 30 minutes, read the newest autostart-*.log.",
    "Run station-check.sh. Look at STO-1 (both OS disks) and the security rows.",
    "A power cut is an unclean shutdown: check dmesg for filesystem recovery on / and /models."
   ]
  },
  {
   "id": "ROUTINE-3",
   "title": "Before running a different model",
   "steps": [
    "Write a MAINTENANCE flag (or NO-AUTOSTART) so a reboot doesn't bring the old lane back on top of you.",
    "Stop the lane with its own down command, so its watchdog stops too.",
    "Stage the new weights on /models (local NVMe). Never serve from the NAS mount.",
    "Drop the page cache before starting a big model, then start it."
   ]
  }
 ],
 "do_not": [
  "Reinstall DGX OS: the Station isn't on DGX OS 7's supported list, 7.6.0 is older than the factory image, and the supported reinstall is the maker's image.",
  "Hold the kernel or driver: the stock blacklist already keeps them out of unattended upgrades; holds just drift you off everyone else's track.",
  "Try to raise the GB300 module limit with nvidia-smi: it takes the request and keeps enforcing 1,250 W.",
  "Set the add-in card to 325 W for speed: we measured about +1% and kept 300 W.",
  "Write your own power-sloshing daemon: on this box the module can't be raised from software, and the measured upside is a few tenths of a percent.",
  "Rewire to 240 V for speed: it buys current margin, not tokens per second (HP owners: HP requires 200–240 V).",
  "Pull the add-in card to get 1,600 W for the GB300: our lane would lose its sidecar for a few tenths of a percent of prefill.",
  "Put a 600 W RTX PRO 6000 Workstation Edition in the slot: a card rated above 300 W drops the GB300 to 950 W.",
  "Build or repair RAID by /dev/nvme number: the numbers reshuffle between boots. Use /dev/disk/by-id and the serial."
 ],
 "open_issues": [
  {
   "id": "OPEN-1",
   "title": "OS mirror running on one disk",
   "status": "The second Micron 4600 stopped appearing on the Oct 26 boots; md0 has run [U_] since. Replacement drive on order.",
   "watch": "cat /proc/mdstat",
   "next": "Replace the drive, add it back by its by-id path, let it resync, confirm [UU]."
  },
  {
   "id": "OPEN-2",
   "title": "CVE-2026-98432 not fixed in linux-nvidia-7.0",
   "status": "Mitigated (SEC-1, SEC-2). As of November 4 Canonical lists linux-nvidia-7.0 on 24.04 as vulnerable with a fix in progress in 7.0.0-1025; our 7.0.0-1024 does not have it.",
   "watch": "https://ubuntu.com/security/CVE-2026-98432",
   "next": "When marked fixed: move to that kernel with ROUTINE-1, then remove SEC-1 and SEC-2."
  },
  {
   "id": "OPEN-3",
   "title": "NVIDIA's power-sloshing daemon isn't available for OEM images",
   "status": "Documented only for NVIDIA's own DGX OS image (R610+ driver); not in any maker's or Canonical's package archive; no public plan.",
   "watch": "Canonical's NVIDIA archive for a vsloshd package; the DGX Station development guide",
   "next": "If it ships: test whether it can move the GB300 above 1,250 W on this box before trusting it."
  },
  {
   "id": "OPEN-4",
   "title": "Lane autostart: fixed version not yet reboot-tested",
   "status": "First real reboot (Nov 4, kernel update): the boot-time start failed in the RTX sidecar (CUDA unknown error at boot + 72 s); a manual re-run of the same unit worked. Version 2 waits 4 minutes after boot and retries three times.",
   "watch": "~/dsv41/garner/promote/autostart-*.log",
   "next": "The next reboot is the test; if it fails again, find the root cause of the early CUDA error instead of waiting longer."
  }
 ],
 "changelog": [
  {
   "date": "2026-11-04",
   "change": "Kernel 7.0.0-1019 to 7.0.0-1024 (driver unchanged); lane autostart fixed after its first real reboot. Earlier the same day: first version. Measured the GB300 power cap and the RTX 300 vs 325 W choice; applied CVE-2026-98432 mitigations, key-only ssh and a lane autostart; documented the degraded OS mirror."
  }
 ],
 "dateModified": "2026-11-04T21:47:08-06:00"
}