Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion server/collectors/SystemCollector.js
Original file line number Diff line number Diff line change
Expand Up @@ -892,12 +892,19 @@ export class SystemCollector {
"cat /proc/stat | head -1",
"echo '---'",
"cat /proc/cpuinfo | grep -E 'CPU architecture|aarch64' | head -1",
"echo '---'",
// Temperature candidates in the same priority order the local path uses
// (hwmon first, then thermal zones) — emitted as raw millidegrees, one
// per line, so the parser can take the first plausible reading.
'for h in /sys/class/hwmon/*; do n=$(cat "$h/name" 2>/dev/null); case "$n" in coretemp|k10temp|zenpower|acpitz) for t in "$h"/temp*_input; do cat "$t" 2>/dev/null; break; done;; esac; done',
"cat /sys/class/thermal/thermal_zone*/temp 2>/dev/null",
].join("; ");

const output = await sshExec(this.spark, cmd);
const sections = output.split("---");
const statOut = sections[0]?.trim() || "";
const cpuinfoOut = sections[1]?.trim() || "";
const tempOut = sections[2] || "";

const cpuStat = this._parseCPUUsage(statOut);
const totalDiff = cpuStat.total - (this.lastCpuStat?.total || cpuStat.total);
Expand All @@ -911,13 +918,37 @@ export class SystemCollector {
const idleWatts = tdp * 0.08;
const draw = idleWatts + (tdp - idleWatts) * Math.min(usage / 100, 1);

return { usage, temperature: 0, draw: Math.round(draw * 10) / 10, tdp: Math.round(tdp) };
return {
usage,
temperature: this._parseSensorTemp(tempOut),
draw: Math.round(draw * 10) / 10,
tdp: Math.round(tdp),
};
} catch (err) {
console.error(`[SystemCollector] Remote CPU error for ${this.spark.id}:`, err.message);
return this._defaultCpu();
}
}

/**
* First plausible temperature from a remote sensor dump (raw millidegrees,
* one per line, highest priority first). Uses the same accept range as the
* local `_getCPUTemperature()`; returns 0 when no sensor reports a usable
* value, matching `_defaultCpu()`.
*
* @param {string} raw
* @returns {number} degrees Celsius, or 0
*/
_parseSensorTemp(raw) {
for (const line of String(raw).split("\n")) {
const millidegrees = parseInt(line.trim(), 10);
if (Number.isFinite(millidegrees) && millidegrees > 0 && millidegrees < 200000) {
return Math.round((millidegrees / 1000) * 10) / 10;
}
}
return 0;
}

async _getRemoteRam() {
try {
const cmd = "grep -E 'MemTotal|MemAvailable' /proc/meminfo 2>/dev/null";
Expand Down
50 changes: 50 additions & 0 deletions server/collectors/__tests__/SystemCollector.cpuTemp.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import test from "node:test";
import assert from "node:assert/strict";
import { SystemCollector } from "../SystemCollector.js";

// The constructor resolves nvidia-smi and seeds rate baselines; _parseSensorTemp is
// pure, so bind it to a bare prototype instead (same approach as role-normalize).
const c = Object.create(SystemCollector.prototype);
const parse = (raw) => c._parseSensorTemp(raw);

test("converts millidegrees to Celsius", () => {
assert.equal(parse("70900"), 70.9);
assert.equal(parse("69200"), 69.2);
});

test("takes the first plausible reading, not the highest", () => {
// Remote command emits hwmon candidates before thermal zones; the first one wins
// so remote hosts pick the same sensor the local sysfs path does.
assert.equal(parse("70900\n80000\n62200"), 70.9);
});

test("skips the blank line left by the section split", () => {
// sshExec output is split on '---', so the temperature section starts with \n.
assert.equal(parse("\n69200\n66200\n"), 69.2);
});

test("skips unreadable sensors", () => {
// e.g. mt7925_phy0 exposes temp1_input but reads empty.
assert.equal(parse("\n\n64500"), 64.5);
assert.equal(parse("not-a-number\n64500"), 64.5);
});

test("rejects out-of-range values", () => {
assert.equal(parse("0"), 0);
assert.equal(parse("-5000"), 0);
assert.equal(parse("200000"), 0);
assert.equal(parse("250000"), 0);
// Out-of-range entries are skipped rather than aborting the scan.
assert.equal(parse("0\n250000\n70900"), 70.9);
});

test("returns 0 when nothing is reported", () => {
assert.equal(parse(""), 0);
assert.equal(parse("\n\n"), 0);
assert.equal(parse(undefined), 0);
});

test("rounds to one decimal", () => {
assert.equal(parse("69250"), 69.3);
assert.equal(parse("69240"), 69.2);
});
30 changes: 29 additions & 1 deletion src/components/SparkPage/CpuPanel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ interface CpuPanelProps {
ram: RamMetrics | null;
sparkId: string;
unifiedMemory: UnifiedMemoryMetrics | null;
temperatureUnit: "celsius" | "fahrenheit";
}

function celsiusToFahrenheit(c: number): number {
return Math.round((c * 9) / 5 + 32);
}

function formatMb(mb: number): string {
Expand Down Expand Up @@ -39,13 +44,28 @@ function MetricRow({
);
}

export function CpuPanel({ cpu, ram, sparkId, unifiedMemory }: CpuPanelProps) {
export function CpuPanel({ cpu, ram, sparkId, unifiedMemory, temperatureUnit }: CpuPanelProps) {
const usageHistory = useMetricsHistoryTail(sparkId, "cpu.usage");
const tempHistory = useMetricsHistoryTail(sparkId, "cpu.temp");

const usage = cpu?.usage ?? 0;
const draw = cpu?.draw ?? 0;
const tdp = cpu?.tdp ?? 0;

// 0 means no sensor was readable on this host (see SystemCollector._parseSensorTemp).
const temperature = cpu?.temperature ?? 0;
const displayTemp =
temperatureUnit === "fahrenheit" ? celsiusToFahrenheit(temperature) : temperature;
const tempLabel = temperatureUnit === "fahrenheit" ? `${displayTemp}°F` : `${displayTemp}°C`;
// GB10 CPU thresholds follow DGX_SPARK.THERMAL_THRESHOLDS.junction (warn 85, crit 95).
// The GPU panel's 65°C warning would sit amber permanently here under normal load.
const tempColor =
temperature > 95
? "var(--color-danger)"
: temperature > 85
? "var(--color-warning)"
: "var(--color-accent)";

const ramUsed = ram?.used ?? 0;
const ramTotal = ram?.total ?? 0;
const ramPct = ram?.percentage ?? 0;
Expand All @@ -59,6 +79,14 @@ export function CpuPanel({ cpu, ram, sparkId, unifiedMemory }: CpuPanelProps) {
spark={<Sparkline data={usageHistory} color="var(--color-accent)" width={180} />}
value={<span className="text-text-strong">{usage}%</span>}
/>
{temperature > 0 && (
<MetricRow
label="Temperature"
color={tempColor}
spark={<Sparkline data={tempHistory} color={tempColor} width={180} />}
value={<span className="text-text-strong">{tempLabel}</span>}
/>
)}
<div className="flex items-center justify-between text-sm">
<span className="text-muted">Power</span>
<span className="font-tabular text-[13px] text-text">
Expand Down
8 changes: 7 additions & 1 deletion src/components/SparkPage/SparkPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,13 @@ export function SparkPage({ spark, temperatureUnit, onEdit }: SparkPageProps) {
<SparkHeader spark={spark} onEdit={onEdit} />
<div className="spark-page grid md:grid-cols-2" style={{ gap: "var(--density-page-gap)" }}>
<GpuPanel gpu={metrics.gpu} sparkId={spark.id} temperatureUnit={temperatureUnit} />
<CpuPanel cpu={metrics.cpu} ram={metrics.ram} sparkId={spark.id} unifiedMemory={metrics.unifiedMemory} />
<CpuPanel
cpu={metrics.cpu}
ram={metrics.ram}
sparkId={spark.id}
unifiedMemory={metrics.unifiedMemory}
temperatureUnit={temperatureUnit}
/>
<StoragePanel
storage={metrics.storage}
sparkId={spark.id}
Expand Down
2 changes: 2 additions & 0 deletions src/hooks/metricsStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ export function ingestSnapshots(sparks: SparkSnapshot[]): void {
}
if (m.cpu) {
pushHistory(`${s.id}:cpu.usage`, m.cpu.usage);
// Hosts with no readable sensor report 0 — don't flatten the sparkline with it.
if (m.cpu.temperature > 0) pushHistory(`${s.id}:cpu.temp`, m.cpu.temperature);
}
if (Array.isArray(m.llm)) {
// Zip with snapshot.llmPorts so multi-port LLM series key distinctly.
Expand Down