https://odysseus-lxcq.onrender.com/static/js/cookbookRunning.js

1// ============================================
2// COOKBOOK RUNNING SUB-MODULE
3// Running tasks tab: task cards, status monitoring,
4// stop/restart, diagnosis, auto-fix, background monitor
5// ============================================
6
7import uiModule from './ui.js';
8import { _diagnose, _showDiagnosis, _clearDiagnosis } from './cookbook-diagnosis.js';
9import { registerMenuDismiss } from './escMenuStack.js';
10import { computeProgressSignal } from './cookbookProgressSignal.js';
11import { portOf, nextFreePort } from './cookbookPorts.js';
12import { topPortalZ } from './toolWindowZOrder.js';
13
14// Human-friendly badge label for a task's internal status. Avoids surfacing
15// the word "error" in the sidebar — a server the user stopped or one that
16// quit cleanly reads as "stopped", not "error".
17function _statusLabel(status, type) {
18  if (status === 'running' && type === 'download') return 'downloading';
19  if (status === 'done' && type === 'download') return 'finished';
20  if (status === 'error') return 'stopped';
21  return status || '';
22}
23
24function _downloadBadgeText(progress) {
25  const raw = String(progress || '').trim();
26  if (!raw) return 'downloading';
27  const pct = raw.match(/(\d+)%/);
28  if (pct) return pct[0];
29  if (/^(?:Downloading|Fetching|Resuming)\s+'[^']+'\s+to\s+'[^']+/i.test(raw)
30    || /^Downloading\s*\(incomplete\b/i.test(raw)) {
31    return 'downloading';
32  }
33  return raw;
34}
35
36// Single source of truth for what a task's status badge shows + its style class.
37// Crucially, a serve task that's still coming up shows its live phase
38// ("loading 45%", "warming up", …) rather than the generic "running" — they're
39// the same state, so the badge shouldn't flip between two different labels on
40// every re-render. Returns { text, cls } where cls is appended after
41// "cookbook-task-status" ('' = the neutral loading style).
42function _taskBadge(task) {
43  if (task._unreachable && task.status === 'running') return { text: 'unreachable', cls: 'cookbook-task-error' };
44  if (task.type === 'download' && task.status === 'running') {
45    return { text: _downloadBadgeText(task.progress), cls: 'cookbook-task-downloading' };
46  }
47  if (task.type === 'serve' && task.status === 'running' && task.progress) {
48    // Same green "running" pill — just with dynamic phase text, so it doesn't
49    // read as a different status while the server is coming up.
50    return { text: task.progress, cls: 'cookbook-task-running' };
51  }
52  return { text: _statusLabel(task.status, task.type), cls: 'cookbook-task-' + task.status };
53}
54
55function _ggufDisplayPartFromPath(path) {
56  const parts = String(path || '').split('/').filter(Boolean);
57  const file = parts[parts.length - 1] || '';
58  const dir = parts.length > 1 ? parts[parts.length - 2] : '';
59  const text = `${dir} ${file}`;
60  const quant = text.match(/\b(?:UD-)?(?:IQ[1-8]_[A-Z0-9]+|Q[2-8]_K_[MLS]|Q[2-8]_[0-9A-Z]+|Q[2-8])\b/i);
61  if (quant) return quant[0].toUpperCase().replace(/^UD-/, '');
62  return file.replace(/\.gguf$/i, '').replace(/-\d{5}-of-\d{5}$/i, '');
63}
64
65function _downloadDisplayName(name, task) {
66  const include = task?.payload?.include || '';
67  if (!include || String(name || '').includes(' · ')) return name;
68  const part = _ggufDisplayPartFromPath(include.replace(/\*/g, ''));
69  return part ? `${name} · ${part}` : name;
70}
71
72function _downloadNameFromPayload(name, payload) {
73  const rawName = String(name || '').trim();
74  // Defensive: failed/restarted downloads can inherit the wrapper executable
75  // name if older state was saved from a command preview. The row title should
76  // always be the model/repo, never "bash", "python", or a live HF progress
77  // line like "Downloading 'vae/...' to '/mnt/...".
78  const looksLikeLauncher = /^(?:bash|sh|zsh|python|python3|pwsh|powershell|cmd|tmux)$/i.test(rawName);
79  const looksLikeProgressLine = /^(?:Downloading|Fetching|Resuming)\s+'[^']+'\s+to\s+'[^']+/i.test(rawName)
80    || /^Downloading\s*\(incomplete\b/i.test(rawName);
81  const base = (!rawName || looksLikeLauncher || looksLikeProgressLine)
82    ? String(payload?.repo_id || payload?.repo || '').split('/').pop()
83    : rawName;
84  const include = payload?.include || '';
85  if (!include || String(base || '').includes(' · ')) return base || rawName || 'download';
86  const part = _ggufDisplayPartFromPath(String(include).replace(/\*/g, ''));
87  return part ? `${base} · ${part}` : (base || rawName || 'download');
88}
89
90function _taskDisplayName(task) {
91  const name = String(task?.name || '').trim();
92  if (task?.type === 'download') return _downloadDisplayName(_downloadNameFromPayload(name, task?.payload), task);
93  if (task?.type !== 'serve') return name;
94  const gguf = task?.payload?._fields?.gguf_file || task?.payload?.gguf_file || '';
95  if (!gguf || name.includes(' · ')) return name;
96  const part = _ggufDisplayPartFromPath(gguf);
97  return part ? `${name} · ${part}` : name;
98}
99
100function _canLaunchDownloadedTask(task) {
101  return task?.type === 'download' && ['done', 'completed'].includes(task.status || '') && !!(task.payload?.repo_id || task.name);
102}
103
104function _downloadServeFields(task) {
105  const include = String(task?.payload?.include || '').trim();
106  if (!include) return null;
107  return {
108    backend: 'llamacpp',
109    _forceBackend: true,
110    _preferredGgufInclude: include,
111  };
112}
113
114// A download task whose tmux output still shows an active per-shard line
115// (e.g. "model-00012-of-00082.safetensors: 56%|") is NOT actually finished —
116// the cookbook just lost track. The clear pill becomes a "reconnect" affordance
117// in that case (click → revive the row + reattach the poll loop).
118function _downloadOutputLooksActive(task) {
119  if (!task || task.type !== 'download') return false;

120  const out = task.output || '';
121  if (!out) return false;
122  if (out.includes('DOWNLOAD_OK') || out.includes('DOWNLOAD_FAILED')) return false;
123  // An active shard line: filename + a colon + a percentage that isn't 100%.
124  // We catch any in-flight shard or "Downloading 'X' to ..." line (no %).
125  return /model-\d+-of-\d+\.[a-z]+:\s+(?!100%)\d+%/i.test(out)
126      || /Downloading\s+'[^']+'\s+to\s+'[^']*\.incomplete'/i.test(out);
127}
128
129function _canClearTask(task) {
130  if (!task || task.status === 'running') return false;
131  if (task.type === 'serve' && (task.status === 'ready' || (!['error', 'crashed', 'failed', 'completed'].includes(task.status) && _serveOutputLooksReady(task)))) return false;
132  // If the tmux output still shows an in-flight download, the task isn't
133  // actually finished — hide the clear/check pill so it doesn't show on a
134  // task that's still doing work. (The next render will reflect this and
135  // ideally the self-heal flips status back to running.)
136  if (_downloadOutputLooksActive(task)) return false;
137  return ['done', 'completed', 'stopped', 'error', 'crashed', 'failed'].includes(task.status);
138}
139
140function _clearPillLabel(task) {
141  if (_downloadOutputLooksActive(task)) return 'reconnect';
142  return 'clear';
143}
144
145function _venvRootFromPath(path) {
146  let p = (path || '').toString().trim().replace(/\/+$/, '');
147  if (!p) return '';
148  p = p.replace(/\/bin\/(?:activate|python(?:3(?:\.\d+)?)?|vllm|pip(?:3)?)$/i, '');
149  return p;
150}
151
152// A pip dependency/driver install (payload._dep) reports success with the
153// runner's "=== Process exited with code 0 ===" sentinel and pip's
154// "Successfully installed" line — never the HuggingFace download markers
155// (DONE / 100% / /snapshots/ / DOWNLOAD_OK) that the download heuristics look
156// for. Without this, a clean install whose tmux pane has already gone away is
157// misread as crashed/stopped even though pip exited 0. Prefer the authoritative
158// exit-code sentinel; fall back to pip's success line when no sentinel was
159// captured (and there's no install error in the same output).
160function _depInstallSucceeded(output) {
161  const text = String(output || '');
162  if (!text) return false;
163  const exitMatch = text.match(/=== Process exited with code (-?\d+) ===/);
164  if (exitMatch) return Number(exitMatch[1]) === 0;
165  return /\b(?:Successfully installed|Requirement already satisfied)\b/.test(text)
166    && !/\bERROR\b|No matching distribution|Could not find a version|Traceback \(most recent call last\)/.test(text);
167}
168
169function _shouldOfferCrashReport(task) {
170  if (!task) return false;
171  if (task._unreachable && task.type === 'serve') return true;
172  return ['error', 'crashed', 'failed'].includes(task.status);
173}
174
175function _serveTaskLooksAwqOnLocalBackend(task, outputText = '') {
176  const repo = `${task?.payload?.repo_id || ''} ${task?.name || ''}`.toLowerCase();
177  const cmd = `${task?.payload?._cmd || ''} ${outputText || ''}`.toLowerCase();
178  return /\b(awq|gptq|fp8)\b/.test(repo) && /(llama-server|llama_cpp\.server|ollama|ggml_cuda_enable_unified_memory)/.test(cmd);
179}
180
181function _serveTaskLooksAwqWithoutUsableAccelerator(task, outputText = '') {
182  const repo = `${task?.payload?.repo_id || ''} ${task?.name || ''}`.toLowerCase();
183  const out = String(outputText || '').toLowerCase();
184  return /\b(awq|gptq|fp8)\b/.test(repo)
185    && /(no accelerator|no cuda runtime|failed to infer device type|triton is not supported|0 active driver)/i.test(out);
186}
187
188async function _openDownloadForGgufTask(task) {
189  const raw = task?.payload?.repo_id || task?.name || '';
190  const modelName = String(raw)
191    .split('/').pop()
192    .replace(/[-_](?:AWQ|GPTQ|FP8|4bit|8bit|Int4|Int8).*$/i, '')
193    .replace(/[-_]+$/g, '')
194    || String(raw).split('/').pop()
195    || raw;
196  const cookbook = window.cookbookModule;
197  if (cookbook && typeof cookbook.open === 'function') {
198    cookbook.open({ tab: 'Search' });
199  } else {
200    document.getElementById('tool-cookbook-btn')?.click();
201  }
202  setTimeout(async () => {
203    const modal = document.getElementById('cookbook-modal');
204    const tab = modal?.querySelector('.cookbook-tab[data-backend="Search"]');
205    if (tab && !tab.classList.contains('active')) tab.click();
206    const search = document.getElementById('hwfit-search');
207    if (search) {
208      search.value = modelName;
209      search.dispatchEvent(new Event('input', { bubbles: true }));
210      search.focus();
211    }
212    const quant = document.getElementById('hwfit-quant');
213    if (quant) {
214      quant.value = 'Q4_K_M';
215      quant.dispatchEvent(new Event('change', { bubbles: true }));
216    }
217    try {
218      const hwfit = await import('./cookbook-hwfit.js');
219      if (typeof hwfit._hwfitFetch === 'function') hwfit._hwfitFetch(true);
220    } catch {}
221  }, 80);
222}
223
224function _terminalServeDiagnosis(task, outputText) {
225  const out = String(outputText || task?.output || '');
226  if (!task || task.type !== 'serve' || !['stopped', 'error', 'crashed', 'failed'].includes(task.status) || !out.trim()) return null;
227  // Suppress the crash diagnosis when the output proves the server
228  // actually became reachable — e.g. an early `exit 127` from a failed
229  // build attempt was followed by the shim/Python fallback successfully
230  // starting Uvicorn. Without this, the user sees a confusing "build
231  // stopped before the server became reachable" toast while the server
232  // is right there serving requests.
233  if (_serveOutputLooksReady(task)) return null;
234  // Pip tasks (Reinstall vLLM, Upgrade torch, etc.) ride on the serve task
235  // type so they get a tmux session + show up in Running tab — but they are
236  // NOT serve invocations. Their output is pip's own; the generic
237  // "Serve stopped before the model became reachable" message + Edit-serve

238  // fix make no sense. Bail so the panel just shows pip's output.
239  const _isPipTask = ((task.payload?.repo_id || '').startsWith('pip-'))
240    || /python3? -m pip\b/.test(task.payload?._cmd || '');
241  if (_isPipTask) return null;
242  if (_serveTaskLooksAwqOnLocalBackend(task, out)) {
243    return {
244      message: 'AWQ/GPTQ/FP8 cannot be served through llama.cpp/Ollama unified-memory mode.',
245      suggestion: 'Suggested action: use vLLM/SGLang on a compatible CUDA/ROCm GPU server, or download a GGUF version for llama.cpp/Ollama/unified-memory serving.',
246      fixes: [
247        { label: 'Find GGUF download', action: () => _openDownloadForGgufTask(task) },
248        { label: 'Edit serve', action: (panel) => _openServeEditForTask(task) },
249      ],
250    };
251  }
252  if (_serveTaskLooksAwqWithoutUsableAccelerator(task, out)) {
253    return {
254      message: 'AWQ/GPTQ/FP8 needs a working vLLM/SGLang accelerator path; this server did not expose one.',
255      suggestion: 'Suggested action: choose a CUDA/ROCm server where vLLM/SGLang can see the GPU, or download a GGUF version and serve it with llama.cpp/Ollama.',
256      fixes: [
257        { label: 'Find GGUF download', action: () => _openDownloadForGgufTask(task) },
258        { label: 'Edit serve', action: (panel) => _openServeEditForTask(task) },
259      ],
260    };
261  }
262  return _diagnose(out) || {
263    message: /Native llama-server not found|building llama-server|llama\.cpp/i.test(out)
264      ? 'llama.cpp build stopped before the server became reachable.'
265      : 'Serve stopped before the model became reachable.',
266    suggestion: /Native llama-server not found|building llama-server|llama\.cpp/i.test(out)
267      ? 'Suggested action: copy the troubleshooting bundle, then edit serve settings. For the quickest local/CPU path, use Ollama or a prebuilt llama-server; source builds can take several minutes and fail if build dependencies are incomplete.'
268      : 'Suggested action: copy the troubleshooting bundle, then edit serve settings or relaunch with a CPU/backend fallback.',
269    fixes: [{ label: 'Edit serve', action: (panel) => _openServeEditForTask(task) }],
270  };
271}
272
273function _redactCrashReportText(text) {
274  if (!text) return '';
275  return String(text)
276    .replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]{12,}/gi, '$1[redacted]')
277    .replace(/\b(hf_[A-Za-z0-9]{16,})\b/g, '[redacted-hf-token]')
278    .replace(/\b(sk-[A-Za-z0-9_-]{16,})\b/g, '[redacted-api-key]')
279    .replace(/\b(xox[baprs]-[A-Za-z0-9-]{16,})\b/g, '[redacted-slack-token]')
280    .replace(/\b(AIza[0-9A-Za-z_-]{20,})\b/g, '[redacted-google-key]')
281    .replace(/\b((?:HF_TOKEN|HUGGING_FACE_HUB_TOKEN|OPENAI_API_KEY|ANTHROPIC_API_KEY|BRAVE_API_KEY|TAVILY_API_KEY|SERPER_API_KEY|GOOGLE_API_KEY|API_KEY|TOKEN|PASSWORD)\s*=\s*)(['"]?)[^\s'"\\]+/gi, '$1$2[redacted]')
282    .replace(/\b(--(?:api-key|token|hf-token|password)\s+)([^\s]+)/gi, '$1[redacted]');
283}
284
285function _lastLines(text, count = 160) {
286  const clean = _redactCrashReportText(text || '').trimEnd();
287  if (!clean) return '(no captured output)';
288  return clean.split('\n').slice(-count).join('\n');
289}
290
291function _codeFence(text) {
292  return String(text || '').replace(/```/g, '` ` `');
293}
294
295function _taskHostLabel(task) {
296  if (!task?.remoteHost) return 'local';
297  return task.remoteHost + (task.sshPort ? `:${task.sshPort}` : '');
298}
299
300function _taskPort(task) {
301  return portOf(task?.payload?._cmd || '');
302}
303
304function _buildCrashReport(task, outputText) {
305  const capturedOutput = outputText || task?.output || '';
306  const cmd = _redactCrashReportText(task?.payload?._cmd || '');
307  const diag = _diagnose(capturedOutput);
308  const started = task?.ts ? new Date(task.ts).toISOString() : '';
309  const report = [
310    '## Odysseus Cookbook crash report',
311    '',
312    'Please review this report for secrets before posting it publicly.',
313    '',
314    '### Task',
315    `- ID: \`${task?.sessionId || task?.id || 'unknown'}\``,
316    `- Type: \`${task?.type || 'unknown'}\``,
317    `- Status: \`${task?._unreachable ? 'unreachable' : (task?.status || 'unknown')}\``,
318    `- Model/repo: \`${task?.payload?.repo_id || task?.name || 'unknown'}\``,
319    `- Host: \`${_taskHostLabel(task)}\``,
320  ];
321  if (task?.platform) report.push(`- Platform: \`${task.platform}\``);
322  if (started) report.push(`- Started: \`${started}\``);
323  const port = _taskPort(task);
324  if (port) report.push(`- Port: \`${port}\``);
325  if (diag?.message) report.push(`- Diagnosis: ${diag.message}`);
326  if (cmd) {
327    report.push('', '### Command', '```bash', _codeFence(cmd), '```');
328  }
329  report.push('', '### Last captured output', '```text', _codeFence(_lastLines(capturedOutput)), '```');
330  return report.join('\n');
331}
332
333// Shared state/functions injected by init()

334let _envState;
335let _sshCmd;
336let _getPort;
337let _sshPrefix;
338let _getPlatform;
339let _isWindows;
340let _buildEnvPrefix;
341let _loadPresets;
342let _savePresets;
343let _copyText;
344let _persistEnvState;
345let _refreshDependencies;
346let _serverByVal;
347let _serverKey;
348let _selectedServer;
349let modelLogo;
350let esc;
351let _detectBackend;
352let _detectToolParser;
353let _detectModelOptimizations;
354let _buildServeCmd;
355
356function _taskServerSelection(task) {
357  const host = task?.remoteHost || task?.payload?.remote_host || '';
358  const savedKey = task?.remoteServerKey || task?.payload?.remote_server_key || '';
359  const server = (savedKey ? _serverByVal(savedKey) : null)
360    || (host ? _serverByVal(host) : null)
361    || (host ? _envState.servers.find(s => s.host === host) : null)
362    || null;
363  const key = server ? (_serverKey ? _serverKey(server) : savedKey) : (savedKey || (host || 'local'));
364  return { host, server, key };
365}
366
367function _serverColorForTaskGroup(key, tasks) {
368  const firstTask = Array.isArray(tasks) ? tasks[0] : null;
369  const host = firstTask?.remoteHost || firstTask?.payload?.remote_host || '';
370  const savedKey = firstTask?.remoteServerKey || firstTask?.payload?.remote_server_key || key || '';
371  const server = (savedKey ? _serverByVal?.(savedKey) : null)
372    || (key ? _serverByVal?.(key) : null)
373    || (host ? _serverByVal?.(host) : null)
374    || (key === 'local' || !key ? (_envState?.servers || []).find(s => !s.host || String(s.host).toLowerCase() === 'local') : null)
375    || null;
376  const color = String(server?.color || '').trim();
377  return /^#[0-9a-fA-F]{6}$/.test(color) ? color : '';
378}
379
380function _serverHeaderStyle(color) {
381  if (!color) return '';
382  const c = color.toLowerCase();
383  const accent = (c === '#ffffff' || c === '#f8fafc') ? '#cbd5e1'
384    : (c === '#111827' || c === '#000000') ? '#64748b'
385    : color;
386  return ` style="--cookbook-server-color:${esc(color)};--cookbook-server-accent:${esc(accent)};"`;
387}
388
389function _shouldAutoExpandTaskOutput(task) {
390  return task?.type === 'download'
391    && !task?.payload?._dep
392    && ['running', 'queued', 'error', 'crashed'].includes(String(task?.status || ''));
393}
394
395function _selectTaskServer(task) {
396  const { host, server, key } = _taskServerSelection(task);
397  _envState.remoteHost = host;
398  _envState.remoteServerKey = key === 'local' ? '' : key;
399  if (server) {
400    _envState.env = server.env || 'none';
401    _envState.envPath = server.envPath || '';
402    _envState.platform = server.platform || '';
403  } else if (!host) {
404    _envState.env = 'none';
405    _envState.envPath = '';
406    _envState.platform = '';
407  }
408  document.querySelectorAll('#hwfit-server-select, #hwfit-dl-server, #hwfit-cache-server, #hwfit-deps-server').forEach(sel => {
409    if (!sel || sel.tagName !== 'SELECT') return;
410    const wanted = key || (host || 'local');
411    if ([...sel.options].some(o => o.value === wanted)) sel.value = wanted;
412    else if (host && [...sel.options].some(o => o.value === host)) sel.value = host;
413    else sel.value = host ? wanted : 'local';
414  });
415  return { host, server, key };
416}
417
418// When a new action is started (download / dependency / serve), this holds the
419// new task's id so the next render collapses every other card and leaves only
420// the new one open. Consumed (cleared) by _renderRunningTab.
421let _soloExpandTaskId = null;
422
423// Storage keys
424const TASKS_KEY = 'cookbook-tasks';
425const STORAGE_KEY = 'cookbook-presets';
426const SERVE_STATE_KEY = 'cookbook-serve-state';
427const SERVE_FAVORITES_KEY = 'cookbook-serve-favorite-models';
428
429// Polling / timeout intervals
430const TASK_POLL_INTERVAL_MS = 3000;       // delay between reconnect-loop iterations
431const BG_MONITOR_INTERVAL_MS = 10000;     // background task status poll
432const STALE_PROGRESS_MS = 5 * 60 * 1000;  // download with no progress this long = stale
433const STARTUP_STALE_PROGRESS_MS = 45 * 1000; // 0%-forever startup stall: retry much sooner
434

435// ── Phase detection (mirrors Python _parse_serve_phase in cookbook_routes.py) ──
436// Single source of truth for serve task status. KEEP IN SYNC with the Python version.
437export function _parseServePhase(snapshot) {
438  if (!snapshot) return {};
439  // Strip newlines so tmux line-wrapping doesn't break regex matching
440  const flat = snapshot.replace(/\s+/g, ' ');
441  const loadMatches = [...flat.matchAll(/Loading safetensors.*?(\d+)%/g)];
442  // "Downloading (incomplete total...)" tracks real aggregate bytes; prefer it
443  // over "Fetching N files" which only counts fully-closed files and lags badly
444  // with hf_transfer's parallel-chunk strategy (often sits at 0/N for most of the run).
445  const downloadingMatches = [...flat.matchAll(/Downloading.*?(\d+)%/g)];
446  const fetchingMatches = [...flat.matchAll(/Fetching.*?(\d+)%/g)];
447  const dlMatches = downloadingMatches.length ? downloadingMatches : fetchingMatches;
448  // "Avg generation throughput: X tokens/s, Running: N reqs"
449  const tpsMatches = [...flat.matchAll(/(?:Avg )?generation throughput:\s*([\d.]+)\s*tokens\/s.*?Running:\s*(\d+)\s*reqs/g)];
450
451  // Throughput FIRST — its log line contains "GPU KV cache usage" which would
452  // otherwise false-match the warmup check
453  if (tpsMatches.length) {
454    const m = tpsMatches[tpsMatches.length - 1];
455    const tps = parseFloat(m[1]);
456    const reqs = parseInt(m[2]);
457    return {
458      phase: reqs > 0 ? `${m[1]} tok/s` : 'idle',
459      status: 'ready',
460      tps,
461      reqs,
462    };
463  }
464  if (flat.includes('Application startup complete')) {
465    return { phase: 'ready', status: 'ready' };
466  }
467  if (/Ollama API ready on port\s+\d+/i.test(flat)) {
468    return { phase: 'ready', status: 'ready' };
469  }
470  const llamaBuildMatches = [...flat.matchAll(/\[\s*(\d{1,3})%\]\s*(?:Building|Linking)/gi)];
471  if (llamaBuildMatches.length) {
472    const pct = Math.min(100, parseInt(llamaBuildMatches[llamaBuildMatches.length - 1][1], 10));
473    return { phase: `building llama.cpp ${pct}%`, status: 'running', pct };
474  }
475  if (/Native llama-server not found|building from source/i.test(flat)) {
476    if (/Cloning into ['"]?llama\.cpp/i.test(flat) && !/Receiving objects:\s*100%/i.test(flat)) {
477      return { phase: 'cloning llama.cpp', status: 'running' };
478    }
479    if (/Configuring incomplete|CMake Error/i.test(flat)) {
480      return {};
481    }
482    if (/CMAKE_BUILD_TYPE|Detecting CXX|Found Threads|Including CPU backend|CUDA nvcc found|building llama-server/i.test(flat)) {
483      return { phase: 'configuring llama.cpp', status: 'running' };
484    }
485    return { phase: 'building llama.cpp', status: 'running' };
486  }
487  // HTTP access logs (e.g. GET /v1/models 200 OK) mean the server is up
488  if (/(?:GET|POST)\s+\/[^\s]*\s+HTTP\/[\d.]+"\s*\d{3}/.test(flat)) {
489    return { phase: 'idle', status: 'ready' };
490  }
491  if (flat.includes('Loading weights took')) {
492    return { phase: 'initializing', status: 'running' };
493  }
494  // "GPU KV cache" alone (during allocation) — not "GPU KV cache usage" (runtime log)
495  if (flat.includes('GPU KV cache') && !flat.includes('GPU KV cache usage')) {
496    return { phase: 'warming up', status: 'running' };
497  }
498  if (loadMatches.length) {
499    const pct = parseInt(loadMatches[loadMatches.length - 1][1]);
500    return { phase: `loading ${pct}%`, status: 'running', pct };
501  }
502  if (dlMatches.length) {
503    const pct = parseInt(dlMatches[dlMatches.length - 1][1]);
504    return { phase: `downloading ${pct}%`, status: 'running', pct };
505  }
506  return {};
507}
508
509// ── Port auto-increment ──
510
511function _nextAvailablePort() {
512  const tasks = _loadTasks();
513  const presets = _loadPresets();
514  const usedPorts = new Set();
515  tasks.forEach(t => {
516    if (t.type === 'serve' && (t.status === 'running' || t.status === 'queued')) {
517      const p = _taskPort(t);
518      if (p) usedPorts.add(parseInt(p));
519    }
520  });
521  presets.forEach(p => {
522    if (p.port) usedPorts.add(parseInt(p.port));
523  });
524  return nextFreePort(usedPorts);
525}
526
527// ── Endpoint cleanup ──
528
529async function _removeEndpointByUrl(baseUrl) {
530  try {
531    const res = await fetch('/api/model-endpoints', { credentials: 'same-origin' });
532    if (!res.ok) return;
533    const endpoints = await res.json();
534    const hostPort = baseUrl.replace(/^https?:\/\//, '').replace(/\/.*$/, '');
535    const ep = endpoints.find(e => e.base_url === baseUrl)
536            || endpoints.find(e => e.base_url.includes(hostPort));
537    if (ep) {
538      await fetch(`/api/model-endpoints/${ep.id}`, { method: 'DELETE', credentials: 'same-origin' });
539      _refreshModelsAfterEndpointChange();
540    }
541  } catch {}
542}
543
544function _refreshModelsAfterEndpointChange() {
545  const pickerLabel = document.getElementById('model-picker-label');
546  if (pickerLabel) {
547    pickerLabel.dataset.prevHtml = pickerLabel.innerHTML;
548    pickerLabel.innerHTML = '<span style="opacity:0.4;">refreshing…</span>';
549  }
550  if (window.modelsModule && window.modelsModule.refreshModels) {
551    window.modelsModule.refreshModels(false);
552  }
553  setTimeout(() => {
554    if (!window.sessionModule) return;
555    const currentModel = window.sessionModule.getCurrentModel ? window.sessionModule.getCurrentModel() : null;
556    if (currentModel) {
557      const items = (window.modelsModule && window.modelsModule.getCachedItems) ? window.modelsModule.getCachedItems() : [];
558      const allModels = [];

559      items.forEach(item => {
560        if (item.offline) return;
561        (item.models || []).concat(item.models_extra || []).forEach(m => allModels.push({ mid: m, url: item.url, endpointId: item.endpoint_id }));
562      });
563      const stillExists = allModels.some(m => m.mid === currentModel);
564      if (!stillExists && allModels.length > 0) {
565        const fallback = allModels[0];
566        if (window.sessionModule.createDirectChat) {
567          window.sessionModule.createDirectChat(fallback.url, fallback.mid, fallback.endpointId);
568        }
569      }
570    }
571    if (window.sessionModule.updateModelPicker) {
572      window.sessionModule.updateModelPicker();
573    }
574  }, 1500);
575}
576
577function _appendCookbookEndpointScope(fd, remoteHost) {
578  const host = String(remoteHost || '').trim();
579  if (!host || host === 'local' || host === 'localhost' || host === '127.0.0.1') {
580    fd.append('container_local', 'true');
581  }
582}
583
584function _connectHostFromRemote(remoteHost, fallback = 'localhost') {
585  const host = String(remoteHost || '').trim();
586  if (!host || host === 'local') return fallback;
587  return host.includes('@') ? host.split('@').pop() : host;
588}
589
590function _isAnyBindHost(host) {
591  const h = String(host || '').trim().toLowerCase();
592  return h === '0.0.0.0' || h === '::' || h === '[::]';
593}
594
595function _endpointFromAdvertisedUrl(rawUrl, currentHost, fallbackPort = '11434') {
596  try {
597    const u = new URL(rawUrl);
598    const host = _isAnyBindHost(u.hostname) ? currentHost : (u.hostname || currentHost);
599    const port = u.port || fallbackPort;
600    const bracketedHost = host.includes(':') && !host.startsWith('[') ? `[${host}]` : host;
601    return { host, port, baseUrl: `${u.protocol}//${bracketedHost}${port ? `:${port}` : ''}/v1` };
602  } catch {
603    return null;
604  }
605}
606
607function _serveExpectedModel(task) {
608  const fields = task?.payload?._fields || {};
609  return String(
610    fields.served_model_name ||
611    fields.model_path ||
612    task?.payload?.repo_id ||
613    task?.model ||
614    task?.name ||
615    ''
616  ).trim();
617}
618
619function _modelIdMatchesExpected(modelId, expected) {
620  const got = String(modelId || '').trim().toLowerCase();
621  const want = String(expected || '').trim().toLowerCase();
622  if (!got || !want) return true;
623  if (got === want) return true;
624  const gotBase = got.split('/').pop();
625  const wantBase = want.split('/').pop();
626  return gotBase === wantBase || got.includes(wantBase) || want.includes(gotBase);
627}
628
629function _endpointMatchesServe(ep, task) {
630  const expected = _serveExpectedModel(task);
631  const models = [...(ep?.models || []), ...(ep?.pinned_models || [])];
632  if (!models.length) return true;
633  return models.some(mid => _modelIdMatchesExpected(mid, expected));
634}
635
636function _markServeEndpointMismatch(task, ep, host, port) {
637  const expected = _serveExpectedModel(task);
638  const actual = (ep?.models || []).join(', ') || 'no models';
639  const msg = `Port ${host}:${port} answered, but it is serving ${actual}, not ${expected || task?.name || 'the launched model'}. The new serve likely failed or the port is occupied by an older server.`;
640  _updateTask(task.sessionId || task.session_id, {
641    status: 'error',
642    _serveReady: false,
643    _endpointAdded: false,
644    output: `${task.output || ''}\n\n${msg}`.trim(),
645  });
646  uiModule.showError(msg);
647}
648
649function _appendPinnedServeModel(fd, task) {
650  const expected = _serveExpectedModel(task);
651  if (expected) fd.append('pinned_models', expected);
652}
653
654function _isImageServeTask(task) {
655  const cmd = String(task?.payload?._cmd || '');
656  return cmd.includes('diffusion_server') || cmd.includes('mlx_image_server');
657}
658
659// ── Download queue — runs one at a time per server ──
660
661function _processQueue() {
662  const tasks = _loadPrunedTasks();
663  const running = tasks.filter(t => t.type === 'download' && t.status === 'running');
664  const queued = tasks.filter(t => t.type === 'download' && t.status === 'queued');
665  if (!queued.length) return;
666
667  const busyHosts = new Set(running.map(t => t.remoteHost || 'local'));
668
669  for (const task of queued) {
670    const host = task.remoteHost || 'local';
671    if (busyHosts.has(host)) continue;
672    busyHosts.add(host);
673    _startQueuedDownload(task);
674  }
675}
676
677async function _startQueuedDownload(task) {
678  if (!task.payload) {
679    _updateTask(task.sessionId, { status: 'error', output: 'No payload' });
680    _renderRunningTab();
681    return;
682  }
683  // Flip to 'running' SYNCHRONOUSLY (before the async POST) so a concurrent
684  // _processQueue — or a second "Start now" — can't see it as still 'queued' and
685  // launch the same download a second time. Without this, finishing another
686  // download mid-POST re-queued this one into a duplicate task.
687  {
688    const _pre = _loadTasks();
689    const _pt = _pre.find(t => t.sessionId === task.sessionId);
690    if (_pt) {
691      if (_pt.status === 'running' && _pt._startLaunched) return;  // already being started
692      _pt.status = 'running';
693      _pt._startLaunched = true;
694      _saveTasks(_pre);
695    }
696  }
697  try {
698    const res = await fetch('/api/model/download', {
699      method: 'POST', credentials: 'same-origin',
700      headers: { 'Content-Type': 'application/json' },
701      body: JSON.stringify(task.payload),
702    });
703    if (!res.ok) {
704      const errText = await res.text().catch(() => '');
705      _updateTask(task.sessionId, { status: 'error', output: `HTTP ${res.status}: ${errText.slice(0, 200)}` });
706      _renderRunningTab();
707      return;
708    }
709    const data = await res.json();
710    if (!data.ok) {
711      _updateTask(task.sessionId, { status: 'error', output: data.error || 'Unknown error' });
712      _renderRunningTab();
713      return;
714    }
715    const oldId = task.sessionId;
716    const launchedTask = { ...task, sessionId: data.session_id, id: data.session_id, status: 'running' };

717    const key = _downloadDedupeKey(launchedTask);
718    let found = false;
719    const tasks = _loadTasks().filter(t => {
720      if (t.sessionId === oldId) {
721        found = true;
722        t.sessionId = data.session_id;
723        t.id = data.session_id;
724        t.status = 'running';
725        t._startLaunched = true;
726        return true;
727      }
728      if (t.sessionId === data.session_id) return false;
729      return !(key && t.type === 'download' && t.status === 'queued' && _downloadDedupeKey(t) === key);
730    });
731    if (!found) tasks.push(_redactTaskForStorage(launchedTask));
732    _saveTasks(tasks);
733    _renderRunningTab();
734    _startBackgroundMonitor();
735    await new Promise(r => setTimeout(r, 2000));
736    _renderRunningTab();
737  } catch (e) {
738    _updateTask(task.sessionId, { status: 'error', output: e.message || 'Network error' });
739    _renderRunningTab();
740  }
741}
742
743// ── Task CRUD ──
744
745function _serveOutputLooksReady(task) {
746  const out = String(task?.output || '');
747  return !!task?._serveReady
748    || /Application startup complete/i.test(out)
749    || /Ollama API ready on port\s+\d+/i.test(out)
750    || /(?:GET|POST)\s+\/[^\s]*\s+HTTP\/[\d.]+"\s*2\d\d/i.test(out);
751}
752
753function _normalizeTaskForDisplay(task) {
754  if (!task || typeof task !== 'object') return task;
755  // Pip tasks (Reinstall vLLM / Upgrade torch / etc.) ride on the serve task
756  // type so they get tmux + the Running tab. They are NOT serves — their
757  // "ready" markers are pip's `Successfully installed` / `Requirement already
758  // satisfied`, not "Application startup complete".
759  const _isPipTask = ((task.payload?.repo_id || '').startsWith('pip-'))
760    || /python3? -m pip\b/.test(task.payload?._cmd || '');
761  if (_isPipTask) {
762    // Override stale status: any pip task whose output carries pip's own
763    // success markers gets displayed as `done` regardless of what's in
764    // localStorage. Old pre-fix runs landed in error/stopped state and
765    // stuck there even after we taught the rest of the flow about pip
766    // tasks — this is the catch-all that flips them to Finished on render.
767    const out = String(task.output || '');
768    const ranOk = /Successfully installed|Requirement already (?:satisfied|up-to-date)/i.test(out)

769      && !/error:|ERROR:/.test(out.slice(-1024));
770    if (ranOk && task.status !== 'done' && task.status !== 'running') {
771      return { ...task, status: 'done' };
772    }
773    return task;
774  }
775  if (task.type === 'serve' && task.status === 'done' && !_serveOutputLooksReady(task)) {
776    return { ...task, status: 'error' };
777  }
778  return task;
779}
780
781export function _loadTasks() {
782  try { return (JSON.parse(localStorage.getItem(TASKS_KEY)) || []).map(_normalizeTaskForDisplay); }
783  catch { return []; }
784}
785
786function _downloadRepoKey(task) {
787  return String(task?.payload?.repo_id || task?.repo_id || task?.repo || task?.name || '').trim();
788}
789
790function _downloadHostKey(task) {
791  return String(task?.remoteHost || task?.payload?.remote_host || 'local').trim() || 'local';
792}
793
794function _downloadDedupeKey(task) {
795  if (!task || task.type !== 'download') return '';
796  const repo = _downloadRepoKey(task);
797  if (!repo) return '';
798  return `${_downloadHostKey(task)}\n${repo}`;
799}
800
801function _pruneQueuedDownloadDuplicates(tasks) {
802  if (!Array.isArray(tasks) || !tasks.length) return tasks || [];
803  const launched = new Set();
804  for (const task of tasks) {
805    if (task?.type !== 'download' || task.status === 'queued') continue;
806    const key = _downloadDedupeKey(task);
807    if (key) launched.add(key);
808  }
809
810  let changed = false;
811  const seenQueued = new Set();
812  const next = tasks.filter(task => {
813    if (task?.type !== 'download' || task.status !== 'queued') return true;
814    const key = _downloadDedupeKey(task);
815    if (!key) return true;
816    if (launched.has(key) || seenQueued.has(key)) {
817      changed = true;
818      return false;
819    }
820    seenQueued.add(key);
821    return true;
822  });
823  return changed ? next : tasks;
824}
825
826function _loadPrunedTasks() {
827  const tasks = _loadTasks();
828  const pruned = _pruneQueuedDownloadDuplicates(tasks);
829  if (pruned !== tasks) _saveTasks(pruned);
830  return pruned;
831}
832
833// Tombstones for removed tasks. Without these, removing a task only deletes it
834// locally — but the server still has it (its own POST guard even re-preserves
835// recently-added ones), so the next sync/poll merges it right back ("I removed
836// it and it came back"). A tombstone makes the removal stick: merges skip any
837// id the user removed, until the entry expires.
838const _REMOVED_KEY = 'cookbook-removed-tasks';
839const _TOMBSTONE_TTL_MS = 24 * 3600 * 1000;
840function _loadTombstones() {
841  try {
842    const tomb = JSON.parse(localStorage.getItem(_REMOVED_KEY)) || {};
843    const now = Date.now();
844    let changed = false;
845    for (const k in tomb) {
846      if (now - tomb[k] > _TOMBSTONE_TTL_MS) {
847        delete tomb[k];
848        changed = true;
849      }
850    }
851    if (changed) localStorage.setItem(_REMOVED_KEY, JSON.stringify(tomb));
852    return tomb;
853  }
854  catch { return {}; }
855}
856function _saveTombstones(tomb) {
857  localStorage.setItem(_REMOVED_KEY, JSON.stringify(tomb || {}));
858}
859function _tombstoneTask(id) {
860  if (!id) return;
861  const tomb = _loadTombstones();
862  const now = Date.now();
863  tomb[id] = now;
864  for (const k in tomb) { if (now - tomb[k] > _TOMBSTONE_TTL_MS) delete tomb[k]; }
865  _saveTombstones(tomb);
866}
867function _isTombstoned(id) {
868  const ts = _loadTombstones()[id];
869  return ts != null && (Date.now() - ts) <= _TOMBSTONE_TTL_MS;
870}
871
872function _redactStoredText(value) {
873  return String(value || '')
874    .replace(/hf_[A-Za-z0-9]{20,}/g, '[redacted-token]')
875    .replace(/((?:api[_-]?key|token|authorization|password|passwd|secret)\s*[=:]\s*)(["']?)[^\s"']+/gi, '$1$2[redacted]');
876}
877
878function _isServeOutputPlaceholder(value) {
879  const text = String(value || '').trim();
880  return !text || /^Launched via agent\s+—\s+waiting for tmux output/i.test(text);
881}
882
883function _redactTaskForStorage(task) {
884  if (!task || typeof task !== 'object') return task;
885  const safe = { ...task };
886  if (typeof safe.output === 'string') safe.output = _redactStoredText(safe.output);
887  if (safe.payload && typeof safe.payload === 'object') {
888    safe.payload = { ...safe.payload };
889    delete safe.payload.hf_token;
890    delete safe.payload.hfToken;
891    if (typeof safe.payload._cmd === 'string') safe.payload._cmd = _redactStoredText(safe.payload._cmd);
892    if (typeof safe.payload.cmd === 'string') safe.payload.cmd = _redactStoredText(safe.payload.cmd);
893  }
894  return safe;
895}
896
897function _stripStateSecrets(state) {
898  const safe = { ...state };
899  if (safe.env && typeof safe.env === 'object') {
900    const { hfToken, ...env }

900 = safe.env;
901    delete env.hostPlatform;
902    safe.env = env;
903  }
904  if (Array.isArray(safe.tasks)) safe.tasks = safe.tasks.map(_redactTaskForStorage);
905  return safe;
906}
907
908export function _saveTasks(tasks) {
909  localStorage.setItem(TASKS_KEY, JSON.stringify((tasks || []).map(_redactTaskForStorage)));
910  _syncToServer();
911}
912
913export function _addTask(sessionId, name, type, payload) {
914  let tasks = _loadTasks();
915  const remoteHost = (payload && payload.remote_host) || _envState.remoteHost || '';
916  const remoteServerKey = (payload && payload.remote_server_key) || '';
917  const remoteServerName = (payload && payload.remote_server_name) || '';
918  const sshPort = (payload && payload.ssh_port) || _getPort(remoteServerKey || remoteHost) || '';
919  const platform = (payload && payload.platform) || _getPlatform(remoteServerKey || remoteHost) || '';
920  // Serving a model supersedes its finished download — clear the matching
921  // finished download card (covers serving directly from the Serve tab, not just
922  // via the download card's "Serve →" button).
923  if (type === 'serve' && payload && payload.repo_id) {
924    const _repoId = payload.repo_id;
925    tasks = tasks.filter(t => !(t.type === 'download' && t.status === 'done' && t.payload && t.payload.repo_id === _repoId));
926  }
927  if (type === 'download' && payload && payload.repo_id) {
928    const key = _downloadDedupeKey({ type: 'download', payload, remoteHost });
929    tasks = tasks.filter(t => {
930      if (t.sessionId === sessionId) return false;
931      return !(key && t.type === 'download' && t.status === 'queued' && _downloadDedupeKey(t) === key);
932    });
933  }
934  const task = _redactTaskForStorage({ id: sessionId, sessionId, name, type, status: 'running', output: '', ts: Date.now(), payload: payload || null, remoteHost, remoteServerKey, remoteServerName, sshPort, platform });
935  tasks.push(task);
936  _saveTasks(tasks);
937  // New action → collapse all other cards, leave only this one open.
938  _soloExpandTaskId = sessionId;
939  _renderRunningTab();
940  // Always start the background monitor when a task is added — works even
941  // when modal is closed and ensures the sidebar shows live status immediately
942  _startBackgroundMonitor();
943  // Switch to Running tab
944  const body = document.querySelector('#cookbook-modal .cookbook-body');
945  if (body) {
946    const tab = body.querySelector('.cookbook-tab[data-backend="Running"]');
947    if (tab) tab.click();
948  }
949  return task;
950}
951
952function _updateTask(sessionId, updates) {
953  const tasks = _loadTasks();
954  const task = tasks.find(t => t.sessionId === sessionId);
955  if (task) {
956    Object.assign(task, updates);
957    _saveTasks(tasks);
958  }
959  if ('status' in updates || '_unreachable' in updates) {
960    _refreshServerDots();
961  }
962  if (updates.status && updates.status !== 'running') {
963    const el = document.querySelector(`.cookbook-task[data-task-id="${sessionId}"]`);
964    if (el) {
965      if (el._uptimeInterval) { clearInterval(el._uptimeInterval); el._uptimeInterval = null; }
966      const wave = el.querySelector('.cookbook-task-wave');
967      if (wave) wave.style.display = 'none';
968      const uptime = el.querySelector('.cookbook-task-uptime');
969      if (uptime) uptime.style.display = 'none';
970    }
971  }
972}
973
974function _refreshDepsAfterInstall(task) {
975  if (!task || task.type !== 'download' || !task.payload?._dep) return;
976  try {
977    _refreshDependencies?.({ host: task.remoteHost || '', port: task.sshPort || '', venv: task.payload?.env_path || '' });
978  } catch {}
979}
980
981export function _removeTask(sessionId) {
982  _tombstoneTask(sessionId);  // so sync/poll can't resurrect it
983  const tasks = _loadTasks().filter(t => t.sessionId !== sessionId);
984  _saveTasks(tasks);
985  _renderRunningTab();
986}
987
988// Fade/slide the task card out, then remove it — so the smooth exit is the same
989// whether a task auto-stops or the user removes/kills it manually.
990function _animateOutThenRemove(el, sessionId) {
991  if (!el || !el.style) { _removeTask(sessionId); return; }
992  if (el._abort) el._abort.abort();
993  el.style.transition = 'opacity 0.35s ease, transform 0.35s ease';
994  el.style.opacity = '0';
995  el.style.transform = 'translateX(-10px)';
996  setTimeout(() => _removeTask(sessionId), 360);
997}
998
999// ── tmux / Windows session commands ──
1000
1001function _taskRemoteHost(task) {
1002  return task?.remoteHost || task?.payload?.remote_host || '';
1003}
1004
1005function _remoteTmuxPrefix() {
1006  return 'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; ';
1007}
1008
1009export function _tmuxCmd(task, tmuxArgs) {
1010  if (_isWindows(task)) {
1011    return _winSessionCmd(task, tmuxArgs);
1012  }
1013  const host = _taskRemoteHost(task);
1014  if (host) {
1015    return `ssh ${_sshPrefix(_getPort(task))}${host} '${_remoteTmuxPrefix()}tmux ${tmuxArgs}' 2>/dev/null`;
1016  }
1017  return `tmux ${tmuxArgs} 2>/dev/null`;
1018}
1019
1020function _winSessionCmd(task, tmuxArgs) {
1021  const host = _taskRemoteHost(task);
1022  const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux';
1023  const sid = task.sessionId;
1024  const pf = _sshPrefix(_getPort(task));
1025  if (tmuxArgs.includes('capture-pane')) {
1026    const lines = tmuxArgs.match(/-S\s*-?(\d+)/)?.[1] || '200';
1027    const ps = host
1028      ? `Get-Content '${sd}\\${sid}.log' -Tail ${lines} -ErrorAction SilentlyContinue`
1029      : `Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.log') -Tail ${lines} -ErrorAction SilentlyContinue`;

1030    return _winPowerShellCmd(task, ps);
1031  }
1032  if (tmuxArgs.includes('has-session')) {
1033    const ps = host
1034      ? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Get-Process -Id $p -ErrorAction SilentlyContinue | Out-Null; if ($?) { exit 0 } else { exit 1 } } else { exit 1 }`
1035      : `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Get-Process -Id $p -ErrorAction SilentlyContinue | Out-Null; if ($?) { exit 0 } else { exit 1 } } else { exit 1 }`;
1036    return _winPowerShellCmd(task, ps);
1037  }
1038  if (tmuxArgs.includes('kill-session')) {
1039    const ps = _winSessionStopTreePs(task);
1040    return _winPowerShellCmd(task, ps);
1041  }
1042  if (tmuxArgs.includes('send-keys') && tmuxArgs.includes('C-c')) {
1043    const ps = host
1044      ? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -ErrorAction SilentlyContinue }`
1045      : `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -ErrorAction SilentlyContinue }`;
1046    return _winPowerShellCmd(task, ps);
1047  }
1048  return host ? `ssh ${pf}${host} '${_remoteTmuxPrefix()}tmux ${tmuxArgs}' 2>/dev/null` : `tmux ${tmuxArgs} 2>/dev/null`;
1049}
1050
1051function _winPowerShellCmd(task, ps) {
1052  const command = `powershell -Command "${ps}"`;
1053  const host = _taskRemoteHost(task);
1054  if (!host) return command;
1055  return `ssh ${_sshPrefix(_getPort(task))}${host} ${_shQuote(command)}`;
1056}
1057
1058function _winSessionStopTreePs(task) {
1059  const host = _taskRemoteHost(task);
1060  const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux';
1061  const sid = task.sessionId;
1062  const stopTree = `function Stop-Tree([int]$Id) { Get-CimInstance Win32_Process -Filter ('ParentProcessId = ' + $Id) -ErrorAction SilentlyContinue | ForEach-Object { Stop-Tree ([int]$_.ProcessId) }; Stop-Process -Id $Id -Force -ErrorAction SilentlyContinue }`;
1063  return host
1064    ? `${stopTree}; $p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p -match '^\\d+$') { Stop-Tree ([int]$p) }; Remove-Item '${sd}\\${sid}.*' -Force -ErrorAction SilentlyContinue`
1065    : `${stopTree}; $p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p -match '^\\d+$') { Stop-Tree ([int]$p) }; Remove-Item (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.*') -Force -ErrorAction SilentlyContinue`;
1066}
1067
1068export function _tmuxGracefulKill(task) {
1069  if (_isWindows(task)) {
1070    const ps = _winSessionStopTreePs(task);
1071    return _winPowerShellCmd(task, ps);
1072  }
1073  const host = _taskRemoteHost(task);
1074  if (host) {
1075    return `ssh ${_sshPrefix(_getPort(task))}${host} '${_remoteTmuxPrefix()}tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null'`;
1076  }
1077  return `tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null`;
1078}
1079
1080// Force-kill escalation: SIGKILL the tmux pane's owning PID and any children,
1081// then nuke the session. Use AFTER the graceful kill when the process is
1082// still detected — vLLM sometimes ignores SIGINT during model init, and a
1083// stuck CUDA context can survive `tmux kill-session` alone.
1084export function _tmuxForceKill(task) {
1085  if (_isWindows(task)) {
1086    // Windows graceful path already does Stop-Process -Force, so the same
1087    // command serves as the "force" variant.
1088    return _tmuxGracefulKill(task);
1089  }
1090  const sid = task.sessionId;
1091  const inner =
1092    `PIDS=$(tmux list-panes -t ${sid} -F "#{pane_pid}" 2>/dev/null); ` +
1093    `if [ -n "$PIDS" ]; then ` +
1094    `  for P in $PIDS; do ` +
1095    `    pkill -KILL -P "$P" 2>/dev/null; ` +
1096    `    kill -9 "$P" 2>/dev/null; ` +
1097    `  done; ` +
1098    `fi; ` +
1099    `tmux kill-session -t ${sid} 2>/dev/null`;
1100  const host = _taskRemoteHost(task);
1101  if (host) {
1102    return `ssh ${_sshPrefix(_getPort(task))}${host} ${_shQuote(_remoteTmuxPrefix() + inner)}`;
1103  }
1104  return inner;
1105}
1106
1107// Returns a shell snippet that prints "ALIVE" if the tmux session still
1108// exists (or its main PID is still listed in /proc), "DEAD" otherwise.
1109// Used by the Stop-all escalation to decide whether to force-kill.

1110export function _tmuxIsAliveCheck(task) {
1111  if (_isWindows(task)) {
1112    // Skip the check on Windows — the graceful path already force-kills.
1113    return null;
1114  }
1115  const sid = task.sessionId;
1116  const inner = `if tmux has-session -t ${sid} 2>/dev/null; then echo ALIVE; else echo DEAD; fi`;
1117  const host = _taskRemoteHost(task);
1118  if (host) {
1119    return `ssh ${_sshPrefix(_getPort(task))}${host} ${_shQuote(_remoteTmuxPrefix() + inner)}`;
1120  }
1121  return inner;
1122}
1123
1124function _shQuote(value) {
1125  return "'" + String(value ?? '').replace(/'/g, "'\\''") + "'";
1126}
1127
1128function _taskLooksOllama(task, outputText = '') {
1129  const haystack = `${task?.payload?.backend || ''} ${task?.payload?._cmd || ''} ${task?.payload?._fields?.backend || ''} ${outputText || ''}`;
1130  return /\bollama\b/i.test(haystack) || /Ollama API ready on port\s+\d+/i.test(haystack);
1131}
1132
1133function _ollamaBaseUrlForTask(task, outputText = '') {
1134  const out = String(outputText || '');
1135  const ready = out.match(/Ollama API ready on port\s+\d+:\s*(http:\/\/[^\s]+)/i);
1136  if (ready) return ready[1].replace(/\/+$/, '');
1137  const cmd = String(task?.payload?._cmd || '');
1138  const host = cmd.match(/OLLAMA_HOST=([^\s]+)/)?.[1] || '';
1139  const port = host.match(/:(\d+)$/)?.[1] || '11434';
1140  return `http://127.0.0.1:${port}`;
1141}
1142
1143function _ollamaModelForTask(task) {
1144  return String(task?.payload?.model || task?.payload?.repo_id || task?.name || '').trim();
1145}
1146
1147function _ollamaUnloadCommand(task, outputText = '') {
1148  if (!_taskLooksOllama(task, outputText)) return '';
1149  const model = _ollamaModelForTask(task);
1150  if (!model) return '';
1151  const base = _ollamaBaseUrlForTask(task, outputText);
1152  const body = JSON.stringify({ model, prompt: '', keep_alive: 0, stream: false });
1153  const inner = `curl -sf -X POST ${_shQuote(base + '/api/generate')} -H 'Content-Type: application/json' -d ${_shQuote(body)} >/dev/null 2>&1 || true`;
1154  const host = _taskRemoteHost(task);
1155  if (host) {
1156    return `ssh ${_sshPrefix(_getPort(task))}${host} ${_shQuote(inner)}`;
1157  }
1158  return inner;
1159}
1160
1161function _endpointUrlForTask(task, outputText = '') {
1162  if (_taskLooksOllama(task, outputText)) {
1163    return _ollamaBaseUrlForTask(task, outputText) + '/v1';
1164  }
1165  const host = _connectHostFromRemote(_taskRemoteHost(task));
1166  const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
1167  const port = portMatch ? portMatch[1] : '8000';
1168  return `http://${host}:${port}/v1`;
1169}
1170
1171// ── Wave animation ──
1172
1173const _waveFrames = ['▁▂▃', '▂▃▄', '▃▄▅', '▄▅▆', '▅▆▅', '▆▅▄', '▅▄▃', '▄▃▂', '▃▂▁'];
1174let _waveIdx = 0;
1175let _waveTimer = null;
1176const _waveEls = new Set();
1177
1178function _startWaveSync() {
1179  if (_waveTimer) return;
1180  _waveTimer = setInterval(() => {
1181    _waveIdx = (_waveIdx + 1) % _waveFrames.length;
1182    for (const el of _waveEls) {
1183      if (!el.isConnected) { _waveEls.delete(el); continue; }
1184      if (el.style.display !== 'none') el.textContent = _waveFrames[_waveIdx];
1185    }
1186    if (!_waveEls.size) { clearInterval(_waveTimer); _waveTimer = null; }
1187  }, 200);
1188}
1189
1190function _registerWaveEl(el) { _waveEls.add(el); _startWaveSync(); }
1191
1192// ── Notifications ──
1193
1194function _showCookbookNotif(isError = false) {
1195  const dot = document.getElementById('cookbook-notif-dot');
1196  if (dot) {
1197    dot.style.display = '';
1198    dot.classList.toggle('cookbook-notif-error', isError);
1199  }
1200  const btn = document.getElementById('tool-cookbook-btn');
1201  if (btn) { btn.style.opacity = '1'; btn.classList.add('cookbook-notif-active'); }
1202  const railBtn = document.getElementById('rail-cookbook');
1203  if (railBtn) {
1204    railBtn.classList.remove('rail-notify-success', 'rail-notify-error');
1205    railBtn.classList.add('rail-notify', isError ? 'rail-notify-error' : 'rail-notify-success', 'cookbook-notif-active');
1206  }
1207  if (window._syncRailDynamic) window._syncRailDynamic();
1208}
1209
1210export function _clearCookbookNotif() {
1211  const dot = document.getElementById('cookbook-notif-dot');
1212  if (dot) dot.style.display = 'none';
1213  const btn = document.getElementById('tool-cookbook-btn');
1214  if (btn) { btn.style.opacity = ''; btn.classList.remove('cookbook-notif-active'); }
1215  const railBtn = document.getElementById('rail-cookbook');
1216  if (railBtn) {
1217    railBtn.classList.remove('rail-notify', 'rail-notify-success', 'cookbook-notif-active');
1218  }
1219  if (window._syncRailDynamic) window._syncRailDynamic();
1220}
1221
1222// ── Presets helper (for save-from-task) ──
1223
1224// A preset must carry the venv + activated GPUs, not just the command — without
1225// them a relaunch has no environment activated and no GPU pinning, so a config
1226// that worked when saved fails on reload. Pull them from the launch payload
1227// (_env/_envPath/_gpus, captured by _launchServeTask) and fold them into the
1228// serve-form `fields` the Serve panel restores from.
1229function _presetEnvFields(task) {
1230  const p = task.payload || {};
1231  const fields = { ...(p._fields || {}) };
1232  // The Serve panel's venv field is a path; conda/venv both activate from it.
1233  if (p._envPath && (p._env === 'venv' || p._env === 'conda')) fields.venv = fields.venv || p._envPath;
1234  if (p._gpus) fields.gpus = p._gpus;
1235  return {
1236    fields: Object.keys(fields).length ? fields : undefined,
1237    env: p._env || '',
1238    envPath: p._envPath || '',
1239    gpus: p._gpus || '',
1240  };
1241}
1242
1243function _redactPresetForStorage(preset) {
1244  if (!preset || typeof preset !== 'object') return preset;
1245  const safe = { ...preset };
1246  if (typeof safe.cmd === 'string') safe.cmd = _redactStoredText(safe.cmd);
1247  if (typeof safe.command === 'string') safe.command = _redactStoredText(safe.command);
1248  delete safe.hf_token;
1249  delete safe.hfToken;
1250  return safe;
1251}
1252
1253function _saveTaskAsPreset(task, label) {
1254  const host = task.remoteHost || 'localhost';

1255  const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
1256  const port = portMatch ? portMatch[1] : '8000';
1257  const presets = _loadPresets();
1258  if (presets.some(p => p.cmd === task.payload._cmd)) return false;
1259  presets.push(_redactPresetForStorage({ name: task.name, model: task.payload.repo_id, backend: 'vllm', host, port, cmd: task.payload._cmd, remoteHost: task.remoteHost || '', label: label || task.name, ..._presetEnvFields(task) }));
1260  _savePresets(presets.map(_redactPresetForStorage));
1261  return true;
1262}
1263
1264// Same model-matching as cookbookServe's _presetsForModel, so the auto-save cap
1265// counts the exact slots the Serve tab shows for this model.
1266function _presetsForModelLocal(presets, repo) {
1267  const short = (repo || '').split('/').pop();
1268  return presets.filter(p => {
1269    const pm = p.model || '', pn = p.name || '';
1270    return pm === repo || pn === repo || pm.split('/').pop() === short || pn === short;
1271  });
1272}
1273
1274// Build a short auto-label from the launched command so an auto-saved config is
1275// recognizable in the Saved dropdown (e.g. "TP2 · 16k ctx · AWQ").
1276function _autoConfigLabel(task) {
1277  const cmd = task.payload?._cmd || '';
1278  const bits = [];
1279  const tp = cmd.match(/--tensor-parallel-size[=\s]+(\d+)/);
1280  if (tp && tp[1] !== '1') bits.push('TP' + tp[1]);
1281  const ml = cmd.match(/--max-model-len[=\s]+(\d+)/);
1282  if (ml) { const n = parseInt(ml[1]); bits.push((n >= 1024 ? Math.round(n / 1024) + 'k' : n) + ' ctx'); }
1283  const q = (task.name || '').match(/AWQ|GPTQ|FP8|Q4|Q5|Q6|Q8|INT8|INT4/i);
1284  if (q) bits.push(q[0].toUpperCase());
1285  return bits.length ? bits.join(' · ') : 'working';
1286}
1287
1288// Auto-save a serve config the moment its endpoint registers successfully, and
1289// flag it confirmed-working. Dedups by exact command: if the same settings are
1290// already saved we just upgrade that slot's badge instead of duplicating it.
1291// Runs at most once per task.
1292function _autoSaveWorkingConfig(task) {
1293  if (!task || task.type !== 'serve' || !task.payload?._cmd) return;
1294  if (task._autoSaved) return;
1295  const cmd = task.payload._cmd;
1296  // Diffusion/image servers aren't vLLM presets — skip them.
1297  if (cmd.includes('diffusion_server') || cmd.includes('mlx_image_server')) { task._autoSaved = true; return; }
1298  const model = task.payload.repo_id || task.name;
1299  const presets = _loadPresets();
1300  const existing = presets.find(p => p.cmd === cmd);
1301  if (existing) {
1302    task._autoSaved = true;
1303    if (!existing.confirmedWorking) { existing.confirmedWorking = true; _savePresets(presets.map(_redactPresetForStorage)); }
1304    return;   // already saved → just confirm it, no duplicate, no toast
1305  }
1306  // Respect the per-model cap the manual save flow uses (max 5).
1307  if (_presetsForModelLocal(presets, model).length >= 5) { task._autoSaved = true; return; }
1308  const host = task.remoteHost || 'localhost';

1309  const portMatch = cmd.match(/--port[=\s]+(\d+)/);
1310  const port = portMatch ? portMatch[1] : '8000';
1311  presets.push(_redactPresetForStorage({
1312    name: task.name, model, backend: 'vllm', host, port,
1313    cmd, remoteHost: task.remoteHost || '',
1314    label: _autoConfigLabel(task), confirmedWorking: true, autoSaved: true,
1315    ..._presetEnvFields(task),
1316  }));
1317  _savePresets(presets.map(_redactPresetForStorage));
1318  task._autoSaved = true;
1319  uiModule.showToast('Saved working config');
1320}
1321
1322// ── Cross-device sync ──
1323
1324let _syncTimer = null;
1325function _syncToServer() {
1326  // Debounce to coalesce bursts of writes, but keep latency low so the server
1327  // is effectively authoritative across devices
1328  clearTimeout(_syncTimer);
1329  _syncTimer = setTimeout(async () => {
1330    try {
1331      // Don't push a not-yet-hydrated state. A legit state always has at
1332      // least the "Local" server, so an empty servers list means we loaded
1333      // before GET /state populated _envState — syncing it would wipe the
1334      // saved servers. (The server has an anti-wipe guard too; this avoids
1335      // the needless round-trip.)
1336      if (!_envState || !Array.isArray(_envState.servers) || _envState.servers.length === 0) return;
1337      const state = {
1338        tasks: _loadTasks(),
1339        removedTasks: _loadTombstones(),
1340        presets: _loadPresets(),
1341        env: _envState,
1342        serveState: null,
1343        serveFavorites: [],
1344      };
1345      try { state.serveState = JSON.parse(localStorage.getItem(SERVE_STATE_KEY)); } catch {}
1346      try {
1347        const favorites = JSON.parse(localStorage.getItem(SERVE_FAVORITES_KEY) || '[]');
1348        state.serveFavorites = Array.isArray(favorites) ? favorites.filter(Boolean).map(String) : [];
1349      } catch {}
1350      await fetch('/api/cookbook/state', {
1351        method: 'POST', credentials: 'same-origin',
1352        headers: { 'Content-Type': 'application/json' },
1353        body: JSON.stringify(_stripStateSecrets(state)),
1354      });
1355    } catch {}
1356  }, 400);
1357}
1358
1359document.addEventListener('cookbook:state-dirty', () => {
1360  _syncToServer();
1361});
1362
1363// Normalize state from server: collapse legacy duplicate keys to canonical form.
1364// - server.modelDir (singular) → server.modelDirs[0] (canonical)
1365// - strip ✕/✖ pollution from modelDirs
1366// - dedupe modelDirs
1367function _normalizeState(state) {
1368  if (!state || typeof state !== 'object') return state;
1369  if (state.env && Array.isArray(state.env.servers)) {
1370    for (const s of state.env.servers) {
1371      // Collapse legacy modelDir → modelDirs
1372      let dirs = Array.isArray(s.modelDirs) ? s.modelDirs : [];
1373      if (s.modelDir && !dirs.includes(s.modelDir)) dirs.push(s.modelDir);
1374      dirs = dirs
1375        .map(d => (d || '').replaceAll('\u2715', '').replaceAll('\u2716', '').trim())
1376        .filter(Boolean);
1377      if (!dirs.includes('~/.cache/huggingface/hub')) dirs.unshift('~/.cache/huggingface/hub');
1378      s.modelDirs = [...new Set(dirs)];
1379      delete s.modelDir; // Drop the legacy singular form
1380      // A download target that's no longer in the dir list falls back to the
1381      // default HF cache (empty) so we never download into an unscanned dir.
1382      if (s.downloadDir && !s.modelDirs.includes(s.downloadDir)) s.downloadDir = '';
1383    }
1384  }
1385  return state;
1386}
1387
1388export async function _syncFromServer() {
1389  try {
1390    const res = await fetch('/api/cookbook/state', { credentials: 'same-origin' });
1391    if (!res.ok) return false;
1392    const state = _normalizeState(await res.json());
1393    if (!state || !state.env) return false;
1394
1395    const localTasks = _loadTasks();
1396    const serverTasks = state.tasks || [];
1397    const serverTombstones = (state.removedTasks && typeof state.removedTasks === 'object') ? state.removedTasks : {};
1398    const localTombstones = _loadTombstones();
1399    const mergedTombstones = { ...serverTombstones, ...localTombstones };
1400    for (const [id, ts] of Object.entries(serverTombstones)) {
1401      if (localTombstones[id] == null || Number(ts) > Number(localTombstones[id])) mergedTombstones[id] = ts;
1402    }
1403    _saveTombstones(mergedTombstones);
1404
1405    const localIds = new Set(localTasks.map(t => t.sessionId));
1406    const merged = localTasks.filter(t => !_isTombstoned(t.sessionId));
1407    for (const t of serverTasks) {
1408      if (!localIds.has(t.sessionId) && !_isTombstoned(t.sessionId)) {
1409        merged.push(t);
1410      }
1411    }
1412    localStorage.setItem(TASKS_KEY, JSON.stringify(merged.map(_redactTaskForStorage)));
1413
1414    if (state.env) {
1415      // The active server selection (remoteHost + its env/path/platform) is a
1416      // per-device, live choice. NEVER let the server's stored copy overwrite
1417      // it here — doing so silently snapped the active host back to whatever was
1418      // saved server-side, so downloads/scans ignored what the user just
1419      // picked. Sync only the shared non-secret settings (servers list, gpus, paths).
1420      const { remoteHost: _rh, env: _e, envPath: _ep, platform: _pf, ...settings } = state.env;
1421      delete settings.hfToken;
1422      Object.assign(_envState, settings);
1423      const selected = (_envState.remoteServerKey && _serverByVal?.(_envState.remoteServerKey))
1424        || (_envState.remoteHost ? (_envState.servers || []).find(s => s.host === _envState.remoteHost) : null);
1425      if (selected) {
1426        _envState.env = selected.env || 'none';
1427        _envState.envPath = selected.envPath || '';
1428        _envState.platform = selected.platform || '';
1429      } else if (!_envState.remoteHost) {
1430        const local = (_envState.servers || []).find(s => !s.host || s.host === 'local');
1431        _envState.env = local?.env || 'none';
1432        _envState.envPath = local?.envPath || '';
1433        _envState.platform = local?.platform || '';
1434      }
1435      const { hfToken, ...safeState } = _envState;
1436      localStorage.setItem('cookbook-last-state', JSON.stringify(safeState));
1437    }
1438    if (state.presets) {
1439      localStorage.setItem(STORAGE_KEY, JSON.stringify(state.presets));
1440    }
1441    if (state.serveState) {
1442      localStorage.setItem(SERVE_STATE_KEY, JSON.stringify(state.serveState));
1443    }
1444    if (Array.isArray(state.serveFavorites)) {
1445      localStorage.setItem(SERVE_FAVORITES_KEY, JSON.stringify(state.serveFavorites.filter(

1445Boolean).map(String)));
1446    }
1447    document.dispatchEvent(new CustomEvent('cookbook:state-synced', { detail: state }));
1448    return true;
1449  } catch { return false; }
1450}
1451
1452// ── Retry download ──
1453
1454// Bounded auto-retry counter for downloads, keyed by model — network blips on
1455// big multi-file downloads are common and HF resumes from the .incomplete parts.
1456const _dlRetryCount = new Map();
1457const _DL_MAX_AUTO_RETRY = 2;
1458
1459// Kill + relaunch a task (download or serve). Shared by the ⋮ → Restart action
1460// and the click-to-retry on a stalled download badge.
1461async function _retryTask(el, task) {
1462  if (el && el._abort) el._abort.abort();
1463  const badge = el?.querySelector('.cookbook-task-status');
1464  if (badge) { badge.textContent = 'restarting...'; badge.className = 'cookbook-task-status'; }
1465  try {
1466    await fetch('/api/shell/exec', {
1467      method: 'POST', credentials: 'same-origin',
1468      headers: { 'Content-Type': 'application/json' },
1469      body: JSON.stringify({ command: _tmuxGracefulKill(task) }),
1470    });
1471  } catch {}
1472  if (task.payload) {
1473    if (task.type === 'serve' && task.payload._cmd) {
1474      _removeTask(task.sessionId);
1475      _launchServeTask(task.name, task.payload.repo_id, task.payload._cmd, task.payload._fields, task.remoteHost || '');
1476    } else {
1477      uiModule.showToast('Retrying download — progress may look reset while HuggingFace checks cached files, then it should resume.', 7000);
1478      _updateTask(task.sessionId, {
1479        status: 'running',
1480        output: `${task.output || ''}\n\n[odysseus] Retrying download. Progress may briefly look like a fresh download while HuggingFace checks cached/incomplete files; cached partial files will be reused when available.`.trim(),
1481        _retrying: true,
1482      });
1483      _retryDownload(task.name, task.payload, task.sessionId);
1484    }
1485  }
1486}
1487
1488async function _retryDownload(name, payload, replaceSessionId = '') {
1489  try {
1490    // A retry means the fast hf_transfer path already failed once — fall back to
1491    // the plain, reliable downloader for this and any further attempt (it resumes
1492    // from the cached .incomplete files, so no progress is lost).
1493    const _payload = { ...(payload || {}), disable_hf_transfer: true };
1494    const res = await fetch('/api/model/download', {
1495      method: 'POST', credentials: 'same-origin',
1496      headers: { 'Content-Type': 'application/json' },
1497      body: JSON.stringify(_payload),
1498    });
1499    if (!res.ok) {
1500      uiModule.showToast('Download failed: HTTP ' + res.status);
1501      if (replaceSessionId) _updateTask(replaceSessionId, { status: 'crashed', _retrying: false });
1502      return;
1503    }
1504    const data = await res.json();
1505    if (!data.ok) {
1506      uiModule.showToast('Download failed: ' + (data.error || ''));
1507      if (replaceSessionId) _updateTask(replaceSessionId, { status: 'crashed', _retrying: false });
1508      return;
1509    }
1510    if (replaceSessionId) {
1511      const tasks = _loadTasks();
1512      const task = tasks.find(t => t.sessionId === replaceSessionId);
1513      if (task) {
1514        task.name = _downloadNameFromPayload(name || task.name, _payload);
1515        task.id = data.session_id;
1516        task.sessionId = data.session_id;
1517        task.status = 'running';
1518        task.output = '';
1519        task.ts = Date.now();
1520        task.payload = _payload;
1521        task._retrying = false;
1522        _saveTasks(tasks);
1523        _soloExpandTaskId = data.session_id;
1524        _renderRunningTab();
1525        _startBackgroundMonitor();
1526      } else {
1527        _addTask(data.session_id, name, 'download', _payload);
1528      }
1529    } else {
1530      _addTask(data.session_id, name, 'download', _payload);
1531    }
1532    uiModule.showToast(`Downloading ${name}...`);
1533  } catch (e) {
1534    uiModule.showToast('Download failed: ' + e.message);
1535    if (replaceSessionId) _updateTask(replaceSessionId, { status: 'crashed', _retrying: false });
1536  }
1537}
1538
1539// ── Serve auto-fix (kill + relaunch with env var) ──
1540
1541// Block stacked retries: once any "Retry with X" is clicked for a task, ignore
1542// every further retry click for it. Each retry fires its own _launchServeTask,
1543// so clicking several options — or one repeatedly during the fade-out / while a
1544// relaunch was loading — used to stack up multiple servers (e.g. 6 launches).
1545// The flag rides on the card element (removed right after), so it can't re-arm.
1546function _guardServeRetry(panel, taskEl) {
1547  if (!taskEl || taskEl.dataset.retrying) return false;

1548  taskEl.dataset.retrying = '1';
1549  panel.querySelectorAll('button').forEach(b => {
1550    b.disabled = true;
1551    b.style.opacity = '0.5';
1552    b.style.pointerEvents = 'none';
1553  });
1554  return true;
1555}
1556
1557export async function _serveAutoFix(panel, envVar) {
1558  const taskEl = panel.closest('.cookbook-task');
1559  if (!taskEl) return;
1560  const taskId = taskEl.dataset.taskId;
1561  const tasks = _loadTasks();
1562  const task = tasks.find(t => t.sessionId === taskId);
1563  if (!task || !task.payload) return;
1564  if (!_guardServeRetry(panel, taskEl)) return;
1565
1566  const killCmd = _tmuxCmd(task, `kill-session -t ${taskId}`);
1567  try {
1568    await fetch('/api/shell/exec', {
1569      method: 'POST', credentials: 'same-origin',
1570      headers: { 'Content-Type': 'application/json' },
1571      body: JSON.stringify({ command: killCmd }),
1572    });
1573  } catch {}
1574
1575  _animateOutThenRemove(taskEl, taskId);
1576
1577  const origCmd = task.payload._cmd || '';
1578  const newCmd = `export ${envVar} && ${origCmd}`;
1579
1580  const origHost = _envState.remoteHost;
1581  if (task.remoteHost) _envState.remoteHost = task.remoteHost;
1582  try {
1583    uiModule.showToast(`Retrying with ${envVar}...`);
1584    await _launchServeTask(task.name, task.payload.repo_id, newCmd);
1585  } finally {
1586    // Always restore — otherwise a thrown launch leaves the global host stuck
1587    // on this serve task, so later downloads/scans hit it.
1588    _envState.remoteHost = origHost;
1589  }
1590}
1591
1592// Open the Serve panel pre-filled for a task — the same flow as the task's
1593// Edit button, but optionally with a modified command (used by the diagnosis
1594// "Retry with X" buttons so a retry lands in the editable Serve panel with the
1595// adjusted setting, instead of blindly relaunching).
1596async function _openServeEditForTask(task, cmdOverride, fieldOverrides = null) {
1597  const repo = task.payload?.repo_id;
1598  if (!repo) { uiModule.showToast('No model info on this task'); return; }
1599  const cmd = cmdOverride || task.payload?._cmd;
1600  // A modified cmd must be re-parsed; otherwise prefer the exact launch fields.
1601  let fields = cmdOverride
1602    ? _parseServeCmdToFields(cmd)
1603    : (task.payload?._fields || (cmd ? _parseServeCmdToFields(cmd) : null));
1604  if (fieldOverrides && typeof fieldOverrides === 'object') {
1605    fields = { ...(fields || {}), ...fieldOverrides };
1606  }
1607  fields = { ...(fields || {}), _replaceTaskId: task.sessionId };
1608  // Switch the active server to the exact profile this serve ran on. The
1609  // dropdown stores stable srv: keys, not raw host strings, so preserving only
1610  // task.remoteHost can relaunch against the local container by accident.
1611  _selectTaskServer(task);
1612  try {
1613    const { openServePanelForRepo } = await import('./cookbookServe.js');
1614    await openServePanelForRepo(repo, fields);
1615  } catch (err) {
1616    console.error('[cookbook] open serve panel failed', err);
1617    uiModule.showToast('Could not open serve panel');
1618  }
1619}
1620
1621export async function _serveAutoRetryReplace(panel, flag, value) {
1622  const taskEl = panel.closest('.cookbook-task');
1623  if (!taskEl) return;
1624  const taskId = taskEl.dataset.taskId;
1625  const tasks = _loadTasks();
1626  const task = tasks.find(t => t.sessionId === taskId);
1627  if (!task || !task.payload || !task.payload._cmd) return;
1628  if (!_guardServeRetry(panel, taskEl)) return;
1629
1630  try {
1631    await fetch('/api/shell/exec', {
1632      method: 'POST', credentials: 'same-origin',
1633      headers: { 'Content-Type': 'application/json' },
1634      body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${taskId}`) }),
1635    });
1636  } catch {}
1637
1638  _animateOutThenRemove(taskEl, taskId);
1639
1640  let newCmd = task.payload._cmd;
1641  if (flag === '--cuda-graph-backend-decode') {
1642    newCmd = newCmd.replace(/\s+--cuda-graph-max-bs-decode(?:\s+\S+|=\S+)/g, '');
1643  } else if (flag === '--cuda-graph-max-bs-decode') {
1644    newCmd = newCmd.replace(/\s+--cuda-graph-backend-decode(?:\s+\S+|=\S+)/g, '');
1645  }
1646  const re = new RegExp(flag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\s+\\S+');
1647  if (re.test(newCmd)) {
1648    newCmd = newCmd.replace(re, `${flag} ${value}`);
1649  } else {
1650    newCmd += ` ${flag} ${value}`;
1651  }
1652
1653  const origHost = _envState.remoteHost;
1654  if (task.remoteHost) _envState.remoteHost = task.remoteHost;
1655  try {

1656    uiModule.showToast(`Retrying with ${flag} ${value}...`);
1657    await _launchServeTask(task.name, task.payload.repo_id, newCmd);
1658  } finally {
1659    _envState.remoteHost = origHost;
1660  }
1661}
1662
1663export async function _serveAutoRetryRemove(panel, flag) {
1664  const taskEl = panel.closest('.cookbook-task');
1665  if (!taskEl) return;
1666  const taskId = taskEl.dataset.taskId;
1667  const tasks = _loadTasks();
1668  const task = tasks.find(t => t.sessionId === taskId);
1669  if (!task || !task.payload || !task.payload._cmd) return;
1670  if (!_guardServeRetry(panel, taskEl)) return;
1671
1672  try {
1673    await fetch('/api/shell/exec', {
1674      method: 'POST', credentials: 'same-origin',
1675      headers: { 'Content-Type': 'application/json' },
1676      body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${taskId}`) }),
1677    });
1678  } catch {}
1679
1680  _animateOutThenRemove(taskEl, taskId);
1681
1682  let newCmd = task.payload._cmd;
1683  const re = new RegExp(flag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\s+\\S+');
1684  newCmd = newCmd.replace(re, '').replace(/\s{2,}/g, ' ').trim();
1685
1686  const origHost = _envState.remoteHost;
1687  if (task.remoteHost) _envState.remoteHost = task.remoteHost;
1688  try {
1689    uiModule.showToast(`Retrying without ${flag}...`);
1690    await _launchServeTask(task.name, task.payload.repo_id, newCmd);
1691  } finally {
1692    _envState.remoteHost = origHost;
1693  }
1694}
1695
1696export async function _serveAutoRetry(panel, flag) {
1697  const taskEl = panel.closest('.cookbook-task');
1698  if (!taskEl) return;
1699  const taskId = taskEl.dataset.taskId;
1700  const tasks = _loadTasks();
1701  const task = tasks.find(t => t.sessionId === taskId);
1702  if (!task || !task.payload || !task.payload._cmd) return;
1703  if (!_guardServeRetry(panel, taskEl)) return;
1704
1705  try {
1706    await fetch('/api/shell/exec', {
1707      method: 'POST', credentials: 'same-origin',
1708      headers: { 'Content-Type': 'application/json' },
1709      body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${taskId}`) }),
1710    });
1711  } catch {}
1712
1713  _animateOutThenRemove(taskEl, taskId);
1714
1715  let newCmd = task.payload._cmd;
1716  if (!newCmd.includes(flag)) {
1717    newCmd += ' ' + flag;
1718  }
1719
1720  const origHost = _envState.remoteHost;
1721  if (task.remoteHost) _envState.remoteHost = task.remoteHost;
1722  try {
1723    uiModule.showToast(`Retrying with ${flag}...`);
1724    await _launchServeTask(task.name, task.payload.repo_id, newCmd);
1725  } finally {
1726    _envState.remoteHost = origHost;
1727  }
1728}
1729
1730// ── Edit-command prompt ──
1731// Shows a small modal with a textarea pre-filled with the current serve cmd.
1732// Resolves to the edited string on Save, or null on Cancel.
1733function _promptEditServeCmd(currentCmd) {
1734  return new Promise((resolve) => {
1735    const overlay = document.createElement('div');
1736    overlay.className = 'cookbook-edit-overlay';
1737    overlay.innerHTML = `
1738      <div class="cookbook-edit-modal">
1739        <div class="cookbook-edit-title">Edit serve command</div>
1740        <textarea class="cookbook-edit-textarea" spellcheck="false"></textarea>
1741        <div class="cookbook-edit-actions">
1742          <button class="cookbook-edit-cancel memory-toolbar-btn">Cancel</button>
1743          <button class="cookbook-edit-save memory-toolbar-btn">Save &amp; relaunch</button>
1744        </div>
1745      </div>`;
1746    const ta = overlay.querySelector('.cookbook-edit-textarea');
1747    ta.value = currentCmd || '';
1748    document.body.appendChild(overlay);
1749    setTimeout(() => { ta.focus(); ta.setSelectionRange(ta.value.length, ta.value.length); }, 0);
1750
1751    const close = (result) => {
1752      overlay.remove();
1753      document.removeEventListener('keydown', onKey);
1754      resolve(result);
1755    };
1756    const onKey = (e) => {
1757      if (e.key === 'Escape') close(null);
1758      else if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) close(ta.value.trim() || null);
1759    };
1760    overlay.querySelector('.cookbook-edit-cancel').addEventListener('click', () => close(null));
1761    overlay.querySelector('.cookbook-edit-save').addEventListener('click', () => close(ta.value.trim() || null));
1762    overlay.addEventListener('click', (e) => { if (e.target === overlay) close(null); });
1763    document.addEventListener('keydown', onKey);
1764  });
1765}
1766
1767// ── Launch serve task ──
1768
1769// Best-effort reconstruction of serve-form field values from a raw launch
1770// command. Fallback for tasks created before _fields capture existed.
1771// Mirrors the regex parser in cookbookServe.js's _loadSlotIntoPanel.
1772function _parseServeCmdToFields(cmd) {
1773  if (!cmd) return null;
1774  const ex = (re) => { const m = cmd.match(re); return m ? m[1] : ''; };
1775    const fields = {
1776      backend: cmd.includes('llama_cpp') || cmd.includes('llama-server') ? 'llamacpp'
1777      : cmd.includes('mlx_image_server') ? 'mlx_image'
1778      : cmd.includes('mlx_lm.server') ? 'mlx'
1779      : cmd.includes('diffusion_server') ? 'diffusers'
1780      : cmd.includes('sglang') ? 'sglang'
1781      : cmd.includes('ollama') ? 'ollama' : 'vllm',
1782    port: ex(/--port\s+(\d+)/) || '8000',
1783    tp: ex(/--tensor-parallel-size\s+(\d+)/) || '1',
1784    ctx: ex(/--max-model-len\s+(\d+)/) || ex(/--context-length\s+(\d+)/) || ex(/--max-tokens\s+(\d+)/) || ex(/--n_ctx\s+(\d+)/) || ex(/-c\s+(\d+)/) || '8192',
1785    gpu_mem: ex(/--gpu-memory-utilization\s+([\d.]+)/) || '0.90',
1786    swap: ex(/--swap-space\s+(\d+)/) || '',
1787    dtype: ex(/--dtype\s+(\w+)/) || 'auto',
1788    vllm_kv_cache_dtype: ex(/--kv-cache-dtype\s+([\w.-]+)/) || 'auto',
1789    max_seqs: ex(/--max-num-seqs\s+(\d+)/) || '',
1790    gpus: ex(/CUDA_VISIBLE_DEVICES=(\S+)/) || '',
1791    cache_type: ex(/(?:--cache-type-k|-ctk)\s+(\S+)/) || '',
1792    llama_fit: ex(/(?:--fit|-fit)\s+(on|off)/) || '',
1793    llama_split_mode: ex(/(?:--split-mode|-sm)\s+(none|layer|row|tensor)/) || '',
1794    llama_tensor_split: ex(/(?:--tensor-split|-ts)\s+([0-9.,]+)/) || '',
1795    llama_main_gpu: ex(/(?:--main-gpu|-mg)\s+(\d+)/) || '',
1796    llama_parallel: ex(/(?:--parallel|-np)\s+(\d+)/) || '',
1797    llama_batch_size: ex(/(?:--batch-size|-b)\s+(\d+)/) || '',
1798    llama_ubatch_size: ex(/(?:--ubatch-size|-ub)\s+(\d+)/) || '',
1799    llama_spec_tokens: ex(/--spec-draft-n-max\s+(\d+)/) || '3',
1800    enforce_eager: cmd.includes('--enforce-eager'),
1801    trust_remote: cmd.includes('--trust-remote-code'),
1802    prefix_cache: cmd.includes('--enable-prefix-caching'),
1803    auto_tool: cmd.includes('--enable-auto-tool-choice'),
1804    flash_attn: /--flash-attn\s+on\b/.test(cmd),
1805    unified_mem: /GGML_CUDA_ENABLE_UNIFIED_MEMORY=1/.test(cmd),
1806    llama_no_mmap: /--no-mmap\b/.test(cmd),
1807    llama_no_warmup: /--no-warmup\b/.test(cmd),
1808    llama_speculative_mtp: /--spec-type\s+\S*draft-mtp/.test(cmd),
1809    speculative: cmd.includes('--speculative-config'),
1810  };
1811  const spec = cmd.match(/--speculative-config\s+'?\{[^}]*"method"\s*:\s*"([^"]+)"[^}]*"num_speculative_tokens"\s*:\s*(\d+)/);
1812  if (spec) { fields.spec_method = spec[1]; fields.spec_tokens = spec[2]; }
1813  return fields;
1814}
1815
1816function _serveCmdNeedsGpuPreflight(cmd, repo) {
1817  const c = String(cmd || '').toLowerCase();
1818  const r = String(repo || '').toLowerCase();
1819  if (!c || /gpu-cleanup|sglang-kernel|mlx-lm|pip\s+install|python\d*\s+-m\s+pip/.test(`${r} ${c}`)) return false;

1820  return /\b(vllm\s+serve|sglang(?:\.launch_server|\s+serve)|mlx_lm\.server|mlx_image_server\.py|diffusion_server\.py|llama-server|llama_cpp\.server|text-generation-launcher|aphrodite|ollama\s+(?:serve|run))\b/.test(c);
1821}
1822
1823function _selectedGpuIndexes(gpus) {
1824  const raw = String(gpus || '').trim();
1825  if (!raw) return null;
1826  const out = new Set();
1827  raw.split(',').forEach(part => {
1828    const p = part.trim();
1829    const range = p.match(/^(\d+)\s*-\s*(\d+)$/);
1830    if (range) {
1831      const a = parseInt(range[1], 10);
1832      const b = parseInt(range[2], 10);
1833      for (let i = Math.min(a, b); i <= Math.max(a, b); i++) out.add(i);
1834      return;
1835    }
1836    const n = parseInt(p, 10);
1837    if (Number.isFinite(n)) out.add(n);
1838  });
1839  return out.size ? out : null;
1840}
1841
1842function _gbFromMb(mb) {
1843  const n = Number(mb || 0);
1844  if (!Number.isFinite(n) || n <= 0) return '';
1845  return n >= 1024 ? `${(n / 1024).toFixed(n >= 10240 ? 0 : 1)}G` : `${Math.round(n)}M`;
1846}
1847
1848function _gpuPreflightIssues(data, selected) {
1849  const backend = String(data?.backend || data?.source || '').toLowerCase();
1850  const isCuda = backend.includes('cuda') || String(data?.source || '').toLowerCase().includes('nvidia');
1851  const rows = Array.isArray(data?.gpus) ? data.gpus : [];
1852  const issues = [];
1853  rows.forEach(g => {
1854    const idx = Number(g?.index);
1855    if (selected && !selected.has(idx)) return;
1856    const procs = Array.isArray(g?.processes) ? g.processes : [];
1857    if (procs.length) {
1858      procs.slice(0, 3).forEach(p => {
1859        const name = String(p?.name || 'process').split(/[\\/]/).pop();
1860        const used = _gbFromMb(p?.used_mb);
1861        issues.push(`GPU ${idx}: ${name}${p?.pid ? ` #${p.pid}` : ''}${used ? ` (${used})` : ''}`);
1862      });
1863      if (procs.length > 3) issues.push(`GPU ${idx}: +${procs.length - 3} more process${procs.length - 3 === 1 ? '' : 'es'}`);
1864      return;
1865    }
1866    const total = Number(g?.total_mb || 0);
1867    const free = Number(g?.free_mb || 0);
1868    const used = Number(g?.used_mb || 0);
1869    const freeRatio = total > 0 ? free / total : 1;
1870    // CUDA can have display/runtime crumbs; warn only for meaningful occupied memory.
1871    if (isCuda && used > 4096 && freeRatio < 0.9) {
1872      issues.push(`GPU ${idx}: ${_gbFromMb(used)} already used (${_gbFromMb(free)} free)`);
1873    } else if (!isCuda && total > 0 && freeRatio < 0.2) {
1874      issues.push(`${g?.name || `GPU ${idx}`}: low free memory (${_gbFromMb(free)} free of ${_gbFromMb(total)})`);
1875    } else if (!isCuda && g?.busy && total <= 0) {
1876      issues.push(`${g?.name || `GPU ${idx}`}: GPU device is busy`);
1877    }
1878  });
1879  return issues;
1880}
1881
1882async function _confirmGpuPreflight(reqBody, shortName, repo, cmd) {
1883  if (!_serveCmdNeedsGpuPreflight(cmd, repo)) return true;
1884  const params = new URLSearchParams();
1885  if (reqBody.remote_host) params.set('host', reqBody.remote_host);
1886  if (reqBody.ssh_port) params.set('ssh_port', reqBody.ssh_port);
1887  try {
1888    const res = await fetch(`/api/cookbook/gpus${params.toString() ? `?${params.toString()}` : ''}`, {
1889      method: 'GET',
1890      credentials: 'same-origin',
1891    });
1892    const data = await res.json().catch(() => null);
1893    if (!res.ok || !data?.ok) return true;
1894    const selected = _selectedGpuIndexes(reqBody.gpus);
1895    const issues = _gpuPreflightIssues(data, selected);
1896    if (!issues.length) return true;
1897    const where = reqBody.remote_host || 'local';
1898    const list = issues.slice(0, 6).join('; ');
1899    const more = issues.length > 6 ? `; +${issues.length - 6} more` : '';
1900    const msg = `GPU preflight found existing load on ${where}: ${list}${more}. Launch ${shortName || 'model'} anyway?`;

1901    const confirm = window.styledConfirm || uiModule?.styledConfirm;
1902    if (confirm) return await confirm(msg, { confirmText: 'Launch anyway', cancelText: 'Cancel' });
1903    return window.confirm ? window.confirm(msg) : true;
1904  } catch (e) {
1905    console.warn('[cookbook] GPU preflight failed; allowing launch', e);
1906    return true;
1907  }
1908}
1909
1910export async function _launchServeTask(shortName, repo, cmd, fields, hostOverride, targetMeta = null) {
1911  // Host resolution mirrors the download path: when the caller passes an explicit
1912  // host (resolved from the dropdown the user actually picked), use it and look
1913  // up that server's port/platform from the shared servers list. Only fall back
1914  // to _envState.remoteHost for legacy callers (diagnosis/pip-update).
1915  const _host = (hostOverride !== undefined) ? (hostOverride || '') : (_envState.remoteHost || '');
1916  const _targetKey = targetMeta?.serverKey || '';
1917  const _hsrv = (_targetKey && _targetKey !== 'local' ? _serverByVal(_targetKey) : null)
1918    || (hostOverride === undefined ? _serverByVal(_envState.remoteServerKey || _host) : null)
1919    || _envState.servers.find(s => s.host === _host) || {};
1920  const _serverMetaKey = _targetKey || (_hsrv && _serverKey ? _serverKey(_hsrv) : '') || (_host || 'local');
1921  const _serverMetaName = targetMeta?.serverName || _hsrv.name || (_host ? _host : 'Local');
1922  const _hplatform = _host ? (_hsrv.platform || '') : (_envState.hostPlatform || '');
1923  const _replaceTaskId = fields?._replaceTaskId || '';
1924  const _launchAnyway = !!targetMeta?.launchAnyway;
1925  if (_replaceTaskId) {
1926    try {
1927      const _old = _loadTasks().find(t => t.sessionId === _replaceTaskId);
1928      if (_old && _old.type === 'serve') {
1929        await fetch('/api/shell/exec', {
1930          method: 'POST', credentials: 'same-origin',
1931          headers: { 'Content-Type': 'application/json' },
1932          body: JSON.stringify({ command: _tmuxGracefulKill(_old) }),
1933        });
1934        _removeTask(_old.sessionId);
1935      }
1936    } catch {}
1937  }
1938  // Replace any serve already targeting this same host:port — you can't run two
1939  // servers on one port, so re-serving (or retrying) should stop & remove the
1940  // old one instead of leaving a dead duplicate behind. (The retry buttons
1941  // already removed their own task, so this is a no-op for them.)
1942  if (!_launchAnyway) try {

1943    const _pm = cmd.match(/--port[=\s]+(\d+)/) || cmd.match(/(?:^|\s)-p[=\s]+(\d+)/);
1944    const _newPort = _pm ? _pm[1] : '';
1945    if (_newPort) {
1946      for (const _t of _loadTasks()) {
1947        if (_t.type !== 'serve' || !_t.payload || !_t.payload._cmd) continue;
1948        const _tm = _t.payload._cmd.match(/--port[=\s]+(\d+)/) || _t.payload._cmd.match(/(?:^|\s)-p[=\s]+(\d+)/);
1949        if ((_tm ? _tm[1] : '') === _newPort && (_t.remoteHost || '') === _host) {
1950          try {
1951            await fetch('/api/shell/exec', {
1952              method: 'POST', credentials: 'same-origin',
1953              headers: { 'Content-Type': 'application/json' },
1954              body: JSON.stringify({ command: _tmuxGracefulKill(_t) }),
1955            });
1956          } catch {}
1957          _removeTask(_t.sessionId);
1958        }
1959      }
1960    }
1961  } catch {}
1962  // Capture the env + GPU pin used for THIS launch BEFORE building the request.
1963  // The serve panel sets _envState.env/envPath/gpus, calls us, then restores them
1964  // synchronously — and our payload is built after an `await`, so reading
1965  // _envState there would see the restored (wrong) values. Persisting these lets
1966  // a saved preset relaunch with the same venv + GPUs (otherwise a confirmed
1967  // working config fails: no venv activation, no GPU pinning).
1968  const _usedEnv = _envState.env;
1969  const _usedEnvPath = _envState.envPath;
1970  const _usedGpus = _envState.gpus || '';
1971  let envPrefix = '';
1972  if (_isWindows()) {
1973    if (_envState.env === 'venv' && _envState.envPath) {
1974      envPrefix = '& ' + (_envState.envPath.endsWith('\\Scripts\\Activate.ps1') ? _envState.envPath : _envState.envPath + '\\Scripts\\Activate.ps1');
1975    } else if (_envState.env === 'conda' && _envState.envPath) {
1976      envPrefix = 'conda activate ' + _envState.envPath;
1977    }
1978  } else {
1979    if (_envState.env === 'venv' && _envState.envPath) {
1980      const p = _venvRootFromPath(_envState.envPath);
1981      envPrefix = 'source ' + (p.endsWith('/bin/activate') ? p : p + '/bin/activate');
1982    } else if (_envState.env === 'conda' && _envState.envPath) {
1983      envPrefix = 'eval "$(conda shell.bash hook)" && conda activate ' + _envState.envPath;
1984    }
1985  }
1986
1987  const reqBody = {
1988    repo_id: repo,
1989    cmd: cmd,
1990    remote_host: _host || undefined,
1991    ssh_port: _getPort(_serverMetaKey || _host) || undefined,
1992    env_prefix: envPrefix || undefined,
1993    hf_token: _envState.hfToken || undefined,
1994    gpus: _usedGpus || undefined,
1995    platform: _hplatform || undefined,
1996  };
1997
1998  try {
1999    const _preflightOk = await _confirmGpuPreflight(reqBody, shortName, repo, cmd);
2000    if (!_preflightOk) {
2001      uiModule.showToast('Launch cancelled — GPU is already in use');
2002      return;
2003    }
2004    const res = await fetch('/api/model/serve', {
2005      method: 'POST', credentials: 'same-origin',
2006      headers: { 'Content-Type': 'application/json' },
2007      body: JSON.stringify(reqBody),
2008    });
2009    const data = await res.json();
2010    if (!data.ok) {
2011      // Two error shapes: `{ok:false, error}` (tmux launch failed) or
2012      // `{detail}` (FastAPI HTTPException). Show whichever is present
2013      // + log full payload so the user can copy the error.
2014      const err = data.error || data.detail || res.statusText || 'unknown';
2015      console.error('[cookbook] /api/model/serve failed', { status: res.status, body: data });
2016      uiModule.showToast('Failed to start: ' + String(err).slice(0, 200), 9000);
2017      return;
2018    }
2019
2020    const _sp = _getPort(_serverMetaKey || _host);
2021    // _fields = the exact structured serve-form values used for this launch,
2022    // so the "Edit / relaunch" button can re-open the Serve panel pre-filled
2023    // with these precise settings (not just the last-used-for-repo state).
2024    const payload = { repo_id: repo, remote_host: _host || undefined, remote_server_key: _serverMetaKey || undefined, remote_server_name: _serverMetaName || undefined, ssh_port: _sp || undefined, _cmd: cmd, _fields: fields || undefined, _env: _usedEnv, _envPath: _usedEnvPath, _gpus: _usedGpus };
2025    _addTask(data.session_id, shortName, 'serve', payload);

2026    uiModule.showToast(`Serving ${shortName}...`);
2027    // Auto-register may have enabled an existing (offline) endpoint for this
2028    // host:port. Refresh the picker so the row is no longer dimmed, and the
2029    // user doesn't see "offline" on a serve they just started.
2030    try { _refreshModelsAfterEndpointChange(); } catch (_) {}
2031  } catch (e) {
2032    uiModule.showToast('Failed: ' + e.message);
2033  }
2034}
2035
2036// ── Render Running tab ──
2037
2038export function _renderRunningTab() {
2039  // Auto-clear the sidebar notif (the bright-icon highlight) when no tasks
2040  // are actively running or errored. _showCookbookNotif fires on each task
2041  // event but the matching clear only ran on modal-open, so the highlight
2042  // persisted indefinitely after tasks finished in the background.
2043  try {
2044    const _activeTasks = _loadPrunedTasks().filter(t => t.status === 'running' || t.status === 'queued' || t.status === 'error');
2045    if (!_activeTasks.length) _clearCookbookNotif();
2046  } catch {}
2047
2048  const body = document.querySelector('#cookbook-modal .cookbook-body');
2049  if (!body) return;
2050
2051  // Capture expansion state so re-renders don't collapse whatever the user
2052  // had open. Task output: presence of .cookbook-task-collapsed means collapsed.
2053  // Section body: inline display:none means collapsed.
2054  const _collapsedTaskIds = new Set();
2055  const _expandedTaskIds = new Set();  // mobile: tasks the user explicitly opened
2056  body.querySelectorAll('.cookbook-task').forEach(tEl => {
2057    const id = tEl.dataset.taskId;
2058    if (!id) return;
2059    const wrap = tEl.querySelector('.cookbook-output-wrap');
2060    if (!wrap) return;
2061    if (wrap.classList.contains('cookbook-task-collapsed')) _collapsedTaskIds.add(id);
2062    else _expandedTaskIds.add(id);
2063  });
2064  // A new action was just started — collapse every existing card and open only
2065  // the new one (works on both desktop and the mobile collapse-by-default path).
2066  if (_soloExpandTaskId) {
2067    const _allIds = new Set([..._collapsedTaskIds, ..._expandedTaskIds]);
2068    _collapsedTaskIds.clear();
2069    _expandedTaskIds.clear();
2070    _allIds.forEach(id => { if (id !== _soloExpandTaskId) _collapsedTaskIds.add(id); });
2071    _expandedTaskIds.add(_soloExpandTaskId);
2072    _soloExpandTaskId = null;
2073  }
2074  // On mobile, task outputs start COLLAPSED — having every running window
2075  // expanded on entry meant a lot of tapping to collapse them. User-expanded
2076  // ones are re-opened from _expandedTaskIds below.
2077  const _mobileCollapseDefault = window.innerWidth <= 768;
2078  const _collapsedSectionIds = new Set();
2079  body.querySelectorAll('.cookbook-section-body').forEach(sb => {
2080    if (sb.style.display === 'none' && sb.id) _collapsedSectionIds.add(sb.id);
2081  });
2082
2083  const tasks = _loadTasks();
2084  const hasContent = tasks.length > 0;
2085  // Count anything that's really active: explicit 'running'/'queued' status,
2086  // OR a download whose tmux output is still showing live shard progress.
2087  // Without the output check, a task whose status got stuck at 'done' /
2088  // 'crashed' (before auto-reconnect catches it) would read as "Running 0"
2089  // even when the model is actively downloading on the host.
2090  const activeCount = tasks.filter(t =>
2091    t.status === 'running'
2092    || t.status === 'queued'
2093    || _downloadOutputLooksActive(t)
2094  ).length;
2095  const activeCountHtml = activeCount ? ` <span class="cookbook-tab-count">${activeCount}</span>` : '';
2096
2097  let tabBar = body.querySelector('.cookbook-tabs');
2098  if (!tabBar) return;
2099  let runTab = tabBar.querySelector('.cookbook-tab[data-backend="Running"]');
2100  if (hasContent && !runTab) {
2101    runTab = document.createElement('button');
2102    runTab.className = 'cookbook-tab';
2103    runTab.dataset.backend = 'Running';
2104    const _errCount = tasks.filter(t => t.status === 'error' || t.status === 'crashed').length;
2105    runTab.innerHTML = `Active${activeCountHtml}${_errCount ? `<span class="cookbook-tab-error-dot"></span>` : ''}`;
2106    tabBar.insertBefore(runTab, tabBar.firstChild);
2107    runTab.addEventListener('click', () => {
2108      tabBar.querySelectorAll('.cookbook-tab').forEach(t => t.classList.remove('active'));
2109      runTab.classList.add('active');
2110      body.querySelectorAll('.cookbook-group').forEach(g => {
2111        g.classList.toggle('hidden', g.dataset.backendGroup !== 'Running');
2112      });
2113      setTimeout(() => _renderRunningTab(), 0);
2114    });
2115  } else if (runTab) {
2116    const _errCount2 = tasks.filter(t => t.status === 'error' || t.status === 'crashed').length;
2117    runTab.innerHTML = tasks.length ? `Active${activeCountHtml}${_errCount2 ? '<span class="cookbook-tab-error-dot"></span>' : ''}` : 'Active';
2118    if (!hasContent) {
2119      if (runTab.classList.contains('active')) {
2120        const wfTab = tabBar.querySelector('.cookbook-tab[data-backend="Search"]');
2121        if (wfTab) wfTab.click();
2122      }
2123      runTab.remove();
2124    }
2125  }
2126
2127  let group = body.querySelector('.cookbook-group[data-backend-group="Running"]');
2128  if (hasContent && !group) {
2129    group = document.createElement('div');
2130    group.className = 'cookbook-group hidden';
2131    group.dataset.backendGroup = 'Running';
2132    // No `flex:1` on the card — with overflow:visible (forced via #cookbook-modal
2133    // .cookbook-group > .admin-card), flex:1 collapsed the card to body height
2134    // and the body's scrollHeight stopped tracking the overflowing children.
2135    // Sized-to-content means cookbook-body's overflow-y:auto kicks in naturally.
2136    group.innerHTML = '<div class="admin-card" style="display:flex;flex-direction:column;">' +
2137      '<div style="display:flex;align-items:baseline;gap:8px;margin-bottom:2px;">' +
2138      '<h2 style="margin:0;padding:0;line-height:1;">Active <span id="running-count" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal">

2138' + activeCount + '</span></h2>' +
2139      '</div>' +
2140      '<p class="memory-desc doclib-desc" style="margin-top:6px;">Active downloads, installs and model launches.</p>' +
2141      '</div>';
2142    const firstGroup = body.querySelector('.cookbook-group');
2143    if (firstGroup) body.insertBefore(group, firstGroup);
2144    else body.appendChild(group);
2145  }
2146
2147  if (!group) return;
2148
2149  const countEl = group.querySelector('#running-count');
2150  if (countEl) countEl.textContent = activeCount;
2151
2152  if (!hasContent) {
2153    group.remove();
2154    return;
2155  }
2156
2157  const _adminCard = group.querySelector('.admin-card');
2158  function _ensureSection(cls, label, items) {
2159    let sec = group.querySelector('.' + cls);
2160    if (!sec) {
2161      sec = document.createElement('div');
2162      sec.className = cls;
2163      (_adminCard || group).appendChild(sec);
2164    }
2165    if (!items || !items.length) {
2166      sec.style.display = 'none';
2167      return sec;
2168    }
2169    sec.style.display = '';
2170    return sec;
2171  }
2172
2173  // Group tasks by server
2174  const _taskServerKey = (task) => task?.remoteServerKey || task?.remoteHost || '';
2175  const _serverName = (keyOrTask) => {
2176    if (keyOrTask && typeof keyOrTask === 'object') {
2177      const task = keyOrTask;
2178      if (task.remoteServerName) return task.remoteServerName;
2179      const srv = task.remoteServerKey ? _serverByVal(task.remoteServerKey) : null;
2180      if (srv?.name) return srv.name;
2181      if (!task.remoteHost) return 'Local';
2182      return (_envState.servers.find(s => s.host === task.remoteHost)?.name) || task.remoteHost;
2183    }
2184    const key = keyOrTask || '';
2185    if (!key || key === 'local') return 'Local';
2186    const srv = _serverByVal(key);
2187    return srv?.name || key;
2188  };
2189  const serverGroups = {};
2190  for (const t of tasks) {
2191    const key = _taskServerKey(t);
2192    if (!serverGroups[key]) serverGroups[key] = { name: _serverName(t), serve: [], download: [] };
2193    serverGroups[key][t.type === 'serve' ? 'serve' : 'download'].push(t);
2194  }
2195
2196
2197  // ── Server-grouped sections ──
2198  group.querySelectorAll('.cookbook-serve-section, .cookbook-dl-section').forEach(el => el.remove());
2199
2200  const serverKeys = Object.keys(serverGroups).sort((a, b) => {
2201    if (!a) return -1; if (!b) return 1;
2202    return serverGroups[a].name.localeCompare(serverGroups[b].name);
2203  });
2204
2205  // Prune stale server sections: a server that no longer has ANY tasks isn't in
2206  // serverKeys, so its section header/dropdown would otherwise linger until the
2207  // user manually cleared it. Drop those automatically on each render.
2208  const _liveSafeKeys = new Set(serverKeys.map(k => (k || 'local').replace(/[^a-zA-Z0-9-]/g, '_')));
2209  (_adminCard || group).querySelectorAll('[class*="cookbook-server-section-"]').forEach(el => {
2210    const cls = [...el.classList].find(c => c.startsWith('cookbook-server-section-'));
2211    if (cls && !_liveSafeKeys.has(cls.replace('cookbook-server-section-', ''))) el.remove();
2212  });
2213
2214  for (const key of serverKeys) {
2215    const sg = serverGroups[key];
2216    const allTasks = [...sg.serve, ...sg.download];
2217    const safeKey = (key || 'local').replace(/[^a-zA-Z0-9-]/g, '_');
2218    const sectionCls = `cookbook-server-section-${safeKey}`;
2219    const bodyId = `server-body-${safeKey}`;
2220    let sec = _ensureSection(sectionCls, sg.name, allTasks);
2221    if (allTasks.length && !sec.querySelector('.cookbook-section-header')) {
2222      const clearId = `clear-server-${key || 'local'}`;
2223      // Glowy status dot next to the server name (like the Settings server card):
2224      // green when reachable, red if any serve task on it is crashed/unreachable.
2225      const _secDot = (key && allTasks.some(_serveTaskFailed)) ? 'fail' : 'ok';
2226      const _dotTitle = key ? (_secDot === 'fail' ? 'Server not responding' : 'Reachable') : 'Local (this machine)';
2227      const _srvColor = _serverColorForTaskGroup(key || 'local', allTasks);
2228      sec.insertAdjacentHTML('afterbegin', `<div class="cookbook-section-header${_srvColor ? ' has-server-color' : ''}" data-collapse="${bodyId}"${_serverHeaderStyle(_srvColor)}><svg class="cookbook-section-chevron" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round"><polyline points="6 9 12 15 18 9"/></svg><span class="cookbook-srv-status ${_secDot}" title="${_dotTitle}" style="flex-shrink:0;position:relative;top:0px;"></span><span class="cookbook-section-title" style="margin:0;">${esc(sg.name)}</span><button class="cookbook-btn cookbook-stop-all-btn" data-stop-server="${esc(key)}" title="Stop all running servers">

2228<svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" stroke="none" aria-hidden="true" style="vertical-align:-1px;margin-right:4px;"><rect x="5" y="5" width="14" height="14" rx="1.5"/></svg>Stop all</button><button class="cookbook-btn cookbook-clear-btn" data-clear-server="${esc(key)}" title="Clear finished tasks"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="vertical-align:-1px;margin-right:4px;"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/></svg>Clear finished</button></div><div id="${bodyId}" class="cookbook-section-body"></div>`);
2229    }
2230  }
2231
2232  // Wire clear all buttons
2233  group.querySelectorAll('[data-clear-server]').forEach(btn => {
2234    if (btn._bound) return;
2235    btn._bound = true;
2236    btn.addEventListener('click', async (e) => {
2237      e.stopPropagation();  // don't toggle the section collapse (was an inline onclick, blocked by CSP)
2238      const host = btn.dataset.clearServer;
2239      const allTasks = _loadTasks();
2240      const toRemove = allTasks.filter(t => _taskServerKey(t) === host && _canClearTask(t));
2241      // Bail with a clear message instead of silently doing nothing when
2242      // every task on this server is still running (nothing finished to
2243      // clear yet) — the previous behavior looked like the button was dead.
2244      if (!toRemove.length) {
2245        const stillRunning = allTasks.filter(t => _taskServerKey(t) === host && t.status === 'running').length;
2246        const _msg = stillRunning
2247          ? `No finished tasks on ${_serverName(host)} — ${stillRunning} still running. Stop them first to clear.`
2248          : `No finished tasks on ${_serverName(host)}.`;
2249        if (window.uiModule?.showToast) window.uiModule.showToast(_msg);
2250        else alert(_msg);
2251        return;
2252      }
2253      if (!await window.styledConfirm(`Clear ${toRemove.length} finished task${toRemove.length === 1 ? '' : 's'} on ${_serverName(host)}?`, { confirmText: 'Clear' })) return;
2254      toRemove.forEach(t => _tombstoneTask(t.sessionId));
2255      const remaining = allTasks.filter(t => _taskServerKey(t) !== host || !_canClearTask(t));
2256      _saveTasks(remaining);
2257      // Fade/slide each finished card out (same exit as the per-card clear)
2258      // instead of yanking them instantly.
2259      toRemove.forEach(t => {
2260        const el = document.querySelector(`.cookbook-task[data-task-id="${t.sessionId}"]`);
2261        if (el) {
2262          if (el._abort) el._abort.abort();
2263          if (el._uptimeInterval) clearInterval(el._uptimeInterval);
2264          el.style.transition = 'opacity 0.35s ease, transform 0.35s ease';
2265          el.style.opacity = '0';
2266          el.style.transform = 'translateX(-10px)';
2267        }
2268      });
2269      // After the animation, remove the cards and tidy up the now-empty section.
2270      setTimeout(() => {
2271        toRemove.forEach(t => document.querySelector(`.cookbook-task[data-task-id="${t.sessionId}"]`)?.remove());
2272        // If this server's section is now empty (only finished tasks lived here),
2273        // remove the whole section so its header/title doesn't linger.
2274        const _sk = (host || 'local').replace(/[^a-zA-Z0-9-]/g, '_');
2275        const _sec = group.querySelector(`.cookbook-server-section-${_sk}`);
2276        if (_sec && !_sec.querySelector('.cookbook-task')) _sec.remove();
2277        if (!remaining.length) _renderRunningTab();
2278      }, 360);
2279    });
2280  });
2281
2282  // Wire "Stop all" buttons — stop every running task on that server.
2283  group.querySelectorAll('[data-stop-server]').forEach(btn => {
2284    if (btn._bound) return;
2285    btn._bound = true;
2286    btn.addEventListener('click', async (e) => {
2287      e.stopPropagation();  // don't toggle the section collapse
2288      const host = btn.dataset.stopServer;
2289      const running = _loadTasks().filter(t => _taskServerKey(t) === host && t.status === 'running');
2290      if (!running.length) { uiModule.showToast(`Nothing running on ${_serverName(host)}`); return; }
2291      if (!await window.styledConfirm(`Stop ${running.length} running task${running.length > 1 ? 's' : ''} on ${_serverName(host)}?`, { confirmText: 'Stop all' })) return;
2292      // Mark every task as user-stopped BEFORE firing the kills so that the
2293      // download auto-retry logic never restarts a task the user just stopped.
2294      running.forEach(t => _updateTask(t.sessionId, { _userStopped: true }));
2295      // Reuse each task's own Stop action so it does the full teardown
2296      // (send C-c, drop the endpoint, mark stopped) consistently.
2297      running.forEach(t => {
2298        const el = document.querySelector(`.cookbook-task[data-task-id="${t.sessionId}"]`);
2299        el?.querySelector('.cookbook-task-action-stop')?.click();
2300      });
2301      uiModule.showToast(`Stopped ${running.length} task${running.length > 1 ? 's' : ''} on ${_serverName(host)}`);
2302    });
2303  });
2304
2305  // Section collapse/expand
2306  group.querySelectorAll('.cookbook-section-header[data-collapse]').forEach(hdr => {
2307    if (hdr._bound) return;
2308    hdr._bound = true;
2309    hdr.addEventListener('click', () => {
2310      const bodyId = hdr.dataset.collapse;
2311      const body = document.getElementById(bodyId);
2312      if (!body) return;
2313      const isHidden = body.style.display === 'none';
2314      body.style.display = isHidden ? '' : 'none';
2315      const chevron = hdr.querySelector('.cookbook-section-chevron');
2316      if (chevron) {
2317        // Collapsed → point right (▶, click to expand); expanded → down (▼).
2318        chevron.style.transform = isHidden ? '' : 'rotate(-90deg)';
2319        chevron.style.opacity = '';
2320      }
2321    });
2322  });
2323
2324  // Only add new tasks or update existing ones
2325  const existingIds = new Set();
2326  group.querySelectorAll('.cookbook-task').forEach(el => {
2327    const id = el.dataset.taskId;
2328    existingIds.add(id);
2329    const task = tasks.find(t => t.sessionId === id);
2330    if (task) {
2331      el.dataset.status = task.status;
2332      const isDone = task.status === 'done';
2333      // Type chip doubles as the "finished" badge once a task completes — both
2334      // download and serve show the same green FINISHED chip.
2335      const typeChip = el.querySelector('.cookbook-task-type');
2336      if (typeChip) {
2337        // Only DOWNLOAD tasks flip to "finished" when done — serve tasks keep
2338        // saying "serve" because the model is still running on that port.
2339        const isDoneDl = isDone && task.type === 'download';
2340        typeChip.textContent = isDoneDl ? 'finished' : task.type;
2341        typeChip.classList.toggle('cookbook-task-type-done', isDoneDl);
2342      }
2343      const badge = el.querySelector('.cookbook-task-status');
2344      if (badge) {
2345        const _bdg = _taskBadge(task);
2346        badge.textContent = _bdg.text;
2347        badge.className = 'cookbook-task-status' + (_bdg.cls ? ' ' + _bdg.cls : '');
2348        badge.style.display = '';
2349      }
2350      // Indicator: spinning wave while running, green check when finished.
2351      const wave = el.querySelector('.cookbook-task-wave');
2352      if (wave) wave.style.display = task.status === 'running' ? '' : 'none';
2353      const check = el.querySelector('.cookbook-task-check');
2354      if (check) {
2355        check.style.display = _canClearTask(task) ? '' : 'none';
2356        const label = check.querySelector('.cookbook-task-done-label');
2357        if (label) label.textContent = _clearPillLabel(task);
2358      }
2359      const startNow = el.querySelector('.cookbook-task-start-now');
2360      if (startNow) startNow.style.display = (task.type === 'download' && task.status === 'queued') ? '' : 'none';
2361      const pre = el.querySelector('.cookbook-output-pre');
2362      if (pre && typeof task.output === 'string' && task.output && pre.textContent !== task.output) {
2363        const atBottom = (pre.scrollHeight - pre.scrollTop - pre.clientHeight) < 40;
2364        pre.textContent = task.output;
2365        if (atBottom) pre.scrollTop = pre.scrollHeight;
2366      }
2367      const terminalDiag = _terminalServeDiagnosis(task, el.querySelector('.cookbook-output-pre')?.textContent || task.output || '');
2368      if (terminalDiag) {
2369        _showDiagnosis(el, terminalDiag, el.querySelector('.cookbook-output-pre')?.textContent || task.output || '');
2370      } else {
2371        const existingDiag = el.querySelector('.cookbook-diagnosis');
2372        // Keep diagnosis for failed tasks even if output was cleared and we
2373        // can no longer re-derive the exact message — removing it would hide
2374        // the crash reason from the user.
2375        if (existingDiag && !['stopped', 'error', 'crashed', 'failed'].includes(task.status)) {
2376          existingDiag.remove();
2377        }
2378      }
2379    }
2380    if (!task) {
2381      if (el._uptimeInterval) { clearInterval(el._uptimeInterval); el._uptimeInterval = null; }
2382      el.remove();
2383    }
2384  });
2385
2386  // Add new task entries
2387  for (const task of tasks) {
2388    if (existingIds.has(task.sessionId)) continue;
2389
2390    const el = document.createElement('div');
2391    el.className = 'cookbook-task' + (task._unreachable && task.status === 'running' ? ' cookbook-task-unreachable' : '');
2392    el.dataset.taskId = task.sessionId;
2393    el.dataset.status = task.status;
2394    el.dataset.type = task.type || '';
2395
2396    const _bdg = _taskBadge(task);
2397    const _bdgTitle = (task._unreachable && task.status === 'running') ? ' title="Server not responding — it may have crashed"' : '';
2398    const displayName = _taskDisplayName(task);
2399    const logoName = task.type === 'download' ? (task.payload?.repo_id || task.name) : task.name;
2400    el.innerHTML = `
2401      <div class="cookbook-task-header">

2402        <span class="cookbook-task-type${(task.status === 'done' && task.type === 'download') ? ' cookbook-task-type-done' : ''}" data-type="${esc(task.type)}">${esc((task.status === 'done' && task.type === 'download') ? 'finished' : task.type)}</span>
2403        <span class="cookbook-task-name">${modelLogo(logoName)}${esc(displayName)}</span>
2404        <span class="cookbook-task-indicator"><span class="cookbook-task-wave" style="display:${task.status === 'running' ? '' : 'none'}"></span>${_canLaunchDownloadedTask(task) ? '<button type="button" class="cookbook-task-serve-btn" title="Open in Launch"><svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round"><polygon points="13 2 3 14 12 14 11 22 21 10 12 10 13 2"/></svg><span>Launch</span></button>' : ''}<span class="cookbook-task-check" title="Clear" style="display:${_canClearTask(task) ? '' : 'none'}"><svg class="cookbook-task-check-ico" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="#50fa7b" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg><svg class="cookbook-task-clear-ico" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg><span class="cookbook-task-done-label">${esc(_clearPillLabel(task))}</span><span class="cookbook-task-clear-label">clear</span></span></span>
2405        <button type="button" class="cookbook-task-start-now" title="Start this queued download now" style="display:${(task.type === 'download' && task.status === 'queued') ? '' : 'none'}">

2405<svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true"><polygon points="8 5 19 12 8 19 8 5"/></svg><span>start now</span></button>
2406        <span class="cookbook-task-status ${_bdg.cls}"${_bdgTitle}>${esc(_bdg.text)}</span>
2407        <button type="button" class="cookbook-task-menu-btn" title="Actions">&#8942;</button>
2408      </div>
2409      <div class="cookbook-task-sub"><span class="cookbook-task-session">${esc(task.sessionId)}</span><span class="cookbook-task-uptime" style="display:${((task.type === 'serve' || task.type === 'download') && task.status === 'running') ? '' : 'none'}"></span>${(task.type === 'download') ? `<span class="cookbook-task-dldir" title="Download destination" style="font-size:9px;color:var(--fg-muted);font-family:'Fira Code',monospace;opacity:0.4;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;max-width:40ch;">Dir: ${esc(task.payload?.local_dir || '~/.cache/huggingface/hub')}</span>` : ''}</div>
2410      <div class="cookbook-output-wrap cookbook-task-collapsible${(_mobileCollapseDefault && !_shouldAutoExpandTaskOutput(task)) ? ' cookbook-task-collapsed' : ''}"><pre class="cookbook-output-pre">${esc(task.output || '')}</pre><button type="button" class="copy-code cookbook-output-copy"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg></button></div>
2411    `;
2412
2413    const _waveEl = el.querySelector('.cookbook-task-wave');
2414    if (_waveEl && task.status === 'running') _registerWaveEl(_waveEl);
2415
2416    const terminalDiag = _terminalServeDiagnosis(task, task.output || '');
2417    if (terminalDiag) _showDiagnosis(el, terminalDiag, task.output || '');
2418    if (!terminalDiag && (task.status === 'error' || task.status === 'crashed') && task._backendDiagnosis) {
2419      _showDiagnosis(el, task._backendDiagnosis, task.output || '');
2420    }
2421
2422    const _uptimeEl = el.querySelector('.cookbook-task-uptime');
2423    if (_uptimeEl && (task.type === 'serve' || task.type === 'download') && task.status === 'running') {
2424      const _startedAt = task.ts || Date.now();
2425      const _prefix = task.type === 'download' ? 'downloading' : 'uptime';
2426      el._uptimeInterval = setInterval(() => {
2427        const secs = Math.floor((Date.now() - _startedAt) / 1000);
2428        const h = Math.floor(secs / 3600);
2429        const m = Math.floor((secs % 3600) / 60);
2430        const s = secs % 60;
2431        const _timer = h > 0
2432          ? `${_prefix}: ${h}h ${String(m).padStart(2,'0')}m`
2433          : `${_prefix}: ${m}m ${String(s).padStart(2,'0')}s`;
2434        // ETA — only for downloads, only when we have a meaningful overall %.
2435        // Reads the badge text (which already shows the true overall % we
2436        // compute in the live-polling block) and back-derives a remaining-time
2437        // estimate from elapsed/done. Hidden until pct >= 3% so the early-job
2438        // wild estimates don't show.
2439        let _eta = '';
2440        if (task.type === 'download') {
2441          const _badge = el.querySelector('.cookbook-task-status');
2442          const _m = _badge && /^(\d+)%/.exec(_badge.textContent || '');
2443          const _pct = _m ? parseInt(_m[1], 10) : 0;
2444          if (_pct >= 3 && _pct < 100 && secs > 5) {
2445            const _totalSec = Math.round(secs * (100 / _pct));
2446            const _remain = Math.max(0, _totalSec - secs);
2447            const _eh = Math.floor(_remain / 3600);
2448            const _em = Math.floor((_remain % 3600) / 60);
2449            const _es = _remain % 60;
2450            _eta = _eh > 0
2451              ? ` · ETA ${_eh}h ${String(_em).padStart(2,'0')}m`
2452              : (_em > 0 ? ` · ETA ${_em}m ${String(_es).padStart(2,'0')}s` : ` · ETA ${_es}s`);
2453          }
2454        }
2455        _uptimeEl.textContent = _timer + _eta;
2456      }, 1000);
2457    }
2458
2459    // Re-open the Serve panel for this model, pre-filled with the EXACT
2460    // settings this instance launched with, and on the SERVER it runs on.
2461    const _openEdit = () => _openServeEditForTask(task);
2462    el.addEventListener('cookbook:edit-serve', (e) => {
2463      e.stopPropagation();
2464      _openServeEditForTask(task, null, e.detail?.fields || null);
2465    });
2466

2467    // Finished download → an explicit "Serve →" button jumps straight to the
2468    // Serve tab with this model pre-selected (on the server it downloaded to).
2469    if (task.type === 'download') {
2470      const _serveBtn = el.querySelector('.cookbook-task-serve-btn');
2471      if (_serveBtn) {
2472        _serveBtn.addEventListener('click', async (e) => {
2473          e.stopPropagation();
2474          const repo = task.payload?.repo_id || task.name;
2475          if (!repo) { uiModule.showToast('No model info on this task'); return; }
2476          // Point the active server at the exact profile it downloaded to.
2477          _selectTaskServer(task);
2478          try {
2479            const { openServePanelForRepo } = await import('./cookbookServe.js');
2480            await openServePanelForRepo(repo, _downloadServeFields(task));
2481            // Serving it supersedes the finished download — clear the card from
2482            // the Running tab (smooth exit) now that we've jumped to Serve.
2483            _animateOutThenRemove(el, task.sessionId);
2484          } catch (err) { uiModule.showToast('Could not open Serve: ' + err.message); }
2485        });
2486      }
2487    }
2488
2489    // Finished tasks show a green check — make it click-to-clear so the user can
2490    // dismiss a completed download/update (we no longer auto-remove them). It
2491    // morphs to a red ✕ on hover (see CSS).
2492    const _clearChk = el.querySelector('.cookbook-task-check');
2493    if (_clearChk) {
2494      _clearChk.addEventListener('click', (e) => {
2495        e.stopPropagation();
2496        // If the output still shows an active shard line, the task isn't
2497        // actually finished — clicking is "reconnect" (flip back to running
2498        // + let _reconnectTask reattach to the live tmux session), not
2499        // "clear". The pill label already reflects this via _clearPillLabel.
2500        if (_downloadOutputLooksActive(task)) {
2501          const _fresh = _loadTasks();
2502          const _ft = _fresh.find(t => t.sessionId === task.sessionId);
2503          if (_ft) {
2504            _ft.status = 'running';
2505            _ft._selfHealed = true;
2506            _saveTasks(_fresh);
2507          }
2508          // Visually flip without waiting for a full re-render — same path the
2509          // self-heal uses on cookbook open.
2510          const _chk = el.querySelector('.cookbook-task-check');
2511          if (_chk) _chk.style.display = 'none';
2512          const _wave = el.querySelector('.cookbook-task-wave');
2513          if (_wave) _wave.style.display = '';
2514          const _up = el.querySelector('.cookbook-task-uptime');
2515          if (_up) _up.style.display = '';
2516          el.dataset.status = 'running';
2517          _renderRunningTab();
2518          return;
2519        }
2520        // Otherwise: real clear. Kill the tmux session as belt-and-suspenders,
2521        // then animate out + remove the row.
2522        try {
2523          fetch('/api/shell/exec', {
2524            method: 'POST', credentials: 'same-origin',
2525            headers: { 'Content-Type': 'application/json' },
2526            body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${task.sessionId}`) }),
2527          }).catch(() => {});
2528        } catch {}
2529        _animateOutThenRemove(el, task.sessionId);
2530      });
2531    }
2532
2533    const _startNowBtn = el.querySelector('.cookbook-task-start-now');
2534    if (_startNowBtn) {
2535      _startNowBtn.addEventListener('click', (e) => {
2536        e.stopPropagation();
2537        _startQueuedDownload(task);
2538      });
2539    }
2540
2541    // Wire header click to collapse/expand output
2542    el.querySelector('.cookbook-task-header').addEventListener('click', (e) => {
2543      if (e.target.closest('button')) return;
2544      const wrap = el.querySelector('.cookbook-output-wrap');
2545      if (!wrap) return;
2546      const isOpening = wrap.classList.contains('cookbook-task-collapsed');
2547      wrap.classList.toggle('cookbook-task-collapsed');
2548      if (isOpening) {
2549        _expandedTaskIds.add(task.sessionId);
2550        _collapsedTaskIds.delete(task.sessionId);
2551        if (task.sessionId && ['serve', 'download'].includes(task.type || '')) {
2552          _reconnectTask(el, task);
2553        }
2554      } else {
2555        _collapsedTaskIds.add(task.sessionId);
2556        _expandedTaskIds.delete(task.sessionId);
2557        if (el._abort) {
2558          try { el._abort.abort(); } catch {}
2559          el._abort = null;
2560        }
2561      }
2562    });
2563
2564    // Wire menu button (also fire from a long-press anywhere on the card so
2565    // mobile users don't have to hit the small ⋮ target precisely).
2566    const menuBtn = el.querySelector('.cookbook-task-menu-btn');
2567    if (menuBtn) {

2568      // Long-press detection on the card: ~500ms hold without scroll movement
2569      // re-uses the menu button's click path (so we don't duplicate logic).
2570      let _lpTimer = null;
2571      let _lpStartY = 0;
2572      let _lpCanceled = false;
2573      const _lpStart = (e) => {
2574        _lpCanceled = false;
2575        _lpStartY = (e.touches?.[0]?.clientY) ?? 0;
2576        _lpTimer = setTimeout(() => {
2577          if (_lpCanceled) return;
2578          _lpCanceled = true;  // suppress the subsequent click-through
2579          try { menuBtn.click(); } catch {}
2580        }, 500);
2581      };
2582      const _lpCancel = () => {
2583        if (_lpTimer) { clearTimeout(_lpTimer); _lpTimer = null; }
2584      };
2585      const _lpMove = (e) => {
2586        const y = (e.touches?.[0]?.clientY) ?? 0;
2587        if (Math.abs(y - _lpStartY) > 8) _lpCancel();
2588      };
2589      el.addEventListener('touchstart', (e) => {
2590        // Skip if the user is starting touch on a button / link inside the
2591        // card — those already have their own tap handlers.
2592        if (e.target.closest('button, a, input, textarea, .cookbook-task-dropdown')) return;
2593        _lpStart(e);
2594      }, { passive: true });
2595      el.addEventListener('touchmove', _lpMove, { passive: true });
2596      el.addEventListener('touchend', _lpCancel, { passive: true });
2597      el.addEventListener('touchcancel', _lpCancel, { passive: true });
2598      let _lastTouchMenuOpenAt = 0;
2599      const _openTaskMenu = (e) => {
2600        e.stopPropagation();
2601        if (e.type === 'click' && Date.now() - _lastTouchMenuOpenAt < 550) return;
2602        const existing = document.querySelector('.cookbook-task-dropdown');
2603        if (existing && existing._anchor === menuBtn) {
2604          if (typeof existing._dismiss === 'function') existing._dismiss();
2605          else existing.remove();
2606          return;
2607        }
2608        document.querySelectorAll('.cookbook-task-dropdown').forEach(d => { if (typeof d._dismiss === 'function') d._dismiss(); else d.remove(); });
2609
2610        const dropdown = document.createElement('div');
2611        dropdown.className = 'cookbook-task-dropdown';
2612        dropdown._anchor = menuBtn;
2613        menuBtn.classList.add('cookbook-menu-active');
2614
2615        const items = [];
2616        // ── Run section ─────────────────────────────────────────────
2617        // Queued download: let the user jump the queue and start it immediately
2618        // (downloads otherwise run one-at-a-time per server).
2619        if (task.type === 'download' && task.status === 'queued') {
2620          items.push({ group: 'run', label: 'Start now', action: 'start-now', custom: () => {
2621            _startQueuedDownload(task);
2622            _renderRunningTab();
2623          }});
2624        }
2625        if (task.status !== 'running' && task.status !== 'queued') {
2626          items.push({ group: 'run', label: 'Reconnect tmux', action: 'reconnect' });
2627        }
2628        items.push({ group: 'run', label: 'Restart', action: 'retry' });
2629        // ── Edit section ────────────────────────────────────────────
2630        // Merged "Edit & relaunch" — opens the structured serve panel
2631        // pre-filled with this task's config. The old standalone "Edit
2632        // cmd & relaunch" raw-text dialog is now reachable from inside
2633        // that panel (Show command). Single entry-point per task.
2634        if (task.type === 'serve' && task.payload?.repo_id) {
2635          items.push({ group: 'edit', label: 'Edit & relaunch', action: 'edit-panel', tooltip: 'Open the Serve config panel pre-filled with this task — pick a different backend, change GPUs, edit env vars or the raw cmd, then Launch.', custom: () => _openEdit() });
2636        }
2637        if (task.type === 'serve' && task.payload?._cmd) {
2638          items.push({ group: 'edit', label: 'Save serve', action: 'save', custom: () => {
2639            if (!_saveTaskAsPreset(task)) { uiModule.showToast('Already saved'); return; }
2640            uiModule.showToast('Saved to presets');
2641            _renderRunningTab();
2642          }});
2643        }
2644        // ── Endpoint section ────────────────────────────────────────
2645        // Manual endpoint registration — fallback for when auto-add fails
2646        // (e.g. probe timeout on a remote that's slow). Forces adding this
2647        // serve to the model-endpoints list regardless of prior flag state.
2648        if (task.type === 'serve' && task.payload?._cmd) {
2649          items.push({ group: 'endpoint', label: 'Register endpoint', action: 'register-endpoint', custom: async () => {
2650            const host = _connectHostFromRemote(task.remoteHost);

2651            const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
2652            const port = portMatch ? portMatch[1] : '8000';
2653            const baseUrl = `http://${host}:${port}/v1`;
2654            try {
2655              // Check existing first — offer to overwrite if present
2656              const eps = await (await fetch('/api/model-endpoints', { credentials: 'same-origin' })).json();
2657              const existing = eps.find(e => e.base_url === baseUrl);
2658              if (existing) {
2659                uiModule.showToast(`Already registered as "${existing.name}"`);
2660                task._endpointAdded = true;
2661                _updateTask(task.sessionId, { _endpointAdded: true });
2662                _refreshModelsAfterEndpointChange();
2663                // If it's still offline (registered before the server finished
2664                // loading), keep probing until it answers instead of leaving it
2665                // stuck offline until a manual delete/re-add.
2666                if (existing.id && !(existing.models || []).length) _probeEndpointUntilOnline(existing.id, host, port);
2667                return;
2668              }
2669              const fd = new FormData();
2670              fd.append('base_url', baseUrl);
2671              fd.append('name', task.name);
2672              fd.append('skip_probe', 'true');
2673              _appendCookbookEndpointScope(fd, task.remoteHost || '');
2674              if (_isImageServeTask(task)) fd.append('model_type', 'image');
2675              const res = await fetch('/api/model-endpoints', { method: 'POST', credentials: 'same-origin', body: fd });
2676              if (res.ok) {
2677                task._endpointAdded = true;
2678                _updateTask(task.sessionId, { _endpointAdded: true });
2679                uiModule.showToast(`Endpoint registered: ${host}:${port}`);
2680                _refreshModelsAfterEndpointChange();
2681                // Added with skip_probe → probe until the (possibly still
2682                // warming) server answers, so it flips online on its own.
2683                const _ep = await res.json().catch(() => ({}));
2684                if (_ep && _ep.id) _probeEndpointUntilOnline(_ep.id, host, port);
2685              } else {
2686                const body = await res.text().catch(() => '');
2687                uiModule.showError(`Register failed: ${res.status} ${body.slice(0, 140)}`);
2688              }
2689            } catch (e) {
2690              uiModule.showError(`Register failed: ${e.message || e}`);
2691            }
2692          }});
2693        }
2694        // ── Copy section ────────────────────────────────────────────
2695        if (_isWindows(task)) {
2696          const host = task.remoteHost;
2697          const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux';
2698          const logCmd = host
2699            ? `ssh ${_sshPrefix(_getPort(task))}${host} "powershell -Command \\"Get-Content '${sd}\\${task.sessionId}.log' -Wait\\""`
2700            : `powershell -Command "Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${task.sessionId}.log') -Wait"`;
2701          items.push({ group: 'copy', label: 'Copy log cmd', action: 'copy-tmux', custom: () => {
2702            _copyText(logCmd);
2703          }});
2704        } else {
2705          // Just the tmux command itself — no ssh wrapper.
2706          const tmuxAttach = `tmux attach -t ${task.sessionId}`;
2707          items.push({ group: 'copy', label: 'Copy tmux', action: 'copy-tmux', custom: () => {
2708            _copyText(tmuxAttach);
2709          }});
2710        }
2711        if (_shouldOfferCrashReport(task)) {
2712          items.push({ group: 'copy', label: 'Copy crash report', action: 'copy-crash-report', custom: () => {
2713            const out = (el.querySelector('.cookbook-output-pre')?.textContent || task.output || '');
2714            _copyText(_buildCrashReport(task, out));
2715            uiModule.showToast('Copied crash report');
2716          }});
2717        }
2718        // Copy the last 50 lines of the task's output/log.
2719        items.push({ group: 'copy', label: 'Copy last 50 lines', action: 'copy-log', custom: () => {
2720          const out = (el.querySelector('.cookbook-output-pre')?.textContent || task.output || '');
2721          const last = out.split('\n').slice(-50).join('\n');
2722          if (!last.trim()) {
2723            uiModule.showToast('No log content available yet');
2724            return;
2725          }
2726          _copyText(last);
2727          uiModule.showToast('Copied last 50 lines');
2728        }});
2729        // Label matches behavior — the kill handler ALWAYS first kills
2730        // the live tmux session and (for serve tasks) deletes the
2731        // matching model-endpoint, THEN animates the task card out.
2732        // Just "Remove" hid that it stops the live serve too.
2733        // ── Danger section ──────────────────────────────────────────
2734        const _isLive = task.type === 'serve' && ['running', 'ready', 'loading', 'warming', 'starting'].includes(task.status || '');
2735        items.push({
2736          group: 'danger',
2737          label: _isLive ? 'Stop and remove' : 'Remove',
2738          action: 'kill',
2739          tooltip: _isLive
2740            ? 'Kill the live tmux session, deregister the chat endpoint, and remove this row'
2741            : 'Remove this row',
2742          danger: true,
2743        });
2744        // Cancel = mobile-only dismiss item. Same pattern as the email kebab.
2745        items.push({ group: 'danger', label: 'Cancel', action: 'cancel', mobileOnly: true, custom: () => {} });
2746
2747        const _MENU_ICONS = {
2748          'start-now': '<polygon points="6 4 20 12 6 20 6 4"/>',
2749          reconnect: '<path d="M1 4v6h6"/><path d="M3.5 15a9 9 0 1 0 2.1-9.4L1 10"/>',
2750          retry: '<path d="M1 4v6h6"/><path d="M3.5 15a9 9 0 1 0 2.1-9.4L1 10"/>',
2751          stop: '<rect x="6" y="6" width="12" height="12" rx="1"/>',
2752          edit: '<path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"/><path d="M18.5 2.5a2.12 2.12 0 0 1 3 3L12 15l-4 1 1-4z"/>',
2753          'edit-panel': '<path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"/><path d="M18.5 2.5a2.12 2.12 0 0 1 3 3L12 15l-4 1 1-4z"/>',
2754          'register-endpoint': '<circle cx="12" cy="12" r="9"/><path d="M12 8v8M8 12h8"/>',
2755          save: '<path d="M19 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11l5 5v11a2 2 0 0 1-2 2z"/><path d="M17 21v-8H7v8M7 3v5h8"/>',
2756          'copy-tmux': '<rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/>',
2757          'copy-crash-report': '<path d="M10.3 2.3 1.8 17a2 2 0 0 0 1.7 3h17a2 2 0 0 0 1.7-3L13.7 2.3a2 2 0 0 0-3.4 0z"/><path d="M12 8v5M12 17h.01"/>',
2758          'copy-log': '<rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/>',
2759          kill: '<path d="M3 6h18"/><path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6m3 0V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"/>',
2760          cancel: '<line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/>',
2761        };
2762        let _lastGroup = null;
2763        for (const item of items) {
2764          // Insert a thin divider whenever the group changes, so the
2765          // user can visually scan Run / Edit / Endpoint / Copy / Danger
2766          // blocks instead of one long undifferentiated list.
2767          if (item.group && _lastGroup && item.group !== _lastGroup) {
2768            const sep = document.createElement('div');
2769            sep.className = 'cookbook-dropdown-divider';
2770            sep.style.cssText = 'height:1px;margin:4px 6px;background:color-mix(in srgb, var(--fg) 12%, transparent);pointer-events:none;';
2771            dropdown.appendChild(sep);
2772          }
2773          _lastGroup = item.group || _lastGroup;
2774          const div = document.createElement('div');
2775          div.className = 'dropdown-item-compact'
2776            + (item.danger ? ' cookbook-dropdown-danger' : '')
2777            + (item.mobileOnly ? ' dropdown-cancel-mobile' : '');
2778          div.style.cssText = 'display:flex;align-items:center;gap:8px;';
2779          if (item.tooltip) div.title = item.tooltip;
2780          const ic = _MENU_ICONS[item.action] || '';
2781          div.innerHTML = `<span style="display:inline-flex;flex-shrink:0;

2781opacity:0.7;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">${ic}</svg></span><span>${item.label}</span>`;
2782          div.addEventListener('click', () => {
2783            _cleanup();
2784            if (item.custom) { item.custom(); return; }
2785            el.querySelector('.cookbook-task-action-' + item.action)?.click();
2786          });
2787          dropdown.appendChild(div);
2788        }
2789
2790        const rect = menuBtn.getBoundingClientRect();
2791        dropdown.style.position = 'fixed';
2792        dropdown.style.zIndex = String(topPortalZ());
2793        dropdown.style.top = rect.bottom + 2 + 'px';
2794        dropdown.style.right = (window.innerWidth - rect.right) + 'px';
2795        document.body.appendChild(dropdown);
2796        // Clamp into the *visible* area. On mobile (esp. Firefox) window.innerHeight
2797        // includes the strip hidden under the dynamic toolbar, so a menu that "fits"
2798        // by innerHeight still lands off-screen at the bottom. visualViewport gives
2799        // the real visible region. Flip above the button if there's no room below,
2800        // else clamp to the bottom edge.
2801        {
2802          const vv = window.visualViewport;
2803          const viewTop = vv ? vv.offsetTop : 0;
2804          const viewBottom = vv ? vv.offsetTop + vv.height : window.innerHeight;
2805          const dh = dropdown.offsetHeight;
2806          const m = 8;
2807          let top = rect.bottom + 2;
2808          if (top + dh > viewBottom - m) {
2809            const above = rect.top - 2 - dh;
2810            top = above >= viewTop + m ? above : Math.max(viewTop + m, viewBottom - dh - m);
2811          }
2812          dropdown.style.top = top + 'px';
2813        }
2814
2815        const closeHandler = (ev) => {
2816          if (!dropdown.contains(ev.target) && ev.target !== menuBtn && !menuBtn.contains(ev.target)) {
2817            _cleanup();
2818          }
2819        };
2820        // Close on scroll too — once the page scrolls, the dropdown's
2821        // fixed position no longer matches the originating ⋮ button, so
2822        // it visually drifts. Matches the email kebab behaviour.
2823        const scrollClose = () => _cleanup();
2824        let _unreg = () => {};
2825        const _cleanup = () => {
2826          _unreg(); _unreg = () => {};
2827          dropdown.remove();
2828          menuBtn.classList.remove('cookbook-menu-active');
2829          document.removeEventListener('click', closeHandler);
2830          window.removeEventListener('scroll', scrollClose, true);
2831          window.visualViewport?.removeEventListener('scroll', scrollClose);
2832        };
2833        dropdown._dismiss = _cleanup;
2834        setTimeout(() => {
2835          document.addEventListener('click', closeHandler);
2836          window.addEventListener('scroll', scrollClose, true);
2837          window.visualViewport?.addEventListener('scroll', scrollClose);
2838        }, 0);
2839        _unreg = registerMenuDismiss(_cleanup);
2840      };
2841      menuBtn.addEventListener('click', _openTaskMenu);
2842      menuBtn.addEventListener('touchend', (e) => {
2843        e.preventDefault();
2844        _lastTouchMenuOpenAt = Date.now();
2845        _openTaskMenu(e);
2846      }, { passive: false });
2847    }
2848
2849    // Hidden action buttons for menu dispatch
2850    const _actionBtns = document.createElement('div');
2851    _actionBtns.style.display = 'none';
2852    _actionBtns.innerHTML = `
2853      <button class="cookbook-task-action-reconnect"></button>
2854      <button class="cookbook-task-action-retry"></button>
2855      <button class="cookbook-task-action-stop"></button>
2856      <button class="cookbook-task-action-kill"></button>
2857    `;
2858    el.appendChild(_actionBtns);
2859
2860    // Wire reconnect
2861    el.querySelector('.cookbook-task-action-reconnect').addEventListener('click', () => {
2862      _updateTask(task.sessionId, { status: 'running' });
2863      el.dataset.status = 'running';
2864      const badge = el.querySelector('.cookbook-task-status');
2865      if (badge) { badge.textContent = _statusLabel('running', task.type); badge.className = 'cookbook-task-status cookbook-task-running'; }
2866      _reconnectTask(el, task);
2867    });
2868
2869    // Wire stop
2870    el.querySelector('.cookbook-task-action-stop').addEventListener('click', async () => {
2871      // Abort the reconnect loop before sending kill so that a DOWNLOAD_FAILED
2872      // marker written by the shell wrapper (on SIGINT/non-zero exit) cannot
2873      // trigger an auto-retry after a manual stop.
2874      if (el._abort) el._abort.abort();
2875      const badge = el.querySelector('.cookbook-task-status');
2876      if (badge) { badge.textContent = 'stopping...'; badge.className = 'cookbook-task-status cookbook-task-stopping'; }
2877      el.dataset.status = 'stopped';
2878      _updateTask(task.sessionId, { _userStopped: true });
2879      const outputText = el.querySelector('.cookbook-output-pre')?.textContent || task.output || '';
2880      // Drop the model endpoint so the picker stops listing it.
2881      if (task.type === 'serve' && task.payload) {
2882        _removeEndpointByUrl(_endpointUrlForTask(task, outputText));
2883      }
2884      const ollamaUnload = _ollamaUnloadCommand(task, outputText);
2885      if (ollamaUnload) {
2886        try {
2887          await fetch('/api/shell/exec', {
2888            method: 'POST', credentials: 'same-origin',
2889            headers: { 'Content-Type': 'application/json' },
2890            body: JSON.stringify({ command: ollamaUnload }),
2891          });
2892        } catch {}
2893      }
2894      // Gracefully stop (C-c, then kill the session) so it's fully down...

2895      try {
2896        await fetch('/api/shell/exec', {
2897          method: 'POST', credentials: 'same-origin',
2898          headers: { 'Content-Type': 'application/json' },
2899          body: JSON.stringify({ command: _tmuxGracefulKill(task) }),
2900        });
2901      } catch {}
2902      // ...then smoothly fade/slide the card out and auto-remove it — no manual
2903      // ⋮ → Remove needed.
2904      _animateOutThenRemove(el, task.sessionId);
2905    });
2906
2907    // Wire kill — awaits the SSH/tmux kill and verifies the session is
2908    // actually gone before removing the row. Previously fire-and-forget,
2909    // which meant a failed kill (wrong remoteHost, SSH error, tmux server
2910    // already exited) silently left the live serve running while the
2911    // row disappeared from the UI.
2912    el.querySelector('.cookbook-task-action-kill').addEventListener('click', async () => {
2913      const outputText = el.querySelector('.cookbook-output-pre')?.textContent || task.output || '';
2914      const isLive = task.type === 'serve' && ['running', 'ready', 'loading', 'warming', 'starting'].includes(task.status || '');
2915      const ollamaUnload = _ollamaUnloadCommand(task, outputText);
2916      if (ollamaUnload) {
2917        try {
2918          await fetch('/api/shell/exec', {
2919            method: 'POST', credentials: 'same-origin',
2920            headers: { 'Content-Type': 'application/json' },
2921            body: JSON.stringify({ command: ollamaUnload }),
2922          });
2923        } catch (_) { /* unload best-effort */ }
2924      }
2925      let killOk = true;
2926      try {
2927        const r = await fetch('/api/shell/exec', {
2928          method: 'POST', credentials: 'same-origin',
2929          headers: { 'Content-Type': 'application/json' },
2930          body: JSON.stringify({ command: _tmuxGracefulKill(task) }),
2931        });
2932        if (r.ok) {
2933          const out = await r.json();
2934          // Don't trust exit_code alone — tmux kill returns 0 even when
2935          // there was nothing to kill. Verify the session is actually gone.
2936          if (task.sessionId && isLive) {
2937            try {
2938              const probe = await fetch('/api/shell/exec', {
2939                method: 'POST', credentials: 'same-origin',
2940                headers: { 'Content-Type': 'application/json' },
2941                body: JSON.stringify({ command: _tmuxCmd(task, `has-session -t ${task.sessionId}`) }),
2942              });
2943              if (probe.ok) {
2944                const pj = await probe.json();
2945                // has-session exits 0 when session STILL exists; non-zero = gone.
2946                if ((pj.exit_code || 0) === 0) killOk = false;
2947              }
2948            } catch (_) { /* probe best-effort; trust kill */ }
2949          }
2950        } else {
2951          killOk = false;
2952        }
2953      } catch (_) { killOk = false; }
2954      if (!killOk) {
2955        try { uiModule.showToast('Kill failed — session may still be running. Check `tmux ls` on the server.', 'error'); } catch (_) {}
2956        return;  // leave the row so the user can retry
2957      }
2958      if (task.type === 'serve' && task.payload) {
2959        const endpointUrl = _endpointUrlForTask(task, outputText);
2960        _removeEndpointByUrl(endpointUrl);
2961        const modelName = task.payload.model || task.name || '';
2962        if (modelName) {
2963          fetch('/api/model-endpoints', { credentials: 'same-origin' })
2964            .then(r => r.json())
2965            .then(eps => {
2966              const ep = eps.find(e => e.name === modelName || e.base_url === endpointUrl);
2967              if (ep) fetch(`/api/model-endpoints/${ep.id}`, { method: 'DELETE', credentials: 'same-origin' }).then(() => _refreshModelsAfterEndpointChange());
2968            }).catch(() => {});
2969        }
2970      }
2971      _animateOutThenRemove(el, task.sessionId);
2972    });
2973
2974    // Wire retry
2975    el.querySelector('.cookbook-task-action-retry').addEventListener('click', () => _retryTask(el, task));
2976
2977    // Wire copy button
2978    el.querySelector('.cookbook-output-copy').addEventListener('click', (e) => {
2979      e.stopPropagation();
2980      const text = el.querySelector('.cookbook-output-pre')?.textContent || '';
2981      if (!text.trim()) {
2982        uiModule.showToast('No log content available yet');
2983        return;
2984      }
2985      _copyText(text).then(() => {
2986        const btn = el.querySelector('.cookbook-output-copy');
2987        const origHTML = btn.innerHTML;
2988        btn.innerHTML = '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg>';
2989        btn.classList.add('copied');
2990        setTimeout(() => { btn.innerHTML = origHTML; btn.classList.remove('copied'); }, 1500);
2991      });
2992    });
2993
2994    // Route to the right server section body
2995    const serverBodyId = `server-body-${(_taskServerKey(task) || 'local').replace(/[^a-zA-Z0-9-]/g, '_')}`;

2996    const targetBody = document.getElementById(serverBodyId);
2997    if (targetBody) targetBody.appendChild(el);
2998    else group.appendChild(el);
2999
3000    // Auto-attach the tmux output stream for any task whose underlying
3001    // session could still be alive — not just 'running'. Scheduler-
3002    // launched serves transition to 'ready' as soon as /v1/models
3003    // responds; without this, the user opens the Running tab and sees
3004    // only the placeholder ("Launched by scheduled task …") because
3005    // _reconnectTask never fires for status 'ready'/'loading'/'warming'.
3006    const _wrapForStream = el.querySelector('.cookbook-output-wrap');
3007    const _streamExpanded = _wrapForStream && !_wrapForStream.classList.contains('cookbook-task-collapsed');
3008    if (_isRunningTabVisible() && _streamExpanded && task.sessionId && ['serve', 'download'].includes(task.type || '')) {
3009      _reconnectTask(el, task);
3010    }
3011  }
3012
3013  if (tasks.some(t => t.status === 'running')) _startWaveSync();
3014
3015  // Re-apply captured expansion state so re-renders don't fold open tasks/sections.
3016  _collapsedTaskIds.forEach((id) => {
3017    const wrap = body.querySelector(`.cookbook-task[data-task-id="${id}"] .cookbook-output-wrap`);
3018    if (wrap) wrap.classList.add('cookbook-task-collapsed');
3019  });
3020  // Mobile defaults to collapsed (above), so re-open whatever the user had
3021  // explicitly expanded before this re-render.
3022  if (_mobileCollapseDefault) {
3023    _expandedTaskIds.forEach((id) => {
3024      const wrap = body.querySelector(`.cookbook-task[data-task-id="${id}"] .cookbook-output-wrap`);
3025      if (wrap) wrap.classList.remove('cookbook-task-collapsed');
3026    });
3027  }
3028  _collapsedSectionIds.forEach((sid) => {
3029    const sb = document.getElementById(sid);
3030    if (sb) sb.style.display = 'none';
3031    const hdr = body.querySelector(`.cookbook-section-header[data-collapse="${sid}"]`);
3032    const chevron = hdr?.querySelector('.cookbook-section-chevron');
3033    if (chevron) { chevron.style.transform = 'rotate(-90deg)'; chevron.style.opacity = ''; }
3034  });
3035}
3036
3037// ── Reconnect task (polling loop) ──
3038
3039async function _reconnectTask(el, task) {
3040  if (!el || !task) return;
3041  const wrap = el.querySelector('.cookbook-output-wrap');
3042  if (!_isRunningTabVisible() || !wrap || wrap.classList.contains('cookbook-task-collapsed')) return;
3043  if (el._abort && !el._abort.signal?.aborted) return;
3044  const output = el.querySelector('.cookbook-output-pre');
3045  if (!output) return;
3046  const controller = new AbortController();
3047  el._abort = controller;
3048  let failCount = 0;
3049
3050  while (!controller.signal.aborted) {
3051    const liveWrap = el.querySelector('.cookbook-output-wrap');
3052    if (!el.isConnected || !_isRunningTabVisible() || !liveWrap || liveWrap.classList.contains('cookbook-task-collapsed')) {
3053      controller.abort();
3054      break;
3055    }
3056    try {
3057      const res = await fetch('/api/shell/exec', {
3058        method: 'POST', credentials: 'same-origin',
3059        headers: { 'Content-Type': 'application/json' },
3060        body: JSON.stringify({ command: _tmuxCmd(task, `capture-pane -t ${task.sessionId} -p -S -500`), timeout: 15 }),
3061      });
3062      const data = await res.json();
3063
3064      if (data.exit_code !== 0) {
3065        failCount++;
3066        if (failCount < 5) {
3067          await new Promise(r => setTimeout(r, 3000));
3068          continue;
3069        }
3070        try {
3071          const verify = await fetch('/api/shell/exec', {
3072            method: 'POST', credentials: 'same-origin',
3073            headers: { 'Content-Type': 'application/json' },
3074            body: JSON.stringify({ command: _tmuxCmd(task, `has-session -t ${task.sessionId}`) }),
3075          });
3076          const vData = await verify.json();
3077          if (vData.exit_code === 0) {
3078            failCount = 0;
3079            await new Promise(r => setTimeout(r, 5000));
3080            continue;
3081          }
3082        } catch {
3083          await new Promise(r => setTimeout(r, 10000));
3084          continue;
3085        }
3086
3087        const lastOutput = output.textContent || '';
3088        // Pip tasks (Reinstall vLLM / Upgrade torch / etc.) must skip the
3089        // generic serve `_diagnose` step. Their output is pip's own and the
3090        // error patterns there (torch ABI traceback, "No module named torch",
3091        // etc.) are routinely matched against the previous tmux scrollback,
3092        // tagging a clean pip success as a crashed serve. Detection is the
3093        // same shape as the looksSuccessful branch below.
3094        const _isPipTaskDiag = ((task.payload?.repo_id || '').startsWith('pip-'))
3095          || /python3? -m pip\b/.test(task.payload?._cmd || '');
3096        const diag = _isPipTaskDiag ? null : _diagnose(lastOutput);
3097        if (diag) {
3098          let diagEl = el.querySelector('.cookbook-diagnosis');
3099          if (!diagEl) {
3100            diagEl = document.createElement('div');
3101            diagEl.className = 'cookbook-diagnosis';
3102            el.appendChild(diagEl);
3103          }
3104          _showDiagnosis(el, diag, lastOutput);
3105          _updateTask(task.sessionId, { status: 'error' });
3106          el.dataset.status = 'error';
3107          const badge = el.querySelector('.cookbook-task-status');
3108          if (badge) { badge.textContent = _statusLabel('error', task.type); badge.className = 'cookbook-task-status cookbook-task-error'; }
3109          _showCookbookNotif(true);
3110        } else {
3111          const downloadLooksSuccessful = !lastOutput.includes('DOWNLOAD_FAILED')
3112            && (lastOutput.includes('DONE') || lastOutput.includes('100%') || lastOutput.includes('/snapshots/') || lastOutput.includes('Download complete') || lastOutput.includes('DOWNLOAD_OK'));
3113          // Pip install / reinstall tasks are launched via _launchServeTask (so
3114          // they show up in the Running tab + use tmux) but they aren't real

3115          // serves — the cmd is `python3 -m pip ...` and the success markers
3116          // are pip's own. Without this branch, a successful reinstall ends
3117          // with no "Uvicorn running on" line and gets mis-flagged as a crashed
3118          // serve.
3119          const _isPipTask = ((task.payload?.repo_id || '').startsWith('pip-'))
3120            || /python3? -m pip\b/.test(task.payload?._cmd || '');
3121          const pipLooksSuccessful = _isPipTask
3122            && /Successfully installed|Requirement already (?:satisfied|up-to-date)/i.test(lastOutput)
3123            && !/error:|ERROR:/.test(lastOutput.slice(-1024));
3124          const serveLooksReady = task.type === 'serve' && _serveOutputLooksReady({ ...task, output: lastOutput });
3125          // Dependency installs are tracked as download tasks but finish with a
3126          // pip exit-0 sentinel, not HF download markers — check that too.
3127          // Standalone pip-* serves finish with pip's own success line, not
3128          // HF or "Uvicorn running on".
3129          const depInstallSucceeded = !!task.payload?._dep && _depInstallSucceeded(lastOutput);
3130          const looksSuccessful = depInstallSucceeded
3131            || (task.type === 'download'
3132              ? downloadLooksSuccessful
3133              : (_isPipTask ? pipLooksSuccessful : serveLooksReady));
3134          if (!lastOutput.trim() || !looksSuccessful) {
3135            _updateTask(task.sessionId, { status: 'crashed' });
3136            el.dataset.status = 'crashed';
3137            const badge = el.querySelector('.cookbook-task-status');
3138            if (badge) { badge.textContent = _statusLabel('crashed', task.type); badge.className = 'cookbook-task-status cookbook-task-crashed'; }
3139            if (_isPipTask) {
3140              // Pip tasks: don't run the serve diagnosis (which would yell
3141              // "Serve stopped before the model became reachable"). Show a
3142              // pip-tailored message; the user can read pip's own error output
3143              // directly above.
3144              const _ranOk = /Successfully installed|Requirement already (?:satisfied|up-to-date)/i.test(lastOutput);
3145              if (!_ranOk) {
3146                _showDiagnosis(el, {
3147                  message: 'Pip install did not finish with a success marker. Check the output for the underlying error.',
3148                  suggestion: 'Suggested action: copy the troubleshooting bundle. Common causes: missing build deps, network blip, mismatched torch ABI.',
3149                  fixes: [],
3150                }, lastOutput);
3151              }
3152            } else if (task.type === 'serve') {
3153              const diag = _diagnose(lastOutput) || {
3154                message: _serveTaskLooksAwqOnLocalBackend(task, lastOutput)
3155                  ? 'AWQ/GPTQ/FP8 cannot be served through llama.cpp/Ollama unified-memory mode.'
3156                  : /Native llama-server not found|building llama-server|llama\.cpp/i.test(lastOutput)
3157                  ? 'llama.cpp build stopped before the server became reachable.'
3158                  : 'Serve stopped before the model became reachable.',
3159                suggestion: _serveTaskLooksAwqOnLocalBackend(task, lastOutput)
3160                  ? 'Suggested action: use vLLM/SGLang on a compatible CUDA/ROCm GPU server, or download a GGUF version for llama.cpp/Ollama/unified-memory serving.'
3161                  : /Native llama-server not found|building llama-server|llama\.cpp/i.test(lastOutput)
3162                  ? 'Suggested action: copy the troubleshooting bundle, then edit serve settings. For the quickest local/CPU path, use Ollama or a prebuilt llama-server; source builds can take several minutes and fail if build dependencies are incomplete.'
3163                  : 'Suggested action: copy the troubleshooting bundle, then edit serve settings or relaunch with a CPU/backend fallback.',
3164                fixes: [{ label: 'Edit serve', action: (panel) => _openServeEditForTask(task) }],
3165              };
3166              _showDiagnosis(el, diag, lastOutput);
3167            } else if (task.type === 'download') {
3168              const isDisk = /no space left|disk quota|enospc/i.test(lastOutput);
3169              const isNetwork = /connection|timeout|timed out|incompleteread|chunkedencoding|reset by peer|protocolerror|all connection attempts failed/i.test(lastOutput);
3170              const progressMatch = String(lastOutput || '').match(/(\d+)%\|/);
3171              const nearDone = progressMatch && Number(progressMatch[1]) >= 80;
3172              // Reconnect: most "crashed" downloads near the end are actually
3173              // finished — we just missed the DOWNLOAD_OK / /snapshots/ marker
3174              // because output rolled over, or the tmux session ended a tick
3175              // before we polled. Probing has-session and re-attaching to
3176              // capture-pane lets the existing _reconnectTask flow pick up
3177              // the real state (running, finished, or truly dead).
3178              const _reconnectFix = {
3179                label: 'Reconnect tmux',
3180                action: () => {
3181                  _updateTask(task.sessionId, { status: 'running' });
3182                  el.dataset.status = 'running';
3183                  const badge2 = el.querySelector('.cookbook-task-status');
3184                  if (badge2) { badge2.textContent = _statusLabel('running', task.type); badge2.className = 'cookbook-task-status'; }
3185                  const _diagEl = el.querySelector('.cookbook-diagnosis');
3186                  if (_diagEl) _diagEl.remove();
3187                  const _wave = el.querySelector('.cookbook-task-wave'); if (_wave) _wave.style.display = '';
3188                  const _up = el.querySelector('.cookbook-task-uptime'); if (_up) _up.style.display = '';
3189                  _reconnectTask(el, task);
3190                },
3191              };
3192              const diag = {
3193                message: isDisk
3194                  ? 'Download stopped because this server ran out of disk space.'
3195                  : isNetwork
3196                  ? 'Download stopped after the HuggingFace connection was interrupted.'
3197                  : nearDone
3198                  ? 'Download stopped near the end before the final completion marker was captured.'
3199                  : 'Download stopped before HuggingFace reported completion.',
3200                suggestion: isDisk
3201                  ? 'Suggested action: free disk space, then retry the download. HuggingFace resumes incomplete files when possible.'
3202                  : nearDone
3203                  ? 'Suggested action: hit Reconnect first — the download may have finished after the output buffer rolled over. Retry only if reconnect cannot recover.'
3204                  : 'Suggested action: hit Reconnect to re-attach to the tmux session. If that fails, retry — HuggingFace resumes incomplete files when possible.',
3205                fixes: isDisk
3206                  ? [
3207                      { label: 'Retry download', action: () => _retryTask(el, task) },
3208                      { label: 'Copy last 50 lines', action: () => {

3209                        const last = String(lastOutput || '').split('\n').slice(-50).join('\n');
3210                        _copyText(last || 'No download log available.');
3211                      } },
3212                    ]
3213                  : [
3214                      _reconnectFix,
3215                      { label: 'Retry download', action: () => _retryTask(el, task) },
3216                      { label: 'Copy last 50 lines', action: () => {
3217                        const last = String(lastOutput || '').split('\n').slice(-50).join('\n');
3218                        _copyText(last || 'No download log available.');
3219                      } },
3220                    ],
3221              };
3222              _showDiagnosis(el, diag, lastOutput);
3223              // Auto-probe: if the tmux session is still alive (download
3224              // genuinely still in progress), _selfHealStaleTasks flips the
3225              // task back to running and the diagnosis disappears without
3226              // the user needing to click Reconnect.
3227              if (nearDone) setTimeout(() => { _selfHealStaleTasks().catch(() => {}); }, 1200);
3228            }
3229            _showCookbookNotif(true);
3230          } else {
3231            // Strong completion markers — `DOWNLOAD_OK` is emitted by our
3232            // downloader wrapper AFTER the model snapshot is on disk, and
3233            // `/snapshots/` only appears once HF has resolved the cached
3234            // tree. Either is conclusive. Finalize as done immediately, skip
3235            // the 30s debounce — the debounce only exists to guard against
3236            // ambiguous markers (bare "100%" / "Download complete") which can
3237            // appear mid-stream during multi-file downloads.
3238            const _strongDone = task.type === 'download'
3239              && (lastOutput.includes('DOWNLOAD_OK') || lastOutput.includes('/snapshots/'));
3240            if (_strongDone) {
3241              _updateTask(task.sessionId, { status: 'done', _doneConfirmAt: null, _lastStatusFlipAt: Date.now() });
3242              el.dataset.status = 'done';
3243              const badge = el.querySelector('.cookbook-task-status');
3244              if (badge) { badge.textContent = _statusLabel('done', task.type); badge.className = 'cookbook-task-status cookbook-task-done'; }
3245              const _chk = el.querySelector('.cookbook-task-check'); if (_chk) _chk.style.display = '';
3246              const _sb = el.querySelector('.cookbook-task-serve-btn'); if (_sb) _sb.style.display = '';
3247              _showCookbookNotif();
3248              _refreshDepsAfterInstall(task);
3249              _renderRunningTab();
3250              _processQueue();
3251              break;
3252            }
3253            // Debounce the done flip. Tmux capture-pane can fail transiently
3254            // (network blip, ssh reconnect), and the verify has-session right
3255            // above can briefly report dead even when the session is in the
3256            // middle of finalizing. Marking done immediately + the periodic
3257            // _selfHealStaleTasks then flipping back to running causes the
3258            // status badge to oscillate between Finished and Downloading.
3259            // Wait 30s and re-probe: only finalize as done if tmux is STILL
3260            // gone. If the session resurfaces, restart _reconnectTask so live
3261            // capture resumes without the user seeing a fake "done" first.
3262            if (!task._doneConfirmAt) {
3263              _updateTask(task.sessionId, { _doneConfirmAt: Date.now() + 30000 });
3264              setTimeout(async () => {
3265                try {
3266                  const fresh = _loadTasks().find(t => t.sessionId === task.sessionId);
3267                  if (!fresh) return;
3268                  let stillAlive = false;
3269                  try {
3270                    const probe = await fetch('/api/shell/exec', {
3271                      method: 'POST', credentials: 'same-origin',
3272                      headers: { 'Content-Type': 'application/json' },
3273                      body: JSON.stringify({ command: _tmuxCmd(task, `has-session -t ${task.sessionId}`), timeout: 5 }),
3274                    });
3275                    const pData = await probe.json();
3276                    stillAlive = pData.exit_code === 0;
3277                  } catch { /* network blip — treat as inconclusive, prefer running */ stillAlive = true; }
3278                  if (stillAlive) {
3279                    _updateTask(task.sessionId, { status: 'running', _doneConfirmAt: null, _lastStatusFlipAt: Date.now() });
3280                    const _el = document.querySelector(`.cookbook-task[data-task-id="${task.sessionId}"]`);
3281                    if (_el) {
3282                      _el.dataset.status = 'running';
3283                      const _badge = _el.querySelector('.cookbook-task-status');
3284                      if (_badge) { _badge.textContent = _statusLabel('running', task.type); _badge.className = 'cookbook-task-status'; }
3285                      const _wave = _el.querySelector('.cookbook-task-wave'); if (_wave) _wave.style.display = '';
3286                      const _up = _el.querySelector('.cookbook-task-uptime'); if (_up) _up.style.display = '';
3287                      _reconnectTask(_el, _loadTasks().find(t => t.sessionId === task.sessionId));
3288                    }
3289                    return;
3290                  }
3291                  _updateTask(task.sessionId, { status: 'done', _doneConfirmAt: null, _lastStatusFlipAt: Date.now() });
3292                  const _el = document.querySelector(`.cookbook-task[data-task-id="${task.sessionId}"]`);
3293                  if (_el) {
3294                    _clearDiagnosis(_el);
3295                    _el.dataset.status = 'done';
3296                    const _badge = _el.querySelector('.cookbook-task-status');
3297                    if (_badge) { _badge.textContent = _statusLabel('done', task.type); _badge.className = 'cookbook-task-status cookbook-task-done'; }
3298                    const _chk = _el.querySelector('.cookbook-task-check'); if (_chk) _chk.style.display = '';
3299                    const _sb = _el.querySelector('.cookbook-task-serve-btn'); if (_sb) _sb.style.display = '';
3300                  }
3301                  _showCookbookNotif();
3302                  _refreshDepsAfterInstall(task);
3303                  _renderRunningTab();
3304                  _processQueue();
3305                } catch { /* swallow — next polling cycle will retry */ }
3306              }, 30000);
3307            }
3308          }
3309        }
3310        _renderRunningTab();
3311        _processQueue();
3312        break;
3313      }
3314
3315      const snapshot = (data.stdout || '').trim();
3316      if (snapshot) {
3317        // Only auto-scroll to bottom if the user was already there. When
3318        // they've scrolled up to read earlier output, leave their position
3319        // alone so a fresh snapshot doesn't yank them back to the tail.
3320        // 40px tolerance covers sub-pixel rounding + the moment between
3321        // releasing the scrollbar and the next poll arriving.
3322        const _atBottom = (output.scrollHeight - output.scrollTop - output.clientHeight) < 40;
3323        output.textContent = snapshot;
3324        if (_atBottom) output.scrollTop = output.scrollHeight;
3325
3326        // Live status parsing for download tasks
3327        if (task.type === 'download') {
3328          const badge = el.querySelector('.cookbook-task-status');
3329          if (badge) {

3330            const completed = (snapshot.match(/Download complete/g) || []).length;
3331            const downloading = snapshot.match(/Downloading '([^']+)'/g) || [];
3332            const totalFiles = downloading.length;
3333            const pctMatches = [...snapshot.matchAll(/(\d+)%\|/g)];
3334            const lastPct = pctMatches.length ? pctMatches[pctMatches.length - 1][1] : null;
3335            const speedMatch = [...snapshot.matchAll(/([\d.]+)(?:MB|GB)\/s/g)];
3336            const lastSpeed = speedMatch.length ? speedMatch[speedMatch.length - 1][0] : null;
3337            // hf_transfer prints "Downloading (incomplete total...): 73% | 1.81G/2.49G"
3338            // — the real aggregate byte progress. The "Fetching N files" line (often
3339            // last in the output) sits at 0%, so lastPct/_fetchPct can read 0 even at
3340            // 73% done. Prefer this aggregate when present.
3341            const _dlAggMatches = [...snapshot.matchAll(/Downloading\s*\(incomplete[^)]*\):\s*(\d+)%/g)];
3342            const _dlAgg = _dlAggMatches.length ? parseInt(_dlAggMatches[_dlAggMatches.length - 1][1]) : null;
3343
3344            // Stale download detection.
3345            // Use the DOWNLOADED-BYTE count ("1.81G" from "1.81G/2.49G") as the
3346            // progress signal: it climbs continuously while transferring (even when
3347            // the % plateaus during a big hf_transfer chunk) and FREEZES when stuck.
3348            // The % alone plateaus (false stall), and a frozen frame still shows a
3349            // stale speed/ETA — so keying off speed masked real stalls (that's why a
3350            // 97%-stuck download went undetected). Bytes are the honest signal; fall
3351            // back to %/aggregate only when no byte counter is present.
3352            const _byteMatches = [...snapshot.matchAll(/([\d.]+\s?[KMGT])B?\s*\/\s*[\d.]+\s?[KMGT]B?/gi)];
3353            const _bytes = _byteMatches.length ? _byteMatches[_byteMatches.length - 1][1].replace(/\s/g, '') : null;
3354            // When there's no byte counter (pip resolve / native build phase of a
3355            // dependency install), key off the output tail so new build lines count
3356            // as progress — otherwise a long quiet build is falsely declared stale
3357            // and restarted mid-build, looping forever (#1568).
3358            const curProgress = computeProgressSignal(_bytes, _dlAgg, lastPct, snapshot);
3359            const _fetchPctMatches = [...snapshot.matchAll(/Fetching\s+\d+\s+files:\s*(\d+)%/g)];
3360            const _fetchPct = _fetchPctMatches.length ? parseInt(_fetchPctMatches[_fetchPctMatches.length - 1][1]) : null;
3361            const isPipDep = !!(task.payload && task.payload._dep);
3362            const _startupStalled = !_bytes && ((_dlAgg === 0) || (_fetchPct === 0)) && curProgress === '0';
3363            const _STALE_TIMEOUT = _startupStalled ? STARTUP_STALE_PROGRESS_MS : STALE_PROGRESS_MS;
3364            if (!el._lastProgress) { el._lastProgress = curProgress; el._lastProgressTime = Date.now(); }
3365            if (curProgress !== el._lastProgress) {
3366              el._lastProgress = curProgress;
3367              el._lastProgressTime = Date.now();
3368            } else if (!isPipDep && Date.now() - (el._lastProgressTime || 0) > _STALE_TIMEOUT && task._autoRestarted) {
3369              const mins = Math.floor((Date.now() - (el._lastProgressTime || 0)) / 60000);
3370              // Already auto-restarted once and stalled again — make the badge a
3371              // one-click retry (resumes from the cached partial files) so the
3372              // user doesn't have to dig into the ⋮ menu.
3373              badge.textContent = `stalled ${mins}m ↻`;
3374              badge.className = 'cookbook-task-status cookbook-task-error';
3375              badge.title = 'Click to retry — resumes where it stopped';
3376              badge.style.cursor = 'pointer';
3377              if (!badge._retryBound) {
3378                badge._retryBound = true;
3379                badge.addEventListener('click', (e) => { e.stopPropagation(); _retryTask(el, task); });
3380              }
3381            } else if (!isPipDep && Date.now() - (el._lastProgressTime || 0) > _STALE_TIMEOUT && !task._autoRestarted) {
3382              task._autoRestarted = true;
3383              _updateTask(task.sessionId, { _autoRestarted: true });

3384              badge.textContent = _startupStalled ? '0% stall — retrying' : 'stale — restarting';
3385              badge.className = 'cookbook-task-status cookbook-task-error';
3386              _showCookbookNotif(true);
3387              try {
3388                await fetch('/api/shell/exec', {
3389                  method: 'POST', credentials: 'same-origin',
3390                  headers: { 'Content-Type': 'application/json' },
3391                  body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${task.sessionId}`) }),
3392                });
3393              } catch {}
3394              try {
3395                // Reuse original payload so the full repo_id (e.g. "Qwen/Qwen3.5-...")
3396                // is preserved — rebuilding from task.repo/task.name drops the org prefix.
3397                const dlPayload = task.payload
3398                  ? { ...task.payload }
3399                  : { repo_id: task.repo || task.name, remote_host: task.remoteHost || '' };
3400                if (_envState.hfToken) dlPayload.hf_token = _envState.hfToken;
3401                // Stalled with hf_transfer — restart on the reliable downloader.
3402                dlPayload.disable_hf_transfer = true;
3403                // Don't overwrite env_prefix — task.payload already has the correct
3404                // "source <path>" form. The bare envPath would miss the `source` and
3405                // the venv never activates (so hf CLI falls off PATH).
3406                const res = await fetch('/api/model/download', {
3407                  method: 'POST', credentials: 'same-origin',
3408                  headers: { 'Content-Type': 'application/json' },
3409                  body: JSON.stringify(dlPayload),
3410                });
3411                const data = await res.json();
3412                if (data.ok && data.session_id) {
3413                  _updateTask(task.sessionId, { sessionId: data.session_id, status: 'running', output: '' });
3414                  task.sessionId = data.session_id;
3415                  el._lastProgress = null;
3416                  el._lastProgressTime = Date.now();
3417                  badge.textContent = 'restarted';
3418                  badge.className = 'cookbook-task-status cookbook-task-running';
3419                  continue;
3420                }
3421              } catch {}
3422              badge.textContent = 'stale — restart failed';
3423              badge.className = 'cookbook-task-status cookbook-task-error';
3424              _showCookbookNotif(true);
3425              break;
3426            }
3427
3428            // When the snapshot includes a shard-of-N marker (e.g.
3429            // "model-00006-of-00082.safetensors"), TRUE overall progress is
3430            // ((shard-1) + currentShardFraction) / totalShards. Before, _dlAgg
3431            // (hf_transfer's per-current-shard aggregate, e.g. 53% of shard 6)
3432            // was treated as overall and the row read "53%" while only 5 of
3433            // 82 shards were actually done.
3434            const _shardPat = [...snapshot.matchAll(/model-(\d+)-of-(\d+)\.(?:safetensors|bin)/g)];
3435            const _lastShard = _shardPat.length ? _shardPat[_shardPat.length - 1] : null;
3436            const _curShardNum = _lastShard ? parseInt(_lastShard[1], 10) : null;
3437            const _totalShards = _lastShard ? parseInt(_lastShard[2], 10) : null;
3438            const _useShardAgg = _curShardNum && _totalShards && _totalShards > 1;
3439
3440            // HF's own "Fetching N files: X%" aggregate counts ALL files,
3441            // including ones already finished in a previous session (resume) —
3442            // so on a resumed download it reflects the true overall progress,
3443            // whereas completed/totalFiles only see this session's files (→ 0%).
3444            // Take the higher of the two so resume doesn't read as 0%.
3445            if (_useShardAgg) {
3446              // Multi-shard download: compute TRUE overall as completed shards
3447              // plus the current shard's fraction. _dlAgg / lastPct represent
3448              // *this shard's* progress, not the whole download.
3449              const curShardFrac = (_dlAgg != null)
3450                ? _dlAgg / 100
3451                : (lastPct ? parseInt(lastPct, 10) / 100 : 0);
3452              let overallPct = Math.round((((_curShardNum - 1) + curShardFrac) / _totalShards) * 100);
3453              if (_fetchPct != null) overallPct = Math.max(overallPct, _fetchPct);
3454              let text = `${overallPct}%`;
3455              if (lastSpeed) text += ` · ${lastSpeed}`;
3456              badge.textContent = text;
3457              badge.className = 'cookbook-task-status cookbook-task-running';
3458            } else if (_dlAgg != null) {
3459              // Real aggregate byte progress — most accurate; take the max of all signals.
3460              let pct = _dlAgg;
3461              if (_fetchPct != null) pct = Math.max(pct, _fetchPct);
3462              let text = `${pct}%`;
3463              if (lastSpeed) text += ` · ${lastSpeed}`;
3464              badge.textContent = text;
3465              badge.className = 'cookbook-task-status cookbook-task-running';
3466            } else if (totalFiles > 0 && completed < totalFiles) {
3467              const curFilePct = lastPct ? parseInt(lastPct) / 100 : 0;
3468              let overallPct = Math.round(((completed + curFilePct) / totalFiles) * 100);
3469              if (_fetchPct != null) overallPct = Math.max(overallPct, _fetchPct);
3470              let text = `${overallPct}%`;
3471              if (lastSpeed) text += ` · ${lastSpeed}`;
3472              badge.textContent = text;
3473              badge.className = 'cookbook-task-status cookbook-task-running';
3474            } else if (_fetchPct != null && _fetchPct < 100) {
3475              // Resume start: only the aggregate is meaningful yet.
3476              let text = `${_fetchPct}%`;
3477              if (lastSpeed) text += ` · ${lastSpeed}`;
3478              badge.textContent = text;
3479              badge.className = 'cookbook-task-status cookbook-task-running';
3480            } else if (completed > 0 && completed >= totalFiles) {
3481              badge.textContent = 'finishing';
3482              badge.className = 'cookbook-task-status cookbook-task-running';
3483            }
3484            if (snapshot.includes('DOWNLOAD_FAILED')) {
3485              // The wrapper prints DOWNLOAD_FAILED but exits 0, and per-file
3486              // "Download complete"/"100%" lines make it look successful — so
3487              // catch the explicit failure marker and handle it.
3488              // A gated/auth failure can NEVER be fixed by retrying (the HF token
3489              // is sent, but its account isn't approved for this repo) — skip the
3490              // auto-retries and surface the gated diagnosis straight away.
3491              const _accessDenied = /Access to model.*is restricted|gated repo|GatedRepoError|401 Unauthorized|403 Forbidden|not in the authorized list|awaiting a review|must (?:be authenticated|have access)/i.test(snapshot);
3492              const _dlKey = task.payload?.repo_id || task.name;
3493              const _dlN = _dlRetryCount.get(_dlKey) || 0;
3494              if (!controller.signal.aborted && !_accessDenied && task.type === 'download' && task.payload && _dlN < _DL_MAX_AUTO_RETRY) {
3495                // Auto-retry: kill the dead session and re-launch (resumes from
3496                // the cached .incomplete files) after a short delay.
3497                _dlRetryCount.set(_dlKey, _dlN + 1);
3498                badge.textContent = `retrying (${_dlN + 1}/${_DL_MAX_AUTO_RETRY})…`;
3499                badge.className = 'cookbook-task-status cookbook-task-running';
3500                uiModule.showToast(`Download interrupted — retrying (${_dlN + 1}/${_DL_MAX_AUTO_RETRY}

3500), resumes where it stopped…`, 6000);
3501                const _p = task.payload, _nm = task.name;
3502                try {
3503                  await fetch('/api/shell/exec', {
3504                    method: 'POST', credentials: 'same-origin',
3505                    headers: { 'Content-Type': 'application/json' },
3506                    body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${task.sessionId}`) }),
3507                  });
3508                } catch {}
3509                _removeTask(task.sessionId);
3510                setTimeout(() => { _retryDownload(_nm, _p); }, 8000);
3511                break;
3512              }
3513              // Out of auto-retries (or not a download) — surface the error; the
3514              // card's Retry button stays available to resume manually.
3515              badge.textContent = _statusLabel('error', task.type);
3516              badge.className = 'cookbook-task-status cookbook-task-error';
3517              _updateTask(task.sessionId, { status: 'error' });
3518              el.dataset.status = 'error';
3519              // Explain a gated/access failure with actionable buttons (request
3520              // access on HF, check token) — otherwise it's just raw red text.
3521              if (_accessDenied) {
3522                const _diag = _diagnose(snapshot);
3523                if (_diag) {
3524                  let diagEl = el.querySelector('.cookbook-diagnosis');
3525                  if (!diagEl) { diagEl = document.createElement('div'); diagEl.className = 'cookbook-diagnosis'; el.appendChild(diagEl); }
3526                  _showDiagnosis(el, _diag, snapshot);
3527                }
3528              }
3529              _showCookbookNotif(true);
3530              break;
3531            }
3532            if (snapshot.includes('DOWNLOAD_OK') || (snapshot.includes('/snapshots/') && completed >= totalFiles && totalFiles > 0)) {
3533              _clearDiagnosis(el);
3534              _dlRetryCount.delete(task.payload?.repo_id || task.name);
3535              badge.textContent = _statusLabel('done', task.type);
3536              badge.className = 'cookbook-task-status cookbook-task-done';
3537              // Flip the type chip from "download" to the green "finished"
3538              // badge so the header reads as completed without a stale label.
3539              const _typeChip = el.querySelector('.cookbook-task-type');
3540              if (_typeChip) { _typeChip.textContent = 'finished'; _typeChip.classList.add('cookbook-task-type-done'); }
3541              _updateTask(task.sessionId, { status: 'done' });
3542              const _sb2 = el.querySelector('.cookbook-task-serve-btn'); if (_sb2) _sb2.style.display = '';
3543              _showCookbookNotif();
3544              _refreshDepsAfterInstall(task);
3545              fetch('/api/shell/exec', {
3546                method: 'POST', credentials: 'same-origin',
3547                headers: { 'Content-Type': 'application/json' },
3548                body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${task.sessionId}`) }),
3549              }).catch(() => {});
3550              _processQueue();
3551              break;
3552            }
3553          }
3554        }
3555
3556        // Live status parsing for serve tasks — uses shared _parseServePhase
3557        if (task.type === 'serve') {
3558          const badge = el.querySelector('.cookbook-task-status');
3559          if (badge) {
3560            const info = _parseServePhase(snapshot);
3561            if (info.status === 'ready' && !task._serveReady) {
3562              task._serveReady = true;
3563              _updateTask(task.sessionId, { _serveReady: true });
3564              // The auto-registered endpoint was marked offline while the
3565              // server was coming up. Now that it's reachable, nudge the
3566              // picker to re-probe so the offline pill clears without the
3567              // user having to reopen Settings or refresh the page.
3568              try { _refreshModelsAfterEndpointChange(); } catch (_) {}
3569            }
3570            if (info.phase) {
3571              badge.textContent = info.phase;
3572              // Always the green "running" style — loading/warming is the same
3573              // state, just with dynamic text (don't switch to a neutral style).
3574              badge.className = 'cookbook-task-status cookbook-task-running';
3575              // Live output reporting 'ready' is direct proof the server is up —
3576              // clear a stale "unreachable" flag here too. The HTTP probe c

3576an lag,
3577              // miss a remote endpoint, or cache a down result, leaving the card
3578              // stuck red even after the server recovered ("doesn't recheck").
3579              if (info.status === 'ready' && task._unreachable) {
3580                task._unreachable = false;
3581                _updateTask(task.sessionId, { _unreachable: false });
3582                el.classList.remove('cookbook-task-unreachable');
3583                _refreshServerDots();
3584              }
3585              // Persist the loading phase so a re-render keeps showing "loading 45%"
3586              // instead of resetting the badge to the generic "running". Clear it
3587              // once ready so the badge falls back to "running".
3588              if (info.status !== 'ready') {
3589                if (task.progress !== info.phase) _updateTask(task.sessionId, { progress: info.phase });
3590              } else if (task.progress) {
3591                _updateTask(task.sessionId, { progress: '' });
3592              }
3593            }
3594          }
3595        }
3596
3597        // Run error diagnosis on serve tasks
3598        const diag = _diagnose(snapshot);
3599        if (diag) {
3600          let diagEl = el.querySelector('.cookbook-diagnosis');
3601          if (!diagEl) {
3602            diagEl = document.createElement('div');
3603            diagEl.className = 'cookbook-diagnosis';
3604            el.appendChild(diagEl);
3605          }
3606          _showDiagnosis(el, diag, snapshot);
3607        }
3608        // Detect serve ready — auto-add to model endpoints. Don't flip
3609        // `_endpointAdded` until the POST succeeds; otherwise a transient
3610        // error silently prevents any future retry. An in-flight guard
3611        // prevents a second poll from firing a duplicate POST before the
3612        // first one's dedup check can observe the newly-added row.
3613        if (task.type === 'serve' && !task._endpointAdded && !task._endpointAddInFlight && task._serveReady) {
3614          task._endpointAddInFlight = true;
3615          let host = _connectHostFromRemote(task.remoteHost);
3616          const portMatch = task.payload?._cmd?.match(/--port[=\s]+(\d+)/)
3617            || task.payload?._cmd?.match(/(?:^|\s)-p[=\s]+(\d+)/)
3618            || snapshot.match(/Uvicorn running on\D*?:(\d+)/i)
3619            || snapshot.match(/running on\D*?:(\d+)/i)
3620            || snapshot.match(/listening on\D*?:(\d+)/i)
3621            || snapshot.match(/port[:=\s]+(\d+)/i);
3622          let port = portMatch ? portMatch[1] : '8000';
3623          let baseUrl = `http://${host}:${port}/v1`;
3624          const ollamaUrlMatch = snapshot.match(/Ollama API ready on port\s+\d+:\s*(http:\/\/[^\s]+)/i);
3625          if (ollamaUrlMatch) {
3626            const endpoint = _endpointFromAdvertisedUrl(ollamaUrlMatch[1], host, '11434');
3627            if (endpoint) ({ host, port, baseUrl } = endpoint);
3628          }
3629          fetch('/api/model-endpoints', { credentials: 'same-origin' })
3630            .then(r => r.json())
3631            .then(async (eps) => {
3632              // Match only exact base_url — don't dedup by friendly name,
3633              // because other endpoints may happen to share a model name.
3634              const exists = eps.some(e => e.base_url === baseUrl);
3635              if (exists) {
3636                // Already registered — e.g. the backend pre-registers diffusion
3637                // endpoints server-side. Mark so we don't retry, but STILL
3638                // refresh the picker (and probe until online) so the new model
3639                // shows up without the user having to manually refresh.
3640                const _ex = eps.find(e => e.base_url === baseUrl);
3641                if (_ex && !_endpointMatchesServe(_ex, task)) {
3642                  _markServeEndpointMismatch(task, _ex, host, port);
3643                  return null;
3644                }
3645                task._endpointAdded = true;
3646                _updateTask(task.sessionId, { _endpointAdded: true });
3647                _autoSaveWorkingConfig(task);   // endpoint live → remember these settings
3648                if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(false);
3649                if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
3650                window.dispatchEvent(new CustomEvent('ge:model-endpoints-updated', { detail: { baseUrl, host, port, model: task.name } }));
3651                if (_ex && _ex.id && !(_ex.models || []).length) _probeEndpointUntilOnline(_ex.id, host, port);
3652                return null;
3653              }
3654              const _isDiffusion = _isImageServeTask(task);
3655              const fd = new FormData();
3656              fd.append('base_url', baseUrl);
3657              fd.append('name', task.name);
3658              fd.append('skip_probe', 'true');
3659              _appendCookbookEndpointScope(fd, task.remoteHost || '');
3660              _appendPinnedServeModel(fd, task);
3661              if (_isDiffusion) fd.append('model_type', 'image');
3662              return fetch('/api/model-endpoints', { method: 'POST', credentials: 'same-origin', body: fd });
3663            })
3664            .then(async (res) => {
3665              if (res && res.ok) {
3666                // Flip the flag only on confirmed success
3667                task._endpointAdded = true;
3668                _updateTask(task.sessionId, { _endpointAdded: true });
3669                _autoSaveWorkingConfig(task);   // endpoint live → remember these settings
3670                uiModule.showToast(`Model endpoint added: ${host}:${port}`);
3671                // Retry-probe until the warming server answers, so it
3672                // flips online without a manual enable/disable toggle.
3673                const _epData = await res.json().catch(() => ({}));
3674                if (_epData && _epData.id && !(_epData.models || []).length) {
3675                  _probeEndpointUntilOnline(_epData.id, host, port);
3676                }
3677                window.dispatchEvent(new CustomEvent('ge:model-endpoints-updated', { detail: { baseUrl, host, port, model: task.name } }));
3678                const _trySelectModel = async (attempt) => {
3679                  if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(false);
3680                  const items = window.modelsModule?.getCachedItems?.() || [];
3681                  for (const item of items) {
3682                    if (item.offline) continue;
3683                    const url = item.url || '';
3684                    if (url.includes(host) || url.includes(port)) {
3685                      const mid = (item.models || [])[0];
3686                      if (mid && window.sessionModule?.createDirectChat) {
3687                        window.sessionModule.createDirectChat(url, mid, item.endpoint_id);
3688                        if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
3689                        uiModule.showToast(`Switched to ${mid.split('/').pop()}`);
3690                        return;
3691                      }
3692                    }
3693                  }
3694                  if (attempt < 3) setTimeout(() => _trySelectModel(attempt + 1), 2000);
3695                  else if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
3696                };
3697                setTimeout(() => _trySelectModel(0), 1000);
3698              } else if (res && !res.ok) {
3699                const body = await res.text().catch(() => '');
3700                console.warn('Endpoint auto-add failed', res.status, body);
3701                uiModule.showError(`Auto-register endpoint failed (${res.status}). Use ⋮ → Register endpoint to retry.`);
3702              }
3703            })
3704            .catch((e) => {
3705              console.warn('Endpoint auto-add error', e);
3706              uiModule.showError(`Auto-register endpoint error: ${e.message || e}. Use ⋮ → Register endpoint to retry.`);
3707            })
3708            .finally(() => { task._endpointAddInFlight = false; });
3709          _updateTask(task.sessionId, { status: 'running' });
3710          const badge = el.querySelector('.cookbook-task-status');
3711          if (badge) { badge.textContent = 'running'; badge.className = 'cookbook-task-status cookbook-task-running'; }
3712          _showCookbookNotif();
3713        }
3714        // Detect process exit

3715        if (snapshot.includes('=== Process exited with code')) {
3716          const codeMatch = snapshot.match(/=== Process exited with code (\d+)/);
3717          const code = codeMatch ? parseInt(codeMatch[1]) : -1;
3718          // Serve tasks that exit without reaching ready state are always errors —
3719          // a serve process should run indefinitely
3720          const status = (task.type === 'serve' && !task._serveReady) ? 'error'
3721            : (code === 0 ? 'done' : 'error');
3722          _updateTask(task.sessionId, { status });
3723          const badge = el.querySelector('.cookbook-task-status');
3724          if (badge) { badge.textContent = status; badge.className = `cookbook-task-status cookbook-task-${status}`; }
3725          _renderRunningTab();
3726        }
3727        _updateTask(task.sessionId, { output: snapshot.slice(-5000) });
3728      }
3729    } catch {
3730      failCount++;
3731      if (failCount > 10) break;
3732      await new Promise(r => setTimeout(r, 10000));
3733      continue;
3734    }
3735
3736    failCount = 0;
3737    await new Promise(r => setTimeout(r, TASK_POLL_INTERVAL_MS));
3738  }
3739}
3740

3741// ── Background monitor ──
3742
3743let _bgMonitorInterval = null;
3744let _bgPollInFlight = false;
3745const BG_LEADER_KEY = 'odysseus-cookbook-bg-leader';
3746const BG_LEADER_ID = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
3747const BG_LEADER_TTL_MS = 15000;
3748
3749function _hasLiveTasks(tasks = null) {
3750  const list = tasks || _loadTasks();
3751  return list.some(t =>
3752    t.status === 'running'
3753    || t.status === 'queued'
3754    || t.status === 'ready'
3755    || _downloadOutputLooksActive(t)
3756  );
3757}
3758
3759function _isRunningTabVisible() {
3760  const modal = document.getElementById('cookbook-modal');
3761  if (!modal || modal.classList.contains('hidden')) return false;
3762  const activeTab = modal.querySelector('.cookbook-tab.active')?.dataset?.backend || '';
3763  return activeTab === 'Running';
3764}
3765
3766function _isCookbookVisible() {
3767  try {
3768    if (window.cookbookModule && typeof window.cookbookModule.isVisible === 'function') {
3769      return !!window.cookbookModule.isVisible();
3770    }
3771  } catch (_) {}
3772  const modal = document.getElementById('cookbook-modal');
3773  return !!modal && !modal.classList.contains('hidden');
3774}
3775
3776function _foregroundChatBusy() {
3777  try {
3778    return !!window.__odysseusChatBusy || Date.now() < (window.__odysseusChatBusyUntil || 0);
3779  } catch {
3780    return false;
3781  }
3782}
3783
3784function _claimBackgroundLeader() {
3785  if (document.visibilityState !== 'visible') return false;
3786  const now = Date.now();
3787  try {
3788    const raw = localStorage.getItem(BG_LEADER_KEY);
3789    const current = raw ? JSON.parse(raw) : null;
3790    if (
3791      !current
3792      || !current.id
3793      || current.id === BG_LEADER_ID
3794      || now - Number(current.ts || 0) > BG_LEADER_TTL_MS
3795    ) {
3796      localStorage.setItem(BG_LEADER_KEY, JSON.stringify({ id: BG_LEADER_ID, ts: now }));
3797      return true;
3798    }
3799    return current.id === BG_LEADER_ID;
3800  } catch (_) {
3801    return true;
3802  }
3803}
3804
3805function _canBackgroundPoll() {
3806  if (_foregroundChatBusy()) return false;
3807  if (document.visibilityState !== 'visible') return false;
3808  return _claimBackgroundLeader();
3809}
3810
3811// Reachability check for running serve tasks. The tmux pane can stay alive
3812// while the model server inside it has crashed (so no "Process exited" line
3813// ever appears) — leaving the card showing "running" forever. So we actively
3814// probe the registered endpoint (same /probe-local the model picker uses) and
3815// flag the card "unreachable" (red) when the server stops answering.
3816let _serveReachabilityInFlight = false;
3817let _serveReachabilityLastAt = 0;
3818async function _checkServeReachability() {
3819  // This reaches out to local model servers. Keep it out of the normal chat
3820  // path unless the user is actively looking at the Running tab.
3821  if (_foregroundChatBusy()) return;
3822  if (!_isRunningTabVisible()) return;
3823  const now = Date.now();
3824  if (_serveReachabilityInFlight || now - _serveReachabilityLastAt < 10000) return;
3825  _serveReachabilityInFlight = true;
3826  _serveReachabilityLastAt = now;
3827  let serveTasks;
3828  try {
3829    serveTasks = _loadTasks().filter(t => t.type === 'serve' && t.status === 'running');
3830  } catch {
3831    _serveReachabilityInFlight = false;
3832    return;
3833  }
3834  if (!serveTasks.length) {
3835    _serveReachabilityInFlight = false;
3836    return;
3837  }
3838  let eps = [], probe = {};
3839  try {
3840    [eps, probe] = await Promise.all([
3841      fetch('/api/model-endpoints', { credentials: 'same-origin' }).then(r => r.json()).catch(() => []),
3842      fetch('/api/model-endpoints/probe-local', { credentials: 'same-origin' }).then(r => r.json()).catch(() => ({})),
3843    ]);
3844    for (const task of serveTasks) {
3845      const host = _connectHostFromRemote(task.remoteHost);
3846      const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
3847      const port = portMatch ? portMatch[1] : '8000';
3848      const baseUrl = `http://${host}:${port}/v1`;
3849      const ep = (eps || []).find(e => e.base_url === baseUrl);
3850      if (!ep) continue;                       // not registered yet — can't judge
3851      const pr = probe[ep.id];
3852      if (!pr || pr.alive === undefined) continue;  // not probed (non-local) — skip
3853      // Record the first time it actually answers. Until then the server is still
3854      // LOADING/warming (the endpoint can get registered on the 300s timeout for a
3855      // big model that hasn't finished loading), and a not-yet-answering server is
3856      // not "unreachable" — flagging it as such while you're launching is a fal

3856se
3857      // alarm. Only treat it as unreachable once it has been reachable at least once.
3858      if (pr.alive === true && !task._everReachable) {
3859        task._everReachable = true;
3860        _updateTask(task.sessionId, { _everReachable: true });
3861      }
3862      const unreachable = pr.alive === false;
3863      if (unreachable && !task._everReachable) continue;  // still coming up, not crashed
3864      if (!!task._unreachable !== unreachable) {
3865        _updateTask(task.sessionId, { _unreachable: unreachable });
3866      }
3867      const el = document.querySelector(`.cookbook-task[data-task-id="${task.sessionId}"]`);
3868      if (el) {
3869        el.classList.toggle('cookbook-task-unreachable', unreachable);
3870        const badge = el.querySelector('.cookbook-task-status');
3871        if (badge) {
3872          if (unreachable) {
3873            badge.textContent = 'unreachable';
3874            badge.className = 'cookbook-task-status cookbook-task-error';
3875            badge.title = pr.error || 'Server not responding — it may have crashed';
3876          } else if (badge.textContent === 'unreachable') {
3877            // Recovered — restore the normal running label.
3878            badge.textContent = _statusLabel('running', task.type);
3879            badge.className = 'cookbook-task-status cookbook-task-running';
3880            badge.title = '';
3881          }
3882        }
3883      }
3884      if (unreachable) _showCookbookNotif(true);
3885    }
3886    _refreshServerDots();
3887  } catch {
3888    // Non-fatal: the normal task status poll continues separately.
3889  } finally {
3890    _serveReachabilityInFlight = false;
3891  }
3892}
3893
3894function _serveTaskFailed(task) {
3895  if (!task || task.type !== 'serve') return false;
3896  return !!task._unreachable || ['error', 'crashed', 'failed'].includes(task.status);
3897}
3898
3899function _setServerDot(dot, failed, title) {
3900  if (!dot) return;
3901  dot.classList.toggle('fail', !!failed);
3902  dot.classList.toggle('ok', !failed);
3903  dot.title = title;
3904}
3905
3906function _syncSettingsServerDots(byKey) {
3907  document.querySelectorAll('.cookbook-server-entry').forEach(entry => {
3908    const hostEl = entry.querySelector('.cookbook-srv-host');
3909    const dot = entry.querySelector('.cookbook-srv-status');
3910    const msg = entry.querySelector('.cookbook-srv-test-msg');
3911    if (!hostEl || !dot) return;
3912
3913    const host = hostEl.value?.trim() || '';
3914    if (!host || hostEl.readOnly || hostEl.disabled) {
3915      _setServerDot(dot, false, 'Local (this machine)');
3916      return;
3917    }
3918
3919    const list = byKey[host] || [];
3920    if (!list.length) return;
3921
3922    const failed = list.some(_serveTaskFailed);
3923    _setServerDot(dot, failed, failed ? 'Server not responding - running serve may have crashed' : 'Reachable');
3924    if (!msg) return;
3925
3926    if (failed) {
3927      msg.textContent = 'Server not responding';
3928      msg.title = 'Server not responding - running serve may have crashed';
3929      msg.style.color = 'var(--red,#e06c75)';
3930      msg.style.opacity = '0.75';
3931    } else if (/failed|crashed|not responding|unreachable/i.test(msg.textContent || '')) {
3932      msg.textContent = 'Reachable';
3933      msg.title = 'Reachable';
3934      msg.style.color = 'var(--green,#50fa7b)';
3935      msg.style.opacity = '0.75';
3936    }
3937  });
3938}
3939
3940// Keep each server section's status dot (green ↔ red) in sync with the live

3941// health of its SERVE tasks. The header dot is only built once, so without
3942// this it got stuck on its first value. Downloads never count because they have
3943// no endpoint to be "unreachable".
3944function _refreshServerDots() {
3945  let tasks;
3946  try { tasks = _loadTasks(); } catch { return; }
3947  const byKey = {};
3948  const _taskServerKeyForDot = (task) => task?.remoteServerKey || task?.remoteHost || '';
3949  for (const t of tasks) { (byKey[_taskServerKeyForDot(t)] = byKey[_taskServerKeyForDot(t)] || []).push(t); }
3950  document.querySelectorAll('.cookbook-section-header').forEach(header => {
3951    const dot = header.querySelector('.cookbook-srv-status');
3952    if (!dot) return;
3953    const key = header.querySelector('[data-stop-server]')?.dataset.stopServer || '';
3954    const list = byKey[key] || [];
3955    const fail = !!key && list.some(_serveTaskFailed);
3956    _setServerDot(dot, fail, key ? (fail ? 'Server not responding' : 'Reachable') : 'Local (this machine)');
3957  });
3958  _syncSettingsServerDots(byKey);
3959}
3960
3961// Self-heal: scan persisted download tasks marked done/error/crashed and
3962// check whether their tmux session is still alive on the host. If yes —
3963// the task isn't actually finished, the cookbook just lost the in-flight
3964// status during restart — flip status back to 'running' so _reconnectTask
3965// picks it up. The one-shot guard is enforced by callers (open path) or
3966// time-throttled inside (background-monitor path).
3967let _selfHealRan = false;
3968let _selfHealLastTs = 0;
3969export async function _selfHealStaleTasks(opts = {}) {
3970  // Open-path call: one-shot per page load.
3971  if (opts.oneShot) {
3972    if (_selfHealRan) return;
3973    _selfHealRan = true;
3974  } else {
3975    // Background-monitor call: throttle to once every 8s (the bg monitor
3976    // itself fires every 10s, so this almost always fires too, but the
3977    // guard keeps a fast manual call from doubling up).
3978    const now = Date.now();
3979    if (now - _selfHealLastTs < 4000) return;
3980    _selfHealLastTs = now;
3981  }
3982  const tasks = _loadTasks();
3983  const candidates = tasks.filter(t => {
3984    if (t.type !== 'download') return false;
3985    if (!['done', 'error', 'crashed', 'stopped'].includes(t.status)) return false;
3986    if (!t.sessionId || String(t.sessionId).startsWith('queue-')) return false;
3987    // Finished downloads with strong completion markers (DOWNLOAD_OK or HF
3988    // /snapshots/ resolution) are demonstrably done — do not flip them back
3989    // to running just because the tmux session is still alive (e.g., a
3990    // long-lived shell that hosted the download or a flapping SSH that
3991    // reports the session as up). This was the main source of finished↔
3992    // downloading oscillation on a flaky connection.
3993    if (t.status === 'done' && /DOWNLOAD_OK|\/snapshots\//.test(t.output || '')) return false;
3994    // Cooldown: never flip the same task more than once every 45s. A flapping
3995    // SSH connection used to drive the badge back-and-forth on every probe
3996    // cycle; this enforces a stable view between flaps.
3997    if (t._lastStatusFlipAt && (Date.now() - t._lastStatusFlipAt < 45000)) return false;
3998    return true;
3999  });
4000  if (!candidates.length) return;
4001  let flipped = 0;
4002  for (const t of candidates) {
4003    try {
4004      const res = await fetch('/api/shell/exec', {
4005        method: 'POST', credentials: 'same-origin',
4006        headers: { 'Content-Type': 'application/json' },
4007        body: JSON.stringify({ command: _tmuxCmd(t, `has-session -t ${t.sessionId}`), timeout: 5 }),
4008      });
4009      const data = await res.json();
4010      if (data.exit_code === 0) {
4011        // Session still alive → the task is actually still running.
4012        const fresh = _loadTasks();
4013        const ft = fresh.find(x => x.sessionId === t.sessionId);
4014        if (ft && ft.status !== 'running') {
4015          ft.status = 'running';
4016          ft._selfHealed = true;
4017          ft._lastStatusFlipAt = Date.now();
4018          _saveTasks(fresh);
4019          flipped++;
4020          const _el = document.querySelector(`.cookbook-task[data-task-id="${t.sessionId}"]`);
4021          if (_el) {
4022            const _chk = _el.querySelector('.cookbook-task-check');
4023            if (_chk) _chk.style.display = 'none';
4024            const _wave = _el.querySelector('.cookbook-task-wave');
4025            if (_wave) _wave.style.display = '';
4026            const _up = _el.querySelector('.cookbook-task-uptime');
4027            if (_up) _up.style.display = '';
4028            _el.dataset.status = 'running';
4029          }
4030        }
4031      }
4032    } catch { /* network blip — skip this one */ }
4033  }
4034  if (flipped) {
4035    console.log(`[cookbook] auto-reconnect: revived ${flipped} task(s) whose tmux session was still alive`);
4036    _renderRunningTab();
4037  }
4038}
4039

4040export function _startBackgroundMonitor() {
4041  if (_bgMonitorInterval) return;
4042  _bgMonitorInterval = setInterval(() => {
4043    if (!_canBackgroundPoll()) return;
4044    _pollBackgroundStatus();
4045    _checkServeReachability();
4046    // Auto-reconnect: every cycle, look for download tasks marked finished/
4047    // crashed/etc. whose tmux session is actually still running, and flip
4048    // them back to running. Internally throttled to 8s so a manual call from
4049    // the open path or a fast invocation doesn't double up.
4050    if (_hasLiveTasks() || _isRunningTabVisible()) {
4051      _selfHealStaleTasks().catch(() => {});
4052    }
4053  }, BG_MONITOR_INTERVAL_MS);
4054  if (_canBackgroundPoll()) {
4055    _pollBackgroundStatus();
4056    _checkServeReachability();
4057  }
4058}
4059
4060function _stopBackgroundMonitor() {
4061  if (_bgMonitorInterval) {
4062    clearInterval(_bgMonitorInterval);
4063    _bgMonitorInterval = null;
4064  }
4065  const statusEl = document.getElementById('cookbook-bg-status');
4066  if (statusEl) statusEl.style.display = 'none';
4067}
4068
4069// Retry-probe a freshly-added endpoint until its model server answers.
4070// A model that just reached "ready" in the cookbook often can't satisfy
4071// the 1s add-time probe (remote, weights still mmap-ing), so it's added
4072// offline. This polls the per-endpoint /probe (which uses a longer
4073// server-side timeout + persists cached_models) every few seconds until
4074// the endpoint reports models, then refreshes the picker. Bounded so a
4075// genuinely-dead server doesn't poll forever.
4076async function _probeEndpointUntilOnline(epId, host, port) {
4077  if (!_isCookbookVisible() || _foregroundChatBusy()) return;
4078  if (!epId) return;
4079  // Big models (e.g. 70B+) can take several minutes to load weights before
4080  // the server answers /v1/models. Probe for up to ~5 min, easing the
4081  // interval out so we're not hammering during a long warmup.
4082  const MAX_TRIES = 40;
4083  for (let i = 0; i < MAX_TRIES; i++) {
4084    const interval = i < 12 ? 5000 : 10000;   // 5s for the first minute, then 10s
4085    await new Promise(r => setTimeout(r, interval));
4086    if (!_isCookbookVisible() || _foregroundChatBusy()) return;
4087    try {
4088      // Hit the probe endpoint — it re-probes server-side and updates
4089      // cached_models. We consume (and discard) the SSE stream.
4090      const probeRes = await fetch(`/api/model-endpoints/${epId}/probe`, { credentials: 'same-origin' }).catch(() => null);
4091      if (probeRes && probeRes.status === 404) return;
4092      if (probeRes) await probeRes.text().catch(() => {});
4093      const eps = await fetch('/api/model-endpoints', { credentials: 'same-origin' }).then(r => r.json()).catch(() => []);
4094      const ep = (eps || []).find(e => e.id === epId);
4095      if (ep && (ep.models || []).length) {
4096        if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(false);
4097        if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
4098        window.dispatchEvent(new CustomEvent('ge:model-endpoints-updated', {
4099          detail: { baseUrl: ep.base_url || `http://${host}:${port}/v1`, host, port, model: (ep.models || [])[0] || '' },
4100        }));
4101        uiModule.showToast(`${host}:${port} is online`);
4102        return;
4103      }
4104    } catch (_) { /* keep retrying */ }
4105  }
4106}
4107
4108async function _pollBackgroundStatus() {
4109  if (!_canBackgroundPoll() || _bgPollInFlight) return;
4110  _bgPollInFlight = true;
4111  try {
4112    // Pull any tasks the server knows about that aren't in localStorage
4113    // yet (e.g. agent-spawned downloads/serves). Without this merge,
4114    // _syncToServer keeps clobbering server-added tasks on every poll.
4115    try {
4116      const stateRes = await fetch('/api/cookbook/state', { credentials: 'same-origin' });
4117      if (stateRes.ok) {
4118        const serverState = await stateRes.json();
4119        const serverTasks = (serverState && Array.isArray(serverState.tasks)) ? serverState.tasks : [];
4120        if (serverTasks.length) {
4121          const localTasks = _loadTasks();
4122          const localIds = new Set(localTasks.map(t => t.sessionId));
4123          const merged = [...localTasks];
4124          let added = 0;
4125          for (const t of serverTasks) {
4126            if (t && t.sessionId && !localIds.has(t.sessionId) && !_isTombstoned(t.sessionId)) {
4127              merged.push(t);
4128              added++;
4129            }
4130          }
4131          if (added > 0) {
4132            localStorage.setItem(TASKS_KEY, JSON.stringify(merged.map(_redactTaskForStorage)));
4133            _renderRunningTab();
4134          }
4135        }
4136      }
4137    } catch (_) { /* non-fatal */ }
4138
4139    const res = await fetch('/api/cookbook/tasks/status', { credentials: 'same-origin' });
4140    if (!res.ok) return;
4141    const data = await res.json();
4142    const tasks = data.tasks || [];
4143
4144    // Reconcile the authoritative tmux/process status back into the persisted
4145    // client task list. The Running-tab reconnect loop also does this, but it
4146    // only exists while cards are rendered; after a page refresh or closed modal
4147    // dependency installs could finish server-side while localStorage stayed
4148    // stuck at "running".
4149    try {
4150      const statusById = new Map(tasks.map(t => [t.session_id, t]));
4151      const localTasks = _loadTasks();
4152      let changed = false;

4153      const completedDeps = [];
4154      const localIds = new Set(localTasks.map(t => t.sessionId).filter(Boolean));
4155      for (const live of tasks) {
4156        const sid = live?.session_id;
4157        if (!sid || localIds.has(sid) || _isTombstoned(sid)) continue;
4158        const liveType = live.type || 'download';
4159        const liveStatus = live.status === 'completed' ? 'done' : (live.status || 'running');
4160        const name = live.model || sid;
4161        const remoteHost = live.remote && live.remote !== 'local' ? live.remote : '';
4162        localTasks.push(_redactTaskForStorage({
4163          id: sid,
4164          sessionId: sid,
4165          name,
4166          type: liveType,
4167          status: liveStatus,
4168          progress: live.progress || '',
4169          output: live.output_tail || '',
4170          ts: Date.now(),
4171          payload: {
4172            repo_id: name,
4173            remote_host: remoteHost,
4174            _cmd: live.cmd || '(adopted from live tmux status)',
4175          },
4176          remoteHost,
4177          _adoptedExternally: true,
4178        }));
4179        localIds.add(sid);
4180        changed = true;
4181      }
4182      for (const task of localTasks) {
4183        const live = statusById.get(task.sessionId);
4184        if (!live) continue;
4185        const updates = {};
4186        // A finished dependency install whose tmux pane is gone is reported
4187        // "stopped" by the backend (its pip package is never in the HF cache the
4188        // dead-session check inspects). Recover "done" from the retained output's
4189        // exit-0 sentinel so a clean install isn't downgraded to crashed.
4190        const combinedOutput = `${task.output || ''}\n${live.output_tail || ''}`;
4191        const depDone = !!task.payload?._dep && _depInstallSucceeded(combinedOutput);
4192        // A finished model download whose tmux pane is gone is also reported
4193        // "stopped" (the dead-session check can miss the landed snapshot).
4194        // Recover "done" from the terminal `DOWNLOAD_OK` sentinel — emitted
4195        // only after the runner exits 0 — so a completed download isn't
4196        // downgraded to crashed. This background poll runs blind (no live
4197        // stream to debounce against), so unlike the reconnect loop it keys
4198        // off the conclusive exit sentinel only, never the `/snapshots/` path,
4199        // which can be printed mid-stream for multi-file downloads.
4200        const downloadDone = task.type === 'download'
4201          && String(combinedOutput || '').includes('DOWNLOAD_OK');
4202        const serveReady = task.type === 'serve'
4203          && (live.status === 'ready' || _serveOutputLooksReady({ ...task, output: live.output_tail || task.output || '' }));
4204        const completedByOutput = depDone || downloadDone;
4205        const nextStatus = completedByOutput
4206          ? 'done'
4207          : (serveReady
4208          ? 'ready'
4209          : (live.status === 'completed'
4210          ? 'done'
4211          : (live.status === 'error'
4212            ? 'error'
4213            : (live.status === 'stopped'
4214                ? ((depDone || downloadDone) ? 'done' : (task.type === 'download' ? 'crashed' : 'stopped'))
4215                : null))));
4216        if (nextStatus && task.status !== nextStatus) {
4217          updates.status = nextStatus;
4218          if (nextStatus === 'done' && task.payload?._dep) completedDeps.push(task);
4219        }
4220        if (serveReady && !task._serveReady) {
4221          updates._serveReady = true;
4222        }
4223        if ((live.status === 'running' || live.status === 'ready') && task.status !== live.status && !serveReady && !completedByOutput) {
4224          updates.status = live.status === 'ready' ? 'ready' : 'running';
4225        }
4226        if (live.progress && live.progress !== task.progress) updates.progress = live.progress;
4227        if (live.exit_code != null && live.exit_code !== task.exit_code) updates.exit_code = live.exit_code;
4228        if (live.output_tail) {
4229          const previous = String(task.output || '');
4230          const tail = String(live.output_tail || '');
4231          if (tail && !previous.endsWith(tail)) {
4232            updates.output = _isServeOutputPlaceholder(previous)
4233              ? tail.slice(-5000)
4234              : `${previous ? `${previous}\n` : ''}${tail}`.slice(-5000);
4235          }
4236        }
4237        if (live.diagnosis && !task._diagnosisDismissed) {
4238          updates._backendDiagnosis = live.diagnosis;
4239        }
4240        if (live.cmd && !task.payload?._cmd) {
4241          updates.payload = { ...(task.payload || {}), _cmd: live.cmd };
4242        }
4243        if (Object.keys(updates).length) {
4244          Object.assign(task, updates);
4245          changed = true;
4246        }
4247      }
4248      if (changed) {
4249        _saveTasks(localTasks);
4250        _renderRunningTab();
4251        for (const task of localTasks) {
4252          if (!task._backendDiagnosis) continue;
4253          const el = document.querySelector(`[data-session-id="${CSS.escape(task.sessionId)}"]`);
4254          if (!el || el.querySelector('.cookbook-diagnosis')) continue;
4255          _showDiagnosis(el, task._backendDiagnosis, task.output || '');
4256        }
4257        completedDeps.forEach(t => _refreshDepsAfterInstall(t));
4258      }
4259    } catch (_) { /* non-fatal: background status should never break polling */ }
4260
4261    const statusEl = document.getElementById('cookbook-bg-status');
4262    const activeTasks = tasks.filter(t => t.status === 'running' || t.status === 'ready');
4263    const errorTasks = tasks.filter(t => t.status === 'error');
4264    const completedTasks = tasks.filter(t => t.status === 'completed');
4265
4266    // Auto-add serve endpoints that became ready (works even when modal is closed)
4267    const readyServes = tasks.filter(t => t.type === 'serve' && t.status === 'ready');
4268    for (const t of readyServes) {
4269      const localTasks = _loadTasks();
4270      const localTask = localTasks.find(lt => lt.sessionId === t.session_id);
4271
4272      let host = _connectHostFromRemote(localTask?.remoteHost || t.remote);

4273      const portMatch = localTask?.payload?._cmd?.match(/--port\s+(\d+)/)
4274        || localTask?.payload?._cmd?.match(/OLLAMA_HOST=[^\s:]+:(\d+)/);
4275      let port = portMatch ? portMatch[1] : '8000';
4276      let baseUrl = `http://${host}:${port}/v1`;
4277      const snapshot = t.output || localTask?.output || '';
4278      const ollamaUrlMatch = snapshot.match(/Ollama API ready on port\s+\d+:\s*(http:\/\/[^\s]+)/i);
4279      if (ollamaUrlMatch) {
4280        const endpoint = _endpointFromAdvertisedUrl(ollamaUrlMatch[1], host, '11434');
4281        if (endpoint) ({ host, port, baseUrl } = endpoint);
4282      }
4283      const _isDiffusion = _isImageServeTask(localTask);
4284
4285      _updateTask(t.session_id, { _serveReady: true });
4286      if (localTask) _autoSaveWorkingConfig(localTask);   // remember working settings (modal may be closed)
4287
4288      // Auto-detect function-calling support from the serve cmd.
4289      // vLLM emits OpenAI-style tool_calls only when launched with
4290      // `--enable-auto-tool-choice`; local-only models otherwise
4291      // hallucinate a fake [TOOL_CALL]...[/TOOL_CALL] text format
4292      // the backend can't parse.
4293      const _cmd = localTask?.payload?._cmd || '';
4294      const _supportsTools = _cmd.includes('--enable-auto-tool-choice') || _isDiffusion === false && /(?:^|\s)(?:deepseek|gpt-[45o]|claude|gemini|qwen3|qwen2\.5|mixtral|llama-[34]|minimax|kimi|hermes|glm-4)/i.test(t.model);
4295
4296      fetch('/api/model-endpoints', { credentials: 'same-origin' })
4297        .then(r => r.json())
4298        .then(eps => {
4299          const hostPort = `${host}:${port}`;
4300          const existing = eps.find(e => e.base_url === baseUrl || e.base_url.includes(hostPort) || e.name === t.model);
4301          if (existing) {
4302            const taskForMatch = localTask || { sessionId: t.session_id, name: t.model, model: t.model, payload: { repo_id: t.model, _cmd } };
4303            if (!_endpointMatchesServe(existing, taskForMatch)) {
4304              _markServeEndpointMismatch(taskForMatch, existing, host, port);
4305              return null;
4306            }
4307            _updateTask(t.session_id, { _endpointAdded: true });
4308            // Already registered — but it may be showing offline because
4309            // it was added while the server was still warming. Kick a
4310            // re-probe so it flips online without manual toggle.
4311            if (!(existing.models || []).length) _probeEndpointUntilOnline(existing.id, host, port);
4312            return null;
4313          }
4314          const fd = new FormData();
4315          fd.append('base_url', baseUrl);
4316          fd.append('name', t.model);
4317          fd.append('skip_probe', 'true');
4318          _appendCookbookEndpointScope(fd, localTask?.remoteHost || t.remote || '');
4319          _appendPinnedServeModel(fd, localTask || { name: t.model, model: t.model, payload: { repo_id: t.model, _cmd } });
4320          if (_isDiffusion) fd.append('model_type', 'image');
4321          if (_supportsTools) fd.append('supports_tools', 'true');
4322          return fetch('/api/model-endpoints', { method: 'POST', credentials: 'same-origin', body: fd });
4323        })
4324        .then(async (res) => {
4325          if (res && res.ok) {
4326            _updateTask(t.session_id, { _endpointAdded: true });
4327            uiModule.showToast(`Model endpoint added: ${host}:${port}`);
4328            const data = await res.json().catch(() => ({}));
4329            // A just-started server often can't answer the 1s add-time
4330            // probe, so it lands "offline". Retry-probe in the background
4331            // until /v1/models responds — no manual enable/disable needed.
4332            if (data && data.id) _probeEndpointUntilOnline(data.id, host, port);
4333            if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(false);
4334            if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
4335          }
4336        })
4337        .catch(() => {});
4338    }
4339
4340    if (errorTasks.length > 0) {
4341      _showCookbookNotif(true);
4342    } else if (completedTasks.length > 0) {
4343      _showCookbookNotif(false);
4344    } else if (activeTasks.length > 0) {
4345      _showCookbookNotif(false);
4346    } else {
4347      _clearCookbookNotif();

4348      _stopBackgroundMonitor();
4349    }
4350
4351    if (statusEl) {
4352      if (activeTasks.length > 0) {
4353        const t = activeTasks[0];
4354        if (t.type === 'serve') {
4355          if (t.progress) {
4356            // Show serve phase from backend (e.g. "loading 45%", "warming up", "idle", "12.5 tok/s")
4357            statusEl.textContent = t.progress;
4358          } else if (t.status === 'ready') {
4359            statusEl.textContent = 'ready';
4360          } else {
4361            statusEl.textContent = 'cooking';
4362          }
4363        } else {
4364          const _dlText = _downloadBadgeText(t.progress);
4365          statusEl.textContent = _dlText === 'downloading' ? 'downloading' : `downloading ${_dlText}`;
4366        }
4367        statusEl.style.display = '';
4368      } else if (errorTasks.length > 0) {
4369        statusEl.textContent = 'error';
4370        statusEl.style.display = '';
4371        statusEl.style.color = 'var(--color-error, #f44)';
4372      } else if (completedTasks.length > 0) {
4373        statusEl.textContent = 'done';
4374        statusEl.style.display = '';
4375        statusEl.style.color = 'var(--color-success, #4caf50)';
4376      } else {
4377        statusEl.style.display = 'none';
4378        statusEl.style.color = '';
4379      }
4380    }
4381    // Also clear the sidebar/rail icon highlight when no tasks are alive.
4382    // Without this, the cookbook icon stays at full opacity ("highlighted")
4383    // indefinitely once any task fires the notif, because the modal-open
4384    // clear only runs when the user actually reopens Cookbook.
4385    if (!activeTasks.length && !errorTasks.length) {
4386      _clearCookbookNotif();
4387    }
4388  } catch (e) {
4389    // Silent fail
4390  } finally {
4391    _bgPollInFlight = false;
4392  }
4393}
4394
4395// ── Init: receive shared state/functions ──
4396
4397export function initRunning(shared) {
4398  _envState = shared._envState;
4399  _sshCmd = shared._sshCmd;
4400  _getPort = shared._getPort;
4401  _sshPrefix = shared._sshPrefix;
4402  _getPlatform = shared._getPlatform;
4403  _isWindows = shared._isWindows;
4404  _buildEnvPrefix = shared._buildEnvPrefix;
4405  _loadPresets = shared._loadPresets;
4406  _savePresets = shared._savePresets;
4407  _copyText = shared._copyText;
4408  _persistEnvState = shared._persistEnvState;
4409  _refreshDependencies = shared._refreshDependencies;
4410  _serverByVal = shared._serverByVal;
4411  _serverKey = shared._serverKey;
4412  _selectedServer = shared._selectedServer;
4413  modelLogo = shared.modelLogo;
4414  esc = shared.esc;
4415  _detectBackend = shared._detectBackend;
4416  _detectToolParser = shared._detectToolParser;
4417  _detectModelOptimizations = shared._detectModelOptimizations;
4418  _buildServeCmd = shared._buildServeCmd;
4419
4420  // App boot: pull authoritative state from server, but don't start the
4421  // running-task monitor unless there is real work to watch. Starting it
4422  // unconditionally made a plain Cookbook open keep probing stale tmux/SSH
4423  // sessions, which is expensive when a saved remote host is unreachable.
4424  (async () => {
4425    try {
4426      await _syncFromServer();
4427    } catch {}
4428    if (_hasLiveTasks()) _startBackgroundMonitor();
4429  })();
4430}
4431
4432// Also export _retryDownload and _nextAvailablePort for use by other modules
4433export { _retryDownload, _nextAvailablePort, _processQueue, _taskPort };