cookbookRunning.js

js jayreddinjr-jr-odysseus-hf.hf.space collected 2026-10-04 17:00:10 UTC 185,046 bytes, 3,732 lines download raw bytes

1// ============================================
2// COOKBOOK RUNNING SUB-MODULE
3// Running tasks tab: task cards, status monitoring,
4// stop/restart, diagnosis, auto-fix, background monitor
5// ============================================
6
7import uiModule from './ui.js';
8import { _diagnose, _showDiagnosis, _clearDiagnosis } from './cookbook-diagnosis.js';
9import { registerMenuDismiss } from './escMenuStack.js';
10import { computeProgressSignal } from './cookbookProgressSignal.js';
11
12// Human-friendly badge label for a task's internal status. Avoids surfacing
13// the word "error" in the sidebar â a server the user stopped or one that
14// quit cleanly reads as "stopped", not "error".
15function _statusLabel(status, type) {
16  if (status === 'running' && type === 'download') return 'downloading';
17  if (status === 'done' && type === 'download') return 'finished';
18  if (status === 'error') return 'stopped';
19  return status || '';
20}
21
22// Single source of truth for what a task's status badge shows + its style class.
23// Crucially, a serve task that's still coming up shows its live phase
24// ("loading 45%", "warming up", â¦) rather than the generic "running" â they're
25// the same state, so the badge shouldn't flip between two different labels on
26// every re-render. Returns { text, cls } where cls is appended after
27// "cookbook-task-status" ('' = the neutral loading style).
28function _taskBadge(task) {
29  if (task._unreachable && task.status === 'running') return { text: 'unreachable', cls: 'cookbook-task-error' };
30  if (task.type === 'serve' && task.status === 'running' && task.progress) {
31    // Same green "running" pill â just with dynamic phase text, so it doesn't
32    // read as a different status while the server is coming up.
33    return { text: task.progress, cls: 'cookbook-task-running' };
34  }
35  return { text: _statusLabel(task.status, task.type), cls: 'cookbook-task-' + task.status };
36}
37
38// A download task whose tmux output still shows an active per-shard line
39// (e.g. "model-00012-of-00082.safetensors: 56%|") is NOT actually finished â
40// the cookbook just lost track. The clear pill becomes a "reconnect" affordance
41// in that case (click â revive the row + reattach the poll loop).
42function _downloadOutputLooksActive(task) {
43  if (!task || task.type !== 'download') return false;
44  const out = task.output || '';
45  if (!out) return false;
46  if (out.includes('DOWNLOAD_OK') || out.includes('DOWNLOAD_FAILED')) return false;
47  // An active shard line: filename + a colon + a percentage that isn't 100%.
48  // We catch any in-flight shard or "Downloading 'X' to ..." line (no %).
49  return /model-\d+-of-\d+\.[a-z]+:\s+(?!100%)\d+%/i.test(out)

50      || /Downloading\s+'[^']+'\s+to\s+'[^']*\.incomplete'/i.test(out);
51}
52
53function _canClearTask(task) {
54  if (!task || task.status === 'running') return false;
55  if (task.type === 'serve' && (task.status === 'ready' || task._serveReady)) return false;
56  // If the tmux output still shows an in-flight download, the task isn't
57  // actually finished â hide the clear/check pill so it doesn't show on a
58  // task that's still doing work. (The next render will reflect this and
59  // ideally the self-heal flips status back to running.)
60  if (_downloadOutputLooksActive(task)) return false;
61  return ['done', 'stopped', 'error', 'crashed', 'failed'].includes(task.status);
62}
63
64function _clearPillLabel(task) {
65  if (_downloadOutputLooksActive(task)) return 'reconnect';
66  return 'clear';
67}
68
69// A pip dependency/driver install (payload._dep) reports success with the
70// runner's "=== Process exited with code 0 ===" sentinel and pip's
71// "Successfully installed" line â never the HuggingFace download markers
72// (DONE / 100% / /snapshots/ / DOWNLOAD_OK) that the download heuristics look
73// for. Without this, a clean install whose tmux pane has already gone away is
74// misread as crashed/stopped even though pip exited 0. Prefer the authoritative
75// exit-code sentinel; fall back to pip's success line when no sentinel was
76// captured (and there's no install error in the same output).
77function _depInstallSucceeded(output) {
78  const text = String(output || '');
79  if (!text) return false;
80  const exitMatch = text.match(/=== Process exited with code (-?\d+) ===/);
81  if (exitMatch) return Number(exitMatch[1]) === 0;
82  return /\b(?:Successfully installed|Requirement already satisfied)\b/.test(text)
83    && !/\bERROR\b|No matching distribution|Could not find a version|Traceback \(most recent call last\)/.test(text);
84}
85
86function _shouldOfferCrashReport(task) {
87  if (!task) return false;
88  if (task._unreachable && task.type === 'serve') return true;
89  return ['error', 'crashed', 'failed'].includes(task.status);
90}
91
92function _serveTaskLooksAwqOnLocalBackend(task, outputText = '') {
93  const repo = `${task?.payload?.repo_id || ''} ${task?.name || ''}`.toLowerCase();
94  const cmd = `${task?.payload?._cmd || ''} ${outputText || ''}`.toLowerCase();
95  return /\b(awq|gptq|fp8)\b/.test(repo) && /(llama-server|llama_cpp\.server|ollama|ggml_cuda_enable_unified_memory)/.test(cmd);
96}
97
98function _serveTaskLooksAwqWithoutUsableAccelerator(task, outputText = '') {
99  const repo = `${task?.payload?.repo_id || ''} ${task?.name || ''}`.toLowerCase();
100  const out = String(outputText || '').toLowerCase();
101  return /\b(awq|gptq|fp8)\b/.test(repo)
102    && /(no accelerator|no cuda runtime|failed to infer device type|triton is not supported|0 active driver)/i.test(out);
103}
104
105async function _openDownloadForGgufTask(task) {
106  const raw = task?.payload?.repo_id || task?.name || '';
107  const modelName = String(raw)
108    .split('/').pop()
109    .replace(/[-_](?:AWQ|GPTQ|FP8|4bit|8bit|Int4|Int8).*$/i, '')
110    .replace(/[-_]+$/g, '')
111    || String(raw).split('/').pop()
112    || raw;
113  const cookbook = window.cookbookModule;
114  if (cookbook && typeof cookbook.open === 'function') {
115    cookbook.open({ tab: 'Search' });
116  } else {
117    document.getElementById('tool-cookbook-btn')?.click();
118  }
119  setTimeout(async () => {
120    const modal = document.getElementById('cookbook-modal');
121    const tab = modal?.querySelector('.cookbook-tab[data-backend="Search"]');
122    if (tab && !tab.classList.contains('active')) tab.click();
123    const search = document.getElementById('hwfit-search');
124    if (search) {
125      search.value = modelName;
126      search.dispatchEvent(new Event('input', { bubbles: true }));
127      search.focus();
128    }
129    const quant = document.getElementById('hwfit-quant');
130    if (quant) {
131      quant.value = 'Q4_K_M';
132      quant.dispatchEvent(new Event('change', { bubbles: true }));
133    }
134    try {
135      const hwfit = await import('./cookbook-hwfit.js');
136      if (typeof hwfit._hwfitFetch === 'function') hwfit._hwfitFetch(true);
137    } catch {}
138  }, 80);
139}
140
141function _terminalServeDiagnosis(task, outputText) {
142  const out = String(outputText || task?.output || '');
143  if (!task || task.type !== 'serve' || !['stopped', 'error', 'crashed', 'failed'].includes(task.status) || !out.trim()) return null;
144  // Pip tasks (Reinstall vLLM, Upgrade torch, etc.) ride on the serve task
145  // type so they get a tmux session + show up in Running tab â but they are
146  // NOT serve invocations. Their output is pip's own; the generic
147  // "Serve stopped before the model became reachable" message + Edit-serve

148  // fix make no sense. Bail so the panel just shows pip's output.
149  const _isPipTask = ((task.payload?.repo_id || '').startsWith('pip-'))
150    || /python3? -m pip\b/.test(task.payload?._cmd || '');
151  if (_isPipTask) return null;
152  if (_serveTaskLooksAwqOnLocalBackend(task, out)) {
153    return {
154      message: 'AWQ/GPTQ/FP8 cannot be served through llama.cpp/Ollama unified-memory mode.',
155      suggestion: 'Suggested action: use vLLM/SGLang on a compatible CUDA/ROCm GPU server, or download a GGUF version for llama.cpp/Ollama/unified-memory serving.',
156      fixes: [
157        { label: 'Find GGUF download', action: () => _openDownloadForGgufTask(task) },
158        { label: 'Edit serve', action: (panel) => _openServeEditForTask(task) },
159      ],
160    };
161  }
162  if (_serveTaskLooksAwqWithoutUsableAccelerator(task, out)) {
163    return {
164      message: 'AWQ/GPTQ/FP8 needs a working vLLM/SGLang accelerator path; this server did not expose one.',
165      suggestion: 'Suggested action: choose a CUDA/ROCm server where vLLM/SGLang can see the GPU, or download a GGUF version and serve it with llama.cpp/Ollama.',
166      fixes: [
167        { label: 'Find GGUF download', action: () => _openDownloadForGgufTask(task) },
168        { label: 'Edit serve', action: (panel) => _openServeEditForTask(task) },
169      ],
170    };
171  }
172  return _diagnose(out) || {
173    message: /Native llama-server not found|building llama-server|llama\.cpp/i.test(out)
174      ? 'llama.cpp build stopped before the server became reachable.'
175      : 'Serve stopped before the model became reachable.',
176    suggestion: /Native llama-server not found|building llama-server|llama\.cpp/i.test(out)
177      ? 'Suggested action: copy the troubleshooting bundle, then edit serve settings. For the quickest local/CPU path, use Ollama or a prebuilt llama-server; source builds can take several minutes and fail if build dependencies are incomplete.'
178      : 'Suggested action: copy the troubleshooting bundle, then edit serve settings or relaunch with a CPU/backend fallback.',
179    fixes: [{ label: 'Edit serve', action: (panel) => _openServeEditForTask(task) }],
180  };
181}
182
183function _redactCrashReportText(text) {
184  if (!text) return '';
185  return String(text)
186    .replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]{12,}/gi, '$1[redacted]')
187    .replace(/\b(hf_[A-Za-z0-9]{16,})\b/g, '[redacted-hf-token]')
188    .replace(/\b(sk-[A-Za-z0-9_-]{16,})\b/g, '[redacted-api-key]')
189    .replace(/\b(xox[baprs]-[A-Za-z0-9-]{16,})\b/g, '[redacted-slack-token]')
190    .replace(/\b(AIza[0-9A-Za-z_-]{20,})\b/g, '[redacted-google-key]')
191    .replace(/\b((?:HF_TOKEN|HUGGING_FACE_HUB_TOKEN|OPENAI_API_KEY|ANTHROPIC_API_KEY|BRAVE_API_KEY|TAVILY_API_KEY|SERPER_API_KEY|GOOGLE_API_KEY|API_KEY|TOKEN|PASSWORD)\s*=\s*)(['"]?)[^\s'"\\]+/gi, '$1$2[redacted]')
192    .replace(/\b(--(?:api-key|token|hf-token|password)\s+)([^\s]+)/gi, '$1[redacted]');
193}
194
195function _lastLines(text, count = 160) {
196  const clean = _redactCrashReportText(text || '').trimEnd();
197  if (!clean) return '(no captured output)';
198  return clean.split('\n').slice(-count).join('\n');
199}
200
201function _codeFence(text) {
202  return String(text || '').replace(/```/g, '` ` `');
203}
204
205function _taskHostLabel(task) {
206  if (!task?.remoteHost) return 'local';
207  return task.remoteHost + (task.sshPort ? `:${task.sshPort}` : '');
208}
209
210function _taskPort(task) {
211  const cmd = task?.payload?._cmd || '';

212  const match = cmd.match(/--port\s+(\d+)/);
213  return match ? match[1] : '';
214}
215
216function _buildCrashReport(task, outputText) {
217  const capturedOutput = outputText || task?.output || '';
218  const cmd = _redactCrashReportText(task?.payload?._cmd || '');
219  const diag = _diagnose(capturedOutput);
220  const started = task?.ts ? new Date(task.ts).toISOString() : '';
221  const report = [
222    '## Odysseus Cookbook crash report',
223    '',
224    'Please review this report for secrets before posting it publicly.',
225    '',
226    '### Task',
227    `- ID: \`${task?.sessionId || task?.id || 'unknown'}\``,
228    `- Type: \`${task?.type || 'unknown'}\``,
229    `- Status: \`${task?._unreachable ? 'unreachable' : (task?.status || 'unknown')}\``,
230    `- Model/repo: \`${task?.payload?.repo_id || task?.name || 'unknown'}\``,
231    `- Host: \`${_taskHostLabel(task)}\``,
232  ];
233  if (task?.platform) report.push(`- Platform: \`${task.platform}\``);
234  if (started) report.push(`- Started: \`${started}\``);
235  const port = _taskPort(task);
236  if (port) report.push(`- Port: \`${port}\``);
237  if (diag?.message) report.push(`- Diagnosis: ${diag.message}`);
238  if (cmd) {
239    report.push('', '### Command', '```bash', _codeFence(cmd), '```');
240  }
241  report.push('', '### Last captured output', '```text', _codeFence(_lastLines(capturedOutput)), '```');
242  return report.join('\n');
243}
244
245// Shared state/functions injected by init()
246let _envState;
247let _sshCmd;
248let _getPort;
249let _sshPrefix;
250let _getPlatform;
251let _isWindows;
252let _buildEnvPrefix;
253let _loadPresets;
254let _savePresets;
255let _copyText;
256let _persistEnvState;
257let _refreshDependencies;
258let modelLogo;
259let esc;
260let _detectBackend;
261let _detectToolParser;
262let _detectModelOptimizations;
263let _buildServeCmd;
264
265// When a new action is started (download / dependency / serve), this holds the
266// new task's id so the next render collapses every other card and leaves only
267// the new one open. Consumed (cleared) by _renderRunningTab.
268let _soloExpandTaskId = null;
269
270// Storage keys
271const TASKS_KEY = 'cookbook-tasks';
272const STORAGE_KEY = 'cookbook-presets';
273const SERVE_STATE_KEY = 'cookbook-serve-state';
274
275// Polling / timeout intervals
276const TASK_POLL_INTERVAL_MS = 3000;       // delay between reconnect-loop iterations
277const BG_MONITOR_INTERVAL_MS = 5000;      // background task status poll
278const STALE_PROGRESS_MS = 5 * 60 * 1000;  // download with no progress this long = stale
279const STARTUP_STALE_PROGRESS_MS = 45 * 1000; // 0%-forever startup stall: retry much sooner
280

281// ââ Phase detection (mirrors Python _parse_serve_phase in cookbook_routes.py) ââ
282// Single source of truth for serve task status. KEEP IN SYNC with the Python version.
283export function _parseServePhase(snapshot) {
284  if (!snapshot) return {};
285  // Strip newlines so tmux line-wrapping doesn't break regex matching
286  const flat = snapshot.replace(/\s+/g, ' ');
287  const loadMatches = [...flat.matchAll(/Loading safetensors.*?(\d+)%/g)];
288  // "Downloading (incomplete total...)" tracks real aggregate bytes; prefer it
289  // over "Fetching N files" which only counts fully-closed files and lags badly
290  // with hf_transfer's parallel-chunk strategy (often sits at 0/N for most of the run).
291  const downloadingMatches = [...flat.matchAll(/Downloading.*?(\d+)%/g)];
292  const fetchingMatches = [...flat.matchAll(/Fetching.*?(\d+)%/g)];
293  const dlMatches = downloadingMatches.length ? downloadingMatches : fetchingMatches;
294  // "Avg generation throughput: X tokens/s, Running: N reqs"
295  const tpsMatches = [...flat.matchAll(/(?:Avg )?generation throughput:\s*([\d.]+)\s*tokens\/s.*?Running:\s*(\d+)\s*reqs/g)];
296
297  // Throughput FIRST â its log line contains "GPU KV cache usage" which would
298  // otherwise false-match the warmup check
299  if (tpsMatches.length) {
300    const m = tpsMatches[tpsMatches.length - 1];
301    const tps = parseFloat(m[1]);
302    const reqs = parseInt(m[2]);
303    return {
304      phase: reqs > 0 ? `${m[1]} tok/s` : 'idle',
305      status: 'ready',
306      tps,
307      reqs,
308    };
309  }
310  if (flat.includes('Application startup complete')) {
311    return { phase: 'ready', status: 'ready' };
312  }
313  if (/Ollama API ready on port\s+\d+/i.test(flat)) {
314    return { phase: 'ready', status: 'ready' };
315  }
316  const llamaBuildMatches = [...flat.matchAll(/\[\s*(\d{1,3})%\]\s*(?:Building|Linking)/gi)];
317  if (llamaBuildMatches.length) {
318    const pct = Math.min(100, parseInt(llamaBuildMatches[llamaBuildMatches.length - 1][1], 10));
319    return { phase: `building llama.cpp ${pct}%`, status: 'running', pct };
320  }
321  if (/Native llama-server not found|building from source/i.test(flat)) {
322    if (/Cloning into ['"]?llama\.cpp/i.test(flat) && !/Receiving objects:\s*100%/i.test(flat)) {
323      return { phase: 'cloning llama.cpp', status: 'running' };
324    }
325    if (/Configuring incomplete|CMake Error/i.test(flat)) {
326      return {};
327    }
328    if (/CMAKE_BUILD_TYPE|Detecting CXX|Found Threads|Including CPU backend|CUDA nvcc found|building llama-server/i.test(flat)) {
329      return { phase: 'configuring llama.cpp', status: 'running' };
330    }
331    return { phase: 'building llama.cpp', status: 'running' };
332  }
333  // HTTP access logs (e.g. GET /v1/models 200 OK) mean the server is up
334  if (/(?:GET|POST)\s+\/[^\s]*\s+HTTP\/[\d.]+"\s*\d{3}/.test(flat)) {
335    return { phase: 'idle', status: 'ready' };
336  }
337  if (flat.includes('Loading weights took')) {
338    return { phase: 'initializing', status: 'running' };
339  }
340  // "GPU KV cache" alone (during allocation) â not "GPU KV cache usage" (runtime log)
341  if (flat.includes('GPU KV cache') && !flat.includes('GPU KV cache usage')) {
342    return { phase: 'warming up', status: 'running' };
343  }
344  if (loadMatches.length) {
345    const pct = parseInt(loadMatches[loadMatches.length - 1][1]);
346    return { phase: `loading ${pct}%`, status: 'running', pct };
347  }
348  if (dlMatches.length) {
349    const pct = parseInt(dlMatches[dlMatches.length - 1][1]);
350    return { phase: `downloading ${pct}%`, status: 'running', pct };
351  }
352  return {};
353}
354
355// ââ Port auto-increment ââ
356
357function _nextAvailablePort() {
358  const tasks = _loadTasks();
359  const presets = _loadPresets();
360  const usedPorts = new Set();
361  tasks.forEach(t => {
362    if (t.type === 'serve' && (t.status === 'running' || t.status === 'queued')) {
363      const m = t.payload?._cmd?.match(/--port\s+(\d+)/);
364      if (m) usedPorts.add(parseInt(m[1]));
365    }
366  });
367  presets.forEach(p => {
368    if (p.port) usedPorts.add(parseInt(p.port));
369  });
370  let port = 8000;
371  while (usedPorts.has(port)) port++;
372  return String(port);
373}
374
375// ââ Endpoint cleanup ââ
376
377async function _removeEndpointByUrl(baseUrl) {
378  try {
379    const res = await fetch('/api/model-endpoints', { credentials: 'same-origin' });
380    if (!res.ok) return;
381    const endpoints = await res.json();
382    const hostPort = baseUrl.replace(/^https?:\/\//, '').replace(/\/.*$/, '');
383    const ep = endpoints.find(e => e.base_url === baseUrl)
384            || endpoints.find(e => e.base_url.includes(hostPort));
385    if (ep) {
386      await fetch(`/api/model-endpoints/${ep.id}`, { method: 'DELETE', credentials: 'same-origin' });
387      _refreshModelsAfterEndpointChange();
388    }
389  } catch {}
390}
391
392function _refreshModelsAfterEndpointChange() {
393  const pickerLabel = document.getElementById('model-picker-label');
394  if (pickerLabel) {
395    pickerLabel.dataset.prevHtml = pickerLabel.innerHTML;
396    pickerLabel.innerHTML = '<span style="opacity:0.4;">refreshingâ¦</span>';
397  }
398  if (window.modelsModule && window.modelsModule.refreshModels) {
399    window.modelsModule.refreshModels(true);
400  }
401  setTimeout(() => {
402    if (!window.sessionModule) return;
403    const currentModel = window.sessionModule.getCurrentModel ? window.sessionModule.getCurrentModel() : null;
404    if (currentModel) {
405      const items = (window.modelsModule && window.modelsModule.getCachedItems) ? window.modelsModule.getCachedItems() : [];
406      const allModels = [];

407      items.forEach(item => {
408        if (item.offline) return;
409        (item.models || []).concat(item.models_extra || []).forEach(m => allModels.push({ mid: m, url: item.url, endpointId: item.endpoint_id }));
410      });
411      const stillExists = allModels.some(m => m.mid === currentModel);
412      if (!stillExists && allModels.length > 0) {
413        const fallback = allModels[0];
414        if (window.sessionModule.createDirectChat) {
415          window.sessionModule.createDirectChat(fallback.url, fallback.mid, fallback.endpointId);
416        }
417      }
418    }
419    if (window.sessionModule.updateModelPicker) {
420      window.sessionModule.updateModelPicker();
421    }
422  }, 1500);
423}
424
425function _appendCookbookEndpointScope(fd, remoteHost) {
426  const host = String(remoteHost || '').trim();
427  if (!host || host === 'local' || host === 'localhost' || host === '127.0.0.1') {
428    fd.append('container_local', 'true');
429  }
430}
431
432function _connectHostFromRemote(remoteHost, fallback = 'localhost') {
433  const host = String(remoteHost || '').trim();
434  if (!host || host === 'local') return fallback;
435  return host.includes('@') ? host.split('@').pop() : host;
436}
437
438function _isAnyBindHost(host) {
439  const h = String(host || '').trim().toLowerCase();
440  return h === '0.0.0.0' || h === '::' || h === '[::]';
441}
442
443function _endpointFromAdvertisedUrl(rawUrl, currentHost, fallbackPort = '11434') {
444  try {
445    const u = new URL(rawUrl);
446    const host = _isAnyBindHost(u.hostname) ? currentHost : (u.hostname || currentHost);
447    const port = u.port || fallbackPort;
448    const bracketedHost = host.includes(':') && !host.startsWith('[') ? `[${host}]` : host;
449    return { host, port, baseUrl: `${u.protocol}//${bracketedHost}${port ? `:${port}` : ''}/v1` };
450  } catch {
451    return null;
452  }
453}
454
455// ââ Download queue â runs one at a time per server ââ
456
457function _processQueue() {
458  const tasks = _loadPrunedTasks();
459  const running = tasks.filter(t => t.type === 'download' && t.status === 'running');
460  const queued = tasks.filter(t => t.type === 'download' && t.status === 'queued');
461  if (!queued.length) return;
462
463  const busyHosts = new Set(running.map(t => t.remoteHost || 'local'));
464
465  for (const task of queued) {
466    const host = task.remoteHost || 'local';
467    if (busyHosts.has(host)) continue;
468    busyHosts.add(host);
469    _startQueuedDownload(task);
470  }
471}
472
473async function _startQueuedDownload(task) {
474  if (!task.payload) {
475    _updateTask(task.sessionId, { status: 'error', output: 'No payload' });
476    _renderRunningTab();
477    return;
478  }
479  // Flip to 'running' SYNCHRONOUSLY (before the async POST) so a concurrent
480  // _processQueue â or a second "Start now" â can't see it as still 'queued' and
481  // launch the same download a second time. Without this, finishing another
482  // download mid-POST re-queued this one into a duplicate task.
483  {
484    const _pre = _loadTasks();
485    const _pt = _pre.find(t => t.sessionId === task.sessionId);
486    if (_pt) {
487      if (_pt.status === 'running' && _pt._startLaunched) return;  // already being started
488      _pt.status = 'running';
489      _pt._startLaunched = true;
490      _saveTasks(_pre);
491    }
492  }
493  try {
494    const res = await fetch('/api/model/download', {
495      method: 'POST', credentials: 'same-origin',
496      headers: { 'Content-Type': 'application/json' },
497      body: JSON.stringify(task.payload),
498    });
499    if (!res.ok) {
500      const errText = await res.text().catch(() => '');
501      _updateTask(task.sessionId, { status: 'error', output: `HTTP ${res.status}: ${errText.slice(0, 200)}` });
502      _renderRunningTab();
503      return;
504    }
505    const data = await res.json();
506    if (!data.ok) {
507      _updateTask(task.sessionId, { status: 'error', output: data.error || 'Unknown error' });
508      _renderRunningTab();
509      return;
510    }
511    const oldId = task.sessionId;
512    const launchedTask = { ...task, sessionId: data.session_id, id: data.session_id, status: 'running' };
513    const key = _downloadDedupeKey(launchedTask);
514    let found = false;
515    const tasks = _loadTasks().filter(t => {
516      if (t.sessionId === oldId) {
517        found = true;
518        t.sessionId = data.session_id;
519        t.id = data.session_id;
520        t.status = 'running';
521        t._startLaunched = true;
522        return true;
523      }
524      if (t.sessionId === data.session_id) return false;

525      return !(key && t.type === 'download' && t.status === 'queued' && _downloadDedupeKey(t) === key);
526    });
527    if (!found) tasks.push(_stripTaskSecrets(launchedTask));
528    _saveTasks(tasks);
529    _renderRunningTab();
530    _startBackgroundMonitor();
531    await new Promise(r => setTimeout(r, 2000));
532    _renderRunningTab();
533  } catch (e) {
534    _updateTask(task.sessionId, { status: 'error', output: e.message || 'Network error' });
535    _renderRunningTab();
536  }
537}
538
539// ââ Task CRUD ââ
540
541function _serveOutputLooksReady(task) {
542  const out = String(task?.output || '');
543  return !!task?._serveReady
544    || /Application startup complete/i.test(out)
545    || /Ollama API ready on port\s+\d+/i.test(out)
546    || /(?:GET|POST)\s+\/[^\s]*\s+HTTP\/[\d.]+"\s*2\d\d/i.test(out);
547}
548
549function _normalizeTaskForDisplay(task) {
550  if (!task || typeof task !== 'object') return task;
551  // Pip tasks (Reinstall vLLM / Upgrade torch / etc.) ride on the serve task
552  // type so they get tmux + the Running tab. They are NOT serves â their
553  // "ready" markers are pip's `Successfully installed` / `Requirement already
554  // satisfied`, not "Application startup complete".
555  const _isPipTask = ((task.payload?.repo_id || '').startsWith('pip-'))
556    || /python3? -m pip\b/.test(task.payload?._cmd || '');
557  if (_isPipTask) {
558    // Override stale status: any pip task whose output carries pip's own
559    // success markers gets displayed as `done` regardless of what's in
560    // localStorage. Old pre-fix runs landed in error/stopped state and
561    // stuck there even after we taught the rest of the flow about pip
562    // tasks â this is the catch-all that flips them to Finished on render.
563    const out = String(task.output || '');
564    const ranOk = /Successfully installed|Requirement already (?:satisfied|up-to-date)/i.test(out)

565      && !/error:|ERROR:/.test(out.slice(-1024));
566    if (ranOk && task.status !== 'done' && task.status !== 'running') {
567      return { ...task, status: 'done' };
568    }
569    return task;
570  }
571  if (task.type === 'serve' && task.status === 'done' && !_serveOutputLooksReady(task)) {
572    return { ...task, status: 'error' };
573  }
574  return task;
575}
576
577export function _loadTasks() {
578  try { return (JSON.parse(localStorage.getItem(TASKS_KEY)) || []).map(_normalizeTaskForDisplay); }
579  catch { return []; }
580}
581
582function _downloadRepoKey(task) {
583  return String(task?.payload?.repo_id || task?.repo_id || task?.repo || task?.name || '').trim();
584}
585
586function _downloadHostKey(task) {
587  return String(task?.remoteHost || task?.payload?.remote_host || 'local').trim() || 'local';
588}
589
590function _downloadDedupeKey(task) {
591  if (!task || task.type !== 'download') return '';
592  const repo = _downloadRepoKey(task);
593  if (!repo) return '';
594  return `${_downloadHostKey(task)}\n${repo}`;
595}
596
597function _pruneQueuedDownloadDuplicates(tasks) {
598  if (!Array.isArray(tasks) || !tasks.length) return tasks || [];
599  const launched = new Set();
600  for (const task of tasks) {
601    if (task?.type !== 'download' || task.status === 'queued') continue;
602    const key = _downloadDedupeKey(task);
603    if (key) launched.add(key);
604  }
605
606  let changed = false;
607  const seenQueued = new Set();
608  const next = tasks.filter(task => {
609    if (task?.type !== 'download' || task.status !== 'queued') return true;
610    const key = _downloadDedupeKey(task);
611    if (!key) return true;
612    if (launched.has(key) || seenQueued.has(key)) {
613      changed = true;
614      return false;
615    }
616    seenQueued.add(key);
617    return true;
618  });
619  return changed ? next : tasks;
620}
621
622function _loadPrunedTasks() {
623  const tasks = _loadTasks();
624  const pruned = _pruneQueuedDownloadDuplicates(tasks);
625  if (pruned !== tasks) _saveTasks(pruned);
626  return pruned;
627}
628
629// Tombstones for removed tasks. Without these, removing a task only deletes it
630// locally â but the server still has it (its own POST guard even re-preserves
631// recently-added ones), so the next sync/poll merges it right back ("I removed
632// it and it came back"). A tombstone makes the removal stick: merges skip any
633// id the user removed, until the entry expires.
634const _REMOVED_KEY = 'cookbook-removed-tasks';
635const _TOMBSTONE_TTL_MS = 24 * 3600 * 1000;
636function _loadTombstones() {
637  try { return JSON.parse(localStorage.getItem(_REMOVED_KEY)) || {}; }
638  catch { return {}; }
639}
640function _tombstoneTask(id) {
641  if (!id) return;
642  const tomb = _loadTombstones();
643  const now = Date.now();
644  tomb[id] = now;
645  for (const k in tomb) { if (now - tomb[k] > _TOMBSTONE_TTL_MS) delete tomb[k]; }
646  localStorage.setItem(_REMOVED_KEY, JSON.stringify(tomb));
647}
648function _isTombstoned(id) {
649  const ts = _loadTombstones()[id];
650  return ts != null && (Date.now() - ts) <= _TOMBSTONE_TTL_MS;
651}
652
653function _stripTaskSecrets(task) {
654  if (!task || typeof task !== 'object') return task;
655  const safe = { ...task };
656  if (safe.payload && typeof safe.payload === 'object') {
657    safe.payload = { ...safe.payload };
658    delete safe.payload.hf_token;
659  }
660  return safe;
661}
662
663function _stripStateSecrets(state) {
664  const safe = { ...state };
665  if (safe.env && typeof safe.env === 'object') {
666    const { hfToken, ...env } = safe.env;
667    if (hfToken) env.hfToken = hfToken;
668    safe.env = env;
669  }
670  if (Array.isArray(safe.tasks)) safe.tasks = safe.tasks.map(_stripTaskSecrets);
671  return safe;
672}
673
674export function _saveTasks(tasks) {
675  localStorage.setItem(TASKS_KEY, JSON.stringify((tasks || []).map(_stripTaskSecrets)));
676  _syncToServer();
677}
678
679export function _addTask(sessionId, name, type, payload) {
680  let tasks = _loadTasks();
681  const remoteHost = (payload && payload.remote_host) || _envState.remoteHost || '';
682  const sshPort = (payload && payload.ssh_port) || _getPort(remoteHost) || '';
683  const platform = (payload && payload.platform) || _getPlatform(remoteHost) || '';
684  // Serving a model supersedes its finished download â clear the matching
685  // finished download card (covers serving directly from the Serve tab, not just
686  // via the download card's "Serve â" button).
687  if (type === 'serve' && payload && payload.repo_id) {
688    const _repoId = payload.repo_id;
689    tasks = tasks.filter(t => !(t.type === 'download' && t.status === 'done' && t.payload && t.payload.repo_id === _repoId));
690  }
691  if (type === 'download' && payload && payload.repo_id) {

692    const key = _downloadDedupeKey({ type: 'download', payload, remoteHost });
693    tasks = tasks.filter(t => {
694      if (t.sessionId === sessionId) return false;
695      return !(key && t.type === 'download' && t.status === 'queued' && _downloadDedupeKey(t) === key);
696    });
697  }
698  const task = _stripTaskSecrets({ id: sessionId, sessionId, name, type, status: 'running', output: '', ts: Date.now(), payload: payload || null, remoteHost, sshPort, platform });
699  tasks.push(task);
700  _saveTasks(tasks);
701  // New action â collapse all other cards, leave only this one open.
702  _soloExpandTaskId = sessionId;
703  _renderRunningTab();
704  // Always start the background monitor when a task is added â works even
705  // when modal is closed and ensures the sidebar shows live status immediately
706  _startBackgroundMonitor();
707  // Switch to Running tab
708  const body = document.querySelector('#cookbook-modal .cookbook-body');
709  if (body) {
710    const tab = body.querySelector('.cookbook-tab[data-backend="Running"]');
711    if (tab) tab.click();
712  }
713  return task;
714}
715
716function _updateTask(sessionId, updates) {
717  const tasks = _loadTasks();
718  const task = tasks.find(t => t.sessionId === sessionId);
719  if (task) {
720    Object.assign(task, updates);
721    _saveTasks(tasks);
722  }
723  if ('status' in updates || '_unreachable' in updates) {
724    _refreshServerDots();
725  }
726  if (updates.status && updates.status !== 'running') {
727    const el = document.querySelector(`.cookbook-task[data-task-id="${sessionId}"]`);
728    if (el) {
729      if (el._uptimeInterval) { clearInterval(el._uptimeInterval); el._uptimeInterval = null; }
730      const wave = el.querySelector('.cookbook-task-wave');
731      if (wave) wave.style.display = 'none';
732      const uptime = el.querySelector('.cookbook-task-uptime');
733      if (uptime) uptime.style.display = 'none';
734    }
735  }
736}
737
738function _refreshDepsAfterInstall(task) {
739  if (!task || task.type !== 'download' || !task.payload?._dep) return;
740  try {
741    _refreshDependencies?.({ host: task.remoteHost || '', port: task.sshPort || '', venv: task.payload?.env_path || '' });
742  } catch {}
743}
744
745export function _removeTask(sessionId) {
746  _tombstoneTask(sessionId);  // so sync/poll can't resurrect it
747  const tasks = _loadTasks().filter(t => t.sessionId !== sessionId);
748  _saveTasks(tasks);
749  _renderRunningTab();
750}
751
752// Fade/slide the task card out, then remove it â so the smooth exit is the same
753// whether a task auto-stops or the user removes/kills it manually.
754function _animateOutThenRemove(el, sessionId) {
755  if (!el || !el.style) { _removeTask(sessionId); return; }
756  if (el._abort) el._abort.abort();
757  el.style.transition = 'opacity 0.35s ease, transform 0.35s ease';
758  el.style.opacity = '0';
759  el.style.transform = 'translateX(-10px)';
760  setTimeout(() => _removeTask(sessionId), 360);
761}
762
763// ââ tmux / Windows session commands ââ
764
765export function _tmuxCmd(task, tmuxArgs) {
766  if (_isWindows(task)) {
767    return _winSessionCmd(task, tmuxArgs);
768  }
769  if (task.remoteHost) {
770    return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} 'tmux ${tmuxArgs}' 2>/dev/null`;
771  }
772  return `tmux ${tmuxArgs} 2>/dev/null`;
773}
774
775function _winSessionCmd(task, tmuxArgs) {
776  const host = task.remoteHost;
777  const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux';
778  const sid = task.sessionId;
779  const pf = _sshPrefix(_getPort(task));
780  if (tmuxArgs.includes('capture-pane')) {
781    const lines = tmuxArgs.match(/-S\s*-?(\d+)/)?.[1] || '200';
782    const ps = host
783      ? `Get-Content '${sd}\\${sid}.log' -Tail ${lines} -ErrorAction SilentlyContinue`
784      : `Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.log') -Tail ${lines} -ErrorAction SilentlyContinue`;
785    return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`;
786  }
787  if (tmuxArgs.includes('has-session')) {
788    const ps = host
789      ? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Get-Process -Id $p -ErrorAction SilentlyContinue | Out-Null; if ($?) { exit 0 } else { exit 1 } } else { exit 1 }`
790      : `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Get-Process -Id $p -ErrorAction SilentlyContinue | Out-Null; if ($?) { exit 0 } else { exit 1 } } else { exit 1 }`;
791    return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`;
792  }
793  if (tmuxArgs.includes('kill-session')) {
794    const ps = host
795      ? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -Force -ErrorAction SilentlyContinue }; Remove-Item '${sd}\\${sid}.*' -Force -ErrorAction SilentlyContinue`
796      : `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -Force -ErrorAction SilentlyContinue }; Remove-Item (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.*') -Force -ErrorAction SilentlyContinue`;
797    return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`;
798  }

799  if (tmuxArgs.includes('send-keys') && tmuxArgs.includes('C-c')) {
800    const ps = host
801      ? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -ErrorAction SilentlyContinue }`
802      : `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -ErrorAction SilentlyContinue }`;
803    return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`;
804  }
805  return host ? `ssh ${pf}${host} 'tmux ${tmuxArgs}' 2>/dev/null` : `tmux ${tmuxArgs} 2>/dev/null`;
806}
807
808function _tmuxGracefulKill(task) {
809  if (_isWindows(task)) {
810    const host = task.remoteHost;
811    const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux';
812    const sid = task.sessionId;
813    const pf = _sshPrefix(_getPort(task));
814    const ps = host
815      ? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -Force -ErrorAction SilentlyContinue }; Remove-Item '${sd}\\${sid}.*' -Force -ErrorAction SilentlyContinue`
816      : `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -Force -ErrorAction SilentlyContinue }; Remove-Item (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.*') -Force -ErrorAction SilentlyContinue`;
817    return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`;
818  }
819  if (task.remoteHost) {
820    return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} 'tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null'`;
821  }
822  return `tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null`;
823}
824
825function _shQuote(value) {
826  return "'" + String(value ?? '').replace(/'/g, "'\\''") + "'";
827}
828
829function _taskLooksOllama(task, outputText = '') {
830  const haystack = `${task?.payload?.backend || ''} ${task?.payload?._cmd || ''} ${task?.payload?._fields?.backend || ''} ${outputText || ''}`;
831  return /\bollama\b/i.test(haystack) || /Ollama API ready on port\s+\d+/i.test(haystack);
832}
833
834function _ollamaBaseUrlForTask(task, outputText = '') {
835  const out = String(outputText || '');
836  const ready = out.match(/Ollama API ready on port\s+\d+:\s*(http:\/\/[^\s]+)/i);
837  if (ready) return ready[1].replace(/\/+$/, '');
838  const cmd = String(task?.payload?._cmd || '');
839  const host = cmd.match(/OLLAMA_HOST=([^\s]+)/)?.[1] || '';
840  const port = host.match(/:(\d+)$/)?.[1] || '11434';
841  return `http://127.0.0.1:${port}`;
842}
843
844function _ollamaModelForTask(task) {
845  return String(task?.payload?.model || task?.payload?.repo_id || task?.name || '').trim();
846}
847
848function _ollamaUnloadCommand(task, outputText = '') {
849  if (!_taskLooksOllama(task, outputText)) return '';
850  const model = _ollamaModelForTask(task);
851  if (!model) return '';
852  const base = _ollamaBaseUrlForTask(task, outputText);
853  const body = JSON.stringify({ model, prompt: '', keep_alive: 0, stream: false });
854  const inner = `curl -sf -X POST ${_shQuote(base + '/api/generate')} -H 'Content-Type: application/json' -d ${_shQuote(body)} >/dev/null 2>&1 || true`;
855  if (task.remoteHost) {
856    return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} ${_shQuote(inner)}`;
857  }
858  return inner;
859}
860
861function _endpointUrlForTask(task, outputText = '') {
862  if (_taskLooksOllama(task, outputText)) {
863    return _ollamaBaseUrlForTask(task, outputText) + '/v1';
864  }
865  const host = _connectHostFromRemote(task.remoteHost);
866  const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
867  const port = portMatch ? portMatch[1] : '8000';
868  return `http://${host}:${port}/v1`;
869}
870
871// ââ Wave animation ââ
872
873const _waveFrames = ['âââ', 'âââ', 'âââ', 'âââ', 'âââ', 'âââ', 'âââ', 'âââ', 'âââ'];
874let _waveIdx = 0;
875let _waveTimer = null;
876const _waveEls = new Set();
877
878function _startWaveSync() {
879  if (_waveTimer) return;
880  _waveTimer = setInterval(() => {
881    _waveIdx = (_waveIdx + 1) % _waveFrames.length;
882    for (const el of _waveEls) {
883      if (!el.isConnected) { _waveEls.delete(el); continue; }
884      if (el.style.display !== 'none') el.textContent = _waveFrames[_waveIdx];
885    }
886    if (!_waveEls.size) { clearInterval(_waveTimer); _waveTimer = null; }
887  }, 200);
888}
889
890function _registerWaveEl(el) { _waveEls.add(el); _startWaveSync(); }
891
892// ââ Notifications ââ
893
894function _showCookbookNotif(isError = false) {
895  const dot = document.getElementById('cookbook-notif-dot');
896  if (dot) {
897    dot.style.display = '';
898    dot.classList.toggle('cookbook-notif-error', isError);
899  }
900  const btn = document.getElementById('tool-cookbook-btn');
901  if (btn) { btn.style.opacity = '1'; btn.classList.add('cookbook-notif-active'); }
902  const railBtn = document.getElementById('rail-cookbook');
903  if (railBtn) {
904    railBtn.classList.remove('rail-notify-success', 'rail-notify-error');
905    railBtn.classList.add('rail-notify', isError ? 'rail-notify-error' : 'rail-notify-success', 'cookbook-notif-active');
906  }
907  if (window._syncRailDynamic) window._syncRailDynamic();
908}
909
910export function _clearCookbookNotif() {
911  const dot = document.getElementById('cookbook-notif-dot');
912  if (dot) dot.style.display = 'none';
913  const btn = document.getElementById('tool-cookbook-btn');
914  if (btn) { btn.style.opacity = ''; btn.classList.remove('cookbook-notif-active'); }
915  const railBtn = document.getElementById('rail-cookbook');
916  if (railBtn) {
917    railBtn.classList.remove('rail-notify', 'rail-notify-success', 'cookbook-notif-active');
918  }
919  if (window._syncRailDynamic) window._syncRailDynamic();
920}
921
922// ââ Presets helper (for save-from-task) ââ
923
924// A preset must carry the venv + activated GPUs, not just the command â without
925// them a relaunch has no environment activated and no GPU pinning, so a config
926// that worked when saved fails on reload. Pull them from the launch payload
927// (_env/_envPath/_gpus, captured by _launchServeTask) and fold them into the
928// serve-form `fields` the Serve panel restores from.
929function _presetEnvFields(task) {
930  const p = task.payload || {};
931  const fields = { ...(p._fields || {}) };
932  // The Serve panel's venv field is a path; conda/venv both activate from it.
933  if (p._envPath && (p._env === 'venv' || p._env === 'conda')) fields.venv = fields.venv || p._envPath;
934  if (p._gpus) fields.gpus = p._gpus;
935  return {
936    fields: Object.keys(fields).length ? fields : undefined,
937    env: p._env || '',
938    envPath: p._envPath || '',
939    gpus: p._gpus || '',
940  };
941}
942
943function _saveTaskAsPreset(task, label) {
944  const host = task.remoteHost || 'localhost';

945  const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
946  const port = portMatch ? portMatch[1] : '8000';
947  const presets = _loadPresets();
948  if (presets.some(p => p.cmd === task.payload._cmd)) return false;
949  presets.push({ name: task.name, model: task.payload.repo_id, backend: 'vllm', host, port, cmd: task.payload._cmd, remoteHost: task.remoteHost || '', label: label || task.name, ..._presetEnvFields(task) });
950  _savePresets(presets);
951  return true;
952}
953
954// Same model-matching as cookbookServe's _presetsForModel, so the auto-save cap
955// counts the exact slots the Serve tab shows for this model.
956function _presetsForModelLocal(presets, repo) {
957  const short = (repo || '').split('/').pop();
958  return presets.filter(p => {
959    const pm = p.model || '', pn = p.name || '';
960    return pm === repo || pn === repo || pm.split('/').pop() === short || pn === short;
961  });
962}
963
964// Build a short auto-label from the launched command so an auto-saved config is
965// recognizable in the Saved dropdown (e.g. "TP2 Â· 16k ctx Â· AWQ").
966function _autoConfigLabel(task) {
967  const cmd = task.payload?._cmd || '';
968  const bits = [];
969  const tp = cmd.match(/--tensor-parallel-size[=\s]+(\d+)/);
970  if (tp && tp[1] !== '1') bits.push('TP' + tp[1]);
971  const ml = cmd.match(/--max-model-len[=\s]+(\d+)/);
972  if (ml) { const n = parseInt(ml[1]); bits.push((n >= 1024 ? Math.round(n / 1024) + 'k' : n) + ' ctx'); }
973  const q = (task.name || '').match(/AWQ|GPTQ|FP8|Q4|Q5|Q6|Q8|INT8|INT4/i);
974  if (q) bits.push(q[0].toUpperCase());
975  return bits.length ? bits.join(' Â· ') : 'working';
976}
977
978// Auto-save a serve config the moment its endpoint registers successfully, and
979// flag it confirmed-working. Dedups by exact command: if the same settings are
980// already saved we just upgrade that slot's badge instead of duplicating it.
981// Runs at most once per task.
982function _autoSaveWorkingConfig(task) {
983  if (!task || task.type !== 'serve' || !task.payload?._cmd) return;
984  if (task._autoSaved) return;
985  const cmd = task.payload._cmd;
986  // Diffusion/image servers aren't vLLM presets â skip them.
987  if (cmd.includes('diffusion_server')) { task._autoSaved = true; return; }
988  const model = task.payload.repo_id || task.name;
989  const presets = _loadPresets();
990  const existing = presets.find(p => p.cmd === cmd);
991  if (existing) {
992    task._autoSaved = true;
993    if (!existing.confirmedWorking) { existing.confirmedWorking = true; _savePresets(presets); }
994    return;   // already saved â just confirm it, no duplicate, no toast
995  }
996  // Respect the per-model cap the manual save flow uses (max 5).
997  if (_presetsForModelLocal(presets, model).length >= 5) { task._autoSaved = true; return; }
998  const host = task.remoteHost || 'localhost';

999  const portMatch = cmd.match(/--port[=\s]+(\d+)/);
1000  const port = portMatch ? portMatch[1] : '8000';
1001  presets.push({
1002    name: task.name, model, backend: 'vllm', host, port,
1003    cmd, remoteHost: task.remoteHost || '',
1004    label: _autoConfigLabel(task), confirmedWorking: true, autoSaved: true,
1005    ..._presetEnvFields(task),
1006  });
1007  _savePresets(presets);
1008  task._autoSaved = true;
1009  uiModule.showToast('Saved working config');
1010}
1011
1012// ââ Cross-device sync ââ
1013
1014let _syncTimer = null;
1015function _syncToServer() {
1016  // Debounce to coalesce bursts of writes, but keep latency low so the server
1017  // is effectively authoritative across devices
1018  clearTimeout(_syncTimer);
1019  _syncTimer = setTimeout(async () => {
1020    try {
1021      // Don't push a not-yet-hydrated state. A legit state always has at
1022      // least the "Local" server, so an empty servers list means we loaded
1023      // before GET /state populated _envState â syncing it would wipe the
1024      // saved servers. (The server has an anti-wipe guard too; this avoids
1025      // the needless round-trip.)
1026      if (!_envState || !Array.isArray(_envState.servers) || _envState.servers.length === 0) return;
1027      const state = {
1028        tasks: _loadTasks(),
1029        presets: _loadPresets(),
1030        env: _envState,
1031        serveState: null,
1032      };
1033      try { state.serveState = JSON.parse(localStorage.getItem(SERVE_STATE_KEY)); } catch {}
1034      await fetch('/api/cookbook/state', {
1035        method: 'POST', credentials: 'same-origin',
1036        headers: { 'Content-Type': 'application/json' },
1037        body: JSON.stringify(_stripStateSecrets(state)),
1038      });
1039    } catch {}
1040  }, 400);
1041}
1042
1043// Normalize state from server: collapse legacy duplicate keys to canonical form.
1044// - server.modelDir (singular) â server.modelDirs[0] (canonical)
1045// - strip â/â pollution from modelDirs
1046// - dedupe modelDirs
1047function _normalizeState(state) {
1048  if (!state || typeof state !== 'object') return state;
1049  if (state.env && Array.isArray(state.env.servers)) {
1050    for (const s of state.env.servers) {
1051      // Collapse legacy modelDir â modelDirs
1052      let dirs = Array.isArray(s.modelDirs) ? s.modelDirs : [];
1053      if (s.modelDir && !dirs.includes(s.modelDir)) dirs.push(s.modelDir);
1054      dirs = dirs
1055        .map(d => (d || '').replaceAll('\u2715', '').replaceAll('\u2716', '').trim())
1056        .filter(Boolean);
1057      if (!dirs.includes('~/.cache/huggingface/hub')) dirs.unshift('~/.cache/huggingface/hub');
1058      s.modelDirs = [...new Set(dirs)];
1059      delete s.modelDir; // Drop the legacy singular form
1060      // A download target that's no longer in the dir list falls back to the
1061      // default HF cache (empty) so we never download into an unscanned dir.
1062      if (s.downloadDir && !s.modelDirs.includes(s.downloadDir)) s.downloadDir = '';
1063    }
1064  }
1065  return state;
1066}
1067
1068export async function _syncFromServer() {
1069  try {
1070    const res = await fetch('/api/cookbook/state', { credentials: 'same-origin' });
1071    if (!res.ok) return false;
1072    const state = _normalizeState(await res.json());
1073    if (!state || !state.env) return false;
1074
1075    const localTasks = _loadTasks();
1076    const serverTasks = state.tasks || [];
1077
1078    const localIds = new Set(localTasks.map(t => t.sessionId));
1079    const merged = [...localTasks];
1080    for (const t of serverTasks) {
1081      if (!localIds.has(t.sessionId) && !_isTombstoned(t.sessionId)) {
1082        merged.push(t);
1083      }
1084    }
1085    localStorage.setItem(TASKS_KEY, JSON.stringify(merged.map(_stripTaskSecrets)));
1086
1087    if (state.env) {
1088      // The active server selection (remoteHost + its env/path/platform) is a
1089      // per-device, live choice. NEVER let the server's stored copy overwrite
1090      // it here â doing so silently snapped the active host back to whatever was
1091      // saved server-side, so downloads/scans ignored what the user just
1092      // picked. Sync only the shared non-secret settings (servers list, gpus, paths).
1093      const { remoteHost: _rh, env: _e, envPath: _ep, platform: _pf, ...settings } = state.env;
1094      delete settings.hfToken;
1095      Object.assign(_envState, settings);
1096      const { hfToken, ...safeState } = _envState;
1097      localStorage.setItem('cookbook-last-state', JSON.stringify(safeState));
1098    }
1099    if (state.presets) {
1100      localStorage.setItem(STORAGE_KEY, JSON.stringify(state.presets));
1101    }
1102    if (state.serveState) {
1103      localStorage.setItem(SERVE_STATE_KEY, JSON.stringify(state.serveState));
1104    }
1105    return true;
1106  } catch { return false; }
1107}
1108
1109// ââ Retry download ââ
1110
1111// Bounded auto-retry counter for downloads, keyed by model â network blips on
1112// big multi-file downloads are common and HF resumes from the .incomplete parts.
1113const _dlRetryCount = new Map();
1114const _DL_MAX_AUTO_RETRY = 2;
1115
1116// Kill + relaunch a task (download or serve). Shared by the â® â Restart action

1117// and the click-to-retry on a stalled download badge.
1118async function _retryTask(el, task) {
1119  if (el && el._abort) el._abort.abort();
1120  const badge = el?.querySelector('.cookbook-task-status');
1121  if (badge) { badge.textContent = 'restarting...'; badge.className = 'cookbook-task-status'; }
1122  try {
1123    await fetch('/api/shell/exec', {
1124      method: 'POST', credentials: 'same-origin',
1125      headers: { 'Content-Type': 'application/json' },
1126      body: JSON.stringify({ command: _tmuxGracefulKill(task) }),
1127    });
1128  } catch {}
1129  if (task.payload) {
1130    if (task.type === 'serve' && task.payload._cmd) {
1131      _removeTask(task.sessionId);
1132      _launchServeTask(task.name, task.payload.repo_id, task.payload._cmd, task.payload._fields, task.remoteHost || '');
1133    } else {
1134      uiModule.showToast('Retrying download â progress may look reset while HuggingFace checks cached files, then it should resume.', 7000);
1135      _updateTask(task.sessionId, {
1136        status: 'running',
1137        output: `${task.output || ''}\n\n[odysseus] Retrying download. Progress may briefly look like a fresh download while HuggingFace checks cached/incomplete files; cached partial files will be reused when available.`.trim(),
1138        _retrying: true,
1139      });
1140      _retryDownload(task.name, task.payload, task.sessionId);
1141    }
1142  }
1143}
1144
1145async function _retryDownload(name, payload, replaceSessionId = '') {
1146  try {
1147    // A retry means the fast hf_transfer path already failed once â fall back to
1148    // the plain, reliable downloader for this and any further attempt (it resumes
1149    // from the cached .incomplete files, so no progress is lost).
1150    const _payload = { ...(payload || {}), disable_hf_transfer: true };
1151    const res = await fetch('/api/model/download', {
1152      method: 'POST', credentials: 'same-origin',
1153      headers: { 'Content-Type': 'application/json' },
1154      body: JSON.stringify(_payload),
1155    });
1156    if (!res.ok) {
1157      uiModule.showToast('Download failed: HTTP ' + res.status);
1158      if (replaceSessionId) _updateTask(replaceSessionId, { status: 'crashed', _retrying: false });
1159      return;
1160    }
1161    const data = await res.json();
1162    if (!data.ok) {
1163      uiModule.showToast('Download failed: ' + (data.error || ''));
1164      if (replaceSessionId) _updateTask(replaceSessionId, { status: 'crashed', _retrying: false });
1165      return;
1166    }
1167    if (replaceSessionId) {
1168      const tasks = _loadTasks();
1169      const task = tasks.find(t => t.sessionId === replaceSessionId);
1170      if (task) {
1171        task.id = data.session_id;
1172        task.sessionId = data.session_id;
1173        task.status = 'running';
1174        task.output = '';
1175        task.ts = Date.now();
1176        task.payload = _payload;
1177        task._retrying = false;
1178        _saveTasks(tasks);
1179        _soloExpandTaskId = data.session_id;
1180        _renderRunningTab();
1181        _startBackgroundMonitor();
1182      } else {
1183        _addTask(data.session_id, name, 'download', _payload);
1184      }
1185    } else {
1186      _addTask(data.session_id, name, 'download', _payload);
1187    }
1188    uiModule.showToast(`Downloading ${name}...`);
1189  } catch (e) {
1190    uiModule.showToast('Download failed: ' + e.message);
1191    if (replaceSessionId) _updateTask(replaceSessionId, { status: 'crashed', _retrying: false });
1192  }
1193}
1194
1195// ââ Serve auto-fix (kill + relaunch with env var) ââ
1196
1197// Block stacked retries: once any "Retry with X" is clicked for a task, ignore
1198// every further retry click for it. Each retry fires its own _launchServeTask,
1199// so clicking several options â or one repeatedly during the fade-out / while a
1200// relaunch was loading â used to stack up multiple servers (e.g. 6 launches).
1201// The flag rides on the card element (removed right after), so it can't re-arm.
1202function _guardServeRetry(panel, taskEl) {
1203  if (!taskEl || taskEl.dataset.retrying) return false;
1204  taskEl.dataset.retrying = '1';
1205  panel.querySelectorAll('button').forEach(b => {
1206    b.disabled = true;
1207    b.style.opacity = '0.5';
1208    b.style.pointerEvents = 'none';
1209  });
1210  return true;
1211}
1212
1213export async function _serveAutoFix(panel, envVar) {
1214  const taskEl = panel.closest('.cookbook-task');
1215  if (!taskEl) return;
1216  const taskId = taskEl.dataset.taskId;
1217  const tasks = _loadTasks();
1218  const task = tasks.find(t => t.sessionId === taskId);
1219  if (!task || !task.payload) return;
1220  if (!_guardServeRetry(panel, taskEl)) return;
1221
1222  const killCmd = _tmuxCmd(task, `kill-session -t ${taskId}`);
1223  try {
1224    await fetch('/api/shell/exec', {
1225      method: 'POST', credentials: 'same-origin',
1226      headers: { 'Content-Type': 'application/json' },
1227      body: JSON.stringify({ command: killCmd }),
1228    });
1229  } catch {}
1230
1231  _animateOutThenRemove(taskEl, taskId);
1232
1233  const origCmd = task.payload._cmd || '';

1234  const newCmd = `export ${envVar} && ${origCmd}`;
1235
1236  const origHost = _envState.remoteHost;
1237  if (task.remoteHost) _envState.remoteHost = task.remoteHost;
1238  try {
1239    uiModule.showToast(`Retrying with ${envVar}...`);
1240    await _launchServeTask(task.name, task.payload.repo_id, newCmd);
1241  } finally {
1242    // Always restore â otherwise a thrown launch leaves the global host stuck
1243    // on this serve task, so later downloads/scans hit it.
1244    _envState.remoteHost = origHost;
1245  }
1246}
1247
1248// Open the Serve panel pre-filled for a task â the same flow as the task's
1249// Edit button, but optionally with a modified command (used by the diagnosis
1250// "Retry with X" buttons so a retry lands in the editable Serve panel with the
1251// adjusted setting, instead of blindly relaunching).
1252async function _openServeEditForTask(task, cmdOverride, fieldOverrides = null) {
1253  const repo = task.payload?.repo_id;
1254  if (!repo) { uiModule.showToast('No model info on this task'); return; }
1255  const cmd = cmdOverride || task.payload?._cmd;
1256  // A modified cmd must be re-parsed; otherwise prefer the exact launch fields.
1257  let fields = cmdOverride
1258    ? _parseServeCmdToFields(cmd)
1259    : (task.payload?._fields || (cmd ? _parseServeCmdToFields(cmd) : null));
1260  if (fieldOverrides && typeof fieldOverrides === 'object') {
1261    fields = { ...(fields || {}), ...fieldOverrides };
1262  }
1263  // Switch the active server to the one this serve ran on (mirrors _openEdit).
1264  const _tHost = task.remoteHost || '';
1265  _envState.remoteHost = _tHost;
1266  const _tSrv = _envState.servers.find(s => s.host === _tHost);
1267  if (_tSrv) { _envState.env = _tSrv.env || 'none'; _envState.envPath = _tSrv.envPath || ''; _envState.platform = _tSrv.platform || ''; }
1268  else if (!_tHost) { _envState.env = 'none'; _envState.envPath = ''; _envState.platform = ''; }
1269  document.querySelectorAll('#hwfit-server-select, #hwfit-dl-server, #hwfit-cache-server, #hwfit-deps-server').forEach(sel => {
1270    if (!sel || sel.tagName !== 'SELECT') return;
1271    sel.value = _tHost || 'local';
1272  });
1273  try {
1274    const { openServePanelForRepo } = await import('./cookbookServe.js');
1275    await openServePanelForRepo(repo, fields);
1276  } catch (err) {
1277    console.error('[cookbook] open serve panel failed', err);
1278    uiModule.showToast('Could not open serve panel');
1279  }
1280}
1281
1282export async function _serveAutoRetryReplace(panel, flag, value) {
1283  const taskEl = panel.closest('.cookbook-task');
1284  if (!taskEl) return;
1285  const taskId = taskEl.dataset.taskId;
1286  const tasks = _loadTasks();
1287  const task = tasks.find(t => t.sessionId === taskId);
1288  if (!task || !task.payload || !task.payload._cmd) return;
1289  if (!_guardServeRetry(panel, taskEl)) return;
1290
1291  try {
1292    await fetch('/api/shell/exec', {
1293      method: 'POST', credentials: 'same-origin',
1294      headers: { 'Content-Type': 'application/json' },
1295      body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${taskId}`) }),
1296    });
1297  } catch {}
1298
1299  _animateOutThenRemove(taskEl, taskId);
1300
1301  let newCmd = task.payload._cmd;
1302  const re = new RegExp(flag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\s+\\S+');
1303  if (re.test(newCmd)) {
1304    newCmd = newCmd.replace(re, `${flag} ${value}`);
1305  } else {
1306    newCmd += ` ${flag} ${value}`;
1307  }
1308
1309  const origHost = _envState.remoteHost;
1310  if (task.remoteHost) _envState.remoteHost = task.remoteHost;
1311  try {
1312    uiModule.showToast(`Retrying with ${flag} ${value}...`);
1313    await _launchServeTask(task.name, task.payload.repo_id, newCmd);
1314  } finally {
1315    _envState.remoteHost = origHost;
1316  }
1317}
1318
1319export async function _serveAutoRetryRemove(panel, flag) {
1320  const taskEl = panel.closest('.cookbook-task');
1321  if (!taskEl) return;
1322  const taskId = taskEl.dataset.taskId;
1323  const tasks = _loadTasks();
1324  const task = tasks.find(t => t.sessionId === taskId);
1325  if (!task || !task.payload || !task.payload._cmd) return;
1326  if (!_guardServeRetry(panel, taskEl)) return;
1327
1328  try {
1329    await fetch('/api/shell/exec', {
1330      method: 'POST', credentials: 'same-origin',
1331      headers: { 'Content-Type': 'application/json' },
1332      body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${taskId}`) }),
1333    });
1334  } catch {}
1335
1336  _animateOutThenRemove(taskEl, taskId);
1337

1338  let newCmd = task.payload._cmd;
1339  const re = new RegExp(flag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\s+\\S+');
1340  newCmd = newCmd.replace(re, '').replace(/\s{2,}/g, ' ').trim();
1341
1342  const origHost = _envState.remoteHost;
1343  if (task.remoteHost) _envState.remoteHost = task.remoteHost;
1344  try {
1345    uiModule.showToast(`Retrying without ${flag}...`);
1346    await _launchServeTask(task.name, task.payload.repo_id, newCmd);
1347  } finally {
1348    _envState.remoteHost = origHost;
1349  }
1350}
1351
1352export async function _serveAutoRetry(panel, flag) {
1353  const taskEl = panel.closest('.cookbook-task');
1354  if (!taskEl) return;
1355  const taskId = taskEl.dataset.taskId;
1356  const tasks = _loadTasks();
1357  const task = tasks.find(t => t.sessionId === taskId);
1358  if (!task || !task.payload || !task.payload._cmd) return;
1359  if (!_guardServeRetry(panel, taskEl)) return;
1360
1361  try {
1362    await fetch('/api/shell/exec', {
1363      method: 'POST', credentials: 'same-origin',
1364      headers: { 'Content-Type': 'application/json' },
1365      body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${taskId}`) }),
1366    });
1367  } catch {}
1368
1369  _animateOutThenRemove(taskEl, taskId);
1370
1371  let newCmd = task.payload._cmd;
1372  if (!newCmd.includes(flag)) {
1373    newCmd += ' ' + flag;
1374  }
1375
1376  const origHost = _envState.remoteHost;
1377  if (task.remoteHost) _envState.remoteHost = task.remoteHost;
1378  try {
1379    uiModule.showToast(`Retrying with ${flag}...`);
1380    await _launchServeTask(task.name, task.payload.repo_id, newCmd);
1381  } finally {
1382    _envState.remoteHost = origHost;
1383  }
1384}
1385
1386// ââ Edit-command prompt ââ
1387// Shows a small modal with a textarea pre-filled with the current serve cmd.
1388// Resolves to the edited string on Save, or null on Cancel.
1389function _promptEditServeCmd(currentCmd) {
1390  return new Promise((resolve) => {
1391    const overlay = document.createElement('div');
1392    overlay.className = 'cookbook-edit-overlay';
1393    overlay.innerHTML = `
1394      <div class="cookbook-edit-modal">
1395        <div class="cookbook-edit-title">Edit serve command</div>
1396        <textarea class="cookbook-edit-textarea" spellcheck="false"></textarea>
1397        <div class="cookbook-edit-actions">
1398          <button class="cookbook-edit-cancel memory-toolbar-btn">Cancel</button>
1399          <button class="cookbook-edit-save memory-toolbar-btn">Save &amp; relaunch</button>
1400        </div>
1401      </div>`;
1402    const ta = overlay.querySelector('.cookbook-edit-textarea');
1403    ta.value = currentCmd || '';
1404    document.body.appendChild(overlay);
1405    setTimeout(() => { ta.focus(); ta.setSelectionRange(ta.value.length, ta.value.length); }, 0);
1406
1407    const close = (result) => {
1408      overlay.remove();
1409      document.removeEventListener('keydown', onKey);
1410      resolve(result);
1411    };
1412    const onKey = (e) => {
1413      if (e.key === 'Escape') close(null);
1414      else if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) close(ta.value.trim() || null);
1415    };
1416    overlay.querySelector('.cookbook-edit-cancel').addEventListener('click', () => close(null));
1417    overlay.querySelector('.cookbook-edit-save').addEventListener('click', () => close(ta.value.trim() || null));
1418    overlay.addEventListener('click', (e) => { if (e.target === overlay) close(null); });
1419    document.addEventListener('keydown', onKey);
1420  });
1421}
1422
1423// ââ Launch serve task ââ
1424
1425// Best-effort reconstruction of serve-form field values from a raw launch
1426// command. Fallback for tasks created before _fields capture existed.
1427// Mirrors the regex parser in cookbookServe.js's _loadSlotIntoPanel.
1428function _parseServeCmdToFields(cmd) {
1429  if (!cmd) return null;
1430  const ex = (re) => { const m = cmd.match(re); return m ? m[1] : ''; };
1431  const fields = {
1432    backend: cmd.includes('llama_cpp') || cmd.includes('llama-server') ? 'llamacpp'
1433      : cmd.includes('diffusion_server') ? 'diffusers'
1434      : cmd.includes('sglang') ? 'sglang'
1435      : cmd.includes('ollama') ? 'ollama' : 'vllm',
1436    port: ex(/--port\s+(\d+)/) || '8000',
1437    tp: ex(/--tensor-parallel-size\s+(\d+)/) || '1',
1438    ctx: ex(/--max-model-len\s+(\d+)/) || ex(/--n_ctx\s+(\d+)/) || ex(/-c\s+(\d+)/) || '8192',
1439    gpu_mem: ex(/--gpu-memory-utilization\s+([\d.]+)/) || '0.90',
1440    swap: ex(/--swap-space\s+(\d+)/) || '',
1441    dtype: ex(/--dtype\s+(\w+)/) || 'auto',
1442    vllm_kv_cache_dtype: ex(/--kv-cache-dtype\s+([\w.-]+)/) || 'auto',
1443    max_seqs: ex(/--max-num-seqs\s+(\d+)/) || '',
1444    gpus: ex(/CUDA_VISIBLE_DEVICES=(\S+)/) || '',
1445    cache_type: ex(/(?:--cache-type-k|-ctk)\s+(\S+)/) || '',
1446    llama_fit: ex(/(?:--fit|-fit)\s+(on|off)/) || '',
1447    llama_split_mode: ex(/(?:--split-mode|-sm)\s+(none|layer|row|tensor)/) || '',
1448    llama_tensor_split: ex(/(?:--tensor-split|-ts)\s+([0-9.,]+)/) || '',
1449    llama_main_gpu: ex(/(?:--main-gpu|-mg)\s+(\d+)/) || '',
1450    llama_parallel: ex(/(?:--parallel|-np)\s+(\d+)/) || '',
1451    llama_batch_size: ex(/(?:--batch-size|-b)\s+(\d+)/) || '',
1452    llama_ubatch_size: ex(/(?:--ubatch-size|-ub)\s+(\d+)/) || '',
1453    llama_spec_tokens: ex(/--spec-draft-n-max\s+(\d+)/) || '3',
1454    enforce_eager: cmd.includes('--enforce-eager'),
1455    trust_remote: cmd.includes('--trust-remote-code'),
1456    prefix_cache: cmd.includes('--enable-prefix-caching'),
1457    auto_tool: cmd.includes('--enable-auto-tool-choice'),
1458    flash_attn: /--flash-attn\s+on\b/.test(cmd),
1459    unified_mem: /GGML_CUDA_ENABLE_UNIFIED_MEMORY=1/.test(cmd),
1460    llama_no_mmap: /--no-mmap\b/.test(cmd),
1461    llama_no_warmup: /--no-warmup\b/.test(cmd),
1462    llama_speculative_mtp: /--spec-type\s+\S*draft-mtp/.test(cmd),
1463    speculative: cmd.includes('--speculative-config'),
1464  };
1465  const spec = cmd.match(/--speculative-config\s+'?\{[^}]*"method"\s*:\s*"([^"]+)"[^}]*"num_speculative_tokens"\s*:\s*(\d+)/);
1466  if (spec) { fields.spec_method = spec[1]; fields.spec_tokens = spec[2]; }
1467  return fields;
1468}
1469
1470export async function _launchServeTask(shortName, repo, cmd, fields, hostOverride) {
1471  // Host resolution mirrors the download path: when the caller passes an explicit
1472  // host (resolved from the dropdown the user actually picked), use it and look
1473  // up that server's port/platform from the shared servers list. Only fall back
1474  // to _envState.remoteHost for legacy callers (diagnosis/pip-update).
1475  const _host = (hostOverride !== undefined) ? (hostOverride || '') : (_envState.remoteHost || '');
1476  const _hsrv = _envState.servers.find(s => s.host === _host) || {};
1477  const _hplatform = _host ? (_hsrv.platform || '') : (_envState.platform || '');
1478
1479  // Replace any serve already targeting this same host:port â you can't run two
1480  // servers on one port, so re-serving (or retrying) should stop & remove the
1481  // old one instead of leaving a dead duplicate behind. (The retry buttons
1482  // already removed their own task, so this is a no-op for them.)
1483  try {

1484    const _pm = cmd.match(/--port[=\s]+(\d+)/) || cmd.match(/(?:^|\s)-p[=\s]+(\d+)/);
1485    const _newPort = _pm ? _pm[1] : '';
1486    if (_newPort) {
1487      for (const _t of _loadTasks()) {
1488        if (_t.type !== 'serve' || !_t.payload || !_t.payload._cmd) continue;
1489        const _tm = _t.payload._cmd.match(/--port[=\s]+(\d+)/) || _t.payload._cmd.match(/(?:^|\s)-p[=\s]+(\d+)/);
1490        if ((_tm ? _tm[1] : '') === _newPort && (_t.remoteHost || '') === _host) {
1491          try {
1492            await fetch('/api/shell/exec', {
1493              method: 'POST', credentials: 'same-origin',
1494              headers: { 'Content-Type': 'application/json' },
1495              body: JSON.stringify({ command: _tmuxGracefulKill(_t) }),
1496            });
1497          } catch {}
1498          _removeTask(_t.sessionId);
1499        }
1500      }
1501    }
1502  } catch {}
1503  // Capture the env + GPU pin used for THIS launch BEFORE building the request.
1504  // The serve panel sets _envState.env/envPath/gpus, calls us, then restores them
1505  // synchronously â and our payload is built after an `await`, so reading
1506  // _envState there would see the restored (wrong) values. Persisting these lets
1507  // a saved preset relaunch with the same venv + GPUs (otherwise a confirmed
1508  // working config fails: no venv activation, no GPU pinning).
1509  const _usedEnv = _envState.env;
1510  const _usedEnvPath = _envState.envPath;
1511  const _usedGpus = _envState.gpus || '';
1512  let envPrefix = '';
1513  if (_isWindows()) {
1514    if (_envState.env === 'venv' && _envState.envPath) {
1515      envPrefix = '& ' + (_envState.envPath.endsWith('\\Scripts\\Activate.ps1') ? _envState.envPath : _envState.envPath + '\\Scripts\\Activate.ps1');
1516    } else if (_envState.env === 'conda' && _envState.envPath) {
1517      envPrefix = 'conda activate ' + _envState.envPath;
1518    }
1519  } else {
1520    if (_envState.env === 'venv' && _envState.envPath) {
1521      const p = _envState.envPath;
1522      envPrefix = 'source ' + (p.endsWith('/bin/activate') ? p : p + '/bin/activate');
1523    } else if (_envState.env === 'conda' && _envState.envPath) {
1524      envPrefix = 'eval "$(conda shell.bash hook)" && conda activate ' + _envState.envPath;
1525    }
1526  }
1527
1528  const reqBody = {
1529    repo_id: repo,
1530    cmd: cmd,
1531    remote_host: _host || undefined,
1532    ssh_port: _getPort(_host) || undefined,
1533    env_prefix: envPrefix || undefined,
1534    hf_token: _envState.hfToken || undefined,
1535    gpus: _envState.gpus || undefined,
1536    platform: _hplatform || undefined,
1537  };
1538
1539  try {
1540    const res = await fetch('/api/model/serve', {
1541      method: 'POST', credentials: 'same-origin',
1542      headers: { 'Content-Type': 'application/json' },
1543      body: JSON.stringify(reqBody),
1544    });
1545    const data = await res.json();
1546    if (!data.ok) {
1547      // Two error shapes: `{ok:false, error}` (tmux launch failed) or
1548      // `{detail}` (FastAPI HTTPException). Show whichever is present
1549      // + log full payload so the user can copy the error.
1550      const err = data.error || data.detail || res.statusText || 'unknown';
1551      console.error('[cookbook] /api/model/serve failed', { status: res.status, body: data });
1552      uiModule.showToast('Failed to start: ' + String(err).slice(0, 200), 9000);
1553      return;
1554    }
1555
1556    const _sp = _getPort(_host);
1557    // _fields = the exact structured serve-form values used for this launch,
1558    // so the "Edit / relaunch" button can re-open the Serve panel pre-filled
1559    // with these precise settings (not just the last-used-for-repo state).
1560    const payload = { repo_id: repo, remote_host: _host || undefined, ssh_port: _sp || undefined, _cmd: cmd, _fields: fields || undefined, _env: _usedEnv, _envPath: _usedEnvPath, _gpus: _usedGpus };
1561    _addTask(data.session_id, shortName, 'serve', payload);

1562    uiModule.showToast(`Serving ${shortName}...`);
1563  } catch (e) {
1564    uiModule.showToast('Failed: ' + e.message);
1565  }
1566}
1567
1568// ââ Render Running tab ââ
1569
1570export function _renderRunningTab() {
1571  // Auto-clear the sidebar notif (the bright-icon highlight) when no tasks
1572  // are actively running or errored. _showCookbookNotif fires on each task
1573  // event but the matching clear only ran on modal-open, so the highlight
1574  // persisted indefinitely after tasks finished in the background.
1575  try {
1576    const _activeTasks = _loadPrunedTasks().filter(t => t.status === 'running' || t.status === 'queued' || t.status === 'error');
1577    if (!_activeTasks.length) _clearCookbookNotif();
1578  } catch {}
1579
1580  const body = document.querySelector('#cookbook-modal .cookbook-body');
1581  if (!body) return;
1582
1583  // Capture expansion state so re-renders don't collapse whatever the user
1584  // had open. Task output: presence of .cookbook-task-collapsed means collapsed.
1585  // Section body: inline display:none means collapsed.
1586  const _collapsedTaskIds = new Set();
1587  const _expandedTaskIds = new Set();  // mobile: tasks the user explicitly opened
1588  body.querySelectorAll('.cookbook-task').forEach(tEl => {
1589    const id = tEl.dataset.taskId;
1590    if (!id) return;
1591    const wrap = tEl.querySelector('.cookbook-output-wrap');
1592    if (!wrap) return;
1593    if (wrap.classList.contains('cookbook-task-collapsed')) _collapsedTaskIds.add(id);
1594    else _expandedTaskIds.add(id);
1595  });
1596  // A new action was just started â collapse every existing card and open only
1597  // the new one (works on both desktop and the mobile collapse-by-default path).
1598  if (_soloExpandTaskId) {
1599    const _allIds = new Set([..._collapsedTaskIds, ..._expandedTaskIds]);
1600    _collapsedTaskIds.clear();
1601    _expandedTaskIds.clear();
1602    _allIds.forEach(id => { if (id !== _soloExpandTaskId) _collapsedTaskIds.add(id); });
1603    _expandedTaskIds.add(_soloExpandTaskId);
1604    _soloExpandTaskId = null;
1605  }
1606  // On mobile, task outputs start COLLAPSED â having every running window
1607  // expanded on entry meant a lot of tapping to collapse them. User-expanded
1608  // ones are re-opened from _expandedTaskIds below.
1609  const _mobileCollapseDefault = window.innerWidth <= 768;
1610  const _collapsedSectionIds = new Set();
1611  body.querySelectorAll('.cookbook-section-body').forEach(sb => {
1612    if (sb.style.display === 'none' && sb.id) _collapsedSectionIds.add(sb.id);
1613  });
1614
1615  const tasks = _loadTasks();
1616  const hasContent = tasks.length > 0;
1617  // Count anything that's really active: explicit 'running'/'queued' status,
1618  // OR a download whose tmux output is still showing live shard progress.
1619  // Without the output check, a task whose status got stuck at 'done' /
1620  // 'crashed' (before auto-reconnect catches it) would read as "Running 0"
1621  // even when the model is actively downloading on the host.
1622  const activeCount = tasks.filter(t =>
1623    t.status === 'running'
1624    || t.status === 'queued'
1625    || _downloadOutputLooksActive(t)
1626  ).length;
1627  const activeCountHtml = activeCount ? ` <span class="cookbook-tab-count">${activeCount}</span>` : '';
1628
1629  let tabBar = body.querySelector('.cookbook-tabs');
1630  if (!tabBar) return;
1631  let runTab = tabBar.querySelector('.cookbook-tab[data-backend="Running"]');
1632  if (hasContent && !runTab) {
1633    runTab = document.createElement('button');
1634    runTab.className = 'cookbook-tab';
1635    runTab.dataset.backend = 'Running';
1636    const _errCount = tasks.filter(t => t.status === 'error' || t.status === 'crashed').length;
1637    runTab.innerHTML = `Running${activeCountHtml}${_errCount ? `<span class="cookbook-tab-error-dot"></span>` : ''}`;
1638    tabBar.insertBefore(runTab, tabBar.firstChild);
1639    runTab.addEventListener('click', () => {
1640      tabBar.querySelectorAll('.cookbook-tab').forEach(t => t.classList.remove('active'));
1641      runTab.classList.add('active');
1642      body.querySelectorAll('.cookbook-group').forEach(g => {
1643        g.classList.toggle('hidden', g.dataset.backendGroup !== 'Running');
1644      });
1645    });
1646  } else if (runTab) {
1647    const _errCount2 = tasks.filter(t => t.status === 'error' || t.status === 'crashed').length;
1648    runTab.innerHTML = tasks.length ? `Running${activeCountHtml}${_errCount2 ? '<span class="cookbook-tab-error-dot"></span>' : ''}` : 'Running';
1649    if (!hasContent) {
1650      if (runTab.classList.contains('active')) {
1651        const wfTab = tabBar.querySelector('.cookbook-tab[data-backend="Search"]');
1652        if (wfTab) wfTab.click();
1653      }
1654      runTab.remove();
1655    }
1656  }
1657
1658  let group = body.querySelector('.cookbook-group[data-backend-group="Running"]');
1659  if (hasContent && !group) {
1660    group = document.createElement('div');
1661    group.className = 'cookbook-group hidden';
1662    group.dataset.backendGroup = 'Running';
1663    group.innerHTML = '<div class="admin-card" style="flex:1;display:flex;flex-direction:column;overflow:hidden;">' +
1664      '<div style="display:flex;align-items:baseline;gap:8px;margin-bottom:2px;">' +
1665      '<h2 style="margin:0;padding:0;line-height:1;">

1665Running <span id="running-count" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal">' + activeCount + '</span></h2>' +
1666      '</div>' +
1667      '<p class="memory-desc doclib-desc" style="margin-top:6px;">Active downloads and serving processes.</p>' +
1668      '</div>';
1669    const firstGroup = body.querySelector('.cookbook-group');
1670    if (firstGroup) body.insertBefore(group, firstGroup);
1671    else body.appendChild(group);
1672  }
1673
1674  if (!group) return;
1675
1676  const countEl = group.querySelector('#running-count');
1677  if (countEl) countEl.textContent = activeCount;
1678
1679  if (!hasContent) {
1680    group.remove();
1681    return;
1682  }
1683
1684  const _adminCard = group.querySelector('.admin-card');
1685  function _ensureSection(cls, label, items) {
1686    let sec = group.querySelector('.' + cls);
1687    if (!sec) {
1688      sec = document.createElement('div');
1689      sec.className = cls;
1690      (_adminCard || group).appendChild(sec);
1691    }
1692    if (!items || !items.length) {
1693      sec.style.display = 'none';
1694      return sec;
1695    }
1696    sec.style.display = '';
1697    return sec;
1698  }
1699
1700  // Group tasks by server
1701  const _serverName = (host) => {
1702    if (!host) return 'Local';
1703    const srv = _envState.servers.find(s => s.host === host);
1704    return srv?.name || host;
1705  };
1706  const serverGroups = {};
1707  for (const t of tasks) {
1708    const key = t.remoteHost || '';
1709    if (!serverGroups[key]) serverGroups[key] = { name: _serverName(key), serve: [], download: [] };
1710    serverGroups[key][t.type === 'serve' ? 'serve' : 'download'].push(t);
1711  }
1712
1713
1714  // ââ Server-grouped sections ââ
1715  group.querySelectorAll('.cookbook-serve-section, .cookbook-dl-section').forEach(el => el.remove());
1716
1717  const serverKeys = Object.keys(serverGroups).sort((a, b) => {
1718    if (!a) return -1; if (!b) return 1;
1719    return serverGroups[a].name.localeCompare(serverGroups[b].name);
1720  });
1721
1722  // Prune stale server sections: a server that no longer has ANY tasks isn't in
1723  // serverKeys, so its section header/dropdown would otherwise linger until the
1724  // user manually cleared it. Drop those automatically on each render.
1725  const _liveSafeKeys = new Set(serverKeys.map(k => (k || 'local').replace(/[^a-zA-Z0-9-]/g, '_')));
1726  (_adminCard || group).querySelectorAll('[class*="cookbook-server-section-"]').forEach(el => {
1727    const cls = [...el.classList].find(c => c.startsWith('cookbook-server-section-'));
1728    if (cls && !_liveSafeKeys.has(cls.replace('cookbook-server-section-', ''))) el.remove();
1729  });
1730
1731  for (const key of serverKeys) {
1732    const sg = serverGroups[key];
1733    const allTasks = [...sg.serve, ...sg.download];
1734    const safeKey = (key || 'local').replace(/[^a-zA-Z0-9-]/g, '_');
1735    const sectionCls = `cookbook-server-section-${safeKey}`;
1736    const bodyId = `server-body-${safeKey}`;
1737    let sec = _ensureSection(sectionCls, sg.name, allTasks);
1738    if (allTasks.length && !sec.querySelector('.cookbook-section-header')) {
1739      const clearId = `clear-server-${key || 'local'}`;
1740      // Glowy status dot next to the server name (like the Settings server card):
1741      // green when reachable, red if any serve task on it is crashed/unreachable.
1742      const _secDot = (key && allTasks.some(_serveTaskFailed)) ? 'fail' : 'ok';
1743      const _dotTitle = key ? (_secDot === 'fail' ? 'Server not responding' : 'Reachable') : 'Local (this machine)';
1744      sec.insertAdjacentHTML('afterbegin', `<div class="cookbook-section-header" data-collapse="${bodyId}"><svg class="cookbook-section-chevron" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round"><polyline points="6 9 12 15 18 9"/></svg><span class="cookbook-srv-status ${_secDot}" title="${_dotTitle}" style="flex-shrink:0;position:relative;top:0px;"></span><span class="cookbook-section-title" style="margin:0;">${esc(sg.name)}</span><button class="cookbook-btn cookbook-stop-all-btn" data-stop-server="${esc(key)}">Stop all</button><button class="cookbook-btn cookbook-clear-btn" data-clear-server="${esc(key)}">Clear finished</button></div><div id="${bodyId}" class="cookbook-section-body"></div>`);
1745    }
1746  }
1747
1748  // Wire clear all buttons
1749  group.querySelectorAll('[data-clear-server]').forEach(btn => {
1750    if (btn._bound) return;
1751    btn._bound = true;
1752    btn.addEventListener('click', async (e) => {
1753      e.stopPropagation();  // don't toggle the section collapse (was an inline onclick, blocked by CSP)
1754      const host = btn.dataset.clearServer;
1755      if (!await window.styledConfirm(`Clear finished tasks on ${_serverName(host)}?`, { confirmText: 'Clear' })) return;
1756      const allTasks = _loadTasks();
1757      const toRemove = allTasks.filter(t => (t.remoteHost || '') === host && _canClearTask(t));
1758      const remaining = allTasks.filter(t => (t.remoteHost || '') !== host || !_canClearTask(t));
1759      _saveTasks(remaining);
1760      // Fade/slide each finished card out (same exit as the per-card clear)
1761      // instead of yanking them instantly.
1762      toRemove.forEach(t => {
1763        const el = document.querySelector(`.cookbook-task[data-task-id="${t.sessionId}"]`);
1764        if (el) {
1765          if (el._abort) el._abort.abort();
1766          if (el._uptimeInterval) clearInterval(el._uptimeInterval);
1767          el.style.transition = 'opacity 0.35s ease, transform 0.35s ease';
1768          el.style.opacity = '0';
1769          el.style.transform = 'translateX(-10px)';
1770        }
1771      });
1772      // After the animation, remove the cards and tidy up the now-empty section.
1773      setTimeout(() => {
1774        toRemove.forEach(t => document.querySelector(`.cookbook-task[data-task-id="${t.sessionId}"]`)?.remove());
1775        // If this server's section is now empty (only finished tasks lived here),
1776        // remove the whole section so its header/title doesn't linger.
1777        const _sk = (host || 'local').replace(/[^a-zA-Z0-9-]/g, '_');
1778        const _sec = group.querySelector(`.cookbook-server-section-${_sk}`);
1779        if (_sec && !_sec.querySelector('.cookbook-task')) _sec.remove();
1780        if (!remaining.length) _renderRunningTab();
1781      }, 360);
1782    });
1783  });
1784
1785  // Wire "Stop all" buttons â stop every running task on that server.
1786  group.querySelectorAll('[data-stop-server]').forEach(btn => {
1787    if (btn._bound) return;
1788    btn._bound = true;
1789    btn.addEventListener('click', async (e) => {
1790      e.stopPropagation();  // don't toggle the section collapse
1791      const host = btn.dataset.stopServer;
1792      const running = _loadTasks().filter(t => (t.remoteHost || '') === host && t.status === 'running');

1793      if (!running.length) { uiModule.showToast(`Nothing running on ${_serverName(host)}`); return; }
1794      if (!await window.styledConfirm(`Stop ${running.length} running task${running.length > 1 ? 's' : ''} on ${_serverName(host)}?`, { confirmText: 'Stop all' })) return;
1795      // Mark every task as user-stopped BEFORE firing the kills so that the
1796      // download auto-retry logic never restarts a task the user just stopped.
1797      running.forEach(t => _updateTask(t.sessionId, { _userStopped: true }));
1798      // Reuse each task's own Stop action so it does the full teardown
1799      // (send C-c, drop the endpoint, mark stopped) consistently.
1800      running.forEach(t => {
1801        const el = document.querySelector(`.cookbook-task[data-task-id="${t.sessionId}"]`);
1802        el?.querySelector('.cookbook-task-action-stop')?.click();
1803      });
1804      uiModule.showToast(`Stopped ${running.length} task${running.length > 1 ? 's' : ''} on ${_serverName(host)}`);
1805    });
1806  });
1807
1808  // Section collapse/expand
1809  group.querySelectorAll('.cookbook-section-header[data-collapse]').forEach(hdr => {
1810    if (hdr._bound) return;
1811    hdr._bound = true;
1812    hdr.addEventListener('click', () => {
1813      const bodyId = hdr.dataset.collapse;
1814      const body = document.getElementById(bodyId);
1815      if (!body) return;
1816      const isHidden = body.style.display === 'none';
1817      body.style.display = isHidden ? '' : 'none';
1818      const chevron = hdr.querySelector('.cookbook-section-chevron');
1819      if (chevron) {
1820        // Collapsed â point right (â¶, click to expand); expanded â down (â¼).
1821        chevron.style.transform = isHidden ? '' : 'rotate(-90deg)';
1822        chevron.style.opacity = '';
1823      }
1824    });
1825  });
1826
1827  // Only add new tasks or update existing ones
1828  const existingIds = new Set();
1829  group.querySelectorAll('.cookbook-task').forEach(el => {
1830    const id = el.dataset.taskId;
1831    existingIds.add(id);
1832    const task = tasks.find(t => t.sessionId === id);
1833    if (task) {
1834      el.dataset.status = task.status;
1835      const isDone = task.status === 'done';
1836      // Type chip doubles as the "finished" badge once a task completes â both
1837      // download and serve show the same green FINISHED chip.
1838      const typeChip = el.querySelector('.cookbook-task-type');
1839      if (typeChip) {
1840        // Only DOWNLOAD tasks flip to "finished" when done â serve tasks keep
1841        // saying "serve" because the model is still running on that port.
1842        const isDoneDl = isDone && task.type === 'download';
1843        typeChip.textContent = isDoneDl ? 'finished' : task.type;
1844        typeChip.classList.toggle('cookbook-task-type-done', isDoneDl);
1845      }
1846      const badge = el.querySelector('.cookbook-task-status');
1847      if (badge) {
1848        const _bdg = _taskBadge(task);
1849        badge.textContent = _bdg.text;
1850        badge.className = 'cookbook-task-status' + (_bdg.cls ? ' ' + _bdg.cls : '');
1851        badge.style.display = '';
1852      }
1853      // Indicator: spinning wave while running, green check when finished.
1854      const wave = el.querySelector('.cookbook-task-wave');
1855      if (wave) wave.style.display = task.status === 'running' ? '' : 'none';
1856      const check = el.querySelector('.cookbook-task-check');
1857      if (check) {
1858        check.style.display = _canClearTask(task) ? '' : 'none';
1859        const label = check.querySelector('.cookbook-task-done-label');
1860        if (label) label.textContent = _clearPillLabel(task);
1861      }
1862      const startNow = el.querySelector('.cookbook-task-start-now');
1863      if (startNow) startNow.style.display = (task.type === 'download' && task.status === 'queued') ? '' : 'none';
1864      const terminalDiag = _terminalServeDiagnosis(task, el.querySelector('.cookbook-output-pre')?.textContent || task.output || '');
1865      if (terminalDiag) {
1866        _showDiagnosis(el, terminalDiag, el.querySelector('.cookbook-output-pre')?.textContent || task.output || '');
1867      } else {
1868        const existingDiag = el.querySelector('.cookbook-diagnosis');
1869        // Keep diagnosis for failed tasks even if output was cleared and we
1870        // can no longer re-derive the exact message â removing it would hide
1871        // the crash reason from the user.
1872        if (existingDiag && !['stopped', 'error', 'crashed', 'failed'].includes(task.status)) {
1873          existingDiag.remove();
1874        }
1875      }
1876    }
1877    if (!task) {
1878      if (el._uptimeInterval) { clearInterval(el._uptimeInterval); el._uptimeInterval = null; }
1879      el.remove();
1880    }
1881  });
1882
1883  // Add new task entries
1884  for (const task of tasks) {
1885    if (existingIds.has(task.sessionId)) continue;
1886
1887    const el = document.createElement('div');
1888    el.className = 'cookbook-task' + (task._unreachable && task.status === 'running' ? ' cookbook-task-unreachable' : '');
1889    el.dataset.taskId = task.sessionId;
1890    el.dataset.status = task.status;
1891    el.dataset.type = task.type || '';
1892
1893    const _bdg = _taskBadge(task);
1894    const _bdgTitle = (task._unreachable && task.status === 'running') ? ' title="Server not responding â it may have crashed"' : '';
1895    el.innerHTML = `
1896      <div class="cookbook-task-header">

1897        <span class="cookbook-task-type${(task.status === 'done' && task.type === 'download') ? ' cookbook-task-type-done' : ''}" data-type="${esc(task.type)}">${esc((task.status === 'done' && task.type === 'download') ? 'finished' : task.type)}</span>
1898        <span class="cookbook-task-name">${modelLogo(task.name)}${esc(task.name)}</span>
1899        <span class="cookbook-task-indicator"><span class="cookbook-task-wave" style="display:${task.status === 'running' ? '' : 'none'}"></span><span class="cookbook-task-check" title="Clear" style="display:${_canClearTask(task) ? '' : 'none'}"><svg class="cookbook-task-check-ico" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="#50fa7b" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg><svg class="cookbook-task-clear-ico" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg><span class="cookbook-task-done-label">${esc(_clearPillLabel(task))}</span><span class="cookbook-task-clear-label">clear</span></span></span>
1900        <button type="button" class="cookbook-task-start-now" title="Start this queued download now" style="display:${(task.type === 'download' && task.status === 'queued') ? '' : 'none'}">

1900<svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true"><polygon points="8 5 19 12 8 19 8 5"/></svg><span>start now</span></button>
1901        <span class="cookbook-task-status ${_bdg.cls}"${_bdgTitle}>${esc(_bdg.text)}</span>
1902        <button class="cookbook-task-menu-btn" title="Actions">&#8942;</button>
1903      </div>
1904      <div class="cookbook-task-sub"><span class="cookbook-task-session">${esc(task.sessionId)}</span><span class="cookbook-task-uptime" style="display:${((task.type === 'serve' || task.type === 'download') && task.status === 'running') ? '' : 'none'}"></span>${(task.type === 'download') ? `<span class="cookbook-task-dldir" title="Download destination" style="font-size:9px;color:var(--fg-muted);font-family:'Fira Code',monospace;opacity:0.4;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;max-width:40ch;">Dir: ${esc(task.payload?.local_dir || '~/.cache/huggingface/hub')}</span>` : ''}</div>
1905      <div class="cookbook-output-wrap cookbook-task-collapsible${_mobileCollapseDefault ? ' cookbook-task-collapsed' : ''}"><pre class="cookbook-output-pre">${esc(task.output || '')}</pre><button type="button" class="copy-code cookbook-output-copy"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg></button></div>
1906    `;
1907
1908    const _waveEl = el.querySelector('.cookbook-task-wave');
1909    if (_waveEl && task.status === 'running') _registerWaveEl(_waveEl);
1910
1911    const terminalDiag = _terminalServeDiagnosis(task, task.output || '');
1912    if (terminalDiag) _showDiagnosis(el, terminalDiag, task.output || '');
1913    if (!terminalDiag && (task.status === 'error' || task.status === 'crashed') && task._backendDiagnosis) {
1914      _showDiagnosis(el, task._backendDiagnosis, task.output || '');
1915    }
1916
1917    const _uptimeEl = el.querySelector('.cookbook-task-uptime');
1918    if (_uptimeEl && (task.type === 'serve' || task.type === 'download') && task.status === 'running') {
1919      const _startedAt = task.ts || Date.now();
1920      const _prefix = task.type === 'download' ? 'downloading' : 'uptime';
1921      el._uptimeInterval = setInterval(() => {
1922        const secs = Math.floor((Date.now() - _startedAt) / 1000);
1923        const h = Math.floor(secs / 3600);
1924        const m = Math.floor((secs % 3600) / 60);
1925        const s = secs % 60;
1926        const _timer = h > 0
1927          ? `${_prefix}: ${h}h ${String(m).padStart(2,'0')}m`
1928          : `${_prefix}: ${m}m ${String(s).padStart(2,'0')}s`;
1929        // ETA â only for downloads, only when we have a meaningful overall %.
1930        // Reads the badge text (which already shows the true overall % we
1931        // compute in the live-polling block) and back-derives a remaining-time
1932        // estimate from elapsed/done. Hidden until pct >= 3% so the early-job
1933        // wild estimates don't show.
1934        let _eta = '';
1935        if (task.type === 'download') {
1936          const _badge = el.querySelector('.cookbook-task-status');
1937          const _m = _badge && /^(\d+)%/.exec(_badge.textContent || '');
1938          const _pct = _m ? parseInt(_m[1], 10) : 0;
1939          if (_pct >= 3 && _pct < 100 && secs > 5) {
1940            const _totalSec = Math.round(secs * (100 / _pct));
1941            const _remain = Math.max(0, _totalSec - secs);
1942            const _eh = Math.floor(_remain / 3600);
1943            const _em = Math.floor((_remain % 3600) / 60);
1944            const _es = _remain % 60;
1945            _eta = _eh > 0
1946              ? ` Â· ETA ${_eh}h ${String(_em).padStart(2,'0')}m`
1947              : (_em > 0 ? ` Â· ETA ${_em}m ${String(_es).padStart(2,'0')}s` : ` Â· ETA ${_es}s`);
1948          }
1949        }
1950        _uptimeEl.textContent = _timer + _eta;
1951      }, 1000);
1952    }
1953
1954    // Re-open the Serve panel for this model, pre-filled with the EXACT
1955    // settings this instance launched with, and on the SERVER it runs on.
1956    const _openEdit = () => _openServeEditForTask(task);
1957    el.addEventListener('cookbook:edit-serve', (e) => {
1958      e.stopPropagation();
1959      _openServeEditForTask(task, null, e.detail?.fields || null);
1960    });
1961

1962    // Finished download â an explicit "Serve â" button jumps straight to the
1963    // Serve tab with this model pre-selected (on the server it downloaded to).
1964    if (task.type === 'download') {
1965      const _serveBtn = el.querySelector('.cookbook-task-serve-btn');
1966      if (_serveBtn) {
1967        _serveBtn.addEventListener('click', async (e) => {
1968          e.stopPropagation();
1969          const repo = task.payload?.repo_id || task.name;
1970          if (!repo) { uiModule.showToast('No model info on this task'); return; }
1971          // Point the active server at the one it downloaded to.
1972          const _tHost = task.remoteHost || '';
1973          _envState.remoteHost = _tHost;
1974          const _tSrv = _envState.servers.find(s => s.host === _tHost);
1975          if (_tSrv) { _envState.env = _tSrv.env || 'none'; _envState.envPath = _tSrv.envPath || ''; _envState.platform = _tSrv.platform || ''; }
1976          else if (!_tHost) { _envState.env = 'none'; _envState.envPath = ''; _envState.platform = ''; }
1977          document.querySelectorAll('#hwfit-server-select, #hwfit-dl-server, #hwfit-cache-server, #hwfit-deps-server').forEach(sel => {
1978            if (sel && sel.tagName === 'SELECT') sel.value = _tHost || 'local';
1979          });
1980          try {
1981            const { openServePanelForRepo } = await import('./cookbookServe.js');
1982            await openServePanelForRepo(repo);
1983            // Serving it supersedes the finished download â clear the card from
1984            // the Running tab (smooth exit) now that we've jumped to Serve.
1985            _animateOutThenRemove(el, task.sessionId);
1986          } catch (err) { uiModule.showToast('Could not open Serve: ' + err.message); }
1987        });
1988      }
1989    }
1990
1991    // Finished tasks show a green check â make it click-to-clear so the user can
1992    // dismiss a completed download/update (we no longer auto-remove them). It
1993    // morphs to a red â on hover (see CSS).
1994    const _clearChk = el.querySelector('.cookbook-task-check');
1995    if (_clearChk) {
1996      _clearChk.addEventListener('click', (e) => {
1997        e.stopPropagation();
1998        // If the output still shows an active shard line, the task isn't
1999        // actually finished â clicking is "reconnect" (flip back to running
2000        // + let _reconnectTask reattach to the live tmux session), not
2001        // "clear". The pill label already reflects this via _clearPillLabel.
2002        if (_downloadOutputLooksActive(task)) {
2003          const _fresh = _loadTasks();
2004          const _ft = _fresh.find(t => t.sessionId === task.sessionId);
2005          if (_ft) {
2006            _ft.status = 'running';
2007            _ft._selfHealed = true;
2008            _saveTasks(_fresh);
2009          }
2010          // Visually flip without waiting for a full re-render â same path the
2011          // self-heal uses on cookbook open.
2012          const _chk = el.querySelector('.cookbook-task-check');
2013          if (_chk) _chk.style.display = 'none';
2014          const _wave = el.querySelector('.cookbook-task-wave');
2015          if (_wave) _wave.style.display = '';
2016          const _up = el.querySelector('.cookbook-task-uptime');
2017          if (_up) _up.style.display = '';
2018          el.dataset.status = 'running';
2019          _renderRunningTab();
2020          return;
2021        }
2022        // Otherwise: real clear. Kill the tmux session as belt-and-suspenders,
2023        // then animate out + remove the row.
2024        try {
2025          fetch('/api/shell/exec', {
2026            method: 'POST', credentials: 'same-origin',
2027            headers: { 'Content-Type': 'application/json' },
2028            body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${task.sessionId}`) }),
2029          }).catch(() => {});
2030        } catch {}
2031        _animateOutThenRemove(el, task.sessionId);
2032      });
2033    }
2034
2035    const _startNowBtn = el.querySelector('.cookbook-task-start-now');
2036    if (_startNowBtn) {
2037      _startNowBtn.addEventListener('click', (e) => {
2038        e.stopPropagation();
2039        _startQueuedDownload(task);
2040      });
2041    }
2042
2043    // Wire header click to collapse/expand output
2044    el.querySelector('.cookbook-task-header').addEventListener('click', (e) => {
2045      if (e.target.closest('button')) return;
2046      const wrap = el.querySelector('.cookbook-output-wrap');
2047      if (wrap) wrap.classList.toggle('cookbook-task-collapsed');
2048    });
2049
2050    // Wire menu button (also fire from a long-press anywhere on the card so
2051    // mobile users don't have to hit the small â® target precisely).
2052    const menuBtn = el.querySelector('.cookbook-task-menu-btn');
2053    if (menuBtn) {

2054      // Long-press detection on the card: ~500ms hold without scroll movement
2055      // re-uses the menu button's click path (so we don't duplicate logic).
2056      let _lpTimer = null;
2057      let _lpStartY = 0;
2058      let _lpCanceled = false;
2059      const _lpStart = (e) => {
2060        _lpCanceled = false;
2061        _lpStartY = (e.touches?.[0]?.clientY) ?? 0;
2062        _lpTimer = setTimeout(() => {
2063          if (_lpCanceled) return;
2064          _lpCanceled = true;  // suppress the subsequent click-through
2065          try { menuBtn.click(); } catch {}
2066        }, 500);
2067      };
2068      const _lpCancel = () => {
2069        if (_lpTimer) { clearTimeout(_lpTimer); _lpTimer = null; }
2070      };
2071      const _lpMove = (e) => {
2072        const y = (e.touches?.[0]?.clientY) ?? 0;
2073        if (Math.abs(y - _lpStartY) > 8) _lpCancel();
2074      };
2075      el.addEventListener('touchstart', (e) => {
2076        // Skip if the user is starting touch on a button / link inside the
2077        // card â those already have their own tap handlers.
2078        if (e.target.closest('button, a, input, textarea, .cookbook-task-dropdown')) return;
2079        _lpStart(e);
2080      }, { passive: true });
2081      el.addEventListener('touchmove', _lpMove, { passive: true });
2082      el.addEventListener('touchend', _lpCancel, { passive: true });
2083      el.addEventListener('touchcancel', _lpCancel, { passive: true });
2084      menuBtn.addEventListener('click', (e) => {
2085        e.stopPropagation();
2086        document.querySelectorAll('.cookbook-task-dropdown').forEach(d => { if (typeof d._dismiss === 'function') d._dismiss(); else d.remove(); });
2087
2088        const dropdown = document.createElement('div');
2089        dropdown.className = 'cookbook-task-dropdown';
2090
2091        const items = [];
2092        // Queued download: let the user jump the queue and start it immediately
2093        // (downloads otherwise run one-at-a-time per server).
2094        if (task.type === 'download' && task.status === 'queued') {
2095          items.push({ label: 'Start now', action: 'start-now', custom: () => {
2096            _startQueuedDownload(task);
2097            _renderRunningTab();
2098          }});
2099        }
2100        if (task.status !== 'running' && task.status !== 'queued') {
2101          items.push({ label: 'Reconnect', action: 'reconnect' });
2102        }
2103        if (task.status === 'running') {
2104          items.push({ label: 'Stop', action: 'stop', danger: true });
2105        }
2106        items.push({ label: 'Restart', action: 'retry' });
2107        // Edit serve â open the full serve panel (same as the edit icon),
2108        // switching to this task's server first so the model is found.
2109        if (task.type === 'serve' && task.payload?.repo_id) {
2110          items.push({ label: 'Edit in serve panel', action: 'edit-panel', tooltip: 'Open the full Serve config panel pre-filled with this task â pick a different backend, change GPUs, edit env vars, then Launch from there', custom: () => _openEdit() });
2111        }
2112        // Save serve â save current launch config as a preset.
2113        if (task.type === 'serve' && task.payload?._cmd) {
2114          items.push({ label: 'Save serve', action: 'save', custom: () => {
2115            if (!_saveTaskAsPreset(task)) { uiModule.showToast('Already saved'); return; }
2116            uiModule.showToast('Saved to presets');
2117            _renderRunningTab();
2118          }});
2119        }
2120        // Edit command â only meaningful for serve tasks that aren't running.
2121        // Lets the user tweak flags after a crash/error and relaunch.
2122        if (task.type === 'serve' && task.status !== 'running' && task.payload?._cmd) {
2123          items.push({ label: 'Edit cmd & relaunch', action: 'edit', tooltip: 'Edit the raw vllm/llama-server cmd string in a dialog and relaunch immediately on the same host', custom: async () => {
2124            const newCmd = await _promptEditServeCmd(task.payload._cmd);
2125            if (newCmd == null) return; // cancelled
2126            try {
2127              await fetch('/api/shell/exec', {
2128                method: 'POST', credentials: 'same-origin',
2129                headers: { 'Content-Type': 'application/json' },
2130                body: JSON.stringify({ command: _tmuxGracefulKill(task) }),
2131              });
2132            } catch {}
2133            _removeTask(task.sessionId);
2134            // Relaunch on the task's OWN host, not the current global selection.
2135            _launchServeTask(task.name, task.payload.repo_id, newCmd, task.payload._fields, task.remoteHost || '');
2136          }});
2137        }
2138        // Manual endpoint registration â fallback for when auto-add fails
2139        // (e.g. probe timeout on a remote that's slow). Forces adding this
2140        // serve to the model-endpoints list regardless of prior flag state.
2141        if (task.type === 'serve' && task.payload?._cmd) {
2142          items.push({ label: 'Register endpoint', action: 'register-endpoint', custom: async () => {
2143            const host = _connectHostFromRemote(task.remoteHost);

2144            const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
2145            const port = portMatch ? portMatch[1] : '8000';
2146            const baseUrl = `http://${host}:${port}/v1`;
2147            try {
2148              // Check existing first â offer to overwrite if present
2149              const eps = await (await fetch('/api/model-endpoints', { credentials: 'same-origin' })).json();
2150              const existing = eps.find(e => e.base_url === baseUrl);
2151              if (existing) {
2152                uiModule.showToast(`Already registered as "${existing.name}"`);
2153                task._endpointAdded = true;
2154                _updateTask(task.sessionId, { _endpointAdded: true });
2155                _refreshModelsAfterEndpointChange();
2156                // If it's still offline (registered before the server finished
2157                // loading), keep probing until it answers instead of leaving it
2158                // stuck offline until a manual delete/re-add.
2159                if (existing.id && !(existing.models || []).length) _probeEndpointUntilOnline(existing.id, host, port);
2160                return;
2161              }
2162              const fd = new FormData();
2163              fd.append('base_url', baseUrl);
2164              fd.append('name', task.name);
2165              fd.append('skip_probe', 'true');
2166              _appendCookbookEndpointScope(fd, task.remoteHost || '');
2167              if (task.payload?._cmd?.includes('diffusion_server')) fd.append('model_type', 'image');
2168              const res = await fetch('/api/model-endpoints', { method: 'POST', credentials: 'same-origin', body: fd });
2169              if (res.ok) {
2170                task._endpointAdded = true;
2171                _updateTask(task.sessionId, { _endpointAdded: true });
2172                uiModule.showToast(`Endpoint registered: ${host}:${port}`);
2173                _refreshModelsAfterEndpointChange();
2174                // Added with skip_probe â probe until the (possibly still
2175                // warming) server answers, so it flips online on its own.
2176                const _ep = await res.json().catch(() => ({}));
2177                if (_ep && _ep.id) _probeEndpointUntilOnline(_ep.id, host, port);
2178              } else {
2179                const body = await res.text().catch(() => '');
2180                uiModule.showError(`Register failed: ${res.status} ${body.slice(0, 140)}`);
2181              }
2182            } catch (e) {
2183              uiModule.showError(`Register failed: ${e.message || e}`);
2184            }
2185          }});
2186        }
2187        if (_isWindows(task)) {
2188          const host = task.remoteHost;
2189          const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux';
2190          const logCmd = host
2191            ? `ssh ${_sshPrefix(_getPort(task))}${host} "powershell -Command \\"Get-Content '${sd}\\${task.sessionId}.log' -Wait\\""`
2192            : `powershell -Command "Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${task.sessionId}.log') -Wait"`;
2193          items.push({ label: 'Copy log cmd', action: 'copy-tmux', custom: () => {
2194            _copyText(logCmd);
2195          }});
2196        } else {
2197          // Just the tmux command itself â no ssh wrapper.
2198          const tmuxAttach = `tmux attach -t ${task.sessionId}`;
2199          items.push({ label: 'Copy tmux', action: 'copy-tmux', custom: () => {
2200            _copyText(tmuxAttach);
2201          }});
2202        }
2203        if (_shouldOfferCrashReport(task)) {
2204          items.push({ label: 'Copy crash report', action: 'copy-crash-report', custom: () => {
2205            const out = (el.querySelector('.cookbook-output-pre')?.textContent || task.output || '');
2206            _copyText(_buildCrashReport(task, out));
2207            uiModule.showToast('Copied crash report');
2208          }});
2209        }
2210        // Copy the last 50 lines of the task's output/log.
2211        items.push({ label: 'Copy last 50 lines', action: 'copy-log', custom: () => {
2212          const out = (el.querySelector('.cookbook-output-pre')?.textContent || task.output || '');
2213          const last = out.split('\n').slice(-50).join('\n');
2214          if (!last.trim()) {
2215            uiModule.showToast('No log content available yet');
2216            return;
2217          }
2218          _copyText(last);
2219          uiModule.showToast('Copied last 50 lines');
2220        }});
2221        // Label matches behavior â the kill handler ALWAYS first kills
2222        // the live tmux session and (for serve tasks) deletes the
2223        // matching model-endpoint, THEN animates the task card out.
2224        // Just "Remove" hid that it stops the live serve too.
2225        const _isLive = task.type === 'serve' && ['running', 'ready', 'loading', 'warming', 'starting'].includes(task.status || '');
2226        items.push({
2227          label: _isLive ? 'Stop and remove' : 'Remove',
2228          action: 'kill',
2229          tooltip: _isLive
2230            ? 'Kill the live tmux session, deregister the chat endpoint, and remove this row'
2231            : 'Remove this row',
2232          danger: true,
2233        });
2234        // Cancel = mobile-only dismiss item. Same pattern as the email kebab:
2235        // the `dropdown-cancel-mobile` class is hidden on desktop and styled
2236        // as a separated bottom row on mobile (border-top + extra padding).
2237        items.push({ label: 'Cancel', action: 'cancel', mobileOnly: true, custom: () => {} });
2238
2239        const _MENU_ICONS = {
2240          'start-now': '<polygon points="6 4 20 12 6 20 6 4"/>',
2241          reconnect: '<path d="M1 4v6h6"/><path d="M3.5 15a9 9 0 1 0 2.1-9.4L1 10"/>',
2242          retry: '<path d="M1 4v6h6"/><path d="M3.5 15a9 9 0 1 0 2.1-9.4L1 10"/>',
2243          stop: '<rect x="6" y="6" width="12" height="12" rx="1"/>',
2244          edit: '<path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"/><path d="M18.5 2.5a2.12 2.12 0 0 1 3 3L12 15l-4 1 1-4z"/>',
2245          'edit-panel': '<path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"/><path d="M18.5 2.5a2.12 2.12 0 0 1 3 3L12 15l-4 1 1-4z"/>',
2246          'register-endpoint': '<circle cx="12" cy="12" r="9"/><path d="M12 8v8M8 12h8"/>',
2247          save: '<path d="M19 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11l5 5v11a2 2 0 0 1-2 2z"/><path d="M17 21v-8H7v8M7 3v5h8"/>',
2248          'copy-tmux': '<rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/>',
2249          'copy-crash-report': '<path d="M10.3 2.3 1.8 17a2 2 0 0 0 1.7 3h17a2 2 0 0 0 1.7-3L13.7 2.3a2 2 0 0 0-3.4 0z"/><path d="M12 8v5M12 17h.01"/>',
2250          'copy-log': '<rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/>',
2251          kill: '<path d="M3 6h18"/><path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6m3 0V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"/>',
2252          cancel: '<line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/>',
2253        };
2254        for (const item of items) {
2255          const div = document.createElement('div');
2256          div.className = 'dropdown-item-compact'
2257            + (item.danger ? ' cookbook-dropdown-danger' : '')
2258            + (item.mobileOnly ? ' dropdown-cancel-mobile' : '');
2259          div.style.cssText = 'display:flex;align-items:center;gap:8px;';
2260          if (item.tooltip) div.title = item.tooltip;
2261          const ic = _MENU_ICONS[item.action] || '';
2262          div.innerHTML = `<span style="display:inline-flex;flex-shrink:0;

2262opacity:0.7;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">${ic}</svg></span><span>${item.label}</span>`;
2263          div.addEventListener('click', () => {
2264            _cleanup();
2265            if (item.custom) { item.custom(); return; }
2266            el.querySelector('.cookbook-task-action-' + item.action)?.click();
2267          });
2268          dropdown.appendChild(div);
2269        }
2270
2271        const rect = menuBtn.getBoundingClientRect();
2272        dropdown.style.position = 'fixed';
2273        dropdown.style.top = rect.bottom + 2 + 'px';
2274        dropdown.style.right = (window.innerWidth - rect.right) + 'px';
2275        document.body.appendChild(dropdown);
2276        // Clamp into the *visible* area. On mobile (esp. Firefox) window.innerHeight
2277        // includes the strip hidden under the dynamic toolbar, so a menu that "fits"
2278        // by innerHeight still lands off-screen at the bottom. visualViewport gives
2279        // the real visible region. Flip above the button if there's no room below,
2280        // else clamp to the bottom edge.
2281        {
2282          const vv = window.visualViewport;
2283          const viewTop = vv ? vv.offsetTop : 0;
2284          const viewBottom = vv ? vv.offsetTop + vv.height : window.innerHeight;
2285          const dh = dropdown.offsetHeight;
2286          const m = 8;
2287          let top = rect.bottom + 2;
2288          if (top + dh > viewBottom - m) {
2289            const above = rect.top - 2 - dh;
2290            top = above >= viewTop + m ? above : Math.max(viewTop + m, viewBottom - dh - m);
2291          }
2292          dropdown.style.top = top + 'px';
2293        }
2294
2295        const closeHandler = (ev) => {
2296          if (!dropdown.contains(ev.target) && ev.target !== menuBtn) {
2297            _cleanup();
2298          }
2299        };
2300        // Close on scroll too â once the page scrolls, the dropdown's
2301        // fixed position no longer matches the originating â® button, so
2302        // it visually drifts. Matches the email kebab behaviour.
2303        const scrollClose = () => _cleanup();
2304        let _unreg = () => {};
2305        const _cleanup = () => {
2306          _unreg(); _unreg = () => {};
2307          dropdown.remove();
2308          document.removeEventListener('click', closeHandler);
2309          window.removeEventListener('scroll', scrollClose, true);
2310          window.visualViewport?.removeEventListener('scroll', scrollClose);
2311        };
2312        dropdown._dismiss = _cleanup;
2313        setTimeout(() => {
2314          document.addEventListener('click', closeHandler);
2315          window.addEventListener('scroll', scrollClose, true);
2316          window.visualViewport?.addEventListener('scroll', scrollClose);
2317        }, 0);
2318        _unreg = registerMenuDismiss(_cleanup);
2319      });
2320    }
2321
2322    // Hidden action buttons for menu dispatch
2323    const _actionBtns = document.createElement('div');
2324    _actionBtns.style.display = 'none';
2325    _actionBtns.innerHTML = `
2326      <button class="cookbook-task-action-reconnect"></button>
2327      <button class="cookbook-task-action-retry"></button>
2328      <button class="cookbook-task-action-stop"></button>
2329      <button class="cookbook-task-action-kill"></button>
2330    `;
2331    el.appendChild(_actionBtns);
2332
2333    // Wire reconnect
2334    el.querySelector('.cookbook-task-action-reconnect').addEventListener('click', () => {
2335      _updateTask(task.sessionId, { status: 'running' });
2336      el.dataset.status = 'running';
2337      const badge = el.querySelector('.cookbook-task-status');
2338      if (badge) { badge.textContent = _statusLabel('running', task.type); badge.className = 'cookbook-task-status cookbook-task-running'; }
2339      _reconnectTask(el, task);
2340    });
2341
2342    // Wire stop
2343    el.querySelector('.cookbook-task-action-stop').addEventListener('click', async () => {
2344      // Abort the reconnect loop before sending kill so that a DOWNLOAD_FAILED
2345      // marker written by the shell wrapper (on SIGINT/non-zero exit) cannot
2346      // trigger an auto-retry after a manual stop.
2347      if (el._abort) el._abort.abort();
2348      const badge = el.querySelector('.cookbook-task-status');
2349      if (badge) { badge.textContent = 'stopping...'; badge.className = 'cookbook-task-status cookbook-task-stopping'; }
2350      el.dataset.status = 'stopped';
2351      _updateTask(task.sessionId, { _userStopped: true });
2352      const outputText = el.querySelector('.cookbook-output-pre')?.textContent || task.output || '';
2353      // Drop the model endpoint so the picker stops listing it.
2354      if (task.type === 'serve' && task.payload) {
2355        _removeEndpointByUrl(_endpointUrlForTask(task, outputText));
2356      }
2357      const ollamaUnload = _ollamaUnloadCommand(task, outputText);
2358      if (ollamaUnload) {
2359        try {
2360          await fetch('/api/shell/exec', {
2361            method: 'POST', credentials: 'same-origin',
2362            headers: { 'Content-Type': 'application/json' },
2363            body: JSON.stringify({ command: ollamaUnload }),
2364          });
2365        } catch {}
2366      }
2367      // Gracefully stop (C-c, then kill the session) so it's fully down...

2368      try {
2369        await fetch('/api/shell/exec', {
2370          method: 'POST', credentials: 'same-origin',
2371          headers: { 'Content-Type': 'application/json' },
2372          body: JSON.stringify({ command: _tmuxGracefulKill(task) }),
2373        });
2374      } catch {}
2375      // ...then smoothly fade/slide the card out and auto-remove it â no manual
2376      // â® â Remove needed.
2377      _animateOutThenRemove(el, task.sessionId);
2378    });
2379
2380    // Wire kill â awaits the SSH/tmux kill and verifies the session is
2381    // actually gone before removing the row. Previously fire-and-forget,
2382    // which meant a failed kill (wrong remoteHost, SSH error, tmux server
2383    // already exited) silently left the live serve running while the
2384    // row disappeared from the UI.
2385    el.querySelector('.cookbook-task-action-kill').addEventListener('click', async () => {
2386      const outputText = el.querySelector('.cookbook-output-pre')?.textContent || task.output || '';
2387      const isLive = task.type === 'serve' && ['running', 'ready', 'loading', 'warming', 'starting'].includes(task.status || '');
2388      const ollamaUnload = _ollamaUnloadCommand(task, outputText);
2389      if (ollamaUnload) {
2390        try {
2391          await fetch('/api/shell/exec', {
2392            method: 'POST', credentials: 'same-origin',
2393            headers: { 'Content-Type': 'application/json' },
2394            body: JSON.stringify({ command: ollamaUnload }),
2395          });
2396        } catch (_) { /* unload best-effort */ }
2397      }
2398      let killOk = true;
2399      try {
2400        const r = await fetch('/api/shell/exec', {
2401          method: 'POST', credentials: 'same-origin',
2402          headers: { 'Content-Type': 'application/json' },
2403          body: JSON.stringify({ command: _tmuxGracefulKill(task) }),
2404        });
2405        if (r.ok) {
2406          const out = await r.json();
2407          // Don't trust exit_code alone â tmux kill returns 0 even when
2408          // there was nothing to kill. Verify the session is actually gone.
2409          if (task.sessionId && isLive) {
2410            try {
2411              const probe = await fetch('/api/shell/exec', {
2412                method: 'POST', credentials: 'same-origin',
2413                headers: { 'Content-Type': 'application/json' },
2414                body: JSON.stringify({ command: _tmuxCmd(task, `has-session -t ${task.sessionId}`) }),
2415              });
2416              if (probe.ok) {
2417                const pj = await probe.json();
2418                // has-session exits 0 when session STILL exists; non-zero = gone.
2419                if ((pj.exit_code || 0) === 0) killOk = false;
2420              }
2421            } catch (_) { /* probe best-effort; trust kill */ }
2422          }
2423        } else {
2424          killOk = false;
2425        }
2426      } catch (_) { killOk = false; }
2427      if (!killOk) {
2428        try { uiModule.showToast('Kill failed â session may still be running. Check `tmux ls` on the server.', 'error'); } catch (_) {}
2429        return;  // leave the row so the user can retry
2430      }
2431      if (task.type === 'serve' && task.payload) {
2432        const endpointUrl = _endpointUrlForTask(task, outputText);
2433        _removeEndpointByUrl(endpointUrl);
2434        const modelName = task.payload.model || task.name || '';
2435        if (modelName) {
2436          fetch('/api/model-endpoints', { credentials: 'same-origin' })
2437            .then(r => r.json())
2438            .then(eps => {
2439              const ep = eps.find(e => e.name === modelName || e.base_url === endpointUrl);
2440              if (ep) fetch(`/api/model-endpoints/${ep.id}`, { method: 'DELETE', credentials: 'same-origin' }).then(() => _refreshModelsAfterEndpointChange());
2441            }).catch(() => {});
2442        }
2443      }
2444      _animateOutThenRemove(el, task.sessionId);
2445    });
2446
2447    // Wire retry
2448    el.querySelector('.cookbook-task-action-retry').addEventListener('click', () => _retryTask(el, task));
2449
2450    // Wire copy button
2451    el.querySelector('.cookbook-output-copy').addEventListener('click', (e) => {
2452      e.stopPropagation();
2453      const text = el.querySelector('.cookbook-output-pre')?.textContent || '';
2454      if (!text.trim()) {
2455        uiModule.showToast('No log content available yet');
2456        return;
2457      }
2458      _copyText(text).then(() => {
2459        const btn = el.querySelector('.cookbook-output-copy');
2460        const origHTML = btn.innerHTML;
2461        btn.innerHTML = '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg>';
2462        btn.classList.add('copied');
2463        setTimeout(() => { btn.innerHTML = origHTML; btn.classList.remove('copied'); }, 1500);
2464      });
2465    });
2466
2467    // Route to the right server section body
2468    const serverBodyId = `server-body-${(task.remoteHost || 'local').replace(/[^a-zA-Z0-9-]/g, '_')}`;

2469    const targetBody = document.getElementById(serverBodyId);
2470    if (targetBody) targetBody.appendChild(el);
2471    else group.appendChild(el);
2472
2473    // Auto-attach the tmux output stream for any task whose underlying
2474    // session could still be alive â not just 'running'. Scheduler-
2475    // launched serves transition to 'ready' as soon as /v1/models
2476    // responds; without this, the user opens the Running tab and sees
2477    // only the placeholder ("Launched by scheduled task â¦") because
2478    // _reconnectTask never fires for status 'ready'/'loading'/'warming'.
2479    if (['running', 'ready', 'loading', 'warming', 'starting'].includes(task.status)) {
2480      _reconnectTask(el, task);
2481    }
2482  }
2483
2484  if (tasks.some(t => t.status === 'running')) _startWaveSync();
2485
2486  // Re-apply captured expansion state so re-renders don't fold open tasks/sections.
2487  _collapsedTaskIds.forEach((id) => {
2488    const wrap = body.querySelector(`.cookbook-task[data-task-id="${id}"] .cookbook-output-wrap`);
2489    if (wrap) wrap.classList.add('cookbook-task-collapsed');
2490  });
2491  // Mobile defaults to collapsed (above), so re-open whatever the user had
2492  // explicitly expanded before this re-render.
2493  if (_mobileCollapseDefault) {
2494    _expandedTaskIds.forEach((id) => {
2495      const wrap = body.querySelector(`.cookbook-task[data-task-id="${id}"] .cookbook-output-wrap`);
2496      if (wrap) wrap.classList.remove('cookbook-task-collapsed');
2497    });
2498  }
2499  _collapsedSectionIds.forEach((sid) => {
2500    const sb = document.getElementById(sid);
2501    if (sb) sb.style.display = 'none';
2502    const hdr = body.querySelector(`.cookbook-section-header[data-collapse="${sid}"]`);
2503    const chevron = hdr?.querySelector('.cookbook-section-chevron');
2504    if (chevron) { chevron.style.transform = 'rotate(-90deg)'; chevron.style.opacity = ''; }
2505  });
2506}
2507
2508// ââ Reconnect task (polling loop) ââ
2509
2510async function _reconnectTask(el, task) {
2511  const output = el.querySelector('.cookbook-output-pre');
2512  const controller = new AbortController();
2513  el._abort = controller;
2514  let failCount = 0;
2515
2516  while (!controller.signal.aborted) {
2517    if (!el.isConnected) {
2518      controller.abort();
2519      break;
2520    }
2521    try {
2522      const res = await fetch('/api/shell/exec', {
2523        method: 'POST', credentials: 'same-origin',
2524        headers: { 'Content-Type': 'application/json' },
2525        body: JSON.stringify({ command: _tmuxCmd(task, `capture-pane -t ${task.sessionId} -p -S -200`), timeout: 15 }),
2526      });
2527      const data = await res.json();
2528
2529      if (data.exit_code !== 0) {
2530        failCount++;
2531        if (failCount < 5) {
2532          await new Promise(r => setTimeout(r, 3000));
2533          continue;
2534        }
2535        try {
2536          const verify = await fetch('/api/shell/exec', {
2537            method: 'POST', credentials: 'same-origin',
2538            headers: { 'Content-Type': 'application/json' },
2539            body: JSON.stringify({ command: _tmuxCmd(task, `has-session -t ${task.sessionId}`) }),
2540          });
2541          const vData = await verify.json();
2542          if (vData.exit_code === 0) {
2543            failCount = 0;
2544            await new Promise(r => setTimeout(r, 5000));
2545            continue;
2546          }
2547        } catch {
2548          await new Promise(r => setTimeout(r, 10000));
2549          continue;
2550        }
2551
2552        const lastOutput = output.textContent || '';
2553        // Pip tasks (Reinstall vLLM / Upgrade torch / etc.) must skip the
2554        // generic serve `_diagnose` step. Their output is pip's own and the
2555        // error patterns there (torch ABI traceback, "No module named torch",
2556        // etc.) are routinely matched against the previous tmux scrollback,
2557        // tagging a clean pip success as a crashed serve. Detection is the
2558        // same shape as the looksSuccessful branch below.
2559        const _isPipTaskDiag = ((task.payload?.repo_id || '').startsWith('pip-'))
2560          || /python3? -m pip\b/.test(task.payload?._cmd || '');
2561        const diag = _isPipTaskDiag ? null : _diagnose(lastOutput);
2562        if (diag) {
2563          let diagEl = el.querySelector('.cookbook-diagnosis');
2564          if (!diagEl) {
2565            diagEl = document.createElement('div');
2566            diagEl.className = 'cookbook-diagnosis';
2567            el.appendChild(diagEl);
2568          }
2569          _showDiagnosis(el, diag, lastOutput);
2570          _updateTask(task.sessionId, { status: 'error' });
2571          el.dataset.status = 'error';
2572          const badge = el.querySelector('.cookbook-task-status');
2573          if (badge) { badge.textContent = _statusLabel('error', task.type); badge.className = 'cookbook-task-status cookbook-task-error'; }
2574          _showCookbookNotif(true);
2575        } else {
2576          const downloadLooksSuccessful = !lastOutput.includes('DOWNLOAD_FAILED')
2577            && (lastOutput.includes('DONE') || lastOutput.includes('100%') || lastOutput.includes('/snapshots/') || lastOutput.includes('Download complete') || lastOutput.includes('DOWNLOAD_OK'));
2578          // Pip install / reinstall tasks are launched via _launchServeTask (so
2579          // they show up in the Running tab + use tmux) but they aren't real

2580          // serves â the cmd is `python3 -m pip ...` and the success markers
2581          // are pip's own. Without this branch, a successful reinstall ends
2582          // with no "Uvicorn running on" line and gets mis-flagged as a crashed
2583          // serve.
2584          const _isPipTask = ((task.payload?.repo_id || '').startsWith('pip-'))
2585            || /python3? -m pip\b/.test(task.payload?._cmd || '');
2586          const pipLooksSuccessful = _isPipTask
2587            && /Successfully installed|Requirement already (?:satisfied|up-to-date)/i.test(lastOutput)
2588            && !/error:|ERROR:/.test(lastOutput.slice(-1024));
2589          const serveLooksReady = task.type === 'serve' && _serveOutputLooksReady({ ...task, output: lastOutput });
2590          // Dependency installs are tracked as download tasks but finish with a
2591          // pip exit-0 sentinel, not HF download markers â check that too.
2592          // Standalone pip-* serves finish with pip's own success line, not
2593          // HF or "Uvicorn running on".
2594          const depInstallSucceeded = !!task.payload?._dep && _depInstallSucceeded(lastOutput);
2595          const looksSuccessful = depInstallSucceeded
2596            || (task.type === 'download'
2597              ? downloadLooksSuccessful
2598              : (_isPipTask ? pipLooksSuccessful : serveLooksReady));
2599          if (!lastOutput.trim() || !looksSuccessful) {
2600            _updateTask(task.sessionId, { status: 'crashed' });
2601            el.dataset.status = 'crashed';
2602            const badge = el.querySelector('.cookbook-task-status');
2603            if (badge) { badge.textContent = _statusLabel('crashed', task.type); badge.className = 'cookbook-task-status cookbook-task-crashed'; }
2604            if (_isPipTask) {
2605              // Pip tasks: don't run the serve diagnosis (which would yell
2606              // "Serve stopped before the model became reachable"). Show a
2607              // pip-tailored message; the user can read pip's own error output
2608              // directly above.
2609              const _ranOk = /Successfully installed|Requirement already (?:satisfied|up-to-date)/i.test(lastOutput);
2610              if (!_ranOk) {
2611                _showDiagnosis(el, {
2612                  message: 'Pip install did not finish with a success marker. Check the output for the underlying error.',
2613                  suggestion: 'Suggested action: copy the troubleshooting bundle. Common causes: missing build deps, network blip, mismatched torch ABI.',
2614                  fixes: [],
2615                }, lastOutput);
2616              }
2617            } else if (task.type === 'serve') {
2618              const diag = _diagnose(lastOutput) || {
2619                message: _serveTaskLooksAwqOnLocalBackend(task, lastOutput)
2620                  ? 'AWQ/GPTQ/FP8 cannot be served through llama.cpp/Ollama unified-memory mode.'
2621                  : /Native llama-server not found|building llama-server|llama\.cpp/i.test(lastOutput)
2622                  ? 'llama.cpp build stopped before the server became reachable.'
2623                  : 'Serve stopped before the model became reachable.',
2624                suggestion: _serveTaskLooksAwqOnLocalBackend(task, lastOutput)
2625                  ? 'Suggested action: use vLLM/SGLang on a compatible CUDA/ROCm GPU server, or download a GGUF version for llama.cpp/Ollama/unified-memory serving.'
2626                  : /Native llama-server not found|building llama-server|llama\.cpp/i.test(lastOutput)
2627                  ? 'Suggested action: copy the troubleshooting bundle, then edit serve settings. For the quickest local/CPU path, use Ollama or a prebuilt llama-server; source builds can take several minutes and fail if build dependencies are incomplete.'
2628                  : 'Suggested action: copy the troubleshooting bundle, then edit serve settings or relaunch with a CPU/backend fallback.',
2629                fixes: [{ label: 'Edit serve', action: (panel) => _openServeEditForTask(task) }],
2630              };
2631              _showDiagnosis(el, diag, lastOutput);
2632            } else if (task.type === 'download') {
2633              const isDisk = /no space left|disk quota|enospc/i.test(lastOutput);
2634              const isNetwork = /connection|timeout|timed out|incompleteread|chunkedencoding|reset by peer|protocolerror|all connection attempts failed/i.test(lastOutput);
2635              const progressMatch = String(lastOutput || '').match(/(\d+)%\|/);
2636              const nearDone = progressMatch && Number(progressMatch[1]) >= 80;
2637              // Reconnect: most "crashed" downloads near the end are actually
2638              // finished â we just missed the DOWNLOAD_OK / /snapshots/ marker
2639              // because output rolled over, or the tmux session ended a tick
2640              // before we polled. Probing has-session and re-attaching to
2641              // capture-pane lets the existing _reconnectTask flow pick up
2642              // the real state (running, finished, or truly dead).
2643              const _reconnectFix = {
2644                label: 'Reconnect',
2645                action: () => {
2646                  _updateTask(task.sessionId, { status: 'running' });
2647                  el.dataset.status = 'running';
2648                  const badge2 = el.querySelector('.cookbook-task-status');
2649                  if (badge2) { badge2.textContent = _statusLabel('running', task.type); badge2.className = 'cookbook-task-status'; }
2650                  const _diagEl = el.querySelector('.cookbook-diagnosis');
2651                  if (_diagEl) _diagEl.remove();
2652                  const _wave = el.querySelector('.cookbook-task-wave'); if (_wave) _wave.style.display = '';
2653                  const _up = el.querySelector('.cookbook-task-uptime'); if (_up) _up.style.display = '';
2654                  _reconnectTask(el, task);
2655                },
2656              };
2657              const diag = {
2658                message: isDisk
2659                  ? 'Download stopped because this server ran out of disk space.'
2660                  : isNetwork
2661                  ? 'Download stopped after the HuggingFace connection was interrupted.'
2662                  : nearDone
2663                  ? 'Download stopped near the end before the final completion marker was captured.'
2664                  : 'Download stopped before HuggingFace reported completion.',
2665                suggestion: isDisk
2666                  ? 'Suggested action: free disk space, then retry the download. HuggingFace resumes incomplete files when possible.'
2667                  : nearDone
2668                  ? 'Suggested action: hit Reconnect first â the download may have finished after the output buffer rolled over. Retry only if reconnect cannot recover.'
2669                  : 'Suggested action: hit Reconnect to re-attach to the tmux session. If that fails, retry â HuggingFace resumes incomplete files when possible.',
2670                fixes: isDisk
2671                  ? [
2672                      { label: 'Retry download', action: () => _retryTask(el, task) },
2673                      { label: 'Copy last 50 lines', action: () => {

2674                        const last = String(lastOutput || '').split('\n').slice(-50).join('\n');
2675                        _copyText(last || 'No download log available.');
2676                      } },
2677                    ]
2678                  : [
2679                      _reconnectFix,
2680                      { label: 'Retry download', action: () => _retryTask(el, task) },
2681                      { label: 'Copy last 50 lines', action: () => {
2682                        const last = String(lastOutput || '').split('\n').slice(-50).join('\n');
2683                        _copyText(last || 'No download log available.');
2684                      } },
2685                    ],
2686              };
2687              _showDiagnosis(el, diag, lastOutput);
2688              // Auto-probe: if the tmux session is still alive (download
2689              // genuinely still in progress), _selfHealStaleTasks flips the
2690              // task back to running and the diagnosis disappears without
2691              // the user needing to click Reconnect.
2692              if (nearDone) setTimeout(() => { _selfHealStaleTasks().catch(() => {}); }, 1200);
2693            }
2694            _showCookbookNotif(true);
2695          } else {
2696            // Strong completion markers â `DOWNLOAD_OK` is emitted by our
2697            // downloader wrapper AFTER the model snapshot is on disk, and
2698            // `/snapshots/` only appears once HF has resolved the cached
2699            // tree. Either is conclusive. Finalize as done immediately, skip
2700            // the 30s debounce â the debounce only exists to guard against
2701            // ambiguous markers (bare "100%" / "Download complete") which can
2702            // appear mid-stream during multi-file downloads.
2703            const _strongDone = task.type === 'download'
2704              && (lastOutput.includes('DOWNLOAD_OK') || lastOutput.includes('/snapshots/'));
2705            if (_strongDone) {
2706              _updateTask(task.sessionId, { status: 'done', _doneConfirmAt: null, _lastStatusFlipAt: Date.now() });
2707              el.dataset.status = 'done';
2708              const badge = el.querySelector('.cookbook-task-status');
2709              if (badge) { badge.textContent = _statusLabel('done', task.type); badge.className = 'cookbook-task-status cookbook-task-done'; }
2710              const _chk = el.querySelector('.cookbook-task-check'); if (_chk) _chk.style.display = '';
2711              const _sb = el.querySelector('.cookbook-task-serve-btn'); if (_sb) _sb.style.display = '';
2712              _showCookbookNotif();
2713              _refreshDepsAfterInstall(task);
2714              _renderRunningTab();
2715              _processQueue();
2716              break;
2717            }
2718            // Debounce the done flip. Tmux capture-pane can fail transiently
2719            // (network blip, ssh reconnect), and the verify has-session right
2720            // above can briefly report dead even when the session is in the
2721            // middle of finalizing. Marking done immediately + the periodic
2722            // _selfHealStaleTasks then flipping back to running causes the
2723            // status badge to oscillate between Finished and Downloading.
2724            // Wait 30s and re-probe: only finalize as done if tmux is STILL
2725            // gone. If the session resurfaces, restart _reconnectTask so live
2726            // capture resumes without the user seeing a fake "done" first.
2727            if (!task._doneConfirmAt) {
2728              _updateTask(task.sessionId, { _doneConfirmAt: Date.now() + 30000 });
2729              setTimeout(async () => {
2730                try {
2731                  const fresh = _loadTasks().find(t => t.sessionId === task.sessionId);
2732                  if (!fresh) return;
2733                  let stillAlive = false;
2734                  try {
2735                    const probe = await fetch('/api/shell/exec', {
2736                      method: 'POST', credentials: 'same-origin',
2737                      headers: { 'Content-Type': 'application/json' },
2738                      body: JSON.stringify({ command: _tmuxCmd(task, `has-session -t ${task.sessionId}`), timeout: 5 }),
2739                    });
2740                    const pData = await probe.json();
2741                    stillAlive = pData.exit_code === 0;
2742                  } catch { /* network blip â treat as inconclusive, prefer running */ stillAlive = true; }
2743                  if (stillAlive) {
2744                    _updateTask(task.sessionId, { status: 'running', _doneConfirmAt: null, _lastStatusFlipAt: Date.now() });
2745                    const _el = document.querySelector(`.cookbook-task[data-task-id="${task.sessionId}"]`);
2746                    if (_el) {
2747                      _el.dataset.status = 'running';
2748                      const _badge = _el.querySelector('.cookbook-task-status');
2749                      if (_badge) { _badge.textContent = _statusLabel('running', task.type); _badge.className = 'cookbook-task-status'; }
2750                      const _wave = _el.querySelector('.cookbook-task-wave'); if (_wave) _wave.style.display = '';
2751                      const _up = _el.querySelector('.cookbook-task-uptime'); if (_up) _up.style.display = '';
2752                      _reconnectTask(_el, _loadTasks().find(t => t.sessionId === task.sessionId));
2753                    }
2754                    return;
2755                  }
2756                  _updateTask(task.sessionId, { status: 'done', _doneConfirmAt: null, _lastStatusFlipAt: Date.now() });
2757                  const _el = document.querySelector(`.cookbook-task[data-task-id="${task.sessionId}"]`);
2758                  if (_el) {
2759                    _clearDiagnosis(_el);
2760                    _el.dataset.status = 'done';
2761                    const _badge = _el.querySelector('.cookbook-task-status');
2762                    if (_badge) { _badge.textContent = _statusLabel('done', task.type); _badge.className = 'cookbook-task-status cookbook-task-done'; }
2763                    const _chk = _el.querySelector('.cookbook-task-check'); if (_chk) _chk.style.display = '';
2764                    const _sb = _el.querySelector('.cookbook-task-serve-btn'); if (_sb) _sb.style.display = '';
2765                  }
2766                  _showCookbookNotif();
2767                  _refreshDepsAfterInstall(task);
2768                  _renderRunningTab();
2769                  _processQueue();
2770                } catch { /* swallow â next polling cycle will retry */ }
2771              }, 30000);
2772            }
2773          }
2774        }
2775        _renderRunningTab();
2776        _processQueue();
2777        break;
2778      }
2779
2780      const snapshot = (data.stdout || '').trim();
2781      if (snapshot) {
2782        // Only auto-scroll to bottom if the user was already there. When
2783        // they've scrolled up to read earlier output, leave their position
2784        // alone so a fresh snapshot doesn't yank them back to the tail.
2785        // 40px tolerance covers sub-pixel rounding + the moment between
2786        // releasing the scrollbar and the next poll arriving.
2787        const _atBottom = (output.scrollHeight - output.scrollTop - output.clientHeight) < 40;
2788        output.textContent = snapshot;
2789        if (_atBottom) output.scrollTop = output.scrollHeight;
2790
2791        // Live status parsing for download tasks
2792        if (task.type === 'download') {
2793          const badge = el.querySelector('.cookbook-task-status');
2794          if (badge) {

2795            const completed = (snapshot.match(/Download complete/g) || []).length;
2796            const downloading = snapshot.match(/Downloading '([^']+)'/g) || [];
2797            const totalFiles = downloading.length;
2798            const pctMatches = [...snapshot.matchAll(/(\d+)%\|/g)];
2799            const lastPct = pctMatches.length ? pctMatches[pctMatches.length - 1][1] : null;
2800            const speedMatch = [...snapshot.matchAll(/([\d.]+)(?:MB|GB)\/s/g)];
2801            const lastSpeed = speedMatch.length ? speedMatch[speedMatch.length - 1][0] : null;
2802            // hf_transfer prints "Downloading (incomplete total...): 73% | 1.81G/2.49G"
2803            // â the real aggregate byte progress. The "Fetching N files" line (often
2804            // last in the output) sits at 0%, so lastPct/_fetchPct can read 0 even at
2805            // 73% done. Prefer this aggregate when present.
2806            const _dlAggMatches = [...snapshot.matchAll(/Downloading\s*\(incomplete[^)]*\):\s*(\d+)%/g)];
2807            const _dlAgg = _dlAggMatches.length ? parseInt(_dlAggMatches[_dlAggMatches.length - 1][1]) : null;
2808
2809            // Stale download detection.
2810            // Use the DOWNLOADED-BYTE count ("1.81G" from "1.81G/2.49G") as the
2811            // progress signal: it climbs continuously while transferring (even when
2812            // the % plateaus during a big hf_transfer chunk) and FREEZES when stuck.
2813            // The % alone plateaus (false stall), and a frozen frame still shows a
2814            // stale speed/ETA â so keying off speed masked real stalls (that's why a
2815            // 97%-stuck download went undetected). Bytes are the honest signal; fall
2816            // back to %/aggregate only when no byte counter is present.
2817            const _byteMatches = [...snapshot.matchAll(/([\d.]+\s?[KMGT])B?\s*\/\s*[\d.]+\s?[KMGT]B?/gi)];
2818            const _bytes = _byteMatches.length ? _byteMatches[_byteMatches.length - 1][1].replace(/\s/g, '') : null;
2819            // When there's no byte counter (pip resolve / native build phase of a
2820            // dependency install), key off the output tail so new build lines count
2821            // as progress â otherwise a long quiet build is falsely declared stale
2822            // and restarted mid-build, looping forever (#1568).
2823            const curProgress = computeProgressSignal(_bytes, _dlAgg, lastPct, snapshot);
2824            const _fetchPctMatches = [...snapshot.matchAll(/Fetching\s+\d+\s+files:\s*(\d+)%/g)];
2825            const _fetchPct = _fetchPctMatches.length ? parseInt(_fetchPctMatches[_fetchPctMatches.length - 1][1]) : null;
2826            const isPipDep = !!(task.payload && task.payload._dep);
2827            const _startupStalled = !_bytes && ((_dlAgg === 0) || (_fetchPct === 0)) && curProgress === '0';
2828            const _STALE_TIMEOUT = _startupStalled ? STARTUP_STALE_PROGRESS_MS : STALE_PROGRESS_MS;
2829            if (!el._lastProgress) { el._lastProgress = curProgress; el._lastProgressTime = Date.now(); }
2830            if (curProgress !== el._lastProgress) {
2831              el._lastProgress = curProgress;
2832              el._lastProgressTime = Date.now();
2833            } else if (!isPipDep && Date.now() - (el._lastProgressTime || 0) > _STALE_TIMEOUT && task._autoRestarted) {
2834              const mins = Math.floor((Date.now() - (el._lastProgressTime || 0)) / 60000);
2835              // Already auto-restarted once and stalled again â make the badge a
2836              // one-click retry (resumes from the cached partial files) so the
2837              // user doesn't have to dig into the â® menu.
2838              badge.textContent = `stalled ${mins}m â»`;
2839              badge.className = 'cookbook-task-status cookbook-task-error';
2840              badge.title = 'Click to retry â resumes where it stopped';
2841              badge.style.cursor = 'pointer';
2842              if (!badge._retryBound) {
2843                badge._retryBound = true;
2844                badge.addEventListener('click', (e) => { e.stopPropagation(); _retryTask(el, task); });
2845              }
2846            } else if (!isPipDep && Date.now() - (el._lastProgressTime || 0) > _STALE_TIMEOUT && !task._autoRestarted) {
2847              task._autoRestarted = true;
2848              _updateTask(task.sessionId, { _autoRestarted: true });

2849              badge.textContent = _startupStalled ? '0% stall â retrying' : 'stale â restarting';
2850              badge.className = 'cookbook-task-status cookbook-task-error';
2851              _showCookbookNotif(true);
2852              try {
2853                await fetch('/api/shell/exec', {
2854                  method: 'POST', credentials: 'same-origin',
2855                  headers: { 'Content-Type': 'application/json' },
2856                  body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${task.sessionId}`) }),
2857                });
2858              } catch {}
2859              try {
2860                // Reuse original payload so the full repo_id (e.g. "Qwen/Qwen3.5-...")
2861                // is preserved â rebuilding from task.repo/task.name drops the org prefix.
2862                const dlPayload = task.payload
2863                  ? { ...task.payload }
2864                  : { repo_id: task.repo || task.name, remote_host: task.remoteHost || '' };
2865                if (_envState.hfToken) dlPayload.hf_token = _envState.hfToken;
2866                // Stalled with hf_transfer â restart on the reliable downloader.
2867                dlPayload.disable_hf_transfer = true;
2868                // Don't overwrite env_prefix â task.payload already has the correct
2869                // "source <path>" form. The bare envPath would miss the `source` and
2870                // the venv never activates (so hf CLI falls off PATH).
2871                const res = await fetch('/api/model/download', {
2872                  method: 'POST', credentials: 'same-origin',
2873                  headers: { 'Content-Type': 'application/json' },
2874                  body: JSON.stringify(dlPayload),
2875                });
2876                const data = await res.json();
2877                if (data.ok && data.session_id) {
2878                  _updateTask(task.sessionId, { sessionId: data.session_id, status: 'running', output: '' });
2879                  task.sessionId = data.session_id;
2880                  el._lastProgress = null;
2881                  el._lastProgressTime = Date.now();
2882                  badge.textContent = 'restarted';
2883                  badge.className = 'cookbook-task-status cookbook-task-running';
2884                  continue;
2885                }
2886              } catch {}
2887              badge.textContent = 'stale â restart failed';
2888              badge.className = 'cookbook-task-status cookbook-task-error';
2889              _showCookbookNotif(true);
2890              break;
2891            }
2892
2893            // When the snapshot includes a shard-of-N marker (e.g.
2894            // "model-00006-of-00082.safetensors"), TRUE overall progress is
2895            // ((shard-1) + currentShardFraction) / totalShards. Before, _dlAgg
2896            // (hf_transfer's per-current-shard aggregate, e.g. 53% of shard 6)
2897            // was treated as overall and the row read "53%" while only 5 of
2898            // 82 shards were actually done.
2899            const _shardPat = [...snapshot.matchAll(/model-(\d+)-of-(\d+)\.(?:safetensors|bin)/g)];
2900            const _lastShard = _shardPat.length ? _shardPat[_shardPat.length - 1] : null;
2901            const _curShardNum = _lastShard ? parseInt(_lastShard[1], 10) : null;
2902            const _totalShards = _lastShard ? parseInt(_lastShard[2], 10) : null;
2903            const _useShardAgg = _curShardNum && _totalShards && _totalShards > 1;
2904
2905            // HF's own "Fetching N files: X%" aggregate counts ALL files,
2906            // including ones already finished in a previous session (resume) â
2907            // so on a resumed download it reflects the true overall progress,
2908            // whereas completed/totalFiles only see this session's files (â 0%).
2909            // Take the higher of the two so resume doesn't read as 0%.
2910            if (_useShardAgg) {
2911              // Multi-shard download: compute TRUE overall as completed shards
2912              // plus the current shard's fraction. _dlAgg / lastPct represent
2913              // *this shard's* progress, not the whole download.
2914              const curShardFrac = (_dlAgg != null)
2915                ? _dlAgg / 100
2916                : (lastPct ? parseInt(lastPct, 10) / 100 : 0);
2917              let overallPct = Math.round((((_curShardNum - 1) + curShardFrac) / _totalShards) * 100);
2918              if (_fetchPct != null) overallPct = Math.max(overallPct, _fetchPct);
2919              let text = `${overallPct}%`;
2920              if (lastSpeed) text += ` Â· ${lastSpeed}`;
2921              badge.textContent = text;
2922              badge.className = 'cookbook-task-status cookbook-task-running';
2923            } else if (_dlAgg != null) {
2924              // Real aggregate byte progress â most accurate; take the max of all signals.
2925              let pct = _dlAgg;
2926              if (_fetchPct != null) pct = Math.max(pct, _fetchPct);
2927              let text = `${pct}%`;
2928              if (lastSpeed) text += ` Â· ${lastSpeed}`;
2929              badge.textContent = text;
2930              badge.className = 'cookbook-task-status cookbook-task-running';
2931            } else if (totalFiles > 0 && completed < totalFiles) {
2932              const curFilePct = lastPct ? parseInt(lastPct) / 100 : 0;
2933              let overallPct = Math.round(((completed + curFilePct) / totalFiles) * 100);
2934              if (_fetchPct != null) overallPct = Math.max(overallPct, _fetchPct);
2935              let text = `${overallPct}%`;
2936              if (lastSpeed) text += ` Â· ${lastSpeed}`;
2937              badge.textContent = text;
2938              badge.className = 'cookbook-task-status cookbook-task-running';
2939            } else if (_fetchPct != null && _fetchPct < 100) {
2940              // Resume start: only the aggregate is meaningful yet.
2941              let text = `${_fetchPct}%`;
2942              if (lastSpeed) text += ` Â· ${lastSpeed}`;
2943              badge.textContent = text;
2944              badge.className = 'cookbook-task-status cookbook-task-running';
2945            } else if (completed > 0 && completed >= totalFiles) {
2946              badge.textContent = 'finishing';
2947              badge.className = 'cookbook-task-status cookbook-task-running';
2948            }
2949            if (snapshot.includes('DOWNLOAD_FAILED')) {
2950              // The wrapper prints DOWNLOAD_FAILED but exits 0, and per-file
2951              // "Download complete"/"100%" lines make it look successful â so
2952              // catch the explicit failure marker and handle it.
2953              // A gated/auth failure can NEVER be fixed by retrying (the HF token
2954              // is sent, but its account isn't approved for this repo) â skip the
2955              // auto-retries and surface the gated diagnosis straight away.
2956              const _accessDenied = /Access to model.*is restricted|gated repo|GatedRepoError|401 Unauthorized|403 Forbidden|not in the authorized list|awaiting a review|must (?:be authenticated|have access)/i.test(snapshot);
2957              const _dlKey = task.payload?.repo_id || task.name;
2958              const _dlN = _dlRetryCount.get(_dlKey) || 0;
2959              if (!controller.signal.aborted && !_accessDenied && task.type === 'download' && task.payload && _dlN < _DL_MAX_AUTO_RETRY) {
2960                // Auto-retry: kill the dead session and re-launch (resumes from
2961                // the cached .incomplete files) after a short delay.
2962                _dlRetryCount.set(_dlKey, _dlN + 1);
2963                badge.textContent = `retrying (${_dlN + 1}/${_DL_MAX_AUTO_RETRY})â¦`;
2964                badge.className = 'cookbook-task-status cookbook-task-running';
2965                uiModule.showToast(`Download interrupted â retrying (${_dlN + 1}/${_DL_MAX_AUTO_RETRY}

2965), resumes where it stoppedâ¦`, 6000);
2966                const _p = task.payload, _nm = task.name;
2967                try {
2968                  await fetch('/api/shell/exec', {
2969                    method: 'POST', credentials: 'same-origin',
2970                    headers: { 'Content-Type': 'application/json' },
2971                    body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${task.sessionId}`) }),
2972                  });
2973                } catch {}
2974                _removeTask(task.sessionId);
2975                setTimeout(() => { _retryDownload(_nm, _p); }, 8000);
2976                break;
2977              }
2978              // Out of auto-retries (or not a download) â surface the error; the
2979              // card's Retry button stays available to resume manually.
2980              badge.textContent = _statusLabel('error', task.type);
2981              badge.className = 'cookbook-task-status cookbook-task-error';
2982              _updateTask(task.sessionId, { status: 'error' });
2983              el.dataset.status = 'error';
2984              // Explain a gated/access failure with actionable buttons (request
2985              // access on HF, check token) â otherwise it's just raw red text.
2986              if (_accessDenied) {
2987                const _diag = _diagnose(snapshot);
2988                if (_diag) {
2989                  let diagEl = el.querySelector('.cookbook-diagnosis');
2990                  if (!diagEl) { diagEl = document.createElement('div'); diagEl.className = 'cookbook-diagnosis'; el.appendChild(diagEl); }
2991                  _showDiagnosis(el, _diag, snapshot);
2992                }
2993              }
2994              _showCookbookNotif(true);
2995              break;
2996            }
2997            if (snapshot.includes('DOWNLOAD_OK') || (snapshot.includes('/snapshots/') && completed >= totalFiles && totalFiles > 0)) {
2998              _clearDiagnosis(el);
2999              _dlRetryCount.delete(task.payload?.repo_id || task.name);
3000              badge.textContent = _statusLabel('done', task.type);
3001              badge.className = 'cookbook-task-status cookbook-task-done';
3002              // Flip the type chip from "download" to the green "finished"
3003              // badge so the header reads as completed without a stale label.
3004              const _typeChip = el.querySelector('.cookbook-task-type');
3005              if (_typeChip) { _typeChip.textContent = 'finished'; _typeChip.classList.add('cookbook-task-type-done'); }
3006              _updateTask(task.sessionId, { status: 'done' });
3007              const _sb2 = el.querySelector('.cookbook-task-serve-btn'); if (_sb2) _sb2.style.display = '';
3008              _showCookbookNotif();
3009              _refreshDepsAfterInstall(task);
3010              fetch('/api/shell/exec', {
3011                method: 'POST', credentials: 'same-origin',
3012                headers: { 'Content-Type': 'application/json' },
3013                body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${task.sessionId}`) }),
3014              }).catch(() => {});
3015              _processQueue();
3016              break;
3017            }
3018          }
3019        }
3020
3021        // Live status parsing for serve tasks â uses shared _parseServePhase
3022        if (task.type === 'serve') {
3023          const badge = el.querySelector('.cookbook-task-status');
3024          if (badge) {
3025            const info = _parseServePhase(snapshot);
3026            if (info.status === 'ready' && !task._serveReady) {
3027              task._serveReady = true;
3028              _updateTask(task.sessionId, { _serveReady: true });
3029            }
3030            if (info.phase) {
3031              badge.textContent = info.phase;
3032              // Always the green "running" style â loading/warming is the same
3033              // state, just with dynamic text (don't switch to a neutral style).
3034              badge.className = 'cookbook-task-status cookbook-task-running';
3035              // Live output reporting 'ready' is direct proof the server is up â
3036              // clear a stale "unreachable" flag here too. The HTTP probe can lag,
3037              // miss a remote endpoint, or cache a down result, leaving the card
3038              // stuck red even after the server recovered ("doesn't recheck").
3039              if (info.status === 'ready' && task._unreachable) {
3040                task._unreachable = false;

3041                _updateTask(task.sessionId, { _unreachable: false });
3042                el.classList.remove('cookbook-task-unreachable');
3043                _refreshServerDots();
3044              }
3045              // Persist the loading phase so a re-render keeps showing "loading 45%"
3046              // instead of resetting the badge to the generic "running". Clear it
3047              // once ready so the badge falls back to "running".
3048              if (info.status !== 'ready') {
3049                if (task.progress !== info.phase) _updateTask(task.sessionId, { progress: info.phase });
3050              } else if (task.progress) {
3051                _updateTask(task.sessionId, { progress: '' });
3052              }
3053            }
3054          }
3055        }
3056
3057        // Run error diagnosis on serve tasks
3058        const diag = _diagnose(snapshot);
3059        if (diag) {
3060          let diagEl = el.querySelector('.cookbook-diagnosis');
3061          if (!diagEl) {
3062            diagEl = document.createElement('div');
3063            diagEl.className = 'cookbook-diagnosis';
3064            el.appendChild(diagEl);
3065          }
3066          _showDiagnosis(el, diag, snapshot);
3067        }
3068        // Detect serve ready â auto-add to model endpoints. Don't flip
3069        // `_endpointAdded` until the POST succeeds; otherwise a transient
3070        // error silently prevents any future retry. An in-flight guard
3071        // prevents a second poll from firing a duplicate POST before the
3072        // first one's dedup check can observe the newly-added row.
3073        if (task.type === 'serve' && !task._endpointAdded && !task._endpointAddInFlight && task._serveReady) {
3074          task._endpointAddInFlight = true;
3075          let host = _connectHostFromRemote(task.remoteHost);
3076          const portMatch = task.payload?._cmd?.match(/--port[=\s]+(\d+)/)
3077            || task.payload?._cmd?.match(/(?:^|\s)-p[=\s]+(\d+)/)
3078            || snapshot.match(/Uvicorn running on\D*?:(\d+)/i)
3079            || snapshot.match(/running on\D*?:(\d+)/i)
3080            || snapshot.match(/listening on\D*?:(\d+)/i)
3081            || snapshot.match(/port[:=\s]+(\d+)/i);
3082          let port = portMatch ? portMatch[1] : '8000';
3083          let baseUrl = `http://${host}:${port}/v1`;
3084          const ollamaUrlMatch = snapshot.match(/Ollama API ready on port\s+\d+:\s*(http:\/\/[^\s]+)/i);
3085          if (ollamaUrlMatch) {
3086            const endpoint = _endpointFromAdvertisedUrl(ollamaUrlMatch[1], host, '11434');
3087            if (endpoint) ({ host, port, baseUrl } = endpoint);
3088          }
3089          fetch('/api/model-endpoints', { credentials: 'same-origin' })
3090            .then(r => r.json())
3091            .then(async (eps) => {
3092              // Match only exact base_url â don't dedup by friendly name,
3093              // because other endpoints may happen to share a model name.
3094              const exists = eps.some(e => e.base_url === baseUrl);
3095              if (exists) {
3096                // Already registered â e.g. the backend pre-registers diffusion
3097                // endpoints server-side. Mark so we don't retry, but STILL
3098                // refresh the picker (and probe until online) so the new model
3099                // shows up without the user having to manually refresh.
3100                task._endpointAdded = true;
3101                _updateTask(task.sessionId, { _endpointAdded: true });
3102                _autoSaveWorkingConfig(task);   // endpoint live â remember these settings
3103                if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(true);
3104                if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
3105                window.dispatchEvent(new CustomEvent('ge:model-endpoints-updated', { detail: { baseUrl, host, port, model: task.name } }));
3106                const _ex = eps.find(e => e.base_url === baseUrl);
3107                if (_ex && _ex.id && !(_ex.models || []).length) _probeEndpointUntilOnline(_ex.id, host, port);
3108                return null;
3109              }
3110              const _isDiffusion = task.payload?._cmd?.includes('diffusion_server');
3111              const fd = new FormData();
3112              fd.append('base_url', baseUrl);
3113              fd.append('name', task.name);
3114              fd.append('skip_probe', 'true');
3115              _appendCookbookEndpointScope(fd, task.remoteHost || '');
3116              if (_isDiffusion) fd.append('model_type', 'image');
3117              return fetch('/api/model-endpoints', { method: 'POST', credentials: 'same-origin', body: fd });
3118            })
3119            .then(async (res) => {
3120              if (res && res.ok) {
3121                // Flip the flag only on confirmed success
3122                task._endpointAdded = true;
3123                _updateTask(task.sessionId, { _endpointAdded: true });
3124                _autoSaveWorkingConfig(task);   // endpoint live â remember these settings
3125                uiModule.showToast(`Model endpoint added: ${host}:${port}`);
3126                // Retry-probe until the warming server answers, so it
3127                // flips online without a manual enable/disable toggle.
3128                const _epData = await res.json().catch(() => ({}));
3129                if (_epData && _epData.id && !(_epData.models || []).length) {
3130                  _probeEndpointUntilOnline(_epData.id, host, port);
3131                }
3132                window.dispatchEvent(new CustomEvent('ge:model-endpoints-updated', { detail: { baseUrl, host, port, model: task.name } }));
3133                const _trySelectModel = async (attempt) => {
3134                  if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(true);
3135                  const items = window.modelsModule?.getCachedItems?.() || [];
3136                  for (const item of items) {
3137                    if (item.offline) continue;
3138                    const url = item.url || '';
3139                    if (url.includes(host) || url.includes(port)) {
3140                      const mid = (item.models || [])[0];
3141                      if (mid && window.sessionModule?.createDirectChat) {
3142                        window.sessionModule.createDirectChat(url, mid, item.endpoint_id);
3143                        if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
3144                        uiModule.showToast(`Switched to ${mid.split('/').pop()}`);
3145                        return;
3146                      }
3147                    }
3148                  }
3149                  if (attempt < 3) setTimeout(() => _trySelectModel(attempt + 1), 2000);
3150                  else if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
3151                };
3152                setTimeout(() => _trySelectModel(0), 1000);
3153              } else if (res && !res.ok) {
3154                const body = await res.text().catch(() => '');
3155                console.warn('Endpoint auto-add failed', res.status, body);
3156                uiModule.showError(`Auto-register endpoint failed (${res.status}). Use â® â Register endpoint to retry.`);
3157              }
3158            })
3159            .catch((e) => {
3160              console.warn('Endpoint auto-add error', e);
3161              uiModule.showError(`Auto-register endpoint error: ${e.message || e}. Use â® â Register endpoint to retry.`);
3162            })
3163            .finally(() => { task._endpointAddInFlight = false; });
3164          _updateTask(task.sessionId, { status: 'running' });
3165          const badge = el.querySelector('.cookbook-task-status');
3166          if (badge) { badge.textContent = 'running'; badge.className = 'cookbook-task-status cookbook-task-running'; }
3167          _showCookbookNotif();
3168        }
3169        // Detect process exit

3170        if (snapshot.includes('=== Process exited with code')) {
3171          const codeMatch = snapshot.match(/=== Process exited with code (\d+)/);
3172          const code = codeMatch ? parseInt(codeMatch[1]) : -1;
3173          // Serve tasks that exit without reaching ready state are always errors â
3174          // a serve process should run indefinitely
3175          const status = (task.type === 'serve' && !task._serveReady) ? 'error'
3176            : (code === 0 ? 'done' : 'error');
3177          _updateTask(task.sessionId, { status });
3178          const badge = el.querySelector('.cookbook-task-status');
3179          if (badge) { badge.textContent = status; badge.className = `cookbook-task-status cookbook-task-${status}`; }
3180          _renderRunningTab();
3181        }
3182        _updateTask(task.sessionId, { output: snapshot.slice(-5000) });
3183      }
3184    } catch {
3185      failCount++;
3186      if (failCount > 10) break;
3187      await new Promise(r => setTimeout(r, 10000));
3188      continue;
3189    }
3190
3191    failCount = 0;
3192    await new Promise(r => setTimeout(r, TASK_POLL_INTERVAL_MS));
3193  }
3194}
3195

3196// ââ Background monitor ââ
3197
3198let _bgMonitorInterval = null;
3199
3200// Reachability check for running serve tasks. The tmux pane can stay alive
3201// while the model server inside it has crashed (so no "Process exited" line
3202// ever appears) â leaving the card showing "running" forever. So we actively
3203// probe the registered endpoint (same /probe-local the model picker uses) and
3204// flag the card "unreachable" (red) when the server stops answering.
3205async function _checkServeReachability() {
3206  let serveTasks;
3207  try {
3208    serveTasks = _loadTasks().filter(t => t.type === 'serve' && t.status === 'running');
3209  } catch { return; }
3210  if (!serveTasks.length) return;
3211  let eps = [], probe = {};
3212  try {
3213    [eps, probe] = await Promise.all([
3214      fetch('/api/model-endpoints', { credentials: 'same-origin' }).then(r => r.json()).catch(() => []),
3215      fetch('/api/model-endpoints/probe-local', { credentials: 'same-origin' }).then(r => r.json()).catch(() => ({})),
3216    ]);
3217  } catch { return; }
3218  for (const task of serveTasks) {
3219    const host = _connectHostFromRemote(task.remoteHost);
3220    const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
3221    const port = portMatch ? portMatch[1] : '8000';
3222    const baseUrl = `http://${host}:${port}/v1`;
3223    const ep = (eps || []).find(e => e.base_url === baseUrl);
3224    if (!ep) continue;                       // not registered yet â can't judge
3225    const pr = probe[ep.id];
3226    if (!pr || pr.alive === undefined) continue;  // not probed (non-local) â skip
3227    // Record the first time it actually answers. Until then the server is still
3228    // LOADING/warming (the endpoint can get registered on the 300s timeout for a
3229    // big model that hasn't finished loading), and a not-yet-answering server is
3230    // not "unreachable" â flagging it as such while you're launching is a false
3231    // alarm. Only treat it as unreachable once it has been reachable at least once.
3232    if (pr.alive === true && !task._everReachable) {
3233      task._everReachable = true;
3234      _updateTask(task.sessionId, { _everReachable: true });
3235    }
3236    const unreachable = pr.alive === false;
3237    if (unreachable && !task._everReachable) continue;  // still coming up, not crashed
3238    if (!!task._unreachable !== unreachable) {
3239      _updateTask(task.sessionId, { _unreachable: unreachable });
3240    }
3241    const el = document.querySelector(`.cookbook-task[data-task-id="${task.sessionId}"]`);
3242    if (el) {
3243      el.classList.toggle('cookbook-task-unreachable', unreachable);
3244      const badge = el.querySelector('.cookbook-task-status');
3245      if (badge) {
3246        if (unreachable) {
3247          badge.textContent = 'unreachable';
3248          badge.className = 'cookbook-task-status cookbook-task-error';
3249          badge.title = pr.error || 'Server not responding â it may have crashed';
3250        } else if (badge.textContent === 'unreachable') {
3251          // Recovered â restore the normal running label.
3252          badge.textContent = _statusLabel('running', task.type);
3253          badge.className = 'cookbook-task-status cookbook-task-running';
3254          badge.title = '';
3255        }
3256      }
3257    }
3258    if (unreachable) _showCookbookNotif(true);
3259  }
3260  _refreshServerDots();
3261}
3262
3263function _serveTaskFailed(task) {
3264  if (!task || task.type !== 'serve') return false;
3265  return !!task._unreachable || ['error', 'crashed', 'failed'].includes(task.status);
3266}
3267
3268function _setServerDot(dot, failed, title) {
3269  if (!dot) return;
3270  dot.classList.toggle('fail', !!failed);
3271  dot.classList.toggle('ok', !failed);
3272  dot.title = title;
3273}
3274
3275function _syncSettingsServerDots(byKey) {
3276  document.querySelectorAll('.cookbook-server-entry').forEach(entry => {
3277    const hostEl = entry.querySelector('.cookbook-srv-host');
3278    const dot = entry.querySelector('.cookbook-srv-status');
3279    const msg = entry.querySelector('.cookbook-srv-test-msg');
3280    if (!hostEl || !dot) return;
3281
3282    const host = hostEl.value?.trim() || '';
3283    if (!host || hostEl.readOnly || hostEl.disabled) {
3284      _setServerDot(dot, false, 'Local (this machine)');
3285      return;
3286    }
3287
3288    const list = byKey[host] || [];
3289    if (!list.length) return;
3290
3291    const failed = list.some(_serveTaskFailed);
3292    _setServerDot(dot, failed, failed ? 'Server not responding - running serve may have crashed' : 'Reachable');
3293    if (!msg) return;
3294
3295    if (failed) {
3296      msg.textContent = 'Server not responding';
3297      msg.title = 'Server not responding - running serve may have crashed';
3298      msg.style.color = 'var(--red,#e06c75)';
3299      msg.style.opacity = '0.75';
3300    } else if (/failed|crashed|not responding|unreachable/i.test(msg.textContent || '')) {
3301      msg.textContent = 'Reachable';
3302      msg.title = 'Reachable';
3303      msg.style.color = 'var(--green,#50fa7b)';
3304      msg.style.opacity = '0.75';
3305    }
3306  });
3307}
3308
3309// Keep each server section's status dot (green â red) in sync with the live

3310// health of its SERVE tasks. The header dot is only built once, so without
3311// this it got stuck on its first value. Downloads never count because they have
3312// no endpoint to be "unreachable".
3313function _refreshServerDots() {
3314  let tasks;
3315  try { tasks = _loadTasks(); } catch { return; }
3316  const byKey = {};
3317  for (const t of tasks) { (byKey[t.remoteHost || ''] = byKey[t.remoteHost || ''] || []).push(t); }
3318  document.querySelectorAll('.cookbook-section-header').forEach(header => {
3319    const dot = header.querySelector('.cookbook-srv-status');
3320    if (!dot) return;
3321    const key = header.querySelector('[data-stop-server]')?.dataset.stopServer || '';
3322    const list = byKey[key] || [];
3323    const fail = !!key && list.some(_serveTaskFailed);
3324    _setServerDot(dot, fail, key ? (fail ? 'Server not responding' : 'Reachable') : 'Local (this machine)');
3325  });
3326  _syncSettingsServerDots(byKey);
3327}
3328
3329// Self-heal: scan persisted download tasks marked done/error/crashed and
3330// check whether their tmux session is still alive on the host. If yes â
3331// the task isn't actually finished, the cookbook just lost the in-flight
3332// status during restart â flip status back to 'running' so _reconnectTask
3333// picks it up. The one-shot guard is enforced by callers (open path) or
3334// time-throttled inside (background-monitor path).
3335let _selfHealRan = false;
3336let _selfHealLastTs = 0;
3337export async function _selfHealStaleTasks(opts = {}) {
3338  // Open-path call: one-shot per page load.
3339  if (opts.oneShot) {
3340    if (_selfHealRan) return;
3341    _selfHealRan = true;
3342  } else {
3343    // Background-monitor call: throttle to once every 8s (the bg monitor
3344    // itself fires every 10s, so this almost always fires too, but the
3345    // guard keeps a fast manual call from doubling up).
3346    const now = Date.now();
3347    if (now - _selfHealLastTs < 4000) return;
3348    _selfHealLastTs = now;
3349  }
3350  const tasks = _loadTasks();
3351  const candidates = tasks.filter(t => {
3352    if (t.type !== 'download') return false;
3353    if (!['done', 'error', 'crashed', 'stopped'].includes(t.status)) return false;
3354    if (!t.sessionId || String(t.sessionId).startsWith('queue-')) return false;
3355    // Finished downloads with strong completion markers (DOWNLOAD_OK or HF
3356    // /snapshots/ resolution) are demonstrably done â do not flip them back
3357    // to running just because the tmux session is still alive (e.g., a
3358    // long-lived shell that hosted the download or a flapping SSH that
3359    // reports the session as up). This was the main source of finishedâ
3360    // downloading oscillation on a flaky connection.
3361    if (t.status === 'done' && /DOWNLOAD_OK|\/snapshots\//.test(t.output || '')) return false;
3362    // Cooldown: never flip the same task more than once every 45s. A flapping
3363    // SSH connection used to drive the badge back-and-forth on every probe
3364    // cycle; this enforces a stable view between flaps.
3365    if (t._lastStatusFlipAt && (Date.now() - t._lastStatusFlipAt < 45000)) return false;
3366    return true;
3367  });
3368  if (!candidates.length) return;
3369  let flipped = 0;
3370  for (const t of candidates) {
3371    try {
3372      const res = await fetch('/api/shell/exec', {
3373        method: 'POST', credentials: 'same-origin',
3374        headers: { 'Content-Type': 'application/json' },
3375        body: JSON.stringify({ command: _tmuxCmd(t, `has-session -t ${t.sessionId}`), timeout: 5 }),
3376      });
3377      const data = await res.json();
3378      if (data.exit_code === 0) {
3379        // Session still alive â the task is actually still running.
3380        const fresh = _loadTasks();
3381        const ft = fresh.find(x => x.sessionId === t.sessionId);
3382        if (ft && ft.status !== 'running') {
3383          ft.status = 'running';
3384          ft._selfHealed = true;
3385          ft._lastStatusFlipAt = Date.now();
3386          _saveTasks(fresh);
3387          flipped++;
3388          const _el = document.querySelector(`.cookbook-task[data-task-id="${t.sessionId}"]`);
3389          if (_el) {
3390            const _chk = _el.querySelector('.cookbook-task-check');
3391            if (_chk) _chk.style.display = 'none';
3392            const _wave = _el.querySelector('.cookbook-task-wave');
3393            if (_wave) _wave.style.display = '';
3394            const _up = _el.querySelector('.cookbook-task-uptime');
3395            if (_up) _up.style.display = '';
3396            _el.dataset.status = 'running';
3397          }
3398        }
3399      }
3400    } catch { /* network blip â skip this one */ }
3401  }
3402  if (flipped) {
3403    console.log(`[cookbook] auto-reconnect: revived ${flipped} task(s) whose tmux session was still alive`);
3404    _renderRunningTab();
3405  }
3406}
3407

3408export function _startBackgroundMonitor() {
3409  if (_bgMonitorInterval) return;
3410  _bgMonitorInterval = setInterval(() => {
3411    _pollBackgroundStatus();
3412    _checkServeReachability();
3413    // Auto-reconnect: every cycle, look for download tasks marked finished/
3414    // crashed/etc. whose tmux session is actually still running, and flip
3415    // them back to running. Internally throttled to 8s so a manual call from
3416    // the open path or a fast invocation doesn't double up.
3417    _selfHealStaleTasks().catch(() => {});
3418  }, BG_MONITOR_INTERVAL_MS);
3419  _pollBackgroundStatus();
3420  _checkServeReachability();
3421}
3422
3423function _stopBackgroundMonitor() {
3424  if (_bgMonitorInterval) {
3425    clearInterval(_bgMonitorInterval);
3426    _bgMonitorInterval = null;
3427  }
3428  const statusEl = document.getElementById('cookbook-bg-status');
3429  if (statusEl) statusEl.style.display = 'none';
3430}
3431
3432// Retry-probe a freshly-added endpoint until its model server answers.
3433// A model that just reached "ready" in the cookbook often can't satisfy
3434// the 1s add-time probe (remote, weights still mmap-ing), so it's added
3435// offline. This polls the per-endpoint /probe (which uses a longer
3436// server-side timeout + persists cached_models) every few seconds until
3437// the endpoint reports models, then refreshes the picker. Bounded so a
3438// genuinely-dead server doesn't poll forever.
3439async function _probeEndpointUntilOnline(epId, host, port) {
3440  if (!epId) return;
3441  // Big models (e.g. 70B+) can take several minutes to load weights before
3442  // the server answers /v1/models. Probe for up to ~5 min, easing the
3443  // interval out so we're not hammering during a long warmup.
3444  const MAX_TRIES = 40;
3445  for (let i = 0; i < MAX_TRIES; i++) {
3446    const interval = i < 12 ? 5000 : 10000;   // 5s for the first minute, then 10s
3447    await new Promise(r => setTimeout(r, interval));
3448    try {
3449      // Hit the probe endpoint â it re-probes server-side and updates
3450      // cached_models. We consume (and discard) the SSE stream.
3451      await fetch(`/api/model-endpoints/${epId}/probe`, { credentials: 'same-origin' }).then(r => r.text()).catch(() => {});
3452      const eps = await fetch('/api/model-endpoints', { credentials: 'same-origin' }).then(r => r.json()).catch(() => []);
3453      const ep = (eps || []).find(e => e.id === epId);
3454      if (ep && (ep.models || []).length) {
3455        if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(true);
3456        if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
3457        window.dispatchEvent(new CustomEvent('ge:model-endpoints-updated', {
3458          detail: { baseUrl: ep.base_url || `http://${host}:${port}/v1`, host, port, model: (ep.models || [])[0] || '' },
3459        }));
3460        uiModule.showToast(`${host}:${port} is online`);
3461        return;
3462      }
3463    } catch (_) { /* keep retrying */ }
3464  }
3465}
3466
3467async function _pollBackgroundStatus() {
3468  try {
3469    // Pull any tasks the server knows about that aren't in localStorage
3470    // yet (e.g. agent-spawned downloads/serves). Without this merge,
3471    // _syncToServer keeps clobbering server-added tasks on every poll.
3472    try {
3473      const stateRes = await fetch('/api/cookbook/state', { credentials: 'same-origin' });
3474      if (stateRes.ok) {
3475        const serverState = await stateRes.json();
3476        const serverTasks = (serverState && Array.isArray(serverState.tasks)) ? serverState.tasks : [];
3477        if (serverTasks.length) {
3478          const localTasks = _loadTasks();
3479          const localIds = new Set(localTasks.map(t => t.sessionId));
3480          const merged = [...localTasks];
3481          let added = 0;
3482          for (const t of serverTasks) {
3483            if (t && t.sessionId && !localIds.has(t.sessionId) && !_isTombstoned(t.sessionId)) {
3484              merged.push(t);
3485              added++;
3486            }
3487          }
3488          if (added > 0) {
3489            localStorage.setItem(TASKS_KEY, JSON.stringify(merged.map(_stripTaskSecrets)));
3490            _renderRunningTab();
3491          }
3492        }
3493      }
3494    } catch (_) { /* non-fatal */ }
3495
3496    const res = await fetch('/api/cookbook/tasks/status', { credentials: 'same-origin' });
3497    if (!res.ok) return;
3498    const data = await res.json();
3499    const tasks = data.tasks || [];
3500
3501    // Reconcile the authoritative tmux/process status back into the persisted
3502    // client task list. The Running-tab reconnect loop also does this, but it
3503    // only exists while cards are rendered; after a page refresh or closed modal
3504    // dependency installs could finish server-side while localStorage stayed
3505    // stuck at "running".
3506    try {
3507      const statusById = new Map(tasks.map(t => [t.session_id, t]));
3508      const localTasks = _loadTasks();
3509      let changed = false;

3510      const completedDeps = [];
3511      for (const task of localTasks) {
3512        const live = statusById.get(task.sessionId);
3513        if (!live) continue;
3514        const updates = {};
3515        // A finished dependency install whose tmux pane is gone is reported
3516        // "stopped" by the backend (its pip package is never in the HF cache the
3517        // dead-session check inspects). Recover "done" from the retained output's
3518        // exit-0 sentinel so a clean install isn't downgraded to crashed.
3519        const depDone = !!task.payload?._dep && _depInstallSucceeded(task.output);
3520        const nextStatus = live.status === 'completed'
3521          ? 'done'
3522          : (live.status === 'error'
3523            ? 'error'
3524            : (live.status === 'stopped'
3525                ? (depDone ? 'done' : (task.type === 'download' ? 'crashed' : 'stopped'))
3526                : null));
3527        if (nextStatus && task.status !== nextStatus) {
3528          updates.status = nextStatus;
3529          if (nextStatus === 'done' && task.payload?._dep) completedDeps.push(task);
3530        }
3531        if ((live.status === 'running' || live.status === 'ready') && task.status !== live.status) {
3532          updates.status = live.status === 'ready' ? 'ready' : 'running';
3533        }
3534        if (live.progress && live.progress !== task.progress) updates.progress = live.progress;
3535        if (live.output_tail) {
3536          const previous = String(task.output || '');
3537          const tail = String(live.output_tail || '');
3538          if (tail && !previous.endsWith(tail)) {
3539            updates.output = `${previous ? `${previous}\n` : ''}${tail}`.slice(-5000);
3540          }
3541        }
3542        if (live.diagnosis && !task._diagnosisDismissed) {
3543          updates._backendDiagnosis = live.diagnosis;
3544        }
3545        if (live.cmd && !task.payload?._cmd) {
3546          updates.payload = { ...(task.payload || {}), _cmd: live.cmd };
3547        }
3548        if (Object.keys(updates).length) {
3549          Object.assign(task, updates);
3550          changed = true;
3551        }
3552      }
3553      if (changed) {
3554        _saveTasks(localTasks);
3555        _renderRunningTab();
3556        for (const task of localTasks) {
3557          if (!task._backendDiagnosis) continue;
3558          const el = document.querySelector(`[data-session-id="${CSS.escape(task.sessionId)}"]`);
3559          if (!el || el.querySelector('.cookbook-diagnosis')) continue;
3560          _showDiagnosis(el, task._backendDiagnosis, task.output || '');
3561        }
3562        completedDeps.forEach(t => _refreshDepsAfterInstall(t));
3563      }
3564    } catch (_) { /* non-fatal: background status should never break polling */ }
3565
3566    const statusEl = document.getElementById('cookbook-bg-status');
3567    const activeTasks = tasks.filter(t => t.status === 'running' || t.status === 'ready');
3568    const errorTasks = tasks.filter(t => t.status === 'error');
3569    const completedTasks = tasks.filter(t => t.status === 'completed');
3570
3571    // Auto-add serve endpoints that became ready (works even when modal is closed)
3572    const readyServes = tasks.filter(t => t.type === 'serve' && t.status === 'ready');
3573    for (const t of readyServes) {
3574      const localTasks = _loadTasks();
3575      const localTask = localTasks.find(lt => lt.sessionId === t.session_id);
3576      if (localTask && localTask._endpointAdded) continue;
3577
3578      let host = _connectHostFromRemote(localTask?.remoteHost || t.remote);
3579      const portMatch = localTask?.payload?._cmd?.match(/--port\s+(\d+)/)
3580        || localTask?.payload?._cmd?.match(/OLLAMA_HOST=[^\s:]+:(\d+)/);
3581      let port = portMatch ? portMatch[1] : '8000';
3582      let baseUrl = `http://${host}:${port}/v1`;
3583      const snapshot = t.output || localTask?.output || '';
3584      const ollamaUrlMatch = snapshot.match(/Ollama API ready on port\s+\d+:\s*(http:\/\/[^\s]+)/i);
3585      if (ollamaUrlMatch) {
3586        const endpoint = _endpointFromAdvertisedUrl(ollamaUrlMatch[1], host, '11434');
3587        if (endpoint) ({ host, port, baseUrl } = endpoint);
3588      }
3589      const _isDiffusion = localTask?.payload?._cmd?.includes('diffusion_server');
3590
3591      _updateTask(t.session_id, { _serveReady: true, _endpointAdded: true });
3592      if (localTask) _autoSaveWorkingConfig(localTask);   // remember working settings (modal may be closed)
3593
3594      // Auto-detect function-calling support from the serve cmd.
3595      // vLLM emits OpenAI-style tool_calls only when launched with
3596      // `--enable-auto-tool-choice`; local-only models otherwise
3597      // hallucinate a fake [TOOL_CALL]...[/TOOL_CALL] text format
3598      // the backend can't parse.
3599      const _cmd = localTask?.payload?._cmd || '';

3600      const _supportsTools = _cmd.includes('--enable-auto-tool-choice') || _isDiffusion === false && /(?:^|\s)(?:deepseek|gpt-[45o]|claude|gemini|qwen3|qwen2\.5|mixtral|llama-[34]|minimax|kimi|hermes|glm-4)/i.test(t.model);
3601
3602      fetch('/api/model-endpoints', { credentials: 'same-origin' })
3603        .then(r => r.json())
3604        .then(eps => {
3605          const hostPort = `${host}:${port}`;
3606          const existing = eps.find(e => e.base_url === baseUrl || e.base_url.includes(hostPort) || e.name === t.model);
3607          if (existing) {
3608            // Already registered â but it may be showing offline because
3609            // it was added while the server was still warming. Kick a
3610            // re-probe so it flips online without manual toggle.
3611            if (!(existing.models || []).length) _probeEndpointUntilOnline(existing.id, host, port);
3612            return null;
3613          }
3614          const fd = new FormData();
3615          fd.append('base_url', baseUrl);
3616          fd.append('name', t.model);
3617          fd.append('skip_probe', 'true');
3618          _appendCookbookEndpointScope(fd, localTask?.remoteHost || t.remote || '');
3619          if (_isDiffusion) fd.append('model_type', 'image');
3620          if (_supportsTools) fd.append('supports_tools', 'true');
3621          return fetch('/api/model-endpoints', { method: 'POST', credentials: 'same-origin', body: fd });
3622        })
3623        .then(async (res) => {
3624          if (res && res.ok) {
3625            uiModule.showToast(`Model endpoint added: ${host}:${port}`);
3626            const data = await res.json().catch(() => ({}));
3627            // A just-started server often can't answer the 1s add-time
3628            // probe, so it lands "offline". Retry-probe in the background
3629            // until /v1/models responds â no manual enable/disable needed.
3630            if (data && data.id) _probeEndpointUntilOnline(data.id, host, port);
3631            if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(true);
3632            if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
3633          }
3634        })
3635        .catch(() => {});
3636    }
3637
3638    if (errorTasks.length > 0) {
3639      _showCookbookNotif(true);
3640    } else if (completedTasks.length > 0) {
3641      _showCookbookNotif(false);
3642    } else if (activeTasks.length > 0) {
3643      _showCookbookNotif(false);
3644    } else {
3645      _clearCookbookNotif();
3646      _stopBackgroundMonitor();
3647    }
3648
3649    if (statusEl) {
3650      if (activeTasks.length > 0) {
3651        const t = activeTasks[0];
3652        if (t.type === 'serve') {
3653          if (t.progress) {
3654            // Show serve phase from backend (e.g. "loading 45%", "warming up", "idle", "12.5 tok/s")
3655            statusEl.textContent = t.progress;
3656          } else if (t.status === 'ready') {
3657            statusEl.textContent = 'ready';
3658          } else {
3659            statusEl.textContent = 'cooking';
3660          }
3661        } else {
3662          var _dlProgress = '';
3663          if (t.progress) {
3664            var _pctMatch = t.progress.match(/(\d+)%/);
3665            _dlProgress = _pctMatch ? ` ${_pctMatch[0]}` : '';
3666          }
3667          statusEl.textContent = `downloading${_dlProgress}`;
3668        }
3669        statusEl.style.display = '';
3670      } else if (errorTasks.length > 0) {
3671        statusEl.textContent = 'error';
3672        statusEl.style.display = '';
3673        statusEl.style.color = 'var(--color-error, #f44)';
3674      } else if (completedTasks.length > 0) {
3675        statusEl.textContent = 'done';
3676        statusEl.style.display = '';
3677        statusEl.style.color = 'var(--color-success, #4caf50)';
3678      } else {
3679        statusEl.style.display = 'none';
3680        statusEl.style.color = '';
3681      }
3682    }
3683    // Also clear the sidebar/rail icon highlight when no tasks are alive.
3684    // Without this, the cookbook icon stays at full opacity ("highlighted")
3685    // indefinitely once any task fires the notif, because the modal-open
3686    // clear only runs when the user actually reopens Cookbook.
3687    if (!activeTasks.length && !errorTasks.length) {
3688      _clearCookbookNotif();
3689    }
3690  } catch (e) {
3691    // Silent fail
3692  }
3693}
3694
3695// ââ Init: receive shared state/functions ââ
3696
3697export function initRunning(shared) {
3698  _envState = shared._envState;
3699  _sshCmd = shared._sshCmd;
3700  _getPort = shared._getPort;
3701  _sshPrefix = shared._sshPrefix;
3702  _getPlatform = shared._getPlatform;
3703  _isWindows = shared._isWindows;
3704  _buildEnvPrefix = shared._buildEnvPrefix;
3705  _loadPresets = shared._loadPresets;
3706  _savePresets = shared._savePresets;
3707  _copyText = shared._copyText;
3708  _persistEnvState = shared._persistEnvState;
3709  _refreshDependencies = shared._refreshDependencies;
3710  modelLogo = shared.modelLogo;
3711  esc = shared.esc;
3712  _detectBackend = shared._detectBackend;
3713  _detectToolParser = shared._detectToolParser;
3714  _detectModelOptimizations = shared._detectModelOptimizations;

3715  _buildServeCmd = shared._buildServeCmd;
3716
3717  // App boot: pull authoritative state from server, then auto-start
3718  // the background monitor unconditionally. Used to gate on "already
3719  // has running tasks" but that meant when the agent (or anyone)
3720  // added a task after boot, the UI never noticed. 10s poll of a
3721  // small status endpoint is cheap and gives the agent + the UI a
3722  // shared live picture.
3723  (async () => {
3724    try {
3725      await _syncFromServer();
3726    } catch {}
3727    _startBackgroundMonitor();
3728  })();
3729}
3730
3731// Also export _retryDownload and _nextAvailablePort for use by other modules
3732export { _retryDownload, _nextAvailablePort, _processQueue };

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.