1import { 2 abortStatusCheck, 3 eventSource, 4 event_types, 5 getRequestHeaders, 6 getStoppingStrings, 7 main_api, 8 max_context, 9 online_status, 10 resultCheckStatus, 11 saveSettingsDebounced, 12 setGenerationParamsFromPreset, 13 setOnlineStatus, 14 startStatusLoading, 15 substituteParams, 16} from '../script.js'; 17import { deriveTemplatesFromChatTemplate } from './chat-templates.js'; 18import { t } from './i18n.js'; 19import { autoSelectInstructPreset, selectContextPreset, selectInstructPreset } from './instruct-mode.js'; 20import { BIAS_CACHE, createNewLogitBiasEntry, displayLogitBias, getLogitBiasListResult } from './logit-bias.js'; 21 22import { power_user, registerDebugFunction } from './power-user.js'; 23import { getActiveManualApiSamplers, loadApiSelectedSamplers, isSamplerManualPriorityEnabled } from './samplerSelect.js'; 24import { SECRET_KEYS, writeSecret } from './secrets.js'; 25import { getEventSourceStream } from './sse-stream.js'; 26import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer, loadAphroditeModels, loadDreamGenModels, loadFeatherlessModels, loadGenericModels, loadInfermaticAIModels, loadLlamaCppModels, loadMancerModels, loadOllamaModels, loadOpenRouterModels, loadTabbyModels, loadTogetherAIModels, loadVllmModels, updateOpenRouterProvidersWarning } from './textgen-models.js'; 27import { ENCODE_TOKENIZERS, TEXTGEN_TOKENIZERS, TOKENIZER_SUPPORTED_KEY, getTextTokens, getTokenizerBestMatch, tokenizers } from './tokenizers.js'; 28import { AbortReason } from './util/AbortReason.js'; 29import { getSortableDelay, onlyUnique, arraysEqual, isObject } from './utils.js'; 30 31export const textgen_types = { 32 OOBA: 'ooba', 33 MANCER: 'mancer', 34 VLLM: 'vllm', 35 APHRODITE: 'aphrodite', 36 TABBY: 'tabby', 37 KOBOLDCPP: 'koboldcpp', 38 TOGETHERAI: 'togetherai', 39 LLAMACPP: 'llamacpp', 40 OLLAMA: 'ollama', 41 INFERMATICAI: 'infermaticai', 42 DREAMGEN: 'dreamgen', 43 OPENROUTER: 'openrouter', 44 FEATHERLESS: 'featherless', 45 HUGGINGFACE: 'huggingface', 46 GENERIC: 'generic', 47}; 48 49const { 50 GENERIC, 51 MANCER, 52 VLLM, 53 APHRODITE, 54 TABBY, 55 TOGETHERAI, 56 OOBA, 57 OLLAMA, 58 LLAMACPP, 59 INFERMATICAI, 60 DREAMGEN, 61 OPENROUTER, 62 KOBOLDCPP, 63 HUGGINGFACE, 64 FEATHERLESS, 65} = textgen_types; 66 67const LLAMACPP_DEFAULT_ORDER = [ 68 'penalties', 69 'dry', 70 'top_n_sigma', 71 'top_k', 72 'typ_p', 73 'top_p', 74 'min_p', 75 'xtc', 76 'temperature', 77 'adaptive_p', 78]; 79const OOBA_DEFAULT_ORDER = [ 80 'repetition_penalty', 81 'presence_penalty', 82 'frequency_penalty', 83 'dry', 84 'temperature', 85 'dynamic_temperature', 86 'quadratic_sampling', 87 'top_n_sigma', 88 'top_k', 89 'top_p', 90 'typical_p', 91 'epsilon_cutoff', 92 'eta_cutoff', 93 'tfs', 94 'top_a', 95 'min_p', 96 'adaptive_p', 97 'mirostat', 98 'xtc', 99 'encoder_repetition_penalty', 100 'no_repeat_ngram', 101]; 102export const APHRODITE_DEFAULT_ORDER = [ 103 'dry', 104 'penalties', 105 'no_repeat_ngram', 106 'temperature', 107 'top_nsigma', 108 'top_p_top_k', 109 'top_a', 110 'min_p', 111 'tfs', 112 'eta_cutoff', 113 'epsilon_cutoff', 114 'typical_p', 115 'quadratic', 116 'xtc', 117]; 118const BIAS_KEY = '#textgenerationwebui_api-settings'; 119 120// Maybe let it be configurable in the future? 121// (7 days later) The future has come. 122const MANCER_SERVER_KEY = 'mancer_server'; 123const MANCER_SERVER_DEFAULT = 'https://neuro.mancer.tech'; 124export let MANCER_SERVER = localStorage.getItem(MANCER_SERVER_KEY) ?? MANCER_SERVER_DEFAULT; 125export let TOGETHERAI_SERVER = 'https://api.together.xyz'; 126export let INFERMATICAI_SERVER = 'https://api.totalgpt.ai'; 127export let DREAMGEN_SERVER = 'https://dreamgen.com'; 128export let OPENROUTER_SERVER = 'https://openrouter.ai/api'; 129export let FEATHERLESS_SERVER = 'https://api.featherless.ai/v1'; 130 131export const SERVER_INPUTS = { 132 [textgen_types.OOBA]: '#textgenerationwebui_api_url_text', 133 [textgen_types.VLLM]: '#vllm_api_url_text', 134 [textgen_types.APHRODITE]: '#aphrodite_api_url_text', 135 [textgen_types.TABBY]: '#tabby_api_url_text', 136 [textgen_types.KOBOLDCPP]: '#koboldcpp_api_url_text', 137 [textgen_types.LLAMACPP]: '#llamacpp_api_url_text', 138 [textgen_types.OLLAMA]: '#ollama_api_url_text', 139 [textgen_types.HUGGINGFACE]: '#huggingface_api_url_text',
140 [textgen_types.GENERIC]: '#generic_api_url_text', 141}; 142 143const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5]; 144export const textgenerationwebui_settings = { 145 temp: 0.7, 146 temperature_last: true, 147 top_p: 0.5, 148 top_k: 40, 149 top_a: 0, 150 tfs: 1, 151 epsilon_cutoff: 0, 152 eta_cutoff: 0, 153 typical_p: 1, 154 min_p: 0, 155 rep_pen: 1.2, 156 rep_pen_range: 0, 157 rep_pen_decay: 0, 158 rep_pen_slope: 1, 159 no_repeat_ngram_size: 0, 160 penalty_alpha: 0, 161 num_beams: 1, 162 length_penalty: 1, 163 min_length: 0, 164 encoder_rep_pen: 1, 165 freq_pen: 0, 166 presence_pen: 0, 167 skew: 0, 168 do_sample: true, 169 early_stopping: false, 170 dynatemp: false, 171 min_temp: 0, 172 max_temp: 2.0, 173 dynatemp_exponent: 1.0, 174 smoothing_factor: 0.0, 175 smoothing_curve: 1.0, 176 dry_allowed_length: 2, 177 dry_multiplier: 0.0, 178 dry_base: 1.75, 179 dry_sequence_breakers: '["\\n", ":", "\\"", "*"]', 180 dry_penalty_last_n: 0, 181 max_tokens_second: 0, 182 seed: -1, 183 preset: 'Default', 184 add_bos_token: true, 185 stopping_strings: [], 186 //truncation_length: 2048, 187 ban_eos_token: false, 188 skip_special_tokens: true, 189 include_reasoning: true, 190 streaming: false, 191 mirostat_mode: 0, 192 mirostat_tau: 5, 193 mirostat_eta: 0.1, 194 guidance_scale: 1, 195 negative_prompt: '', 196 grammar_string: '', 197 json_schema: null, 198 json_schema_allow_empty: false, 199 banned_tokens: '', 200 global_banned_tokens: '', 201 send_banned_tokens: true, 202 sampler_priority: OOBA_DEFAULT_ORDER, 203 samplers: LLAMACPP_DEFAULT_ORDER, 204 samplers_priorities: APHRODITE_DEFAULT_ORDER, 205 ignore_eos_token: false, 206 spaces_between_special_tokens: true, 207 speculative_ngram: false, 208 type: textgen_types.OOBA, 209 mancer_model: 'mytholite', 210 togetherai_model: 'Gryphe/MythoMax-L2-13b', 211 infermaticai_model: '', 212 ollama_model: '', 213 openrouter_model: 'openrouter/auto', 214 openrouter_providers: [], 215 openrouter_quantizations: [], 216 vllm_model: '', 217 aphrodite_model: '', 218 dreamgen_model: 'lucid-v1-extra-large/text', 219 tabby_model: '', 220 llamacpp_model: '', 221 sampler_order: KOBOLDCPP_ORDER, 222 logit_bias: [], 223 n: 1, 224 server_urls: {}, 225 custom_model: '', 226 bypass_status_check: false, 227 openrouter_allow_fallbacks: true, 228 xtc_threshold: 0.1, 229 xtc_probability: 0, 230 nsigma: 0.0, 231 min_keep: 0, 232 featherless_model: '', 233 generic_model: '', 234 extensions: {}, 235 adaptive_target: -0.01, 236 adaptive_decay: 0.9, 237}; 238 239export { 240 showSamplerControls as showTGSamplerControls, 241}; 242 243export let textgenerationwebui_banned_in_macros = []; 244 245export let textgenerationwebui_presets = []; 246export let textgenerationwebui_preset_names = []; 247 248export const setting_names = [ 249 'temp', 250 'temperature_last', 251 'rep_pen', 252 'rep_pen_range', 253 'rep_pen_decay', 254 'rep_pen_slope', 255 'no_repeat_ngram_size', 256 'top_k', 257 'top_p', 258 'top_a', 259 'tfs', 260 'epsilon_cutoff', 261 'eta_cutoff', 262 'typical_p', 263 'min_p', 264 'penalty_alpha', 265 'num_beams', 266 'length_penalty', 267 'min_length', 268 'dynatemp', 269 'min_temp', 270 'max_temp', 271 'dynatemp_exponent', 272 'smoothing_factor', 273 'smoothing_curve', 274 'dry_allowed_length', 275 'dry_multiplier', 276 'dry_base', 277 'dry_sequence_breakers', 278 'dry_penalty_last_n', 279 'max_tokens_second', 280 'encoder_rep_pen', 281 'freq_pen', 282 'presence_pen', 283 'skew', 284 'do_sample', 285 'early_stopping', 286 'seed', 287 'add_bos_token', 288 'ban_eos_token', 289 'skip_special_tokens', 290 'include_reasoning', 291 'streaming', 292 'mirostat_mode', 293 'mirostat_tau', 294 'mirostat_eta', 295 'guidance_scale', 296 'negative_prompt', 297 'grammar_string', 298 'json_schema', 299 'banned_tokens', 300 'global_banned_tokens', 301 'send_banned_tokens', 302 'ignore_eos_token', 303 'spaces_between_special_tokens', 304 'speculative_ngram', 305 'sampler_order', 306 'sampler_priority', 307 'samplers', 308 'samplers_priorities', 309 'n', 310 'logit_bias', 311 'custom_model', 312 'bypass_status_check', 313 'openrouter_allow_fallbacks', 314 'xtc_threshold', 315 'xtc_probability', 316 'nsigma', 317 'min_keep', 318 'generic_model', 319 'extensions', 320 'json_schema_allow_empty', 321 'adaptive_target', 322 'adaptive_decay', 323]; 324 325const DYNATEMP_BLOCK = document.getElementById('dynatemp_block_ooba'); 326 327export function validateTextGenUrl() { 328 const selector = SERVER_INPUTS[textgenerationwebui_settings.type]; 329 330 if (!selector) { 331 return; 332 } 333 334 const control = $(selector); 335 const url = String(control.val()).trim(); 336 const formattedUrl = formatTextGenURL(url); 337 338 if (!formattedUrl) { 339 toastr.error(t`Enter a valid API URL`, 'Text Completion API'); 340 return; 341 } 342 343 control.val(formattedUrl); 344} 345 346/** 347 * Gets the API URL for the selected text generation type. 348 * @param {string} type If it's set, ignores active type 349 * @returns {string} API URL 350 */ 351export function getTextGenServer(type = null) { 352 const selectedType = type ?? textgenerationwebui_settings.type; 353 switch (selectedType) { 354 case FEATHERLESS: 355 return FEATHERLESS_SERVER; 356 case MANCER: 357 return MANCER_SERVER; 358 case TOGETHERAI: 359 return TOGETHERAI_SERVER; 360 case INFERMATICAI: 361 return INFERMATICAI_SERVER; 362 case DREAMGEN: 363 return DREAMGEN_SERVER; 364 case OPENROUTER: 365 return OPENROUTER_SERVER; 366 default: 367 return textgenerationwebui_settings.server_urls[selectedType] ?? ''; 368 } 369} 370 371async function selectPreset(name) { 372 const preset = textgenerationwebui_presets[textgenerationwebui_preset_names.indexOf(name)]; 373 374 if (!preset) { 375 return; 376 } 377 378 textgenerationwebui_settings.preset = name; 379 for (const name of setting_names) { 380 const value = preset[name]; 381 setSettingByName(name, value, true); 382 } 383 setGenerationParamsFromPreset(preset); 384 BIAS_CACHE.delete(BIAS_KEY); 385 displayLogitBias(preset.logit_bias, BIAS_KEY); 386 saveSettingsDebounced(); 387} 388 389export function formatTextGenURL(value) { 390 try { 391 const noFormatTypes = [MANCER, TOGETHERAI, INFERMATICAI, DREAMGEN, OPENROUTER]; 392 if (noFormatTypes.includes(textgenerationwebui_settings.type)) { 393 return value; 394 } 395 396 const url = new URL(value); 397 return url.toString(); 398 } catch { 399 // Just using URL as a validation check 400 } 401 return null; 402} 403 404function convertPresets(presets) { 405 return Array.isArray(presets) ? presets.map((p) => JSON.parse(p)) : []; 406} 407 408function getTokenizerForTokenIds() { 409 const bestMatchTokenizer = getTokenizerBestMatch('textgenerationwebui'); 410 if (bestMatchTokenizer === tokenizers.API_TEXTGENERATIONWEBUI) { 411 return tokenizers.API_CURRENT; 412 } 413 414 if (power_user.tokenizer === tokenizers.API_CURRENT && TEXTGEN_TOKENIZERS.includes(textgenerationwebui_settings.type)) { 415 return tokenizers.API_CURRENT; 416 } 417 418 if (ENCODE_TOKENIZERS.includes(power_user.tokenizer)) { 419 return power_user.tokenizer; 420 } 421 422 if (textgenerationwebui_settings.type === OPENROUTER) { 423 return getCurrentOpenRouterModelTokenizer(); 424 } 425 426 if (textgenerationwebui_settings.type === DREAMGEN) { 427 return getCurrentDreamGenModelTokenizer(); 428 } 429 430 return tokenizers.LLAMA; 431} 432 433/** 434 * Gets the custom token bans from settings and macros. 435 * @param {TextCompletionSettings} settings Text completion settings to use 436 * @typedef {{banned_tokens: string, banned_strings: string[]}} TokenBanResult 437 * @returns {TokenBanResult} String with comma-separated banned token IDs 438 */ 439function getCustomTokenBans(settings = null) { 440 settings = settings ?? textgenerationwebui_settings; 441 if (!settings.send_banned_tokens || (!settings.banned_tokens && !sett
441ings.global_banned_tokens && !textgenerationwebui_banned_in_macros.length)) { 442 return { 443 banned_tokens: '', 444 banned_strings: [], 445 }; 446 } 447 448 const tokenizer = getTokenizerForTokenIds(); 449 const banned_tokens = []; 450 const banned_strings = []; 451 const sequences = [] 452 .concat(settings.banned_tokens.split('\n')) 453 .concat(settings.global_banned_tokens.split('\n')) 454 .concat(textgenerationwebui_banned_in_macros) 455 .filter(x => x.length > 0) 456 .filter(onlyUnique) 457 .map(x => substituteParams(x)); 458 459 //debug 460 if (textgenerationwebui_banned_in_macros.length) { 461 console.log('=== Found banned word sequences in the macros:', textgenerationwebui_banned_in_macros, 'Resulting array of banned sequences (will be used this generation turn):', sequences); 462 } 463 464 //clean old temporary bans found in macros before, for the next generation turn. 465 textgenerationwebui_banned_in_macros = []; 466 467 for (const line of sequences) { 468 // Raw token ids, JSON serialized 469 if (line.startsWith('[') && line.endsWith(']')) { 470 try { 471 const tokens = JSON.parse(line); 472 473 if (Array.isArray(tokens) && tokens.every(t => Number.isInteger(t))) { 474 banned_tokens.push(...tokens); 475 } else { 476 throw new Error('Not an array of integers'); 477 } 478 } catch (err) { 479 console.log(`Failed to parse bad word token list: ${line}`, err); 480 } 481 } else if (line.startsWith('"') && line.endsWith('"')) { 482 // Remove the enclosing quotes 483 484 banned_strings.push(line.slice(1, -1)); 485 } else { 486 try { 487 const tokens = getTextTokens(tokenizer, line); 488 banned_tokens.push(...tokens); 489 } catch { 490 console.log(`Could not tokenize raw text: ${line}`); 491 } 492 } 493 } 494 495 return { 496 banned_tokens: banned_tokens.filter(onlyUnique).map(x => String(x)).join(','), 497 banned_strings: banned_strings, 498 }; 499} 500 501/** 502 * Sets the banned strings kill switch toggle. 503 * @param {boolean} isEnabled Kill switch state 504 * @param {string} title Label title 505 */ 506function toggleBannedStringsKillSwitch(isEnabled, title) { 507 $('#send_banned_tokens_textgenerationwebui').prop('checked', isEnabled); 508 $('#send_banned_tokens_label').find('.menu_button').toggleClass('toggleEnabled', isEnabled).prop('title', title); 509 textgenerationwebui_settings.send_banned_tokens = isEnabled; 510 saveSettingsDebounced(); 511} 512 513/** 514 * Calculates logit bias object from the logit bias list. 515 * @param {TextCompletionSettings} settings Text completion settings 516 * @returns {object} Logit bias object 517 */ 518function calculateLogitBias(settings = null) { 519 settings = settings ?? textgenerationwebui_settings; 520 521 if (!Array.isArray(settings.logit_bias) || settings.logit_bias.length === 0) { 522 return {}; 523 } 524 525 const tokenizer = getTokenizerForTokenIds(); 526 const result = {}; 527 528 /** 529 * Adds bias to the logit bias object. 530 * @param {number} bias 531 * @param {number[]} sequence 532 * @returns {object} Accumulated logit bias object 533 */ 534 function addBias(bias, sequence) { 535 if (sequence.length === 0) { 536 return; 537 } 538 539 for (const logit of sequence) { 540 const key = String(logit); 541 result[key] = bias; 542 } 543 544 return result; 545 } 546 547 getLogitBiasListResult(settings.logit_bias, tokenizer, addBias); 548 549 return result; 550} 551 552export async function loadTextGenSettings(data, loadedSettings) { 553 await loadApiSelectedSamplers(); 554 textgenerationwebui_presets = convertPresets(data.textgenerationwebui_presets); 555 textgenerationwebui_preset_names = data.textgenerationwebui_preset_names ?? []; 556 Object.assign(textgenerationwebui_settings, loadedSettings.textgenerationwebui_settings ?? {}); 557 558 if (loadedSettings.api_server_textgenerationwebui) { 559 for (const type of Object.keys(SERVER_INPUTS)) { 560 textgenerationwebui_settings.server_urls[type] = loadedSettings.api_server_textgenerationwebui; 561 } 562 delete loadedSettings.api_server_textgenerationwebui; 563 } 564 565 for (const [type, selector] of Object.entries(SERVER_INPUTS)) { 566 const control = $(selector); 567 control.val(textgenerationwebui_settings.server_urls[type] ?? '').on('input', function () { 568 textgenerationwebui_settings.server_urls[type] = String($(this).val()).trim(); 569 saveSettingsDebounced(); 570 }); 571 } 572 573 if (loadedSettings.api_use_mancer_webui) { 574 textgenerationwebui_settings.type = MANCER; 575 } 576 577 for (const name of textgenerationwebui_preset_names) { 578 const option = document.createElement('option'); 579 option.value = name; 580 option.innerText = name; 581 $('#settings_preset_textgenerationwebui').append(option); 582 } 583 584 if (textgenerationwebui_settings.preset) { 585 $('#settings_preset_textgenerationwebui').val(textgenerationwebui_settings.preset); 586 } 587 588 for (const i of setting_names) { 589 const value = textgenerationwebui_settings[i]; 590 setSettingByName(i, value); 591 } 592 593 $('#textgen_type').val(textgenerationwebui_settings.type); 594 $('#openrouter_providers_text').val(textgenerationwebui_settings.openrouter_providers).trigger('change'); 595 $('#openrouter_quantizations_text').val(textgenerationwebui_settings.openrouter_quantizations).trigger('change'); 596 showSamplerControls(textgenerationwebui_settings.type); 597 BIAS_CACHE.delete(BIAS_KEY); 598 displayLogitBias(textgenerationwebui_settings.logit_bias, BIAS_KEY); 599 600 registerDebugFunction('change-mancer-url', 'Change Mancer base URL', 'Change Mancer API server base URL', () => { 601 const result = prompt(`Enter Mancer base URL\nDefault: ${MANCER_SERVER_DEFAULT}`, MANCER_SERVER); 602 603 if (result) { 604 localStorage.setItem(MANCER_SERVER_KEY, result); 605 MANCER_SERVER = result; 606 } 607 }); 608} 609 610/** 611 * Sorts the sampler items by the given order. 612 * @param {any[]} orderArray Sampler order array. 613 */ 614function sortKoboldItemsByOrder(orderArray) { 615 console.debug('Preset samplers order: ' + orderArray); 616 const $draggableItems = $('#koboldcpp_order'); 617 618 for (let i = 0; i < orderArray.length; i++) { 619 const index = orderArray[i]; 620 const $item = $draggableItems.find(`[data-id="${index}"]`).detach(); 621 $draggableItems.append($item); 622 } 623} 624 625function sortLlamacppItemsByOrder(orderArray) { 626 console.debug('Preset samplers order: ', orderArray); 627 const $container = $('#llamacpp_samplers_sortable'); 628 629 orderArray.forEach((name) => { 630 const $item = $container.find(`[data-name="${name}"]`).detach(); 631 $container.append($item); 632 }); 633} 634 635function sortOobaItemsByOrder(orderArray) { 636 console.debug('Preset samplers order: ', orderArray); 637 const $container = $('#sampler_priority_container'); 638 639 orderArray.forEach((name) => { 640 const $item = $container.find(`[data-name="${name}"]`).detach(); 641 $container.append($item); 642 }); 643} 644 645/** 646 * Sorts the Aphrodite sampler items by the given order. 647 * @param {string[]} orderArray Sampler order array. 648 */ 649function sortAphroditeItemsByOrder(orderArray) { 650 console.debug('Preset samplers order: ', orderArray); 651 const $container = $('#sampler_priority_container_aphrodite'); 652 653 orderArray.forEach((name) => { 654 const $item = $container.find(`[data-name="${name}"]`).detach(); 655 $container.append($item); 656 }); 657} 658 659async function getStatusTextgen() { 660 const url = '/api/backends/text-completions/status'; 661 662 const endpoint = getTextGenServer(); 663 664 if (!endpoint) { 665 console.warn('No endpoint for status check'); 666 setOnlineStatus('no_connection'); 667 return resultCheckStatus(); 668 } 669 670 // Clear logit bias cache 671 BIAS_CACHE.delete(BIAS_KEY); 672
673 if ([textgen_types.GENERIC, textgen_types.OOBA].includes(textgenerationwebui_settings.type) && textgenerationwebui_settings.bypass_status_check) { 674 setOnlineStatus(t`Status check bypassed`); 675 return resultCheckStatus(); 676 } 677 678 try { 679 const response = await fetch(url, { 680 method: 'POST', 681 headers: getRequestHeaders(), 682 body: JSON.stringify({ 683 api_server: endpoint, 684 api_type: textgenerationwebui_settings.type, 685 }), 686 signal: abortStatusCheck.signal, 687 }); 688 689 const data = await response.json(); 690 691 if (textgenerationwebui_settings.type === textgen_types.MANCER) { 692 loadMancerModels(data?.data); 693 setOnlineStatus(textgenerationwebui_settings.mancer_model); 694 } else if (textgenerationwebui_settings.type === textgen_types.TOGETHERAI) { 695 loadTogetherAIModels(data?.data); 696 setOnlineStatus(textgenerationwebui_settings.togetherai_model); 697 } else if (textgenerationwebui_settings.type === textgen_types.OLLAMA) { 698 loadOllamaModels(data?.data); 699 setOnlineStatus(textgenerationwebui_settings.ollama_model || t`Connected`); 700 } else if (textgenerationwebui_settings.type === textgen_types.INFERMATICAI) { 701 loadInfermaticAIModels(data?.data); 702 setOnlineStatus(textgenerationwebui_settings.infermaticai_model); 703 } else if (textgenerationwebui_settings.type === textgen_types.DREAMGEN) { 704 loadDreamGenModels(data?.data); 705 setOnlineStatus(textgenerationwebui_settings.dreamgen_model); 706 } else if (textgenerationwebui_settings.type === textgen_types.OPENROUTER) { 707 loadOpenRouterModels(data?.data); 708 setOnlineStatus(textgenerationwebui_settings.openrouter_model); 709 } else if (textgenerationwebui_settings.type === textgen_types.VLLM) { 710 loadVllmModels(data?.data); 711 setOnlineStatus(textgenerationwebui_settings.vllm_model); 712 } else if (textgenerationwebui_settings.type === textgen_types.APHRODITE) { 713 loadAphroditeModels(data?.data); 714 setOnlineStatus(textgenerationwebui_settings.aphrodite_model); 715 } else if (textgenerationwebui_settings.type === textgen_types.FEATHERLESS) { 716 loadFeatherlessModels(data?.data); 717 setOnlineStatus(textgenerationwebui_settings.featherless_model); 718 } else if (textgenerationwebui_settings.type === textgen_types.TABBY) { 719 loadTabbyModels(data?.data); 720 setOnlineStatus(textgenerationwebui_settings.tabby_model || data?.result); 721 } else if (textgenerationwebui_settings.type === textgen_types.LLAMACPP) { 722 loadLlamaCppModels(data?.data); 723 setOnlineStatus(textgenerationwebui_settings.llamacpp_model || data?.result || t`Connected`); 724 } else if (textgenerationwebui_settings.type === textgen_types.GENERIC) { 725 loadGenericModels(data?.data); 726 setOnlineStatus(textgenerationwebui_settings.generic_model || data?.result || t`Connected`); 727 } else { 728 setOnlineStatus(data?.result); 729 } 730 731 if (!online_status) { 732 setOnlineStatus('no_connection'); 733 } 734 735 power_user.chat_template_hash = ''; 736 737 // Determine instruct mode preset 738 const autoSelected = autoSelectInstructPreset(online_status); 739 740 const supportsTokenization = response.headers.get('x-supports-tokenization') === 'true'; 741 supportsTokenization ? sessionStorage.setItem(TOKENIZER_SUPPORTED_KEY, 'true') : sessionStorage.removeItem(TOKENIZER_SUPPORTED_KEY); 742 743 const wantsInstructDerivation = !autoSelected && (power_user.instruct.enabled && power_user.instruct_derived); 744 const wantsContextDerivation = !autoSelected && power_user.context_derived; 745 const wantsContextSize = power_user.context_size_derived; 746 const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(textgenerationwebui_settings.type); 747 748 if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation || wantsContextSize)) { 749 const model = textgenerationwebui_settings.type === textgen_types.LLAMACPP 750 ? textgenerationwebui_settings.llamacpp_model 751 : undefined; 752 753 const response = await fetch('/api/backends/text-completions/props', { 754 method: 'POST', 755 headers: getRequestHeaders(), 756 body: JSON.stringify({ 757 api_server: endpoint, 758 api_type: textgenerationwebui_settings.type, 759 model: model, 760 }), 761 }); 762 763 if (response.ok) { 764 const data = await response.json(); 765 if (data) { 766 const { chat_template, chat_template_hash } = data; 767 power_user.chat_template_hash = chat_template_hash; 768 769 if (wantsContextSize && 'default_generation_settings' in data) { 770 const backend_max_context = data.default_generation_settings.n_ctx; 771 if (backend_max_context && typeof backend_max_context === 'number') { 772 const old_value = max_context; 773 if (max_context !== backend_max_context) { 774 setGenerationParamsFromPreset({ max_length: backend_max_context }); 775 } 776 if (old_value !== max_context) { 777 console.log(`Auto-switched max context from ${old_value} to ${max_context}`); 778 toastr.info(`${old_value} ⇒ ${max_context}`, 'Context Size Changed'); 779 } 780 } 781 } 782 console.log(`We have chat template ${chat_template.split('\n')[0]}...`); 783 const savedTemplate = power_user.model_templates_mappings[chat_template_hash]; 784 const derivedTemplate = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash); 785 const { context, instruct } = savedTemplate ?? derivedTemplate; 786 787 if (wantsContextDerivation && context) { 788 selectContextPreset(context, { isAuto: true }); 789 } 790 if (wantsInstructDerivation && power_user.instruct.enabled && instruct) { 791 selectInstructPreset(instruct, { isAuto: true }); 792 } 793 } 794 } 795 } 796 797 // We didn't get a 200 status code, but the endpoint has an explanation. Which means it DID connect, but I digress. 798 if (online_status === 'no_connection' && data.response) { 799 toastr.error(data.response, t`API Error`, { timeOut: 5000, preventDuplicates: true }); 800 } 801 } catch (err) { 802 if (err instanceof AbortReason) { 803 console.info('Status check aborted.', err.reason); 804 } else { 805 console.error('Error getting status', err); 806 } 807 setOnlineStatus('no_connection'); 808 } 809 810 return resultCheckStatus(); 811} 812
813export function initTextGenSettings() { 814 $('#send_banned_tokens_textgenerationwebui').on('change', function () { 815 const checked = !!$(this).prop('checked'); 816 toggleBannedStringsKillSwitch(checked, 817 checked 818 ? t`Banned tokens/strings are being sent in the request.` 819 : t`Banned tokens/strings are NOT being sent in the request.`); 820 }); 821 822 $('#koboldcpp_order').sortable({ 823 delay: getSortableDelay(), 824 stop: function () { 825 const order = []; 826 $('#koboldcpp_order').children().each(function () { 827 order.push($(this).data('id')); 828 }); 829 textgenerationwebui_settings.sampler_order = order; 830 console.log('Samplers reordered:', textgenerationwebui_settings.sampler_order); 831 saveSettingsDebounced(); 832 }, 833 }); 834 835 $('#koboldcpp_default_order').on('click', function () { 836 textgenerationwebui_settings.sampler_order = KOBOLDCPP_ORDER; 837 sortKoboldItemsByOrder(textgenerationwebui_settings.sampler_order); 838 saveSettingsDebounced(); 839 }); 840 841 $('#llamacpp_samplers_sortable').sortable({ 842 delay: getSortableDelay(), 843 stop: function () { 844 const order = []; 845 $('#llamacpp_samplers_sortable').children().each(function () { 846 order.push($(this).data('name')); 847 }); 848 textgenerationwebui_settings.samplers = order; 849 console.log('Samplers reordered:', textgenerationwebui_settings.samplers); 850 saveSettingsDebounced(); 851 }, 852 }); 853 854 $('#llamacpp_samplers_default_order').on('click', function () { 855 sortLlamacppItemsByOrder(LLAMACPP_DEFAULT_ORDER); 856 textgenerationwebui_settings.samplers = LLAMACPP_DEFAULT_ORDER; 857 console.log('Default samplers order loaded:', textgenerationwebui_settings.samplers); 858 saveSettingsDebounced(); 859 }); 860 861 $('#sampler_priority_container').sortable({ 862 delay: getSortableDelay(), 863 stop: function () { 864 const order = []; 865 $('#sampler_priority_container').children().each(function () { 866 order.push($(this).data('name')); 867 }); 868 textgenerationwebui_settings.sampler_priority = order; 869 console.log('Samplers reordered:', textgenerationwebui_settings.sampler_priority); 870 saveSettingsDebounced(); 871 }, 872 }); 873 874 $('#sampler_priority_container_aphrodite').sortable({ 875 delay: getSortableDelay(), 876 stop: function () { 877 const order = []; 878 $('#sampler_priority_container_aphrodite').children().each(function () { 879 order.push($(this).data('name')); 880 }); 881 textgenerationwebui_settings.samplers_priorities = order; 882 console.log('Samplers reordered:', textgenerationwebui_settings.samplers_priorities); 883 saveSettingsDebounced(); 884 }, 885 }); 886 887 $('#tabby_json_schema').on('input', function () { 888 const json_schema_string = String($(this).val()); 889 890 if (json_schema_string) { 891 try { 892 textgenerationwebui_settings.json_schema = JSON.parse(json_schema_string); 893 } catch { 894 textgenerationwebui_settings.json_schema = null; 895 } 896 } else { 897 textgenerationwebui_settings.json_schema = null; 898 } 899 900 saveSettingsDebounced(); 901 }); 902 903 $('#textgenerationwebui_default_order').on('click', function () { 904 sortOobaItemsByOrder(OOBA_DEFAULT_ORDER); 905 textgenerationwebui_settings.sampler_priority = OOBA_DEFAULT_ORDER; 906 console.log('Default samplers order loaded:', textgenerationwebui_settings.sampler_priority); 907 saveSettingsDebounced(); 908 }); 909 910 $('#aphrodite_default_order').on('click', function () { 911 sortAphroditeItemsByOrder(APHRODITE_DEFAULT_ORDER); 912 textgenerationwebui_settings.samplers_priorities = APHRODITE_DEFAULT_ORDER; 913 console.log('Default samplers order loaded:', textgenerationwebui_settings.samplers_priorities); 914 saveSettingsDebounced(); 915 }); 916 917 $('#textgen_type').on('change', function () { 918 const type = String($(this).val()); 919 textgenerationwebui_settings.type = type; 920 921 if ([VLLM, APHRODITE, INFERMATICAI].includes(textgenerationwebui_settings.type)) { 922 $('#mirostat_mode_textgenerationwebui').attr('step', 2); //Aphro disallows mode 1 923 $('#do_sample_textgenerationwebui').prop('checked', true); //Aphro should always do sample; 'otherwise set temp to 0 to mimic no sample' 924 $('#ban_eos_token_textgenerationwebui').prop('checked', false); //Aphro should not ban EOS, just ignore it; 'add token '2' to ban list do to this' 925 //special handling for vLLM/Aphrodite topK -1 disable state 926 $('#top_k_textgenerationwebui').attr('min', -1); 927 if ($('#top_k_textgenerationwebui').val() === '0' || textgenerationwebui_settings.top_k === 0) { 928 textgenerationwebui_settings.top_k = -1; 929 $('#top_k_textgenerationwebui').val('-1').trigger('input'); 930 } 931 } else { 932 $('#mirostat_mode_textgenerationwebui').attr('step', 1); 933 //undo special vLLM/Aphrodite setup for topK 934 $('#top_k_textgenerationwebui').attr('min', 0); 935 if ($('#top_k_textgenerationwebui').val() === '-1' || textgenerationwebui_settings.top_k === -1) { 936 textgenerationwebui_settings.top_k = 0; 937 $('#top_k_textgenerationwebui').val('0').trigger('input'); 938 } 939 } 940 941 showSamplerControls(type); 942 setOnlineStatus('no_connection'); 943 BIAS_CACHE.delete(BIAS_KEY); 944 945 $('#main_api').trigger('change'); 946 947 if (!SERVER_INPUTS[type] || textgenerationwebui_settings.server_urls[type]) { 948 $('#api_button_textgenerationwebui').trigger('click'); 949 } 950 951 saveSettingsDebounced(); 952 }); 953 954 $('#settings_preset_textgenerationwebui').on('change', async function () { 955 const presetName = $(this).val(); 956 await selectPreset(presetName); 957 await eventSource.emit(event_types.PRESET_CHANGED, { apiId: 'textgenerationwebui', name: presetName }); 958 }); 959 960 $('#samplerResetButton').off('click').on('click', function () { 961 const inputs = { 962 'temp_textgenerationwebui': 1, 963 'top_k_textgenerationwebui': [INFERMATICAI, APHRODITE, VLLM].includes(textgenerationwebui_settings.type) ? -1 : 0, 964 'top_p_textgenerationwebui': 1, 965 'min_p_textgenerationwebui': 0, 966 'rep_pen_textgenerationwebui': 1, 967 'rep_pen_range_textgenerationwebui': 0, 968 'rep_pen_decay_textgenerationwebui': 0, 969 'dynatemp_textgenerationwebui': false, 970 'seed_textgenerationwebui': -1, 971 'ban_eos_token_textgenerationwebui': false, 972 'do_sample_textgenerationwebui': true, 973 'add_bos_token_textgenerationwebui': true, 974 'temperature_last_textgenerationwebui': true, 975 'skip_special_tokens_textgenerationwebui': true, 976 'include_reasoning_textgenerationwebui': true, 977 'top_a_textgenerationwebui': 0, 978 'top_a_counter_textgenerationwebui': 0, 979 'mirostat_mode_textgenerationwebui': 0, 980 'mirostat_tau_textgenerationwebui': 5, 981 'mirostat_eta_textgenerationwebui': 0.1, 982 'tfs_textgenerationwebui': 1, 983 'epsilon_cutoff_textgenerationwebui': 0, 984 'eta_cutoff_textgenerationwebui': 0, 985 'encoder_rep_pen_textgenerationwebui': 1, 986 'freq_pen_textgenerationwebui': 0, 987 'presence_pen_textgenerationwebui': 0, 988 'skew_textgenerationwebui': 0, 989 'no_repeat_ngram_size_textgenerationwebui': 0, 990 'speculative_ngram_textgenerationwebui': false, 991 'min_length_textgenerationwebui': 0, 992 'num_beams_textgenerationwebui': 1, 993 'length_penalty_textgenerationwebui': 1, 994 'penalty_alpha_textgenerationwebui': 0, 995 'typical_p_textgenerationwebui': 1, // Added entry 996 'guidance_scale_textgenerationwebui': 1, 997 'smoothing_factor_textgenerationwebui': 0, 998 'smoothing_curve_textgenerationwebui': 1, 999 'dry_allowed_length_textgenerationwebui': 2, 1000 'dry_multiplier_textgenerationwebui': 0, 1001 'dry_base_textgenerationwebui': 1.75, 1002 'dry_penalty_last_n_textgenerationwebui': 0, 1003 'xtc_threshold_textgenerationwebui': 0.1, 1004 'xtc_probability_textgenerationwebui': 0, 1005 'nsigma_textgenerationwebui': 0, 1006 'min_keep_textgenerationwebui': 0, 1007 'adaptive_target_textgenerationwebui': -0.01, 1008 'adaptive_decay_textgenerationwebui': 0.9, 1009 }; 1010 1011 for (const [id, value] of Object.entries(inputs)) { 1012 const inputElement = $(`#${id}`); 1013 const valueToSet = typeof value === 'boolean' ? String(value) : value; 1014 if (inputElement.prop('type') === 'checkbox') { 1015 inputElement.prop('checked', value).trigger('input'); 1016 } else if (inputElement.prop('type') === 'number') { 1017 inputElement.val(valueToSet).trigger('input'); 1018 } else { 1019 inputElement.val(valueToSet).trigger('input');
1020 if (power_user.enableZenSliders) { 1021 let masterElementID = inputElement.prop('id'); 1022 console.log(masterElementID); 1023 let zenSlider = $(`#${masterElementID}_zenslider`).slider(); 1024 zenSlider.slider('option', 'value', value); 1025 zenSlider.slider('option', 'slide') 1026 .call(zenSlider, null, { 1027 handle: $('.ui-slider-handle', zenSlider), value: value, 1028 }); 1029 } 1030 } 1031 } 1032 }); 1033 1034 for (const i of setting_names) { 1035 $(`#${i}_textgenerationwebui`).attr('x-setting-id', i); 1036 $(document).on('input', `#${i}_textgenerationwebui`, function () { 1037 const isCheckbox = $(this).attr('type') == 'checkbox'; 1038 const isText = $(this).attr('type') == 'text' || $(this).is('textarea'); 1039 const id = $(this).attr('x-setting-id'); 1040 1041 if (isCheckbox) { 1042 const value = $(this).prop('checked'); 1043 textgenerationwebui_settings[id] = value; 1044 } else if (isText) { 1045 const value = $(this).val(); 1046 textgenerationwebui_settings[id] = value; 1047 } else { 1048 const value = Number($(this).val()); 1049 $(`#${id}_counter_textgenerationwebui`).val(value); 1050 textgenerationwebui_settings[id] = value; 1051 //special handling for vLLM/Aphrodite using -1 as disabled instead of 0 1052 if ($(this).attr('id') === 'top_k_textgenerationwebui' && [INFERMATICAI, APHRODITE, VLLM].includes(textgenerationwebui_settings.type) && value === 0) { 1053 textgenerationwebui_settings[id] = -1; 1054 $(this).val(-1); 1055 } 1056 } 1057 saveSettingsDebounced(); 1058 }); 1059 } 1060 1061 $('#textgen_logit_bias_new_entry').on('click', () => createNewLogitBiasEntry(textgenerationwebui_settings.logit_bias, BIAS_KEY)); 1062 1063 $('#openrouter_providers_text').on('change', function () { 1064 const selectedProviders = $(this).val(); 1065 1066 // Not a multiple select? 1067 if (!Array.isArray(selectedProviders)) { 1068 return; 1069 } 1070 1071 textgenerationwebui_settings.openrouter_providers = selectedProviders; 1072 1073 updateOpenRouterProvidersWarning('#openrouter_providers_text'); 1074 saveSettingsDebounced(); 1075 }); 1076 1077 $('#openrouter_allow_fallbacks_textgenerationwebui').on('input', function () { 1078 updateOpenRouterProvidersWarning('#openrouter_providers_text'); 1079 }); 1080 1081 $('#openrouter_quantizations_text').on('change', function () { 1082 const selectedQuantizations = $(this).val(); 1083 1084 // Not a multiple select? 1085 if (!Array.isArray(selectedQuantizations)) { 1086 return; 1087 } 1088 1089 textgenerationwebui_settings.openrouter_quantizations = selectedQuantizations; 1090 1091 saveSettingsDebounced(); 1092 }); 1093 1094 $('#api_button_textgenerationwebui').on('click', async function (e) { 1095 const keys = [ 1096 { id: 'api_key_mancer', secret: SECRET_KEYS.MANCER }, 1097 { id: 'api_key_vllm', secret: SECRET_KEYS.VLLM }, 1098 { id: 'api_key_aphrodite', secret: SECRET_KEYS.APHRODITE }, 1099 { id: 'api_key_tabby', secret: SECRET_KEYS.TABBY }, 1100 { id: 'api_key_togetherai', secret: SECRET_KEYS.TOGETHERAI }, 1101 { id: 'api_key_ooba', secret: SECRET_KEYS.OOBA }, 1102 { id: 'api_key_infermaticai', secret: SECRET_KEYS.INFERMATICAI }, 1103 { id: 'api_key_dreamgen', secret: SECRET_KEYS.DREAMGEN }, 1104 { id: 'api_key_openrouter-tg', secret: SECRET_KEYS.OPENROUTER }, 1105 { id: 'api_key_koboldcpp', secret: SECRET_KEYS.KOBOLDCPP }, 1106 { id: 'api_key_llamacpp', secret: SECRET_KEYS.LLAMACPP }, 1107 { id: 'api_key_featherless', secret: SECRET_KEYS.FEATHERLESS }, 1108 { id: 'api_key_huggingface', secret: SECRET_KEYS.HUGGINGFACE }, 1109 { id: 'api_key_generic', secret: SECRET_KEYS.GENERIC }, 1110 ]; 1111 1112 for (const key of keys) {
1113 const keyValue = String($(`#${key.id}`).val()).trim(); 1114 if (keyValue.length) { 1115 await writeSecret(key.secret, keyValue); 1116 } 1117 } 1118 1119 validateTextGenUrl(); 1120 startStatusLoading(); 1121 saveSettingsDebounced(); 1122 getStatusTextgen(); 1123 }); 1124} 1125 1126/** 1127 * Hides and shows preset samplers from the left panel. 1128 * @param {string?} apiType API Type selected in API Connections - Currently selected one by default 1129 * @returns void 1130 */ 1131function showSamplerControls(apiType = null) { 1132 $('#textgenerationwebui_api-settings [data-tg-samplers], #textgenerationwebui_api [data-tg-samplers]').each(function (idx, elem) { 1133 const typeSpecificControlled = $(elem).data('tg-type') !== undefined; 1134 1135 if (!typeSpecificControlled) $(this).show(); 1136 }); 1137 1138 showTypeSpecificControls(apiType ?? textgenerationwebui_settings.type); 1139 1140 const prioritizeManualSamplerSelect = isSamplerManualPriorityEnabled(apiType ?? textgenerationwebui_settings.type); 1141 const samplersActivatedManually = getActiveManualApiSamplers(apiType ?? textgenerationwebui_settings.type); 1142 1143 if (!samplersActivatedManually?.length || !prioritizeManualSamplerSelect) return; 1144 1145 $('#textgenerationwebui_api-settings [data-tg-samplers], #textgenerationwebui_api [data-tg-samplers]').each(function () { 1146 const tgSamplers = $(this).attr('data-tg-samplers').split(',').map(x => x.trim()).filter(str => str !== ''); 1147 1148 for (const tgSampler of tgSamplers) { 1149 if (samplersActivatedManually.includes(tgSampler)) { 1150 $(this).show(); 1151 return; 1152 } else { 1153 $(this).hide(); 1154 } 1155 } 1156 }); 1157} 1158 1159function showTypeSpecificControls(apiType) { 1160 $('[data-tg-type]').each(function () { 1161 const mode = String($(this).attr('data-tg-type-mode') ?? '').toLowerCase().trim(); 1162 const tgTypes = $(this).attr('data-tg-type').split(',').map(x => x.trim()); 1163 1164 if (mode === 'except') { 1165 $(this)[tgTypes.includes(apiType) ? 'hide' : 'show'](); 1166 return; 1167 } 1168 1169 for (const tgType of tgTypes) { 1170 if (tgType === apiType || tgType == 'all') { 1171 $(this).show(); 1172 return; 1173 } else { 1174 $(this).hide(); 1175 } 1176 } 1177 }); 1178} 1179 1180/** 1181 * Inserts missing items from the source array into the target array. 1182 * @param {any[]} source - Source array 1183 * @param {any[]} target - Target array 1184 * @returns {void} 1185 */ 1186function insertMissingArrayItems(source, target) { 1187 if (source === target || !Array.isArray(source) || !Array.isArray(target)) { 1188 return; 1189 } 1190 1191 for (const item of source) { 1192 if (!target.includes(item)) { 1193 const index = source.indexOf(item); 1194 target.splice(index, 0, item); 1195 } 1196 } 1197} 1198 1199function setSettingByName(setting, value, trigger) { 1200 if ('extensions' === setting) { 1201 value = value || {}; 1202 textgenerationwebui_settings.extensions = value; 1203 return; 1204 } 1205 1206 if ('json_schema' === setting) { 1207 textgenerationwebui_settings.json_schema = value ?? null; 1208 $('#tabby_json_schema').val(value ? JSON.stringify(textgenerationwebui_settings.json_schema, null, 2) : ''); 1209 return; 1210 } 1211 1212 if (value === null || value === undefined) { 1213 return; 1214 } 1215 1216 if ('sampler_order' === setting) { 1217 value = Array.isArray(value) ? value : KOBOLDCPP_ORDER; 1218 sortKoboldItemsByOrder(value); 1219 textgenerationwebui_settings.sampler_order = value; 1220 return; 1221 } 1222 1223 if ('sampler_priority' === setting) { 1224 value = Array.isArray(value) ? value : OOBA_DEFAULT_ORDER; 1225 insertMissingArrayItems(OOBA_DEFAULT_ORDER, value); 1226 sortOobaItemsByOrder(value); 1227 textgenerationwebui_settings.sampler_priority = value; 1228 return; 1229 } 1230 1231 if ('samplers_priorities' === setting) { 1232 value = Array.isArray(value) ? value : APHRODITE_DEFAULT_ORDER; 1233 insertMissingArrayItems(APHRODITE_DEFAULT_ORDER, value); 1234 sortAphroditeItemsByOrder(value); 1235 textgenerationwebui_settings.samplers_priorities = value; 1236 return; 1237 } 1238 1239 if ('samplers' === setting) { 1240 value = Array.isArray(value) ? value : LLAMACPP_DEFAULT_ORDER; 1241 insertMissingArrayItems(LLAMACPP_DEFAULT_ORDER, value); 1242 sortLlamacppItemsByOrder(value); 1243 textgenerationwebui_settings.samplers = value; 1244 return; 1245 } 1246 1247 if ('logit_bias' === setting) { 1248 textgenerationwebui_settings.logit_bias = Array.isArray(value) ? value : []; 1249 return; 1250 } 1251 1252 const isCheckbox = $(`#${setting}_textgenerationwebui`).attr('type') == 'checkbox'; 1253 const isText = $(`#${setting}_textgenerationwebui`).attr('type') == 'text' || $(`#${setting}_textgenerationwebui`).is('textarea'); 1254 if (isCheckbox) { 1255 const val = Boolean(value); 1256 $(`#${setting}_textgenerationwebui`).prop('checked', val); 1257 1258 if ('send_banned_tokens' === setting) { 1259 $(`#${setting}_textgenerationwebui`).trigger('change'); 1260 } 1261 } else if (isText) { 1262 $(`#${setting}_textgenerationwebui`).val(value); 1263 } else { 1264 const val = parseFloat(value); 1265 $(`#${setting}_textgenerationwebui`).val(val); 1266 $(`#${setting}_counter_textgenerationwebui`).val(val);
1267 if (power_user.enableZenSliders) { 1268 let zenSlider = $(`#${setting}_textgenerationwebui_zenslider`).slider(); 1269 zenSlider.slider('option', 'value', val); 1270 zenSlider.slider('option', 'slide') 1271 .call(zenSlider, null, { 1272 handle: $('.ui-slider-handle', zenSlider), value: val, 1273 }); 1274 } 1275 } 1276 1277 if (trigger) { 1278 $(`#${setting}_textgenerationwebui`).trigger('input'); 1279 } 1280} 1281 1282/** 1283 * Sends a streaming request for textgenerationwebui. 1284 * @param {object} generate_data 1285 * @param {AbortSignal} signal 1286 * @returns {Promise<(function(): AsyncGenerator<{swipes: [], text: string, toolCalls: [], logprobs: {token: string, topLogprobs: Candidate[]}|null}, void, *>)|*>} 1287 * @throws {Error} - If the response status is not OK, or from within the generator 1288 */ 1289export async function generateTextGenWithStreaming(generate_data, signal) { 1290 generate_data.stream = true; 1291 1292 const response = await fetch('/api/backends/text-completions/generate', { 1293 headers: { 1294 ...getRequestHeaders(), 1295 }, 1296 body: JSON.stringify(generate_data), 1297 method: 'POST', 1298 signal: signal, 1299 }); 1300 1301 if (!response.ok) { 1302 tryParseStreamingError(response, await response.text()); 1303 throw new Error(`Got response status ${response.status}`); 1304 } 1305 1306 const eventStream = getEventSourceStream(); 1307 response.body.pipeThrough(eventStream); 1308 const reader = eventStream.readable.getReader(); 1309 1310 return async function* streamData() { 1311 let text = ''; 1312 /** @type {import('./logprobs.js').TokenLogprobs | null} */ 1313 let logprobs = null; 1314 const swipes = []; 1315 const toolCalls = []; 1316 const state = { reasoning: '' }; 1317 while (true) { 1318 const { done, value } = await reader.read(); 1319 if (done) return; 1320 if (value.data === '[DONE]') return; 1321 1322 tryParseStreamingError(response, value.data); 1323 1324 let data = JSON.parse(value.data); 1325 1326 if (data?.choices?.[0]?.index > 0) { 1327 const swipeIndex = data.choices[0].index - 1; 1328 swipes[swipeIndex] = (swipes[swipeIndex] || '') + data.choices[0].text; 1329 } else if (data?.index > 0) { 1330 // llama.cpp streaming swipe 1331 const swipeIndex = data.index - 1; 1332 swipes[swipeIndex] = (swipes[swipeIndex] || '') + data.content; 1333 } else { 1334 const newText = data?.choices?.[0]?.text || data?.content || ''; 1335 text += newText; 1336 logprobs = parseTextgenLogprobs(newText, data.choices?.[0]?.logprobs || data?.completion_probabilities); 1337 state.reasoning += data?.choices?.[0]?.reasoning ?? data?.choices?.[0]?.thinking ?? ''; 1338 } 1339 1340 yield { text, swipes, logprobs, toolCalls, state }; 1341 } 1342 }; 1343} 1344 1345/** 1346 * parseTextgenLogprobs converts a logprobs object returned from a textgen API 1347 * for a single token into a TokenLogprobs object used by the Token 1348 * Probabilities feature. 1349 * @param {string} token - the text of the token that the logprobs are for 1350 * @param {Object} logprobs - logprobs object returned from the API 1351 * @returns {import('./logprobs.js').TokenLogprobs | null} - converted logprobs 1352 */ 1353export function parseTextgenLogprobs(token, logprobs) { 1354 if (!logprobs) { 1355 return null; 1356 } 1357 1358 switch (textgenerationwebui_settings.type) { 1359 case KOBOLDCPP: 1360 case TABBY: 1361 case VLLM: 1362 case APHRODITE: 1363 case MANCER: 1364 case INFERMATICAI: 1365 case OOBA: { 1366 /** @type {Record<string, number>[]} */ 1367 const topLogprobs = logprobs.top_logprobs; 1368 if (!topLogprobs?.length) { 1369 return null; 1370 } 1371 const candidates = Object.entries(topLogprobs[0]); 1372 return { token, topLogprobs: candidates }; 1373 } 1374 case LLAMACPP: { 1375 if (!logprobs?.length) { 1376 return null; 1377 } 1378 1379 // 3 cases: 1380 // 1. Before commit 6c5bc06, "probs" key with "tok_str"/"prob", and probs are [0, 1] so use them directly. 1381 // 2. After commit 6c5bc06 but before commit 89d604f broke logprobs (they all return the first token's logprobs) 1382 // We don't know the llama.cpp version so we can't do much about this. 1383 // 3. After commit 89d604f uses OpenAI-compatible format with "completion_probabilities" and "token"/"logprob" keys. 1384 // Note that it is also the *actual* logprob (negative number), so we need to convert to [0, 1]. 1385 if (logprobs?.[0]?.probs) { 1386 const candidates = logprobs?.[0]?.probs?.map(x => [x.tok_str, x.prob]); 1387 if (!candidates) { 1388 return null; 1389 } 1390 return { token, topLogprobs: candidates }; 1391 } else if (logprobs?.[0].top_logprobs) { 1392 const candidates = logprobs?.[0]?.top_logprobs?.map(x => [x.token, Math.exp(x.logprob)]); 1393 if (!candidates) { 1394 return null; 1395 } 1396 return { token, topLogprobs: candidates }; 1397 } 1398 return null; 1399 } 1400 default: 1401 return null; 1402 } 1403} 1404 1405export function parseTabbyLogprobs(data) { 1406 const text = data?.choices?.[0]?.text; 1407 const offsets = data?.choices?.[0]?.logprobs?.text_offset; 1408 1409 if (!text || !offsets) { 1410 return null; 1411 } 1412 1413 // Convert string offsets list to tokens 1414 const tokens = offsets?.map((offset, index) => { 1415 const nextOffset = offsets[index + 1] || text.length; 1416 return text.substring(offset, nextOffset); 1417 }); 1418 1419 const topLogprobs = data?.choices?.[0]?.logprobs?.top_logprobs?.map(x => ({ top_logprobs: [x] })); 1420 return tokens?.map((token, index) => parseTextgenLogprobs(token, topLogprobs[index])) || null; 1421} 1422 1423/** 1424 * Parses errors in streaming responses and displays them in toastr. 1425 * @param {Response} response - Response from the server. 1426 * @param {string} decoded - Decoded response body. 1427 * @returns {void} Nothing. 1428 * @throws {Error} If the response contains an error message, throws Error with the message. 1429 */ 1430function tryParseStreamingError(response, decoded) { 1431 let data = {}; 1432 1433 try { 1434 data = JSON.parse(decoded); 1435 } catch { 1436 // No JSON. Do nothing. 1437 } 1438 1439 const message = data?.error?.message || data?.error || data?.message || data?.detail; 1440 1441 if (message) { 1442 toastr.error(message, 'Text Completion API'); 1443 throw new Error(message); 1444 } 1445} 1446 1447/** 1448 * Converts a string of comma-separated integers to an array of integers. 1449 * @param {string} string Input string 1450 * @returns {number[]} Array of integers 1451 */ 1452function toIntArray(string) { 1453 if (!string) { 1454 return []; 1455 } 1456 1457 return string.split(',').map(x => parseInt(x)).filter(x => !isNaN(x)); 1458} 1459 1460/** 1461 * Gets the text generation model specified by the given text completion settings 1462 * @param {TextCompletionSettings} settings Text completion settings to use 1463 * @returns {string} model name 1464 */ 1465export function getTextGenModel(settings = null) { 1466 settings = settings ?? textgenerationwebui_settings; 1467 switch (settings.type) { 1468 case OOBA: 1469 if (settings.custom_model) { 1470 return settings.custom_model; 1471 } 1472 break; 1473 case GENERIC: 1474 if (settings.generic_model) { 1475 return settings.generic_model; 1476 } 1477 break; 1478 case MANCER: 1479 return settings.mancer_model; 1480 case TOGETHERAI: 1481 return settings.togetherai_model; 1482 case INFERMATICAI: 1483 return settings.infermaticai_model; 1484 case DREAMGEN: 1485 return settings.dreamgen_model; 1486 case OPENROUTER: 1487 return settings.openrouter_model; 1488 case VLLM: 1489 return settings.vllm_model; 1490 case APHRODITE: 1491 return settings.aphrodite_model; 1492 case OLLAMA: 1493 if (!settings.ollama_model) { 1494 toastr.error(t`No Ollama model selected.`, 'Text Completion API'); 1495 throw new Error('No Ollama model selected'); 1496 } 1497 return settings.ollama_model; 1498 case FEATHERLESS: 1499 return settings.featherless_model; 1500 case HUGGINGFACE: 1501 return 'tgi'; 1502 case TABBY: 1503 if (settings.tabby_model) { 1504 return settings.tabby_model; 1505 } 1506 break; 1507 case LLAMACPP: 1508 if (settings.llamacpp_model) { 1509 return settings.llamacpp_model; 1510 } 1511 break; 1512 default: 1513 return undefined; 1514 } 1515 1516 return undefined; 1517} 1518 1519export function isJsonSchemaSupported() { 1520 return [TABBY, LLAMACPP].includes(textgenerationwebui_settings.type) && main_api === 'textgenerationwebui'; 1521} 1522
1523/** 1524 * Returns whether dynamic temperature is supported by the given text completion settings 1525 * @param {TextCompletionSettings} settings Text completion settings to use 1526 * @returns {boolean} Whether dynamic temperature supported 1527 */ 1528function isDynamicTemperatureSupported(settings = null) { 1529 settings = settings ?? textgenerationwebui_settings; 1530 return settings.dynatemp && DYNATEMP_BLOCK?.dataset?.tgType?.includes(settings.type); 1531} 1532 1533/** 1534 * Gets the number of logprobs to request based on the selected type. 1535 * @param {string} type If it's set, ignores active type 1536 * @returns {number} Number of logprobs to request 1537 */ 1538export function getLogprobsNumber(type = null) { 1539 const selectedType = type ?? textgenerationwebui_settings.type; 1540 if (selectedType === VLLM || selectedType === INFERMATICAI) { 1541 return 5; 1542 } 1543 1544 return 10; 1545} 1546 1547/** 1548 * Replaces {{macro}} in a comma-separated or serialized JSON array string. 1549 * @param {string} str Input string 1550 * @returns {string} Output string 1551 */ 1552export function replaceMacrosInList(str) { 1553 if (!str || typeof str !== 'string') { 1554 return str; 1555 } 1556 1557 try { 1558 const array = JSON.parse(str); 1559 if (!Array.isArray(array)) { 1560 throw new Error('Not an array'); 1561 } 1562 for (let i = 0; i < array.length; i++) { 1563 array[i] = substituteParams(array[i]); 1564 } 1565 return JSON.stringify(array); 1566 } catch { 1567 const array = str.split(','); 1568 for (let i = 0; i < array.length; i++) { 1569 array[i] = substituteParams(array[i]); 1570 } 1571 return array.join(','); 1572 } 1573} 1574 1575/** 1576 * Build the generation parameter object for an text completion request 1577 * @param {TextCompletionSettings} settings Text completion settings to use 1578 * @param {string} model Model to use 1579 * @param {string} finalPrompt The final prompt to send 1580 * @param {number} maxTokens Max allowed generation tokens 1581 * @param {boolean} isImpersonate Whether this is for an impersonation 1582 * @param {boolean} isContinue Whether this is for a continue 1583 * @param {object} cfgValues Additional parameters (guidanceScale, negativePrompt) 1584 * @param {string} type Request type (impersonate, quiet, continue, etc) 1585 * @returns {object} Final generation parameters object appropriate for the text completion source 1586 */ 1587export function createTextGenGenerationData(settings, model, finalPrompt = null, maxTokens = null, isImpersonate = false, isContinue = false, cfgValues = null, type = 'quiet') { 1588 settings = settings ?? textgenerationwebui_settings; 1589 model = model ?? getTextGenModel(settings); 1590 1591 const canMultiSwipe = !isContinue && !isImpersonate && type !== 'quiet'; 1592 const dynatemp = isDynamicTemperatureSupported(settings); 1593 const { banned_tokens, banned_strings } = getCustomTokenBans(settings); 1594 const jsonSchema = isObject(settings.json_schema) 1595 ? settings.json_schema_allow_empty 1596 ? settings.json_schema 1597 : Object.keys(settings.json_schema).length > 0 ? settings.json_schema : undefined 1598 : undefined; 1599 1600 let params = { 1601 'prompt': finalPrompt, 1602 'model': model, 1603 'max_new_tokens': maxTokens, 1604 'max_tokens': maxTokens, 1605 'logprobs': power_user.request_token_probabilities ? getLogprobsNumber(settings.type) : undefined, 1606 'temperature': dynatemp ? (settings.min_temp + settings.max_temp) / 2 : settings.temp, 1607 'top_p': settings.top_p, 1608 'typical_p': settings.typical_p, 1609 'typical': settings.typical_p, 1610 'sampler_seed': settings.seed >= 0 ? settings.seed : undefined, 1611 'min_p': settings.min_p, 1612 'repetition_penalty': settings.rep_pen, 1613 'frequency_penalty': settings.freq_pen, 1614 'presence_penalty': settings.presence_pen, 1615 'top_k': settings.top_k, 1616 'skew': settings.skew, 1617 'min_length': settings.type === OOBA ? settings.min_length : undefined, 1618 'minimum_message_content_tokens': settings.type === DREAMGEN ? settings.min_length : undefined, 1619 'min_tokens': settings.min_length, 1620 'num_beams': settings.type === OOBA ? settings.num_beams : undefined, 1621 'length_penalty': settings.type === OOBA ? settings.length_penalty : undefined, 1622 'early_stopping': settings.type === OOBA ? settings.early_stopping : undefined, 1623 'add_bos_token': settings.add_bos_token, 1624 'dynamic_temperature': dynatemp ? true : undefined, 1625 'dynatemp_low': dynatemp ? settings.min_temp : undefined, 1626 'dynatemp_high': dynatemp ? settings.max_temp : undefined, 1627 'dynatemp_range': dynatemp ? (settings.max_temp - settings.min_temp) / 2 : undefined, 1628 'dynatemp_exponent': dynatemp ? settings.dynatemp_exponent : undefined, 1629 'smoothing_factor': settings.smoothing_factor, 1630 'smoothing_curve': settings.smoothing_curve, 1631 'dry_allowed_length': settings.dry_allowed_length, 1632 'dry_multiplier': settings.dry_multiplier, 1633 'dry_base': settings.dry_base, 1634 'dry_sequence_breakers': replaceMacrosInList(settings.dry_sequence_breakers), 1635 'dry_penalty_last_n': settings.dry_penalty_last_n, 1636 'max_tokens_second': settings.max_tokens_second,
1637 'sampler_priority': settings.type === OOBA ? settings.sampler_priority : undefined, 1638 'samplers': settings.type === LLAMACPP ? settings.samplers : undefined, 1639 'stopping_strings': getStoppingStrings(isImpersonate, isContinue), 1640 'stop': getStoppingStrings(isImpersonate, isContinue), 1641 'truncation_length': max_context, 1642 'ban_eos_token': settings.ban_eos_token, 1643 'skip_special_tokens': settings.skip_special_tokens, 1644 'include_reasoning': settings.include_reasoning, 1645 'top_a': settings.top_a, 1646 'tfs': settings.tfs, 1647 'epsilon_cutoff': [OOBA, MANCER].includes(settings.type) ? settings.epsilon_cutoff : undefined, 1648 'eta_cutoff': [OOBA, MANCER].includes(settings.type) ? settings.eta_cutoff : undefined, 1649 'mirostat_mode': settings.mirostat_mode, 1650 'mirostat_tau': settings.mirostat_tau, 1651 'mirostat_eta': settings.mirostat_eta, 1652 'custom_token_bans': [APHRODITE, MANCER].includes(settings.type) ? 1653 toIntArray(banned_tokens) : 1654 banned_tokens, 1655 'banned_strings': banned_strings, 1656 'api_type': settings.type, 1657 'api_server': getTextGenServer(settings.type), 1658 'sampler_order': settings.type === textgen_types.KOBOLDCPP ? settings.sampler_order : undefined, 1659 'xtc_threshold': settings.xtc_threshold, 1660 'xtc_probability': settings.xtc_probability, 1661 'nsigma': settings.nsigma, 1662 'top_n_sigma': settings.nsigma, 1663 'min_keep': settings.min_keep, 1664 'adaptive_target': settings.adaptive_target, 1665 'adaptive_decay': settings.adaptive_decay, 1666 parseSequenceBreakers: function () { 1667 try { 1668 return JSON.parse(this.dry_sequence_breakers); 1669 } catch { 1670 if (typeof this.dry_sequence_breakers === 'string') { 1671 return this.dry_sequence_breakers.split(','); 1672 } 1673 return undefined; 1674 } 1675 }, 1676 }; 1677 const nonAphroditeParams = { 1678 'rep_pen': settings.rep_pen, 1679 'rep_pen_range': settings.rep_pen_range, 1680 'repetition_decay': settings.type === TABBY ? settings.rep_pen_decay : undefined, 1681 'repetition_penalty_range': settings.rep_pen_range, 1682 'encoder_repetition_penalty': settings.type === OOBA ? settings.encoder_rep_pen : undefined, 1683 'no_repeat_ngram_size': settings.type === OOBA ? settings.no_repeat_ngram_size : undefined, 1684 'penalty_alpha': settings.type === OOBA ? settings.penalty_alpha : undefined, 1685 'temperature_last': (settings.type === OOBA || settings.type === APHRODITE || settings.type == TABBY) ? settings.temperature_last : undefined, 1686 'speculative_ngram': settings.type === TABBY ? settings.speculative_ngram : undefined, 1687 'do_sample': settings.type === OOBA ? settings.do_sample : undefined, 1688 'seed': settings.seed >= 0 ? settings.seed : undefined, 1689 'guidance_scale': cfgValues?.guidanceScale?.value ?? settings.guidance_scale ?? 1, 1690 'negative_prompt': cfgValues?.negativePrompt ?? substituteParams(settings.negative_prompt) ?? '', 1691 'grammar_string': settings.grammar_string || undefined, 1692 'json_schema': [TABBY, LLAMACPP].includes(settings.type) ? jsonSchema : undefined, 1693 // llama.cpp aliases. In case someone wants to use LM Studio as Text Completion API 1694 'repeat_penalty': settings.rep_pen, 1695 'repeat_last_n': settings.rep_pen_range, 1696 'n_predict': maxTokens, 1697 'num_predict': maxTokens, 1698 'num_ctx': max_context, 1699 'mirostat': settings.mirostat_mode, 1700 'ignore_eos': settings.ban_eos_token, 1701 'n_probs': power_user.request_token_probabilities ? 10 : undefined, 1702 'rep_pen_slope': settings.rep_pen_slope, 1703 }; 1704 const vllmParams = { 1705 'n': canMultiSwipe ? settings.n : 1, 1706 'ignore_eos': settings.ignore_eos_token, 1707 'spaces_between_special_tokens': settings.spaces_between_special_tokens, 1708 'seed': settings.seed >= 0 ? settings.seed : undefined, 1709 }; 1710 const aphroditeParams = { 1711 'n': canMultiSwipe ? settings.n : 1, 1712 'frequency_penalty': settings.freq_pen, 1713 'presence_penalty': settings.presence_pen, 1714 'repetition_penalty': settings.rep_pen, 1715 'seed': settings.seed >= 0 ? settings.seed : undefined, 1716 'stop': getStoppingStrings(isImpersonate, isContinue), 1717 'temperature': dynatemp ? (settings.min_temp + settings.max_temp) / 2 : settings.temp, 1718 'temperature_last': settings.temperature_last, 1719 'top_p': settings.top_p, 1720 'top_k': settings.top_k, 1721 'top_a': settings.top_a, 1722 'min_p': settings.min_p, 1723 'tfs': settings.tfs, 1724 'eta_cutoff': settings.eta_cutoff, 1725 'epsilon_cutoff': settings.epsilon_cutoff, 1726 'typical_p': settings.typical_p, 1727 'smoothing_factor': settings.smoothing_factor, 1728 'smoothing_curve': settings.smoothing_curve, 1729 'ignore_eos': settings.ignore_eos_token, 1730 'min_tokens': settings.min_length, 1731 'skip_special_tokens': settings.skip_special_tokens, 1732 'spaces_between_special_tokens': settings.spaces_between_special_tokens, 1733 'guided_grammar': settings.grammar_string || undefined, 1734 'guided_json': jsonSchema || undefined, 1735 'early_stopping': false, // hacks 1736 'include_stop_str_in_output': false, 1737 'dynatemp_min': dynatemp ? settings.min_temp : undefined, 1738 'dynatemp_max': dynatemp ? settings.max_temp : undefined, 1739 'dynatemp_exponent': dynatemp ? settings.dynatemp_exponent : undefined, 1740 'xtc_threshold': settings.xtc_threshold, 1741 'xtc_probability': settings.xtc_probability, 1742 'nsigma': settings.nsigma, 1743 'custom_token_bans': toIntArray(banned_tokens), 1744 'no_repeat_ngram_size': settings.no_repeat_ngram_size,
1745 'sampler_priority': settings.type === APHRODITE && !arraysEqual( 1746 settings.samplers_priorities, 1747 APHRODITE_DEFAULT_ORDER) 1748 ? settings.samplers_priorities 1749 : undefined, 1750 }; 1751 1752 if (settings.type === OPENROUTER) { 1753 params.provider = settings.openrouter_providers; 1754 params.quantizations = settings.openrouter_quantizations; 1755 params.allow_fallbacks = settings.openrouter_allow_fallbacks; 1756 } 1757 1758 if (settings.type === KOBOLDCPP) { 1759 params.grammar = settings.grammar_string || undefined; 1760 params.grammar_retain_state = (settings.grammar_string && !!isContinue) ? true : undefined; 1761 params.trim_stop = true; 1762 params.dry_sequence_breakers = params.parseSequenceBreakers(); 1763 } 1764 1765 if (settings.type === HUGGINGFACE) { 1766 params.top_p = Math.min(Math.max(Number(params.top_p), 0.0), 0.999); 1767 params.stop = Array.isArray(params.stop) ? params.stop.slice(0, 4) : []; 1768 nonAphroditeParams.seed = settings.seed >= 0 ? settings.seed : Math.floor(Math.random() * Math.pow(2, 32)); 1769 } 1770 1771 if (settings.type === MANCER) { 1772 params.n = canMultiSwipe ? settings.n : 1; 1773 params.epsilon_cutoff /= 1000; 1774 params.eta_cutoff /= 1000; 1775 params.dynatemp_mode = params.dynamic_temperature ? 1 : 0; 1776 params.dynatemp_min = params.dynatemp_low; 1777 params.dynatemp_max = params.dynatemp_high; 1778 delete params.dynatemp_low; 1779 delete params.dynatemp_high; 1780 params.dry_sequence_breakers = params.parseSequenceBreakers(); 1781 } 1782 1783 if (settings.type === TABBY || settings.type === LLAMACPP) { 1784 params.n = canMultiSwipe ? settings.n : 1; 1785 } 1786 1787 switch (settings.type) { 1788 case VLLM: 1789 case INFERMATICAI: 1790 params = Object.assign(params, vllmParams); 1791 break; 1792 1793 case APHRODITE: 1794 // set params to aphroditeParams 1795 params = Object.assign(params, aphroditeParams); 1796 break; 1797 1798 default: 1799 params = Object.assign(params, nonAphroditeParams); 1800 break; 1801 } 1802 1803 if (Array.isArray(settings.logit_bias) && settings.logit_bias.length) { 1804 const logitBias = BIAS_CACHE.get(BIAS_KEY) || calculateLogitBias(settings); 1805 BIAS_CACHE.set(BIAS_KEY, logitBias); 1806 params.logit_bias = logitBias; 1807 } 1808 1809 if (settings.type === LLAMACPP || settings.type === OLLAMA) { 1810 // Convert bias and token bans to array of arrays 1811 const logitBiasArray = (params.logit_bias && typeof params.logit_bias === 'object' && Object.keys(params.logit_bias).length > 0) 1812 ? Object.entries(params.logit_bias).map(([key, value]) => [Number(key), value]) 1813 : []; 1814 const tokenBans = toIntArray(banned_tokens); 1815 logitBiasArray.push(...tokenBans.map(x => [Number(x), false])); 1816 const sequenceBreakers = params.parseSequenceBreakers(); 1817 const llamaCppParams = { 1818 'logit_bias': logitBiasArray, 1819 // Conflicts with ooba's grammar_string 1820 'grammar': settings.grammar_string, 1821 'cache_prompt': true, 1822 'dry_sequence_breakers': sequenceBreakers, 1823 }; 1824 params = Object.assign(params, llamaCppParams); 1825 if (!Array.isArray(sequenceBreakers) || sequenceBreakers.length === 0) { 1826 delete params.dry_sequence_breakers; 1827 } 1828 } 1829 1830 // Grammar conflicts with with json_schema 1831 if ([LLAMACPP, APHRODITE].includes(settings.type)) { 1832 if (jsonSchema) { 1833 delete params.grammar_string; 1834 delete params.grammar; 1835 delete params.guided_grammar; 1836 } else { 1837 delete params.json_schema; 1838 delete params.guided_json; 1839 } 1840 } 1841 return params; 1842} 1843 1844export async function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate, isContinue, cfgValues, type) { 1845 const model = getTextGenModel(textgenerationwebui_settings); 1846 const params = createTextGenGenerationData(textgenerationwebui_settings, model, finalPrompt, maxTokens, isImpersonate, isContinue, cfgValues, type); 1847 await eventSource.emit(event_types.TEXT_COMPLETION_SETTINGS_READY, params); 1848 return params; 1849}
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.