1788 lines
62 KiB
Rust
1788 lines
62 KiB
Rust
use super::model::{ModelData, ProviderModels};
|
|
use super::oauth::OAuthConfig;
|
|
use std::cmp::Ordering;
|
|
|
|
use crate::utils::fetch;
|
|
|
|
use anyhow::{Result, bail};
|
|
use fancy_regex::Regex;
|
|
use log::warn;
|
|
use serde::{Deserialize, Serialize};
|
|
use serde_json::Value;
|
|
use std::collections::{HashMap, HashSet};
|
|
use std::fs;
|
|
use std::path::Path;
|
|
use std::sync::{Arc, LazyLock, Mutex};
|
|
|
|
pub const QUIRKS_YAML: &str = include_str!("../../quirks.yaml");
|
|
|
|
pub static ALL_QUIRKS: LazyLock<Vec<ProviderQuirks>> =
|
|
LazyLock::new(|| serde_yaml::from_str(QUIRKS_YAML).expect("quirks.yaml must be valid"));
|
|
|
|
const MODELSDEV_API_URL: &str = "https://models.dev/api.json";
|
|
const OPENROUTER_API_URL: &str = "https://openrouter.ai/api/v1/models";
|
|
|
|
struct SourceIds {
|
|
provider: &'static str,
|
|
modelsdev: Option<&'static str>,
|
|
openrouter: Option<&'static str>,
|
|
}
|
|
|
|
const PROVIDER_SOURCES: [SourceIds; 24] = [
|
|
SourceIds {
|
|
provider: "openai",
|
|
modelsdev: Some("openai"),
|
|
openrouter: Some("openai"),
|
|
},
|
|
SourceIds {
|
|
provider: "gemini",
|
|
modelsdev: Some("google"),
|
|
openrouter: Some("google"),
|
|
},
|
|
SourceIds {
|
|
provider: "claude",
|
|
modelsdev: Some("anthropic"),
|
|
openrouter: Some("anthropic"),
|
|
},
|
|
SourceIds {
|
|
provider: "mistral",
|
|
modelsdev: Some("mistral"),
|
|
openrouter: Some("mistralai"),
|
|
},
|
|
SourceIds {
|
|
provider: "ai21",
|
|
modelsdev: None,
|
|
openrouter: None,
|
|
},
|
|
SourceIds {
|
|
provider: "cohere",
|
|
modelsdev: Some("cohere"),
|
|
openrouter: Some("cohere"),
|
|
},
|
|
SourceIds {
|
|
provider: "xai",
|
|
modelsdev: Some("xai"),
|
|
openrouter: Some("x-ai"),
|
|
},
|
|
SourceIds {
|
|
provider: "perplexity",
|
|
modelsdev: Some("perplexity"),
|
|
openrouter: Some("perplexity"),
|
|
},
|
|
SourceIds {
|
|
provider: "groq",
|
|
modelsdev: Some("groq"),
|
|
openrouter: Some("groq"),
|
|
},
|
|
SourceIds {
|
|
provider: "vertexai",
|
|
modelsdev: Some("google-vertex"),
|
|
openrouter: Some("google"),
|
|
},
|
|
SourceIds {
|
|
provider: "bedrock",
|
|
modelsdev: Some("amazon-bedrock"),
|
|
openrouter: Some("amazon"),
|
|
},
|
|
SourceIds {
|
|
provider: "cloudflare",
|
|
modelsdev: Some("cloudflare-workers-ai"),
|
|
openrouter: None,
|
|
},
|
|
SourceIds {
|
|
provider: "ernie",
|
|
modelsdev: None,
|
|
openrouter: None,
|
|
},
|
|
SourceIds {
|
|
provider: "qianwen",
|
|
modelsdev: Some("alibaba"),
|
|
openrouter: Some("qwen"),
|
|
},
|
|
SourceIds {
|
|
provider: "hunyuan",
|
|
modelsdev: None,
|
|
openrouter: None,
|
|
},
|
|
SourceIds {
|
|
provider: "moonshot",
|
|
modelsdev: Some("moonshotai"),
|
|
openrouter: Some("moonshotai"),
|
|
},
|
|
SourceIds {
|
|
provider: "deepseek",
|
|
modelsdev: Some("deepseek"),
|
|
openrouter: Some("deepseek"),
|
|
},
|
|
SourceIds {
|
|
provider: "zhipuai",
|
|
modelsdev: Some("zhipuai"),
|
|
openrouter: Some("zhipuai"),
|
|
},
|
|
SourceIds {
|
|
provider: "minimax",
|
|
modelsdev: Some("minimax"),
|
|
openrouter: Some("minimax"),
|
|
},
|
|
SourceIds {
|
|
provider: "openrouter",
|
|
modelsdev: None,
|
|
openrouter: None,
|
|
},
|
|
SourceIds {
|
|
provider: "github",
|
|
modelsdev: None,
|
|
openrouter: None,
|
|
},
|
|
SourceIds {
|
|
provider: "deepinfra",
|
|
modelsdev: Some("deepinfra"),
|
|
openrouter: None,
|
|
},
|
|
SourceIds {
|
|
provider: "jina",
|
|
modelsdev: None,
|
|
openrouter: None,
|
|
},
|
|
SourceIds {
|
|
provider: "voyageai",
|
|
modelsdev: None,
|
|
openrouter: None,
|
|
},
|
|
];
|
|
|
|
pub struct FetchedSources {
|
|
pub modelsdev: Option<String>,
|
|
pub openrouter: Option<String>,
|
|
}
|
|
|
|
pub async fn fetch_sources() -> FetchedSources {
|
|
let (modelsdev, openrouter) = tokio::join!(fetch(MODELSDEV_API_URL), fetch(OPENROUTER_API_URL));
|
|
FetchedSources {
|
|
modelsdev: modelsdev
|
|
.inspect_err(|err| warn!("Failed to fetch {MODELSDEV_API_URL}: {err}"))
|
|
.ok(),
|
|
openrouter: openrouter
|
|
.inspect_err(|err| warn!("Failed to fetch {OPENROUTER_API_URL}: {err}"))
|
|
.ok(),
|
|
}
|
|
}
|
|
|
|
struct SourcedModel {
|
|
data: ModelData,
|
|
release_date: Option<String>,
|
|
}
|
|
|
|
fn clamp_input_tokens(value: u64) -> Option<usize> {
|
|
(1..=50_000_000).contains(&value).then_some(value as usize)
|
|
}
|
|
|
|
fn clamp_output_tokens(value: i64) -> Option<isize> {
|
|
(1..=10_000_000).contains(&value).then_some(value as isize)
|
|
}
|
|
|
|
fn clamp_price(value: f64) -> Option<f64> {
|
|
(0.0..=10_000.0).contains(&value).then_some(value)
|
|
}
|
|
|
|
fn modelsdev_provider_models(api: &Value, provider_id: &str) -> Vec<SourcedModel> {
|
|
let Some(models) = api[provider_id]["models"].as_object() else {
|
|
return Vec::new();
|
|
};
|
|
models
|
|
.iter()
|
|
.map(|(name, entry)| {
|
|
let mut data = ModelData::new(name);
|
|
data.max_input_tokens = entry["limit"]["context"]
|
|
.as_u64()
|
|
.and_then(clamp_input_tokens);
|
|
data.max_output_tokens = entry["limit"]["output"]
|
|
.as_i64()
|
|
.and_then(clamp_output_tokens);
|
|
data.input_price = entry["cost"]["input"].as_f64().and_then(clamp_price);
|
|
data.output_price = entry["cost"]["output"].as_f64().and_then(clamp_price);
|
|
data.cache_read_price = entry["cost"]["cache_read"].as_f64().and_then(clamp_price);
|
|
data.cache_write_price = entry["cost"]["cache_write"].as_f64().and_then(clamp_price);
|
|
data.supports_vision = entry["modalities"]["input"]
|
|
.as_array()
|
|
.is_some_and(|modes| modes.iter().any(|mode| mode.as_str() == Some("image")));
|
|
data.supports_function_calling = entry["tool_call"].as_bool().unwrap_or(false);
|
|
data.reasoning_levels = entry["reasoning_options"]
|
|
.as_array()
|
|
.and_then(|options| {
|
|
options
|
|
.iter()
|
|
.find(|option| option["type"].as_str() == Some("effort"))
|
|
})
|
|
.and_then(|option| option["values"].as_array())
|
|
.map(|values| {
|
|
values
|
|
.iter()
|
|
.filter_map(|value| value.as_str().map(str::to_string))
|
|
.collect()
|
|
})
|
|
.unwrap_or_default();
|
|
let release_date = entry["release_date"].as_str().map(str::to_string);
|
|
SourcedModel { data, release_date }
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
fn openrouter_price(value: &Value) -> Option<f64> {
|
|
let per_token: f64 = value.as_str()?.parse().ok()?;
|
|
clamp_price((per_token * 1_000_000.0 * 10_000.0).round() / 10_000.0)
|
|
}
|
|
|
|
fn openrouter_model_data(name: &str, entry: &Value) -> ModelData {
|
|
let mut data = ModelData::new(name);
|
|
data.max_input_tokens = entry["context_length"]
|
|
.as_u64()
|
|
.and_then(clamp_input_tokens);
|
|
data.max_output_tokens = entry["top_provider"]["max_completion_tokens"]
|
|
.as_i64()
|
|
.and_then(clamp_output_tokens);
|
|
data.input_price = openrouter_price(&entry["pricing"]["prompt"]);
|
|
data.output_price = openrouter_price(&entry["pricing"]["completion"]);
|
|
data.cache_read_price = openrouter_price(&entry["pricing"]["input_cache_read"]);
|
|
data.cache_write_price = openrouter_price(&entry["pricing"]["input_cache_write"]);
|
|
data.supports_vision = entry["architecture"]["modality"]
|
|
.as_str()
|
|
.is_some_and(|modality| modality.contains("image"));
|
|
data.supports_function_calling = entry["supported_parameters"]
|
|
.as_array()
|
|
.is_some_and(|params| params.iter().any(|param| param.as_str() == Some("tools")));
|
|
data
|
|
}
|
|
|
|
fn openrouter_index(payload: &Value) -> HashMap<&str, &Value> {
|
|
payload["data"]
|
|
.as_array()
|
|
.map(|entries| {
|
|
entries
|
|
.iter()
|
|
.filter_map(|entry| Some((entry["id"].as_str()?, entry)))
|
|
.collect()
|
|
})
|
|
.unwrap_or_default()
|
|
}
|
|
|
|
fn openrouter_native_models(payload: &Value) -> Vec<SourcedModel> {
|
|
payload["data"]
|
|
.as_array()
|
|
.map(|entries| {
|
|
entries
|
|
.iter()
|
|
.filter_map(|entry| {
|
|
let id = entry["id"].as_str()?;
|
|
Some(SourcedModel {
|
|
data: openrouter_model_data(id, entry),
|
|
release_date: None,
|
|
})
|
|
})
|
|
.collect()
|
|
})
|
|
.unwrap_or_default()
|
|
}
|
|
|
|
fn fill_from_openrouter(data: &mut ModelData, entry: &Value) {
|
|
let filler = openrouter_model_data(&data.name, entry);
|
|
if data.max_input_tokens.is_none() {
|
|
data.max_input_tokens = filler.max_input_tokens;
|
|
}
|
|
if data.max_output_tokens.is_none() {
|
|
data.max_output_tokens = filler.max_output_tokens;
|
|
}
|
|
if data.input_price.is_none() {
|
|
data.input_price = filler.input_price;
|
|
}
|
|
if data.output_price.is_none() {
|
|
data.output_price = filler.output_price;
|
|
}
|
|
if data.cache_read_price.is_none() {
|
|
data.cache_read_price = filler.cache_read_price;
|
|
}
|
|
if data.cache_write_price.is_none() {
|
|
data.cache_write_price = filler.cache_write_price;
|
|
}
|
|
if !data.supports_vision {
|
|
data.supports_vision = filler.supports_vision;
|
|
}
|
|
if !data.supports_function_calling {
|
|
data.supports_function_calling = filler.supports_function_calling;
|
|
}
|
|
}
|
|
|
|
/// Outcome of a catalog merge. `degraded` lists API-mapped providers that fell
|
|
/// back to their hand-owned quirks models because API data was unavailable or
|
|
/// empty; `failed` lists providers that emitted nothing at all.
|
|
pub struct CatalogBuild {
|
|
pub providers: Vec<ProviderModels>,
|
|
pub degraded: Vec<String>,
|
|
pub failed: Vec<String>,
|
|
}
|
|
|
|
/// Merges API-sourced model data with hand-owned quirks into the full
|
|
/// provider catalog, in the fixed provider order of [`PROVIDER_SOURCES`].
|
|
///
|
|
/// Per provider: API models (models.dev primary, OpenRouter gap-fill) are
|
|
/// curated by `exclude` globs and `rules`, hand-owned `models:` entries are
|
|
/// merged in field-wise (hand-owned wins, inheriting the API model's release
|
|
/// date), and all models are sorted newest-first by release date
|
|
/// (name-ascending tiebreak). Undated models sort last: hand-owned ones in
|
|
/// quirks.yaml order (preserving curation for quirks-owned providers), then
|
|
/// API ones name-ascending. `variants` clones are inserted immediately after
|
|
/// their base model. Providers without usable data are routed to `degraded`
|
|
/// or `failed` instead of erroring; a provider never emits zero models.
|
|
pub fn build_catalog(sources: &FetchedSources, quirks: &[ProviderQuirks]) -> CatalogBuild {
|
|
let modelsdev: Option<Value> = sources
|
|
.modelsdev
|
|
.as_deref()
|
|
.and_then(|payload| serde_json::from_str(payload).ok());
|
|
let openrouter: Option<Value> = sources
|
|
.openrouter
|
|
.as_deref()
|
|
.and_then(|payload| serde_json::from_str(payload).ok());
|
|
let or_index = openrouter
|
|
.as_ref()
|
|
.map(openrouter_index)
|
|
.unwrap_or_default();
|
|
|
|
let mut providers = Vec::new();
|
|
let mut degraded = Vec::new();
|
|
let mut failed = Vec::new();
|
|
|
|
for source in &PROVIDER_SOURCES {
|
|
let provider_quirks = quirks_for(quirks, source.provider);
|
|
let api_mapped = source.provider == "openrouter" || source.modelsdev.is_some();
|
|
|
|
let mut api_models: Vec<SourcedModel> = if source.provider == "openrouter" {
|
|
openrouter
|
|
.as_ref()
|
|
.map(openrouter_native_models)
|
|
.unwrap_or_default()
|
|
} else if let (Some(id), Some(api)) = (source.modelsdev, modelsdev.as_ref()) {
|
|
let mut models = modelsdev_provider_models(api, id);
|
|
if let Some(prefix) = source.openrouter {
|
|
for model in &mut models {
|
|
if let Some(entry) =
|
|
or_index.get(format!("{prefix}/{}", model.data.name).as_str())
|
|
{
|
|
fill_from_openrouter(&mut model.data, entry);
|
|
}
|
|
}
|
|
}
|
|
models
|
|
} else {
|
|
Vec::new()
|
|
};
|
|
|
|
if let Some(provider_quirks) = provider_quirks {
|
|
api_models.retain(|model| {
|
|
!provider_quirks
|
|
.exclude
|
|
.iter()
|
|
.any(|pattern| glob_matches(pattern, &model.data.name))
|
|
});
|
|
for model in &mut api_models {
|
|
apply_rules(provider_quirks, &mut model.data);
|
|
}
|
|
}
|
|
|
|
let hand_owned = provider_quirks
|
|
.map(|provider_quirks| provider_quirks.models.clone())
|
|
.unwrap_or_default();
|
|
|
|
if api_models.is_empty() {
|
|
if hand_owned.is_empty() {
|
|
failed.push(source.provider.to_string());
|
|
continue;
|
|
}
|
|
if api_mapped {
|
|
degraded.push(source.provider.to_string());
|
|
}
|
|
}
|
|
|
|
let mut merged_hand = Vec::with_capacity(hand_owned.len());
|
|
for hand in hand_owned {
|
|
match api_models
|
|
.iter()
|
|
.position(|model| model.data.name == hand.name)
|
|
{
|
|
Some(position) => {
|
|
let api = api_models.remove(position);
|
|
merged_hand.push(SourcedModel {
|
|
data: overlay_hand_owned(hand, api.data),
|
|
release_date: api.release_date,
|
|
});
|
|
}
|
|
None => merged_hand.push(SourcedModel {
|
|
data: hand,
|
|
release_date: None,
|
|
}),
|
|
}
|
|
}
|
|
|
|
api_models.sort_by(|a, b| match (&a.release_date, &b.release_date) {
|
|
(Some(a_date), Some(b_date)) => b_date
|
|
.cmp(a_date)
|
|
.then_with(|| a.data.name.cmp(&b.data.name)),
|
|
(Some(_), None) => Ordering::Less,
|
|
(None, Some(_)) => Ordering::Greater,
|
|
(None, None) => a.data.name.cmp(&b.data.name),
|
|
});
|
|
|
|
let mut all_models = merged_hand;
|
|
all_models.extend(api_models);
|
|
all_models.sort_by(|a, b| match (&a.release_date, &b.release_date) {
|
|
(Some(a_date), Some(b_date)) => b_date
|
|
.cmp(a_date)
|
|
.then_with(|| a.data.name.cmp(&b.data.name)),
|
|
(Some(_), None) => Ordering::Less,
|
|
(None, Some(_)) => Ordering::Greater,
|
|
(None, None) => Ordering::Equal,
|
|
});
|
|
let bases = all_models.into_iter().map(|model| model.data);
|
|
|
|
let variants = provider_quirks
|
|
.map(|provider_quirks| provider_quirks.variants.as_slice())
|
|
.unwrap_or_default();
|
|
let mut models = Vec::new();
|
|
for base in bases {
|
|
let mut clones = Vec::new();
|
|
for variant in variants {
|
|
if !glob_matches(&variant.r#match, &base.name) {
|
|
continue;
|
|
}
|
|
let mut clone = base.clone();
|
|
clone.name = format!("{}{}", base.name, variant.suffix);
|
|
// Without an explicit real_name the clone would send its
|
|
// suffixed name on the wire; default to the base's wire name.
|
|
if clone.real_name.is_none() {
|
|
clone.real_name = Some(base.name.clone());
|
|
}
|
|
apply_variant(variant, &mut clone);
|
|
clones.push(clone);
|
|
}
|
|
models.push(base);
|
|
models.append(&mut clones);
|
|
}
|
|
|
|
providers.push(ProviderModels {
|
|
provider: source.provider.to_string(),
|
|
oauth: provider_quirks.and_then(|provider_quirks| provider_quirks.oauth.clone()),
|
|
models,
|
|
});
|
|
}
|
|
|
|
CatalogBuild {
|
|
providers,
|
|
degraded,
|
|
failed,
|
|
}
|
|
}
|
|
|
|
pub fn build_catalog_strict(
|
|
sources: &FetchedSources,
|
|
quirks: &[ProviderQuirks],
|
|
) -> Result<Vec<ProviderModels>> {
|
|
let build = build_catalog(sources, quirks);
|
|
if !build.degraded.is_empty() || !build.failed.is_empty() {
|
|
bail!(
|
|
"model catalog build incomplete: degraded providers [{}], failed providers [{}]",
|
|
build.degraded.join(", "),
|
|
build.failed.join(", ")
|
|
);
|
|
}
|
|
|
|
Ok(build.providers)
|
|
}
|
|
|
|
/// Replaces or inserts the entry of every degraded/failed provider from the
|
|
/// embedded catalog, preserving the fixed provider order of
|
|
/// [`PROVIDER_SOURCES`]. A synced override must never carry fewer models for a
|
|
/// provider than the embedded catalog: an upstream outage would otherwise
|
|
/// shrink the model list until the next successful sync.
|
|
pub fn backfill_embedded_floor(build: &mut CatalogBuild, embedded: &[ProviderModels]) {
|
|
let needs_backfill: HashSet<&str> = build
|
|
.degraded
|
|
.iter()
|
|
.chain(build.failed.iter())
|
|
.map(String::as_str)
|
|
.collect();
|
|
if needs_backfill.is_empty() {
|
|
return;
|
|
}
|
|
let mut providers = Vec::with_capacity(PROVIDER_SOURCES.len());
|
|
for source in &PROVIDER_SOURCES {
|
|
let built = build
|
|
.providers
|
|
.iter()
|
|
.position(|provider| provider.provider == source.provider)
|
|
.map(|position| build.providers.remove(position));
|
|
if needs_backfill.contains(source.provider)
|
|
&& let Some(entry) = embedded
|
|
.iter()
|
|
.find(|provider| provider.provider == source.provider)
|
|
{
|
|
providers.push(entry.clone());
|
|
continue;
|
|
}
|
|
if let Some(built) = built {
|
|
providers.push(built);
|
|
}
|
|
}
|
|
build.providers = providers;
|
|
}
|
|
|
|
const GENERATED_HEADER: &str = "# GENERATED by 'coyote --generate-models'\n# DO NOT hand-edit model data.\n\
|
|
# Hand-owned behavior lives in quirks.yaml; this file is the embedded offline fallback.\n";
|
|
|
|
pub async fn generate_models_file(path: &Path) -> Result<()> {
|
|
let sources = fetch_sources().await;
|
|
if sources.modelsdev.is_none() || sources.openrouter.is_none() {
|
|
bail!(
|
|
"generating the models file requires both catalog sources; see the log for the failed fetch"
|
|
);
|
|
}
|
|
let providers = build_catalog_strict(&sources, &ALL_QUIRKS)?;
|
|
let model_count: usize = providers.iter().map(|provider| provider.models.len()).sum();
|
|
let yaml = serde_yaml::to_string(&providers)?;
|
|
|
|
fs::write(path, format!("{GENERATED_HEADER}{yaml}"))?;
|
|
println!(
|
|
"Wrote {} models across {} providers to {}",
|
|
model_count,
|
|
providers.len(),
|
|
path.display()
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
pub(crate) fn name_overlap(old: &[String], new: &[String]) -> f64 {
|
|
if old.is_empty() {
|
|
return 1.0;
|
|
}
|
|
let new_names: HashSet<&str> = new.iter().map(String::as_str).collect();
|
|
let kept = old
|
|
.iter()
|
|
.filter(|name| new_names.contains(name.as_str()))
|
|
.count();
|
|
kept as f64 / old.len() as f64
|
|
}
|
|
|
|
pub(crate) fn overlay_hand_owned(hand: ModelData, api: ModelData) -> ModelData {
|
|
ModelData {
|
|
name: hand.name,
|
|
model_type: hand.model_type,
|
|
real_name: hand.real_name.or(api.real_name),
|
|
max_input_tokens: hand.max_input_tokens.or(api.max_input_tokens),
|
|
input_price: hand.input_price.or(api.input_price),
|
|
output_price: hand.output_price.or(api.output_price),
|
|
cache_read_price: hand.cache_read_price.or(api.cache_read_price),
|
|
cache_write_price: hand.cache_write_price.or(api.cache_write_price),
|
|
patch: hand.patch.or(api.patch),
|
|
max_output_tokens: hand.max_output_tokens.or(api.max_output_tokens),
|
|
require_max_tokens: hand.require_max_tokens || api.require_max_tokens,
|
|
supports_vision: hand.supports_vision || api.supports_vision,
|
|
supports_function_calling: hand.supports_function_calling || api.supports_function_calling,
|
|
reasoning_levels: if hand.reasoning_levels.is_empty() {
|
|
api.reasoning_levels
|
|
} else {
|
|
hand.reasoning_levels
|
|
},
|
|
default_reasoning_effort: hand
|
|
.default_reasoning_effort
|
|
.or(api.default_reasoning_effort),
|
|
no_stream: hand.no_stream || api.no_stream,
|
|
no_system_message: hand.no_system_message || api.no_system_message,
|
|
system_prompt_prefix: hand.system_prompt_prefix.or(api.system_prompt_prefix),
|
|
max_tokens_per_chunk: hand.max_tokens_per_chunk.or(api.max_tokens_per_chunk),
|
|
default_chunk_size: hand.default_chunk_size.or(api.default_chunk_size),
|
|
max_batch_size: hand.max_batch_size.or(api.max_batch_size),
|
|
}
|
|
}
|
|
|
|
pub(crate) fn extend_catalog_models(
|
|
catalog: &[ModelData],
|
|
overrides: &[ModelData],
|
|
) -> Vec<ModelData> {
|
|
let mut models = catalog.to_vec();
|
|
for override_model in overrides {
|
|
match models.iter_mut().find(|model| {
|
|
model.name == override_model.name && model.model_type == override_model.model_type
|
|
}) {
|
|
Some(model) => *model = overlay_hand_owned(override_model.clone(), model.clone()),
|
|
None => models.push(override_model.clone()),
|
|
}
|
|
}
|
|
|
|
models
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct ProviderQuirks {
|
|
pub provider: String,
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub oauth: Option<OAuthConfig>,
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub exclude: Vec<String>,
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub rules: Vec<QuirkRule>,
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub variants: Vec<QuirkVariant>,
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub models: Vec<ModelData>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct QuirkRule {
|
|
pub r#match: String,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub real_name: Option<String>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub max_input_tokens: Option<usize>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub max_output_tokens: Option<isize>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub input_price: Option<f64>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub output_price: Option<f64>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub patch: Option<Value>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub require_max_tokens: Option<bool>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub supports_vision: Option<bool>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub supports_function_calling: Option<bool>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub reasoning_levels: Option<Vec<String>>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub default_reasoning_effort: Option<String>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub no_stream: Option<bool>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub no_system_message: Option<bool>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub system_prompt_prefix: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct QuirkVariant {
|
|
pub r#match: String,
|
|
pub suffix: String,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub real_name: Option<String>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub max_input_tokens: Option<usize>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub max_output_tokens: Option<isize>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub input_price: Option<f64>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub output_price: Option<f64>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub patch: Option<Value>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub require_max_tokens: Option<bool>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub supports_vision: Option<bool>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub supports_function_calling: Option<bool>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub reasoning_levels: Option<Vec<String>>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub default_reasoning_effort: Option<String>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub no_stream: Option<bool>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub no_system_message: Option<bool>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub system_prompt_prefix: Option<String>,
|
|
}
|
|
|
|
pub fn apply_rules(provider_quirks: &ProviderQuirks, data: &mut ModelData) {
|
|
for rule in &provider_quirks.rules {
|
|
if !glob_matches(&rule.r#match, &data.name) {
|
|
continue;
|
|
}
|
|
if let Some(real_name) = &rule.real_name {
|
|
data.real_name = Some(real_name.clone());
|
|
}
|
|
if let Some(max_input_tokens) = rule.max_input_tokens {
|
|
data.max_input_tokens = Some(max_input_tokens);
|
|
}
|
|
if let Some(max_output_tokens) = rule.max_output_tokens {
|
|
data.max_output_tokens = Some(max_output_tokens);
|
|
}
|
|
if let Some(input_price) = rule.input_price {
|
|
data.input_price = Some(input_price);
|
|
}
|
|
if let Some(output_price) = rule.output_price {
|
|
data.output_price = Some(output_price);
|
|
}
|
|
if let Some(patch) = &rule.patch {
|
|
data.patch = Some(patch.clone());
|
|
}
|
|
if let Some(require_max_tokens) = rule.require_max_tokens {
|
|
data.require_max_tokens = require_max_tokens;
|
|
}
|
|
if let Some(supports_vision) = rule.supports_vision {
|
|
data.supports_vision = supports_vision;
|
|
}
|
|
if let Some(supports_function_calling) = rule.supports_function_calling {
|
|
data.supports_function_calling = supports_function_calling;
|
|
}
|
|
if let Some(reasoning_levels) = &rule.reasoning_levels {
|
|
data.reasoning_levels = reasoning_levels.clone();
|
|
}
|
|
if let Some(default_reasoning_effort) = &rule.default_reasoning_effort {
|
|
data.default_reasoning_effort = Some(default_reasoning_effort.clone());
|
|
}
|
|
if let Some(no_stream) = rule.no_stream {
|
|
data.no_stream = no_stream;
|
|
}
|
|
if let Some(no_system_message) = rule.no_system_message {
|
|
data.no_system_message = no_system_message;
|
|
}
|
|
if let Some(system_prompt_prefix) = &rule.system_prompt_prefix {
|
|
data.system_prompt_prefix = Some(system_prompt_prefix.clone());
|
|
}
|
|
}
|
|
}
|
|
|
|
fn apply_variant(variant: &QuirkVariant, data: &mut ModelData) {
|
|
if let Some(real_name) = &variant.real_name {
|
|
data.real_name = Some(real_name.clone());
|
|
}
|
|
if let Some(max_input_tokens) = variant.max_input_tokens {
|
|
data.max_input_tokens = Some(max_input_tokens);
|
|
}
|
|
if let Some(max_output_tokens) = variant.max_output_tokens {
|
|
data.max_output_tokens = Some(max_output_tokens);
|
|
}
|
|
if let Some(input_price) = variant.input_price {
|
|
data.input_price = Some(input_price);
|
|
}
|
|
if let Some(output_price) = variant.output_price {
|
|
data.output_price = Some(output_price);
|
|
}
|
|
if let Some(patch) = &variant.patch {
|
|
data.patch = Some(patch.clone());
|
|
}
|
|
if let Some(require_max_tokens) = variant.require_max_tokens {
|
|
data.require_max_tokens = require_max_tokens;
|
|
}
|
|
if let Some(supports_vision) = variant.supports_vision {
|
|
data.supports_vision = supports_vision;
|
|
}
|
|
if let Some(supports_function_calling) = variant.supports_function_calling {
|
|
data.supports_function_calling = supports_function_calling;
|
|
}
|
|
if let Some(reasoning_levels) = &variant.reasoning_levels {
|
|
data.reasoning_levels = reasoning_levels.clone();
|
|
}
|
|
if let Some(default_reasoning_effort) = &variant.default_reasoning_effort {
|
|
data.default_reasoning_effort = Some(default_reasoning_effort.clone());
|
|
}
|
|
if let Some(no_stream) = variant.no_stream {
|
|
data.no_stream = no_stream;
|
|
}
|
|
if let Some(no_system_message) = variant.no_system_message {
|
|
data.no_system_message = no_system_message;
|
|
}
|
|
if let Some(system_prompt_prefix) = &variant.system_prompt_prefix {
|
|
data.system_prompt_prefix = Some(system_prompt_prefix.clone());
|
|
}
|
|
}
|
|
|
|
pub fn quirks_for<'a>(
|
|
quirks: &'a [ProviderQuirks],
|
|
client_name: &str,
|
|
) -> Option<&'a ProviderQuirks> {
|
|
quirks
|
|
.iter()
|
|
.find(|quirks| quirks.provider == client_name)
|
|
.or_else(|| {
|
|
quirks
|
|
.iter()
|
|
.find(|quirks| client_name.starts_with(&quirks.provider))
|
|
})
|
|
}
|
|
|
|
static GLOB_CACHE: LazyLock<Mutex<HashMap<String, Arc<Regex>>>> =
|
|
LazyLock::new(|| Mutex::new(HashMap::new()));
|
|
|
|
fn glob_matches(pattern: &str, name: &str) -> bool {
|
|
let regex = {
|
|
let mut cache = GLOB_CACHE
|
|
.lock()
|
|
.unwrap_or_else(|poison| poison.into_inner());
|
|
cache
|
|
.entry(pattern.to_string())
|
|
.or_insert_with(|| Arc::new(compile_glob(pattern)))
|
|
.clone()
|
|
};
|
|
regex.is_match(name).unwrap_or(false)
|
|
}
|
|
|
|
fn compile_glob(pattern: &str) -> Regex {
|
|
let translated = format!(
|
|
"^{}$",
|
|
fancy_regex::escape(pattern)
|
|
.replace("\\*", ".*")
|
|
.replace("\\?", ".")
|
|
);
|
|
Regex::new(&translated).unwrap_or_else(|error| {
|
|
warn!("Invalid model quirk pattern '{pattern}': {error}. It will match nothing.");
|
|
Regex::new("(?!)").expect("'(?!)' is a valid never-matching regex")
|
|
})
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
use crate::client::MODELS_YAML;
|
|
use serde_json::json;
|
|
|
|
const KNOWN_PROVIDERS: [&str; 24] = [
|
|
"openai",
|
|
"gemini",
|
|
"claude",
|
|
"mistral",
|
|
"ai21",
|
|
"cohere",
|
|
"xai",
|
|
"perplexity",
|
|
"groq",
|
|
"vertexai",
|
|
"bedrock",
|
|
"cloudflare",
|
|
"ernie",
|
|
"qianwen",
|
|
"hunyuan",
|
|
"moonshot",
|
|
"deepseek",
|
|
"zhipuai",
|
|
"minimax",
|
|
"openrouter",
|
|
"github",
|
|
"deepinfra",
|
|
"jina",
|
|
"voyageai",
|
|
];
|
|
|
|
const MODELSDEV_FIXTURE: &str = include_str!("testdata/modelsdev_fixture.json");
|
|
const OPENROUTER_FIXTURE: &str = include_str!("testdata/openrouter_fixture.json");
|
|
|
|
fn make_sources(modelsdev: Option<&str>, openrouter: Option<&str>) -> FetchedSources {
|
|
FetchedSources {
|
|
modelsdev: modelsdev.map(str::to_string),
|
|
openrouter: openrouter.map(str::to_string),
|
|
}
|
|
}
|
|
|
|
fn fixture_sources() -> FetchedSources {
|
|
make_sources(Some(MODELSDEV_FIXTURE), Some(OPENROUTER_FIXTURE))
|
|
}
|
|
|
|
fn parse_quirks(yaml: &str) -> Vec<ProviderQuirks> {
|
|
serde_yaml::from_str(yaml).unwrap()
|
|
}
|
|
|
|
fn find_provider<'a>(build: &'a CatalogBuild, provider: &str) -> &'a ProviderModels {
|
|
build
|
|
.providers
|
|
.iter()
|
|
.find(|candidate| candidate.provider == provider)
|
|
.unwrap_or_else(|| panic!("provider '{provider}' missing"))
|
|
}
|
|
|
|
fn find_model<'a>(provider: &'a ProviderModels, name: &str) -> &'a ModelData {
|
|
provider
|
|
.models
|
|
.iter()
|
|
.find(|model| model.name == name)
|
|
.unwrap_or_else(|| panic!("model '{name}' missing"))
|
|
}
|
|
|
|
#[test]
|
|
fn quirks_yaml_parses() {
|
|
assert!(!ALL_QUIRKS.is_empty());
|
|
for quirks in ALL_QUIRKS.iter() {
|
|
assert!(
|
|
KNOWN_PROVIDERS.contains(&quirks.provider.as_str()),
|
|
"unknown provider '{}' in quirks.yaml",
|
|
quirks.provider
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn provider_sources_match_known_providers() {
|
|
let order: Vec<&str> = PROVIDER_SOURCES
|
|
.iter()
|
|
.map(|source| source.provider)
|
|
.collect();
|
|
|
|
assert_eq!(order, KNOWN_PROVIDERS);
|
|
}
|
|
|
|
#[test]
|
|
fn quirks_models_no_dropped_keys() {
|
|
let skipped_when_default = [
|
|
"type",
|
|
"require_max_tokens",
|
|
"supports_vision",
|
|
"supports_function_calling",
|
|
"reasoning_levels",
|
|
"no_stream",
|
|
"no_system_message",
|
|
];
|
|
let raw: serde_yaml::Value = serde_yaml::from_str(QUIRKS_YAML).unwrap();
|
|
for provider in raw.as_sequence().unwrap() {
|
|
let Some(models) = provider
|
|
.get("models")
|
|
.and_then(|models| models.as_sequence())
|
|
else {
|
|
continue;
|
|
};
|
|
for entry in models {
|
|
let parsed: ModelData = serde_yaml::from_value(entry.clone()).unwrap();
|
|
let reserialized = serde_yaml::to_value(&parsed).unwrap();
|
|
let reserialized_keys: Vec<&str> = reserialized
|
|
.as_mapping()
|
|
.unwrap()
|
|
.keys()
|
|
.map(|key| key.as_str().unwrap())
|
|
.collect();
|
|
for key in entry.as_mapping().unwrap().keys() {
|
|
let key = key.as_str().unwrap();
|
|
assert!(
|
|
reserialized_keys.contains(&key) || skipped_when_default.contains(&key),
|
|
"key '{key}' on model '{}' was dropped by ModelData",
|
|
parsed.name
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn rules_apply_in_order() {
|
|
let quirks: ProviderQuirks = serde_yaml::from_str(
|
|
r#"
|
|
provider: openai
|
|
rules:
|
|
- match: "model-*"
|
|
default_reasoning_effort: low
|
|
require_max_tokens: true
|
|
- match: "model-a"
|
|
default_reasoning_effort: high
|
|
"#,
|
|
)
|
|
.unwrap();
|
|
|
|
let mut matched_by_both = ModelData::new("model-a");
|
|
apply_rules(&quirks, &mut matched_by_both);
|
|
assert_eq!(
|
|
matched_by_both.default_reasoning_effort.as_deref(),
|
|
Some("high")
|
|
);
|
|
assert!(matched_by_both.require_max_tokens);
|
|
|
|
let mut matched_by_glob = ModelData::new("model-b");
|
|
apply_rules(&quirks, &mut matched_by_glob);
|
|
assert_eq!(
|
|
matched_by_glob.default_reasoning_effort.as_deref(),
|
|
Some("low")
|
|
);
|
|
|
|
let mut unmatched = ModelData::new("other");
|
|
apply_rules(&quirks, &mut unmatched);
|
|
assert_eq!(unmatched.default_reasoning_effort, None);
|
|
assert!(!unmatched.require_max_tokens);
|
|
}
|
|
|
|
#[test]
|
|
fn glob_semantics() {
|
|
assert!(glob_matches("claude-*", "claude-opus-4-8"));
|
|
assert!(!glob_matches("claude-*", "gpt-4"));
|
|
assert!(glob_matches("claude-*", "claude-"));
|
|
assert!(glob_matches("gpt-?", "gpt-4"));
|
|
assert!(!glob_matches("gpt-?", "gpt-45"));
|
|
assert!(!glob_matches("gpt-?", "gpt-"));
|
|
assert!(glob_matches("o3", "o3"));
|
|
assert!(!glob_matches("o3", "o3-mini"));
|
|
}
|
|
|
|
#[test]
|
|
fn modelsdev_adapter_maps_fields() {
|
|
let api: Value = serde_json::from_str(MODELSDEV_FIXTURE).unwrap();
|
|
|
|
let openai = modelsdev_provider_models(&api, "openai");
|
|
let gpt56 = openai
|
|
.iter()
|
|
.find(|model| model.data.name == "gpt-5.6")
|
|
.unwrap();
|
|
assert_eq!(gpt56.data.max_input_tokens, Some(1_050_000));
|
|
assert_eq!(gpt56.data.max_output_tokens, Some(128_000));
|
|
assert_eq!(gpt56.data.input_price, Some(4.0));
|
|
assert_eq!(gpt56.data.output_price, Some(20.0));
|
|
assert_eq!(gpt56.data.cache_read_price, Some(0.4));
|
|
assert_eq!(gpt56.data.cache_write_price, Some(5.0));
|
|
assert!(gpt56.data.supports_vision);
|
|
assert!(gpt56.data.supports_function_calling);
|
|
assert_eq!(
|
|
gpt56.data.reasoning_levels,
|
|
["none", "low", "medium", "high", "xhigh", "max"]
|
|
);
|
|
assert_eq!(gpt56.release_date.as_deref(), Some("2026-07-09"));
|
|
|
|
let alibaba = modelsdev_provider_models(&api, "alibaba");
|
|
assert_eq!(alibaba.len(), 1);
|
|
let qwen = &alibaba[0];
|
|
assert_eq!(qwen.data.name, "qwen-max");
|
|
assert_eq!(qwen.data.max_input_tokens, Some(32_768));
|
|
assert_eq!(qwen.data.max_output_tokens, Some(8_192));
|
|
assert_eq!(qwen.data.input_price, Some(1.6));
|
|
assert_eq!(qwen.data.output_price, Some(6.4));
|
|
assert_eq!(qwen.data.cache_read_price, None);
|
|
assert_eq!(qwen.data.cache_write_price, None);
|
|
assert!(!qwen.data.supports_vision);
|
|
assert!(qwen.data.supports_function_calling);
|
|
assert!(qwen.data.reasoning_levels.is_empty());
|
|
assert_eq!(qwen.release_date.as_deref(), Some("2024-04-03"));
|
|
|
|
assert!(modelsdev_provider_models(&api, "no-such-provider").is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn openrouter_adapter_maps_fields() {
|
|
let payload: Value = serde_json::from_str(OPENROUTER_FIXTURE).unwrap();
|
|
let index = openrouter_index(&payload);
|
|
|
|
let data = openrouter_model_data("gpt-4.1", index["openai/gpt-4.1"]);
|
|
assert_eq!(data.max_input_tokens, Some(1_047_576));
|
|
assert_eq!(data.max_output_tokens, Some(32_768));
|
|
assert_eq!(data.input_price, Some(2.0));
|
|
assert_eq!(data.output_price, Some(8.0));
|
|
assert_eq!(data.cache_read_price, Some(0.5));
|
|
assert_eq!(data.cache_write_price, None);
|
|
assert!(data.supports_vision);
|
|
assert!(data.supports_function_calling);
|
|
|
|
let qwen = openrouter_model_data("qwen3.8-max", index["qwen/qwen3.8-max"]);
|
|
assert_eq!(qwen.cache_read_price, Some(0.25));
|
|
assert_eq!(qwen.cache_write_price, Some(2.5));
|
|
|
|
// input_cache_write_1h must not leak into cache_write_price
|
|
let sonnet =
|
|
openrouter_model_data("claude-sonnet-4.6", index["anthropic/claude-sonnet-4.6"]);
|
|
assert_eq!(sonnet.cache_write_price, Some(3.75));
|
|
|
|
let rounded = openrouter_model_data(
|
|
"m",
|
|
&json!({"pricing": {"prompt": "0.00000123456789", "completion": "0.0000025"}}),
|
|
);
|
|
assert_eq!(rounded.input_price, Some(1.2346));
|
|
assert_eq!(rounded.output_price, Some(2.5));
|
|
}
|
|
|
|
#[test]
|
|
fn clamps_drop_out_of_range_fields() {
|
|
let api = json!({
|
|
"p": {"models": {"m": {
|
|
"limit": {"context": 100_000_000, "output": 0},
|
|
"cost": {"input": -1.0, "output": 20_000.0, "cache_read": -0.5, "cache_write": 20_000.0},
|
|
}}}
|
|
});
|
|
let models = modelsdev_provider_models(&api, "p");
|
|
assert_eq!(models.len(), 1);
|
|
let data = &models[0].data;
|
|
assert_eq!(data.name, "m");
|
|
assert_eq!(data.max_input_tokens, None);
|
|
assert_eq!(data.max_output_tokens, None);
|
|
assert_eq!(data.input_price, None);
|
|
assert_eq!(data.output_price, None);
|
|
assert_eq!(data.cache_read_price, None);
|
|
assert_eq!(data.cache_write_price, None);
|
|
|
|
let data = openrouter_model_data(
|
|
"m",
|
|
&json!({
|
|
"context_length": 60_000_000,
|
|
"top_provider": {"max_completion_tokens": 20_000_000},
|
|
"pricing": {"prompt": "-0.000001", "completion": "999", "input_cache_read": "-0.000001", "input_cache_write": "999"},
|
|
}),
|
|
);
|
|
assert_eq!(data.max_input_tokens, None);
|
|
assert_eq!(data.max_output_tokens, None);
|
|
assert_eq!(data.input_price, None);
|
|
assert_eq!(data.output_price, None);
|
|
assert_eq!(data.cache_read_price, None);
|
|
assert_eq!(data.cache_write_price, None);
|
|
}
|
|
|
|
#[test]
|
|
fn adapters_never_set_quirks_owned_fields() {
|
|
let api: Value = serde_json::from_str(MODELSDEV_FIXTURE).unwrap();
|
|
let payload: Value = serde_json::from_str(OPENROUTER_FIXTURE).unwrap();
|
|
let mut all: Vec<SourcedModel> = ["openai", "anthropic", "xai", "alibaba"]
|
|
.iter()
|
|
.flat_map(|id| modelsdev_provider_models(&api, id))
|
|
.collect();
|
|
|
|
all.extend(openrouter_native_models(&payload));
|
|
|
|
assert!(!all.is_empty());
|
|
for model in &all {
|
|
let data = &model.data;
|
|
assert_eq!(data.model_type, "chat", "{}", data.name);
|
|
assert!(data.patch.is_none(), "{}", data.name);
|
|
assert!(data.real_name.is_none(), "{}", data.name);
|
|
assert!(data.system_prompt_prefix.is_none(), "{}", data.name);
|
|
assert!(data.max_tokens_per_chunk.is_none(), "{}", data.name);
|
|
assert!(data.default_chunk_size.is_none(), "{}", data.name);
|
|
assert!(data.max_batch_size.is_none(), "{}", data.name);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn oauth_comes_only_from_quirks() {
|
|
let without_quirks = build_catalog(&fixture_sources(), &[]);
|
|
assert!(
|
|
without_quirks
|
|
.providers
|
|
.iter()
|
|
.all(|provider| provider.oauth.is_none())
|
|
);
|
|
|
|
let quirks = parse_quirks(
|
|
r#"
|
|
- provider: xai
|
|
oauth:
|
|
client_id: id
|
|
token_url: https://example.com/token
|
|
"#,
|
|
);
|
|
|
|
let build = build_catalog(&fixture_sources(), &quirks);
|
|
|
|
assert!(find_provider(&build, "xai").oauth.is_some());
|
|
assert!(find_provider(&build, "openai").oauth.is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn merge_precedence_hand_owned_beats_rule_beats_modelsdev_beats_openrouter() {
|
|
let modelsdev = json!({
|
|
"xai": {"models": {
|
|
"m-1": {"limit": {"context": 1000}, "cost": {"input": 1.0, "cache_read": 0.2}},
|
|
"m-2": {"limit": {"context": 1000}},
|
|
}}
|
|
})
|
|
.to_string();
|
|
let openrouter = json!({"data": [
|
|
{
|
|
"id": "x-ai/m-1",
|
|
"context_length": 2000,
|
|
"pricing": {"prompt": "0.000003", "completion": "0.000004", "input_cache_read": "0.0000001", "input_cache_write": "0.000001"},
|
|
"top_provider": {"max_completion_tokens": 512},
|
|
},
|
|
{"id": "x-ai/m-2", "context_length": 2000},
|
|
]})
|
|
.to_string();
|
|
let quirks = parse_quirks(
|
|
r#"
|
|
- provider: xai
|
|
rules:
|
|
- match: "m-1"
|
|
input_price: 5
|
|
max_input_tokens: 3000
|
|
models:
|
|
- name: m-1
|
|
input_price: 7
|
|
cache_read_price: 0.7
|
|
"#,
|
|
);
|
|
let build = build_catalog(&make_sources(Some(&modelsdev), Some(&openrouter)), &quirks);
|
|
let xai = find_provider(&build, "xai");
|
|
|
|
let merged = find_model(xai, "m-1");
|
|
// hand-owned beats the rule, which beat models.dev, which beat openrouter
|
|
assert_eq!(merged.input_price, Some(7.0));
|
|
assert_eq!(merged.cache_read_price, Some(0.7));
|
|
// rule beats models.dev
|
|
assert_eq!(merged.max_input_tokens, Some(3000));
|
|
// openrouter fills fields models.dev left unset
|
|
assert_eq!(merged.output_price, Some(4.0));
|
|
assert_eq!(merged.max_output_tokens, Some(512));
|
|
assert_eq!(merged.cache_write_price, Some(1.0));
|
|
|
|
// models.dev beats openrouter on the same field
|
|
assert_eq!(find_model(xai, "m-2").max_input_tokens, Some(1000));
|
|
}
|
|
|
|
#[test]
|
|
fn excludes_drop_api_models_but_never_hand_owned() {
|
|
let quirks = parse_quirks(
|
|
r#"
|
|
- provider: openai
|
|
exclude:
|
|
- "gpt-4*"
|
|
models:
|
|
- name: gpt-4.1
|
|
input_price: 9
|
|
"#,
|
|
);
|
|
let build = build_catalog(&fixture_sources(), &quirks);
|
|
let openai = find_provider(&build, "openai");
|
|
assert!(openai.models.iter().all(|model| model.name != "gpt-4o"));
|
|
find_model(openai, "gpt-5.6");
|
|
let hand = find_model(openai, "gpt-4.1");
|
|
assert_eq!(hand.input_price, Some(9.0));
|
|
// the API copy was excluded before the merge, so nothing leaked in
|
|
assert_eq!(hand.max_input_tokens, None);
|
|
}
|
|
|
|
#[test]
|
|
fn variant_expansion() {
|
|
let quirks = parse_quirks(
|
|
r#"
|
|
- provider: xai
|
|
variants:
|
|
- match: "grok-4.6"
|
|
suffix: ":thinking"
|
|
max_output_tokens: 24000
|
|
patch:
|
|
body:
|
|
thinking: true
|
|
- match: "grok-4.5"
|
|
suffix: ":fast"
|
|
real_name: grok-4.5-fast
|
|
"#,
|
|
);
|
|
let build = build_catalog(&fixture_sources(), &quirks);
|
|
let xai = find_provider(&build, "xai");
|
|
let names: Vec<&str> = xai.models.iter().map(|model| model.name.as_str()).collect();
|
|
assert_eq!(
|
|
names,
|
|
["grok-4.6", "grok-4.6:thinking", "grok-4.5", "grok-4.5:fast"]
|
|
);
|
|
|
|
let thinking = find_model(xai, "grok-4.6:thinking");
|
|
assert_eq!(thinking.real_name.as_deref(), Some("grok-4.6"));
|
|
assert_eq!(thinking.max_output_tokens, Some(24_000));
|
|
assert_eq!(thinking.patch, Some(json!({"body": {"thinking": true}})));
|
|
|
|
assert_eq!(
|
|
find_model(xai, "grok-4.5:fast").real_name.as_deref(),
|
|
Some("grok-4.5-fast")
|
|
);
|
|
assert_eq!(find_model(xai, "grok-4.6").real_name, None);
|
|
}
|
|
|
|
#[test]
|
|
fn ordering_release_date_desc_hand_owned_interleaved_undated_last() {
|
|
let shuffled_payloads = [
|
|
r#"{"openai":{"models":{
|
|
"b":{"release_date":"2026-05-01"},
|
|
"a":{"release_date":"2026-05-01"},
|
|
"c":{"release_date":"2026-06-01"},
|
|
"z":{},
|
|
"y":{}
|
|
}}}"#,
|
|
r#"{"openai":{"models":{
|
|
"y":{},
|
|
"c":{"release_date":"2026-06-01"},
|
|
"z":{},
|
|
"a":{"release_date":"2026-05-01"},
|
|
"b":{"release_date":"2026-05-01"}
|
|
}}}"#,
|
|
];
|
|
let quirks = parse_quirks(
|
|
r#"
|
|
- provider: openai
|
|
models:
|
|
- name: n2
|
|
- name: n1
|
|
- name: b
|
|
input_price: 9
|
|
"#,
|
|
);
|
|
for payload in shuffled_payloads {
|
|
let build = build_catalog(&make_sources(Some(payload), None), &quirks);
|
|
let openai = find_provider(&build, "openai");
|
|
let names: Vec<&str> = openai
|
|
.models
|
|
.iter()
|
|
.map(|model| model.name.as_str())
|
|
.collect();
|
|
// Dated models sort newest-first with a name-ascending tiebreak;
|
|
// hand-owned "b" inherits its API release date and interleaves
|
|
// instead of pinning to the top. Undated models sort last:
|
|
// hand-owned in quirks order (n2, n1), then API name-ascending.
|
|
assert_eq!(names, ["c", "a", "b", "n2", "n1", "y", "z"]);
|
|
// "b" is still the hand-owned overlay, not the bare API entry.
|
|
assert_eq!(find_model(openai, "b").input_price, Some(9.0));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn catalog_round_trips_through_yaml() {
|
|
let build = build_catalog(&fixture_sources(), &ALL_QUIRKS);
|
|
assert!(!build.providers.is_empty());
|
|
let yaml = serde_yaml::to_string(&build.providers).unwrap();
|
|
let parsed: Vec<ProviderModels> = serde_yaml::from_str(&yaml).unwrap();
|
|
assert_eq!(serde_yaml::to_string(&parsed).unwrap(), yaml);
|
|
}
|
|
|
|
#[test]
|
|
fn cache_prices_omitted_when_absent_round_trip_when_set() {
|
|
let yaml = serde_yaml::to_string(&ModelData::new("m")).unwrap();
|
|
assert!(!yaml.contains("cache_read_price"));
|
|
assert!(!yaml.contains("cache_write_price"));
|
|
|
|
let mut data = ModelData::new("m");
|
|
data.cache_read_price = Some(0.25);
|
|
data.cache_write_price = Some(2.5);
|
|
let parsed: ModelData =
|
|
serde_yaml::from_str(&serde_yaml::to_string(&data).unwrap()).unwrap();
|
|
assert_eq!(parsed.cache_read_price, Some(0.25));
|
|
assert_eq!(parsed.cache_write_price, Some(2.5));
|
|
}
|
|
|
|
#[test]
|
|
fn openrouter_native_models_use_full_ids() {
|
|
let build = build_catalog(&fixture_sources(), &[]);
|
|
let openrouter = find_provider(&build, "openrouter");
|
|
assert_eq!(openrouter.models.len(), 8);
|
|
|
|
let auto = find_model(openrouter, "openrouter/auto");
|
|
assert_eq!(auto.max_input_tokens, Some(2_000_000));
|
|
// null max_completion_tokens and the "-1" sentinel prices are dropped
|
|
assert_eq!(auto.max_output_tokens, None);
|
|
assert_eq!(auto.input_price, None);
|
|
assert_eq!(auto.output_price, None);
|
|
assert!(auto.supports_vision);
|
|
assert!(auto.supports_function_calling);
|
|
|
|
let free = find_model(openrouter, "openrouter/free");
|
|
assert_eq!(free.max_input_tokens, Some(200_000));
|
|
assert_eq!(free.input_price, Some(0.0));
|
|
assert_eq!(free.output_price, Some(0.0));
|
|
|
|
// no release dates upstream, so full ids sort ascending
|
|
let names: Vec<&str> = openrouter
|
|
.models
|
|
.iter()
|
|
.map(|model| model.name.as_str())
|
|
.collect();
|
|
let mut sorted = names.clone();
|
|
sorted.sort_unstable();
|
|
assert_eq!(names, sorted);
|
|
}
|
|
|
|
#[test]
|
|
fn failure_routing_degraded_and_failed() {
|
|
let quirks = parse_quirks(
|
|
r#"
|
|
- provider: gemini
|
|
models: [{name: g-1}]
|
|
- provider: ai21
|
|
models: [{name: jamba}]
|
|
"#,
|
|
);
|
|
|
|
let build = build_catalog(&make_sources(None, None), &quirks);
|
|
|
|
assert_eq!(build.degraded, ["gemini"]);
|
|
assert!(build.failed.contains(&"openai".to_string()));
|
|
assert!(build.failed.contains(&"openrouter".to_string()));
|
|
assert!(build.failed.contains(&"jina".to_string()));
|
|
assert!(!build.failed.contains(&"gemini".to_string()));
|
|
assert!(!build.failed.contains(&"ai21".to_string()));
|
|
assert!(!build.degraded.contains(&"ai21".to_string()));
|
|
|
|
let providers: Vec<&str> = build
|
|
.providers
|
|
.iter()
|
|
.map(|provider| provider.provider.as_str())
|
|
.collect();
|
|
assert_eq!(providers, ["gemini", "ai21"]);
|
|
assert!(
|
|
build
|
|
.providers
|
|
.iter()
|
|
.all(|provider| !provider.models.is_empty())
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn strict_errors_on_degraded_or_failed() {
|
|
let quirks = parse_quirks(
|
|
r#"
|
|
- provider: gemini
|
|
models: [{name: g-1}]
|
|
"#,
|
|
);
|
|
|
|
let err = build_catalog_strict(&make_sources(None, None), &quirks)
|
|
.unwrap_err()
|
|
.to_string();
|
|
|
|
assert!(err.contains("gemini"), "{err}");
|
|
assert!(err.contains("openai"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn strict_succeeds_when_every_provider_resolves() {
|
|
let mut api = serde_json::Map::new();
|
|
for source in &PROVIDER_SOURCES {
|
|
if let Some(id) = source.modelsdev {
|
|
api.insert(id.to_string(), json!({"models": {"m": {}}}));
|
|
}
|
|
}
|
|
let modelsdev = Value::Object(api).to_string();
|
|
let openrouter = json!({"data": [{"id": "openrouter/auto"}]}).to_string();
|
|
let quirks = parse_quirks(
|
|
r#"
|
|
- provider: ai21
|
|
models: [{name: a}]
|
|
- provider: ernie
|
|
models: [{name: e}]
|
|
- provider: hunyuan
|
|
models: [{name: h}]
|
|
- provider: github
|
|
models: [{name: g}]
|
|
- provider: jina
|
|
models: [{name: j}]
|
|
- provider: voyageai
|
|
models: [{name: v}]
|
|
"#,
|
|
);
|
|
|
|
let providers =
|
|
build_catalog_strict(&make_sources(Some(&modelsdev), Some(&openrouter)), &quirks)
|
|
.unwrap();
|
|
let order: Vec<&str> = providers
|
|
.iter()
|
|
.map(|provider| provider.provider.as_str())
|
|
.collect();
|
|
|
|
assert_eq!(order, KNOWN_PROVIDERS);
|
|
}
|
|
|
|
#[test]
|
|
fn name_overlap_fraction_of_old_names_still_present() {
|
|
let old: Vec<String> = ["a", "b", "c", "d"].map(str::to_string).into();
|
|
let new: Vec<String> = ["a", "c", "x"].map(str::to_string).into();
|
|
|
|
assert_eq!(name_overlap(&old, &new), 0.5);
|
|
assert_eq!(name_overlap(&[], &new), 1.0);
|
|
assert_eq!(name_overlap(&old, &[]), 0.0);
|
|
}
|
|
|
|
fn embedded_catalog() -> Vec<ProviderModels> {
|
|
serde_yaml::from_str(MODELS_YAML).unwrap()
|
|
}
|
|
|
|
fn embedded_provider<'a>(providers: &'a [ProviderModels], name: &str) -> &'a ProviderModels {
|
|
providers
|
|
.iter()
|
|
.find(|provider| provider.provider == name)
|
|
.unwrap_or_else(|| panic!("provider '{name}' missing from models.yaml"))
|
|
}
|
|
|
|
#[test]
|
|
fn embedded_catalog_invariants() {
|
|
let providers = embedded_catalog();
|
|
assert!(providers.len() >= 24);
|
|
let order: Vec<&str> = providers
|
|
.iter()
|
|
.map(|provider| provider.provider.as_str())
|
|
.collect();
|
|
assert_eq!(order, KNOWN_PROVIDERS);
|
|
|
|
assert!(embedded_provider(&providers, "xai").oauth.is_some());
|
|
for name in ["jina", "voyageai"] {
|
|
assert!(
|
|
embedded_provider(&providers, name)
|
|
.models
|
|
.iter()
|
|
.any(|model| model.model_type == "embedding"),
|
|
"{name} lost its embedding models"
|
|
);
|
|
}
|
|
assert!(
|
|
embedded_provider(&providers, "cohere")
|
|
.models
|
|
.iter()
|
|
.any(|model| model.model_type == "reranker"),
|
|
"cohere lost its reranker models"
|
|
);
|
|
|
|
for provider in &providers {
|
|
for model in &provider.models {
|
|
// A :thinking model without a patch must at least alias a real
|
|
// wire name (qianwen twins invert a base-model patch instead).
|
|
if model.name.ends_with(":thinking") {
|
|
assert!(
|
|
model.patch.is_some() || model.real_name.is_some(),
|
|
"{}:{} has neither patch nor real_name",
|
|
provider.provider,
|
|
model.name
|
|
);
|
|
}
|
|
if model.model_type == "chat" {
|
|
let serialized = serde_yaml::to_value(model).unwrap();
|
|
assert!(
|
|
serialized
|
|
.as_mapping()
|
|
.unwrap()
|
|
.keys()
|
|
.any(|key| !matches!(key.as_str(), Some("name" | "type"))),
|
|
"{}:{} is an all-default chat entry",
|
|
provider.provider,
|
|
model.name
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
for model in &embedded_provider(&providers, "claude").models {
|
|
if model.model_type == "chat" {
|
|
assert!(
|
|
model.require_max_tokens,
|
|
"claude:{} is missing require_max_tokens",
|
|
model.name
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn embedded_catalog_roundtrip() {
|
|
let providers = embedded_catalog();
|
|
let yaml = serde_yaml::to_string(&providers).unwrap();
|
|
let reparsed: Vec<ProviderModels> = serde_yaml::from_str(&yaml).unwrap();
|
|
assert_eq!(reparsed.len(), providers.len());
|
|
for (reparsed, original) in reparsed.iter().zip(&providers) {
|
|
assert_eq!(reparsed.provider, original.provider);
|
|
assert_eq!(reparsed.models.len(), original.models.len());
|
|
}
|
|
}
|
|
|
|
fn provider_entry(provider: &str, model_names: &[&str]) -> ProviderModels {
|
|
ProviderModels {
|
|
provider: provider.to_string(),
|
|
oauth: None,
|
|
models: model_names
|
|
.iter()
|
|
.map(|name| ModelData::new(name))
|
|
.collect(),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn backfill_replaces_degraded_and_inserts_failed_providers() {
|
|
let mut build = CatalogBuild {
|
|
providers: vec![
|
|
provider_entry("openai", &["gpt-fresh"]),
|
|
provider_entry("gemini", &["quirks-only"]),
|
|
],
|
|
degraded: vec!["gemini".to_string()],
|
|
failed: vec!["claude".to_string()],
|
|
};
|
|
let embedded = vec![
|
|
provider_entry("openai", &["gpt-old-1", "gpt-old-2"]),
|
|
provider_entry("gemini", &["gem-1", "gem-2", "gem-3"]),
|
|
provider_entry("claude", &["claude-1"]),
|
|
];
|
|
|
|
backfill_embedded_floor(&mut build, &embedded);
|
|
|
|
let order: Vec<&str> = build
|
|
.providers
|
|
.iter()
|
|
.map(|provider| provider.provider.as_str())
|
|
.collect();
|
|
assert_eq!(order, ["openai", "gemini", "claude"]);
|
|
assert_eq!(find_provider(&build, "openai").models.len(), 1);
|
|
assert_eq!(find_provider(&build, "gemini").models.len(), 3);
|
|
assert_eq!(find_provider(&build, "claude").models.len(), 1);
|
|
assert_eq!(
|
|
find_provider(&build, "openai").models[0].name,
|
|
"gpt-fresh",
|
|
"providers with live API data must not be touched"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn backfill_keeps_built_entry_when_embedded_lacks_the_provider() {
|
|
let mut build = CatalogBuild {
|
|
providers: vec![provider_entry("gemini", &["quirks-only"])],
|
|
degraded: vec!["gemini".to_string()],
|
|
failed: vec![],
|
|
};
|
|
|
|
backfill_embedded_floor(&mut build, &[]);
|
|
|
|
assert_eq!(find_provider(&build, "gemini").models.len(), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn backfill_without_degraded_or_failed_is_a_no_op() {
|
|
let mut build = CatalogBuild {
|
|
providers: vec![provider_entry("openai", &["gpt-fresh"])],
|
|
degraded: vec![],
|
|
failed: vec![],
|
|
};
|
|
|
|
backfill_embedded_floor(&mut build, &[provider_entry("openai", &["a", "b"])]);
|
|
|
|
assert_eq!(find_provider(&build, "openai").models.len(), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn backfill_restores_embedded_floor_for_unresolved_providers() {
|
|
let embedded: Vec<ProviderModels> = serde_yaml::from_str(MODELS_YAML).unwrap();
|
|
let mut build = build_catalog(&make_sources(Some(MODELSDEV_FIXTURE), None), &ALL_QUIRKS);
|
|
let unresolved: Vec<String> = build
|
|
.degraded
|
|
.iter()
|
|
.chain(build.failed.iter())
|
|
.cloned()
|
|
.collect();
|
|
assert!(!unresolved.is_empty());
|
|
|
|
backfill_embedded_floor(&mut build, &embedded);
|
|
|
|
for provider in unresolved {
|
|
let built = find_provider(&build, &provider);
|
|
let floor = embedded
|
|
.iter()
|
|
.find(|embedded| embedded.provider == provider)
|
|
.unwrap();
|
|
assert!(
|
|
built.models.len() >= floor.models.len(),
|
|
"provider '{provider}' shrank below the embedded floor"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Guards against upstream id-shape drift by comparing live API model
|
|
/// names against the embedded models.yaml. Run manually:
|
|
/// `cargo test idshape_guard_live -- --ignored --nocapture`
|
|
#[tokio::test]
|
|
#[ignore]
|
|
async fn idshape_guard_live() {
|
|
let sources = fetch_sources().await;
|
|
let modelsdev: Value = serde_json::from_str(
|
|
sources
|
|
.modelsdev
|
|
.as_deref()
|
|
.expect("models.dev fetch failed"),
|
|
)
|
|
.unwrap();
|
|
let openrouter: Value = serde_json::from_str(
|
|
sources
|
|
.openrouter
|
|
.as_deref()
|
|
.expect("openrouter fetch failed"),
|
|
)
|
|
.unwrap();
|
|
let embedded: Vec<ProviderModels> = serde_yaml::from_str(MODELS_YAML).unwrap();
|
|
|
|
println!(
|
|
"{:<12} {:>7} {:>5} {:>5}",
|
|
"provider", "overlap", "old", "new"
|
|
);
|
|
for source in &PROVIDER_SOURCES {
|
|
let new: Vec<String> = if source.provider == "openrouter" {
|
|
openrouter_native_models(&openrouter)
|
|
.into_iter()
|
|
.map(|model| model.data.name)
|
|
.collect()
|
|
} else if let Some(id) = source.modelsdev {
|
|
modelsdev_provider_models(&modelsdev, id)
|
|
.into_iter()
|
|
.map(|model| model.data.name)
|
|
.collect()
|
|
} else {
|
|
continue;
|
|
};
|
|
let old: Vec<String> = embedded
|
|
.iter()
|
|
.find(|provider| provider.provider == source.provider)
|
|
.map(|provider| {
|
|
provider
|
|
.models
|
|
.iter()
|
|
.filter(|model| {
|
|
model.model_type == "chat" && !model.name.ends_with(":thinking")
|
|
})
|
|
.map(|model| model.name.clone())
|
|
.collect()
|
|
})
|
|
.unwrap_or_default();
|
|
println!(
|
|
"{:<12} {:>7.2} {:>5} {:>5}",
|
|
source.provider,
|
|
name_overlap(&old, &new),
|
|
old.len(),
|
|
new.len()
|
|
);
|
|
}
|
|
}
|
|
|
|
fn typed_model(name: &str, model_type: &str) -> ModelData {
|
|
ModelData {
|
|
name: name.to_string(),
|
|
model_type: model_type.to_string(),
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn extend_catalog_models_merges_collisions_field_wise() {
|
|
let mut catalog_model = ModelData::new("m");
|
|
catalog_model.max_input_tokens = Some(100_000);
|
|
catalog_model.input_price = Some(1.0);
|
|
catalog_model.supports_vision = true;
|
|
let mut override_model = ModelData::new("m");
|
|
override_model.input_price = Some(9.0);
|
|
|
|
let merged = extend_catalog_models(&[catalog_model], &[override_model]);
|
|
assert_eq!(merged.len(), 1);
|
|
assert_eq!(merged[0].input_price, Some(9.0));
|
|
assert_eq!(merged[0].max_input_tokens, Some(100_000));
|
|
assert!(merged[0].supports_vision);
|
|
}
|
|
|
|
#[test]
|
|
fn extend_catalog_models_appends_net_new_after_catalog() {
|
|
let catalog = [ModelData::new("a"), ModelData::new("b")];
|
|
let merged = extend_catalog_models(&catalog, &[ModelData::new("c")]);
|
|
let names: Vec<&str> = merged.iter().map(|model| model.name.as_str()).collect();
|
|
assert_eq!(names, ["a", "b", "c"]);
|
|
}
|
|
|
|
#[test]
|
|
fn extend_catalog_models_collides_on_name_and_type() {
|
|
let mut catalog_model = typed_model("m", "embedding");
|
|
catalog_model.default_chunk_size = Some(1500);
|
|
let merged = extend_catalog_models(&[catalog_model], &[typed_model("m", "reranker")]);
|
|
assert_eq!(merged.len(), 2);
|
|
assert_eq!(merged[0].model_type, "embedding");
|
|
assert_eq!(merged[0].default_chunk_size, Some(1500));
|
|
assert_eq!(merged[1].model_type, "reranker");
|
|
}
|
|
}
|