feat: Support for secret injection into the global config file (API keys, for example)

config.example.yaml

@@ -10,7 +10,7 @@ keybindings: emacs               # Choose keybinding style (emacs, vi)
 editor: null                     # Specifies the command used to edit input buffer or session. (e.g. vim, emacs, nano).
 wrap: no                         # Controls text wrapping (no, auto, <max-width>)
 wrap_code: false                 # Enables or disables wrapping of code blocks
-vault_password_file: null        # Path to a file containing the password for the Loki vault
+vault_password_file: null        # Path to a file containing the password for the Loki vault (cannot be a secret template)
 
 # ---- function-calling ----
 function_calling: true           # Enables or disables function calling (Globally).
@@ -116,14 +116,14 @@ clients:
   # See https://platform.openai.com/docs/quickstart
   - type: openai
     api_base: https://api.openai.com/v1               # Optional
-    api_key: xxx
+    api_key: '{{OPENAI_API_KEY}}'                     # You can either hard-code or inject secrets from the Loki vault
     organization_id: org-xxx                          # Optional
 
   # For any platform compatible with OpenAI's API
   - type: openai-compatible
     name: ollama
     api_base: http://localhost:11434/v1
-    api_key: xxx                                      # Optional
+    api_key: '{{OLLAMA_API_KEY}}'                     # Optional; You can either hard-code or inject secrets from the Loki vault
     models:
       - name: deepseek-r1
         max_input_tokens: 131072
@@ -141,7 +141,7 @@ clients:
   # See https://ai.google.dev/docs
   - type: gemini
     api_base: https://generativelanguage.googleapis.com/v1beta
-    api_key: xxx
+    api_key: '{{GEMINI_API_KEY}}'                       # You can either hard-code or inject secrets from the Loki vault
     patch:
       chat_completions:
         '.*':
@@ -159,47 +159,47 @@ clients:
   # See https://docs.anthropic.com/claude/reference/getting-started-with-the-api
   - type: claude
     api_base: https://api.anthropic.com/v1              # Optional
-    api_key: xxx
+    api_key: '{{ANTHROPIC_API_KEY}}'                    # You can either hard-code or inject secrets from the Loki vault
 
   # See https://docs.mistral.ai/
   - type: openai-compatible
     name: mistral
     api_base: https://api.mistral.ai/v1
-    api_key: xxx
+    api_key: '{{MISTRAL_API_KEY}}'                      # You can either hard-code or inject secrets from the Loki vault
 
   # See https://docs.x.ai/docs
   - type: openai-compatible
     name: xai
     api_base: https://api.x.ai/v1
-    api_key: xxx
+    api_key: '{{XAI_API_KEY}}'                          # You can either hard-code or inject secrets from the Loki vault
 
   # See https://docs.ai21.com/docs/quickstart
   - type: openai-compatible
     name: ai12
     api_base: https://api.ai21.com/studio/v1
-    api_key: xxx
+    api_key: '{{AI21_API_KEY}}'                         # You can either hard-code or inject secrets from the Loki vault
 
   # See https://docs.cohere.com/docs/the-cohere-platform
   - type: cohere
     api_base: https://api.cohere.ai/v2                  # Optional
-    api_key: xxx
+    api_key: '{{COHERE_API_KEY}}'                       # You can either hard-code or inject secrets from the Loki vault
 
   # See https://docs.perplexity.ai/docs/getting-started
   - type: openai-compatible
     name: perplexity
     api_base: https://api.perplexity.ai
-    api_key: xxx
+    api_key: '{{PERPLEXITY_API_KEY}}'                   # You can either hard-code or inject secrets from the Loki vault
 
   # See https://console.groq.com/docs/quickstart
   - type: openai-compatible
     name: groq
     api_base: https://api.groq.com/openai/v1
-    api_key: xxx
+    api_key: '{{GROQ_API_KEY}}'                         # You can either hard-code or inject secrets from the Loki vault
 
   # See https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart
   - type: azure-openai
     api_base: https://{RESOURCE}.openai.azure.com
-    api_key: xxx
+    api_key: '{{AZURE_OPENAI_API_KEY}}'                 # You can either hard-code or inject secrets from the Loki vault
     models:
       - name: gpt-4o                                    # Model deployment name
         max_input_tokens: 128000
@@ -230,8 +230,8 @@ clients:
 
   # See https://docs.aws.amazon.com/bedrock/latest/userguide/
   - type: bedrock
-    access_key_id: xxx
+    access_key_id: '{{AWS_ACCESS_KEY_ID}}'              # You can either hard-code or inject secrets from the Loki vault
-    secret_access_key: xxx
+    secret_access_key: '{{AWS_SECRET_ACCESS_KEY}}'      # You can either hard-code or inject secrets from the Loki vault
     region: xxx
     session_token: xxx                                  # Optional, only needed for temporary credentials
 
@@ -239,67 +239,67 @@ clients:
   - type: openai-compatible
     name: cloudflare
     api_base: https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/ai/v1
-    api_key: xxx
+    api_key: '{{CLOUDFLARE_API_KEY}}'                    # You can either hard-code or inject secrets from the Loki vault
 
   # See https://cloud.baidu.com/doc/WENXINWORKSHOP/index.html
   - type: openai-compatible
     name: ernie
     api_base: https://qianfan.baidubce.com/v2
-    api_key: xxx
+    api_key: '{{BAIDU_API_KEY}}'                        # You can either hard-code or inject secrets from the Loki vault
 
   # See https://dashscope.aliyun.com/
   - type: openai-compatible
     name: qianwen
     api_base: https://dashscope.aliyuncs.com/compatible-mode/v1
-    api_key: xxx
+    api_key: '{{ALIYUN_API_KEY}}'                       # You can either hard-code or inject secrets from the Loki vault
 
   # See https://cloud.tencent.com/product/hunyuan
   - type: openai-compatible
     name: hunyuan
     api_base: https://api.hunyuan.cloud.tencent.com/v1
-    api_key: xxx
+    api_key: '{{TENCENT_API_KEY}}'                      # You can either hard-code or inject secrets from the Loki vault
 
   # See https://platform.moonshot.cn/docs/intro
   - type: openai-compatible
     name: moonshot
     api_base: https://api.moonshot.cn/v1
-    api_key: xxx
+    api_key: '{{MOONSHOT_API_KEY}}'                     # You can either hard-code or inject secrets from the Loki vault
 
   # See https://platform.deepseek.com/api-docs/
   - type: openai-compatible
     name: deepseek
     api_base: https://api.deepseek.com
-    api_key: xxx
+    api_key: '{{DEEPSEEK_API_KEY}}'                     # You can either hard-code or inject secrets from the Loki vault
 
   # See https://open.bigmodel.cn/dev/howuse/introduction
   - type: openai-compatible
     name: zhipuai
     api_base: https://open.bigmodel.cn/api/paas/v4
-    api_key: xxx
+    api_key: '{{ZHIPUAI_API_KEY}}'                      # You can either hard-code or inject secrets from the Loki vault
 
   # See https://platform.minimaxi.com/document/Fast%20access
   - type: openai-compatible
     name: minimax
     api_base: https://api.minimax.chat/v1
-    api_key: xxx
+    api_key: '{{MINIMAX_API_KEY}}'                      # You can either hard-code or inject secrets from the Loki vault
 
   # See https://openrouter.ai/docs#quick-start
   - type: openai-compatible
     name: openrouter
     api_base: https://openrouter.ai/api/v1
-    api_key: xxx
+    api_key: '{{OPENROUTER_API_KEY}}'                   # You can either hard-code or inject secrets from the Loki vault
 
   # See https://github.com/marketplace/models
   - type: openai-compatible
     name: github
     api_base: https://models.inference.ai.azure.com
-    api_key: xxx
+    api_key: '{{GITHUB_API_KEY}}'                       # You can either hard-code or inject secrets from the Loki vault
 
   # See https://deepinfra.com/docs
   - type: openai-compatible
     name: deepinfra
     api_base: https://api.deepinfra.com/v1/openai
-    api_key: xxx
+    api_key: '{{DEEPINFRA_API_KEY}}'                    # You can either hard-code or inject secrets from the Loki vault
 
 
   # ----- RAG dedicated -----
@@ -308,10 +308,10 @@ clients:
   - type: openai-compatible
     name: jina
     api_base: https://api.jina.ai/v1
-    api_key: xxx
+    api_key: '{{JINA_API_KEY}}'                         # You can either hard-code or inject secrets from the Loki vault
 
   # See https://docs.voyageai.com/docs/introduction
   - type: openai-compatible
     name: voyageai
     api_base: https://api.voyageai.com/v1
-    api_key: xxx
+    api_key: '{{VOYAGEAI_API_KEY}}'                     # You can either hard-code or inject secrets from the Loki vault

src/config/mod.rs

@@ -24,8 +24,9 @@ use crate::utils::*;
 use crate::mcp::{
 	McpRegistry, MCP_INVOKE_META_FUNCTION_NAME_PREFIX, MCP_LIST_META_FUNCTION_NAME_PREFIX,
 };
-use crate::vault::Vault;
+use crate::vault::{interpolate_secrets, Vault};
 use anyhow::{anyhow, bail, Context, Result};
+use fancy_regex::Regex;
 use indexmap::IndexMap;
 use indoc::formatdoc;
 use inquire::{list_option::ListOption, validator::Validation, Confirm, MultiSelect, Select, Text};
@@ -34,6 +35,7 @@ use parking_lot::RwLock;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use std::collections::{HashMap, HashSet};
+use std::sync::LazyLock;
 use std::{
 	env,
 	fs::{
@@ -53,6 +55,9 @@ pub const TEMP_ROLE_NAME: &str = "temp";
 pub const TEMP_RAG_NAME: &str = "temp";
 pub const TEMP_SESSION_NAME: &str = "temp";
 
+static PASSWORD_FILE_SECRET_RE: LazyLock<Regex> =
+	LazyLock::new(|| Regex::new(r#"vault_password_file:.*['|"]?\{\{(.+)}}['|"]?"#).unwrap());
+
 /// Monokai Extended
 const DARK_THEME: &[u8] = include_bytes!("../../assets/monokai-extended.theme.bin");
 const LIGHT_THEME: &[u8] = include_bytes!("../../assets/monokai-extended-light.theme.bin");
@@ -292,12 +297,12 @@ impl Config {
 		abort_signal: AbortSignal,
 	) -> Result<Self> {
 		let config_path = Self::config_file();
-        let mut config = if !config_path.exists() {
+		let (mut config, content) = if !config_path.exists() {
 			match env::var(get_env_name("provider"))
 				.ok()
 				.or_else(|| env::var(get_env_name("platform")).ok())
 			{
-                Some(v) => Self::load_dynamic(&v)?,
+				Some(v) => (Self::load_dynamic(&v)?, String::new()),
 				None => {
 					if *IS_STDOUT_TERMINAL {
 						create_config_file(&config_path).await?;
@@ -309,12 +314,39 @@ impl Config {
 			Self::load_from_file(&config_path)?
 		};
 
-        Agent::install_builtin_agents()?;
+		let setup = async |config: &mut Self| -> Result<()> {
+			let vault = Vault::init(config);
+
+			let (parsed_config, missing_secrets) = interpolate_secrets(&content, &vault);
+			if !missing_secrets.is_empty() && !info_flag {
+				debug!("Global config references secrets that are missing from the vault: {missing_secrets:?}");
+				return Err(anyhow!(formatdoc!(
+                    "
+										Global config file references secrets that are missing from the vault: {:?}
+										Please add these secrets to the vault and try again.",
+                    missing_secrets
+                )));
+			}
+
+			if !parsed_config.is_empty() && !info_flag {
+				debug!("Global config is invalid once secrets are injected: {parsed_config}");
+				let new_config = Self::load_from_str(&parsed_config).with_context(|| {
+					formatdoc!(
+                        "
+												Global config is invalid once secrets are injected.
+												Double check the secret values and file syntax, then try again.
+												"
+                    )
+				})?;
+				*config = new_config.clone();
+			}
 
 			config.working_mode = working_mode;
 			config.info_flag = info_flag;
+			config.vault = vault;
+
+			Agent::install_builtin_agents()?;
 
-        let setup = async |config: &mut Self| -> Result<()> {
 			config.load_envs();
 
 			if let Some(wrap) = config.wrap.clone() {
@@ -329,7 +361,6 @@ impl Config {
 			config.setup_model()?;
 			config.setup_document_loaders();
 			config.setup_user_agent();
-            config.vault = Vault::init(config);
 			Ok(())
 		};
 		let ret = setup(&mut config).await;
@@ -756,12 +787,13 @@ impl Config {
 							bail!("No MCP servers are configured. Please configure MCP servers first before setting 'use_mcp_servers'.");
 						}
 
-                        if servers.split(',').all(|s| {
+						if !servers.split(',').all(|s| {
-                            !registry
+							registry
 								.list_configured_servers()
 								.contains(&s.trim().to_string())
+								|| s == "all"
 						}) {
-                            bail!("None of the specified MCP servers in 'use_mcp_servers' are configured. Please check your MCP server configuration.");
+							bail!("Some of the specified MCP servers in 'use_mcp_servers' are configured. Please check your MCP server configuration.");
 						}
 					}
 				}
@@ -2604,10 +2636,20 @@ impl Config {
 			.with_context(|| format!("Failed to create/append {}", path.display()))
 	}
 
-    fn load_from_file(config_path: &Path) -> Result<Self> {
+	fn load_from_file(config_path: &Path) -> Result<(Self, String)> {
 		let err = || format!("Failed to load config at '{}'", config_path.display());
 		let content = read_to_string(config_path).with_context(err)?;
-        let config: Self = serde_yaml::from_str(&content)
+		let config = Self::load_from_str(&content).with_context(err)?;
+
+		Ok((config, content))
+	}
+
+	fn load_from_str(content: &str) -> Result<Self> {
+		if PASSWORD_FILE_SECRET_RE.is_match(content)? {
+			bail!("secret injection cannot be done on the vault_password_file property");
+		}
+
+		let config: Self = serde_yaml::from_str(content)
 			.map_err(|err| {
 				let err_msg = err.to_string();
 				let err_msg = if err_msg.starts_with(&format!("{CLIENTS_FIELD}: ")) {
@@ -2623,7 +2665,7 @@ impl Config {
 				};
 				anyhow!("{err_msg}")
 			})
-            .with_context(err)?;
+			.with_context(|| "Failed to load config from str")?;
 
 		Ok(config)
 	}

src/mcp/mod.rs

@@ -1,6 +1,6 @@
 use crate::config::Config;
 use crate::utils::{abortable_run_with_spinner, AbortSignal};
-use crate::vault::SECRET_RE;
+use crate::vault::interpolate_secrets;
 use anyhow::{anyhow, Context, Result};
 use futures_util::future::BoxFuture;
 use futures_util::{stream, StreamExt, TryStreamExt};
@@ -92,17 +92,7 @@ impl McpRegistry {
 			return Ok(registry);
 		}
 
-        let mut missing_secrets = vec![];
+		let (parsed_content, missing_secrets) = interpolate_secrets(&content, &config.vault);
-        let parsed_content = SECRET_RE.replace_all(&content, |caps: &fancy_regex::Captures<'_>| {
-            let secret = config.vault.get_secret(&caps[1], false);
-            match secret {
-                Ok(s) => s,
-                Err(_) => {
-                    missing_secrets.push(caps[1].to_string());
-                    "".to_string()
-                }
-            }
-        });
 
 		if !missing_secrets.is_empty() {
 			return Err(anyhow!(formatdoc!(

src/vault/mod.rs

@@ -1,5 +1,7 @@
 mod utils;
 
+pub use utils::interpolate_secrets;
+
 use crate::cli::Cli;
 use crate::config::Config;
 use crate::vault::utils::ensure_password_file_initialized;
@@ -11,7 +13,7 @@ use inquire::{required, Password, PasswordDisplayMode};
 use std::sync::LazyLock;
 use tokio::runtime::Handle;
 
-pub static SECRET_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\{\{(.+)}}").unwrap());
+static SECRET_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\{\{(.+)}}").unwrap());
 
 #[derive(Debug, Default, Clone)]
 pub struct Vault {

src/vault/utils.rs

@@ -1,10 +1,12 @@
 use crate::config::ensure_parent_exists;
+use crate::vault::{Vault, SECRET_RE};
 use anyhow::anyhow;
 use anyhow::Result;
 use gman::providers::local::LocalProvider;
 use indoc::formatdoc;
 use inquire::validator::Validation;
 use inquire::{min_length, required, Confirm, Password, PasswordDisplayMode, Text};
+use std::borrow::Cow;
 use std::path::PathBuf;
 
 pub fn ensure_password_file_initialized(local_provider: &mut LocalProvider) -> Result<()> {
@@ -130,3 +132,19 @@ pub fn ensure_password_file_initialized(local_provider: &mut LocalProvider) -> R
 
 	Ok(())
 }
+
+pub fn interpolate_secrets<'a>(content: &'a str, vault: &Vault) -> (Cow<'a, str>, Vec<String>) {
+	let mut missing_secrets = vec![];
+	let parsed_content = SECRET_RE.replace_all(content, |caps: &fancy_regex::Captures<'_>| {
+		let secret = vault.get_secret(caps[1].trim(), false);
+		match secret {
+			Ok(s) => s,
+			Err(_) => {
+				missing_secrets.push(caps[1].to_string());
+				"".to_string()
+			}
+		}
+	});
+
+	(parsed_content, missing_secrets)
+}