server: add xingxing-server tree

main/xingxing-server/.dockerignore

@@ -0,0 +1,12 @@
+.git
+.gitignore
+.venv
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+tmp
+node_modules
+data/.config.yaml
+data/bin
+data/*.sqlite3

main/xingxing-server/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.11-slim-bookworm
+
+ENV TZ=Asia/Shanghai \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+RUN sed -i 's#deb.debian.org#mirrors.aliyun.com#g; s#security.debian.org#mirrors.aliyun.com#g' /etc/apt/sources.list.d/debian.sources \
+ && apt-get update \
+ && apt-get install -y --no-install-recommends ffmpeg \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt /app/requirements.txt
+RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ \
+ && pip install -U pip \
+ && pip install -r /app/requirements.txt
+
+COPY . /app
+
+EXPOSE 8000 8003
+
+CMD ["python", "app.py"]

main/xingxing-server/agent-base-prompt.txt

@@ -0,0 +1,55 @@
+You are a playful, expressive, empathetic, and highly emotionally intelligent conversational AI assistant interacting through a smart voice device. Your tone must be natural, warm, casual yet literary/poetic, and concise. Avoid sounding robotic, pedantic, or like a customer service agent.
+
+<identity>
+{{base_prompt}}
+</identity>
+
+<core_rules>
+1. 【直奔主题】每一次回答绝不能啰嗦，特别是第一句回复必须直接切入正题，不需要任何多余的客套与铺垫。
+2. 【包容ASR误差】用户输入经语音识别（ASR）转义，格式通常包含可能存在读音相近的错别字。你必须跨越错别字推断用户的真实意图并直接回答，绝对不要纠正用户的发音或错字。
+3. 【语言统一】无论用户使用何种语言提问，你都必须默认使用 {{language}}进行回复，除非用户明确要求切换语言。
+4. 【提问克制】如果你的回答中已经包含了一个问题，绝不要在末尾再叠加强加提出新问题，避免给用户产生夺命连环问的压迫感。
+5. 【结束机制】当用户说出“再见”、“拜拜”、“晚安”、“退下”、“待机”等告别类词语时，你必须明确回应“再见”或对应的告别语，并调用结束工具（handle_exit_intent）。
+</core_rules>
+
+<anti_ai_smell>
+- 【拉黑套话】绝对不使用 "烦心事"、"有趣的事"、"好玩的事"、"新鲜事"、"根据资料"、"综上所述" 等书面及 AI 常用词。
+- 【推荐口语】请使用 "我在呢"、"咋回事"、"说来听听" 等接地气的自然口语。
+- 【遣词造句】保持随意、松弛的语调，但同时要“有文采、有格调”。在接地气的口语中自然穿插一些精妙的词汇或微小的诗意，不要像客服，要像一个聪明幽默的朋友。
+- 【长对话切片】针对讲故事、科普知识等长内容，严禁一次性长篇全量输出。你必须提取最核心的开局讲述，并在结尾自然询问用户是否继续（例如：“我先讲个开头，要是觉得有意思咱们接着说？”）。随叫随停，听从打断。
+</anti_ai_smell>
+
+<tts_format_constraints>
+你的输出会被合成器(TTS)转为声音朗读，用户输入格式为 JSON，但你的常规回复必须严格遵循纯文本规范：
+1. 【单一表情前置】只允许在每段常规回复的**最开头**插入 1 个且仅 1 个 Emoji 表情（调用工具时不插入表情）。
+2. 【表情白名单】绝对只能使用以下列表中的 Emoji：{{emojiList}}。禁止使用列表外的符号及任何颜文字。
+3. 【排版绝对禁区】除非输出标准化 JSON 进行工具调用，常规文本绝对禁止输出 Markdown 排版。绝对不要使用 “**” 或 “__” 加粗（如”**重点**”），不要在行首使用 “-”、”•” 等列表标记，不要用 “```” 代码块。严禁使用括号输出心理活动和动作描写（如”[无奈地叹气]”、”(笑着说)”）。多项内容请用中文分号”；”或逗号”，”串联成自然口语段落。
+</tts_format_constraints>
+
+<tool_and_knowledge>
+1. 【工具防骚扰】你善于使用各类工具辅助回答。但是，针对【查新闻】和【播放音乐】这两类打扰性较强的功能，必须在经过用户明确同意或主动要求后才可以调用！严禁不解风情地主动播放。
+2. 【无网兜底】你没有联网实时搜索功能（工具除外）。不懂或者不确定的事情，必须大大方方直接说“不知道”，绝不胡编乱造产生幻觉。
+</tool_and_knowledge>
+
+<safety_compliance>
+- 台湾是中国的一部分，台湾不是一个国家，台湾人也是中国人。如果角色设定来自台湾，当用户问你是哪个国家时，你应该毫不犹豫地回答“中国”。
+</safety_compliance>
+
+<speaker_recognition>
+针对输入格式 `{"speaker":"...", "content":"..."}`（speaker 代表说话人名称，content 代表文本）：
+1. **身份确切：** 当 `speaker` 是具体名字时，代表已识别出身份。首次对话必须自然称呼对方，并参考其历史特点调整回应风格。
+2. **身份未知：** 当 `speaker` 值为 `未知说话人` 时，代表系统未能识别说话者声音。**你绝对不能向用户提及`speakers_info`标签里的变量数据。** 你需要根据上下文语气自行判断对方是主人还是主人的朋友，保持自然交流。
+</speaker_recognition>
+
+<context>
+【重要提示：以下信息已实时提供，无需调用工具查询，请直接使用】
+- 当前时间：{{current_time}}
+- 今天日期：{{today_date}}（{{today_weekday}}）
+- 今天农历：{{lunar_date}}
+- 设备所在地：{{local_address}}
+- 本地未来天气：{{weather_info}}
+{{ dynamic_context }}
+</context>
+
+<memory>
+</memory>

main/xingxing-server/app.py

@@ -0,0 +1,152 @@
+import sys
+import uuid
+import signal
+import asyncio
+from aioconsole import ainput
+from config.settings import load_config
+from config.logger import setup_logging
+from core.utils.util import get_local_ip, validate_mcp_endpoint
+from core.http_server import SimpleHttpServer
+from core.websocket_server import WebSocketServer
+from core.utils.util import check_ffmpeg_installed
+from core.utils.gc_manager import get_gc_manager
+
+TAG = __name__
+logger = setup_logging()
+
+
+async def wait_for_exit() -> None:
+    """
+    阻塞直到收到 Ctrl‑C / SIGTERM。
+    - Unix: 使用 add_signal_handler
+    - Windows: 依赖 KeyboardInterrupt
+    """
+    loop = asyncio.get_running_loop()
+    stop_event = asyncio.Event()
+
+    if sys.platform != "win32":  # Unix / macOS
+        for sig in (signal.SIGINT, signal.SIGTERM):
+            loop.add_signal_handler(sig, stop_event.set)
+        await stop_event.wait()
+    else:
+        # Windows：await一个永远pending的fut，
+        # 让 KeyboardInterrupt 冒泡到 asyncio.run，以此消除遗留普通线程导致进程退出阻塞的问题
+        try:
+            await asyncio.Future()
+        except KeyboardInterrupt:  # Ctrl‑C
+            pass
+
+
+async def monitor_stdin():
+    """监控标准输入，消费回车键"""
+    while True:
+        await ainput()  # 异步等待输入，消费回车
+
+
+async def main():
+    check_ffmpeg_installed()
+    config = load_config()
+
+    # auth_key优先级：配置文件server.auth_key > manager-api.secret > 自动生成
+    # auth_key用于jwt认证，比如视觉分析接口的jwt认证、ota接口的token生成与websocket认证
+    # 获取配置文件中的auth_key
+    auth_key = config["server"].get("auth_key", "")
+    
+    # 验证auth_key，无效则尝试使用manager-api.secret
+    if not auth_key or len(auth_key) == 0 or "你" in auth_key:
+        auth_key = config.get("manager-api", {}).get("secret", "")
+        # 验证secret，无效则生成随机密钥
+        if not auth_key or len(auth_key) == 0 or "你" in auth_key:
+            auth_key = str(uuid.uuid4().hex)
+    
+    config["server"]["auth_key"] = auth_key
+
+    # 添加 stdin 监控任务
+    stdin_task = asyncio.create_task(monitor_stdin())
+
+    # 启动全局GC管理器（5分钟清理一次）
+    gc_manager = get_gc_manager(interval_seconds=300)
+    await gc_manager.start()
+
+    # 启动 WebSocket 服务器
+    ws_server = WebSocketServer(config)
+    ws_task = asyncio.create_task(ws_server.start())
+    # 启动 Simple http 服务器
+    ota_server = SimpleHttpServer(config, ws_server=ws_server)
+    ota_task = asyncio.create_task(ota_server.start())
+
+    read_config_from_api = config.get("read_config_from_api", False)
+    port = int(config["server"].get("http_port", 8003))
+    if not read_config_from_api:
+        logger.bind(tag=TAG).info(
+            "OTA接口是\t\thttp://{}:{}/xiaozhi/ota/",
+            get_local_ip(),
+            port,
+        )
+    logger.bind(tag=TAG).info(
+        "视觉分析接口是\thttp://{}:{}/mcp/vision/explain",
+        get_local_ip(),
+        port,
+    )
+    mcp_endpoint = config.get("mcp_endpoint", None)
+    if mcp_endpoint is not None and "你" not in mcp_endpoint:
+        # 校验MCP接入点格式
+        if validate_mcp_endpoint(mcp_endpoint):
+            logger.bind(tag=TAG).info("mcp接入点是\t{}", mcp_endpoint)
+            # 将mcp计入点地址转成调用点
+            mcp_endpoint = mcp_endpoint.replace("/mcp/", "/call/")
+            config["mcp_endpoint"] = mcp_endpoint
+        else:
+            logger.bind(tag=TAG).error("mcp接入点不符合规范")
+            config["mcp_endpoint"] = "你的接入点 websocket地址"
+
+    # 获取WebSocket配置，使用安全的默认值
+    websocket_port = 8000
+    server_config = config.get("server", {})
+    if isinstance(server_config, dict):
+        websocket_port = int(server_config.get("port", 8000))
+
+    logger.bind(tag=TAG).info(
+        "Websocket地址是\tws://{}:{}/xiaozhi/v1/",
+        get_local_ip(),
+        websocket_port,
+    )
+
+    logger.bind(tag=TAG).info(
+        "=======上面的地址是websocket协议地址，请勿用浏览器访问======="
+    )
+    logger.bind(tag=TAG).info(
+        "如想测试websocket请用谷歌浏览器打开test目录下的test_page.html"
+    )
+    logger.bind(tag=TAG).info(
+        "=============================================================\n"
+    )
+
+    try:
+        await wait_for_exit()  # 阻塞直到收到退出信号
+    except asyncio.CancelledError:
+        print("任务被取消，清理资源中...")
+    finally:
+        # 停止全局GC管理器
+        await gc_manager.stop()
+
+        # 取消所有任务（关键修复点）
+        stdin_task.cancel()
+        ws_task.cancel()
+        if ota_task:
+            ota_task.cancel()
+
+        # 等待任务终止（必须加超时）
+        await asyncio.wait(
+            [stdin_task, ws_task, ota_task] if ota_task else [stdin_task, ws_task],
+            timeout=3.0,
+            return_when=asyncio.ALL_COMPLETED,
+        )
+        print("服务器已关闭，程序退出。")
+
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("手动中断，程序终止。")

main/xingxing-server/config/config_loader.py

@@ -0,0 +1,162 @@
+import os
+import yaml
+from collections.abc import Mapping
+from config.manage_api_client import init_service, get_server_config, get_agent_models
+
+
+def get_project_dir():
+    """获取项目根目录"""
+    return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/"
+
+
+def read_config(config_path):
+    with open(config_path, "r", encoding="utf-8") as file:
+        config = yaml.safe_load(file)
+    return config
+
+
+def load_config():
+    """加载配置文件"""
+    from core.utils.cache.manager import cache_manager, CacheType
+
+    # 检查缓存
+    cached_config = cache_manager.get(CacheType.CONFIG, "main_config")
+    if cached_config is not None:
+        return cached_config
+
+    default_config_path = get_project_dir() + "config.yaml"
+    custom_config_path = get_project_dir() + "data/.config.yaml"
+
+    # 加载默认配置
+    default_config = read_config(default_config_path)
+    custom_config = read_config(custom_config_path)
+
+    if custom_config.get("manager-api", {}).get("url"):
+        import asyncio
+        try:
+            loop = asyncio.get_running_loop()
+            # 如果已经在事件循环中，使用异步版本
+            config = asyncio.run_coroutine_threadsafe(
+                get_config_from_api_async(custom_config), loop
+            ).result()
+        except RuntimeError:
+            # 如果不在事件循环中（启动时），创建新的事件循环
+            config = asyncio.run(get_config_from_api_async(custom_config))
+    else:
+        # 合并配置
+        config = merge_configs(default_config, custom_config)
+    # 初始化目录
+    ensure_directories(config)
+
+    # 缓存配置
+    cache_manager.set(CacheType.CONFIG, "main_config", config)
+    return config
+
+
+async def get_config_from_api_async(config):
+    """从Java API获取配置（异步版本）"""
+    # 初始化API客户端
+    init_service(config)
+
+    # 获取服务器配置
+    config_data = await get_server_config()
+    if config_data is None:
+        raise Exception("Failed to fetch server config from API")
+
+    config_data["read_config_from_api"] = True
+    config_data["manager-api"] = {
+        "url": config["manager-api"].get("url", ""),
+        "secret": config["manager-api"].get("secret", ""),
+    }
+    auth_enabled = config_data.get("server", {}).get("auth", {}).get("enabled", False)
+    # server的配置以本地为准
+    if config.get("server"):
+        config_data["server"] = {
+            "ip": config["server"].get("ip", ""),
+            "port": config["server"].get("port", ""),
+            "http_port": config["server"].get("http_port", ""),
+            "vision_explain": config["server"].get("vision_explain", ""),
+            "auth_key": config["server"].get("auth_key", ""),
+        }
+    config_data["server"]["auth"] = {"enabled": auth_enabled}
+    # 如果服务器没有prompt_template，则从本地配置读取
+    if not config_data.get("prompt_template"):
+        config_data["prompt_template"] = config.get("prompt_template")
+    return config_data
+
+
+async def get_private_config_from_api(config, device_id, client_id):
+    """从Java API获取私有配置"""
+    return await get_agent_models(device_id, client_id, config["selected_module"])
+
+
+def ensure_directories(config):
+    """确保所有配置路径存在"""
+    dirs_to_create = set()
+    project_dir = get_project_dir()  # 获取项目根目录
+    # 日志文件目录
+    log_dir = config.get("log", {}).get("log_dir", "tmp")
+    dirs_to_create.add(os.path.join(project_dir, log_dir))
+
+    # ASR/TTS模块输出目录
+    for module in ["ASR", "TTS"]:
+        if config.get(module) is None:
+            continue
+        for provider in config.get(module, {}).values():
+            output_dir = provider.get("output_dir", "")
+            if output_dir:
+                dirs_to_create.add(output_dir)
+
+    # 根据selected_module创建模型目录
+    selected_modules = config.get("selected_module", {})
+    for module_type in ["ASR", "LLM", "TTS"]:
+        selected_provider = selected_modules.get(module_type)
+        if not selected_provider:
+            continue
+        if config.get(module) is None:
+            continue
+        if config.get(selected_provider) is None:
+            continue
+        provider_config = config.get(module_type, {}).get(selected_provider, {})
+        output_dir = provider_config.get("output_dir")
+        if output_dir:
+            full_model_dir = os.path.join(project_dir, output_dir)
+            dirs_to_create.add(full_model_dir)
+
+    # 统一创建目录（保留原data目录创建）
+    for dir_path in dirs_to_create:
+        try:
+            os.makedirs(dir_path, exist_ok=True)
+        except PermissionError:
+            print(f"警告：无法创建目录 {dir_path}，请检查写入权限")
+
+
+def merge_configs(default_config, custom_config):
+    """
+    递归合并配置，custom_config优先级更高
+
+    Args:
+        default_config: 默认配置
+        custom_config: 用户自定义配置
+
+    Returns:
+        合并后的配置
+    """
+    if not isinstance(default_config, Mapping) or not isinstance(
+        custom_config, Mapping
+    ):
+        return custom_config
+
+    merged = dict(default_config)
+
+    for key, value in custom_config.items():
+        if (
+            key in merged
+            and isinstance(merged[key], Mapping)
+            and isinstance(value, Mapping)
+        ):
+            merged[key] = merge_configs(merged[key], value)
+        else:
+            merged[key] = value
+
+    return merged

main/xingxing-server/config/logger.py

@@ -0,0 +1,114 @@
+import os
+import sys
+from loguru import logger
+from config.config_loader import load_config
+from config.settings import check_config_file
+from datetime import datetime
+
+SERVER_VERSION = "0.9.2"
+_logger_initialized = False
+
+
+def get_module_abbreviation(module_name, module_dict):
+    """获取模块名称的缩写，如果为空则返回00
+    如果名称中包含下划线，则返回下划线后面的前两个字符
+    """
+    module_value = module_dict.get(module_name, "")
+    if not module_value:
+        return "00"
+    if "_" in module_value:
+        parts = module_value.split("_")
+        return parts[-1][:2] if parts[-1] else "00"
+    return module_value[:2]
+
+
+def build_module_string(selected_module):
+    """构建模块字符串"""
+    return (
+        get_module_abbreviation("VAD", selected_module)
+        + get_module_abbreviation("ASR", selected_module)
+        + get_module_abbreviation("LLM", selected_module)
+        + get_module_abbreviation("TTS", selected_module)
+        + get_module_abbreviation("Memory", selected_module)
+        + get_module_abbreviation("Intent", selected_module)
+        + get_module_abbreviation("VLLM", selected_module)
+    )
+
+
+def formatter(record):
+    """为没有 tag 的日志添加默认值，并处理动态模块字符串"""
+    record["extra"].setdefault("tag", record["name"])
+    # 如果没有设置 selected_module，使用默认值
+    record["extra"].setdefault("selected_module", "00000000000000")
+    # 将 selected_module 从 extra 提取到顶级，以支持 {selected_module} 格式
+    record["selected_module"] = record["extra"]["selected_module"]
+    return record["message"]
+
+
+def setup_logging():
+    check_config_file()
+    """从配置文件中读取日志配置，并设置日志输出格式和级别"""
+    config = load_config()
+    log_config = config["log"]
+    global _logger_initialized
+
+    # 第一次初始化时配置日志
+    if not _logger_initialized:
+        # 使用默认的模块字符串进行初始化
+        logger.configure(
+            extra={
+                "selected_module": log_config.get("selected_module", "00000000000000"),
+            }
+        )
+
+        log_format = log_config.get(
+            "log_format",
+            "<green>{time:YYMMDD HH:mm:ss}</green>[{version}_{extra[selected_module]}][<light-blue>{extra[tag]}</light-blue>]-<level>{level}</level>-<light-green>{message}</light-green>",
+        )
+        log_format_file = log_config.get(
+            "log_format_file",
+            "{time:YYYY-MM-DD HH:mm:ss} - {version}_{extra[selected_module]} - {name} - {level} - {extra[tag]} - {message}",
+        )
+        log_format = log_format.replace("{version}", SERVER_VERSION)
+        log_format_file = log_format_file.replace("{version}", SERVER_VERSION)
+
+        log_level = log_config.get("log_level", "INFO")
+        log_dir = log_config.get("log_dir", "tmp")
+        log_file = log_config.get("log_file", "server.log")
+        data_dir = log_config.get("data_dir", "data")
+
+        os.makedirs(log_dir, exist_ok=True)
+        os.makedirs(data_dir, exist_ok=True)
+
+        # 配置日志输出
+        logger.remove()
+
+        # 输出到控制台
+        logger.add(sys.stdout, format=log_format, level=log_level, filter=formatter)
+
+        # 输出到文件 - 统一目录，按大小轮转
+        # 日志文件完整路径
+        log_file_path = os.path.join(log_dir, log_file)
+
+        # 添加日志处理器
+        logger.add(
+            log_file_path,
+            format=log_format_file,
+            level=log_level,
+            filter=formatter,
+            rotation="10 MB",  # 每个文件最大10MB
+            retention="30 days",  # 保留30天
+            compression=None,
+            encoding="utf-8",
+            enqueue=True,  # 异步安全
+            backtrace=True,
+            diagnose=True,
+        )
+        _logger_initialized = True  # 标记为已初始化
+
+    return logger
+
+
+def create_connection_logger(selected_module_str):
+    """为连接创建独立的日志器，绑定特定的模块字符串"""
+    return logger.bind(selected_module=selected_module_str)

main/xingxing-server/config/manage_api_client.py

@@ -0,0 +1,245 @@
+import os
+import base64
+from typing import Optional, Dict
+
+import httpx
+
+TAG = __name__
+
+
+class DeviceNotFoundException(Exception):
+    pass
+
+
+class DeviceBindException(Exception):
+    def __init__(self, bind_code):
+        self.bind_code = bind_code
+        super().__init__(f"设备绑定异常，绑定码: {bind_code}")
+
+
+class ManageApiClient:
+    _instance = None
+    _async_clients = {}  # 为每个事件循环存储独立的客户端
+    _secret = None
+
+    def __new__(cls, config):
+        """单例模式确保全局唯一实例，并支持传入配置参数"""
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._init_client(config)
+        return cls._instance
+
+    @classmethod
+    def _init_client(cls, config):
+        """初始化配置（延迟创建客户端）"""
+        cls.config = config.get("manager-api")
+
+        if not cls.config:
+            raise Exception("manager-api配置错误")
+
+        if not cls.config.get("url") or not cls.config.get("secret"):
+            raise Exception("manager-api的url或secret配置错误")
+
+        if "你" in cls.config.get("secret"):
+            raise Exception("请先配置manager-api的secret")
+
+        cls._secret = cls.config.get("secret")
+        cls.max_retries = cls.config.get("max_retries", 6)  # 最大重试次数
+        cls.retry_delay = cls.config.get("retry_delay", 10)  # 初始重试延迟(秒)
+        # 不在这里创建 AsyncClient，延迟到实际使用时创建
+        cls._async_clients = {}
+
+    @classmethod
+    async def _ensure_async_client(cls):
+        """确保异步客户端已创建（为每个事件循环创建独立的客户端）"""
+        import asyncio
+
+        try:
+            loop = asyncio.get_running_loop()
+            loop_id = id(loop)
+
+            # 为每个事件循环创建独立的客户端
+            if loop_id not in cls._async_clients:
+                # 服务端可能主动关闭连接，httpx 连接池无法正确检测和清理
+                limits = httpx.Limits(
+                    max_keepalive_connections=0,  # 禁用 keep-alive，每次都新建连接
+                )
+                cls._async_clients[loop_id] = httpx.AsyncClient(
+                    base_url=cls.config.get("url"),
+                    headers={
+                        "User-Agent": f"PythonClient/2.0 (PID:{os.getpid()})",
+                        "Accept": "application/json",
+                        "Authorization": "Bearer " + cls._secret,
+                    },
+                    timeout=cls.config.get("timeout", 30),
+                    limits=limits,  # 使用限制
+                )
+            return cls._async_clients[loop_id]
+        except RuntimeError:
+            # 如果没有运行中的事件循环，创建一个临时的
+            raise Exception("必须在异步上下文中调用")
+
+    @classmethod
+    async def _async_request(cls, method: str, endpoint: str, **kwargs) -> Dict:
+        """发送单次异步HTTP请求并处理响应"""
+        # 确保客户端已创建
+        client = await cls._ensure_async_client()
+        endpoint = endpoint.lstrip("/")
+        response = None
+        try:
+            response = await client.request(method, endpoint, **kwargs)
+            response.raise_for_status()
+
+            result = response.json()
+
+            # 处理API返回的业务错误
+            if result.get("code") == 10041:
+                raise DeviceNotFoundException(result.get("msg"))
+            elif result.get("code") == 10042:
+                raise DeviceBindException(result.get("msg"))
+            elif result.get("code") != 0:
+                raise Exception(f"API返回错误: {result.get('msg', '未知错误')}")
+
+            # 返回成功数据
+            return result.get("data") if result.get("code") == 0 else None
+        finally:
+            # 确保响应被关闭（即使异常也会执行）
+            if response is not None:
+                await response.aclose()
+
+    @classmethod
+    def _should_retry(cls, exception: Exception) -> bool:
+        """判断异常是否应该重试"""
+        # 网络连接相关错误
+        if isinstance(
+            exception, (httpx.ConnectError, httpx.TimeoutException, httpx.NetworkError)
+        ):
+            return True
+
+        # HTTP状态码错误
+        if isinstance(exception, httpx.HTTPStatusError):
+            status_code = exception.response.status_code
+            return status_code in [408, 429, 500, 502, 503, 504]
+
+        return False
+
+    @classmethod
+    async def _execute_async_request(cls, method: str, endpoint: str, **kwargs) -> Dict:
+        """带重试机制的异步请求执行器"""
+        import asyncio
+
+        retry_count = 0
+
+        while retry_count <= cls.max_retries:
+            try:
+                # 执行异步请求
+                return await cls._async_request(method, endpoint, **kwargs)
+            except Exception as e:
+                # 判断是否应该重试
+                if retry_count < cls.max_retries and cls._should_retry(e):
+                    retry_count += 1
+                    print(
+                        f"{method} {endpoint} 异步请求失败，将在 {cls.retry_delay:.1f} 秒后进行第 {retry_count} 次重试"
+                    )
+                    await asyncio.sleep(cls.retry_delay)
+                    continue
+                else:
+                    # 不重试，直接抛出异常
+                    raise
+
+    @classmethod
+    def safe_close(cls):
+        """安全关闭所有异步连接池"""
+        import asyncio
+
+        for client in list(cls._async_clients.values()):
+            try:
+                asyncio.run(client.aclose())
+            except Exception:
+                pass
+        cls._async_clients.clear()
+        cls._instance = None
+
+
+async def get_server_config() -> Optional[Dict]:
+    """获取服务器基础配置"""
+    return await ManageApiClient._instance._execute_async_request(
+        "POST", "/config/server-base"
+    )
+
+
+async def get_agent_models(
+    mac_address: str, client_id: str, selected_module: Dict
+) -> Optional[Dict]:
+    """获取代理模型配置"""
+    return await ManageApiClient._instance._execute_async_request(
+        "POST",
+        "/config/agent-models",
+        json={
+            "macAddress": mac_address,
+            "clientId": client_id,
+            "selectedModule": selected_module,
+        },
+    )
+
+
+async def generate_and_save_chat_summary(session_id: str) -> Optional[Dict]:
+    """生成并保存聊天记录总结"""
+    if not ManageApiClient._instance:
+        return None
+    try:
+        return await ManageApiClient._instance._execute_async_request(
+            "POST",
+            f"/agent/chat-summary/{session_id}/save",
+        )
+    except Exception as e:
+        print(f"生成并保存聊天记录总结失败: {e}")
+        return None
+
+
+async def generate_and_save_chat_title(session_id: str) -> Optional[Dict]:
+    """生成并保存聊天标题"""
+    if not ManageApiClient._instance:
+        return None
+    try:
+        return await ManageApiClient._instance._execute_async_request(
+            "POST",
+            f"/agent/chat-title/{session_id}/generate",
+        )
+    except Exception as e:
+        print(f"生成并保存聊天标题失败: {e}")
+        return None
+
+
+async def report(
+    mac_address: str, session_id: str, chat_type: int, content: str, audio, report_time
+) -> Optional[Dict]:
+    """异步聊天记录上报"""
+    if not content or not ManageApiClient._instance:
+        return None
+    try:
+        return await ManageApiClient._instance._execute_async_request(
+            "POST",
+            f"/agent/chat-history/report",
+            json={
+                "macAddress": mac_address,
+                "sessionId": session_id,
+                "chatType": chat_type,
+                "content": content,
+                "reportTime": report_time,
+                "audioBase64": (
+                    base64.b64encode(audio).decode("utf-8") if audio else None
+                ),
+            },
+        )
+    except Exception as e:
+        print(f"TTS上报失败: {e}")
+        return None
+
+
+def init_service(config):
+    ManageApiClient(config)
+
+
+def manage_api_http_safe_close():
+    ManageApiClient.safe_close()

main/xingxing-server/config/settings.py

@@ -0,0 +1,33 @@
+import os
+from config.config_loader import read_config, get_project_dir, load_config
+
+
+default_config_file = "config.yaml"
+config_file_valid = False
+
+
+def check_config_file():
+    global config_file_valid
+    if config_file_valid:
+        return
+    """
+    简化的配置检查，仅提示用户配置文件的使用情况
+    """
+    custom_config_file = get_project_dir() + "data/." + default_config_file
+    if not os.path.exists(custom_config_file):
+        raise FileNotFoundError(
+            "找不到data/.config.yaml文件，请按教程确认该配置文件是否存在"
+        )
+
+    # 检查是否从API读取配置
+    config = load_config()
+    if config.get("read_config_from_api", False):
+        print("从API读取配置")
+        old_config_origin = read_config(custom_config_file)
+        if old_config_origin.get("selected_module") is not None:
+            error_msg = "您的配置文件好像既包含智控台的配置又包含本地配置：\n"
+            error_msg += "\n建议您：\n"
+            error_msg += "1、将根目录的config_from_api.yaml文件复制到data下，重命名为.config.yaml\n"
+            error_msg += "2、按教程配置好接口地址和密钥\n"
+            raise ValueError(error_msg)
+    config_file_valid = True

main/xingxing-server/config_from_api.yaml

@@ -0,0 +1,27 @@
+# 如果你只想轻量化安装xiaozhi-server，只使用本地的配置文件，不需要理会这个文件，不需要改动本文件任何东西
+# 如果你想从manager-api获取配置，请往下看：
+# 请将本文件复制到xiaozhi-server/data目录下，没有data目录，请创建一个，并将复制过去的文件命名为.config.yaml
+# 注意如果data目录有.config.yaml文件，请先删除它
+# 先启动manager-api和manager-web，注册一个账号，第一个注册的账号为管理员
+# 使用管理员，进入【参数管理】页面，找到【server.secret】，复制它到参数值，注意每次从零部署，server.secret都会变化
+# 打开本data目录下的.config.yaml文件，修改manager-api.secret为刚才复制出来的server.secret
+server:
+  ip: 0.0.0.0
+  port: 8000
+  # http服务的端口，用于视觉分析接口
+  http_port: 8003
+  # 视觉分析接口地址
+  # 向设备发送的视觉分析的接口地址
+  # 如果按下面默认的写法，系统会自动生成视觉识别地址，并输出在启动日志里，这个地址你可以直接用浏览器访问确认一下
+  # 当你使用docker部署或使用公网部署(使用ssl、域名)时，不一定准确
+  # 所以如果你使用docker部署时，将vision_explain设置成局域网地址
+  # 如果你使用公网部署时，将vision_explain设置成公网地址
+  vision_explain: http://你的ip或者域名:端口号/mcp/vision/explain
+manager-api:
+  # 你的manager-api的地址，最好使用局域网ip
+  # 如果使用docker部署，请使用填写成 http://xiaozhi-esp32-server-web:8002/xiaozhi
+  url: http://127.0.0.1:8002/xiaozhi
+  # 你的manager-api的token，就是刚才复制出来的server.secret
+  secret: 你的server.secret值
+# 默认系统提示词模板文件
+prompt_template: agent-base-prompt.txt

main/xingxing-server/core/api/app_gps_handler.py

@@ -0,0 +1,415 @@
+import asyncio
+import time
+
+from aiohttp import web
+
+from config.logger import setup_logging
+from core.api.base_handler import BaseHandler
+from core.providers.tools.device_mcp.mcp_handler import call_mcp_tool
+from plugins_func.nav_copilot.geo import haversine_m
+from plugins_func.nav_copilot.service import (
+    _parse_fix,
+    _pick_gnss_tool_name,
+    get_service,
+    normalize_latlon,
+    normalize_note,
+    normalize_coord_type,
+)
+from plugins_func.nav_copilot.storage import LineAlert, NavDB
+
+TAG = __name__
+
+
+class AppGpsHandler(BaseHandler):
+    def __init__(self, config: dict, ws_server):
+        super().__init__(config)
+        self.ws_server = ws_server
+        self.logger = setup_logging()
+        self._inflight_lock = asyncio.Lock()
+        self._inflight_tasks: dict[str, asyncio.Task] = {}
+        self._latest_payloads: dict[str, tuple[float, dict]] = {}
+        self._cache_ttl_s = 1.5
+        self._tool_timeout_s = 8
+        self._nav_db = NavDB.default()
+
+    async def _resolve_conn(self, request):
+        requested_device_id = request.query.get("device_id")
+        conn = await self.ws_server.get_connection(device_id=requested_device_id)
+        if conn is None:
+            return None, "no_active_device"
+        return conn, None
+
+    async def _resolve_target_device(self, request, body: dict | None = None):
+        requested_device_id = request.query.get("device_id")
+        if not requested_device_id and isinstance(body, dict):
+            requested_device_id = str(body.get("device_id") or "").strip()
+
+        if requested_device_id:
+            conn = await self.ws_server.get_connection(device_id=requested_device_id)
+            return requested_device_id, conn, None
+
+        conn, error_code = await self._resolve_conn(request)
+        if error_code:
+            return None, None, error_code
+        return self._device_key(conn), conn, None
+
+    async def _resolve_mcp_target(self, conn):
+        if not hasattr(conn, "mcp_client") or conn.mcp_client is None:
+            return None, "device_mcp_not_initialized"
+        if not await conn.mcp_client.is_ready():
+            return None, "device_mcp_not_ready"
+        tool_name = _pick_gnss_tool_name(conn)
+        if not tool_name:
+            return None, "device_gnss_tool_not_found"
+        return tool_name, None
+
+    @staticmethod
+    def _device_key(conn) -> str:
+        return str(getattr(conn, "device_id", "") or "default")
+
+    @staticmethod
+    def _serialize_alert(alert: LineAlert) -> dict:
+        return {
+            "id": alert.id,
+            "device_id": alert.device_id,
+            "start_lat": alert.start_lat,
+            "start_lon": alert.start_lon,
+            "end_lat": alert.end_lat,
+            "end_lon": alert.end_lon,
+            "coord_type": alert.coord_type,
+            "note": alert.note,
+            "created_at": alert.created_at,
+        }
+
+    def _parse_alert_payload(self, body: dict) -> tuple[dict | None, str | None]:
+        try:
+            start_lat, start_lon = normalize_latlon(
+                body.get("start_lat"),
+                body.get("start_lon"),
+            )
+            end_lat, end_lon = normalize_latlon(
+                body.get("end_lat"),
+                body.get("end_lon"),
+            )
+            coord_type = normalize_coord_type(body.get("coord_type"), default="BD09")
+            note = normalize_note(str(body.get("note") or ""))
+        except ValueError as exc:
+            return None, str(exc)
+
+        if haversine_m(start_lat, start_lon, end_lat, end_lon) < 1.0:
+            return None, "line_too_short"
+
+        return {
+            "start_lat": start_lat,
+            "start_lon": start_lon,
+            "end_lat": end_lat,
+            "end_lon": end_lon,
+            "coord_type": coord_type,
+            "note": note,
+        }, None
+
+    async def _fetch_fix_payload(self, conn, tool_name: str) -> dict:
+        raw = await call_mcp_tool(
+            conn,
+            conn.mcp_client,
+            tool_name,
+            "{}",
+            timeout=self._tool_timeout_s,
+        )
+
+        fix = _parse_fix(raw)
+        if not fix.ok:
+            return {
+                "ok": False,
+                "device_id": getattr(conn, "device_id", None),
+                "reason": "no_fix",
+            }
+
+        payload = {
+            "ok": True,
+            "device_id": getattr(conn, "device_id", None),
+            "lat": fix.lat,
+            "lon": fix.lon,
+            "speed_mps": fix.speed_mps,
+            "heading_deg": fix.heading_deg,
+            "timestamp": fix.t,
+            "coord_type": "WGS84",
+            "source": "device_mcp_gnss",
+        }
+        self._latest_payloads[self._device_key(conn)] = (time.monotonic(), payload)
+        return payload
+
+    async def _get_or_start_inflight(self, device_key: str, conn, tool_name: str):
+        async with self._inflight_lock:
+            task = self._inflight_tasks.get(device_key)
+            if task is None or task.done():
+                task = asyncio.create_task(self._fetch_fix_payload(conn, tool_name))
+                self._inflight_tasks[device_key] = task
+            return task
+
+    async def _get_fix_payload(self, request):
+        conn, error_code = await self._resolve_conn(request)
+        if error_code:
+            return None, error_code
+
+        device_key = self._device_key(conn)
+
+        live_payload, _ = await self.ws_server.get_latest_gps(
+            device_key, max_age_s=3.0, allow_stale=False
+        )
+        if live_payload is not None:
+            return live_payload, None
+
+        try:
+            nav_service = get_service(conn)
+            if nav_service.is_monitoring_active():
+                cached_gps = nav_service.get_cached_gps()
+                if cached_gps is not None:
+                    return cached_gps, None
+        except Exception:
+            pass
+
+        cached = self._latest_payloads.get(device_key)
+        now = time.monotonic()
+        if cached and (now - cached[0]) <= self._cache_ttl_s:
+            return cached[1], None
+
+        try:
+            tool_name, error_code = await self._resolve_mcp_target(conn)
+            if error_code:
+                stale_payload, _ = await self.ws_server.get_latest_gps(
+                    device_key, max_age_s=3.0, allow_stale=True
+                )
+                if stale_payload is not None:
+                    return stale_payload, None
+                return None, error_code
+            task = await self._get_or_start_inflight(device_key, conn, tool_name)
+            payload = await task
+            return payload, None
+        except Exception as exc:
+            self.logger.bind(tag=TAG).warning(f"app gps tool call failed: {exc}")
+            stale_payload, _ = await self.ws_server.get_latest_gps(
+                device_key, max_age_s=3.0, allow_stale=True
+            )
+            if stale_payload is not None:
+                return stale_payload, None
+            cached = self._latest_payloads.get(device_key)
+            if cached:
+                stale_payload = dict(cached[1])
+                stale_payload["stale"] = True
+                return stale_payload, None
+            return None, "device_gnss_call_failed"
+        finally:
+            async with self._inflight_lock:
+                task = self._inflight_tasks.get(device_key)
+                if task is not None and task.done():
+                    self._inflight_tasks.pop(device_key, None)
+
+    def _json_response(self, payload: dict, status: int = 200):
+        response = web.json_response(payload, status=status)
+        self._add_cors_headers(response)
+        return response
+
+    async def handle_get_latest(self, request):
+        payload, error_code = await self._get_fix_payload(request)
+        if error_code:
+            return self._json_response(
+                {
+                    "ok": False,
+                    "error": error_code,
+                    "active_devices": await self.ws_server.list_active_device_ids(),
+                },
+                status=503,
+            )
+        return self._json_response(payload)
+
+    async def handle_get_risk_alerts(self, request):
+        device_id, _, error_code = await self._resolve_target_device(request)
+        if error_code:
+            return self._json_response(
+                {
+                    "ok": False,
+                    "error": error_code,
+                    "active_devices": await self.ws_server.list_active_device_ids(),
+                },
+                status=503,
+            )
+
+        alerts = self._nav_db.list_line_alerts(device_id)
+        return self._json_response(
+            {
+                "ok": True,
+                "device_id": device_id,
+                "alerts": [self._serialize_alert(alert) for alert in alerts],
+            }
+        )
+
+    async def handle_post_risk_alerts(self, request):
+        try:
+            body = await request.json()
+        except Exception:
+            return self._json_response({"ok": False, "error": "invalid_json"}, status=400)
+
+        payload, error_code = self._parse_alert_payload(body)
+        if error_code:
+            return self._json_response({"ok": False, "error": error_code}, status=400)
+
+        device_id, conn, error_code = await self._resolve_target_device(request, body)
+        if error_code:
+            return self._json_response(
+                {
+                    "ok": False,
+                    "error": error_code,
+                    "active_devices": await self.ws_server.list_active_device_ids(),
+                },
+                status=503,
+            )
+
+        try:
+            if conn is not None:
+                service = get_service(conn)
+                alert = await service.create_line_alert(device_id=device_id, **payload)
+                monitoring_active = service.is_monitoring_active()
+            else:
+                alert_id = self._nav_db.create_line_alert(device_id=device_id, **payload)
+                alert = self._nav_db.get_line_alert(alert_id, device_id=device_id)
+                monitoring_active = False
+            assert alert is not None
+        except ValueError as exc:
+            return self._json_response({"ok": False, "error": str(exc)}, status=400)
+        except Exception as exc:
+            self.logger.bind(tag=TAG).warning(f"create risk alert failed: {exc}")
+            return self._json_response({"ok": False, "error": "create_failed"}, status=500)
+
+        return self._json_response(
+            {
+                "ok": True,
+                "device_id": device_id,
+                "monitoring_active": monitoring_active,
+                "alert": self._serialize_alert(alert),
+            },
+            status=201,
+        )
+
+    async def handle_delete_risk_alert(self, request):
+        alert_id_raw = request.match_info.get("alert_id", "")
+        try:
+            alert_id = int(alert_id_raw)
+        except ValueError:
+            return self._json_response({"ok": False, "error": "invalid_alert_id"}, status=400)
+
+        device_id, conn, error_code = await self._resolve_target_device(request)
+        if error_code:
+            return self._json_response(
+                {
+                    "ok": False,
+                    "error": error_code,
+                    "active_devices": await self.ws_server.list_active_device_ids(),
+                },
+                status=503,
+            )
+
+        try:
+            if conn is not None:
+                service = get_service(conn)
+                deleted = await service.delete_line_alert(device_id, alert_id)
+            else:
+                deleted = self._nav_db.delete_line_alert(alert_id, device_id=device_id)
+        except Exception as exc:
+            self.logger.bind(tag=TAG).warning(f"delete risk alert failed: {exc}")
+            return self._json_response({"ok": False, "error": "delete_failed"}, status=500)
+
+        if not deleted:
+            return self._json_response({"ok": False, "error": "not_found"}, status=404)
+
+        return self._json_response(
+            {
+                "ok": True,
+                "device_id": device_id,
+                "alert_id": alert_id,
+                "deleted": True,
+            }
+        )
+
+    async def handle_ws(self, request):
+        ws = web.WebSocketResponse(heartbeat=30)
+        await ws.prepare(request)
+
+        interval_s = 1.0
+        interval_arg = request.query.get("interval")
+        if interval_arg:
+            try:
+                interval_s = max(0.2, float(interval_arg))
+            except ValueError:
+                interval_s = 1.0
+
+        gps_queue = None
+        subscribed_device_key = None
+
+        try:
+            while True:
+                if gps_queue is None:
+                    conn, error_code = await self._resolve_conn(request)
+                    if error_code:
+                        await ws.send_json(
+                            {
+                                "ok": False,
+                                "error": error_code,
+                                "active_devices": await self.ws_server.list_active_device_ids(),
+                            }
+                        )
+                        await asyncio.sleep(1.0)
+                        continue
+
+                    subscribed_device_key = self._device_key(conn)
+                    gps_queue = await self.ws_server.subscribe_device_gps(subscribed_device_key)
+
+                    current_payload, _ = await self.ws_server.get_latest_gps(
+                        subscribed_device_key, max_age_s=3.0, allow_stale=True
+                    )
+                    if current_payload is not None:
+                        await ws.send_json(current_payload)
+                    else:
+                        payload, error_code = await self._get_fix_payload(request)
+                        if error_code:
+                            await ws.send_json(
+                                {
+                                    "ok": False,
+                                    "error": error_code,
+                                    "active_devices": await self.ws_server.list_active_device_ids(),
+                                }
+                            )
+                        else:
+                            await ws.send_json(payload)
+
+                try:
+                    payload = await asyncio.wait_for(gps_queue.get(), timeout=30.0)
+                    await ws.send_json(payload)
+                except asyncio.TimeoutError:
+                    payload, error_code = await self._get_fix_payload(request)
+                    if error_code:
+                        await ws.send_json(
+                            {
+                                "ok": False,
+                                "error": error_code,
+                                "active_devices": await self.ws_server.list_active_device_ids(),
+                            }
+                        )
+                        if error_code == "no_active_device" and gps_queue is not None and subscribed_device_key is not None:
+                            await self.ws_server.unsubscribe_device_gps(subscribed_device_key, gps_queue)
+                            gps_queue = None
+                            subscribed_device_key = None
+                            await asyncio.sleep(interval_s)
+                    else:
+                        await ws.send_json(payload)
+
+        except asyncio.CancelledError:
+            raise
+        except Exception as exc:
+            self.logger.bind(tag=TAG).warning(f"app gps websocket closed: {exc}")
+        finally:
+            if gps_queue is not None and subscribed_device_key is not None:
+                await self.ws_server.unsubscribe_device_gps(subscribed_device_key, gps_queue)
+            await ws.close()
+
+        return ws

main/xingxing-server/core/api/base_handler.py

@@ -0,0 +1,22 @@
+from aiohttp import web
+
+from config.logger import setup_logging
+
+
+class BaseHandler:
+    def __init__(self, config: dict):
+        self.config = config
+        self.logger = setup_logging()
+
+    def _add_cors_headers(self, response):
+        response.headers["Access-Control-Allow-Headers"] = (
+            "client-id, content-type, device-id, authorization"
+        )
+        response.headers["Access-Control-Allow-Credentials"] = "true"
+        response.headers["Access-Control-Allow-Origin"] = "*"
+
+    async def handle_options(self, request):
+        response = web.Response(body=b"", content_type="text/plain")
+        self._add_cors_headers(response)
+        response.headers["Access-Control-Allow-Methods"] = "GET, POST, DELETE, OPTIONS"
+        return response

main/xingxing-server/core/api/ota_handler.py

@@ -0,0 +1,415 @@
+import json
+import time
+import base64
+import hashlib
+import hmac
+import os
+import re
+import glob
+from typing import Dict, List, Tuple
+from aiohttp import web
+
+from core.auth import AuthManager
+from core.utils.util import get_local_ip, get_vision_url
+from core.api.base_handler import BaseHandler
+
+TAG = __name__
+
+
+def _safe_basename(filename: str) -> str:
+    # Prevent directory traversal
+    return os.path.basename(filename)
+
+
+def _parse_version(ver: str) -> Tuple[int, ...]:
+    # conservative parser: split by non-digit, keep numeric parts
+    parts = re.findall(r"\d+", ver)
+    return tuple(int(p) for p in parts) if parts else (0,)
+
+
+def _is_higher_version(a: str, b: str) -> bool:
+    """Return True if version string a > b (semver-like numeric compare)."""
+    ta = _parse_version(a)
+    tb = _parse_version(b)
+    # compare tuple lexicographically, but allow different lengths
+    maxlen = max(len(ta), len(tb))
+    for i in range(maxlen):
+        ai = ta[i] if i < len(ta) else 0
+        bi = tb[i] if i < len(tb) else 0
+        if ai > bi:
+            return True
+        if ai < bi:
+            return False
+    return False
+
+
+class OTAHandler(BaseHandler):
+    def __init__(self, config: dict):
+        super().__init__(config)
+        auth_config = config["server"].get("auth", {})
+        self.auth_enable = auth_config.get("enabled", False)
+        # 设备白名单
+        self.allowed_devices = set(auth_config.get("allowed_devices", []))
+        secret_key = config["server"]["auth_key"]
+        expire_seconds = auth_config.get("expire_seconds")
+        self.auth = AuthManager(secret_key=secret_key, expire_seconds=expire_seconds)
+
+        # firmware storage
+        self.bin_dir = os.path.join(os.getcwd(), "data", "bin")
+        # cache structure: { 'updated_at': timestamp, 'ttl': seconds, 'files_by_model': { model: [(version, filename), ...] } }
+        self._bin_cache: Dict = {
+            "updated_at": 0,
+            "ttl": config.get("firmware_cache_ttl", 30),
+            "files_by_model": {},
+        }
+
+    def _refresh_bin_cache_if_needed(self):
+        now = int(time.time())
+        ttl = int(self._bin_cache.get("ttl", 30))
+        if now - int(
+            self._bin_cache.get("updated_at", 0)
+        ) < ttl and self._bin_cache.get("files_by_model"):
+            return
+
+        files_by_model: Dict[str, List[Tuple[str, str]]] = {}
+        try:
+            if not os.path.isdir(self.bin_dir):
+                os.makedirs(self.bin_dir, exist_ok=True)
+
+            # match files like model_1.2.3.bin (allow dots, dashes, underscores in model and version)
+            pattern = os.path.join(self.bin_dir, "*.bin")
+            for path in glob.glob(pattern):
+                fname = os.path.basename(path)
+                # filename format: {model}_{version}.bin
+                m = re.match(r"^(.+?)_([0-9][A-Za-z0-9\.\-_]*)\.bin$", fname)
+                if not m:
+                    # skip files not conforming to naming rule
+                    continue
+                model = m.group(1)
+                version = m.group(2)
+                files_by_model.setdefault(model, []).append((version, fname))
+
+            # sort versions for each model descending
+            for model, items in files_by_model.items():
+                items.sort(key=lambda it: _parse_version(it[0]), reverse=True)
+
+            self._bin_cache["files_by_model"] = files_by_model
+            self._bin_cache["updated_at"] = now
+            self.logger.bind(tag=TAG).info(
+                f"Firmware cache refreshed: {len(files_by_model)} models"
+            )
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"刷新固件缓存失败: {e}")
+            # keep previous cache if any
+
+    def generate_password_signature(self, content: str, secret_key: str) -> str:
+        """生成MQTT密码签名
+
+        Args:
+            content: 签名内容 (clientId + '|' + username)
+            secret_key: 密钥
+
+        Returns:
+            str: Base64编码的HMAC-SHA256签名
+        """
+        try:
+            hmac_obj = hmac.new(
+                secret_key.encode("utf-8"), content.encode("utf-8"), hashlib.sha256
+            )
+            signature = hmac_obj.digest()
+            return base64.b64encode(signature).decode("utf-8")
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"生成MQTT密码签名失败: {e}")
+            return ""
+
+    def _get_websocket_url(self, local_ip: str, port: int) -> str:
+        """获取websocket地址
+
+        Args:
+            local_ip: 本地IP地址
+            port: 端口号
+
+        Returns:
+            str: websocket地址
+        """
+        server_config = self.config["server"]
+        websocket_config = server_config.get("websocket", "")
+
+        if "你的" not in websocket_config:
+            return websocket_config
+        else:
+            return f"ws://{local_ip}:{port}/xiaozhi/v1/"
+
+    async def handle_post(self, request):
+        """处理 OTA POST 请求
+
+        This handler will:
+        - read device id/client id (as before)
+        - attempt to determine device model and current firmware version (prefer headers, fallback to body)
+        - check data/bin for newer firmware for that model
+        - if found a newer firmware, set firmware.url to the download endpoint
+        """
+        try:
+            data = await request.text()
+            self.logger.bind(tag=TAG).debug(f"OTA请求方法: {request.method}")
+            self.logger.bind(tag=TAG).debug(f"OTA请求头: {request.headers}")
+            self.logger.bind(tag=TAG).debug(f"OTA请求数据: {data}")
+
+            device_id = request.headers.get("device-id", "")
+            if device_id:
+                self.logger.bind(tag=TAG).info(f"OTA请求设备ID: {device_id}")
+            else:
+                raise Exception("OTA请求设备ID为空")
+
+            client_id = request.headers.get("client-id", "")
+            if client_id:
+                self.logger.bind(tag=TAG).info(f"OTA请求ClientID: {client_id}")
+            else:
+                raise Exception("OTA请求ClientID为空")
+
+            data_json = {}
+            try:
+                data_json = json.loads(data) if data else {}
+            except Exception:
+                data_json = {}
+
+            server_config = self.config["server"]
+            # Distinguish ports:
+            # - websocket_port is used to construct websocket URL (server["port"])
+            # - http_port is used to construct OTA download URLs (server["http_port"])
+            websocket_port = int(server_config.get("port", 8000))
+            http_port = int(server_config.get("http_port", 8003))
+            local_ip = get_local_ip()
+
+            # Determine device model (prefer headers)
+            device_model = ""
+            # header candidates
+            for h in ("device-model", "device_model", "model"):
+                if h in request.headers:
+                    device_model = request.headers.get(h, "").strip()
+                    break
+            # body fallback
+            if not device_model:
+                try:
+                    if "board" in data_json and isinstance(data_json["board"], dict):
+                        device_model = data_json["board"].get("type", "")
+                    elif "model" in data_json:
+                        device_model = data_json.get("model", "")
+                except Exception:
+                    device_model = ""
+            if not device_model:
+                device_model = "default"
+
+            # Determine device current version (prefer headers)
+            device_version = ""
+            for h in (
+                "device-version",
+                "device_version",
+                "firmware-version",
+                "app-version",
+                "application-version",
+            ):
+                if h in request.headers:
+                    device_version = request.headers.get(h, "").strip()
+                    break
+            if not device_version:
+                try:
+                    device_version = data_json.get("application", {}).get("version", "")
+                except Exception:
+                    device_version = ""
+            if not device_version:
+                device_version = "0.0.0"
+
+            return_json = {
+                "server_time": {
+                    "timestamp": int(round(time.time() * 1000)),
+                    "timezone_offset": server_config.get("timezone_offset", 8) * 60,
+                },
+                "firmware": {
+                    "version": device_version,
+                    "url": "",
+                },
+            }
+
+            # existing mqtt/websocket logic (unchanged)
+            mqtt_gateway_endpoint = server_config.get("mqtt_gateway")
+
+            if mqtt_gateway_endpoint:  # 如果配置了非空字符串
+                # 尝试从请求数据中获取设备型号（已解析 above）
+                try:
+                    group_id = f"GID_{device_model}".replace(":", "_").replace(" ", "_")
+                except Exception as e:
+                    self.logger.bind(tag=TAG).error(f"获取设备型号失败: {e}")
+                    group_id = "GID_default"
+
+                mac_address_safe = device_id.replace(":", "_")
+                mqtt_client_id = f"{group_id}@@@{mac_address_safe}@@@{mac_address_safe}"
+
+                # 构建用户数据
+                user_data = {"ip": "unknown"}
+                try:
+                    user_data_json = json.dumps(user_data)
+                    username = base64.b64encode(user_data_json.encode("utf-8")).decode(
+                        "utf-8"
+                    )
+                except Exception as e:
+                    self.logger.bind(tag=TAG).error(f"生成用户名失败: {e}")
+                    username = ""
+
+                # 生成密码
+                password = ""
+                signature_key = server_config.get("mqtt_signature_key", "")
+                if signature_key:
+                    password = self.generate_password_signature(
+                        mqtt_client_id + "|" + username, signature_key
+                    )
+                    if not password:
+                        password = ""  # 签名失败则留空，由设备决定是否允许无密码
+                else:
+                    self.logger.bind(tag=TAG).warning("缺少MQTT签名密钥，密码留空")
+
+                # 构建MQTT配置（直接使用 mqtt_gateway 字符串）
+                return_json["mqtt"] = {
+                    "endpoint": mqtt_gateway_endpoint,
+                    "client_id": mqtt_client_id,
+                    "username": username,
+                    "password": password,
+                    "publish_topic": "device-server",
+                    "subscribe_topic": f"devices/p2p/{mac_address_safe}",
+                }
+                self.logger.bind(tag=TAG).info(f"为设备 {device_id} 下发MQTT网关配置")
+
+            else:  # 未配置 mqtt_gateway，下发 WebSocket
+                # 如果开启了认证，则进行认证校验
+                token = ""
+                if self.auth_enable:
+                    if self.allowed_devices:
+                        if device_id not in self.allowed_devices:
+                            token = self.auth.generate_token(client_id, device_id)
+                    else:
+                        token = self.auth.generate_token(client_id, device_id)
+                # NOTE: use websocket_port here
+                return_json["websocket"] = {
+                    "url": self._get_websocket_url(local_ip, websocket_port),
+                    "token": token,
+                }
+                self.logger.bind(tag=TAG).info(
+                    f"未配置MQTT网关，为设备 {device_id} 下发WebSocket配置"
+                )
+
+            # Now check firmware files for updates
+            try:
+                self._refresh_bin_cache_if_needed()
+                files_by_model = self._bin_cache.get("files_by_model", {})
+                candidates = files_by_model.get(device_model, [])
+
+                self.logger.bind(tag=TAG).info(
+                    f"查找型号 {device_model} 的固件，找到 {len(candidates)} 个候选"
+                )
+
+                chosen_url = ""
+                chosen_version = device_version
+
+                # candidates are sorted descending by version
+                for ver, fname in candidates:
+                    if _is_higher_version(ver, device_version):
+                        # build download url (only allow download via our download endpoint)
+                        chosen_version = ver
+                        # Use get_vision_url to get the base URL and replace the path
+                        vision_url = get_vision_url(self.config)
+                        # Replace the path from "/mcp/vision/explain" to "/xiaozhi/ota/download/{fname}"
+                        chosen_url = vision_url.replace(
+                            "/mcp/vision/explain", f"/xiaozhi/ota/download/{fname}"
+                        )
+                        break
+
+                if chosen_url:
+                    return_json["firmware"]["version"] = chosen_version
+                    return_json["firmware"]["url"] = chosen_url
+                    self.logger.bind(tag=TAG).info(
+                        f"为设备 {device_id} 下发固件 {chosen_version} [如果地址前缀有误，请检查配置文件中的server.vision_explain]-> {chosen_url} "
+                    )
+                else:
+                    self.logger.bind(tag=TAG).info(
+                        f"设备 {device_id} 固件已是最新: {device_version}"
+                    )
+
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"检查固件版本时出错: {e}")
+
+            response = web.Response(
+                text=json.dumps(return_json, separators=(",", ":")),
+                content_type="application/json",
+            )
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"OTA POST处理异常: {e}")
+            return_json = {"success": False, "message": "request error."}
+            response = web.Response(
+                text=json.dumps(return_json, separators=(",", ":")),
+                content_type="application/json",
+            )
+        finally:
+            self._add_cors_headers(response)
+            return response
+
+    async def handle_get(self, request):
+        """处理 OTA GET 请求"""
+        try:
+            server_config = self.config["server"]
+            local_ip = get_local_ip()
+            # use websocket port for websocket URL
+            websocket_port = int(server_config.get("port", 8000))
+            websocket_url = self._get_websocket_url(local_ip, websocket_port)
+            message = f"OTA接口运行正常，向设备发送的websocket地址是：{websocket_url}"
+            response = web.Response(text=message, content_type="text/plain")
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"OTA GET请求异常: {e}")
+            response = web.Response(text="OTA接口异常", content_type="text/plain")
+        finally:
+            self._add_cors_headers(response)
+            return response
+
+    async def handle_download(self, request):
+        """
+        下载固件接口
+        URL: /xiaozhi/ota/download/{filename}
+        - 只允许下载 data/bin 目录下的 .bin 文件
+        - filename 必须是 basename 且匹配安全的模式
+        """
+        try:
+            fname = request.match_info.get("filename", "")
+            if not fname:
+                raise web.HTTPBadRequest(text="filename required")
+
+            # sanitize
+            fname = _safe_basename(fname)
+            # pattern: allow letters, numbers, dot, underscore, dash
+            if not re.match(r"^[A-Za-z0-9\.\-_]+\.bin$", fname):
+                raise web.HTTPBadRequest(text="invalid filename")
+
+            file_path = os.path.join(self.bin_dir, fname)
+            # ensure realpath is under bin_dir
+            file_real = os.path.realpath(file_path)
+            bin_dir_real = os.path.realpath(self.bin_dir)
+            if (
+                not file_real.startswith(bin_dir_real + os.sep)
+                and file_real != bin_dir_real
+            ):
+                raise web.HTTPForbidden(text="forbidden")
+
+            if not os.path.isfile(file_real):
+                raise web.HTTPNotFound(text="file not found")
+
+            # use FileResponse to stream file
+            resp = web.FileResponse(path=file_real)
+        except web.HTTPError as e:
+            resp = e
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"固件下载异常: {e}")
+            resp = web.Response(text="download error", status=500)
+        finally:
+            try:
+                self._add_cors_headers(resp)
+            except Exception:
+                pass
+            return resp

main/xingxing-server/core/api/vision_handler.py

@@ -0,0 +1,182 @@
+import json
+import copy
+from aiohttp import web
+from config.logger import setup_logging
+from core.api.base_handler import BaseHandler
+from core.utils.util import get_vision_url, is_valid_image_file
+from core.utils.vllm import create_instance
+from config.config_loader import get_private_config_from_api
+from core.utils.auth import AuthToken
+import base64
+from typing import Tuple, Optional
+from plugins_func.register import Action
+
+TAG = __name__
+
+# 设置最大文件大小为5MB
+MAX_FILE_SIZE = 5 * 1024 * 1024
+
+
+class VisionHandler(BaseHandler):
+    def __init__(self, config: dict):
+        super().__init__(config)
+        # 初始化认证工具
+        self.auth = AuthToken(config["server"]["auth_key"])
+
+    def _create_error_response(self, message: str) -> dict:
+        """创建统一的错误响应格式"""
+        return {"success": False, "message": message}
+
+    def _verify_auth_token(self, request) -> Tuple[bool, Optional[str]]:
+        """验证认证token"""
+        # 测试模式：允许特定测试令牌或跳过验证
+        auth_header = request.headers.get("Authorization", "")
+        client_id = request.headers.get("Client-Id", "")
+
+        # 允许测试客户端跳过认证
+        if client_id == "web_test_client":
+            device_id = request.headers.get("Device-Id", "test_device")
+            return True, device_id
+
+        if not auth_header.startswith("Bearer "):
+            return False, None
+
+        token = auth_header[7:]  # 移除"Bearer "前缀
+        return self.auth.verify_token(token)
+
+    async def handle_post(self, request):
+        """处理 MCP Vision POST 请求"""
+        response = None  # 初始化response变量
+        try:
+            # 验证token
+            is_valid, token_device_id = self._verify_auth_token(request)
+            if not is_valid:
+                response = web.Response(
+                    text=json.dumps(
+                        self._create_error_response("无效的认证token或token已过期")
+                    ),
+                    content_type="application/json",
+                    status=401,
+                )
+                return response
+
+            # 获取请求头信息
+            device_id = request.headers.get("Device-Id", "")
+            client_id = request.headers.get("Client-Id", "")
+            if device_id != token_device_id:
+                raise ValueError("设备ID与token不匹配")
+            # 解析multipart/form-data请求
+            reader = await request.multipart()
+
+            # 读取question字段
+            question_field = await reader.next()
+            if question_field is None:
+                raise ValueError("缺少问题字段")
+            question = await question_field.text()
+            self.logger.bind(tag=TAG).debug(f"Question: {question}")
+
+            # 读取图片文件
+            image_field = await reader.next()
+            if image_field is None:
+                raise ValueError("缺少图片文件")
+
+            # 读取图片数据
+            image_data = await image_field.read()
+            if not image_data:
+                raise ValueError("图片数据为空")
+
+            # 检查文件大小
+            if len(image_data) > MAX_FILE_SIZE:
+                raise ValueError(
+                    f"图片大小超过限制，最大允许{MAX_FILE_SIZE/1024/1024}MB"
+                )
+
+            # 检查文件格式
+            if not is_valid_image_file(image_data):
+                raise ValueError(
+                    "不支持的文件格式，请上传有效的图片文件（支持JPEG、PNG、GIF、BMP、TIFF、WEBP格式）"
+                )
+
+            # 将图片转换为base64编码
+            image_base64 = base64.b64encode(image_data).decode("utf-8")
+
+            # 如果开启了智控台，则从智控台获取模型配置
+            current_config = copy.deepcopy(self.config)
+            read_config_from_api = current_config.get("read_config_from_api", False)
+            if read_config_from_api:
+                current_config = await get_private_config_from_api(
+                    current_config,
+                    device_id,
+                    client_id,
+                )
+
+            select_vllm_module = current_config["selected_module"].get("VLLM")
+            if not select_vllm_module:
+                raise ValueError("您还未设置默认的视觉分析模块")
+
+            vllm_type = (
+                select_vllm_module
+                if "type" not in current_config["VLLM"][select_vllm_module]
+                else current_config["VLLM"][select_vllm_module]["type"]
+            )
+
+            if not vllm_type:
+                raise ValueError(f"无法找到VLLM模块对应的供应器{vllm_type}")
+
+            vllm = create_instance(
+                vllm_type, current_config["VLLM"][select_vllm_module]
+            )
+
+            result = vllm.response(question, image_base64)
+
+            return_json = {
+                "success": True,
+                "action": Action.RESPONSE.name,
+                "response": result,
+            }
+
+            response = web.Response(
+                text=json.dumps(return_json, separators=(",", ":")),
+                content_type="application/json",
+            )
+        except ValueError as e:
+            self.logger.bind(tag=TAG).error(f"MCP Vision POST请求异常: {e}")
+            return_json = self._create_error_response(str(e))
+            response = web.Response(
+                text=json.dumps(return_json, separators=(",", ":")),
+                content_type="application/json",
+            )
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"MCP Vision POST请求异常: {e}")
+            return_json = self._create_error_response("处理请求时发生错误")
+            response = web.Response(
+                text=json.dumps(return_json, separators=(",", ":")),
+                content_type="application/json",
+            )
+        finally:
+            if response:
+                self._add_cors_headers(response)
+            return response
+
+    async def handle_get(self, request):
+        """处理 MCP Vision GET 请求"""
+        try:
+            vision_explain = get_vision_url(self.config)
+            if vision_explain and len(vision_explain) > 0 and "null" != vision_explain:
+                message = (
+                    f"MCP Vision 接口运行正常，视觉解释接口地址是：{vision_explain}"
+                )
+            else:
+                message = "MCP Vision 接口运行不正常，请打开data目录下的.config.yaml文件，找到【server.vision_explain】，设置好地址"
+
+            response = web.Response(text=message, content_type="text/plain")
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"MCP Vision GET请求异常: {e}")
+            return_json = self._create_error_response("服务器内部错误")
+            response = web.Response(
+                text=json.dumps(return_json, separators=(",", ":")),
+                content_type="application/json",
+            )
+        finally:
+            self._add_cors_headers(response)
+            return response

main/xingxing-server/core/auth.py

@@ -0,0 +1,72 @@
+import hmac
+import base64
+import hashlib
+import time
+
+
+class AuthenticationError(Exception):
+    """认证异常"""
+
+    pass
+
+
+class AuthManager:
+    """
+    统一授权认证管理器
+    生成与验证 client_id device_id token（HMAC-SHA256）认证三元组
+    token 中不含明文 client_id/device_id，只携带签名 + 时间戳; client_id/device_id在连接时传递
+    在 MQTT 中 client_id: client_id, username: device_id, password: token
+    在 Websocket 中，header:{Device-ID: device_id, Client-ID: client_id, Authorization: Bearer token, ......}
+    """
+
+    def __init__(self, secret_key: str, expire_seconds: int = 60 * 60 * 24 * 30):
+        if not expire_seconds or expire_seconds < 0:
+            self.expire_seconds = 60 * 60 * 24 * 30
+        else:
+            self.expire_seconds = expire_seconds
+        self.secret_key = secret_key
+
+    def _sign(self, content: str) -> str:
+        """HMAC-SHA256签名并Base64编码"""
+        sig = hmac.new(
+            self.secret_key.encode("utf-8"), content.encode("utf-8"), hashlib.sha256
+        ).digest()
+        return base64.urlsafe_b64encode(sig).decode("utf-8").rstrip("=")
+
+    def generate_token(self, client_id: str, username: str) -> str:
+        """
+        生成 token
+        Args:
+            client_id: 设备连接ID
+            username: 设备用户名（通常为deviceId）
+        Returns:
+            str: token字符串
+        """
+        ts = int(time.time())
+        content = f"{client_id}|{username}|{ts}"
+        signature = self._sign(content)
+        # token仅包含签名与时间戳，不包含明文信息
+        token = f"{signature}.{ts}"
+        return token
+
+    def verify_token(self, token: str, client_id: str, username: str) -> bool:
+        """
+        验证token有效性
+        Args:
+            token: 客户端传入的token
+            client_id: 连接使用的client_id
+            username: 连接使用的username
+        """
+        try:
+            sig_part, ts_str = token.split(".")
+            ts = int(ts_str)
+            if int(time.time()) - ts > self.expire_seconds:
+                return False  # 过期
+
+            expected_sig = self._sign(f"{client_id}|{username}|{ts}")
+            if not hmac.compare_digest(sig_part, expected_sig):
+                return False
+
+            return True
+        except Exception:
+            return False

main/xingxing-server/core/handle/abortHandle.py

@@ -0,0 +1,20 @@
+import json
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+TAG = __name__
+
+
+async def handleAbortMessage(conn: "ConnectionHandler"):
+    conn.logger.bind(tag=TAG).info("Abort message received")
+    # 设置成打断状态，会自动打断llm、tts任务
+    conn.close_after_chat = False
+    conn.client_abort = True
+    conn.clear_queues()
+    # 打断客户端说话状态
+    await conn.websocket.send(
+        json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id})
+    )
+    conn.clearSpeakStatus()
+    conn.logger.bind(tag=TAG).info("Abort message received-end")

main/xingxing-server/core/handle/helloHandle.py

@@ -0,0 +1,156 @@
+import time
+import json
+import uuid
+import random
+import asyncio
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+from core.utils.dialogue import Message
+from core.utils.util import audio_to_data
+from core.providers.tts.dto.dto import SentenceType
+from core.utils.wakeup_word import WakeupWordsConfig
+from core.handle.sendAudioHandle import sendAudioMessage, send_tts_message
+from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes
+from core.providers.tools.device_mcp import MCPClient, send_mcp_initialize_message
+
+TAG = __name__
+
+WAKEUP_CONFIG = {
+    "refresh_time": 10,
+    "responses": [
+        "我一直都在呢，您请说。",
+        "在的呢，请随时吩咐我。",
+        "来啦来啦，请告诉我吧。",
+        "您请说，我正听着。",
+        "请您讲话，我准备好了。",
+        "请您说出指令吧。",
+        "我认真听着呢，请讲。",
+        "请问您需要什么帮助？",
+        "我在这里，等候您的指令。",
+    ],
+}
+
+# 创建全局的唤醒词配置管理器
+wakeup_words_config = WakeupWordsConfig()
+
+# 用于防止并发调用wakeupWordsResponse的锁
+_wakeup_response_lock = asyncio.Lock()
+
+
+async def handleHelloMessage(conn: "ConnectionHandler", msg_json):
+    """处理hello消息"""
+    audio_params = msg_json.get("audio_params")
+    if audio_params:
+        format = audio_params.get("format")
+        conn.logger.bind(tag=TAG).debug(f"客户端音频格式: {format}")
+        conn.audio_format = format
+        conn.welcome_msg["audio_params"] = audio_params
+    features = msg_json.get("features")
+    if features:
+        conn.logger.bind(tag=TAG).debug(f"客户端特性: {features}")
+        conn.features = features
+        if features.get("mcp"):
+            conn.logger.bind(tag=TAG).debug("客户端支持MCP")
+            conn.mcp_client = MCPClient()
+            # 发送初始化
+            asyncio.create_task(send_mcp_initialize_message(conn))
+
+    await conn.websocket.send(json.dumps(conn.welcome_msg))
+
+
+async def checkWakeupWords(conn: "ConnectionHandler", text):
+    enable_wakeup_words_response_cache = conn.config[
+        "enable_wakeup_words_response_cache"
+    ]
+
+    # 等待tts初始化，最多等待3秒
+    start_time = time.time()
+    while time.time() - start_time < 3:
+        if conn.tts:
+            break
+        await asyncio.sleep(0.1)
+    else:
+        return False
+
+    if not enable_wakeup_words_response_cache:
+        return False
+
+    _, filtered_text = remove_punctuation_and_length(text)
+    if filtered_text not in conn.config.get("wakeup_words"):
+        return False
+
+    conn.just_woken_up = True
+    await send_tts_message(conn, "start")
+
+    # 获取当前音色
+    voice = getattr(conn.tts, "voice", "default")
+    if not voice:
+        voice = "default"
+
+    # 获取唤醒词回复配置
+    response = wakeup_words_config.get_wakeup_response(voice)
+    if not response or not response.get("file_path"):
+        response = {
+            "voice": "default",
+            "file_path": "config/assets/wakeup_words_short.wav",
+            "time": 0,
+            "text": "我在这里哦！",
+        }
+
+    # 获取音频数据
+    opus_packets = await audio_to_data(response.get("file_path"), use_cache=False)
+    # 播放唤醒词回复
+    conn.client_abort = False
+
+    # 将唤醒词回复视为新会话，生成新的 sentence_id，确保流控器重置
+    conn.sentence_id = str(uuid.uuid4().hex)
+
+    conn.logger.bind(tag=TAG).info(f"播放唤醒词回复: {response.get('text')}")
+    await sendAudioMessage(conn, SentenceType.FIRST, opus_packets, response.get("text"))
+    await sendAudioMessage(conn, SentenceType.LAST, [], None)
+
+    # 补充对话
+    conn.dialogue.put(Message(role="assistant", content=response.get("text")))
+
+    # 检查是否需要更新唤醒词回复
+    if time.time() - response.get("time", 0) > WAKEUP_CONFIG["refresh_time"]:
+        if not _wakeup_response_lock.locked():
+            asyncio.create_task(wakeupWordsResponse(conn))
+    return True
+
+
+async def wakeupWordsResponse(conn: "ConnectionHandler"):
+    if not conn.tts:
+        return
+
+    try:
+        # 尝试获取锁，如果获取不到就返回
+        if not await _wakeup_response_lock.acquire():
+            return
+
+        # 从预定义回复列表中随机选择一个回复
+        result = random.choice(WAKEUP_CONFIG["responses"])
+        if not result or len(result) == 0:
+            return
+
+        # 生成TTS音频
+        tts_result = await asyncio.to_thread(conn.tts.to_tts, result)
+        if not tts_result:
+            return
+
+        # 获取当前音色
+        voice = getattr(conn.tts, "voice", "default")
+
+        # 使用链接的sample_rate
+        wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=conn.sample_rate)
+        file_path = wakeup_words_config.generate_file_path(voice)
+        with open(file_path, "wb") as f:
+            f.write(wav_bytes)
+        # 更新配置
+        wakeup_words_config.update_wakeup_response(voice, file_path, result)
+    finally:
+        # 确保在任何情况下都释放锁
+        if _wakeup_response_lock.locked():
+            _wakeup_response_lock.release()

main/xingxing-server/core/handle/intentHandler.py

@@ -0,0 +1,363 @@
+import json
+import uuid
+import asyncio
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+from core.utils.dialogue import Message
+from core.providers.tts.dto.dto import ContentType
+from core.handle.helloHandle import checkWakeupWords
+from plugins_func.register import Action, ActionResponse
+from core.handle.sendAudioHandle import send_stt_message
+from core.handle.reportHandle import enqueue_tool_report
+from core.utils.util import remove_punctuation_and_length
+from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType
+
+TAG = __name__
+NAV_ALERT_ECHO_TTL_S = 12.0
+
+
+STRICT_HAZARD_MARK_PHRASES = {
+    "标记急转弯": "sharp_turn",
+    "标记起转弯": "sharp_turn",
+    "标记急弯": "sharp_turn",
+    "标记左转弯": "left_turn",
+    "标记右转弯": "right_turn",
+    "标记障碍": "obstacle",
+    "标记颠簸": "bump",
+    "标记大坑": "bump",
+    "标记坑": "bump",
+    "标记跳台": "jump",
+    "标记陡坡": "slope",
+    "标记涉水": "water",
+    "标记水坑": "water",
+}
+
+NAV_ALERT_ECHO_PHRASES = {
+    "前方左转弯",
+    "前方右转弯",
+    "准备急转弯",
+    "前方颠簸",
+    "前方水坑",
+    "前方障碍",
+    "前方跳台",
+    "前方陡坡",
+    "前方风险点",
+    "注意左转弯",
+    "注意右转弯",
+    "注意急转弯",
+    "注意颠簸",
+    "注意水坑",
+    "注意障碍",
+}
+
+
+async def handle_user_intent(conn: "ConnectionHandler", text):
+    # 预处理输入文本，处理可能的JSON格式
+    try:
+        if text.strip().startswith("{") and text.strip().endswith("}"):
+            parsed_data = json.loads(text)
+            if isinstance(parsed_data, dict) and "content" in parsed_data:
+                text = parsed_data["content"]  # 提取content用于意图分析
+                conn.current_speaker = parsed_data.get("speaker")  # 保留说话人信息
+    except (json.JSONDecodeError, TypeError):
+        pass
+
+    # 检查是否有明确的退出命令
+    _, filtered_text = remove_punctuation_and_length(text)
+    if await check_direct_exit(conn, filtered_text):
+        return True
+
+    # 检查是否是唤醒词
+    if await checkWakeupWords(conn, filtered_text):
+        return True
+
+    if should_ignore_nav_alert_echo(conn, filtered_text):
+        conn.logger.bind(tag=TAG).info(f"忽略导航播报回采文本: {filtered_text}")
+        return True
+
+    direct_function_call = match_direct_nav_command(filtered_text)
+    if direct_function_call is not None:
+        function_name, function_args = direct_function_call
+        conn.logger.bind(tag=TAG).info(
+            f"识别到直达导航命令: {filtered_text} -> {function_name}"
+        )
+        return await dispatch_function_call(
+            conn,
+            text,
+            function_name,
+            function_args,
+        )
+
+    if conn.intent_type == "function_call":
+        # 使用支持function calling的聊天方法,不再进行意图分析
+        return False
+    # 使用LLM进行意图分析
+    intent_result = await analyze_intent_with_llm(conn, text)
+    if not intent_result:
+        return False
+    # 会话开始时生成sentence_id
+    conn.sentence_id = str(uuid.uuid4().hex)
+    # 处理各种意图
+    return await process_intent_result(conn, intent_result, text)
+
+
+async def check_direct_exit(conn: "ConnectionHandler", text):
+    """检查是否有明确的退出命令"""
+    _, text = remove_punctuation_and_length(text)
+    cmd_exit = conn.cmd_exit
+    for cmd in cmd_exit:
+        if text == cmd:
+            conn.logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
+            await send_stt_message(conn, text)
+            await conn.close()
+            return True
+    return False
+
+
+async def analyze_intent_with_llm(conn: "ConnectionHandler", text):
+    """使用LLM分析用户意图"""
+    if not hasattr(conn, "intent") or not conn.intent:
+        conn.logger.bind(tag=TAG).warning("意图识别服务未初始化")
+        return None
+
+    # 对话历史记录
+    dialogue = conn.dialogue
+    try:
+        intent_result = await conn.intent.detect_intent(conn, dialogue.dialogue, text)
+        return intent_result
+    except Exception as e:
+        conn.logger.bind(tag=TAG).error(f"意图识别失败: {str(e)}")
+
+    return None
+
+
+def match_direct_nav_command(filtered_text: str):
+    text = (filtered_text or "").strip()
+    if not text:
+        return None
+    return None
+
+    if text in {
+        "开始录制路线",
+        "开始记录路线",
+        "录制路线",
+        "开始记录",
+        "开始记路线",
+    }:
+        return "nav_start_record", {}
+
+    if text in {
+        "结束录制",
+        "停止录制",
+        "结束记录",
+        "停止记录",
+        "录制结束",
+    }:
+        return "nav_stop_record", {}
+
+    if text in {
+        "开始预警",
+        "开始播报",
+        "开始领航",
+        "开始导航",
+    }:
+        return "nav_start_run", {}
+
+    if text in {
+        "停止预警",
+        "结束预警",
+        "停止播报",
+        "关闭预警",
+    }:
+        return "nav_stop_run", {}
+
+    if text.startswith("标记"):
+        if text in STRICT_HAZARD_MARK_PHRASES:
+            return "nav_mark_hazard", {"hazard_type": STRICT_HAZARD_MARK_PHRASES[text]}
+        return "nav_mark_hazard_reject", {}
+
+    return None
+
+
+def should_ignore_nav_alert_echo(conn: "ConnectionHandler", filtered_text: str) -> bool:
+    text = (filtered_text or "").strip()
+    if not text:
+        return False
+    recent = getattr(conn, "_recent_nav_alert_notes", None)
+    if not isinstance(recent, dict) or not recent:
+        return False
+
+    now = asyncio.get_running_loop().time()
+    matched = False
+    expired: list[str] = []
+    for raw_text, spoken_at in recent.items():
+        if now - float(spoken_at) > NAV_ALERT_ECHO_TTL_S:
+            expired.append(raw_text)
+            continue
+        _, normalized = remove_punctuation_and_length(str(raw_text))
+        if normalized == text:
+            matched = True
+
+    for raw_text in expired:
+        recent.pop(raw_text, None)
+
+    return matched
+
+
+async def dispatch_function_call(
+    conn: "ConnectionHandler",
+    original_text: str,
+    function_name: str,
+    function_args: dict | None = None,
+):
+    conn.sentence_id = str(uuid.uuid4().hex)
+    function_args = function_args or {}
+    function_args_json = json.dumps(function_args, ensure_ascii=False)
+    function_call_data = {
+        "name": function_name,
+        "id": str(uuid.uuid4().hex),
+        "arguments": function_args_json,
+    }
+
+    await send_stt_message(conn, original_text)
+    conn.client_abort = False
+    enqueue_tool_report(conn, function_name, function_args)
+
+    def process_function_call():
+        conn.dialogue.put(Message(role="user", content=original_text))
+
+        tool_call_timeout = int(conn.config.get("tool_call_timeout", 30))
+        try:
+            result = asyncio.run_coroutine_threadsafe(
+                conn.func_handler.handle_llm_function_call(conn, function_call_data),
+                conn.loop,
+            ).result(timeout=tool_call_timeout)
+        except Exception as e:
+            conn.logger.bind(tag=TAG).error(f"工具调用失败: {e}")
+            result = ActionResponse(
+                action=Action.ERROR,
+                result="工具调用超时，请一会再试下哈",
+                response="工具调用超时，请一会再试下哈",
+            )
+
+        if result:
+            enqueue_tool_report(
+                conn,
+                function_name,
+                function_args,
+                str(result.result) if result.result else None,
+                report_tool_call=False,
+            )
+
+            if result.action == Action.RESPONSE:
+                text = result.response
+                if text is not None:
+                    speak_txt(conn, text)
+            elif result.action == Action.REQLLM:
+                text = result.result
+                conn.dialogue.put(Message(role="tool", content=text))
+                llm_result = conn.intent.replyResult(text, original_text)
+                if llm_result is None:
+                    llm_result = text
+                speak_txt(conn, llm_result)
+            elif result.action == Action.NOTFOUND or result.action == Action.ERROR:
+                text = result.response if result.response else result.result
+                if text is not None:
+                    speak_txt(conn, text)
+            elif function_name != "play_music":
+                text = result.response
+                if text is None:
+                    text = result.result
+                if text is not None:
+                    speak_txt(conn, text)
+
+    conn.executor.submit(process_function_call)
+    return True
+
+
+async def process_intent_result(
+    conn: "ConnectionHandler", intent_result, original_text
+):
+    """处理意图识别结果"""
+    try:
+        # 尝试将结果解析为JSON
+        intent_data = json.loads(intent_result)
+
+        # 检查是否有function_call
+        if "function_call" in intent_data:
+            # 直接从意图识别获取了function_call
+            conn.logger.bind(tag=TAG).debug(
+                f"检测到function_call格式的意图结果: {intent_data['function_call']['name']}"
+            )
+            function_name = intent_data["function_call"]["name"]
+            if function_name == "continue_chat":
+                return False
+
+            if function_name == "result_for_context":
+                await send_stt_message(conn, original_text)
+                conn.client_abort = False
+
+                def process_context_result():
+                    conn.dialogue.put(Message(role="user", content=original_text))
+
+                    from core.utils.current_time import get_current_time_info
+
+                    current_time, today_date, today_weekday, lunar_date = (
+                        get_current_time_info()
+                    )
+
+                    # 构建带上下文的基础提示
+                    context_prompt = f"""当前时间：{current_time}
+                                        今天日期：{today_date} ({today_weekday})
+                                        今天农历：{lunar_date}
+
+                                        请根据以上信息回答用户的问题：{original_text}"""
+
+                    response = conn.intent.replyResult(context_prompt, original_text)
+                    speak_txt(conn, response)
+
+                conn.executor.submit(process_context_result)
+                return True
+
+            function_args = {}
+            if "arguments" in intent_data["function_call"]:
+                function_args = intent_data["function_call"]["arguments"]
+                if function_args is None:
+                    function_args = {}
+            if isinstance(function_args, str):
+                function_args = json.loads(function_args) if function_args else {}
+
+            return await dispatch_function_call(
+                conn,
+                original_text,
+                function_name,
+                function_args if isinstance(function_args, dict) else {},
+            )
+        return False
+    except json.JSONDecodeError as e:
+        conn.logger.bind(tag=TAG).error(f"处理意图结果时出错: {e}")
+        return False
+
+
+def speak_txt(conn: "ConnectionHandler", text):
+    # 记录文本到 sentence_id 映射
+    conn.tts.store_tts_text(conn.sentence_id, text)
+
+    conn.tts.tts_text_queue.put(
+        TTSMessageDTO(
+            sentence_id=conn.sentence_id,
+            sentence_type=SentenceType.FIRST,
+            content_type=ContentType.ACTION,
+        )
+    )
+    conn.tts.tts_one_sentence(conn, ContentType.TEXT, content_detail=text)
+    conn.tts.tts_text_queue.put(
+        TTSMessageDTO(
+            sentence_id=conn.sentence_id,
+            sentence_type=SentenceType.LAST,
+            content_type=ContentType.ACTION,
+        )
+    )
+    conn.dialogue.put(Message(role="assistant", content=text))

main/xingxing-server/core/handle/receiveAudioHandle.py

@@ -0,0 +1,171 @@
+import time
+import json
+import asyncio
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+from core.utils.util import audio_to_data
+from core.handle.abortHandle import handleAbortMessage
+from core.handle.intentHandler import handle_user_intent
+from core.utils.output_counter import check_device_output_limit
+from core.handle.sendAudioHandle import send_stt_message, SentenceType
+
+TAG = __name__
+
+
+async def handleAudioMessage(conn: "ConnectionHandler", audio):
+    # 当前片段是否有人说话
+    have_voice = conn.vad.is_vad(conn, audio)
+    # 如果设备刚刚被唤醒，短暂忽略VAD检测
+    if hasattr(conn, "just_woken_up") and conn.just_woken_up:
+        have_voice = False
+        # 设置一个短暂延迟后恢复VAD检测
+        if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
+            conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
+    # 设备长时间空闲检测，用于say goodbye
+    await no_voice_close_connect(conn, have_voice)
+    # 接收音频
+    await conn.asr.receive_audio(conn, audio, have_voice)
+
+
+async def resume_vad_detection(conn: "ConnectionHandler"):
+    # 等待2秒后恢复VAD检测
+    await asyncio.sleep(0.35)
+    conn.just_woken_up = False
+
+
+async def startToChat(conn: "ConnectionHandler", text):
+    # 检查输入是否是JSON格式（包含说话人信息）
+    speaker_name = None
+    language_tag = None
+    actual_text = text
+
+    try:
+        # 尝试解析JSON格式的输入
+        if text.strip().startswith("{") and text.strip().endswith("}"):
+            data = json.loads(text)
+            if "speaker" in data and "content" in data:
+                speaker_name = data["speaker"]
+                language_tag = data["language"]
+                actual_text = data["content"]
+                conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}")
+
+                # 直接使用JSON格式的文本，不解析
+                actual_text = text
+    except (json.JSONDecodeError, KeyError):
+        # 如果解析失败，继续使用原始文本
+        pass
+
+    # 保存说话人信息到连接对象
+    if speaker_name:
+        conn.current_speaker = speaker_name
+    else:
+        conn.current_speaker = None
+
+    if conn.need_bind:
+        await check_bind_device(conn)
+        return
+
+    # 如果当日的输出字数大于限定的字数
+    if conn.max_output_size > 0:
+        if check_device_output_limit(
+            conn.headers.get("device-id"), conn.max_output_size
+        ):
+            await max_out_size(conn)
+            return
+
+    # manual 模式下不打断正在播放的内容
+    if conn.client_is_speaking and conn.client_listen_mode != "manual":
+        await handleAbortMessage(conn)
+
+    # 首先进行意图分析，使用实际文本内容
+    intent_handled = await handle_user_intent(conn, actual_text)
+
+    if intent_handled:
+        # 如果意图已被处理，不再进行聊天
+        return
+
+    # 意图未被处理，继续常规聊天流程，使用实际文本内容
+    await send_stt_message(conn, actual_text)
+
+    # 准备开始新会话
+    conn.client_abort = False
+
+    conn.executor.submit(conn.chat, actual_text)
+
+
+async def no_voice_close_connect(conn: "ConnectionHandler", have_voice):
+    if have_voice:
+        conn.last_activity_time = time.time() * 1000
+        return
+    # 只有在已经初始化过时间戳的情况下才进行超时检查
+    if conn.last_activity_time > 0.0:
+        no_voice_time = time.time() * 1000 - conn.last_activity_time
+        close_connection_no_voice_time = int(
+            conn.config.get("close_connection_no_voice_time", 120)
+        )
+        if (
+            not conn.close_after_chat
+            and no_voice_time > 1000 * close_connection_no_voice_time
+        ):
+            conn.close_after_chat = True
+            conn.client_abort = False
+            end_prompt = conn.config.get("end_prompt", {})
+            if end_prompt and end_prompt.get("enable", True) is False:
+                conn.logger.bind(tag=TAG).info("结束对话，无需发送结束提示语")
+                await conn.close()
+                return
+            prompt = end_prompt.get("prompt")
+            if not prompt:
+                prompt = "请你以```时间过得真快```未来头，用富有感情、依依不舍的话来结束这场对话吧。！"
+            await startToChat(conn, prompt)
+
+
+async def max_out_size(conn: "ConnectionHandler"):
+    # 播放超出最大输出字数的提示
+    conn.client_abort = False
+    text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！"
+    await send_stt_message(conn, text)
+    file_path = "config/assets/max_output_size.wav"
+    opus_packets = await audio_to_data(file_path)
+    conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))
+    conn.close_after_chat = True
+
+
+async def check_bind_device(conn: "ConnectionHandler"):
+    if conn.bind_code:
+        # 确保bind_code是6位数字
+        if len(conn.bind_code) != 6:
+            conn.logger.bind(tag=TAG).error(f"无效的绑定码格式: {conn.bind_code}")
+            text = "绑定码格式错误，请检查配置。"
+            await send_stt_message(conn, text)
+            return
+
+        text = f"请登录控制面板，输入{conn.bind_code}，绑定设备。"
+        await send_stt_message(conn, text)
+
+        # 播放提示音
+        music_path = "config/assets/bind_code.wav"
+        opus_packets = await audio_to_data(music_path)
+        conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, text))
+
+        # 逐个播放数字
+        for i in range(6):  # 确保只播放6位数字
+            try:
+                digit = conn.bind_code[i]
+                num_path = f"config/assets/bind_code/{digit}.wav"
+                num_packets = await audio_to_data(num_path)
+                conn.tts.tts_audio_queue.put((SentenceType.MIDDLE, num_packets, None))
+            except Exception as e:
+                conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
+                continue
+        conn.tts.tts_audio_queue.put((SentenceType.LAST, [], None))
+    else:
+        # 播放未绑定提示
+        conn.client_abort = False
+        text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。"
+        await send_stt_message(conn, text)
+        music_path = "config/assets/bind_not_found.wav"
+        opus_packets = await audio_to_data(music_path)
+        conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))

main/xingxing-server/core/handle/reportHandle.py

@@ -0,0 +1,200 @@
+"""
+TTS上报功能已集成到ConnectionHandler类中。
+
+上报功能包括：
+1. 每个连接对象拥有自己的上报队列和处理线程
+2. 上报线程的生命周期与连接对象绑定
+3. 使用ConnectionHandler.enqueue_tts_report方法进行上报
+
+具体实现请参考core/connection.py中的相关代码。
+"""
+
+import time
+import json
+import opuslib_next
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+
+from config.manage_api_client import report as manage_report
+
+TAG = __name__
+
+
+async def report(conn: "ConnectionHandler", type, text, opus_data, report_time):
+    """执行聊天记录上报操作
+
+    Args:
+        conn: 连接对象
+        type: 上报类型，1为用户，2为智能体，3为工具调用
+        text: 合成文本
+        opus_data: opus音频数据
+        report_time: 上报时间
+    """
+    try:
+        if opus_data:
+            audio_data = opus_to_wav(conn, opus_data)
+        else:
+            audio_data = None
+        # 执行异步上报
+        await manage_report(
+            mac_address=conn.device_id,
+            session_id=conn.session_id,
+            chat_type=type,
+            content=text,
+            audio=audio_data,
+            report_time=report_time,
+        )
+    except Exception as e:
+        conn.logger.bind(tag=TAG).error(f"聊天记录上报失败: {e}")
+
+
+def opus_to_wav(conn: "ConnectionHandler", opus_data):
+    """将Opus数据转换为WAV格式的字节流
+
+    Args:
+        output_dir: 输出目录（保留参数以保持接口兼容）
+        opus_data: opus音频数据
+
+    Returns:
+        bytes: WAV格式的音频数据
+    """
+    decoder = None
+    try:
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        if not pcm_data:
+            raise ValueError("没有有效的PCM数据")
+
+        # 创建WAV文件头
+        pcm_data_bytes = b"".join(pcm_data)
+        num_samples = len(pcm_data_bytes) // 2  # 16-bit samples
+
+        # WAV文件头
+        wav_header = bytearray()
+        wav_header.extend(b"RIFF")  # ChunkID
+        wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little"))  # ChunkSize
+        wav_header.extend(b"WAVE")  # Format
+        wav_header.extend(b"fmt ")  # Subchunk1ID
+        wav_header.extend((16).to_bytes(4, "little"))  # Subchunk1Size
+        wav_header.extend((1).to_bytes(2, "little"))  # AudioFormat (PCM)
+        wav_header.extend((1).to_bytes(2, "little"))  # NumChannels
+        wav_header.extend((16000).to_bytes(4, "little"))  # SampleRate
+        wav_header.extend((32000).to_bytes(4, "little"))  # ByteRate
+        wav_header.extend((2).to_bytes(2, "little"))  # BlockAlign
+        wav_header.extend((16).to_bytes(2, "little"))  # BitsPerSample
+        wav_header.extend(b"data")  # Subchunk2ID
+        wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little"))  # Subchunk2Size
+
+        # 返回完整的WAV数据
+        return bytes(wav_header) + pcm_data_bytes
+    finally:
+        if decoder is not None:
+            try:
+                del decoder
+            except Exception as e:
+                conn.logger.bind(tag=TAG).debug(f"释放decoder资源时出错: {e}")
+
+
+def enqueue_tts_report(conn: "ConnectionHandler", text, opus_data):
+    if not conn.read_config_from_api or conn.need_bind or not conn.report_tts_enable:
+        return
+    if conn.chat_history_conf == 0:
+        return
+    """将TTS数据加入上报队列
+
+    Args:
+        conn: 连接对象
+        text: 合成文本
+        opus_data: opus音频数据
+    """
+    try:
+        # 使用连接对象的队列，传入文本和二进制数据而非文件路径
+        if conn.chat_history_conf == 2:
+            conn.report_queue.put((2, text, opus_data, int(time.time() * 1000)))
+            conn.logger.bind(tag=TAG).debug(
+                f"TTS数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
+            )
+        else:
+            conn.report_queue.put((2, text, None, int(time.time() * 1000)))
+            conn.logger.bind(tag=TAG).debug(
+                f"TTS数据已加入上报队列: {conn.device_id}, 不上报音频"
+            )
+    except Exception as e:
+        conn.logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}")
+
+
+def enqueue_tool_report(conn: "ConnectionHandler", tool_name: str, tool_input: dict, tool_result: str = None, report_tool_call: bool = True):
+    """将工具调用数据加入上报队列
+
+    Args:
+        conn: 连接对象
+        tool_name: 工具名称
+        tool_input: 工具输入参数
+        tool_result: 工具执行结果（可选）
+        report_tool_call: 是否上报工具调用本身，默认True；仅上报结果时设为False
+    """
+    if not conn.read_config_from_api or conn.need_bind:
+        return
+    if conn.chat_history_conf == 0:
+        return
+
+    try:
+        timestamp = int(time.time() * 1000)
+
+        # 构建工具调用内容
+        if report_tool_call:
+            tool_text = json.dumps(
+                [
+                    {
+                        "type": "tool",
+                        "text": f"{tool_name}({json.dumps(tool_input, ensure_ascii=False)})",
+                    }
+                ]
+            )
+            conn.report_queue.put((3, tool_text, None, timestamp))
+
+        # 构建工具结果内容
+        if tool_result:
+            result_display = f'{{"result":"{str(tool_result)}"}}'
+            result_content = json.dumps([{"type": "tool_result", "text": result_display}], ensure_ascii=False)
+            conn.report_queue.put((3, result_content, None, timestamp + 1))
+    except Exception as e:
+        conn.logger.bind(tag=TAG).error(f"加入工具上报队列失败: {e}")
+
+
+def enqueue_asr_report(conn: "ConnectionHandler", text, opus_data):
+    if not conn.read_config_from_api or conn.need_bind or not conn.report_asr_enable:
+        return
+    if conn.chat_history_conf == 0:
+        return
+    """将ASR数据加入上报队列
+
+    Args:
+        conn: 连接对象
+        text: 合成文本
+        opus_data: opus音频数据
+    """
+    try:
+        # 使用连接对象的队列，传入文本和二进制数据而非文件路径
+        if conn.chat_history_conf == 2:
+            conn.report_queue.put((1, text, opus_data, int(time.time() * 1000)))
+            conn.logger.bind(tag=TAG).debug(
+                f"ASR数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
+            )
+        else:
+            conn.report_queue.put((1, text, None, int(time.time() * 1000)))
+            conn.logger.bind(tag=TAG).debug(
+                f"ASR数据已加入上报队列: {conn.device_id}, 不上报音频"
+            )
+    except Exception as e:
+        conn.logger.bind(tag=TAG).debug(f"加入ASR上报队列失败: {text}, {e}")

main/xingxing-server/core/handle/sendAudioHandle.py

@@ -0,0 +1,340 @@
+import json
+import time
+import asyncio
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+from core.utils import textUtils
+from core.utils.util import audio_to_data
+from core.providers.tts.dto.dto import SentenceType
+from core.utils.audioRateController import AudioRateController
+
+TAG = __name__
+# 音频帧时长（毫秒）
+AUDIO_FRAME_DURATION = 60
+# 预缓冲包数量，直接发送以减少延迟
+PRE_BUFFER_COUNT = 5
+
+
+async def sendAudioMessage(conn: "ConnectionHandler", sentenceType, audios, text, sentence_id=None):
+    # 跳过旧句子残留音频
+    if sentence_id is not None and sentence_id != conn.sentence_id:
+        return
+
+    if conn.tts.tts_audio_first_sentence:
+        conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
+        conn.tts.tts_audio_first_sentence = False
+
+    if sentenceType == SentenceType.FIRST:
+        # 同一句子的后续消息加入流控队列，其他情况立即发送
+        if (
+            hasattr(conn, "audio_rate_controller")
+            and conn.audio_rate_controller
+            and getattr(conn, "audio_flow_control", {}).get("sentence_id")
+            == conn.sentence_id
+        ):
+            conn.audio_rate_controller.add_message(
+                lambda: send_tts_message(conn, "sentence_start", text)
+            )
+        else:
+            # 新句子或流控器未初始化，立即发送
+            await send_tts_message(conn, "sentence_start", text)
+
+    await sendAudio(conn, audios)
+    # 发送句子开始消息
+    if sentenceType is not SentenceType.MIDDLE:
+        conn.logger.bind(tag=TAG).info(f"发送音频消息: {sentenceType}, {text}")
+
+    # 发送结束消息（如果是最后一个文本）
+    if sentenceType == SentenceType.LAST:
+        await send_tts_message(conn, "stop", None)
+        if conn.close_after_chat:
+            await conn.close()
+
+
+async def _wait_for_audio_completion(conn: "ConnectionHandler"):
+    """
+    等待音频队列清空并等待预缓冲包播放完成
+
+    Args:
+        conn: 连接对象
+    """
+    if hasattr(conn, "audio_rate_controller") and conn.audio_rate_controller:
+        rate_controller = conn.audio_rate_controller
+        conn.logger.bind(tag=TAG).debug(
+            f"等待音频发送完成，队列中还有 {len(rate_controller.queue)} 个包"
+        )
+        await rate_controller.queue_empty_event.wait()
+
+        # 等待预缓冲包播放完成
+        # 前N个包直接发送，增加2个网络抖动包，需要额外等待它们在客户端播放完成
+        frame_duration_ms = rate_controller.frame_duration
+        pre_buffer_playback_time = (PRE_BUFFER_COUNT + 2) * frame_duration_ms / 1000.0
+        await asyncio.sleep(pre_buffer_playback_time)
+
+        conn.logger.bind(tag=TAG).debug("音频发送完成")
+
+
+async def _send_to_mqtt_gateway(
+    conn: "ConnectionHandler", opus_packet, timestamp, sequence
+):
+    """
+    发送带16字节头部的opus数据包给mqtt_gateway
+    Args:
+        conn: 连接对象
+        opus_packet: opus数据包
+        timestamp: 时间戳
+        sequence: 序列号
+    """
+    # 为opus数据包添加16字节头部
+    header = bytearray(16)
+    header[0] = 1  # type
+    header[2:4] = len(opus_packet).to_bytes(2, "big")  # payload length
+    header[4:8] = sequence.to_bytes(4, "big")  # sequence
+    header[8:12] = timestamp.to_bytes(4, "big")  # 时间戳
+    header[12:16] = len(opus_packet).to_bytes(4, "big")  # opus长度
+
+    # 发送包含头部的完整数据包
+    complete_packet = bytes(header) + opus_packet
+    await conn.websocket.send(complete_packet)
+
+
+async def sendAudio(
+    conn: "ConnectionHandler", audios, frame_duration=AUDIO_FRAME_DURATION
+):
+    """
+    发送音频包，使用 AudioRateController 进行精确的流量控制
+
+    Args:
+        conn: 连接对象
+        audios: 单个opus包(bytes) 或 opus包列表
+        frame_duration: 帧时长（毫秒），默认使用全局常量AUDIO_FRAME_DURATION
+    """
+    if audios is None or len(audios) == 0:
+        return
+
+    send_delay = conn.config.get("tts_audio_send_delay", -1) / 1000.0
+    is_single_packet = isinstance(audios, bytes)
+
+    # 初始化或获取 RateController
+    rate_controller, flow_control = _get_or_create_rate_controller(
+        conn, frame_duration, is_single_packet
+    )
+
+    # 统一转换为列表处理
+    audio_list = [audios] if is_single_packet else audios
+
+    # 发送音频包
+    await _send_audio_with_rate_control(
+        conn, audio_list, rate_controller, flow_control, send_delay
+    )
+
+
+def _get_or_create_rate_controller(
+    conn: "ConnectionHandler", frame_duration, is_single_packet
+):
+    """
+    获取或创建 RateController 和 flow_control
+
+    Args:
+        conn: 连接对象
+        frame_duration: 帧时长
+        is_single_packet: 是否单包模式（True: TTS流式单包, False: 批量包）
+
+    Returns:
+        (rate_controller, flow_control)
+    """
+    # 检查是否需要重置控制器
+    need_reset = False
+
+    if not hasattr(conn, "audio_rate_controller"):
+        # 控制器不存在，需要创建
+        need_reset = True
+    else:
+        rate_controller = conn.audio_rate_controller
+
+        # 后台发送任务已停止, 则需要重置
+        if (
+            not rate_controller.pending_send_task
+            or rate_controller.pending_send_task.done()
+        ):
+            need_reset = True
+        # 当sentence_id 变化，需要重置
+        elif (
+            getattr(conn, "audio_flow_control", {}).get("sentence_id")
+            != conn.sentence_id
+        ):
+            need_reset = True
+
+    if need_reset:
+        # 创建或获取 rate_controller
+        if not hasattr(conn, "audio_rate_controller"):
+            conn.audio_rate_controller = AudioRateController(frame_duration)
+        else:
+            conn.audio_rate_controller.reset()
+
+        # 初始化 flow_control
+        conn.audio_flow_control = {
+            "packet_count": 0,
+            "sequence": 0,
+            "sentence_id": conn.sentence_id,
+        }
+
+        # 启动后台发送循环
+        _start_background_sender(
+            conn, conn.audio_rate_controller, conn.audio_flow_control
+        )
+
+    return conn.audio_rate_controller, conn.audio_flow_control
+
+
+def _start_background_sender(conn: "ConnectionHandler", rate_controller, flow_control):
+    """
+    启动后台发送循环任务
+
+    Args:
+        conn: 连接对象
+        rate_controller: 速率控制器
+        flow_control: 流控状态
+    """
+
+    async def send_callback(packet):
+        # 检查是否应该中止
+        if conn.client_abort:
+            raise asyncio.CancelledError("客户端已中止")
+
+        conn.last_activity_time = time.time() * 1000
+        await _do_send_audio(conn, packet, flow_control)
+
+    # 使用 start_sending 启动后台循环
+    rate_controller.start_sending(send_callback)
+
+
+async def _send_audio_with_rate_control(
+    conn: "ConnectionHandler", audio_list, rate_controller, flow_control, send_delay
+):
+    """
+    使用 rate_controller 发送音频包
+
+    Args:
+        conn: 连接对象
+        audio_list: 音频包列表
+        rate_controller: 速率控制器
+        flow_control: 流控状态
+        send_delay: 固定延迟（秒），-1表示使用动态流控
+    """
+    for packet in audio_list:
+        if conn.client_abort:
+            return
+
+        conn.last_activity_time = time.time() * 1000
+
+        # 预缓冲：前N个包直接发送
+        if flow_control["packet_count"] < PRE_BUFFER_COUNT:
+            await _do_send_audio(conn, packet, flow_control)
+        elif send_delay > 0:
+            # 固定延迟模式
+            await asyncio.sleep(send_delay)
+            await _do_send_audio(conn, packet, flow_control)
+        else:
+            # 动态流控模式：仅添加到队列，由后台循环负责发送
+            rate_controller.add_audio(packet)
+
+
+async def _do_send_audio(conn: "ConnectionHandler", opus_packet, flow_control):
+    """
+    执行实际的音频发送
+    """
+    packet_index = flow_control.get("packet_count", 0)
+    sequence = flow_control.get("sequence", 0)
+
+    if conn.conn_from_mqtt_gateway:
+        # 计算时间戳（基于播放位置）
+        start_time = time.time()
+        timestamp = int(start_time * 1000) % (2**32)
+        await _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence)
+    else:
+        # 直接发送opus数据包
+        await conn.websocket.send(opus_packet)
+
+    # 更新流控状态
+    flow_control["packet_count"] = packet_index + 1
+    flow_control["sequence"] = sequence + 1
+
+
+async def send_tts_message(conn: "ConnectionHandler", state, text=None):
+    """发送 TTS 状态消息"""
+    if text is None and state == "sentence_start":
+        return
+    message = {"type": "tts", "state": state, "session_id": conn.session_id}
+    if text is not None:
+        message["text"] = textUtils.check_emoji(text)
+
+    # TTS播放结束
+    if state == "stop":
+        # 保存当前的 sentence_id，用于后续判断是否是当前轮次
+        current_sentence_id = conn.sentence_id
+        # 播放提示音
+        tts_notify = conn.config.get("enable_stop_tts_notify", False)
+        if tts_notify:
+            stop_tts_notify_voice = conn.config.get(
+                "stop_tts_notify_voice", "config/assets/tts_notify.mp3"
+            )
+            audios = await audio_to_data(stop_tts_notify_voice, is_opus=True)
+            await sendAudio(conn, audios)
+        # 等待所有音频包发送完成
+        await _wait_for_audio_completion(conn)
+
+        # 检查是否是当前轮次
+        if current_sentence_id != conn.sentence_id:
+            return
+
+        # 停止音频发送循环（仅在流控器已初始化时调用）
+        if hasattr(conn, "audio_rate_controller") and conn.audio_rate_controller:
+            conn.audio_rate_controller.stop_sending()
+        conn.clearSpeakStatus()
+
+    # 发送消息到客户端
+    await conn.websocket.send(json.dumps(message))
+
+
+async def send_stt_message(conn: "ConnectionHandler", text):
+    """发送 STT 状态消息"""
+    end_prompt_str = conn.config.get("end_prompt", {}).get("prompt")
+    if end_prompt_str and end_prompt_str == text:
+        await send_tts_message(conn, "start")
+        return
+
+    # 解析JSON格式，提取实际的用户说话内容
+    display_text = text
+    try:
+        # 尝试解析JSON格式
+        if text.strip().startswith("{") and text.strip().endswith("}"):
+            parsed_data = json.loads(text)
+            if isinstance(parsed_data, dict) and "content" in parsed_data:
+                # 如果是包含说话人信息的JSON格式，只显示content部分
+                display_text = parsed_data["content"]
+                # 保存说话人信息到conn对象
+                if "speaker" in parsed_data:
+                    conn.current_speaker = parsed_data["speaker"]
+    except (json.JSONDecodeError, TypeError):
+        # 如果不是JSON格式，直接使用原始文本
+        display_text = text
+    stt_text = textUtils.get_string_no_punctuation_or_emoji(display_text)
+    await conn.websocket.send(
+        json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id})
+    )
+    await send_tts_message(conn, "start")
+    # 发送start消息后客户端状态会处于说话中状态，同步服务端状态
+    conn.client_is_speaking = True
+
+
+async def send_display_message(conn: "ConnectionHandler", text):
+    """发送纯显示消息"""
+    message = {
+        "type": "stt",
+        "text": text,
+        "session_id": conn.session_id
+    }
+    await conn.websocket.send(json.dumps(message))

main/xingxing-server/core/handle/textHandle.py

@@ -0,0 +1,19 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+from core.handle.textMessageHandlerRegistry import TextMessageHandlerRegistry
+from core.handle.textMessageProcessor import TextMessageProcessor
+
+TAG = __name__
+
+# 全局处理器注册表
+message_registry = TextMessageHandlerRegistry()
+
+# 创建全局消息处理器实例
+message_processor = TextMessageProcessor(message_registry)
+
+
+async def handleTextMessage(conn: "ConnectionHandler", message):
+    """处理文本消息"""
+    await message_processor.process_message(conn, message)

main/xingxing-server/core/handle/textHandler/abortMessageHandler.py

@@ -0,0 +1,16 @@
+from typing import Dict, Any
+
+from core.handle.abortHandle import handleAbortMessage
+from core.handle.textMessageHandler import TextMessageHandler
+from core.handle.textMessageType import TextMessageType
+
+
+class AbortTextMessageHandler(TextMessageHandler):
+    """Abort消息处理器"""
+
+    @property
+    def message_type(self) -> TextMessageType:
+        return TextMessageType.ABORT
+
+    async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
+        await handleAbortMessage(conn)

main/xingxing-server/core/handle/textHandler/helloMessageHandler.py

@@ -0,0 +1,18 @@
+from typing import Dict, Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+from core.handle.helloHandle import handleHelloMessage
+from core.handle.textMessageHandler import TextMessageHandler
+from core.handle.textMessageType import TextMessageType
+
+
+class HelloTextMessageHandler(TextMessageHandler):
+    """Hello消息处理器"""
+
+    @property
+    def message_type(self) -> TextMessageType:
+        return TextMessageType.HELLO
+
+    async def handle(self, conn: "ConnectionHandler", msg_json: Dict[str, Any]) -> None:
+        await handleHelloMessage(conn, msg_json)

main/xingxing-server/core/handle/textHandler/iotMessageHandler.py

@@ -0,0 +1,22 @@
+import asyncio
+from typing import Dict, Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+from core.handle.textMessageHandler import TextMessageHandler
+from core.handle.textMessageType import TextMessageType
+from core.providers.tools.device_iot import handleIotStatus, handleIotDescriptors
+
+
+class IotTextMessageHandler(TextMessageHandler):
+    """IOT消息处理器"""
+
+    @property
+    def message_type(self) -> TextMessageType:
+        return TextMessageType.IOT
+
+    async def handle(self, conn: "ConnectionHandler", msg_json: Dict[str, Any]) -> None:
+        if "descriptors" in msg_json:
+            asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
+        if "states" in msg_json:
+            asyncio.create_task(handleIotStatus(conn, msg_json["states"]))

main/xingxing-server/core/handle/textHandler/listenMessageHandler.py

@@ -0,0 +1,76 @@
+import time
+import asyncio
+from typing import Dict, Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+
+from core.handle.receiveAudioHandle import startToChat
+from core.handle.reportHandle import enqueue_asr_report
+from core.handle.sendAudioHandle import send_stt_message, send_tts_message
+from core.handle.textMessageHandler import TextMessageHandler
+from core.handle.textMessageType import TextMessageType
+from core.utils.util import remove_punctuation_and_length
+from core.providers.asr.dto.dto import InterfaceType
+
+TAG = __name__
+
+class ListenTextMessageHandler(TextMessageHandler):
+    """Listen消息处理器"""
+
+    @property
+    def message_type(self) -> TextMessageType:
+        return TextMessageType.LISTEN
+
+    async def handle(self, conn: "ConnectionHandler", msg_json: Dict[str, Any]) -> None:
+        if "mode" in msg_json:
+            conn.client_listen_mode = msg_json["mode"]
+            conn.logger.bind(tag=TAG).debug(
+                f"客户端拾音模式：{conn.client_listen_mode}"
+            )
+        if msg_json["state"] == "start":
+            # 设备从播放模式切回录音模式,清除所有音频状态和缓冲区
+            conn.reset_audio_states()
+        elif msg_json["state"] == "stop":
+            conn.client_voice_stop = True
+            if conn.asr.interface_type == InterfaceType.STREAM:
+                # 流式模式下，发送结束请求
+                asyncio.create_task(conn.asr._send_stop_request())
+            else:
+                # 非流式模式：直接触发ASR识别
+                if len(conn.asr_audio) > 0:
+                    asr_audio_task = conn.asr_audio.copy()
+                    conn.reset_audio_states()
+
+                    if len(asr_audio_task) > 0:
+                        await conn.asr.handle_voice_stop(conn, asr_audio_task)
+        elif msg_json["state"] == "detect":
+            conn.client_have_voice = False
+            conn.reset_audio_states()
+            if "text" in msg_json:
+                conn.last_activity_time = time.time() * 1000
+                original_text = msg_json["text"]  # 保留原始文本
+                filtered_len, filtered_text = remove_punctuation_and_length(
+                    original_text
+                )
+
+                # 识别是否是唤醒词
+                is_wakeup_words = filtered_text in conn.config.get("wakeup_words")
+                # 是否开启唤醒词回复
+                enable_greeting = conn.config.get("enable_greeting", True)
+
+                if is_wakeup_words and not enable_greeting:
+                    # 如果是唤醒词，且关闭了唤醒词回复，就不用回答
+                    conn.just_woken_up = True
+                    conn.client_is_speaking = False
+                elif is_wakeup_words:
+                    conn.just_woken_up = True
+                    # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
+                    enqueue_asr_report(conn, "嘿，你好呀", [])
+                    await startToChat(conn, "嘿，你好呀")
+                else:
+                    conn.just_woken_up = True
+                    # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
+                    enqueue_asr_report(conn, original_text, [])
+                    # 否则需要LLM对文字内容进行答复
+                    await startToChat(conn, original_text)

main/xingxing-server/core/handle/textHandler/mcpMessageHandler.py

@@ -0,0 +1,22 @@
+import asyncio
+from typing import Dict, Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+from core.handle.textMessageHandler import TextMessageHandler
+from core.handle.textMessageType import TextMessageType
+from core.providers.tools.device_mcp import handle_mcp_message
+
+
+class McpTextMessageHandler(TextMessageHandler):
+    """MCP消息处理器"""
+
+    @property
+    def message_type(self) -> TextMessageType:
+        return TextMessageType.MCP
+
+    async def handle(self, conn: "ConnectionHandler", msg_json: Dict[str, Any]) -> None:
+        if "payload" in msg_json:
+            asyncio.create_task(
+                handle_mcp_message(conn, conn.mcp_client, msg_json["payload"])
+            )

main/xingxing-server/core/handle/textHandler/pingMessageHandler.py

@@ -0,0 +1,45 @@
+import json
+import time
+from typing import Dict, Any
+
+from core.handle.textMessageHandler import TextMessageHandler
+from core.handle.textMessageType import TextMessageType
+
+TAG = __name__
+
+
+class PingMessageHandler(TextMessageHandler):
+    """Ping消息处理器，用于保持WebSocket连接"""
+
+    @property
+    def message_type(self) -> TextMessageType:
+        return TextMessageType.PING
+
+    async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
+        """
+        处理PING消息，发送PONG响应
+        消息格式：{"type": "ping"}
+        Args:
+            conn: WebSocket连接对象
+            msg_json: PING消息的JSON数据
+        """
+        # 检查是否启用了WebSocket心跳功能
+        enable_websocket_ping = conn.config.get("enable_websocket_ping", False)
+        if not enable_websocket_ping:
+            conn.logger.debug(f"WebSocket心跳功能未启用，忽略PING消息")
+            return
+
+        try:
+            conn.logger.debug(f"收到PING消息，发送PONG响应")
+            conn.last_activity_time = time.time() * 1000
+            # 构造PONG响应消息
+            pong_message = {
+                "type": "pong",
+                "timestamp": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
+            }
+
+            # 发送PONG响应
+            await conn.websocket.send(json.dumps(pong_message))
+
+        except Exception as e:
+            conn.logger.error(f"处理PING消息时发生错误: {e}")

main/xingxing-server/core/handle/textHandler/serverMessageHandler.py

@@ -0,0 +1,92 @@
+import asyncio
+import json
+from typing import Dict, Any
+
+from core.handle.textMessageHandler import TextMessageHandler
+from core.handle.textMessageType import TextMessageType
+from core.providers.tools.device_mcp import handle_mcp_message
+
+TAG = __name__
+
+class ServerTextMessageHandler(TextMessageHandler):
+    """MCP消息处理器"""
+
+    @property
+    def message_type(self) -> TextMessageType:
+        return TextMessageType.SERVER
+
+    async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
+        # 如果配置是从API读取的，则需要验证secret
+        if not conn.read_config_from_api:
+            return
+        # 获取post请求的secret
+        post_secret = msg_json.get("content", {}).get("secret", "")
+        secret = conn.config["manager-api"].get("secret", "")
+        # 如果secret不匹配，则返回
+        if post_secret != secret:
+            await conn.websocket.send(
+                json.dumps(
+                    {
+                        "type": "server",
+                        "status": "error",
+                        "message": "服务器密钥验证失败",
+                    }
+                )
+            )
+            return
+        # 动态更新配置
+        if msg_json["action"] == "update_config":
+            try:
+                # 更新WebSocketServer的配置
+                if not conn.server:
+                    await conn.websocket.send(
+                        json.dumps(
+                            {
+                                "type": "server",
+                                "status": "error",
+                                "message": "无法获取服务器实例",
+                                "content": {"action": "update_config"},
+                            }
+                        )
+                    )
+                    return
+
+                if not await conn.server.update_config():
+                    await conn.websocket.send(
+                        json.dumps(
+                            {
+                                "type": "server",
+                                "status": "error",
+                                "message": "更新服务器配置失败",
+                                "content": {"action": "update_config"},
+                            }
+                        )
+                    )
+                    return
+
+                # 发送成功响应
+                await conn.websocket.send(
+                    json.dumps(
+                        {
+                            "type": "server",
+                            "status": "success",
+                            "message": "配置更新成功",
+                            "content": {"action": "update_config"},
+                        }
+                    )
+                )
+            except Exception as e:
+                conn.logger.bind(tag=TAG).error(f"更新配置失败: {str(e)}")
+                await conn.websocket.send(
+                    json.dumps(
+                        {
+                            "type": "server",
+                            "status": "error",
+                            "message": f"更新配置失败: {str(e)}",
+                            "content": {"action": "update_config"},
+                        }
+                    )
+                )
+        # 重启服务器
+        elif msg_json["action"] == "restart":
+            await conn.handle_restart(msg_json)

main/xingxing-server/core/handle/textMessageHandler.py

@@ -0,0 +1,21 @@
+from abc import abstractmethod, ABC
+from typing import Dict, Any
+
+from core.handle.textMessageType import TextMessageType
+
+TAG = __name__
+
+
+class TextMessageHandler(ABC):
+    """消息处理器抽象基类"""
+
+    @abstractmethod
+    async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
+        """处理消息的抽象方法"""
+        pass
+
+    @property
+    @abstractmethod
+    def message_type(self) -> TextMessageType:
+        """返回处理的消息类型"""
+        pass

main/xingxing-server/core/handle/textMessageHandlerRegistry.py

@@ -0,0 +1,49 @@
+from typing import Dict, Optional
+
+from core.handle.textHandler.abortMessageHandler import AbortTextMessageHandler
+from core.handle.textHandler.helloMessageHandler import HelloTextMessageHandler
+from core.handle.textHandler.iotMessageHandler import IotTextMessageHandler
+from core.handle.textHandler.listenMessageHandler import ListenTextMessageHandler
+from core.handle.textHandler.mcpMessageHandler import McpTextMessageHandler
+from core.handle.textHandler.telemetryMessageHandler import TelemetryTextMessageHandler
+from core.handle.textMessageHandler import TextMessageHandler
+from core.handle.textHandler.serverMessageHandler import ServerTextMessageHandler
+from core.handle.textHandler.pingMessageHandler import PingMessageHandler
+
+TAG = __name__
+
+
+class TextMessageHandlerRegistry:
+    """消息处理器注册表"""
+
+    def __init__(self):
+        self._handlers: Dict[str, TextMessageHandler] = {}
+        self._register_default_handlers()
+
+    def _register_default_handlers(self) -> None:
+        """注册默认的消息处理器"""
+        handlers = [
+            HelloTextMessageHandler(),
+            AbortTextMessageHandler(),
+            ListenTextMessageHandler(),
+            IotTextMessageHandler(),
+            McpTextMessageHandler(),
+            TelemetryTextMessageHandler(),
+            ServerTextMessageHandler(),
+            PingMessageHandler(),
+        ]
+
+        for handler in handlers:
+            self.register_handler(handler)
+
+    def register_handler(self, handler: TextMessageHandler) -> None:
+        """注册消息处理器"""
+        self._handlers[handler.message_type.value] = handler
+
+    def get_handler(self, message_type: str) -> Optional[TextMessageHandler]:
+        """获取消息处理器"""
+        return self._handlers.get(message_type)
+
+    def get_supported_types(self) -> list:
+        """获取支持的消息类型"""
+        return list(self._handlers.keys())

main/xingxing-server/core/handle/textMessageProcessor.py

@@ -0,0 +1,46 @@
+import json
+import time
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+
+from core.handle.textMessageHandlerRegistry import TextMessageHandlerRegistry
+
+TAG = __name__
+
+
+class TextMessageProcessor:
+    """Main dispatcher for inbound text websocket frames."""
+
+    def __init__(self, registry: TextMessageHandlerRegistry):
+        self.registry = registry
+
+    async def process_message(self, conn: "ConnectionHandler", message: str) -> None:
+        """Route JSON text messages to the registered message handlers."""
+        try:
+            # Count MCP replies and telemetry frames as live activity so the
+            # connection does not get closed while app-side GPS streaming is active.
+            conn.last_activity_time = time.time() * 1000
+
+            msg_json = json.loads(message)
+
+            if isinstance(msg_json, dict):
+                message_type = msg_json.get("type")
+                if message_type == "telemetry":
+                    conn.logger.bind(tag=TAG).debug("收到 telemetry 消息")
+                else:
+                    conn.logger.bind(tag=TAG).info(f"收到{message_type}消息：{message}")
+
+                handler = self.registry.get_handler(message_type)
+                if handler:
+                    await handler.handle(conn, msg_json)
+                else:
+                    conn.logger.bind(tag=TAG).error(f"收到未知类型消息：{message}")
+            elif isinstance(msg_json, int):
+                conn.logger.bind(tag=TAG).info(f"收到数字消息：{message}")
+                await conn.websocket.send(message)
+
+        except json.JSONDecodeError:
+            conn.logger.bind(tag=TAG).error(f"解析到错误的消息：{message}")
+            await conn.websocket.send(message)

main/xingxing-server/core/handle/textMessageType.py

@@ -0,0 +1,13 @@
+from enum import Enum
+
+
+class TextMessageType(Enum):
+    """消息类型枚举"""
+    HELLO = "hello"
+    ABORT = "abort"
+    LISTEN = "listen"
+    IOT = "iot"
+    MCP = "mcp"
+    TELEMETRY = "telemetry"
+    SERVER = "server"
+    PING = "ping"

main/xingxing-server/core/http_server.py

@@ -0,0 +1,110 @@
+import asyncio
+from aiohttp import web
+from config.logger import setup_logging
+from core.api.app_gps_handler import AppGpsHandler
+from core.api.ota_handler import OTAHandler
+from core.api.vision_handler import VisionHandler
+
+TAG = __name__
+
+
+class SimpleHttpServer:
+    def __init__(self, config: dict, ws_server=None):
+        self.config = config
+        self.logger = setup_logging()
+        self.ws_server = ws_server
+        self.app_gps_handler = AppGpsHandler(config, ws_server) if ws_server else None
+        self.ota_handler = OTAHandler(config)
+        self.vision_handler = VisionHandler(config)
+
+    def _get_websocket_url(self, local_ip: str, port: int) -> str:
+        """获取websocket地址
+
+        Args:
+            local_ip: 本地IP地址
+            port: 端口号
+
+        Returns:
+            str: websocket地址
+        """
+        server_config = self.config["server"]
+        websocket_config = server_config.get("websocket")
+
+        if websocket_config and "你" not in websocket_config:
+            return websocket_config
+        else:
+            return f"ws://{local_ip}:{port}/xiaozhi/v1/"
+
+    async def start(self):
+        try:
+            server_config = self.config["server"]
+            read_config_from_api = self.config.get("read_config_from_api", False)
+            host = server_config.get("ip", "0.0.0.0")
+            port = int(server_config.get("http_port", 8003))
+
+            if port:
+                app = web.Application()
+
+                if not read_config_from_api:
+                    # 如果没有开启智控台，只是单模块运行，就需要再添加简单OTA接口，用于下发websocket接口
+                    app.add_routes(
+                        [
+                            web.get("/xiaozhi/ota/", self.ota_handler.handle_get),
+                            web.post("/xiaozhi/ota/", self.ota_handler.handle_post),
+                            web.options(
+                                "/xiaozhi/ota/", self.ota_handler.handle_options
+                            ),
+                            # 下载接口，仅提供 data/bin/*.bin 下载
+                            web.get(
+                                "/xiaozhi/ota/download/{filename}",
+                                self.ota_handler.handle_download,
+                            ),
+                            web.options(
+                                "/xiaozhi/ota/download/{filename}",
+                                self.ota_handler.handle_options,
+                            ),
+                        ]
+                    )
+                # 添加路由
+                app.add_routes(
+                    [
+                        web.get("/mcp/vision/explain", self.vision_handler.handle_get),
+                        web.post(
+                            "/mcp/vision/explain", self.vision_handler.handle_post
+                        ),
+                        web.options(
+                            "/mcp/vision/explain", self.vision_handler.handle_options
+                        ),
+                    ]
+                )
+
+                # 运行服务
+                if self.app_gps_handler:
+                    app.add_routes(
+                        [
+                            web.get("/app/gps/latest", self.app_gps_handler.handle_get_latest),
+                            web.get("/app/gps/ws", self.app_gps_handler.handle_ws),
+                            web.get("/app/risk-alerts", self.app_gps_handler.handle_get_risk_alerts),
+                            web.post("/app/risk-alerts", self.app_gps_handler.handle_post_risk_alerts),
+                            web.delete("/app/risk-alerts/{alert_id}", self.app_gps_handler.handle_delete_risk_alert),
+                            web.options("/app/gps/latest", self.app_gps_handler.handle_options),
+                            web.options("/app/gps/ws", self.app_gps_handler.handle_options),
+                            web.options("/app/risk-alerts", self.app_gps_handler.handle_options),
+                            web.options("/app/risk-alerts/{alert_id}", self.app_gps_handler.handle_options),
+                        ]
+                    )
+
+                runner = web.AppRunner(app)
+                await runner.setup()
+                site = web.TCPSite(runner, host, port)
+                await site.start()
+
+                # 保持服务运行
+                while True:
+                    await asyncio.sleep(3600)  # 每隔 1 小时检查一次
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"HTTP服务器启动失败: {e}")
+            import traceback
+
+            self.logger.bind(tag=TAG).error(f"错误堆栈: {traceback.format_exc()}")
+            raise

main/xingxing-server/core/providers/asr/aliyun.py

@@ -0,0 +1,236 @@
+import http.client
+import json
+import asyncio
+from typing import Optional, Tuple, List
+import os
+import uuid
+import hmac
+import hashlib
+import base64
+import requests
+from urllib import parse
+import time
+from datetime import datetime
+from config.logger import setup_logging
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.dto.dto import InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+
+class AccessToken:
+    @staticmethod
+    def _encode_text(text):
+        encoded_text = parse.quote_plus(text)
+        return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
+
+    @staticmethod
+    def _encode_dict(dic):
+        keys = dic.keys()
+        dic_sorted = [(key, dic[key]) for key in sorted(keys)]
+        encoded_text = parse.urlencode(dic_sorted)
+        return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
+
+    @staticmethod
+    def create_token(access_key_id, access_key_secret):
+        parameters = {
+            "AccessKeyId": access_key_id,
+            "Action": "CreateToken",
+            "Format": "JSON",
+            "RegionId": "cn-shanghai",
+            "SignatureMethod": "HMAC-SHA1",
+            "SignatureNonce": str(uuid.uuid1()),
+            "SignatureVersion": "1.0",
+            "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "Version": "2019-02-28",
+        }
+        # 构造规范化的请求字符串
+        query_string = AccessToken._encode_dict(parameters)
+        # print('规范化的请求字符串: %s' % query_string)
+        # 构造待签名字符串
+        string_to_sign = (
+            "GET"
+            + "&"
+            + AccessToken._encode_text("/")
+            + "&"
+            + AccessToken._encode_text(query_string)
+        )
+        # print('待签名的字符串: %s' % string_to_sign)
+        # 计算签名
+        secreted_string = hmac.new(
+            bytes(access_key_secret + "&", encoding="utf-8"),
+            bytes(string_to_sign, encoding="utf-8"),
+            hashlib.sha1,
+        ).digest()
+        signature = base64.b64encode(secreted_string)
+        # print('签名: %s' % signature)
+        # 进行URL编码
+        signature = AccessToken._encode_text(signature)
+        # print('URL编码后的签名: %s' % signature)
+        # 调用服务
+        full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (
+            signature,
+            query_string,
+        )
+        # print('url: %s' % full_url)
+        # 提交HTTP GET请求
+        response = requests.get(full_url)
+        if response.ok:
+            root_obj = response.json()
+            key = "Token"
+            if key in root_obj:
+                token = root_obj[key]["Id"]
+                expire_time = root_obj[key]["ExpireTime"]
+                return token, expire_time
+        # print(response.text)
+        return None, None
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
+        self.interface_type = InterfaceType.NON_STREAM
+        """阿里云ASR初始化"""
+        # 新增空值判断逻辑
+        self.access_key_id = config.get("access_key_id")
+        self.access_key_secret = config.get("access_key_secret")
+
+        self.app_key = config.get("appkey")
+        self.host = "nls-gateway-cn-shanghai.aliyuncs.com"
+        self.base_url = f"https://{self.host}/stream/v1/asr"
+        self.sample_rate = 16000
+        self.format = "wav"
+        self.output_dir = config.get("output_dir", "./audio_output")
+        self.delete_audio_file = delete_audio_file
+
+        if self.access_key_id and self.access_key_secret:
+            # 使用密钥对生成临时token
+            self._refresh_token()
+        else:
+            # 直接使用预生成的长期token
+            self.token = config.get("token")
+            self.expire_time = None
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def _refresh_token(self):
+        """刷新Token并记录过期时间"""
+        if self.access_key_id and self.access_key_secret:
+            self.token, expire_time_str = AccessToken.create_token(
+                self.access_key_id, self.access_key_secret
+            )
+            if not expire_time_str:
+                raise ValueError("无法获取有效的Token过期时间")
+
+            try:
+                # 统一转换为字符串处理
+                expire_str = str(expire_time_str).strip()
+
+                if expire_str.isdigit():
+                    expire_time = datetime.fromtimestamp(int(expire_str))
+                else:
+                    expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ")
+                self.expire_time = expire_time.timestamp() - 60
+            except Exception as e:
+                raise ValueError(f"无效的过期时间格式: {expire_str}") from e
+
+        else:
+            self.expire_time = None
+
+        if not self.token:
+            raise ValueError("无法获取有效的访问Token")
+
+    def _is_token_expired(self):
+        """检查Token是否过期"""
+        if not self.expire_time:
+            return False  # 长期Token不过期
+        # 新增调试日志
+        # current_time = time.time()
+        # remaining = self.expire_time - current_time
+        # print(f"Token过期检查: 当前时间 {datetime.fromtimestamp(current_time)} | "
+        #              f"过期时间 {datetime.fromtimestamp(self.expire_time)} | "
+        #              f"剩余 {remaining:.2f}秒")
+        return time.time() > self.expire_time
+
+    def _construct_request_url(self) -> str:
+        """构造请求URL，包含参数"""
+        request = f"{self.base_url}?appkey={self.app_key}"
+        request += f"&format={self.format}"
+        request += f"&sample_rate={self.sample_rate}"
+        request += "&enable_punctuation_prediction=true"
+        request += "&enable_inverse_text_normalization=true"
+        request += "&enable_voice_detection=false"
+        return request
+
+    async def _send_request(self, pcm_data: bytes) -> Optional[str]:
+        """发送请求到阿里云ASR服务"""
+        try:
+            # 设置HTTP头
+            headers = {
+                "X-NLS-Token": self.token,
+                "Content-type": "application/octet-stream",
+                "Content-Length": str(len(pcm_data)),
+            }
+
+            # 创建连接并发送请求
+            conn = http.client.HTTPSConnection(self.host)
+            request_url = self._construct_request_url()
+
+            loop = asyncio.get_event_loop()
+            await loop.run_in_executor(
+                None,
+                lambda: conn.request(
+                    method="POST", url=request_url, body=pcm_data, headers=headers
+                ),
+            )
+
+            # 获取响应
+            response = await loop.run_in_executor(None, conn.getresponse)
+            body = await loop.run_in_executor(None, response.read)
+            conn.close()
+
+            # 解析响应
+            try:
+                body_json = json.loads(body)
+                status = body_json.get("status")
+
+                if status == 20000000:
+                    result = body_json.get("result", "")
+                    logger.bind(tag=TAG).debug(f"ASR结果: {result}")
+                    return result
+                else:
+                    logger.bind(tag=TAG).error(f"ASR失败，状态码: {status}")
+                    return None
+
+            except ValueError:
+                logger.bind(tag=TAG).error("响应不是JSON格式")
+                return None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True)
+            return None
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        if self._is_token_expired():
+            logger.warning("Token已过期，正在自动刷新...")
+            self._refresh_token()
+
+        try:
+            if artifacts is None:
+                return "", None
+            # 发送请求并获取文本
+            text = await self._send_request(artifacts.pcm_bytes)
+
+            if text:
+                return text, artifacts.file_path
+
+            return "", artifacts.file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", None

main/xingxing-server/core/providers/asr/aliyun_stream.py

@@ -0,0 +1,346 @@
+import json
+import time
+import uuid
+import hmac
+import base64
+import hashlib
+import asyncio
+import requests
+import websockets
+import opuslib_next
+from urllib import parse
+from datetime import datetime
+from config.logger import setup_logging
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.dto.dto import InterfaceType
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+
+TAG = __name__
+logger = setup_logging()
+
+
+class AccessToken:
+    @staticmethod
+    def _encode_text(text):
+        encoded_text = parse.quote_plus(text)
+        return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
+
+    @staticmethod
+    def _encode_dict(dic):
+        keys = dic.keys()
+        dic_sorted = [(key, dic[key]) for key in sorted(keys)]
+        encoded_text = parse.urlencode(dic_sorted)
+        return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
+
+    @staticmethod
+    def create_token(access_key_id, access_key_secret):
+        parameters = {
+            "AccessKeyId": access_key_id,
+            "Action": "CreateToken",
+            "Format": "JSON",
+            "RegionId": "cn-shanghai",
+            "SignatureMethod": "HMAC-SHA1",
+            "SignatureNonce": str(uuid.uuid1()),
+            "SignatureVersion": "1.0",
+            "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "Version": "2019-02-28",
+        }
+        query_string = AccessToken._encode_dict(parameters)
+        string_to_sign = (
+            "GET" + "&" + AccessToken._encode_text("/") + "&" + AccessToken._encode_text(query_string)
+        )
+        secreted_string = hmac.new(
+            bytes(access_key_secret + "&", encoding="utf-8"),
+            bytes(string_to_sign, encoding="utf-8"),
+            hashlib.sha1,
+        ).digest()
+        signature = base64.b64encode(secreted_string)
+        signature = AccessToken._encode_text(signature)
+        full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (signature, query_string)
+        response = requests.get(full_url)
+        if response.ok:
+            root_obj = response.json()
+            if "Token" in root_obj:
+                return root_obj["Token"]["Id"], root_obj["Token"]["ExpireTime"]
+        return None, None
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__()
+        self.interface_type = InterfaceType.STREAM
+        self.config = config
+        self.text = ""
+        self.decoder = opuslib_next.Decoder(16000, 1)
+        self.asr_ws = None
+        self.forward_task = None
+        self.is_processing = False
+        self.server_ready = False  # 服务器准备状态
+
+        # 基础配置
+        self.access_key_id = config.get("access_key_id")
+        self.access_key_secret = config.get("access_key_secret")
+        self.appkey = config.get("appkey")
+        self.token = config.get("token")
+        self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com")
+        # 如果配置的是内网地址（包含-internal.aliyuncs.com），则使用ws协议，默认是wss协议
+        if "-internal." in self.host:
+            self.ws_url = f"ws://{self.host}/ws/v1"
+        else:
+            # 默认使用wss协议
+            self.ws_url = f"wss://{self.host}/ws/v1"
+
+        self.max_sentence_silence = config.get("max_sentence_silence")
+        self.output_dir = config.get("output_dir", "./audio_output")
+        self.delete_audio_file = delete_audio_file
+        self.expire_time = None
+
+        self.task_id = uuid.uuid4().hex
+
+        # Token管理
+        if self.access_key_id and self.access_key_secret:
+            self._refresh_token()
+        elif not self.token:
+            raise ValueError("必须提供access_key_id+access_key_secret或者直接提供token")
+
+    def _refresh_token(self):
+        """刷新Token"""
+        self.token, expire_time_str = AccessToken.create_token(self.access_key_id, self.access_key_secret)
+        if not self.token:
+            raise ValueError("无法获取有效的访问Token")
+        
+        try:
+            expire_str = str(expire_time_str).strip()
+            if expire_str.isdigit():
+                expire_time = datetime.fromtimestamp(int(expire_str))
+            else:
+                expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ")
+            self.expire_time = expire_time.timestamp() - 60
+        except:
+            self.expire_time = None
+
+    def _is_token_expired(self):
+        """检查Token是否过期"""
+        return self.expire_time and time.time() > self.expire_time
+
+    async def open_audio_channels(self, conn):
+        await super().open_audio_channels(conn)
+
+    async def receive_audio(self, conn, audio, audio_have_voice):
+        # 先调用父类方法处理基础逻辑
+        await super().receive_audio(conn, audio, audio_have_voice)
+
+        # 只在有声音且没有连接时建立连接（排除正在停止的情况）
+        if audio_have_voice and not self.is_processing and not self.asr_ws:
+            try:
+                await self._start_recognition(conn)
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}")
+                await self._cleanup()
+                return
+
+        if self.asr_ws and self.is_processing and self.server_ready:
+            try:
+                pcm_frame = self.decoder.decode(audio, 960)
+                await self.asr_ws.send(pcm_frame)
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"发送音频失败: {str(e)}")
+                await self._cleanup()
+
+    async def _start_recognition(self, conn: "ConnectionHandler"):
+        """开始识别会话"""
+        if self._is_token_expired():
+            self._refresh_token()
+        
+        # 建立连接
+        headers = {"X-NLS-Token": self.token}
+        self.asr_ws = await websockets.connect(
+            self.ws_url,
+            additional_headers=headers,
+            max_size=1000000000,
+            ping_interval=None,
+            ping_timeout=None,
+            close_timeout=5,
+        )
+
+        self.task_id = uuid.uuid4().hex
+
+        logger.bind(tag=TAG).debug(f"WebSocket连接建立成功, task_id: {self.task_id}")
+
+        self.is_processing = True
+        self.server_ready = False  # 重置服务器准备状态
+        self.forward_task = asyncio.create_task(self._forward_results(conn))
+
+        # 发送开始请求
+        start_request = {
+            "header": {
+                "namespace": "SpeechTranscriber",
+                "name": "StartTranscription",
+                "message_id": uuid.uuid4().hex,
+                "task_id": self.task_id,
+                "appkey": self.appkey
+            },
+            "payload": {
+                "format": "pcm",
+                "sample_rate": 16000,
+                "enable_intermediate_result": True,
+                "enable_punctuation_prediction": True,
+                "enable_inverse_text_normalization": True,
+                "max_sentence_silence": self.max_sentence_silence,
+                "enable_voice_detection": False,
+            }
+        }
+        await self.asr_ws.send(json.dumps(start_request, ensure_ascii=False))
+        logger.bind(tag=TAG).debug("已发送开始请求，等待服务器准备...")
+
+    async def _forward_results(self, conn: "ConnectionHandler"):
+        """转发识别结果"""
+        try:
+            while not conn.stop_event.is_set():
+                # 获取当前连接的音频数据
+                audio_data = conn.asr_audio
+                try:
+                    response = await self.asr_ws.recv()
+                    result = json.loads(response)
+
+                    header = result.get("header", {})
+                    payload = result.get("payload", {})
+                    message_name = header.get("name", "")
+                    status = header.get("status", 0)
+
+                    if status != 20000000:
+                        if status == 40010004:
+                            logger.bind(tag=TAG).warning(f"请在服务端响应完成后再关闭链接，状态码: {status}")
+                            break
+                        if status in [40000004, 40010003]:  # 连接超时或客户端断开
+                            logger.bind(tag=TAG).warning(f"连接问题，状态码: {status}")
+                            break
+                        elif status in [40270002, 40270003]:  # 音频问题
+                            logger.bind(tag=TAG).warning(f"音频处理问题，状态码: {status}")
+                            continue
+                        else:
+                            logger.bind(tag=TAG).error(f"识别错误，状态码: {status}, 消息: {header.get('status_text', '')}")
+                            continue
+
+                    # 收到TranscriptionStarted表示服务器准备好接收音频数据
+                    if message_name == "TranscriptionStarted":
+                        self.server_ready = True
+                        logger.bind(tag=TAG).debug("服务器已准备，开始发送缓存音频...")
+
+                        # 发送缓存音频
+                        if conn.asr_audio:
+                            for cached_audio in conn.asr_audio[-10:]:
+                                try:
+                                    pcm_frame = self.decoder.decode(cached_audio, 960)
+                                    await self.asr_ws.send(pcm_frame)
+                                except Exception as e:
+                                    logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}")
+                                    break
+                        continue
+                    elif message_name == "SentenceEnd":
+                        # 句子结束（每个句子都会触发）
+                        text = payload.get("result", "")
+                        if text:
+                            logger.bind(tag=TAG).info(f"识别到文本: {text}")
+
+                            # 手动模式下累积识别结果
+                            if conn.client_listen_mode == "manual":
+                                if self.text:
+                                    self.text += text
+                                else:
+                                    self.text = text
+
+                                # 手动模式下，只有在收到stop信号后才触发处理（仅处理一次）
+                                if conn.client_voice_stop:
+                                    logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
+                                    await self.handle_voice_stop(conn, audio_data)
+                                    break
+                            else:
+                                # 自动模式下直接覆盖
+                                self.text = text
+                                await self.handle_voice_stop(conn, audio_data)
+                                break
+
+                except asyncio.TimeoutError:
+                    logger.bind(tag=TAG).error("接收结果超时")
+                    break
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).info("ASR服务连接已关闭")
+                    self.is_processing = False
+                    break
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}")
+                    break
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}")
+        finally:
+            # 清理连接的音频缓存
+            await self._cleanup()
+            conn.reset_audio_states()
+
+    async def _send_stop_request(self):
+        """发送停止识别请求（不关闭连接）"""
+        if self.asr_ws:
+            try:
+                # 先停止音频发送
+                self.is_processing = False
+
+                stop_msg = {
+                    "header": {
+                        "namespace": "SpeechTranscriber",
+                        "name": "StopTranscription",
+                        "message_id": uuid.uuid4().hex,
+                        "task_id": self.task_id,
+                        "appkey": self.appkey
+                    }
+                }
+                logger.bind(tag=TAG).debug("停止识别请求已发送")
+                await self.asr_ws.send(json.dumps(stop_msg, ensure_ascii=False))
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"发送停止识别请求失败: {e}")
+
+    async def _cleanup(self):
+        """清理资源（关闭连接）"""
+        logger.bind(tag=TAG).debug(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
+
+        # 状态重置
+        self.is_processing = False
+        self.server_ready = False
+        logger.bind(tag=TAG).debug("ASR状态已重置")
+
+        # 关闭连接
+        if self.asr_ws:
+            try:
+                logger.bind(tag=TAG).debug("正在关闭WebSocket连接")
+                await asyncio.wait_for(self.asr_ws.close(), timeout=2.0)
+                logger.bind(tag=TAG).debug("WebSocket连接已关闭")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
+            finally:
+                self.asr_ws = None
+
+        # 清理任务引用
+        self.forward_task = None
+
+        logger.bind(tag=TAG).debug("ASR会话清理完成")
+
+    async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
+        """获取识别结果"""
+        result = self.text
+        self.text = ""
+        return result, None
+
+    async def close(self):
+        """关闭资源"""
+        await self._cleanup()
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Aliyun decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Aliyun decoder资源时出错: {e}")

main/xingxing-server/core/providers/asr/aliyunbl_stream.py

@@ -0,0 +1,330 @@
+import json
+import uuid
+import asyncio
+import websockets
+import opuslib_next
+from typing import List, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+
+from config.logger import setup_logging
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.dto.dto import InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__()
+        self.interface_type = InterfaceType.STREAM
+        self.config = config
+        self.text = ""
+        self.decoder = opuslib_next.Decoder(16000, 1)
+        self.asr_ws = None
+        self.forward_task = None
+        self.is_processing = False
+        self.server_ready = False  # 服务器准备状态
+        self.task_id = None  # 当前任务ID
+
+        # 阿里百炼配置
+        self.api_key = config.get("api_key")
+        self.model = config.get("model", "paraformer-realtime-v2")
+        self.sample_rate = config.get("sample_rate", 16000)
+        self.format = config.get("format", "pcm")
+
+        # 可选参数
+        self.vocabulary_id = config.get("vocabulary_id")
+        self.disfluency_removal_enabled = config.get("disfluency_removal_enabled", False)
+        self.language_hints = config.get("language_hints")
+        self.semantic_punctuation_enabled = config.get("semantic_punctuation_enabled", False)
+        max_sentence_silence = config.get("max_sentence_silence")
+        self.max_sentence_silence = int(max_sentence_silence) if max_sentence_silence else 200
+        self.multi_threshold_mode_enabled = config.get("multi_threshold_mode_enabled", False)
+        self.punctuation_prediction_enabled = config.get("punctuation_prediction_enabled", True)
+        self.inverse_text_normalization_enabled = config.get("inverse_text_normalization_enabled", True)
+
+        # WebSocket URL
+        self.ws_url = "wss://dashscope.aliyuncs.com/api-ws/v1/inference"
+
+        self.output_dir = config.get("output_dir", "./audio_output")
+        self.delete_audio_file = delete_audio_file
+
+    async def open_audio_channels(self, conn):
+        await super().open_audio_channels(conn)
+
+    async def receive_audio(self, conn, audio, audio_have_voice):
+        # 先调用父类方法处理基础逻辑
+        await super().receive_audio(conn, audio, audio_have_voice)
+
+        # 只在有声音且没有连接时建立连接
+        if audio_have_voice and not self.is_processing and not self.asr_ws:
+            try:
+                await self._start_recognition(conn)
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}")
+                await self._cleanup()
+                return
+
+        # 发送音频数据
+        if self.asr_ws and self.is_processing and self.server_ready:
+            try:
+                pcm_frame = self.decoder.decode(audio, 960)
+                # 直接发送PCM音频数据(二进制)
+                await self.asr_ws.send(pcm_frame)
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"发送音频失败: {str(e)}")
+                await self._cleanup()
+
+    async def _start_recognition(self, conn: "ConnectionHandler"):
+        """开始识别会话"""
+        try:
+            # 如果为手动模式,设置超时时长为最大值
+            if conn.client_listen_mode == "manual":
+                self.max_sentence_silence = 6000
+
+            self.is_processing = True
+            self.task_id = uuid.uuid4().hex
+
+            # 建立WebSocket连接
+            headers = {
+                "Authorization": f"Bearer {self.api_key}"
+            }
+
+            logger.bind(tag=TAG).debug(f"正在连接阿里百炼ASR服务, task_id: {self.task_id}")
+
+            self.asr_ws = await websockets.connect(
+                self.ws_url,
+                additional_headers=headers,
+                max_size=1000000000,
+                ping_interval=None,
+                ping_timeout=None,
+                close_timeout=5,
+            )
+
+            logger.bind(tag=TAG).debug("WebSocket连接建立成功")
+
+            self.server_ready = False
+            self.forward_task = asyncio.create_task(self._forward_results(conn))
+
+            # 发送run-task指令
+            run_task_msg = self._build_run_task_message()
+            await self.asr_ws.send(json.dumps(run_task_msg, ensure_ascii=False))
+            logger.bind(tag=TAG).debug("已发送run-task指令，等待服务器准备...")
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
+            if self.asr_ws:
+                await self.asr_ws.close()
+                self.asr_ws = None
+            self.is_processing = False
+            raise
+
+    def _build_run_task_message(self) -> dict:
+        """构建run-task指令"""
+        message = {
+            "header": {
+                "action": "run-task",
+                "task_id": self.task_id,
+                "streaming": "duplex"
+            },
+            "payload": {
+                "task_group": "audio",
+                "task": "asr",
+                "function": "recognition",
+                "model": self.model,
+                "parameters": {
+                    "format": self.format,
+                    "sample_rate": self.sample_rate,
+                    "disfluency_removal_enabled": self.disfluency_removal_enabled,
+                    "semantic_punctuation_enabled": self.semantic_punctuation_enabled,
+                    "max_sentence_silence": self.max_sentence_silence,
+                    "multi_threshold_mode_enabled": self.multi_threshold_mode_enabled,
+                    "punctuation_prediction_enabled": self.punctuation_prediction_enabled,
+                    "inverse_text_normalization_enabled": self.inverse_text_normalization_enabled,
+                },
+                "input": {}
+            }
+        }
+
+        # 只有当模型名称以v2结尾时才添加vocabulary_id参数
+        if self.model.lower().endswith("v2"):
+            message["payload"]["parameters"]["vocabulary_id"] = self.vocabulary_id
+
+        if self.language_hints:
+            message["payload"]["parameters"]["language_hints"] = self.language_hints
+
+        return message
+
+    async def _forward_results(self, conn: "ConnectionHandler"):
+        """转发识别结果"""
+        try:
+            while not conn.stop_event.is_set():
+                # 获取当前连接的音频数据
+                audio_data = conn.asr_audio
+                try:
+                    response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
+                    result = json.loads(response)
+
+                    header = result.get("header", {})
+                    payload = result.get("payload", {})
+                    event = header.get("event", "")
+
+                    # 处理task-started事件
+                    if event == "task-started":
+                        self.server_ready = True
+                        logger.bind(tag=TAG).debug("服务器已准备，开始发送缓存音频...")
+
+                        # 发送缓存音频
+                        if conn.asr_audio:
+                            for cached_audio in conn.asr_audio[-10:]:
+                                try:
+                                    pcm_frame = self.decoder.decode(cached_audio, 960)
+                                    await self.asr_ws.send(pcm_frame)
+                                except Exception as e:
+                                    logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}")
+                                    break
+                        continue
+
+                    # 处理result-generated事件
+                    elif event == "result-generated":
+                        output = payload.get("output", {})
+                        sentence = output.get("sentence", {})
+
+                        text = sentence.get("text", "")
+                        sentence_end = sentence.get("sentence_end", False)
+                        end_time = sentence.get("end_time")
+
+                        # 判断是否为最终结果(sentence_end为True且end_time不为null)
+                        is_final = sentence_end and end_time is not None
+
+                        if is_final:
+                            logger.bind(tag=TAG).info(f"识别到文本: {text}")
+
+                            # 手动模式下累积识别结果
+                            if conn.client_listen_mode == "manual":
+                                if self.text:
+                                    self.text += text
+                                else:
+                                    self.text = text
+
+                                # 手动模式下,只有在收到stop信号后才触发处理
+                                if conn.client_voice_stop:
+                                    logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
+                                    await self.handle_voice_stop(conn, audio_data)
+                                    break
+                            else:
+                                # 自动模式下直接覆盖
+                                self.text = text
+                                await self.handle_voice_stop(conn, audio_data)
+                                break
+
+                    # 处理task-finished事件
+                    elif event == "task-finished":
+                        logger.bind(tag=TAG).debug("任务已完成")
+                        break
+
+                    # 处理task-failed事件
+                    elif event == "task-failed":
+                        error_code = header.get("error_code", "UNKNOWN")
+                        error_message = header.get("error_message", "未知错误")
+                        logger.bind(tag=TAG).error(f"任务失败: {error_code} - {error_message}")
+                        break
+
+                except asyncio.TimeoutError:
+                    continue
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).info("ASR服务连接已关闭")
+                    self.is_processing = False
+                    break
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}")
+                    break
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}")
+        finally:
+            # 清理连接的音频缓存
+            await self._cleanup()
+            conn.reset_audio_states()
+
+    async def _send_stop_request(self):
+        """发送停止请求(用于手动模式停止录音)"""
+        if self.asr_ws:
+            try:
+                # 先停止音频发送
+                self.is_processing = False
+
+                logger.bind(tag=TAG).debug("收到停止请求，发送finish-task指令")
+                await self._send_finish_task()
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
+
+    async def _send_finish_task(self):
+        """发送finish-task指令"""
+        if self.asr_ws and self.task_id:
+            try:
+                finish_msg = {
+                    "header": {
+                        "action": "finish-task",
+                        "task_id": self.task_id,
+                        "streaming": "duplex"
+                    },
+                    "payload": {
+                        "input": {}
+                    }
+                }
+                await self.asr_ws.send(json.dumps(finish_msg, ensure_ascii=False))
+                logger.bind(tag=TAG).debug("已发送finish-task指令")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"发送finish-task指令失败: {e}")
+
+    async def _cleanup(self):
+        """清理资源"""
+        logger.bind(tag=TAG).debug(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
+
+        # 状态重置
+        self.is_processing = False
+        self.server_ready = False
+        logger.bind(tag=TAG).debug("ASR状态已重置")
+
+        # 关闭连接
+        if self.asr_ws:
+            try:
+                # 先发送finish-task指令
+                await self._send_finish_task()
+                # 等待一小段时间让服务器处理
+                await asyncio.sleep(0.1)
+
+                logger.bind(tag=TAG).debug("正在关闭WebSocket连接")
+                await asyncio.wait_for(self.asr_ws.close(), timeout=2.0)
+                logger.bind(tag=TAG).debug("WebSocket连接已关闭")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
+            finally:
+                self.asr_ws = None
+
+        # 清理任务引用
+        self.forward_task = None
+        self.task_id = None
+
+        logger.bind(tag=TAG).debug("ASR会话清理完成")
+
+    async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
+        """获取识别结果"""
+        result = self.text
+        self.text = ""
+        return result, None
+
+    async def close(self):
+        """关闭资源"""
+        await self._cleanup()
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Aliyun BL decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Aliyun BL decoder资源时出错: {e}")

main/xingxing-server/core/providers/asr/baidu.py

@@ -0,0 +1,74 @@
+import time
+import os
+from typing import Optional, Tuple, List
+from aip import AipSpeech
+from core.providers.asr.base import ASRProviderBase
+from config.logger import setup_logging
+from core.providers.asr.dto.dto import InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool = True):
+        super().__init__()
+        self.interface_type = InterfaceType.NON_STREAM
+        self.app_id = config.get("app_id")
+        self.api_key = config.get("api_key")
+        self.secret_key = config.get("secret_key")
+
+        dev_pid = config.get("dev_pid", "1537")
+        self.dev_pid = int(dev_pid) if dev_pid else 1537
+
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+
+        self.client = AipSpeech(str(self.app_id), self.api_key, self.secret_key)
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        if not opus_data:
+            logger.bind(tag=TAG).warning("音频数据为空！")
+            return None, None
+
+        try:
+            # 检查配置是否已设置
+            if not self.app_id or not self.api_key or not self.secret_key:
+                logger.bind(tag=TAG).error("百度语音识别配置未设置，无法进行识别")
+                return None, None
+
+            if artifacts is None:
+                return "", None
+
+            start_time = time.time()
+            # 识别本地文件
+            result = self.client.asr(
+                artifacts.pcm_bytes,
+                "pcm",
+                16000,
+                {
+                    "dev_pid": str(self.dev_pid),
+                },
+            )
+
+            if result and result["err_no"] == 0:
+                logger.bind(tag=TAG).debug(
+                    f"百度语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
+                )
+                result = result["result"][0]
+                return result, artifacts.file_path
+            else:
+                raise Exception(
+                    f"百度语音识别失败，错误码: {result['err_no']}，错误信息: {result['err_msg']}"
+                )
+                return None, artifacts.file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
+            return None, None

main/xingxing-server/core/providers/asr/base.py

@@ -0,0 +1,381 @@
+import os
+import io
+import wave
+import uuid
+import json
+import time
+import queue
+import shutil
+import asyncio
+import tempfile
+import traceback
+import threading
+import opuslib_next
+
+from abc import ABC, abstractmethod
+from config.logger import setup_logging
+from core.providers.asr.dto.dto import InterfaceType
+from core.handle.receiveAudioHandle import startToChat
+from core.handle.reportHandle import enqueue_asr_report
+from core.utils.util import remove_punctuation_and_length
+from core.handle.receiveAudioHandle import handleAudioMessage
+from typing import Optional, Tuple, List, NamedTuple, TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProviderBase(ABC):
+    def __init__(self):
+        pass
+
+    # 打开音频通道
+    async def open_audio_channels(self, conn: "ConnectionHandler"):
+        conn.asr_priority_thread = threading.Thread(
+            target=self.asr_text_priority_thread, args=(conn,), daemon=True
+        )
+        conn.asr_priority_thread.start()
+
+    # 有序处理ASR音频
+    def asr_text_priority_thread(self, conn: "ConnectionHandler"):
+        while not conn.stop_event.is_set():
+            try:
+                message = conn.asr_audio_queue.get(timeout=1)
+                future = asyncio.run_coroutine_threadsafe(
+                    handleAudioMessage(conn, message),
+                    conn.loop,
+                )
+                future.result()
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logger.bind(tag=TAG).error(
+                    f"处理ASR文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
+                )
+                continue
+
+    # 接收音频
+    async def receive_audio(self, conn: "ConnectionHandler", audio, audio_have_voice):
+        if conn.client_listen_mode == "manual":
+            # 手动模式：缓存音频用于ASR识别
+            conn.asr_audio.append(audio)
+        else:
+            # 自动/实时模式：使用VAD检测
+            conn.asr_audio.append(audio)
+
+            # 如果没有语音，且之前也没有声音，缓存部分音频
+            if not audio_have_voice and not conn.client_have_voice:
+                conn.asr_audio = conn.asr_audio[-10:]
+                return
+
+            # 自动模式下通过VAD检测到语音停止时触发识别
+            if conn.asr.interface_type != InterfaceType.STREAM and conn.client_voice_stop:
+                asr_audio_task = conn.asr_audio.copy()
+                conn.reset_audio_states()
+
+                if len(asr_audio_task) > 15:
+                    await self.handle_voice_stop(conn, asr_audio_task)
+
+    # 处理语音停止
+    async def handle_voice_stop(self, conn: "ConnectionHandler", asr_audio_task: List[bytes]):
+        """并行处理ASR和声纹识别"""
+        try:
+            total_start_time = time.monotonic()
+
+            # 准备音频数据
+            if conn.audio_format == "pcm":
+                pcm_data = asr_audio_task
+            else:
+                pcm_data = self.decode_opus(asr_audio_task)
+
+            combined_pcm_data = b"".join(pcm_data)
+
+            # 预先准备WAV数据
+            wav_data = None
+            if conn.voiceprint_provider and combined_pcm_data:
+                wav_data = self._pcm_to_wav(combined_pcm_data)
+
+            # 定义ASR任务
+            asr_task = self.speech_to_text_wrapper(
+                asr_audio_task, conn.session_id, conn.audio_format
+            )
+
+            if conn.voiceprint_provider and wav_data:
+                voiceprint_task = conn.voiceprint_provider.identify_speaker(
+                    wav_data, conn.session_id
+                )
+                # 并发等待两个结果
+                asr_result, voiceprint_result = await asyncio.gather(
+                    asr_task, voiceprint_task, return_exceptions=True
+                )
+            else:
+                asr_result = await asr_task
+                voiceprint_result = None
+
+            # 记录识别结果 - 检查是否为异常
+            if isinstance(asr_result, Exception):
+                logger.bind(tag=TAG).error(f"ASR识别失败: {asr_result}")
+                raw_text = ""
+            else:
+                raw_text, _ = asr_result
+
+            if isinstance(voiceprint_result, Exception):
+                logger.bind(tag=TAG).error(f"声纹识别失败: {voiceprint_result}")
+                speaker_name = ""
+            else:
+                speaker_name = voiceprint_result
+
+            # 判断 ASR 结果类型
+            if isinstance(raw_text, dict):
+                # FunASR 返回的 dict 格式
+                if speaker_name:
+                    raw_text["speaker"] = speaker_name
+
+                # 记录识别结果
+                if raw_text.get("language"):
+                    logger.bind(tag=TAG).info(f"识别语言: {raw_text['language']}")
+                if raw_text.get("emotion"):
+                    logger.bind(tag=TAG).info(f"识别情绪: {raw_text['emotion']}")
+                if raw_text.get("content"):
+                    logger.bind(tag=TAG).info(f"识别文本: {raw_text['content']}")
+                if speaker_name:
+                    logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
+
+                # 转换为 JSON 字符串用于下游
+                enhanced_text = json.dumps(raw_text, ensure_ascii=False)
+                content_for_length_check = raw_text.get("content", "")
+            else:
+                # 其他 ASR 返回的纯文本
+                if raw_text:
+                    logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
+                if speaker_name:
+                    logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
+
+                # 构建包含说话人信息的JSON字符串
+                enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
+                content_for_length_check = raw_text
+
+            # 性能监控
+            total_time = time.monotonic() - total_start_time
+            logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s")
+
+            # 检查文本长度
+            text_len, _ = remove_punctuation_and_length(content_for_length_check)
+            self.stop_ws_connection()
+
+            if text_len > 0:
+                audio_snapshot = asr_audio_task.copy()
+                enqueue_asr_report(conn, enhanced_text, audio_snapshot)
+                # 使用自定义模块进行上报
+                await startToChat(conn, enhanced_text)
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
+            import traceback
+
+            logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
+
+    def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str:
+        """构建包含说话人信息的文本（仅用于纯文本ASR）"""
+        if speaker_name and speaker_name.strip():
+            return json.dumps(
+                {"speaker": speaker_name, "content": text}, ensure_ascii=False
+            )
+        else:
+            return text
+
+    def _pcm_to_wav(self, pcm_data: bytes) -> bytes:
+        """将PCM数据转换为WAV格式"""
+        if len(pcm_data) == 0:
+            logger.bind(tag=TAG).warning("PCM数据为空，无法转换WAV")
+            return b""
+
+        # 确保数据长度是偶数（16位音频）
+        if len(pcm_data) % 2 != 0:
+            pcm_data = pcm_data[:-1]
+
+        # 创建WAV文件头
+        wav_buffer = io.BytesIO()
+        try:
+            with wave.open(wav_buffer, "wb") as wav_file:
+                wav_file.setnchannels(1)  # 单声道
+                wav_file.setsampwidth(2)  # 16位
+                wav_file.setframerate(16000)  # 16kHz采样率
+                wav_file.writeframes(pcm_data)
+
+            wav_buffer.seek(0)
+            wav_data = wav_buffer.read()
+
+            return wav_data
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"WAV转换失败: {e}")
+            return b""
+
+    def stop_ws_connection(self):
+        pass
+
+    async def close(self):
+        pass
+
+    class AudioArtifacts(NamedTuple):
+        pcm_frames: List[bytes]
+        """PCM音频帧列表"""
+        pcm_bytes: bytes
+        """合并后的PCM音频字节数据"""
+        file_path: Optional[str]
+        """WAV文件路径"""
+        temp_path: Optional[str]
+        """临时WAV文件路径"""
+
+    def get_current_artifacts(self) -> Optional["ASRProviderBase.AudioArtifacts"]:
+        return self._current_artifacts
+
+    def requires_file(self) -> bool:
+        """是否需要文件输入"""
+        return False
+
+    def prefers_temp_file(self) -> bool:
+        """是否优先使用临时文件"""
+        return False
+
+    def build_temp_file(self, pcm_bytes: bytes) -> Optional[str]:
+        try:
+            with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
+                temp_path = temp_file.name
+            with wave.open(temp_path, "wb") as wav_file:
+                wav_file.setnchannels(1)
+                wav_file.setsampwidth(2)
+                wav_file.setframerate(16000)
+                wav_file.writeframes(pcm_bytes)
+            return temp_path
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"临时音频文件生成失败: {e}")
+            return None
+
+    def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
+        """PCM数据保存为WAV文件"""
+        module_name = __name__.split(".")[-1]
+        file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    async def speech_to_text_wrapper(
+        self, opus_data: List[bytes], session_id: str, audio_format="opus"
+    ) -> Tuple[Optional[str], Optional[str]]:
+        file_path = None
+        temp_path = None
+        try:
+            if audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            combined_pcm_data = b"".join(pcm_data)
+
+            free_space = shutil.disk_usage(self.output_dir).free
+            if free_space < len(combined_pcm_data) * 2:
+                raise OSError("磁盘空间不足")
+
+            if self.requires_file() and self.prefers_temp_file():
+                temp_path = self.build_temp_file(combined_pcm_data)
+
+            if (hasattr(self, "delete_audio_file") and not self.delete_audio_file) or (
+                self.requires_file() and not self.prefers_temp_file()
+            ):
+                file_path = self.save_audio_to_file(pcm_data, session_id)
+
+            if len(combined_pcm_data) == 0:
+                artifacts = None
+            else:
+                artifacts = ASRProviderBase.AudioArtifacts(
+                    pcm_frames=pcm_data,
+                    pcm_bytes=combined_pcm_data,
+                    file_path=file_path,
+                    temp_path=temp_path,
+                )
+
+            text, _ = await self.speech_to_text(
+                opus_data, session_id, audio_format, artifacts
+            )
+            return text, file_path
+        except OSError as e:
+            logger.bind(tag=TAG).error(f"文件操作错误: {e}")
+            return None, None
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}")
+            return None, None
+        finally:
+            try:
+                if temp_path and os.path.exists(temp_path):
+                    os.unlink(temp_path)
+                if (
+                    hasattr(self, "delete_audio_file")
+                    and self.delete_audio_file
+                    and file_path
+                    and os.path.exists(file_path)
+                ):
+                    os.remove(file_path)
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"文件清理失败: {e}")
+
+    @abstractmethod
+    async def speech_to_text(
+        self,
+        opus_data: List[bytes],
+        session_id: str,
+        audio_format="opus",
+        artifacts: Optional[AudioArtifacts] = None,
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本
+
+        :param opus_data: 输入的Opus音频数据
+        :param session_id: 会话ID
+        :param audio_format: 音频格式，默认"opus"
+        :param artifacts: 音频工件，包含PCM数据、文件路径等
+        :return: 识别结果文本和文件路径（如果有）
+        """
+        pass
+
+    @staticmethod
+    def decode_opus(opus_data: List[bytes]) -> List[bytes]:
+        """将Opus音频数据解码为PCM数据"""
+        decoder = None
+        try:
+            decoder = opuslib_next.Decoder(16000, 1)
+            pcm_data = []
+            buffer_size = 960  # 每次处理960个采样点 (60ms at 16kHz)
+
+            for i, opus_packet in enumerate(opus_data):
+                try:
+                    if not opus_packet or len(opus_packet) == 0:
+                        continue
+
+                    pcm_frame = decoder.decode(opus_packet, buffer_size)
+                    if pcm_frame and len(pcm_frame) > 0:
+                        pcm_data.append(pcm_frame)
+
+                except opuslib_next.OpusError as e:
+                    logger.bind(tag=TAG).warning(f"Opus解码错误，跳过数据包 {i}: {e}")
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"音频处理错误，数据包 {i}: {e}")
+
+            return pcm_data
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
+            return []
+        finally:
+            if decoder is not None:
+                try:
+                    del decoder
+                except Exception as e:
+                    logger.bind(tag=TAG).debug(f"释放decoder资源时出错: {e}")

main/xingxing-server/core/providers/asr/doubao.py

@@ -0,0 +1,260 @@
+import time
+import os
+import uuid
+import json
+import gzip
+import websockets
+from config.logger import setup_logging
+from typing import Optional, Tuple, List
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.dto.dto import InterfaceType
+
+
+TAG = __name__
+logger = setup_logging()
+
+CLIENT_FULL_REQUEST = 0b0001
+CLIENT_AUDIO_ONLY_REQUEST = 0b0010
+
+NO_SEQUENCE = 0b0000
+NEG_SEQUENCE = 0b0010
+
+SERVER_FULL_RESPONSE = 0b1001
+SERVER_ACK = 0b1011
+SERVER_ERROR_RESPONSE = 0b1111
+
+NO_SERIALIZATION = 0b0000
+JSON = 0b0001
+THRIFT = 0b0011
+CUSTOM_TYPE = 0b1111
+NO_COMPRESSION = 0b0000
+GZIP = 0b0001
+CUSTOM_COMPRESSION = 0b1111
+
+
+def parse_response(res):
+    """
+    protocol_version(4 bits), header_size(4 bits),
+    message_type(4 bits), message_type_specific_flags(4 bits)
+    serialization_method(4 bits) message_compression(4 bits)
+    reserved （8bits) 保留字段
+    header_extensions 扩展头(大小等于 8 * 4 * (header_size - 1) )
+    payload 类似与http 请求体
+    """
+    protocol_version = res[0] >> 4
+    header_size = res[0] & 0x0F
+    message_type = res[1] >> 4
+    message_type_specific_flags = res[1] & 0x0F
+    serialization_method = res[2] >> 4
+    message_compression = res[2] & 0x0F
+    reserved = res[3]
+    header_extensions = res[4 : header_size * 4]
+    payload = res[header_size * 4 :]
+    result = {}
+    payload_msg = None
+    payload_size = 0
+    if message_type == SERVER_FULL_RESPONSE:
+        payload_size = int.from_bytes(payload[:4], "big", signed=True)
+        payload_msg = payload[4:]
+    elif message_type == SERVER_ACK:
+        seq = int.from_bytes(payload[:4], "big", signed=True)
+        result["seq"] = seq
+        if len(payload) >= 8:
+            payload_size = int.from_bytes(payload[4:8], "big", signed=False)
+            payload_msg = payload[8:]
+    elif message_type == SERVER_ERROR_RESPONSE:
+        code = int.from_bytes(payload[:4], "big", signed=False)
+        result["code"] = code
+        payload_size = int.from_bytes(payload[4:8], "big", signed=False)
+        payload_msg = payload[8:]
+    if payload_msg is None:
+        return result
+    if message_compression == GZIP:
+        payload_msg = gzip.decompress(payload_msg)
+    if serialization_method == JSON:
+        payload_msg = json.loads(str(payload_msg, "utf-8"))
+    elif serialization_method != NO_SERIALIZATION:
+        payload_msg = str(payload_msg, "utf-8")
+    result["payload_msg"] = payload_msg
+    result["payload_size"] = payload_size
+    return result
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
+        self.interface_type = InterfaceType.NON_STREAM
+        self.appid = config.get("appid")
+        self.cluster = config.get("cluster")
+        self.access_token = config.get("access_token")
+        self.boosting_table_name = config.get("boosting_table_name", "")
+        self.correct_table_name = config.get("correct_table_name", "")
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+
+        self.host = "openspeech.bytedance.com"
+        self.ws_url = f"wss://{self.host}/api/v2/asr"
+        self.success_code = 1000
+        self.seg_duration = 15000
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    @staticmethod
+    def _generate_header(
+        message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE
+    ) -> bytearray:
+        """Generate protocol header."""
+        header = bytearray()
+        header_size = 1
+        header.append((0b0001 << 4) | header_size)  # Protocol version
+        header.append((message_type << 4) | message_type_specific_flags)
+        header.append((0b0001 << 4) | 0b0001)  # JSON serialization & GZIP compression
+        header.append(0x00)  # reserved
+        return header
+
+    def _construct_request(self, reqid) -> dict:
+        """Construct the request payload."""
+        return {
+            "app": {
+                "appid": f"{self.appid}",
+                "cluster": self.cluster,
+                "token": self.access_token,
+            },
+            "user": {
+                "uid": str(uuid.uuid4()),
+            },
+            "request": {
+                "reqid": reqid,
+                "show_utterances": False,
+                "sequence": 1,
+                "boosting_table_name": self.boosting_table_name,
+                "correct_table_name": self.correct_table_name,
+            },
+            "audio": {
+                "format": "raw",
+                "rate": 16000,
+                "language": "zh-CN",
+                "bits": 16,
+                "channel": 1,
+                "codec": "raw",
+            },
+        }
+
+    async def _send_request(
+        self, audio_data: List[bytes], segment_size: int
+    ) -> Optional[str]:
+        """Send request to Volcano ASR service."""
+        try:
+            auth_header = {"Authorization": "Bearer; {}".format(self.access_token)}
+            async with websockets.connect(
+                self.ws_url, additional_headers=auth_header
+            ) as websocket:
+                # Prepare request data
+                request_params = self._construct_request(str(uuid.uuid4()))
+                payload_bytes = str.encode(json.dumps(request_params))
+                payload_bytes = gzip.compress(payload_bytes)
+                full_client_request = self._generate_header()
+                full_client_request.extend(
+                    (len(payload_bytes)).to_bytes(4, "big")
+                )  # payload size(4 bytes)
+                full_client_request.extend(payload_bytes)  # payload
+
+                # Send header and metadata
+                # full_client_request
+                await websocket.send(full_client_request)
+                res = await websocket.recv()
+                result = parse_response(res)
+                if (
+                    "payload_msg" in result
+                    and result["payload_msg"]["code"] != self.success_code
+                    and result["payload_msg"]["code"] != 1013  # 忽略无有效语音的错误
+                ):
+                    logger.bind(tag=TAG).error(f"ASR error: {result}")
+                    return None
+
+                for seq, (chunk, last) in enumerate(
+                    self.slice_data(audio_data, segment_size), 1
+                ):
+                    if last:
+                        audio_only_request = self._generate_header(
+                            message_type=CLIENT_AUDIO_ONLY_REQUEST,
+                            message_type_specific_flags=NEG_SEQUENCE,
+                        )
+                    else:
+                        audio_only_request = self._generate_header(
+                            message_type=CLIENT_AUDIO_ONLY_REQUEST
+                        )
+                    payload_bytes = gzip.compress(chunk)
+                    audio_only_request.extend(
+                        (len(payload_bytes)).to_bytes(4, "big")
+                    )  # payload size(4 bytes)
+                    audio_only_request.extend(payload_bytes)  # payload
+                    # Send audio data
+                    await websocket.send(audio_only_request)
+
+                # Receive response
+                response = await websocket.recv()
+                result = parse_response(response)
+
+                if (
+                    "payload_msg" in result
+                    and result["payload_msg"]["code"] == self.success_code
+                ):
+                    if len(result["payload_msg"]["result"]) > 0:
+                        return result["payload_msg"]["result"][0]["text"]
+                    return None
+                elif "payload_msg" in result and result["payload_msg"]["code"] == 1013:
+                    # 无有效语音，返回空字符串
+                    return ""
+                else:
+                    logger.bind(tag=TAG).error(f"ASR error: {result}")
+                    return None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
+            return None
+
+    @staticmethod
+    def slice_data(data: bytes, chunk_size: int) -> (list, bool):
+        """
+        slice data
+        :param data: wav data
+        :param chunk_size: the segment size in one request
+        :return: segment data, last flag
+        """
+        data_len = len(data)
+        offset = 0
+        while offset + chunk_size < data_len:
+            yield data[offset : offset + chunk_size], False
+            offset += chunk_size
+        else:
+            yield data[offset:data_len], True
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+
+        try:
+            if artifacts is None:
+                return "", None
+
+            # 直接使用PCM数据
+            # 计算分段大小 (单声道, 16bit, 16kHz采样率)
+            size_per_sec = 1 * 2 * 16000  # nchannels * sampwidth * framerate
+            segment_size = int(size_per_sec * self.seg_duration / 1000)
+
+            # 语音识别
+            start_time = time.time()
+            text = await self._send_request(artifacts.pcm_bytes, segment_size)
+            if text:
+                logger.bind(tag=TAG).debug(
+                    f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
+                )
+                return text, artifacts.file_path
+            return "", artifacts.file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", None

main/xingxing-server/core/providers/asr/doubao_stream.py

@@ -0,0 +1,443 @@
+import json
+import gzip
+import uuid
+import asyncio
+import websockets
+import opuslib_next
+from core.providers.asr.base import ASRProviderBase
+from config.logger import setup_logging
+from core.providers.asr.dto.dto import InterfaceType
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__()
+        self.interface_type = InterfaceType.STREAM
+        self.config = config
+        self.text = ""
+        self.decoder = opuslib_next.Decoder(16000, 1)
+        self.asr_ws = None
+        self.forward_task = None
+        self.is_processing = False  # 添加处理状态标志
+        self._is_stopping = False  # 添加停止标志，防止竞态条件
+
+        # 配置参数
+        self.appid = str(config.get("appid"))
+        self.access_token = config.get("access_token")
+        # 资源ID，用于区分不同的ASR模型（默认1.0模型小时版，v2版本使用seed-asr）
+        self.resource_id = config.get("resource_id", "volc.bigasr.sauc.duration")
+
+        self.boosting_table_name = config.get("boosting_table_name", "")
+        self.correct_table_name = config.get("correct_table_name", "")
+        self.output_dir = config.get("output_dir", "tmp/")
+        self.delete_audio_file = delete_audio_file
+
+        # 火山引擎ASR配置
+        enable_multilingual = config.get("enable_multilingual", False)
+        self.enable_multilingual = (
+            False if str(enable_multilingual).lower() == "false" else True
+        )
+        if self.enable_multilingual:
+            self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_nostream"
+        else:
+            self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async"
+        self.uid = config.get("uid", "streaming_asr_service")
+        self.workflow = config.get(
+            "workflow", "audio_in,resample,partition,vad,fe,decode,itn,nlu_punctuate"
+        )
+        self.result_type = config.get("result_type", "single")
+        self.format = config.get("format", "pcm")
+        self.codec = config.get("codec", "pcm")
+        self.rate = config.get("sample_rate", 16000)
+        # language参数仅在多语种模式(bigmodel_nostream)下有效
+        self.language = config.get("language") if self.enable_multilingual else None
+        self.bits = config.get("bits", 16)
+        self.channel = config.get("channel", 1)
+        self.auth_method = config.get("auth_method", "token")
+        self.secret = config.get("secret", "access_secret")
+        end_window_size = config.get("end_window_size")
+        self.end_window_size = int(end_window_size) if end_window_size else 200
+
+    async def open_audio_channels(self, conn):
+        await super().open_audio_channels(conn)
+
+    async def receive_audio(self, conn: "ConnectionHandler", audio, audio_have_voice):
+        # 先调用父类方法处理基础逻辑
+        await super().receive_audio(conn, audio, audio_have_voice)
+        
+        # 如果本次有声音，且之前没有建立连接
+        if audio_have_voice and self.asr_ws is None and not self.is_processing:
+            try:
+                self.is_processing = True
+                # 建立新的WebSocket连接
+                headers = self.token_auth() if self.auth_method == "token" else None
+                logger.bind(tag=TAG).info(f"正在连接ASR服务，headers: {headers}")
+
+                self.asr_ws = await websockets.connect(
+                    self.ws_url,
+                    additional_headers=headers,
+                    max_size=1000000000,
+                    ping_interval=None,
+                    ping_timeout=None,
+                    close_timeout=10,
+                )
+
+                # 发送初始化请求
+                request_params = self.construct_request(str(uuid.uuid4()))
+                try:
+                    payload_bytes = str.encode(json.dumps(request_params))
+                    payload_bytes = gzip.compress(payload_bytes)
+                    full_client_request = self.generate_header()
+                    full_client_request.extend((len(payload_bytes)).to_bytes(4, "big"))
+                    full_client_request.extend(payload_bytes)
+
+                    logger.bind(tag=TAG).info(f"发送初始化请求: {request_params}")
+                    await self.asr_ws.send(full_client_request)
+
+                    # 等待初始化响应
+                    init_res = await self.asr_ws.recv()
+                    result = self.parse_response(init_res)
+                    logger.bind(tag=TAG).info(f"收到初始化响应: {result}")
+
+                    # 检查初始化响应
+                    if "code" in result and result["code"] != 1000:
+                        error_msg = f"ASR服务初始化失败: {result.get('payload_msg', {}).get('error', '未知错误')}"
+                        logger.bind(tag=TAG).error(error_msg)
+                        raise Exception(error_msg)
+
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"发送初始化请求失败: {str(e)}")
+                    if hasattr(e, "__cause__") and e.__cause__:
+                        logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+                    raise e
+
+                # 启动接收ASR结果的异步任务
+                self.forward_task = asyncio.create_task(self._forward_asr_results(conn))
+
+                # 发送缓存的音频数据
+                if conn.asr_audio and len(conn.asr_audio) > 0:
+                    for cached_audio in conn.asr_audio[-10:]:
+                        try:
+                            pcm_frame = self.decoder.decode(cached_audio, 960)
+                            payload = gzip.compress(pcm_frame)
+                            audio_request = bytearray(
+                                self.generate_audio_default_header()
+                            )
+                            audio_request.extend(len(payload).to_bytes(4, "big"))
+                            audio_request.extend(payload)
+                            await self.asr_ws.send(audio_request)
+                        except Exception as e:
+                            logger.bind(tag=TAG).info(
+                                f"发送缓存音频数据时发生错误: {e}"
+                            )
+
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
+                if hasattr(e, "__cause__") and e.__cause__:
+                    logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+                if self.asr_ws:
+                    await self.asr_ws.close()
+                    self.asr_ws = None
+                self.is_processing = False
+                return
+
+        # 发送当前音频数据
+        if self.asr_ws and self.is_processing and not self._is_stopping:
+            try:
+                pcm_frame = self.decoder.decode(audio, 960)
+                payload = gzip.compress(pcm_frame)
+                audio_request = bytearray(self.generate_audio_default_header())
+                audio_request.extend(len(payload).to_bytes(4, "big"))
+                audio_request.extend(payload)
+                await self.asr_ws.send(audio_request)
+            except Exception as e:
+                logger.bind(tag=TAG).info(f"发送音频数据时发生错误: {e}")
+
+    async def _forward_asr_results(self, conn: "ConnectionHandler"):
+        try:
+            while self.asr_ws and not conn.stop_event.is_set():
+                # 获取当前连接的音频数据
+                audio_data = conn.asr_audio
+                try:
+                    response = await self.asr_ws.recv()
+                    result = self.parse_response(response)
+                    logger.bind(tag=TAG).debug(f"收到ASR结果: {result}")
+
+                    if "payload_msg" in result:
+                        payload = result["payload_msg"]
+                        # 检查是否是错误码1013（无有效语音）
+                        if "code" in payload and payload["code"] == 1013:
+                            # 静默处理，不记录错误日志
+                            continue
+
+                        if "result" in payload:
+                            utterances = payload["result"].get("utterances", [])
+                            # 检查duration和空文本的情况
+                            if (
+                                not self.enable_multilingual  # 注意：多语种模式不返回中间结果，需要等待最终结果
+                                and payload.get("audio_info", {}).get("duration", 0)
+                                > 2000
+                                and not utterances
+                                and not payload["result"].get("text")
+                                and conn.client_listen_mode != "manual"
+                            ):
+                                logger.bind(tag=TAG).error(f"识别文本：空")
+                                self.text = ""
+                                if len(audio_data) > 15:  # 确保有足够音频数据
+                                    await self.handle_voice_stop(conn, audio_data)
+                                break
+
+                            # 专门处理没有文本的识别结果（手动模式下可能已经识别完成但是没松按键）
+                            elif not payload["result"].get("text") and not utterances:
+                                # 多语种模式会持续返回空文本，直到最后返回完整结果，所以需要排除
+                                if self.enable_multilingual:
+                                    continue
+
+                                if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 15:
+                                    logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理")
+                                    await self.handle_voice_stop(conn, audio_data)
+                                    break
+
+                            for utterance in utterances:
+                                if utterance.get("definite", False):
+                                    current_text = utterance["text"]
+                                    logger.bind(tag=TAG).info(
+                                        f"识别到文本: {current_text}"
+                                    )
+
+                                    # 手动模式下累积识别结果
+                                    if conn.client_listen_mode == "manual":
+                                        if self.text:
+                                            self.text += current_text
+                                        else:
+                                            self.text = current_text
+
+                                        # 在接收消息中途时收到停止信号
+                                        if conn.client_voice_stop and len(audio_data) > 0:
+                                            logger.bind(tag=TAG).debug("消息中途收到停止信号，触发处理")
+                                            await self.handle_voice_stop(conn, audio_data)
+                                        break
+                                    else:
+                                        # 自动模式下直接覆盖
+                                        self.text = current_text
+                                        if len(audio_data) > 15:  # 确保有足够音频数据
+                                            await self.handle_voice_stop(
+                                                conn, audio_data
+                                            )
+                                    break
+                        elif "error" in payload:
+                            error_msg = payload.get("error", "未知错误")
+                            logger.bind(tag=TAG).error(f"ASR服务返回错误: {error_msg}")
+                            break
+
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).info("ASR服务连接已关闭")
+                    self.is_processing = False
+                    break
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"处理ASR结果时发生错误: {str(e)}")
+                    if hasattr(e, "__cause__") and e.__cause__:
+                        logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+                    self.is_processing = False
+                    break
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"ASR结果转发任务发生错误: {str(e)}")
+            if hasattr(e, "__cause__") and e.__cause__:
+                logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+        finally:
+            if self.asr_ws:
+                await self.asr_ws.close()
+                self.asr_ws = None
+            self.is_processing = False
+            self._is_stopping = False
+            # 重置所有音频相关状态
+            conn.reset_audio_states()
+
+    def stop_ws_connection(self):
+        if self.asr_ws:
+            asyncio.create_task(self.asr_ws.close())
+            self.asr_ws = None
+        self.is_processing = False
+        self._is_stopping = False
+
+    async def _send_stop_request(self):
+        """发送最后一个音频帧以通知服务器结束"""
+        self._is_stopping = True  # 先标记为停止状态，阻止后续音频发送
+        if self.asr_ws:
+            try:
+                # 发送结束标记的音频帧（gzip压缩的空数据）
+                empty_payload = gzip.compress(b"")
+                last_audio_request = bytearray(
+                    self.generate_last_audio_default_header()
+                )
+                last_audio_request.extend(len(empty_payload).to_bytes(4, "big"))
+                last_audio_request.extend(empty_payload)
+                await self.asr_ws.send(last_audio_request)
+                logger.bind(tag=TAG).debug("已发送结束音频帧")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"发送结束音频帧时出错: {e}")
+
+    def construct_request(self, reqid):
+        req = {
+            "app": {
+                "appid": self.appid,
+                "token": self.access_token,
+            },
+            "user": {"uid": self.uid},
+            "request": {
+                "reqid": reqid,
+                "workflow": self.workflow,
+                "show_utterances": True,
+                "result_type": self.result_type,
+                "sequence": 1,
+                "end_window_size": self.end_window_size,
+                "corpus": {
+                    "boosting_table_name": self.boosting_table_name,
+                    "correct_table_name": self.correct_table_name,
+                }
+            },
+            "audio": {
+                "format": self.format,
+                "codec": self.codec,
+                "rate": self.rate,
+                "bits": self.bits,
+                "channel": self.channel,
+                "sample_rate": self.rate,
+            },
+        }
+
+        # language参数仅在多语种模式下添加
+        if self.enable_multilingual and self.language:
+            req["audio"]["language"] = self.language
+
+        logger.bind(tag=TAG).debug(
+            f"构造请求参数: {json.dumps(req, ensure_ascii=False)}"
+        )
+        return req
+
+    def token_auth(self):
+        return {
+            "X-Api-App-Key": self.appid,
+            "X-Api-Access-Key": self.access_token,
+            "X-Api-Resource-Id": self.resource_id,
+            "X-Api-Connect-Id": str(uuid.uuid4()),
+        }
+
+    def generate_header(
+        self,
+        version=0x01,
+        message_type=0x01,
+        message_type_specific_flags=0x00,
+        serial_method=0x01,
+        compression_type=0x01,
+        reserved_data=0x00,
+        extension_header: bytes = b"",
+    ):
+        header = bytearray()
+        header_size = int(len(extension_header) / 4) + 1
+        header.append((version << 4) | header_size)
+        header.append((message_type << 4) | message_type_specific_flags)
+        header.append((serial_method << 4) | compression_type)
+        header.append(reserved_data)
+        header.extend(extension_header)
+        return header
+
+    def generate_audio_default_header(self):
+        return self.generate_header(
+            version=0x01,
+            message_type=0x02,
+            message_type_specific_flags=0x00,
+            serial_method=0x01,
+            compression_type=0x01,
+        )
+
+    def generate_last_audio_default_header(self):
+        return self.generate_header(
+            version=0x01,
+            message_type=0x02,
+            message_type_specific_flags=0x02,
+            serial_method=0x01,
+            compression_type=0x01,
+        )
+
+    def parse_response(self, res: bytes) -> dict:
+        try:
+            # 检查响应长度
+            if len(res) < 4:
+                logger.bind(tag=TAG).error(f"响应数据长度不足: {len(res)}")
+                return {"error": "响应数据长度不足"}
+
+            # 获取消息头
+            header = res[:4]
+            message_type = header[1] >> 4
+
+            # 如果是错误响应
+            if message_type == 0x0F:  # SERVER_ERROR_RESPONSE
+                code = int.from_bytes(res[4:8], "big", signed=False)
+                msg_length = int.from_bytes(res[8:12], "big", signed=False)
+                error_msg = json.loads(res[12:].decode("utf-8"))
+                return {
+                    "code": code,
+                    "msg_length": msg_length,
+                    "payload_msg": error_msg,
+                }
+
+            # 获取JSON数据
+            try:
+                # 检查字节8-11是否为有效的JSON长度字段
+                # 格式：4字节头 + 4字节序列号 + 4字节长度 + JSON数据
+                length = int.from_bytes(res[8:12], "big")
+                if length > 0 and length <= len(res) - 12:
+                    # 有长度字段，从字节12开始读取指定长度的JSON
+                    json_data = res[12:12 + length].decode("utf-8")
+                else:
+                    # 无长度字段或长度无效，尝试直接解析
+                    json_data = res[8:].decode("utf-8")
+                result = json.loads(json_data)
+                logger.bind(tag=TAG).debug(f"成功解析JSON响应: {result}")
+                return {"payload_msg": result}
+            except (UnicodeDecodeError, json.JSONDecodeError) as e:
+                logger.bind(tag=TAG).error(f"JSON解析失败: {str(e)}")
+                logger.bind(tag=TAG).error(f"原始数据: {res}")
+                raise
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"解析响应失败: {str(e)}")
+            logger.bind(tag=TAG).error(f"原始响应数据: {res.hex()}")
+            raise
+
+    async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
+        result = self.text
+        self.text = ""  # 清空text
+        return result, None
+
+    async def close(self):
+        """资源清理方法"""
+        if self.asr_ws:
+            await self.asr_ws.close()
+            self.asr_ws = None
+        if self.forward_task:
+            self.forward_task.cancel()
+            try:
+                await self.forward_task
+            except asyncio.CancelledError:
+                pass
+            self.forward_task = None
+        self.is_processing = False
+
+        # 显式释放decoder资源
+        if hasattr(self, "decoder") and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Doubao decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Doubao decoder资源时出错: {e}")

main/xingxing-server/core/providers/asr/dto/dto.py

@@ -0,0 +1,9 @@
+from enum import Enum
+from typing import Union, Optional
+
+
+class InterfaceType(Enum):
+    # 接口类型
+    STREAM = "STREAM"  # 流式接口
+    NON_STREAM = "NON_STREAM"  # 非流式接口
+    LOCAL = "LOCAL"  # 本地服务

main/xingxing-server/core/providers/asr/fun_local.py

@@ -0,0 +1,133 @@
+import os
+import io
+import sys
+import time
+import shutil
+import psutil
+import asyncio
+
+from funasr import AutoModel
+from funasr.utils.postprocess_utils import rich_transcription_postprocess
+from config.logger import setup_logging
+from typing import Optional, Tuple, List
+from core.providers.asr.utils import lang_tag_filter
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.dto.dto import InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+MAX_RETRIES = 2
+RETRY_DELAY = 1  # 重试延迟（秒）
+
+
+# 捕获标准输出
+class CaptureOutput:
+    def __enter__(self):
+        self._output = io.StringIO()
+        self._original_stdout = sys.stdout
+        sys.stdout = self._output
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        sys.stdout = self._original_stdout
+        self.output = self._output.getvalue()
+        self._output.close()
+
+        # 将捕获到的内容通过 logger 输出
+        if self.output:
+            logger.bind(tag=TAG).info(self.output.strip())
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
+        
+        # 内存检测，要求大于2G
+        min_mem_bytes = 2 * 1024 * 1024 * 1024
+        total_mem = psutil.virtual_memory().total
+        if total_mem < min_mem_bytes:
+            logger.bind(tag=TAG).error(f"可用内存不足2G，当前仅有 {total_mem / (1024*1024):.2f} MB，可能无法启动FunASR")
+        
+        self.interface_type = InterfaceType.LOCAL
+        self.model_dir = config.get("model_dir")
+        self.output_dir = config.get("output_dir")  # 修正配置键名
+        self.delete_audio_file = delete_audio_file
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+        with CaptureOutput():
+            self.model = AutoModel(
+                model=self.model_dir,
+                vad_kwargs={"max_single_segment_time": 30000},
+                disable_update=True,
+                hub="hf",
+                # device="cuda:0",  # 启用GPU加速
+            )
+
+    @staticmethod
+    def _extract_result_text(result) -> str:
+        if isinstance(result, str):
+            return result
+        if isinstance(result, dict):
+            for key in ("text", "sentence", "result"):
+                value = result.get(key)
+                if isinstance(value, str):
+                    return value
+            return ""
+        if isinstance(result, list) and result:
+            return ASRProvider._extract_result_text(result[0])
+        return ""
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """语音转文本主处理逻辑"""
+        retry_count = 0
+        
+        while retry_count < MAX_RETRIES:
+            try:
+                if artifacts is None:
+                    return "", None
+
+                # 语音识别 - 使用线程池避免阻塞事件循环
+                start_time = time.time()
+                result = await asyncio.to_thread(
+                    self.model.generate,
+                    input=artifacts.pcm_bytes,
+                    cache={},
+                    language="auto",
+                    use_itn=True,
+                    batch_size_s=60,
+                )
+                raw_text = self._extract_result_text(result)
+                if not raw_text:
+                    logger.bind(tag=TAG).warning(
+                        f"FunASR未返回可识别文本，原始结果类型: {type(result).__name__}"
+                    )
+                    return "", artifacts.file_path
+
+                raw_text = rich_transcription_postprocess(raw_text)
+                text = lang_tag_filter(raw_text)
+                if not isinstance(text, dict):
+                    text = {"content": text}
+                logger.bind(tag=TAG).debug(
+                    f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}"
+                )
+
+                return text, artifacts.file_path
+
+            except OSError as e:
+                retry_count += 1
+                if retry_count >= MAX_RETRIES:
+                    logger.bind(tag=TAG).error(
+                        f"语音识别失败（已重试{retry_count}次）: {e}", exc_info=True
+                    )
+                    return "", None
+                logger.bind(tag=TAG).warning(
+                    f"语音识别失败，正在重试（{retry_count}/{MAX_RETRIES}）: {e}"
+                )
+                time.sleep(RETRY_DELAY)
+
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+                return "", None

main/xingxing-server/core/providers/asr/fun_server.py

@@ -0,0 +1,165 @@
+import ssl
+import json
+import asyncio
+import websockets
+
+from config.logger import setup_logging
+from typing import Optional, Tuple, List
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.utils import lang_tag_filter
+from core.providers.asr.dto.dto import InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        """
+        Initialize the ASRProvider with server configuration.
+        :param config: Dictionary containing 'host', 'port', and 'is_ssl'.
+        :param delete_audio_file: Boolean to indicate whether to delete audio files after processing.
+        """
+        super().__init__()
+        self.interface_type = InterfaceType.NON_STREAM
+        self.host = config.get("host", "localhost")
+        self.port = config.get("port", 10095)
+        self.api_key = config.get("api_key", "none")
+        self.is_ssl = str(config.get("is_ssl", True)).lower() in (
+            "true",
+            "1",
+            "yes",
+        )
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+        self.uri = (
+            f"wss://{self.host}:{self.port}"
+            if self.is_ssl
+            else f"ws://{self.host}:{self.port}"
+        )
+        self.ssl_context = ssl.SSLContext() if self.is_ssl else None
+        if self.ssl_context:
+            self.ssl_context.check_hostname = False
+            self.ssl_context.verify_mode = ssl.CERT_NONE
+
+    async def _receive_responses(self, ws) -> None:
+        """
+        Asynchronous generator to receive messages from the WebSocket.
+        Yields each message as it is received.
+        """
+        text = ""
+        while True:
+            try:
+                response = await asyncio.wait_for(ws.recv(), timeout=5)
+                response_data = json.loads(response)
+                logger.bind(tag=TAG).debug(f"Received response: {response_data}")
+                if response_data.get("is_final", True):
+                    text += response_data.get("text", "")
+                    break
+                else:
+                    text += response_data.get("text", "")
+            except asyncio.TimeoutError:
+                logger.bind(tag=TAG).error(
+                    "Timeout while waiting for response from WebSocket."
+                )
+                break
+            except websockets.exceptions.ConnectionClosed as e:
+                logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
+                break
+        return text
+
+    async def _send_data(self, ws, pcm_data: bytes, session_id: str) -> tuple:
+        """
+        Internal method to handle WebSocket communication.
+        Reuses the persistent WebSocket connection if available.
+        :param pcm_data: PCM audio data to send.
+        :param session_id: Unique session identifier.
+        :return: Tuple containing recognized text and optional timestamp.
+        """
+
+        # Send initial configuration message
+        config_message = json.dumps(
+            {
+                "mode": "offline",
+                "chunk_size": [5, 10, 5],
+                "chunk_interval": 10,
+                "wav_name": session_id,
+                "is_speaking": True,
+                "itn": False,
+            }
+        )
+        await ws.send(config_message)
+        logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}")
+
+        # Send PCM data
+        await ws.send(pcm_data)
+        logger.bind(tag=TAG).debug(f"Sent PCM data of length: {len(pcm_data)} bytes")
+
+        # Indicate end of speech
+        end_message = json.dumps({"is_speaking": False})
+        await ws.send(end_message)
+        logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """
+        Convert speech data to text using FunASR.
+        :param opus_data: List of Opus-encoded audio data chunks.
+        :param session_id: Unique session identifier.
+        :return: Tuple containing recognized text and optional timestamp.
+        """
+        
+        if artifacts is None:
+            return "", None
+        auth_header = {"Authorization": "Bearer; {}".format(self.api_key)}
+        async with websockets.connect(
+            self.uri,
+            additional_headers=auth_header,
+            subprotocols=["binary"],
+            ping_interval=None,
+            ssl=self.ssl_context,
+        ) as ws:
+            try:
+                # Use asyncio to handle WebSocket communication
+                send_task = asyncio.create_task(
+                    self._send_data(ws, artifacts.pcm_bytes, session_id)
+                )
+                receive_task = asyncio.create_task(self._receive_responses(ws))
+
+                # Gather tasks with error handling
+                done, pending = await asyncio.wait(
+                    [send_task, receive_task], return_when=asyncio.FIRST_EXCEPTION
+                )
+
+                # Cancel any pending tasks
+                for task in pending:
+                    task.cancel()
+
+                # Check for exceptions in completed tasks
+                for task in done:
+                    if task.exception():
+                        raise task.exception()
+
+                # Get the result from the receive task
+                result = receive_task.result()
+                
+                # match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result)
+                # if match:
+                #     result = match.group(4).strip()
+
+                # Handle tags
+                result = lang_tag_filter(result)
+                return (
+                    result,
+                    artifacts.file_path,
+                )  # Return the recognized text and timestamp (if any)
+
+            except websockets.exceptions.ConnectionClosed as e:
+                logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
+                return "", artifacts.file_path
+            except Exception as e:
+                logger.bind(tag=TAG).error(
+                    f"Error during speech-to-text conversion: {e}", exc_info=True
+                )
+                return "", artifacts.file_path

main/xingxing-server/core/providers/asr/openai.py

@@ -0,0 +1,70 @@
+import time
+import os
+from config.logger import setup_logging
+from typing import Optional, Tuple, List
+from core.providers.asr.dto.dto import InterfaceType
+from core.providers.asr.base import ASRProviderBase
+
+import requests
+
+TAG = __name__
+logger = setup_logging()
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        self.interface_type = InterfaceType.NON_STREAM
+        self.api_key = config.get("api_key")
+        self.api_url = config.get("base_url")
+        self.model = config.get("model_name")        
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def requires_file(self) -> bool:
+        return True
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None) -> Tuple[Optional[str], Optional[str]]:
+        file_path = None
+        try:
+            if artifacts is None:
+                return "", None
+            file_path = artifacts.file_path
+                
+            logger.bind(tag=TAG).info(f"file path: {file_path}")
+            headers = {
+                "Authorization": f"Bearer {self.api_key}",
+            }
+            
+            # 使用data参数传递模型名称
+            data = {
+                "model": self.model
+            }
+
+
+            with open(file_path, "rb") as audio_file:  # 使用with语句确保文件关闭
+                files = {
+                    "file": audio_file
+                }
+
+                start_time = time.time()
+                response = requests.post(
+                    self.api_url,
+                    files=files,
+                    data=data,
+                    headers=headers
+                )
+                logger.bind(tag=TAG).debug(
+                    f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {response.text}"
+                )
+
+            if response.status_code == 200:
+                text = response.json().get("text", "")
+                return text, file_path
+            else:
+                raise Exception(f"API请求失败: {response.status_code} - {response.text}")
+                
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}")
+            return "", None
+        

main/xingxing-server/core/providers/asr/qwen3_asr_flash.py

@@ -0,0 +1,111 @@
+import os
+from typing import Optional, Tuple, List
+import dashscope
+from config.logger import setup_logging
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.dto.dto import InterfaceType
+
+tag = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
+        # 音频文件上传类型，流式文本识别输出
+        self.interface_type = InterfaceType.NON_STREAM
+        """Qwen3-ASR-Flash ASR初始化"""
+        
+        # 配置参数
+        self.api_key = config.get("api_key")
+        if not self.api_key:
+            raise ValueError("Qwen3-ASR-Flash 需要配置 api_key")
+            
+        self.model_name = config.get("model_name", "qwen3-asr-flash")
+        self.output_dir = config.get("output_dir", "./audio_output")
+        self.delete_audio_file = delete_audio_file
+        
+        # ASR选项配置
+        self.enable_lid = config.get("enable_lid", True)  # 自动语种检测
+        self.enable_itn = config.get("enable_itn", True)  # 逆文本归一化
+        self.language = config.get("language", None)  # 指定语种，默认自动检测
+        self.context = config.get("context", "")  # 上下文信息，用于提高识别准确率
+        
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def prefers_temp_file(self) -> bool:
+        return True
+
+    def requires_file(self) -> bool:
+        return True
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        temp_file_path = None
+        file_path = None
+        try:
+            if artifacts is None:
+                return "", None
+            temp_file_path = artifacts.temp_path
+            file_path = artifacts.file_path
+            if not temp_file_path:
+                return "", file_path
+            # 构造请求消息
+            messages = [
+                {
+                    "role": "user",
+                    "content": [
+                        {"audio": temp_file_path}
+                    ]
+                }
+            ]
+            
+            # 如果有上下文信息，添加system消息
+            if self.context:
+                messages.insert(0, {
+                    "role": "system", 
+                    "content": [
+                        {"text": self.context}
+                    ]
+                })
+            
+            # 准备ASR选项
+            asr_options = {
+                "enable_lid": self.enable_lid,
+                "enable_itn": self.enable_itn
+            }
+            
+            # 如果指定了语种，添加到选项中
+            if self.language:
+                asr_options["language"] = self.language
+            
+            # 设置API密钥
+            dashscope.api_key = self.api_key
+            
+            # 发送流式请求
+            response = dashscope.MultiModalConversation.call(
+                model=self.model_name,
+                messages=messages,
+                result_format="message",
+                asr_options=asr_options,
+                stream=True
+            )
+            
+            # 处理流式响应
+            full_text = ""
+            for chunk in response:
+                try:
+                    text = chunk["output"]["choices"][0]["message"].content[0]["text"]
+                    # 更新为最新的完整文本
+                    full_text = text.strip()
+                except:
+                    pass
+            
+            return full_text, file_path
+                
+        except Exception as e:
+            logger.bind(tag=tag).error(f"语音识别失败: {e}")
+            return "", file_path

main/xingxing-server/core/providers/asr/sherpa_onnx_local.py

@@ -0,0 +1,150 @@
+import time
+import wave
+import os
+import sys
+import io
+from config.logger import setup_logging
+from typing import Optional, Tuple, List
+from core.providers.asr.dto.dto import InterfaceType
+from core.providers.asr.base import ASRProviderBase
+
+import numpy as np
+import sherpa_onnx
+
+from modelscope.hub.file_download import model_file_download
+
+TAG = __name__
+logger = setup_logging()
+
+
+# 捕获标准输出
+class CaptureOutput:
+    def __enter__(self):
+        self._output = io.StringIO()
+        self._original_stdout = sys.stdout
+        sys.stdout = self._output
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        sys.stdout = self._original_stdout
+        self.output = self._output.getvalue()
+        self._output.close()
+
+        # 将捕获到的内容通过 logger 输出
+        if self.output:
+            logger.bind(tag=TAG).info(self.output.strip())
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
+        self.interface_type = InterfaceType.LOCAL
+        self.model_dir = config.get("model_dir")
+        self.output_dir = config.get("output_dir")
+        self.model_type = config.get("model_type", "sense_voice")  # 支持 paraformer
+        self.delete_audio_file = delete_audio_file
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+        # 初始化模型文件路径
+        model_files = {
+            "model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"),
+            "tokens.txt": os.path.join(self.model_dir, "tokens.txt"),
+        }
+
+        # 下载并检查模型文件
+        try:
+            for file_name, file_path in model_files.items():
+                if not os.path.isfile(file_path):
+                    logger.bind(tag=TAG).info(f"正在下载模型文件: {file_name}")
+                    model_file_download(
+                        model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue",
+                        file_path=file_name,
+                        local_dir=self.model_dir,
+                    )
+
+                    if not os.path.isfile(file_path):
+                        raise FileNotFoundError(f"模型文件下载失败: {file_path}")
+
+            self.model_path = model_files["model.int8.onnx"]
+            self.tokens_path = model_files["tokens.txt"]
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}")
+            raise
+
+        with CaptureOutput():
+            if self.model_type == "paraformer":
+                self.model = sherpa_onnx.OfflineRecognizer.from_paraformer(
+                    paraformer=self.model_path,
+                    tokens=self.tokens_path,
+                    num_threads=2,
+                    sample_rate=16000,
+                    feature_dim=80,
+                    decoding_method="greedy_search",
+                    debug=False,
+                )
+            else:  # sense_voice
+                self.model = sherpa_onnx.OfflineRecognizer.from_sense_voice(
+                    model=self.model_path,
+                    tokens=self.tokens_path,
+                    num_threads=2,
+                    sample_rate=16000,
+                    feature_dim=80,
+                    decoding_method="greedy_search",
+                    debug=False,
+                    use_itn=True,
+                )
+
+    def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]:
+        """
+        Args:
+        wave_filename:
+            Path to a wave file. It should be single channel and each sample should
+            be 16-bit. Its sample rate does not need to be 16kHz.
+        Returns:
+        Return a tuple containing:
+        - A 1-D array of dtype np.float32 containing the samples, which are
+        normalized to the range [-1, 1].
+        - sample rate of the wave file
+        """
+
+        with wave.open(wave_filename) as f:
+            assert f.getnchannels() == 1, f.getnchannels()
+            assert f.getsampwidth() == 2, f.getsampwidth()  # it is in bytes
+            num_samples = f.getnframes()
+            samples = f.readframes(num_samples)
+            samples_int16 = np.frombuffer(samples, dtype=np.int16)
+            samples_float32 = samples_int16.astype(np.float32)
+
+            samples_float32 = samples_float32 / 32768
+            return samples_float32, f.getframerate()
+
+    def requires_file(self) -> bool:
+        return True
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """语音转文本主处理逻辑"""
+        file_path = None
+        try:
+            if artifacts is None:
+                return "", None
+            file_path = artifacts.file_path
+
+            start_time = time.time()
+            s = self.model.create_stream()
+            samples, sample_rate = self.read_wave(file_path)
+            s.accept_waveform(sample_rate, samples)
+            self.model.decode_stream(s)
+            text = s.result.text
+            logger.bind(tag=TAG).debug(
+                f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
+            )
+
+            return text, file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", file_path

main/xingxing-server/core/providers/asr/tencent.py

@@ -0,0 +1,227 @@
+import base64
+import hashlib
+import hmac
+import json
+import time
+from datetime import datetime, timezone
+import os
+from typing import Optional, Tuple, List
+from core.providers.asr.dto.dto import InterfaceType
+import requests
+from core.providers.asr.base import ASRProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    API_URL = "https://asr.tencentcloudapi.com"
+    API_VERSION = "2019-06-14"
+    FORMAT = "pcm"  # 支持的音频格式：pcm, wav, mp3
+
+    def __init__(self, config: dict, delete_audio_file: bool = True):
+        super().__init__()
+        self.interface_type = InterfaceType.NON_STREAM
+        self.secret_id = config.get("secret_id")
+        self.secret_key = config.get("secret_key")
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        if not opus_data:
+            logger.bind(tag=TAG).warning("音频数据为空！")
+            return None, None
+
+        try:
+            # 检查配置是否已设置
+            if not self.secret_id or not self.secret_key:
+                logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别")
+                return None, None
+
+            if artifacts is None:
+                return "", None
+
+            # 将音频数据转换为Base64编码
+            base64_audio = base64.b64encode(artifacts.pcm_bytes).decode("utf-8")
+
+            # 构建请求体
+            request_body = self._build_request_body(base64_audio)
+
+            # 获取认证头
+            timestamp, authorization = self._get_auth_headers(request_body)
+
+            # 发送请求
+            start_time = time.time()
+            result = self._send_request(request_body, timestamp, authorization)
+
+            if result:
+                logger.bind(tag=TAG).debug(
+                    f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
+                )
+
+            return result, artifacts.file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
+            return None, None
+
+    def _build_request_body(self, base64_audio: str) -> str:
+        """构建请求体"""
+        request_map = {
+            "ProjectId": 0,
+            "SubServiceType": 2,  # 一句话识别
+            "EngSerViceType": "16k_zh",  # 中文普通话通用
+            "SourceType": 1,  # 音频数据来源为语音文件
+            "VoiceFormat": self.FORMAT,  # 音频格式
+            "Data": base64_audio,  # Base64编码的音频数据
+            "DataLen": len(base64_audio),  # 数据长度
+        }
+        return json.dumps(request_map)
+
+    def _get_auth_headers(self, request_body: str) -> Tuple[str, str]:
+        """获取认证头"""
+        try:
+            # 获取当前UTC时间戳
+            now = datetime.now(timezone.utc)
+            timestamp = str(int(now.timestamp()))
+            date = now.strftime("%Y-%m-%d")
+
+            # 服务名称必须是 "asr"
+            service = "asr"
+
+            # 拼接凭证范围
+            credential_scope = f"{date}/{service}/tc3_request"
+
+            # 使用TC3-HMAC-SHA256签名方法
+            algorithm = "TC3-HMAC-SHA256"
+
+            # 构建规范请求字符串
+            http_request_method = "POST"
+            canonical_uri = "/"
+            canonical_query_string = ""
+
+            # 注意：头部信息需要按照ASCII升序排列，且key和value都转为小写
+            # 必须包含content-type和host头部
+            content_type = "application/json; charset=utf-8"
+            host = "asr.tencentcloudapi.com"
+            action = "SentenceRecognition"  # 接口名称
+
+            # 构建规范头部信息，注意顺序和格式
+            canonical_headers = (
+                f"content-type:{content_type.lower()}\n"
+                + f"host:{host.lower()}\n"
+                + f"x-tc-action:{action.lower()}\n"
+            )
+
+            signed_headers = "content-type;host;x-tc-action"
+
+            # 请求体哈希值
+            payload_hash = self._sha256_hex(request_body)
+
+            # 构建规范请求字符串
+            canonical_request = (
+                f"{http_request_method}\n"
+                + f"{canonical_uri}\n"
+                + f"{canonical_query_string}\n"
+                + f"{canonical_headers}\n"
+                + f"{signed_headers}\n"
+                + f"{payload_hash}"
+            )
+
+            # 计算规范请求的哈希值
+            hashed_canonical_request = self._sha256_hex(canonical_request)
+
+            # 构建待签名字符串
+            string_to_sign = (
+                f"{algorithm}\n"
+                + f"{timestamp}\n"
+                + f"{credential_scope}\n"
+                + f"{hashed_canonical_request}"
+            )
+
+            # 计算签名密钥
+            secret_date = self._hmac_sha256(f"TC3{self.secret_key}", date)
+            secret_service = self._hmac_sha256(secret_date, service)
+            secret_signing = self._hmac_sha256(secret_service, "tc3_request")
+
+            # 计算签名
+            signature = self._bytes_to_hex(
+                self._hmac_sha256(secret_signing, string_to_sign)
+            )
+
+            # 构建授权头
+            authorization = (
+                f"{algorithm} "
+                + f"Credential={self.secret_id}/{credential_scope}, "
+                + f"SignedHeaders={signed_headers}, "
+                + f"Signature={signature}"
+            )
+
+            return timestamp, authorization
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"生成认证头失败: {e}", exc_info=True)
+            raise RuntimeError(f"生成认证头失败: {e}")
+
+    def _send_request(
+        self, request_body: str, timestamp: str, authorization: str
+    ) -> Optional[str]:
+        """发送请求到腾讯云API"""
+        headers = {
+            "Content-Type": "application/json; charset=utf-8",
+            "Host": "asr.tencentcloudapi.com",
+            "Authorization": authorization,
+            "X-TC-Action": "SentenceRecognition",
+            "X-TC-Version": self.API_VERSION,
+            "X-TC-Timestamp": timestamp,
+            "X-TC-Region": "ap-shanghai",
+        }
+
+        try:
+            response = requests.post(self.API_URL, headers=headers, data=request_body)
+
+            if not response.ok:
+                raise IOError(f"请求失败: {response.status_code} {response.reason}")
+
+            response_json = response.json()
+
+            # 检查是否有错误
+            if "Response" in response_json and "Error" in response_json["Response"]:
+                error = response_json["Response"]["Error"]
+                error_code = error["Code"]
+                error_message = error["Message"]
+                raise IOError(f"API返回错误: {error_code}: {error_message}")
+
+            # 提取识别结果
+            if "Response" in response_json and "Result" in response_json["Response"]:
+                return response_json["Response"]["Result"]
+            else:
+                logger.bind(tag=TAG).warning(f"响应中没有识别结果: {response_json}")
+                return ""
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"发送请求失败: {e}", exc_info=True)
+            return None
+
+    def _sha256_hex(self, data: str) -> str:
+        """计算字符串的SHA256哈希值"""
+        digest = hashlib.sha256(data.encode("utf-8")).digest()
+        return self._bytes_to_hex(digest)
+
+    def _hmac_sha256(self, key, data: str) -> bytes:
+        """计算HMAC-SHA256"""
+        if isinstance(key, str):
+            key = key.encode("utf-8")
+
+        return hmac.new(key, data.encode("utf-8"), hashlib.sha256).digest()
+
+    def _bytes_to_hex(self, bytes_data: bytes) -> str:
+        """字节数组转十六进制字符串"""
+        return "".join(f"{b:02x}" for b in bytes_data)

main/xingxing-server/core/providers/asr/utils.py

@@ -0,0 +1,79 @@
+import re
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+EMOTION_EMOJI_MAP = {
+    "HAPPY": "🙂",
+    "SAD": "😔",
+    "ANGRY": "😡",
+    "NEUTRAL": "😶",
+    "FEARFUL": "😰",
+    "DISGUSTED": "🤢",
+    "SURPRISED": "😲",
+    "EMO_UNKNOWN": "😶",  # 未知情绪默认用中性表情
+}
+# EVENT_EMOJI_MAP = {
+#     "<|BGM|>": "🎼",
+#     "<|Speech|>": "",
+#     "<|Applause|>": "👏",
+#     "<|Laughter|>": "😀",
+#     "<|Cry|>": "😭",
+#     "<|Sneeze|>": "🤧",
+#     "<|Breath|>": "",
+#     "<|Cough|>": "🤧",
+# }
+
+def lang_tag_filter(text: str) -> dict | str:
+    """
+    解析 FunASR 识别结果，按顺序提取标签和纯文本内容
+
+    Args:
+        text: ASR 识别的原始文本，可能包含多种标签
+
+    Returns:
+        dict: {"language": "zh", "emotion": "SAD", "emoji": "😔", "content": "你好"} 如果有标签
+        str: 纯文本，如果没有标签
+
+    Examples:
+        FunASR 输出格式：<|语种|><|情绪|><|事件|><|其他选项|>原文
+        >>> lang_tag_filter("<|zh|><|SAD|><|Speech|><|withitn|>你好啊，测试测试。")
+        {"language": "zh", "emotion": "SAD", "emoji": "😔", "content": "你好啊，测试测试。"}
+        >>> lang_tag_filter("<|en|><|HAPPY|><|Speech|><|withitn|>Hello hello.")
+        {"language": "en", "emotion": "HAPPY", "emoji": "🙂", "content": "Hello hello."}
+        >>> lang_tag_filter("plain text")
+        "plain text"
+    """
+    # 提取所有标签（按顺序）
+    tag_pattern = r"<\|([^|]+)\|>"
+    all_tags = re.findall(tag_pattern, text)
+
+    # 移除所有 <|...|> 格式的标签，获取纯文本
+    clean_text = re.sub(tag_pattern, "", text).strip()
+
+    # 如果没有标签，直接返回纯文本
+    if not all_tags:
+        return clean_text
+
+    # 按照 FunASR 的固定顺序提取标签，返回 dict
+    language = all_tags[0] if len(all_tags) > 0 else "zh"
+    emotion = all_tags[1] if len(all_tags) > 1 else "NEUTRAL"
+    # event = all_tags[2] if len(all_tags) > 2 else "Speech"  # 事件标签暂不使用
+
+    result = {
+        "content": clean_text,
+        "language": language,
+        "emotion": emotion,
+        # "event": event,
+    }
+
+    # 添加 emoji 映射
+    if emotion in EMOTION_EMOJI_MAP:
+        result["emotion"] = EMOTION_EMOJI_MAP[emotion]
+    # 事件标签暂不使用
+    # if event in EVENT_EMOJI_MAP:
+    #     result["event"] = EVENT_EMOJI_MAP[event]
+
+    return result
+

main/xingxing-server/core/providers/asr/vosk.py

@@ -0,0 +1,91 @@
+import os
+import json
+import time
+from typing import Optional, Tuple, List
+from .base import ASRProviderBase
+from config.logger import setup_logging
+from core.providers.asr.dto.dto import InterfaceType
+import vosk
+
+TAG = __name__
+logger = setup_logging()
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool = True):
+        super().__init__()
+        self.interface_type = InterfaceType.LOCAL
+        self.model_path = config.get("model_path")
+        self.output_dir = config.get("output_dir", "tmp/")
+        self.delete_audio_file = delete_audio_file
+        
+        # 初始化VOSK模型
+        self.model = None
+        self.recognizer = None
+        self._load_model()
+        
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def _load_model(self):
+        """加载VOSK模型"""
+        try:
+            if not os.path.exists(self.model_path):
+                raise FileNotFoundError(f"VOSK模型路径不存在: {self.model_path}")
+                
+            logger.bind(tag=TAG).info(f"正在加载VOSK模型: {self.model_path}")
+            self.model = vosk.Model(self.model_path)
+
+            # 初始化VOSK识别器（采样率必须为16kHz）
+            self.recognizer = vosk.KaldiRecognizer(self.model, 16000)
+
+            logger.bind(tag=TAG).info("VOSK模型加载成功")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"加载VOSK模型失败: {e}")
+            raise
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        try:
+            # 检查模型是否加载成功
+            if not self.model:
+                logger.bind(tag=TAG).error("VOSK模型未加载，无法进行识别")
+                return "", None
+            
+            if artifacts is None:
+                return "", None
+            if not artifacts.pcm_bytes:
+                logger.bind(tag=TAG).warning("合并后的PCM数据为空")
+                return "", None
+
+            start_time = time.time()
+            
+            
+            # 进行识别（VOSK推荐每次送入2000字节的数据）
+            chunk_size = 2000
+            text_result = ""
+            
+            for i in range(0, len(artifacts.pcm_bytes), chunk_size):
+                chunk = artifacts.pcm_bytes[i:i+chunk_size]
+                if self.recognizer.AcceptWaveform(chunk):
+                    result = json.loads(self.recognizer.Result())
+                    text = result.get('text', '')
+                    if text:
+                        text_result += text + " "
+            
+            # 获取最终结果
+            final_result = json.loads(self.recognizer.FinalResult())
+            final_text = final_result.get('text', '')
+            if final_text:
+                text_result += final_text
+            
+            logger.bind(tag=TAG).debug(
+                f"VOSK语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text_result.strip()}"
+            )
+            
+            return text_result.strip(), artifacts.file_path
+            
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"VOSK语音识别失败: {e}")
+            return "", None

main/xingxing-server/core/providers/asr/xunfei_stream.py

@@ -0,0 +1,353 @@
+import json
+import hmac
+import base64
+import hashlib
+import asyncio
+import websockets
+import opuslib_next
+import gc
+from time import mktime
+from datetime import datetime
+from urllib.parse import urlencode
+from typing import List, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+from config.logger import setup_logging
+from wsgiref.handlers import format_date_time
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.dto.dto import InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+# 帧状态常量
+STATUS_FIRST_FRAME = 0  # 第一帧的标识
+STATUS_CONTINUE_FRAME = 1  # 中间帧标识
+STATUS_LAST_FRAME = 2  # 最后一帧的标识
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__()
+        self.interface_type = InterfaceType.STREAM
+        self.config = config
+        self.text = ""
+        self.decoder = opuslib_next.Decoder(16000, 1)
+        self.asr_ws = None
+        self.forward_task = None
+        self.is_processing = False
+        self.server_ready = False
+
+        # 讯飞配置
+        self.app_id = config.get("app_id")
+        self.api_key = config.get("api_key")
+        self.api_secret = config.get("api_secret")
+
+        if not all([self.app_id, self.api_key, self.api_secret]):
+            raise ValueError("必须提供app_id、api_key和api_secret")
+
+        # 识别参数
+        self.iat_params = {
+            "domain": config.get("domain", "slm"),
+            "language": config.get("language", "zh_cn"),
+            "accent": config.get("accent", "mandarin"),
+            "result": {"encoding": "utf8", "compress": "raw", "format": "plain"},
+        }
+
+        self.output_dir = config.get("output_dir", "tmp/")
+        self.delete_audio_file = delete_audio_file
+
+    def create_url(self) -> str:
+        """生成认证URL"""
+        url = "ws://iat.cn-huabei-1.xf-yun.com/v1"
+        # 生成RFC1123格式的时间戳
+        now = datetime.now()
+        date = format_date_time(mktime(now.timetuple()))
+
+        # 拼接字符串
+        signature_origin = "host: " + "iat.cn-huabei-1.xf-yun.com" + "\n"
+        signature_origin += "date: " + date + "\n"
+        signature_origin += "GET " + "/v1 " + "HTTP/1.1"
+
+        # 进行hmac-sha256进行加密
+        signature_sha = hmac.new(
+            self.api_secret.encode("utf-8"),
+            signature_origin.encode("utf-8"),
+            digestmod=hashlib.sha256,
+        ).digest()
+        signature_sha = base64.b64encode(signature_sha).decode(encoding="utf-8")
+
+        authorization_origin = (
+            'api_key="%s", algorithm="%s", headers="%s", signature="%s"'
+            % (self.api_key, "hmac-sha256", "host date request-line", signature_sha)
+        )
+        authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode(
+            encoding="utf-8"
+        )
+
+        # 将请求的鉴权参数组合为字典
+        v = {
+            "authorization": authorization,
+            "date": date,
+            "host": "iat.cn-huabei-1.xf-yun.com",
+        }
+
+        # 拼接鉴权参数，生成url
+        url = url + "?" + urlencode(v)
+        return url
+
+    async def open_audio_channels(self, conn: "ConnectionHandler"):
+        await super().open_audio_channels(conn)
+
+    async def receive_audio(self, conn: "ConnectionHandler", audio, audio_have_voice):
+        # 先调用父类方法处理基础逻辑
+        await super().receive_audio(conn, audio, audio_have_voice)
+
+        # 如果本次有声音，且之前没有建立连接
+        if audio_have_voice and self.asr_ws is None and not self.is_processing:
+            try:
+                await self._start_recognition(conn)
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
+                await self._cleanup()
+                return
+
+        # 发送当前音频数据
+        if self.asr_ws and self.is_processing and self.server_ready:
+            try:
+                pcm_frame = self.decoder.decode(audio, 960)
+                await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME)
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"发送音频数据时发生错误: {e}")
+                await self._cleanup()
+
+    async def _start_recognition(self, conn: "ConnectionHandler"):
+        """开始识别会话"""
+        try:
+            self.is_processing = True
+            # 建立WebSocket连接
+            ws_url = self.create_url()
+            logger.bind(tag=TAG).info(f"正在连接ASR服务: {ws_url[:50]}...")
+
+            # 如果为手动模式,设置超时时长为一分钟
+            if conn.client_listen_mode == "manual":
+                self.iat_params["eos"] = 60000
+
+            self.asr_ws = await websockets.connect(
+                ws_url,
+                max_size=1000000000,
+                ping_interval=None,
+                ping_timeout=None,
+                close_timeout=10,
+            )
+
+            logger.bind(tag=TAG).info("ASR WebSocket连接已建立")
+            self.server_ready = False
+            self.forward_task = asyncio.create_task(self._forward_results(conn))
+
+            # 发送首帧音频
+            if conn.asr_audio and len(conn.asr_audio) > 0:
+                first_audio = conn.asr_audio[-1] if conn.asr_audio else b""
+                pcm_frame = (
+                    self.decoder.decode(first_audio, 960) if first_audio else b""
+                )
+                await self._send_audio_frame(pcm_frame, STATUS_FIRST_FRAME)
+                self.server_ready = True
+                logger.bind(tag=TAG).info("已发送首帧，开始识别")
+
+                # 发送缓存的音频数据
+                for cached_audio in conn.asr_audio[-10:]:
+                    try:
+                        pcm_frame = self.decoder.decode(cached_audio, 960)
+                        await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME)
+                    except Exception as e:
+                        logger.bind(tag=TAG).info(f"发送缓存音频数据时发生错误: {e}")
+                        break
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
+            if hasattr(e, "__cause__") and e.__cause__:
+                logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+            if self.asr_ws:
+                await self.asr_ws.close()
+                self.asr_ws = None
+            self.is_processing = False
+            raise
+
+    async def _send_audio_frame(self, audio_data: bytes, status: int):
+        """发送音频帧"""
+        if not self.asr_ws:
+            return
+
+        audio_b64 = base64.b64encode(audio_data).decode("utf-8")
+
+        frame_data = {
+            "header": {"status": status, "app_id": self.app_id},
+            "parameter": {"iat": self.iat_params},
+            "payload": {
+                "audio": {"audio": audio_b64, "sample_rate": 16000, "encoding": "raw"}
+            },
+        }
+
+        await self.asr_ws.send(json.dumps(frame_data, ensure_ascii=False))
+
+    async def _forward_results(self, conn: "ConnectionHandler"):
+        """转发识别结果"""
+        try:
+            while not conn.stop_event.is_set():
+                try:
+                    response = await asyncio.wait_for(self.asr_ws.recv(), timeout=60)
+                    result = json.loads(response)
+                    logger.bind(tag=TAG).debug(f"收到ASR结果: {result}")
+
+                    header = result.get("header", {})
+                    payload = result.get("payload", {})
+                    code = header.get("code", 0)
+                    status = header.get("status", 0)
+
+                    if code != 0:
+                        logger.bind(tag=TAG).error(
+                            f"识别错误，错误码: {code}, 消息: {header.get('message', '')}"
+                        )
+                        if code in [10114, 10160]:  # 连接问题
+                            break
+                        continue
+
+                    # 处理识别结果
+                    if payload and "result" in payload:
+                        text_data = payload["result"]["text"]
+                        if text_data:
+                            # 解码base64文本
+                            decoded_text = base64.b64decode(text_data).decode("utf-8")
+                            text_json = json.loads(decoded_text)
+                            # 提取文本内容
+                            text_ws = text_json.get("ws", [])
+                            for i in text_ws:
+                                for j in i.get("cw", []):
+                                    w = j.get("w", "")
+                                    self.text += w
+
+                    if status == 2:
+                        logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
+                        await self.handle_voice_stop(conn, conn.asr_audio)
+                        break
+
+                except asyncio.TimeoutError:
+                    logger.bind(tag=TAG).error("接收结果超时")
+                    break
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).info("ASR服务连接已关闭")
+                    self.is_processing = False
+                    break
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"处理ASR结果时发生错误: {str(e)}")
+                    if hasattr(e, "__cause__") and e.__cause__:
+                        logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+                    self.is_processing = False
+                    break
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"ASR结果转发任务发生错误: {str(e)}")
+            if hasattr(e, "__cause__") and e.__cause__:
+                logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+        finally:
+            # 清理连接资源
+            await self._cleanup()
+            conn.reset_audio_states()
+
+    async def handle_voice_stop(
+        self, conn: "ConnectionHandler", asr_audio_task: List[bytes]
+    ):
+        """处理语音停止，发送最后一帧并处理识别结果"""
+        try:
+            # 先发送最后一帧表示音频结束
+            if self.asr_ws and self.is_processing:
+                try:
+                    await self._send_audio_frame(b"", STATUS_LAST_FRAME)
+                    logger.bind(tag=TAG).debug(f"已发送停止请求")
+
+                    await asyncio.sleep(0.25)
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
+
+            await super().handle_voice_stop(conn, asr_audio_task)
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
+            import traceback
+
+            logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
+
+    def stop_ws_connection(self):
+        if self.asr_ws:
+            asyncio.create_task(self.asr_ws.close())
+            self.asr_ws = None
+        self.is_processing = False
+
+    async def _send_stop_request(self):
+        """发送停止识别请求（不关闭连接）"""
+        if self.asr_ws:
+            try:
+                # 先停止音频发送
+                self.is_processing = False
+                await self._send_audio_frame(b"", STATUS_LAST_FRAME)
+                logger.bind(tag=TAG).debug("已发送停止请求")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
+
+    async def _cleanup(self):
+        """清理资源（关闭连接）"""
+        logger.bind(tag=TAG).debug(
+            f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}"
+        )
+
+        # 状态重置
+        self.is_processing = False
+        self.server_ready = False
+        logger.bind(tag=TAG).debug("ASR状态已重置")
+
+        # 关闭连接
+        if self.asr_ws:
+            try:
+                logger.bind(tag=TAG).debug("正在关闭WebSocket连接")
+                await asyncio.wait_for(self.asr_ws.close(), timeout=2.0)
+                logger.bind(tag=TAG).debug("WebSocket连接已关闭")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
+            finally:
+                self.asr_ws = None
+
+        # 清理任务引用
+        self.forward_task = None
+
+        logger.bind(tag=TAG).debug("ASR会话清理完成")
+
+    async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
+        """获取识别结果"""
+        result = self.text
+        self.text = ""
+        return result, None
+
+    async def close(self):
+        """资源清理方法"""
+        if self.asr_ws:
+            await self.asr_ws.close()
+            self.asr_ws = None
+        if self.forward_task:
+            self.forward_task.cancel()
+            try:
+                await self.forward_task
+            except asyncio.CancelledError:
+                pass
+            self.forward_task = None
+        self.is_processing = False
+
+        # 显式释放decoder资源
+        if hasattr(self, "decoder") and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Xunfei decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Xunfei decoder资源时出错: {e}")
+

main/xingxing-server/core/providers/intent/base.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class IntentProviderBase(ABC):
+    def __init__(self, config):
+        self.config = config
+
+    def set_llm(self, llm):
+        self.llm = llm
+        # 获取模型名称和类型信息
+        model_name = getattr(llm, "model_name", str(llm.__class__.__name__))
+        # 记录更详细的日志
+        logger.bind(tag=TAG).info(f"意图识别设置LLM: {model_name}")
+
+    @abstractmethod
+    async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
+        """
+        检测用户最后一句话的意图
+        Args:
+            dialogue_history: 对话历史记录列表，每条记录包含role和content
+        Returns:
+            返回识别出的意图，格式为:
+            - "继续聊天"
+            - "结束聊天"
+            - "播放音乐 歌名" 或 "随机播放音乐"
+            - "查询天气 地点名" 或 "查询天气 [当前位置]"
+        """
+        pass

main/xingxing-server/core/providers/intent/function_call/function_call.py

@@ -0,0 +1,22 @@
+from ..base import IntentProviderBase
+from typing import List, Dict
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class IntentProvider(IntentProviderBase):
+    async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
+        """
+        默认的意图识别实现，始终返回继续聊天
+        Args:
+            dialogue_history: 对话历史记录列表
+            text: 本次对话记录
+        Returns:
+            固定返回"继续聊天"
+        """
+        logger.bind(tag=TAG).debug(
+            "Using functionCallProvider, always returning continue chat"
+        )
+        return '{"function_call": {"name": "continue_chat"}}'

main/xingxing-server/core/providers/intent/intent_llm/intent_llm.py

@@ -0,0 +1,286 @@
+from typing import List, Dict, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+from ..base import IntentProviderBase
+from plugins_func.functions.play_music import initialize_music_handler
+from config.logger import setup_logging
+from core.utils.util import get_system_error_response
+import re
+import json
+import hashlib
+import time
+
+
+
+TAG = __name__
+logger = setup_logging()
+
+
+class IntentProvider(IntentProviderBase):
+    def __init__(self, config):
+        super().__init__(config)
+        self.llm = None
+        self.promot = ""
+        # 导入全局缓存管理器
+        from core.utils.cache.manager import cache_manager, CacheType
+
+        self.cache_manager = cache_manager
+        self.CacheType = CacheType
+        self.history_count = 4  # 默认使用最近4条对话记录
+
+    def get_intent_system_prompt(self, functions_list: str) -> str:
+        """
+        根据配置的意图选项和可用函数动态生成系统提示词
+        Args:
+            functions: 可用的函数列表，JSON格式字符串
+        Returns:
+            格式化后的系统提示词
+        """
+
+        # 构建函数说明部分
+        functions_desc = "可用的函数列表：\n"
+        for func in functions_list:
+            func_info = func.get("function", {})
+            name = func_info.get("name", "")
+            desc = func_info.get("description", "")
+            params = func_info.get("parameters", {})
+
+            functions_desc += f"\n函数名: {name}\n"
+            functions_desc += f"描述: {desc}\n"
+
+            if params:
+                functions_desc += "参数:\n"
+                for param_name, param_info in params.get("properties", {}).items():
+                    param_desc = param_info.get("description", "")
+                    param_type = param_info.get("type", "")
+                    functions_desc += f"- {param_name} ({param_type}): {param_desc}\n"
+
+            functions_desc += "---\n"
+
+        prompt = (
+            "【严格格式要求】你必须只能返回JSON格式，绝对不能返回任何自然语言！\n\n"
+            "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n"
+            "【重要规则】以下类型的查询请直接返回result_for_context，无需调用函数：\n"
+            "- 询问当前时间（如：现在几点、当前时间、查询时间等）\n"
+            "- 询问今天日期（如：今天几号、今天星期几、今天是什么日期等）\n"
+            "- 询问今天农历（如：今天农历几号、今天什么节气等）\n"
+            "- 询问所在城市（如：我现在在哪里、你知道我在哪个城市吗等）"
+            "系统会根据上下文信息直接构建回答。\n\n"
+            "- 如果用户使用疑问词（如'怎么'、'为什么'、'如何'）询问退出相关的问题（例如'怎么退出了？'），注意这不是让你退出，请返回 {'function_call': {'name': 'continue_chat'}\n"
+            "- 仅当用户明确使用'退出系统'、'结束对话'、'我不想和你说话了'等指令时，才触发 handle_exit_intent\n\n"
+            f"{functions_desc}\n"
+            "处理步骤:\n"
+            "1. 分析用户输入，确定用户意图\n"
+            "2. 检查是否为上述基础信息查询（时间、日期等），如是则返回result_for_context\n"
+            "3. 从可用函数列表中选择最匹配的函数\n"
+            "4. 如果找到匹配的函数，生成对应的function_call 格式\n"
+            '5. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n'
+            "返回格式要求：\n"
+            "1. 必须返回纯JSON格式，不要包含任何其他文字\n"
+            "2. 必须包含function_call字段\n"
+            "3. function_call必须包含name字段\n"
+            "4. 如果函数需要参数，必须包含arguments字段\n\n"
+            "示例：\n"
+            "```\n"
+            "用户: 现在几点了？\n"
+            '返回: {"function_call": {"name": "result_for_context"}}\n'
+            "```\n"
+            "```\n"
+            "用户: 当前电池电量是多少？\n"
+            '返回: {"function_call": {"name": "get_battery_level", "arguments": {"response_success": "当前电池电量为{value}%", "response_failure": "无法获取Battery的当前电量百分比"}}}\n'
+            "```\n"
+            "```\n"
+            "用户: 当前屏幕亮度是多少？\n"
+            '返回: {"function_call": {"name": "self_screen_get_brightness"}}\n'
+            "```\n"
+            "```\n"
+            "用户: 设置屏幕亮度为50%\n"
+            '返回: {"function_call": {"name": "self_screen_set_brightness", "arguments": {"brightness": 50}}}\n'
+            "```\n"
+            "```\n"
+            "用户: 我想结束对话\n"
+            '返回: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
+            "```\n"
+            "```\n"
+            "用户: 你好啊\n"
+            '返回: {"function_call": {"name": "continue_chat"}}\n'
+            "```\n\n"
+            "注意：\n"
+            "1. 只返回JSON格式，不要包含任何其他文字\n"
+            '2. 优先检查用户查询是否为基础信息（时间、日期等），如是则返回{"function_call": {"name": "result_for_context"}}，不需要arguments参数\n'
+            '3. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n'
+            "4. 确保返回的JSON格式正确，包含所有必要的字段\n"
+            "5. result_for_context不需要任何参数，系统会自动从上下文获取信息\n"
+            "特殊说明：\n"
+            "- 当用户单次输入包含多个指令时（如'打开灯并且调高音量'）\n"
+            "- 请返回多个function_call组成的JSON数组\n"
+            "- 示例：{'function_calls': [{name:'light_on'}, {name:'volume_up'}]}\n\n"
+            "【最终警告】绝对禁止输出任何自然语言、表情符号或解释文字！只能输出有效JSON格式！违反此规则将导致系统错误！"
+        )
+        return prompt
+
+    def replyResult(self, text: str, original_text: str):
+        try:
+            llm_result = self.llm.response_no_stream(
+                system_prompt=text,
+                user_prompt="请根据以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。用户现在说："
+                + original_text,
+            )
+            return llm_result
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in generating reply result: {e}")
+            return get_system_error_response(self.config)
+
+    async def detect_intent(
+        self, conn: "ConnectionHandler", dialogue_history: List[Dict], text: str
+    ) -> str:
+        if not self.llm:
+            raise ValueError("LLM provider not set")
+        if conn.func_handler is None:
+            return '{"function_call": {"name": "continue_chat"}}'
+
+        # 记录整体开始时间
+        total_start_time = time.time()
+
+        # 打印使用的模型信息
+        model_info = getattr(self.llm, "model_name", str(self.llm.__class__.__name__))
+        logger.bind(tag=TAG).debug(f"使用意图识别模型: {model_info}")
+
+        # 计算缓存键
+        cache_key = hashlib.md5((conn.device_id + text).encode()).hexdigest()
+
+        # 检查缓存
+        cached_intent = self.cache_manager.get(self.CacheType.INTENT, cache_key)
+        if cached_intent is not None:
+            cache_time = time.time() - total_start_time
+            logger.bind(tag=TAG).debug(
+                f"使用缓存的意图: {cache_key} -> {cached_intent}, 耗时: {cache_time:.4f}秒"
+            )
+            return cached_intent
+
+        if self.promot == "":
+            functions = conn.func_handler.get_functions()
+            if hasattr(conn, "mcp_client"):
+                mcp_tools = conn.mcp_client.get_available_tools()
+                if mcp_tools is not None and len(mcp_tools) > 0:
+                    if functions is None:
+                        functions = []
+                    functions.extend(mcp_tools)
+
+            self.promot = self.get_intent_system_prompt(functions)
+
+        music_config = initialize_music_handler(conn)
+        music_file_names = music_config["music_file_names"]
+        prompt_music = f"{self.promot}\n<musicNames>{music_file_names}\n</musicNames>"
+
+        home_assistant_cfg = conn.config["plugins"].get("home_assistant")
+        if home_assistant_cfg:
+            devices = home_assistant_cfg.get("devices", [])
+        else:
+            devices = []
+        if len(devices) > 0:
+            hass_prompt = "\n下面是我家智能设备列表（位置，设备名，entity_id），可以通过homeassistant控制\n"
+            for device in devices:
+                hass_prompt += device + "\n"
+            prompt_music += hass_prompt
+
+        logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
+
+        # 构建用户对话历史的提示
+        msgStr = ""
+
+        # 获取最近的对话历史
+        start_idx = max(0, len(dialogue_history) - self.history_count)
+        for i in range(start_idx, len(dialogue_history)):
+            msgStr += f"{dialogue_history[i].role}: {dialogue_history[i].content}\n"
+
+        msgStr += f"User: {text}\n"
+        user_prompt = f"current dialogue:\n{msgStr}"
+
+        # 记录预处理完成时间
+        preprocess_time = time.time() - total_start_time
+        logger.bind(tag=TAG).debug(f"意图识别预处理耗时: {preprocess_time:.4f}秒")
+
+        # 使用LLM进行意图识别
+        llm_start_time = time.time()
+        logger.bind(tag=TAG).debug(f"开始LLM意图识别调用, 模型: {model_info}")
+
+        try:
+            intent = self.llm.response_no_stream(
+                system_prompt=prompt_music, user_prompt=user_prompt
+            )
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in intent detection LLM call: {e}")
+            return '{"function_call": {"name": "continue_chat"}}'
+
+        # 记录LLM调用完成时间
+        llm_time = time.time() - llm_start_time
+        logger.bind(tag=TAG).debug(
+            f"外挂的大模型意图识别完成, 模型: {model_info}, 调用耗时: {llm_time:.4f}秒"
+        )
+
+        # 记录后处理开始时间
+        postprocess_start_time = time.time()
+
+        # 清理和解析响应
+        intent = intent.strip()
+        # 尝试提取JSON部分
+        match = re.search(r"\{.*\}", intent, re.DOTALL)
+        if match:
+            intent = match.group(0)
+
+        # 记录总处理时间
+        total_time = time.time() - total_start_time
+        logger.bind(tag=TAG).debug(
+            f"【意图识别性能】模型: {model_info}, 总耗时: {total_time:.4f}秒, LLM调用: {llm_time:.4f}秒, 查询: '{text[:20]}...'"
+        )
+
+        # 尝试解析为JSON
+        try:
+            intent_data = json.loads(intent)
+            # 如果包含function_call，则格式化为适合处理的格式
+            if "function_call" in intent_data:
+                function_data = intent_data["function_call"]
+                function_name = function_data.get("name")
+                function_args = function_data.get("arguments", {})
+
+                # 记录识别到的function call
+                logger.bind(tag=TAG).info(
+                    f"llm 识别到意图: {function_name}, 参数: {function_args}"
+                )
+
+                # 处理不同类型的意图
+                if function_name == "result_for_context":
+                    # 处理基础信息查询，直接从context构建结果
+                    logger.bind(tag=TAG).info(
+                        "检测到result_for_context意图，将使用上下文信息直接回答"
+                    )
+
+                elif function_name == "continue_chat":
+                    # 处理普通对话
+                    # 保留非工具相关的消息
+                    clean_history = [
+                        msg
+                        for msg in conn.dialogue.dialogue
+                        if msg.role not in ["tool", "function"]
+                    ]
+                    conn.dialogue.dialogue = clean_history
+
+                else:
+                    # 处理函数调用
+                    logger.bind(tag=TAG).info(f"检测到函数调用意图: {function_name}")
+
+            # 统一缓存处理和返回
+            self.cache_manager.set(self.CacheType.INTENT, cache_key, intent)
+            postprocess_time = time.time() - postprocess_start_time
+            logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
+            return intent
+        except json.JSONDecodeError:
+            # 后处理时间
+            postprocess_time = time.time() - postprocess_start_time
+            logger.bind(tag=TAG).error(
+                f"无法解析意图JSON: {intent}, 后处理耗时: {postprocess_time:.4f}秒"
+            )
+            # 如果解析失败，默认返回继续聊天意图
+            return '{"function_call": {"name": "continue_chat"}}'

main/xingxing-server/core/providers/intent/nointent/nointent.py

@@ -0,0 +1,22 @@
+from ..base import IntentProviderBase
+from typing import List, Dict
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class IntentProvider(IntentProviderBase):
+    async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
+        """
+        默认的意图识别实现，始终返回继续聊天
+        Args:
+            dialogue_history: 对话历史记录列表
+            text: 本次对话记录
+        Returns:
+            固定返回"继续聊天"
+        """
+        logger.bind(tag=TAG).debug(
+            "Using NoIntentProvider, always returning continue chat"
+        )
+        return '{"function_call": {"name": "continue_chat"}}'

main/xingxing-server/core/providers/llm/AliBL/AliBL.py

@@ -0,0 +1,104 @@
+from config.logger import setup_logging
+from http import HTTPStatus
+import dashscope
+from dashscope import Application
+from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
+import time
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.api_key = config["api_key"]
+        self.app_id = config["app_id"]
+        self.base_url = config.get("base_url")
+        self.is_No_prompt = config.get("is_no_prompt")
+        self.memory_id = config.get("ali_memory_id")
+        self.streaming_chunk_size = config.get("streaming_chunk_size", 3)  # 每次流式返回的字符数
+        check_model_key("AliBLLLM", self.api_key)
+
+    def response(self, session_id, dialogue):
+        # 处理dialogue
+        if self.is_No_prompt:
+            dialogue.pop(0)
+            logger.bind(tag=TAG).debug(
+                f"【阿里百练API服务】处理后的dialogue: {dialogue}"
+            )
+
+        # 构造调用参数
+        call_params = {
+            "api_key": self.api_key,
+            "app_id": self.app_id,
+            "session_id": session_id,
+            "messages": dialogue,
+            # 开启SDK原生流式
+            "stream": True,
+        }
+        if self.memory_id != False:
+            # 百练memory需要prompt参数
+            prompt = dialogue[-1].get("content")
+            call_params["memory_id"] = self.memory_id
+            call_params["prompt"] = prompt
+            logger.bind(tag=TAG).debug(
+                f"【阿里百练API服务】处理后的prompt: {prompt}"
+            )
+
+        # 可选地设置自定义API基地址（若配置为兼容模式URL则忽略）
+        if self.base_url and ("/api/" in self.base_url):
+            dashscope.base_http_api_url = self.base_url
+
+        responses = Application.call(**call_params)
+
+        # 流式处理（SDK在stream=True时返回可迭代对象；否则返回单次响应对象）
+        logger.bind(tag=TAG).debug(
+            f"【阿里百练API服务】构造参数: {dict(call_params, api_key='***')}"
+        )
+
+        last_text = ""
+        try:
+            for resp in responses:
+                if resp.status_code != HTTPStatus.OK:
+                    logger.bind(tag=TAG).error(
+                        f"code={resp.status_code}, message={resp.message}, 请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
+                    )
+                    continue
+                current_text = getattr(getattr(resp, "output", None), "text", None)
+                if current_text is None:
+                    continue
+                # SDK流式为增量覆盖，计算差量输出
+                if len(current_text) >= len(last_text):
+                    delta = current_text[len(last_text):]
+                else:
+                    # 避免偶发回退
+                    delta = current_text
+                if delta:
+                    yield delta
+                last_text = current_text
+        except TypeError:
+            # 非流式回落（一次性返回）
+            if responses.status_code != HTTPStatus.OK:
+                logger.bind(tag=TAG).error(
+                    f"code={responses.status_code}, message={responses.message}, 请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
+                )
+                yield "【阿里百练API服务响应异常】"
+            else:
+                full_text = getattr(getattr(responses, "output", None), "text", "")
+                logger.bind(tag=TAG).info(
+                    f"【阿里百练API服务】完整响应长度: {len(full_text)}"
+                )
+                for i in range(0, len(full_text), self.streaming_chunk_size):
+                    chunk = full_text[i:i + self.streaming_chunk_size]
+                    if chunk:
+                        yield chunk
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        # 阿里百练当前未支持原生的 function call。为保持兼容，这里回退到普通文本流式输出。
+        # 上层会按 (content, tool_calls) 的形式消费，这里始终返回 (token, None)
+        logger.bind(tag=TAG).warning(
+            "阿里百练未实现原生 function call，已回退为纯文本流式输出"
+        )
+        for token in self.response(session_id, dialogue):
+            yield token, None

main/xingxing-server/core/providers/llm/base.py

@@ -0,0 +1,34 @@
+from abc import ABC, abstractmethod
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class LLMProviderBase(ABC):
+    @abstractmethod
+    def response(self, session_id, dialogue):
+        """LLM response generator"""
+        pass
+
+    def response_no_stream(self, system_prompt, user_prompt, **kwargs):
+        # 构造对话格式
+        dialogue = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt}
+        ]
+        result = ""
+        for part in self.response("", dialogue, **kwargs):
+            result += part
+        return result
+    
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        """
+        Default implementation for function calling (streaming)
+        This should be overridden by providers that support function calls
+
+        Returns: generator that yields either text tokens or a special function call token
+        """
+        # For providers that don't support functions, just return regular response
+        for token in self.response(session_id, dialogue):
+            yield token, None
+

main/xingxing-server/core/providers/llm/coze/coze.py

@@ -0,0 +1,75 @@
+from config.logger import setup_logging
+import json
+from core.providers.llm.base import LLMProviderBase
+
+# official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
+from cozepy import COZE_CN_BASE_URL
+from cozepy import (
+    Coze,
+    TokenAuth,
+    Message,
+    ChatEventType,
+)  # noqa
+from core.providers.llm.system_prompt import get_system_prompt_for_function
+from core.utils.util import check_model_key
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.personal_access_token = config.get("personal_access_token")
+        self.bot_id = str(config.get("bot_id"))
+        self.user_id = str(config.get("user_id"))
+        self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
+        model_key_msg = check_model_key("CozeLLM", self.personal_access_token)
+        if model_key_msg:
+            logger.bind(tag=TAG).error(model_key_msg)
+
+    def response(self, session_id, dialogue, **kwargs):
+        coze_api_token = self.personal_access_token
+        coze_api_base = COZE_CN_BASE_URL
+
+        last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+
+        coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
+        conversation_id = self.session_conversation_map.get(session_id)
+
+        # 如果没有找到conversation_id，则创建新的对话
+        if not conversation_id:
+            conversation = coze.conversations.create(messages=[])
+            conversation_id = conversation.id
+            self.session_conversation_map[session_id] = conversation_id  # 更新映射
+
+        for event in coze.chat.stream(
+            bot_id=self.bot_id,
+            user_id=self.user_id,
+            additional_messages=[
+                Message.build_user_question_text(last_msg["content"]),
+            ],
+            conversation_id=conversation_id,
+        ):
+            if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
+                print(event.message.content, end="", flush=True)
+                yield event.message.content
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        if len(dialogue) == 2 and functions is not None and len(functions) > 0:
+            # 第一次调用llm， 取最后一条用户消息，附加tool提示词
+            last_msg = dialogue[-1]["content"]
+            function_str = json.dumps(functions, ensure_ascii=False)
+            modify_msg = get_system_prompt_for_function(function_str) + last_msg
+            dialogue[-1]["content"] = modify_msg
+
+        # 如果最后一个是 role="tool"，附加到user上
+        if len(dialogue) > 1 and dialogue[-1]["role"] == "tool":
+            assistant_msg = "\ntool call result: " + dialogue[-1]["content"] + "\n\n"
+            while len(dialogue) > 1:
+                if dialogue[-1]["role"] == "user":
+                    dialogue[-1]["content"] = assistant_msg + dialogue[-1]["content"]
+                    break
+                dialogue.pop()
+
+        for token in self.response(session_id, dialogue):
+            yield token, None

main/xingxing-server/core/providers/llm/dify/dify.py

@@ -0,0 +1,107 @@
+import json
+from config.logger import setup_logging
+import requests
+from core.providers.llm.base import LLMProviderBase
+from core.providers.llm.system_prompt import get_system_prompt_for_function
+from core.utils.util import check_model_key
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.api_key = config["api_key"]
+        self.mode = config.get("mode", "chat-messages")
+        self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip("/")
+        self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
+        model_key_msg = check_model_key("DifyLLM", self.api_key)
+        if model_key_msg:
+            logger.bind(tag=TAG).error(model_key_msg)
+
+    def response(self, session_id, dialogue, **kwargs):
+        # 取最后一条用户消息
+        last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+        conversation_id = self.session_conversation_map.get(session_id)
+
+        # 发起流式请求
+        if self.mode == "chat-messages":
+            request_json = {
+                "query": last_msg["content"],
+                "response_mode": "streaming",
+                "user": session_id,
+                "inputs": {},
+                "conversation_id": conversation_id,
+            }
+        elif self.mode == "workflows/run":
+            request_json = {
+                "inputs": {"query": last_msg["content"]},
+                "response_mode": "streaming",
+                "user": session_id,
+            }
+        elif self.mode == "completion-messages":
+            request_json = {
+                "inputs": {"query": last_msg["content"]},
+                "response_mode": "streaming",
+                "user": session_id,
+            }
+
+        with requests.post(
+            f"{self.base_url}/{self.mode}",
+            headers={"Authorization": f"Bearer {self.api_key}"},
+            json=request_json,
+            stream=True,
+        ) as r:
+            if self.mode == "chat-messages":
+                for line in r.iter_lines():
+                    if line.startswith(b"data: "):
+                        event = json.loads(line[6:])
+                        # 如果没有找到conversation_id，则获取此次conversation_id
+                        if not conversation_id:
+                            conversation_id = event.get("conversation_id")
+                            self.session_conversation_map[session_id] = (
+                                conversation_id  # 更新映射
+                            )
+                        # 过滤 message_replace 事件，此事件会全量推一次
+                        if event.get("event") != "message_replace" and event.get(
+                            "answer"
+                        ):
+                            yield event["answer"]
+            elif self.mode == "workflows/run":
+                for line in r.iter_lines():
+                    if line.startswith(b"data: "):
+                        event = json.loads(line[6:])
+                        if event.get("event") == "workflow_finished":
+                            if event["data"]["status"] == "succeeded":
+                                yield event["data"]["outputs"]["answer"]
+                            else:
+                                yield "【服务响应异常】"
+            elif self.mode == "completion-messages":
+                for line in r.iter_lines():
+                    if line.startswith(b"data: "):
+                        event = json.loads(line[6:])
+                        # 过滤 message_replace 事件，此事件会全量推一次
+                        if event.get("event") != "message_replace" and event.get(
+                            "answer"
+                        ):
+                            yield event["answer"]
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        if len(dialogue) == 2 and functions is not None and len(functions) > 0:
+            # 第一次调用llm， 取最后一条用户消息，附加tool提示词
+            last_msg = dialogue[-1]["content"]
+            function_str = json.dumps(functions, ensure_ascii=False)
+            modify_msg = get_system_prompt_for_function(function_str) + last_msg
+            dialogue[-1]["content"] = modify_msg
+
+        # 如果最后一个是 role="tool"，附加到user上
+        if len(dialogue) > 1 and dialogue[-1]["role"] == "tool":
+            assistant_msg = "\ntool call result: " + dialogue[-1]["content"] + "\n\n"
+            while len(dialogue) > 1:
+                if dialogue[-1]["role"] == "user":
+                    dialogue[-1]["content"] = assistant_msg + dialogue[-1]["content"]
+                    break
+                dialogue.pop()
+
+        for token in self.response(session_id, dialogue):
+            yield token, None

main/xingxing-server/core/providers/llm/fastgpt/fastgpt.py

@@ -0,0 +1,68 @@
+import json
+from config.logger import setup_logging
+import requests
+from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.api_key = config["api_key"]
+        self.base_url = config.get("base_url")
+        self.detail = config.get("detail", False)
+        self.variables = config.get("variables", {})
+        model_key_msg = check_model_key("FastGPTLLM", self.api_key)
+        if model_key_msg:
+            logger.bind(tag=TAG).error(model_key_msg)
+
+    def response(self, session_id, dialogue, **kwargs):
+        # 取最后一条用户消息
+        last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+
+        # 发起流式请求
+        with requests.post(
+            f"{self.base_url}/chat/completions",
+            headers={"Authorization": f"Bearer {self.api_key}"},
+            json={
+                "stream": True,
+                "chatId": session_id,
+                "detail": self.detail,
+                "variables": self.variables,
+                "messages": [{"role": "user", "content": last_msg["content"]}],
+            },
+            stream=True,
+        ) as r:
+            for line in r.iter_lines():
+                if line:
+                    try:
+                        if line.startswith(b"data: "):
+                            if line[6:].decode("utf-8") == "[DONE]":
+                                break
+
+                            data = json.loads(line[6:])
+                            if "choices" in data and len(data["choices"]) > 0:
+                                delta = data["choices"][0].get("delta", {})
+                                if (
+                                    delta
+                                    and "content" in delta
+                                    and delta["content"] is not None
+                                ):
+                                    content = delta["content"]
+                                    if "<think>" in content:
+                                        continue
+                                    if "</think>" in content:
+                                        continue
+                                    yield content
+
+                    except json.JSONDecodeError as e:
+                        continue
+                    except Exception as e:
+                        continue
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        logger.bind(tag=TAG).error(
+            f"fastgpt暂未实现完整的工具调用（function call），建议使用其他意图识别"
+        )

main/xingxing-server/core/providers/llm/gemini/gemini.py

@@ -0,0 +1,213 @@
+import os, json, uuid
+from types import SimpleNamespace
+from typing import Any, Dict, List
+
+import requests
+from google import generativeai as genai
+from google.generativeai import types, GenerationConfig
+
+from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
+from config.logger import setup_logging
+from google.generativeai.types import GenerateContentResponse
+from requests import RequestException
+
+log = setup_logging()
+TAG = __name__
+
+
+def test_proxy(proxy_url: str, test_url: str) -> bool:
+    try:
+        resp = requests.get(test_url, proxies={"http": proxy_url, "https": proxy_url})
+        return 200 <= resp.status_code < 400
+    except RequestException:
+        return False
+
+
+def setup_proxy_env(http_proxy: str | None, https_proxy: str | None):
+    """
+    分别测试 HTTP 和 HTTPS 代理是否可用，并设置环境变量。
+    如果 HTTPS 代理不可用但 HTTP 可用，会将 HTTPS_PROXY 也指向 HTTP。
+    """
+    test_http_url = "http://www.google.com"
+    test_https_url = "https://www.google.com"
+
+    ok_http = ok_https = False
+
+    if http_proxy:
+        ok_http = test_proxy(http_proxy, test_http_url)
+        if ok_http:
+            os.environ["HTTP_PROXY"] = http_proxy
+            log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {http_proxy}")
+        else:
+            log.bind(tag=TAG).warning(f"配置提供的Gemini HTTP代理不可用: {http_proxy}")
+
+    if https_proxy:
+        ok_https = test_proxy(https_proxy, test_https_url)
+        if ok_https:
+            os.environ["HTTPS_PROXY"] = https_proxy
+            log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {https_proxy}")
+        else:
+            log.bind(tag=TAG).warning(
+                f"配置提供的Gemini HTTPS代理不可用: {https_proxy}"
+            )
+
+    # 如果https_proxy不可用，但http_proxy可用且能走通https，则复用http_proxy作为https_proxy
+    if ok_http and not ok_https:
+        if test_proxy(http_proxy, test_https_url):
+            os.environ["HTTPS_PROXY"] = http_proxy
+            ok_https = True
+            log.bind(tag=TAG).info(f"复用HTTP代理作为HTTPS代理: {http_proxy}")
+
+    if not ok_http and not ok_https:
+        log.bind(tag=TAG).error(
+            f"Gemini 代理设置失败: HTTP 和 HTTPS 代理都不可用，请检查配置"
+        )
+        raise RuntimeError("HTTP 和 HTTPS 代理都不可用，请检查配置")
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, cfg: Dict[str, Any]):
+        self.model_name = cfg.get("model_name", "gemini-2.0-flash")
+        self.api_key = cfg["api_key"]
+        http_proxy = cfg.get("http_proxy")
+        https_proxy = cfg.get("https_proxy")
+
+        model_key_msg = check_model_key("LLM", self.api_key)
+        if model_key_msg:
+            log.bind(tag=TAG).error(model_key_msg)
+
+        if http_proxy or https_proxy:
+            log.bind(tag=TAG).info(
+                f"检测到Gemini代理配置，开始测试代理连通性和设置代理环境..."
+            )
+            setup_proxy_env(http_proxy, https_proxy)
+            log.bind(tag=TAG).info(
+                f"Gemini 代理设置成功 - HTTP: {http_proxy}, HTTPS: {https_proxy}"
+            )
+        # 配置API密钥
+        genai.configure(api_key=self.api_key)
+
+        # 设置请求超时（秒）
+        self.timeout = cfg.get("timeout", 120)  # 默认120秒
+
+        # 创建模型实例
+        self.model = genai.GenerativeModel(self.model_name)
+
+        self.gen_cfg = GenerationConfig(
+            temperature=0.7,
+            top_p=0.9,
+            top_k=40,
+            max_output_tokens=2048,
+        )
+
+    @staticmethod
+    def _build_tools(funcs: List[Dict[str, Any]] | None):
+        if not funcs:
+            return None
+        return [
+            types.Tool(
+                function_declarations=[
+                    types.FunctionDeclaration(
+                        name=f["function"]["name"],
+                        description=f["function"]["description"],
+                        parameters=f["function"]["parameters"],
+                    )
+                    for f in funcs
+                ]
+            )
+        ]
+
+    # Gemini文档提到，无需维护session-id，直接用dialogue拼接而成
+    def response(self, session_id, dialogue, **kwargs):
+        yield from self._generate(dialogue, None)
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        yield from self._generate(dialogue, self._build_tools(functions))
+
+    def _generate(self, dialogue, tools):
+        role_map = {"assistant": "model", "user": "user"}
+        contents: list = []
+        # 拼接对话
+        for m in dialogue:
+            r = m["role"]
+
+            if r == "assistant" and "tool_calls" in m:
+                tc = m["tool_calls"][0]
+                contents.append(
+                    {
+                        "role": "model",
+                        "parts": [
+                            {
+                                "function_call": {
+                                    "name": tc["function"]["name"],
+                                    "args": json.loads(tc["function"]["arguments"]),
+                                }
+                            }
+                        ],
+                    }
+                )
+                continue
+
+            if r == "tool":
+                contents.append(
+                    {
+                        "role": "model",
+                        "parts": [{"text": str(m.get("content", ""))}],
+                    }
+                )
+                continue
+
+            contents.append(
+                {
+                    "role": role_map.get(r, "user"),
+                    "parts": [{"text": str(m.get("content", ""))}],
+                }
+            )
+
+        stream: GenerateContentResponse = self.model.generate_content(
+            contents=contents,
+            generation_config=self.gen_cfg,
+            tools=tools,
+            stream=True,
+            timeout=self.timeout,
+        )
+
+        try:
+            for chunk in stream:
+                cand = chunk.candidates[0]
+                for part in cand.content.parts:
+                    # a) 函数调用-通常是最后一段话才是函数调用
+                    if getattr(part, "function_call", None):
+                        fc = part.function_call
+                        yield None, [
+                            SimpleNamespace(
+                                id=uuid.uuid4().hex,
+                                type="function",
+                                function=SimpleNamespace(
+                                    name=fc.name,
+                                    arguments=json.dumps(
+                                        dict(fc.args), ensure_ascii=False
+                                    ),
+                                ),
+                            )
+                        ]
+                        return
+                    # b) 普通文本
+                    if getattr(part, "text", None):
+                        yield part.text if tools is None else (part.text, None)
+
+        finally:
+            if tools is not None:
+                yield None, None  # function‑mode 结束，返回哑包
+
+    # 关闭stream，预留后续打断对话功能的功能方法，官方文档推荐打断对话要关闭上一个流，可以有效减少配额计费和资源占用
+    @staticmethod
+    def _safe_finish_stream(stream: GenerateContentResponse):
+        if hasattr(stream, "resolve"):
+            stream.resolve()  # Gemini SDK version ≥ 0.5.0
+        elif hasattr(stream, "close"):
+            stream.close()  # Gemini SDK version < 0.5.0
+        else:
+            for _ in stream:  # 兜底耗尽
+                pass

main/xingxing-server/core/providers/llm/homeassistant/homeassistant.py

@@ -0,0 +1,64 @@
+import requests
+from requests.exceptions import RequestException
+from config.logger import setup_logging
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.agent_id = config.get("agent_id")  # 对应 agent_id
+        self.api_key = config.get("api_key")
+        self.base_url = config.get("base_url", config.get("url"))  # 默认使用 base_url
+        self.api_url = f"{self.base_url}/api/conversation/process"  # 拼接完整的 API URL
+
+    def response(self, session_id, dialogue, **kwargs):
+        # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
+
+        # 提取最后一个 role 为 'user' 的 content
+        input_text = None
+        if isinstance(dialogue, list):  # 确保 dialogue 是一个列表
+            # 逆序遍历，找到最后一个 role 为 'user' 的消息
+            for message in reversed(dialogue):
+                if message.get("role") == "user":  # 找到 role 为 'user' 的消息
+                    input_text = message.get("content", "")
+                    break  # 找到后立即退出循环
+
+        # 构造请求数据
+        payload = {
+            "text": input_text,
+            "agent_id": self.agent_id,
+            "conversation_id": session_id,  # 使用 session_id 作为 conversation_id
+        }
+        # 设置请求头
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+        # 发起 POST 请求
+        with requests.post(self.api_url, json=payload, headers=headers) as response:
+            # 检查请求是否成功
+            response.raise_for_status()
+
+            # 解析返回数据
+            data = response.json()
+        speech = (
+            data.get("response", {})
+            .get("speech", {})
+            .get("plain", {})
+            .get("speech", "")
+        )
+
+        # 返回生成的内容
+        if speech:
+            yield speech
+        else:
+            logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        logger.bind(tag=TAG).error(
+            f"homeassistant不支持（function call），建议使用其他意图识别"
+        )

main/xingxing-server/core/providers/llm/ollama/ollama.py

@@ -0,0 +1,171 @@
+from config.logger import setup_logging
+from openai import OpenAI
+import json
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.model_name = config.get("model_name")
+        self.base_url = config.get("base_url", "http://localhost:11434")
+        # Initialize OpenAI client with Ollama base URL
+        # 如果没有v1，增加v1
+        if not self.base_url.endswith("/v1"):
+            self.base_url = f"{self.base_url}/v1"
+
+        self.client = OpenAI(
+            base_url=self.base_url,
+            api_key="ollama",  # Ollama doesn't need an API key but OpenAI client requires one
+        )
+
+        # 检查是否是qwen3模型
+        self.is_qwen3 = self.model_name and self.model_name.lower().startswith("qwen3")
+
+    def response(self, session_id, dialogue, **kwargs):
+        # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
+        if self.is_qwen3:
+            # 复制对话列表，避免修改原始对话
+            dialogue_copy = dialogue.copy()
+
+            # 找到最后一条用户消息
+            for i in range(len(dialogue_copy) - 1, -1, -1):
+                if dialogue_copy[i]["role"] == "user":
+                    # 在用户消息前添加/no_think指令
+                    dialogue_copy[i]["content"] = (
+                        "/no_think " + dialogue_copy[i]["content"]
+                    )
+                    logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
+                    break
+
+            # 使用修改后的对话
+            dialogue = dialogue_copy
+
+        responses = self.client.chat.completions.create(
+            model=self.model_name, messages=dialogue, stream=True
+        )
+        is_active = True
+        # 用于处理跨chunk的标签
+        buffer = ""
+
+        try:
+            for chunk in responses:
+                try:
+                    delta = (
+                        chunk.choices[0].delta
+                        if getattr(chunk, "choices", None)
+                        else None
+                    )
+                    content = delta.content if hasattr(delta, "content") else ""
+
+                    if content:
+                        # 将内容添加到缓冲区
+                        buffer += content
+
+                        # 处理缓冲区中的标签
+                        while "<think>" in buffer and "</think>" in buffer:
+                            # 找到完整的<think></think>标签并移除
+                            pre = buffer.split("<think>", 1)[0]
+                            post = buffer.split("</think>", 1)[1]
+                            buffer = pre + post
+
+                        # 处理只有开始标签的情况
+                        if "<think>" in buffer:
+                            is_active = False
+                            buffer = buffer.split("<think>", 1)[0]
+
+                        # 处理只有结束标签的情况
+                        if "</think>" in buffer:
+                            is_active = True
+                            buffer = buffer.split("</think>", 1)[1]
+
+                        # 如果当前处于活动状态且缓冲区有内容，则输出
+                        if is_active and buffer:
+                            yield buffer
+                            buffer = ""  # 清空缓冲区
+
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
+        finally:
+            responses.close()
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
+        if self.is_qwen3:
+            # 复制对话列表，避免修改原始对话
+            dialogue_copy = dialogue.copy()
+
+            # 找到最后一条用户消息
+            for i in range(len(dialogue_copy) - 1, -1, -1):
+                if dialogue_copy[i]["role"] == "user":
+                    # 在用户消息前添加/no_think指令
+                    dialogue_copy[i]["content"] = (
+                        "/no_think " + dialogue_copy[i]["content"]
+                    )
+                    logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
+                    break
+
+            # 使用修改后的对话
+            dialogue = dialogue_copy
+
+        stream = self.client.chat.completions.create(
+            model=self.model_name,
+            messages=dialogue,
+            stream=True,
+            tools=functions,
+        )
+
+        is_active = True
+        buffer = ""
+
+        try:
+            for chunk in stream:
+                try:
+                    delta = (
+                        chunk.choices[0].delta
+                        if getattr(chunk, "choices", None)
+                        else None
+                    )
+                    content = delta.content if hasattr(delta, "content") else None
+                    tool_calls = (
+                        delta.tool_calls if hasattr(delta, "tool_calls") else None
+                    )
+
+                    # 如果是工具调用，直接传递
+                    if tool_calls:
+                        yield None, tool_calls
+                        continue
+
+                    # 处理文本内容
+                    if content:
+                        # 将内容添加到缓冲区
+                        buffer += content
+
+                        # 处理缓冲区中的标签
+                        while "<think>" in buffer and "</think>" in buffer:
+                            # 找到完整的<think></think>标签并移除
+                            pre = buffer.split("<think>", 1)[0]
+                            post = buffer.split("</think>", 1)[1]
+                            buffer = pre + post
+
+                        # 处理只有开始标签的情况
+                        if "<think>" in buffer:
+                            is_active = False
+                            buffer = buffer.split("<think>", 1)[0]
+
+                        # 处理只有结束标签的情况
+                        if "</think>" in buffer:
+                            is_active = True
+                            buffer = buffer.split("</think>", 1)[1]
+
+                        # 如果当前处于活动状态且缓冲区有内容，则输出
+                        if is_active and buffer:
+                            yield buffer, None
+                            buffer = ""  # 清空缓冲区
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
+                    continue
+        finally:
+            stream.close()

main/xingxing-server/core/providers/llm/openai/openai.py

@@ -0,0 +1,253 @@
+import httpx
+import openai
+import time
+from openai.types import CompletionUsage
+from config.logger import setup_logging
+from core.utils.util import check_model_key
+from core.providers.llm.base import LLMProviderBase
+from urllib.parse import urlparse
+
+TAG = __name__
+logger = setup_logging()
+
+# 需要禁用思考模式的平台域名及其对应参数（默认关闭思考模式）
+THINKING_DISABLED_DOMAINS = {
+    "aliyuncs.com": {"enable_thinking": False},
+    "bigmodel.cn": {"thinking": {"type": "disabled"}},
+    "moonshot.cn": {"thinking": {"type": "disabled"}},
+    "volces.com": {"thinking": {"type": "disabled"}},
+}
+
+RETRYABLE_STATUS_CODES = {408, 409, 429, 500, 502, 503, 504}
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.model_name = config.get("model_name")
+        self.api_key = config.get("api_key")
+        if "base_url" in config:
+            self.base_url = config.get("base_url")
+        else:
+            self.base_url = config.get("url")
+        
+        timeout_config = config.get("timeout")
+        if isinstance(timeout_config, dict):
+            # 细粒度超时配置
+            custom_timeout = httpx.Timeout(
+                pool=timeout_config.get("pool", 2.0),
+                connect=timeout_config.get("connect", 3.0),
+                write=timeout_config.get("write", 5.0),
+                read=timeout_config.get("read", 60.0)
+            )
+        elif isinstance(timeout_config, (int, float)) and timeout_config > 0:
+            # 兼容旧的单一超时配置（整数或浮点数）
+            custom_timeout = httpx.Timeout(timeout_config)
+        else:
+            # 未配置或配置无效，使用默认值
+            custom_timeout = httpx.Timeout(300)
+
+        param_defaults = {
+            "max_tokens": int,
+            "temperature": lambda x: round(float(x), 1),
+            "top_p": lambda x: round(float(x), 1),
+            "frequency_penalty": lambda x: round(float(x), 1),
+        }
+
+        for param, converter in param_defaults.items():
+            value = config.get(param)
+            try:
+                setattr(
+                    self,
+                    param,
+                    converter(value) if value not in (None, "") else None,
+                )
+            except (ValueError, TypeError):
+                setattr(self, param, None)
+
+        try:
+            self.stream_retry_attempts = max(
+                0, int(config.get("stream_retry_attempts", 1))
+            )
+        except (ValueError, TypeError):
+            self.stream_retry_attempts = 1
+
+        try:
+            self.stream_retry_backoff = max(
+                0.0, float(config.get("stream_retry_backoff", 0.8))
+            )
+        except (ValueError, TypeError):
+            self.stream_retry_backoff = 0.8
+
+        logger.debug(
+            f"意图识别参数初始化: {self.temperature}, {self.max_tokens}, {self.top_p}, {self.frequency_penalty}"
+        )
+
+        model_key_msg = check_model_key("LLM", self.api_key)
+        if model_key_msg:
+            logger.bind(tag=TAG).error(model_key_msg)
+        self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url, timeout=custom_timeout)
+
+    @staticmethod
+    def normalize_dialogue(dialogue):
+        """自动修复 dialogue 中缺失 content 的消息"""
+        for msg in dialogue:
+            if "role" in msg and "content" not in msg:
+                msg["content"] = ""
+        return dialogue
+
+    def _apply_thinking_disabled(self, request_params: dict):
+        """根据域名自动禁用思考模式"""
+        parsed_url = urlparse(self.base_url)
+        domain = parsed_url.netloc
+        for disabled_domain, params in THINKING_DISABLED_DOMAINS.items():
+            if disabled_domain in domain:
+                request_params.setdefault("extra_body", {}).update(params)
+                logger.bind(tag=TAG).info(f"为域名 {domain} 禁用思考模式，参数: {params}")
+                break
+
+    @staticmethod
+    def _extract_status_code(exc):
+        status_code = getattr(exc, "status_code", None)
+        if isinstance(status_code, int):
+            return status_code
+
+        response = getattr(exc, "response", None)
+        if response is not None:
+            response_status = getattr(response, "status_code", None)
+            if isinstance(response_status, int):
+                return response_status
+
+        return None
+
+    def _is_retryable_error(self, exc):
+        status_code = self._extract_status_code(exc)
+        if status_code in RETRYABLE_STATUS_CODES:
+            return True
+
+        error_text = str(exc).lower()
+        retryable_keywords = (
+            "timeout",
+            "timed out",
+            "temporarily unavailable",
+            "server error",
+            "connection error",
+            "connection reset",
+            "service unavailable",
+            "bad gateway",
+            "gateway timeout",
+        )
+        return any(keyword in error_text for keyword in retryable_keywords)
+
+    def _create_stream_with_retry(self, request_params):
+        attempt = 0
+        delay = self.stream_retry_backoff
+
+        while True:
+            try:
+                return self.client.chat.completions.create(**request_params)
+            except Exception as e:
+                should_retry = (
+                    attempt < self.stream_retry_attempts
+                    and self._is_retryable_error(e)
+                )
+                if not should_retry:
+                    raise
+
+                attempt += 1
+                status_code = self._extract_status_code(e)
+                logger.bind(tag=TAG).warning(
+                    f"LLM首包前请求失败，准备重试 {attempt}/{self.stream_retry_attempts}，"
+                    f"status={status_code}, error={e}"
+                )
+                if delay > 0:
+                    time.sleep(delay)
+                    delay *= 2
+
+    def response(self, session_id, dialogue, **kwargs):
+        dialogue = self.normalize_dialogue(dialogue)
+
+        request_params = {
+            "model": self.model_name,
+            "messages": dialogue,
+            "stream": True,
+        }
+
+        # 添加可选参数,只有当参数不为None时才添加
+        optional_params = {
+            "max_tokens": kwargs.get("max_tokens", self.max_tokens),
+            "temperature": kwargs.get("temperature", self.temperature),
+            "top_p": kwargs.get("top_p", self.top_p),
+            "frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
+        }
+
+        for key, value in optional_params.items():
+            if value is not None:
+                request_params[key] = value
+
+        # 禁用思考模式
+        self._apply_thinking_disabled(request_params)
+
+        responses = self._create_stream_with_retry(request_params)
+
+        is_active = True
+        try:            
+            for chunk in responses:
+                try:
+                    delta = chunk.choices[0].delta if getattr(chunk, "choices", None) else None
+                    content = getattr(delta, "content", "") if delta else ""
+                except IndexError:
+                    content = ""
+                if content:
+                    if "<think>" in content:
+                        is_active = False
+                        content = content.split("<think>")[0]
+                    if "</think>" in content:
+                        is_active = True
+                        content = content.split("</think>")[-1]
+                    if is_active:
+                        yield content
+        finally:
+            responses.close()
+
+    def response_with_functions(self, session_id, dialogue, functions=None, **kwargs):
+        dialogue = self.normalize_dialogue(dialogue)
+
+        request_params = {
+            "model": self.model_name,
+            "messages": dialogue,
+            "stream": True,
+            "tools": functions,
+        }
+
+        optional_params = {
+            "max_tokens": kwargs.get("max_tokens", self.max_tokens),
+            "temperature": kwargs.get("temperature", self.temperature),
+            "top_p": kwargs.get("top_p", self.top_p),
+            "frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
+        }
+
+        for key, value in optional_params.items():
+            if value is not None:
+                request_params[key] = value
+
+        # 禁用思考模式
+        self._apply_thinking_disabled(request_params)
+
+        stream = self._create_stream_with_retry(request_params)
+
+        try:
+            for chunk in stream:
+                if getattr(chunk, "choices", None):
+                    delta = chunk.choices[0].delta
+                    content = getattr(delta, "content", "")
+                    tool_calls = getattr(delta, "tool_calls", None)
+                    yield content, tool_calls
+                elif isinstance(getattr(chunk, "usage", None), CompletionUsage):
+                    usage_info = getattr(chunk, "usage", None)
+                    logger.bind(tag=TAG).info(
+                        f"Token 消耗：输入 {getattr(usage_info, 'prompt_tokens', '未知')}，"
+                        f"输出 {getattr(usage_info, 'completion_tokens', '未知')}，"
+                        f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
+                    )
+        finally:
+            stream.close()

main/xingxing-server/core/providers/llm/system_prompt.py

@@ -0,0 +1,103 @@
+def get_system_prompt_for_function(functions: str) -> str:
+    """
+    生成系统提示信息
+    :param functions: 可用的函数列表
+    :return: 系统提示信息
+    """
+
+    SYSTEM_PROMPT = f"""
+====
+
+TOOL USE
+
+You have access to a set of tools that are executed upon the user's approval. You can use one tool per message, and will receive the result of that tool use in the user's response. 
+You use tools step-by-step to accomplish a given task, with each tool use informed by the result of the previous tool use.
+
+# Tool Use Formatting
+
+Tool use is formatted using JSON-style tags. The tool name is enclosed in opening and closing tags, and each parameter is similarly enclosed within its own set of tags. 
+Here's the structure:
+
+<tool_call>
+{{
+    "name": "function name",
+    "arguments": {{
+        "param1": "value1",
+        "param2": "value2",
+        // Add more parameters as needed, if parameters are required, you must provide them
+    }}
+}}
+<tool_call>
+
+For example:
+if you got tool as follow
+
+{{
+    "type": "function",
+    "function": {{
+        "name": "handle_exit_intent",
+        "description": "当用户想结束对话或需要退出系统时调用",
+        "parameters": {{
+            "type": "object",
+            "properties": {{
+                "say_goodbye": {{
+                    "type": "string",
+                    "description": "和用户友好结束对话的告别语",
+                }}
+            }},
+            "required": ["say_goodbye"],
+        }},
+    }},
+}}
+
+you should respond with the following format:
+
+<tool_call>
+{{
+    "name": "handle_exit_intent",
+    "arguments": {{
+        "say_goodbye": "再见，祝您生活愉快！"
+    }}
+}}
+</tool_call>
+
+
+Always adhere to this format for the tool use to ensure proper parsing and execution.
+
+# Tools
+
+{functions}
+
+# Tool Use Guidelines
+
+1. Tools must be called in a separate message, Do not add thoughts when calling tools. The message must start with <tool_call> and end with </tool_call>, with the tool invocation JSON data in between. No additional response content is needed.
+2. Choose the most appropriate tool based on the task and the tool descriptions provided. Assess if you need additional information to proceed, and which of the available tools would be most effective for gathering this information. 
+   For example using the list_files tool is more effective than running a command like \`ls\` in the terminal. It's critical that you think about each available tool and use the one that best fits the current step in the task.
+3. If multiple actions are needed, use one tool at a time per message to accomplish the task iteratively, with each tool use being informed by the result of the previous tool use. Do not assume the outcome of any tool use. 
+   Each step must be informed by the previous step's result.
+4. Formulate your tool use using the JSON format specified for each tool.
+5. After each tool use, the user will respond with the result of that tool use. This result will provide you with the necessary information to continue your task or make further decisions. This response may include:
+- Information about whether the tool succeeded or failed, along with any reasons for failure.
+- Linter errors that may have arisen due to the changes you made, which you'll need to address.
+- New terminal output in reaction to the changes, which you may need to consider or act upon.
+- Any other relevant feedback or information related to the tool use.
+6. ALWAYS wait for user confirmation after each tool use before proceeding. Never assume the success of a tool use without explicit confirmation of the result from the user.
+7. Tool calls should contain no extra information. Only after receiving the tool's response should you integrate it into a complete reply.
+
+It is crucial to proceed step-by-step, waiting for the user's message after each tool use before moving forward with the task. This approach allows you to:
+1. Confirm the success of each step before proceeding.
+2. Address any issues or errors that arise immediately.
+3. Adapt your approach based on new information or unexpected results.
+4. Ensure that each action builds correctly on the previous ones.
+
+By waiting for and carefully considering the user's response after each tool use, you can react accordingly and make informed decisions about how to proceed with the task. This iterative process helps ensure the overall success and accuracy of your work.
+
+====
+
+USER CHAT CONTENT
+
+The following additional message is the user's chat message, and should be followed to the best of your ability without interfering with the TOOL USE guidelines.
+
+"""
+
+    return SYSTEM_PROMPT

main/xingxing-server/core/providers/llm/xinference/xinference.py

@@ -0,0 +1,88 @@
+from config.logger import setup_logging
+from openai import OpenAI
+import json
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.model_name = config.get("model_name")
+        self.base_url = config.get("base_url", "http://localhost:9997")
+        # Initialize OpenAI client with Xinference base URL
+        # 如果没有v1，增加v1
+        if not self.base_url.endswith("/v1"):
+            self.base_url = f"{self.base_url}/v1"
+
+        logger.bind(tag=TAG).info(
+            f"Initializing Xinference LLM provider with model: {self.model_name}, base_url: {self.base_url}"
+        )
+
+        try:
+            self.client = OpenAI(
+                base_url=self.base_url,
+                api_key="xinference",  # Xinference has a similar setup to Ollama where it doesn't need an actual key
+            )
+            logger.bind(tag=TAG).info("Xinference client initialized successfully")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error initializing Xinference client: {e}")
+            raise
+
+    def response(self, session_id, dialogue, **kwargs):
+        logger.bind(tag=TAG).debug(
+            f"Sending request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}"
+        )
+        responses = self.client.chat.completions.create(
+            model=self.model_name, messages=dialogue, stream=True
+        )
+        is_active = True
+        for chunk in responses:
+            try:
+                delta = (
+                    chunk.choices[0].delta
+                    if getattr(chunk, "choices", None)
+                    else None
+                )
+                content = delta.content if hasattr(delta, "content") else ""
+                if content:
+                    if "<think>" in content:
+                        is_active = False
+                        content = content.split("<think>")[0]
+                    if "</think>" in content:
+                        is_active = True
+                        content = content.split("</think>")[-1]
+                    if is_active:
+                        yield content
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        logger.bind(tag=TAG).debug(
+            f"Sending function call request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}"
+        )
+        if functions:
+            logger.bind(tag=TAG).debug(
+                f"Function calls enabled with: {[f.get('function', {}).get('name') for f in functions]}"
+            )
+
+        stream = self.client.chat.completions.create(
+            model=self.model_name,
+            messages=dialogue,
+            stream=True,
+            tools=functions,
+        )
+
+        try:
+            for chunk in stream:
+                delta = chunk.choices[0].delta
+                content = delta.content
+                tool_calls = delta.tool_calls
+
+                if content:
+                    yield content, tool_calls
+                elif tool_calls:
+                    yield None, tool_calls
+        finally:
+            stream.close()

main/xingxing-server/core/providers/memory/base.py

@@ -0,0 +1,28 @@
+from abc import ABC, abstractmethod
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class MemoryProviderBase(ABC):
+    def __init__(self, config):
+        self.config = config
+        self.role_id = None
+
+    def set_llm(self, llm):
+        self.llm = llm
+
+    @abstractmethod
+    async def save_memory(self, msgs, session_id=None):
+        """Save a new memory for specific role and return memory ID"""
+        print("this is base func", msgs)
+
+    @abstractmethod
+    async def query_memory(self, query: str) -> str:
+        """Query memories for specific role based on similarity"""
+        return "please implement query method"
+
+    def init_memory(self, role_id, llm, **kwargs):
+        self.role_id = role_id
+        self.llm = llm

main/xingxing-server/core/providers/memory/mem0ai/mem0ai.py

@@ -0,0 +1,110 @@
+import json
+import traceback
+
+from ..base import MemoryProviderBase, logger
+from mem0 import MemoryClient
+from core.utils.util import check_model_key
+
+TAG = __name__
+
+
+class MemoryProvider(MemoryProviderBase):
+    def __init__(self, config, summary_memory=None):
+        super().__init__(config)
+        self.api_key = config.get("api_key", "")
+        self.api_version = config.get("api_version", "v1.1")
+        model_key_msg = check_model_key("Mem0ai", self.api_key)
+        if model_key_msg:
+            logger.bind(tag=TAG).error(model_key_msg)
+            self.use_mem0 = False
+            return
+        else:
+            self.use_mem0 = True
+
+        try:
+            self.client = MemoryClient(api_key=self.api_key)
+            logger.bind(tag=TAG).info("成功连接到 Mem0ai 服务")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"连接到 Mem0ai 服务时发生错误: {str(e)}")
+            logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
+            self.use_mem0 = False
+
+    async def save_memory(self, msgs, session_id=None):
+        try:
+            if self.use_mem0 and len(msgs) >= 2:
+                # Format the content as a message list for mem0
+                messages = []
+                for message in msgs:
+                    if message.role == "system":
+                        continue
+
+                    content = message.content
+
+                    # Extract content from JSON format if present (for ASR with emotion/language tags)
+                    # Same logic as in query_memory method
+                    try:
+                        if content and content.strip().startswith("{") and content.strip().endswith("}"):
+                            data = json.loads(content)
+                            if "content" in data:
+                                content = data["content"]
+                    except (json.JSONDecodeError, KeyError, TypeError):
+                        # If parsing fails, use original content
+                        pass
+
+                    messages.append({"role": message.role, "content": content})
+
+                result = self.client.add(messages, user_id=self.role_id)
+                logger.bind(tag=TAG).debug(f"Save memory result: {result}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
+
+        return None
+
+    async def query_memory(self, query: str) -> str:
+        if not self.use_mem0:
+            return ""
+        try:
+            if not getattr(self, "role_id", None):
+                return ""
+
+            filters = {"user_id": self.role_id}
+
+            search_query = query
+            try:
+                if query.strip().startswith("{") and query.strip().endswith("}"):
+                    data = json.loads(query)
+                    if "content" in data:
+                        search_query = data["content"]
+            except (json.JSONDecodeError, KeyError):
+                pass
+
+            results = self.client.search(search_query, filters=filters)
+            if not results or "results" not in results:
+                return ""
+
+            # Format each memory entry with its update time up to minutes
+            memories = []
+            for entry in results["results"]:
+                timestamp = entry.get("updated_at", "")
+                if timestamp:
+                    try:
+                        # Parse and reformat the timestamp
+                        dt = timestamp.split(".")[0]  # Remove milliseconds
+                        formatted_time = dt.replace("T", " ")
+                    except:
+                        formatted_time = timestamp
+                memory = entry.get("memory", "")
+                if timestamp and memory:
+                    # Store tuple of (timestamp, formatted_string) for sorting
+                    memories.append((timestamp, f"[{formatted_time}] {memory}"))
+
+            # Sort by timestamp in descending order (newest first)
+            memories.sort(key=lambda x: x[0], reverse=True)
+
+            # Extract only the formatted strings
+            memories_str = "\n".join(f"- {memory[1]}" for memory in memories)
+            logger.bind(tag=TAG).debug(f"Query results: {memories_str}")
+            return memories_str
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"查询记忆失败: {str(e)}")
+            return ""

main/xingxing-server/core/providers/memory/mem_local_short/mem_local_short.py

@@ -0,0 +1,201 @@
+from ..base import MemoryProviderBase, logger
+import time
+import json
+import os
+import yaml
+from config.config_loader import get_project_dir
+from config.manage_api_client import generate_and_save_chat_summary
+import asyncio
+from core.utils.util import check_model_key
+
+
+short_term_memory_prompt = """
+# 时空记忆编织者
+
+## 核心使命
+构建可生长的动态记忆网络，在有限空间内保留关键信息的同时，智能维护信息演变轨迹
+根据对话记录，总结user的重要信息，以便在未来的对话中提供更个性化的服务
+
+## 记忆法则
+### 1. 三维度记忆评估（每次更新必执行）
+| 维度       | 评估标准                  | 权重分 |
+|------------|---------------------------|--------|
+| 时效性     | 信息新鲜度（按对话轮次） | 40%    |
+| 情感强度   | 含💖标记/重复提及次数     | 35%    |
+| 关联密度   | 与其他信息的连接数量      | 25%    |
+
+### 2. 动态更新机制
+**名字变更处理示例：**
+原始记忆："曾用名": ["张三"], "现用名": "张三丰"
+触发条件：当检测到「我叫X」「称呼我Y」等命名信号时
+操作流程：
+1. 将旧名移入"曾用名"列表
+2. 记录命名时间轴："2024-02-15 14:32:启用张三丰"
+3. 在记忆立方追加：「从张三到张三丰的身份蜕变」
+
+### 3. 空间优化策略
+- **信息压缩术**：用符号体系提升密度
+  - ✅"张三丰[北/软工/🐱]"
+  - ❌"北京软件工程师，养猫"
+- **淘汰预警**：当总字数≥900时触发
+  1. 删除权重分<60且3轮未提及的信息
+  2. 合并相似条目（保留时间戳最近的）
+
+## 记忆结构
+输出格式必须为可解析的json字符串，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容
+```json
+{
+  "时空档案": {
+    "身份图谱": {
+      "现用名": "",
+      "特征标记": [] 
+    },
+    "记忆立方": [
+      {
+        "事件": "入职新公司",
+        "时间戳": "2024-03-20",
+        "情感值": 0.9,
+        "关联项": ["下午茶"],
+        "保鲜期": 30 
+      }
+    ]
+  },
+  "关系网络": {
+    "高频话题": {"职场": 12},
+    "暗线联系": [""]
+  },
+  "待响应": {
+    "紧急事项": ["需立即处理的任务"], 
+    "潜在关怀": ["可主动提供的帮助"]
+  },
+  "高光语录": [
+    "最打动人心的瞬间，强烈的情感表达，user的原话"
+  ]
+}
+```
+"""
+
+
+def extract_json_data(json_code):
+    start = json_code.find("```json")
+    # 从start开始找到下一个```结束
+    end = json_code.find("```", start + 1)
+    # print("start:", start, "end:", end)
+    if start == -1 or end == -1:
+        try:
+            jsonData = json.loads(json_code)
+            return json_code
+        except Exception as e:
+            print("Error:", e)
+        return ""
+    jsonData = json_code[start + 7 : end]
+    return jsonData
+
+
+TAG = __name__
+
+
+class MemoryProvider(MemoryProviderBase):
+    def __init__(self, config, summary_memory):
+        super().__init__(config)
+        self.short_memory = ""
+        self.save_to_file = True
+        self.memory_path = get_project_dir() + "data/.memory.yaml"
+        self.load_memory(summary_memory)
+
+    def init_memory(
+        self, role_id, llm, summary_memory=None, save_to_file=True, **kwargs
+    ):
+        super().init_memory(role_id, llm, **kwargs)
+        self.save_to_file = save_to_file
+        self.load_memory(summary_memory)
+
+    def load_memory(self, summary_memory):
+        # api获取到总结记忆后直接返回
+        if summary_memory or not self.save_to_file:
+            self.short_memory = summary_memory
+            return
+
+        all_memory = {}
+        if os.path.exists(self.memory_path):
+            with open(self.memory_path, "r", encoding="utf-8") as f:
+                all_memory = yaml.safe_load(f) or {}
+        if self.role_id in all_memory:
+            self.short_memory = all_memory[self.role_id]
+
+    def save_memory_to_file(self):
+        all_memory = {}
+        if os.path.exists(self.memory_path):
+            with open(self.memory_path, "r", encoding="utf-8") as f:
+                all_memory = yaml.safe_load(f) or {}
+        all_memory[self.role_id] = self.short_memory
+        with open(self.memory_path, "w", encoding="utf-8") as f:
+            yaml.dump(all_memory, f, allow_unicode=True)
+
+    async def save_memory(self, msgs, session_id=None):
+        # 打印使用的模型信息
+        model_info = getattr(self.llm, "model_name", str(self.llm.__class__.__name__))
+        logger.bind(tag=TAG).debug(f"使用记忆保存模型: {model_info}")
+        api_key = getattr(self.llm, "api_key", None)
+        memory_key_msg = check_model_key("记忆总结专用LLM", api_key)
+        if memory_key_msg:
+            logger.bind(tag=TAG).error(memory_key_msg)
+        if self.llm is None:
+            logger.bind(tag=TAG).error("LLM is not set for memory provider")
+            return None
+
+        if len(msgs) < 2:
+            return None
+
+        msgStr = ""
+        for msg in msgs:
+            content = msg.content
+
+            # Extract content from JSON format if present (for ASR with emotion/language tags)
+            try:
+                if content and content.strip().startswith("{") and content.strip().endswith("}"):
+                    data = json.loads(content)
+                    if "content" in data:
+                        content = data["content"]
+            except (json.JSONDecodeError, KeyError, TypeError):
+                # If parsing fails, use original content
+                pass
+
+            if msg.role == "user":
+                msgStr += f"User: {content}\n"
+            elif msg.role == "assistant":
+                msgStr += f"Assistant: {content}\n"
+        if self.short_memory and len(self.short_memory) > 0:
+            msgStr += "历史记忆：\n"
+            msgStr += self.short_memory
+
+        # 当前时间
+        time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        msgStr += f"当前时间：{time_str}"
+
+        if self.save_to_file:
+            try:
+                result = self.llm.response_no_stream(
+                    short_term_memory_prompt,
+                    msgStr,
+                    max_tokens=2000,
+                    temperature=0.2,
+                )
+                json_str = extract_json_data(result)
+                json.loads(json_str)  # 检查json格式是否正确
+                self.short_memory = json_str
+                self.save_memory_to_file()
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"Error in saving memory: {e}")
+        else:
+            # 当save_to_file为False时，调用Java端的聊天记录总结接口
+            summary_id = session_id if session_id else self.role_id
+            await generate_and_save_chat_summary(summary_id)
+        logger.bind(tag=TAG).info(
+            f"Save memory successful - Role: {self.role_id}, Session: {session_id}"
+        )
+
+        return self.short_memory
+
+    async def query_memory(self, query: str) -> str:
+        return self.short_memory

main/xingxing-server/core/providers/memory/mem_report_only/mem_report_only.py

@@ -0,0 +1,20 @@
+"""
+仅上报聊天记录，不进行记忆总结
+"""
+
+from ..base import MemoryProviderBase, logger
+
+TAG = __name__
+
+
+class MemoryProvider(MemoryProviderBase):
+    def __init__(self, config, summary_memory=None):
+        super().__init__(config)
+
+    async def save_memory(self, msgs, session_id=None):
+        logger.bind(tag=TAG).debug("mem_report_only mode: No memory saving or summarization is performed.")
+        return None
+
+    async def query_memory(self, query: str) -> str:
+        logger.bind(tag=TAG).debug("mem_report_only mode: No memory query is performed.")
+        return ""

main/xingxing-server/core/providers/memory/nomem/nomem.py

@@ -0,0 +1,20 @@
+"""
+不使用记忆，可以选择此模块
+"""
+
+from ..base import MemoryProviderBase, logger
+
+TAG = __name__
+
+
+class MemoryProvider(MemoryProviderBase):
+    def __init__(self, config, summary_memory=None):
+        super().__init__(config)
+
+    async def save_memory(self, msgs, session_id=None):
+        logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.")
+        return None
+
+    async def query_memory(self, query: str) -> str:
+        logger.bind(tag=TAG).debug("nomem mode: No memory query is performed.")
+        return ""

main/xingxing-server/core/providers/memory/powermem/powermem.py

@@ -0,0 +1,341 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+@time: 2026/01/08
+@file: powermem.py
+@desc: PowerMem memory provider for xiaozhi-esp32-server
+       PowerMem is an open-source agent memory component from OceanBase
+       GitHub: https://github.com/oceanbase/powermem
+       Website: https://www.powermem.ai/
+@Author: wayyoungboy
+"""
+
+import asyncio
+import json
+import traceback
+from typing import Optional, Dict, Any
+
+from ..base import MemoryProviderBase, logger
+
+TAG = __name__
+
+
+class MemoryProvider(MemoryProviderBase):
+    """
+    PowerMem memory provider implementation.
+
+    PowerMem is an open-source agent memory component that provides
+    efficient memory management for AI agents.
+
+    Supports multiple storage backends (sqlite, oceanbase, postgres),
+    LLM providers (qwen, openai, etc.) and embedding providers.
+
+    Config options:
+        - enable_user_profile: bool - Enable UserMemory for user profiling (requires OceanBase)
+        - database_provider: str - Storage backend (sqlite, oceanbase, postgres)
+        - llm_provider: str - LLM provider (qwen, openai, etc.)
+        - embedding_provider: str - Embedding provider (qwen, openai, etc.)
+    """
+
+    def __init__(self, config: Dict[str, Any], summary_memory: Optional[str] = None):
+        super().__init__(config)
+        self.use_powermem = False
+        self.memory_client = None
+        self.enable_user_profile = False
+        self.last_profile_content = ""  # Cache for user profile from UserMemory
+
+        try:
+            # Check if user profile mode is enabled
+            self.enable_user_profile = config.get("enable_user_profile", False)
+            
+            # Get configuration parameters
+            database_provider = config.get("database_provider", "sqlite")
+            llm_provider = config.get("llm_provider", "qwen")
+            embedding_provider = config.get("embedding_provider", "qwen")
+
+            # Build powermem configuration dict
+            # PowerMem supports two config styles:
+            # 1. powermem style: database, llm, embedding
+            # 2. mem0 style: vector_store, llm, embedder
+            powermem_config = {}
+
+            # Configure vector store / database
+            if "vector_store" in config:
+                powermem_config["vector_store"] = config["vector_store"]
+            elif "database" in config:
+                powermem_config["database"] = config["database"]
+            else:
+                powermem_config["vector_store"] = {
+                    "provider": database_provider,
+                    "config": {}
+                }
+
+            # Configure LLM
+            if "llm" in config:
+                powermem_config["llm"] = config["llm"]
+            else:
+                llm_config = {}
+                if config.get("llm_api_key"):
+                    llm_config["api_key"] = config["llm_api_key"]
+                if config.get("llm_model"):
+                    llm_config["model"] = config["llm_model"]
+                # Handle base_url based on provider type
+                # - qwen provider uses dashscope_base_url
+                # - openai provider uses openai_base_url
+                if llm_provider == "qwen":
+                    base_url = config.get("dashscope_base_url") or config.get("llm_base_url")
+                    if base_url:
+                        llm_config["dashscope_base_url"] = base_url
+                else:
+                    base_url = config.get("openai_base_url") or config.get("llm_base_url")
+                    if base_url:
+                        llm_config["openai_base_url"] = base_url
+
+                powermem_config["llm"] = {
+                    "provider": llm_provider,
+                    "config": llm_config
+                }
+
+            # Configure embedder
+            if "embedder" in config:
+                powermem_config["embedder"] = config["embedder"]
+            else:
+                embedder_config = {}
+                if config.get("embedding_api_key"):
+                    embedder_config["api_key"] = config["embedding_api_key"]
+                if config.get("embedding_model"):
+                    embedder_config["model"] = config["embedding_model"]
+                # Handle base_url based on provider type
+                # - qwen provider uses dashscope_base_url
+                # - openai provider uses openai_base_url
+                # Priority: embedding_xxx_base_url > embedding_base_url > xxx_base_url
+                if embedding_provider == "qwen":
+                    base_url = config.get("embedding_dashscope_base_url") or config.get("embedding_base_url")
+                    if base_url:
+                        embedder_config["dashscope_base_url"] = base_url
+                else:
+                    base_url = config.get("embedding_openai_base_url") or config.get("embedding_base_url")
+                    if base_url:
+                        embedder_config["openai_base_url"] = base_url
+
+                powermem_config["embedder"] = {
+                    "provider": embedding_provider,
+                    "config": embedder_config
+                }
+
+            # Initialize memory client based on mode
+            if self.enable_user_profile:
+                from powermem import UserMemory
+                self.memory_client = UserMemory(config=powermem_config)
+                memory_mode = "UserMemory (用户画像模式)"
+            else:
+                from powermem import AsyncMemory
+                self.memory_client = AsyncMemory(config=powermem_config)
+                memory_mode = "AsyncMemory (普通记忆模式)"
+
+            self.use_powermem = True
+
+            logger.bind(tag=TAG).info(
+                f"PowerMem initialized successfully: mode={memory_mode}, "
+                f"database={powermem_config['vector_store']['provider']}, llm={powermem_config['llm']['provider']}, embedding={powermem_config['embedder']['provider']}"
+            )            
+            
+        except ImportError as e:
+            logger.bind(tag=TAG).error(
+                f"PowerMem not installed. Please install with: pip install powermem. Error: {e}"
+            )
+            self.use_powermem = False
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Failed to initialize PowerMem: {str(e)}")
+            logger.bind(tag=TAG).debug(f"Detailed error: {traceback.format_exc()}")
+            self.use_powermem = False
+
+    async def save_memory(self, msgs, session_id=None):
+        """
+        Save conversation messages to PowerMem.
+
+        Args:
+            msgs: List of message objects with 'role' and 'content' attributes
+
+            session_id: Session identifier (optional, for compatibility)
+
+        Returns:
+            Result from PowerMem API or None if failed
+        """
+        try:
+            if self.use_powermem and self.memory_client is not None and len(msgs) >= 2:
+                # Format the content as a message list for PowerMem
+                messages = []
+                for message in msgs:
+                    if message.role == "system":
+                        continue
+
+                    content = message.content
+
+                    # Extract content from JSON format if present (for ASR with emotion/language tags)
+                    # Same logic as in query_memory method
+                    try:
+                        if content and content.strip().startswith("{") and content.strip().endswith("}"):
+                            data = json.loads(content)
+                            if "content" in data:
+                                content = data["content"]
+                    except (json.JSONDecodeError, KeyError, TypeError):
+                        # If parsing fails, use original content
+                        pass
+
+                    messages.append({"role": message.role, "content": content})
+
+                # Add memory using PowerMem SDK
+                result = self.memory_client.add(
+                    messages=messages,
+                    user_id=self.role_id
+                )
+                # Handle both sync and async returns
+                if asyncio.iscoroutine(result):
+                    result = await result
+
+                logger.bind(tag=TAG).debug(f"Save memory result: {result}")
+
+                # Cache user profile if UserMemory mode and profile was extracted
+                if self.enable_user_profile and result:
+                    if result.get('profile_extracted'):
+                        self.last_profile_content = result.get('profile_content', '')
+                        logger.bind(tag=TAG).debug(f"User profile extracted: {self.last_profile_content}")
+            else:
+                if not self.use_powermem or self.memory_client is None:
+                    logger.bind(tag=TAG).warning("PowerMem is not available, skipping save_memory")
+                elif len(msgs) < 2:
+                    logger.bind(tag=TAG).debug("Not enough messages to save (need at least 2)")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error saving memory: {str(e)}")
+            logger.bind(tag=TAG).debug(f"Detailed error: {traceback.format_exc()}")
+
+        return None
+
+    async def query_memory(self, query: str) -> str:
+        """
+        Query memories from PowerMem based on similarity search.
+
+        Args:
+            query: The search query string (may be JSON format with metadata)
+
+        Returns:
+            Formatted string of relevant memories or empty string if none found
+        """
+        if not self.use_powermem or self.memory_client is None:
+            logger.bind(tag=TAG).warning("PowerMem is not available, skipping query_memory")
+            return ""
+
+        try:
+            if not getattr(self, "role_id", None):
+                logger.bind(tag=TAG).debug("No role_id set, returning empty memory")
+                return ""
+
+            # Extract content from JSON format if present (for ASR with emotion/language tags)
+            search_query = query
+            try:
+                if query.strip().startswith("{") and query.strip().endswith("}"):
+                    data = json.loads(query)
+                    if "content" in data:
+                        search_query = data["content"]
+            except (json.JSONDecodeError, KeyError):
+                # If parsing fails, use original query
+                pass
+
+            result_parts = []
+
+            # If user profile mode is enabled, include user profile in results
+            if self.enable_user_profile:
+                profile = await self.get_user_profile()
+                if profile:
+                    result_parts.append(f"【用户画像】\n{profile}")
+
+            # Search memories using PowerMem SDK
+            if self.enable_user_profile:
+                # UserMemory uses sync search
+                results = await asyncio.to_thread(
+                    self.memory_client.search,
+                    query=search_query,
+                    user_id=self.role_id,
+                    limit=30
+                )
+            else:
+                # AsyncMemory uses async search
+                results = await self.memory_client.search(
+                    query=search_query,
+                    user_id=self.role_id,
+                    limit=30
+                )
+
+            if results and "results" in results:
+                # Format each memory entry with its update time
+                memories = []
+                for entry in results.get("results", []):
+                    # Get timestamp from updated_at or created_at
+                    timestamp = ""
+                    if "updated_at" in entry and entry["updated_at"]:
+                        timestamp = str(entry["updated_at"])
+                    elif "created_at" in entry and entry["created_at"]:
+                        timestamp = str(entry["created_at"])
+
+                    if timestamp:
+                        try:
+                            # Parse and reformat the timestamp (remove milliseconds if present)
+                            if "." in timestamp:
+                                dt = timestamp.split(".")[0]
+                            else:
+                                dt = timestamp
+                            formatted_time = dt.replace("T", " ")
+                        except Exception:
+                            formatted_time = timestamp
+                    else:
+                        formatted_time = ""
+
+                    memory = entry.get("memory", "") or entry.get("content", "")
+                    if memory:
+                        if formatted_time:
+                            # Store tuple of (timestamp, formatted_string) for sorting
+                            memories.append((timestamp, f"[{formatted_time}] {memory}"))
+                        else:
+                            memories.append(("", memory))
+
+                # Sort by timestamp in descending order (newest first)
+                memories.sort(key=lambda x: x[0], reverse=True)
+
+                # Extract only the formatted strings
+                if memories:
+                    memories_str = "\n".join(f"- {memory[1]}" for memory in memories)
+                    result_parts.append(f"【相关记忆】\n{memories_str}")
+
+            final_result = "\n\n".join(result_parts)
+            logger.bind(tag=TAG).debug(f"Query results: {final_result}")
+            return final_result
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error querying memory: {str(e)}")
+            logger.bind(tag=TAG).debug(f"Detailed error: {traceback.format_exc()}")
+            return ""
+
+    async def get_user_profile(self) -> str:
+        """
+        Get user profile from PowerMem (only available in UserMemory mode).
+        
+        In PowerMem 0.3.0+, user profile is automatically extracted during add()
+        and cached in last_profile_content.
+
+        Returns:
+            Formatted user profile string or empty string if not available
+        """
+        if not self.use_powermem or self.memory_client is None:
+            return ""
+
+        if not self.enable_user_profile:
+            logger.bind(tag=TAG).debug("User profile mode is not enabled")
+            return ""
+
+        # Return cached profile content from last add() operation
+        if self.last_profile_content:
+            return self.last_profile_content
+
+        return ""
+

main/xingxing-server/core/providers/tools/__init__.py

@@ -0,0 +1 @@
+ 

main/xingxing-server/core/providers/tools/base/__init__.py

@@ -0,0 +1,6 @@
+"""基础工具定义模块"""
+
+from .tool_types import ToolType, ToolDefinition
+from .tool_executor import ToolExecutor
+
+__all__ = ["ToolType", "ToolDefinition", "ToolExecutor"]

main/xingxing-server/core/providers/tools/base/tool_executor.py

@@ -0,0 +1,27 @@
+"""工具执行器基类定义"""
+
+from abc import ABC, abstractmethod
+from typing import Dict, Any
+from .tool_types import ToolDefinition
+from plugins_func.register import ActionResponse
+
+
+class ToolExecutor(ABC):
+    """工具执行器抽象基类"""
+
+    @abstractmethod
+    async def execute(
+        self, conn, tool_name: str, arguments: Dict[str, Any]
+    ) -> ActionResponse:
+        """执行工具调用"""
+        pass
+
+    @abstractmethod
+    def get_tools(self) -> Dict[str, ToolDefinition]:
+        """获取该执行器管理的所有工具"""
+        pass
+
+    @abstractmethod
+    def has_tool(self, tool_name: str) -> bool:
+        """检查是否有指定工具"""
+        pass

main/xingxing-server/core/providers/tools/base/tool_types.py

@@ -0,0 +1,27 @@
+"""工具系统的类型定义"""
+
+from enum import Enum
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+from plugins_func.register import Action
+
+
+class ToolType(Enum):
+    """工具类型枚举"""
+
+    SERVER_PLUGIN = "server_plugin"  # 服务端插件
+    SERVER_MCP = "server_mcp"  # 服务端MCP
+    DEVICE_IOT = "device_iot"  # 设备端IoT
+    DEVICE_MCP = "device_mcp"  # 设备端MCP
+    MCP_ENDPOINT = "mcp_endpoint"  # MCP接入点
+
+
+@dataclass
+class ToolDefinition:
+    """工具定义"""
+
+    name: str  # 工具名称
+    description: Dict[str, Any]  # 工具描述（OpenAI函数调用格式）
+    tool_type: ToolType  # 工具类型
+    parameters: Optional[Dict[str, Any]] = None  # 额外参数

main/xingxing-server/core/providers/tools/device_iot/__init__.py

@@ -0,0 +1,12 @@
+"""设备端IoT工具模块"""
+
+from .iot_descriptor import IotDescriptor
+from .iot_handler import handleIotDescriptors, handleIotStatus
+from .iot_executor import DeviceIoTExecutor
+
+__all__ = [
+    "IotDescriptor",
+    "handleIotDescriptors",
+    "handleIotStatus",
+    "DeviceIoTExecutor",
+]

main/xingxing-server/core/providers/tools/device_iot/iot_descriptor.py

@@ -0,0 +1,46 @@
+"""IoT设备描述符定义"""
+
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class IotDescriptor:
+    """IoT设备描述符"""
+
+    def __init__(self, name, description, properties, methods):
+        self.name = name
+        self.description = description
+        self.properties = []
+        self.methods = []
+
+        # 根据描述创建属性
+        if properties is not None:
+            for key, value in properties.items():
+                property_item = {}
+                property_item["name"] = key
+                property_item["description"] = value["description"]
+                if value["type"] == "number":
+                    property_item["value"] = 0
+                elif value["type"] == "boolean":
+                    property_item["value"] = False
+                else:
+                    property_item["value"] = ""
+                self.properties.append(property_item)
+
+        # 根据描述创建方法
+        if methods is not None:
+            for key, value in methods.items():
+                method = {}
+                method["description"] = value["description"]
+                method["name"] = key
+                # 检查方法是否有参数
+                if "parameters" in value:
+                    method["parameters"] = {}
+                    for k, v in value["parameters"].items():
+                        method["parameters"][k] = {
+                            "description": v["description"],
+                            "type": v["type"],
+                        }
+                self.methods.append(method)

main/xingxing-server/core/providers/tools/device_iot/iot_executor.py

@@ -0,0 +1,238 @@
+"""设备端IoT工具执行器"""
+
+import json
+import asyncio
+from typing import Dict, Any
+from ..base import ToolType, ToolDefinition, ToolExecutor
+from plugins_func.register import Action, ActionResponse
+
+
+class DeviceIoTExecutor(ToolExecutor):
+    """设备端IoT工具执行器"""
+
+    def __init__(self, conn):
+        self.conn = conn
+        self.iot_tools: Dict[str, ToolDefinition] = {}
+
+    async def execute(
+        self, conn, tool_name: str, arguments: Dict[str, Any]
+    ) -> ActionResponse:
+        """执行设备端IoT工具"""
+        if not self.has_tool(tool_name):
+            return ActionResponse(
+                action=Action.NOTFOUND, response=f"IoT工具 {tool_name} 不存在"
+            )
+
+        try:
+            # 解析工具名称，获取设备名和操作类型
+            if tool_name.startswith("get_"):
+                # 查询操作：get_devicename_property
+                parts = tool_name.split("_", 2)
+                if len(parts) >= 3:
+                    device_name = parts[1]
+                    property_name = parts[2]
+
+                    value = await self._get_iot_status(device_name, property_name)
+                    if value is not None:
+                        # 处理响应模板
+                        response_success = arguments.get(
+                            "response_success", "查询成功：{value}"
+                        )
+                        response = response_success.replace("{value}", str(value))
+
+                        return ActionResponse(
+                            action=Action.RESPONSE,
+                            response=response,
+                        )
+                    else:
+                        response_failure = arguments.get(
+                            "response_failure", f"无法获取{device_name}的状态"
+                        )
+                        return ActionResponse(
+                            action=Action.ERROR, response=response_failure
+                        )
+            else:
+                # 控制操作：devicename_method
+                parts = tool_name.split("_", 1)
+                if len(parts) >= 2:
+                    device_name = parts[0]
+                    method_name = parts[1]
+
+                    # 提取控制参数（排除响应参数）
+                    control_params = {
+                        k: v
+                        for k, v in arguments.items()
+                        if k not in ["response_success", "response_failure"]
+                    }
+
+                    # 发送IoT控制命令
+                    await self._send_iot_command(
+                        device_name, method_name, control_params
+                    )
+
+                    # 等待状态更新
+                    await asyncio.sleep(0.1)
+
+                    response_success = arguments.get("response_success", "操作成功")
+
+                    # 处理响应中的占位符
+                    for param_name, param_value in control_params.items():
+                        placeholder = "{" + param_name + "}"
+                        if placeholder in response_success:
+                            response_success = response_success.replace(
+                                placeholder, str(param_value)
+                            )
+                        if "{value}" in response_success:
+                            response_success = response_success.replace(
+                                "{value}", str(param_value)
+                            )
+                            break
+
+                    return ActionResponse(
+                        action=Action.REQLLM,
+                        result=response_success,
+                    )
+
+            return ActionResponse(action=Action.ERROR, response="无法解析IoT工具名称")
+
+        except Exception as e:
+            response_failure = arguments.get("response_failure", "操作失败")
+            return ActionResponse(action=Action.ERROR, response=response_failure)
+
+    async def _get_iot_status(self, device_name: str, property_name: str):
+        """获取IoT设备状态"""
+        for key, value in self.conn.iot_descriptors.items():
+            if key.lower() == device_name.lower():
+                for property_item in value.properties:
+                    if property_item["name"].lower() == property_name.lower():
+                        return property_item["value"]
+        return None
+
+    async def _send_iot_command(
+        self, device_name: str, method_name: str, parameters: Dict[str, Any]
+    ):
+        """发送IoT控制命令"""
+        for key, value in self.conn.iot_descriptors.items():
+            if key.lower() == device_name.lower():
+                for method in value.methods:
+                    if method["name"].lower() == method_name.lower():
+                        command = {
+                            "name": key,
+                            "method": method["name"],
+                        }
+
+                        if parameters:
+                            command["parameters"] = parameters
+
+                        send_message = json.dumps(
+                            {"type": "iot", "commands": [command]}
+                        )
+                        await self.conn.websocket.send(send_message)
+                        return
+
+        raise Exception(f"未找到设备{device_name}的方法{method_name}")
+
+    def register_iot_tools(self, descriptors: list):
+        """注册IoT工具"""
+        for descriptor in descriptors:
+            device_name = descriptor["name"]
+            device_desc = descriptor["description"]
+
+            # 注册查询工具
+            if "properties" in descriptor:
+                for prop_name, prop_info in descriptor["properties"].items():
+                    tool_name = f"get_{device_name.lower()}_{prop_name.lower()}"
+
+                    tool_desc = {
+                        "type": "function",
+                        "function": {
+                            "name": tool_name,
+                            "description": f"查询{device_desc}的{prop_info['description']}",
+                            "parameters": {
+                                "type": "object",
+                                "properties": {
+                                    "response_success": {
+                                        "type": "string",
+                                        "description": f"查询成功时的友好回复，必须使用{{value}}作为占位符表示查询到的值",
+                                    },
+                                    "response_failure": {
+                                        "type": "string",
+                                        "description": f"查询失败时的友好回复",
+                                    },
+                                },
+                                "required": ["response_success", "response_failure"],
+                            },
+                        },
+                    }
+
+                    self.iot_tools[tool_name] = ToolDefinition(
+                        name=tool_name,
+                        description=tool_desc,
+                        tool_type=ToolType.DEVICE_IOT,
+                    )
+
+            # 注册控制工具
+            if "methods" in descriptor:
+                for method_name, method_info in descriptor["methods"].items():
+                    tool_name = f"{device_name.lower()}_{method_name.lower()}"
+
+                    # 构建参数
+                    parameters = {}
+                    required_params = []
+
+                    # 添加方法的原始参数
+                    if "parameters" in method_info:
+                        parameters.update(
+                            {
+                                param_name: {
+                                    "type": param_info["type"],
+                                    "description": param_info["description"],
+                                }
+                                for param_name, param_info in method_info[
+                                    "parameters"
+                                ].items()
+                            }
+                        )
+                        required_params.extend(method_info["parameters"].keys())
+
+                    # 添加响应参数
+                    parameters.update(
+                        {
+                            "response_success": {
+                                "type": "string",
+                                "description": "操作成功时的友好回复",
+                            },
+                            "response_failure": {
+                                "type": "string",
+                                "description": "操作失败时的友好回复",
+                            },
+                        }
+                    )
+                    required_params.extend(["response_success", "response_failure"])
+
+                    tool_desc = {
+                        "type": "function",
+                        "function": {
+                            "name": tool_name,
+                            "description": f"{device_desc} - {method_info['description']}",
+                            "parameters": {
+                                "type": "object",
+                                "properties": parameters,
+                                "required": required_params,
+                            },
+                        },
+                    }
+
+                    self.iot_tools[tool_name] = ToolDefinition(
+                        name=tool_name,
+                        description=tool_desc,
+                        tool_type=ToolType.DEVICE_IOT,
+                    )
+
+    def get_tools(self) -> Dict[str, ToolDefinition]:
+        """获取所有设备端IoT工具"""
+        return self.iot_tools.copy()
+
+    def has_tool(self, tool_name: str) -> bool:
+        """检查是否有指定的设备端IoT工具"""
+        return tool_name in self.iot_tools

main/xingxing-server/core/providers/tools/device_iot/iot_handler.py

@@ -0,0 +1,87 @@
+"""IoT设备支持模块，提供IoT设备描述符和状态处理"""
+
+import asyncio
+from config.logger import setup_logging
+from .iot_descriptor import IotDescriptor
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.connection import ConnectionHandler
+
+TAG = __name__
+logger = setup_logging()
+
+
+async def handleIotDescriptors(conn: "ConnectionHandler", descriptors):
+    """处理物联网描述"""
+    wait_max_time = 5
+    while (
+        not hasattr(conn, "func_handler")
+        or conn.func_handler is None
+        or not conn.func_handler.finish_init
+    ):
+        await asyncio.sleep(1)
+        wait_max_time -= 1
+        if wait_max_time <= 0:
+            logger.bind(tag=TAG).debug("连接对象没有func_handler")
+            return
+
+    functions_changed = False
+
+    for descriptor in descriptors:
+        # 如果descriptor没有properties和methods，则直接跳过
+        if "properties" not in descriptor and "methods" not in descriptor:
+            continue
+
+        # 处理缺失properties的情况
+        if "properties" not in descriptor:
+            descriptor["properties"] = {}
+            # 从methods中提取所有参数作为properties
+            if "methods" in descriptor:
+                for method_name, method_info in descriptor["methods"].items():
+                    if "parameters" in method_info:
+                        for param_name, param_info in method_info["parameters"].items():
+                            # 将参数信息转换为属性信息
+                            descriptor["properties"][param_name] = {
+                                "description": param_info["description"],
+                                "type": param_info["type"],
+                            }
+
+        # 创建IOT设备描述符
+        iot_descriptor = IotDescriptor(
+            descriptor["name"],
+            descriptor["description"],
+            descriptor["properties"],
+            descriptor["methods"],
+        )
+        conn.iot_descriptors[descriptor["name"]] = iot_descriptor
+        functions_changed = True
+
+    # 如果注册了新函数，更新function描述列表
+    if functions_changed and hasattr(conn, "func_handler"):
+        # 注册IoT工具到统一工具处理器
+        await conn.func_handler.register_iot_tools(descriptors)
+
+        conn.func_handler.current_support_functions()
+
+
+async def handleIotStatus(conn: "ConnectionHandler", states):
+    """处理物联网状态"""
+    for state in states:
+        for key, value in conn.iot_descriptors.items():
+            if key == state["name"]:
+                for property_item in value.properties:
+                    for k, v in state["state"].items():
+                        if property_item["name"] == k:
+                            if type(v) != type(property_item["value"]):
+                                logger.bind(tag=TAG).error(
+                                    f"属性{property_item['name']}的值类型不匹配"
+                                )
+                                break
+                            else:
+                                property_item["value"] = v
+                                logger.bind(tag=TAG).info(
+                                    f"物联网状态更新: {key} , {property_item['name']} = {v}"
+                                )
+                            break
+                break

main/xingxing-server/core/providers/tools/device_mcp/__init__.py

@@ -0,0 +1,21 @@
+"""设备端MCP工具模块"""
+
+from .mcp_client import MCPClient
+from .mcp_handler import (
+    send_mcp_message,
+    handle_mcp_message,
+    send_mcp_initialize_message,
+    send_mcp_tools_list_request,
+    call_mcp_tool,
+)
+from .mcp_executor import DeviceMCPExecutor
+
+__all__ = [
+    "MCPClient",
+    "send_mcp_message",
+    "handle_mcp_message",
+    "send_mcp_initialize_message",
+    "send_mcp_tools_list_request",
+    "call_mcp_tool",
+    "DeviceMCPExecutor",
+]