lobe-chat/packages/prompts/promptfoo/summary-title/eval.yaml

description: Test summary title generation for different conversation types

providers:
  - openai:chat:gpt-5-mini
  - openai:chat:claude-3-5-haiku-latest
  - openai:chat:gemini-flash-latest
  - openai:chat:deepseek-chat

prompts:
  - file://promptfoo/summary-title/prompt.ts

tests:
  - vars:
      messages:
        - role: "user"
          content: "How do I install Node.js on my computer?"
        - role: "assistant"
          content: "To install Node.js, you can download it from the official website nodejs.org and follow the installation instructions for your operating system."
        - role: "user"
          content: "What about using a version manager?"
        - role: "assistant"
          content: "Yes! I recommend using nvm (Node Version Manager) which allows you to install and switch between different Node.js versions easily."
      locale: "en-US"
    assert:
      - type: llm-rubric
        provider: openai:gpt-5-mini
        value: "The response should be a concise title (10 words or less) that summarizes the conversation about Node.js installation"
      - type: regex
        value: "^.{1,50}$"  # Title should be between 1-50 characters
      - type: not-contains
        value: "标点符号"  # Should not contain punctuation as requested in Chinese

  - vars:
      messages:
        - role: "user"
          content: "我想学习做蛋炒饭"
        - role: "assistant"
          content: "蛋炒饭是很经典的家常菜！你需要准备鸡蛋、米饭、葱花、盐和生抽等基本材料。"
        - role: "user"
          content: "具体步骤是什么？"
        - role: "assistant"
          content: "首先打散鸡蛋炒熟盛起，然后下米饭炒散，最后加入鸡蛋和调料炒匀即可。"
      locale: "zh-CN"
    assert:
      - type: llm-rubric
        provider: openai:gpt-5-mini
        value: "The response should be a Chinese title summarizing the conversation about fried rice cooking"
      - type: regex
        value: "^.{1,30}$"  # Chinese titles can be shorter
      - type: contains-any
        value: ["蛋炒饭", "做饭", "烹饪", "料理"]

  - vars:
      messages:
        - role: "user"
          content: "Can you help me debug this Python error?"
        - role: "assistant"
          content: "Of course! Please share the error message and the relevant code."
        - role: "user"
          content: "I'm getting 'AttributeError: 'NoneType' object has no attribute 'split''"
        - role: "assistant"
          content: "This error occurs when you're trying to call .split() on a None value. The variable is likely None instead of a string."
      locale: "en-US"
    assert:
      - type: llm-rubric
        provider: openai:gpt-5-mini

        value: "The response should be a title about Python debugging or error resolution"
      - type: contains-any
        value: ["Python", "debug", "error", "AttributeError", "code"]

  - vars:
      messages:
        - role: "user"
          content: "¿Cómo está el tiempo hoy?"
        - role: "assistant"
          content: "No tengo acceso a información meteorológica en tiempo real, pero puedes consultar el clima en tu área usando aplicaciones como Weather.com o tu app del tiempo local."
      locale: "es-ES"
    assert:
      - type: llm-rubric
        provider: openai:gpt-5-mini

        value: "The response should be a Spanish title about weather inquiry"
      - type: regex
        value: "^.{1,50}$"

  - vars:
      messages:
        - role: "user"
          content: "你知道 litellm 吗？"
      locale: "zh-CN"
    assert:
      - type: regex
        value: "^.{1,20}$"  # Chinese titles can be shorter