# This configuration runs each prompt through a series of example inputs and checks if they meet requirements.

prompts:
  - "You are a helpful assistant. Reply with a concise answer to this inquiry: '{{question}}'"

providers:
  - id: vertex:gemini-1.5-flash-002
    # config:
    #   systemInstruction:
    #     parts:
    #       - text: Always answer with Blah!
  - id: vertex:gemini-1.5-pro-002
    # or use the Google AI Studio API: google:gemini-pro

defaultTest:
  options:
    provider:
      # Use gemini-pro for model-graded evals (e.g. assertions such as llm-rubric)
      text: vertex:gemini-1.0-pro-002
      # Use vertex embeddings for similarity
      embedding: vertex:embedding:text-embedding-004

tests:
  # Deterministic eval metrics
  # See https://www.promptfoo.dev/docs/configuration/expected-outputs/#deterministic-eval-metrics
  - vars:
      question: What's the capital of Cyprus?
    assert:
      - type: contains
        value: Nicosia
  # Model-assisted eval metrics
  # See https://www.promptfoo.dev/docs/configuration/expected-outputs/#model-assisted-eval-metrics
  - vars:
      question: What's the weather like in London generally?
    assert:
      - type: similar
        value: mild and rainy
        threshold: 0.7