# Abstention Environment
# Trains a policy model to abstain from answering when unsure rather than
# hallucinating.  Uses a three-tier reward: correct (1.0) > abstain (lambda) >
# incorrect (0.0).  Verification uses an LLM judge (OMNISCIENCE_GRADER) instead
# of string matching.

abstention:
  resources_servers:
    abstention:
      entrypoint: app.py
      domain: rlhf
      verified: false
      description: Train models to abstain when unsure using three-tier reward on HotPotQA with LLM judge
      value: Improve calibration by rewarding abstention over incorrect answers

      # LLM judge model server
      judge_model_server:
        type: responses_api_models
        name: genrm_model
      judge_responses_create_params:
        input: []
        max_output_tokens: 64
        temperature: 0.0
        top_p: 1.0

      # Reward for abstaining (lambda). Must satisfy 0 < lambda < 1.
      abstention_reward: 0.5

      # Token the model outputs inside \boxed{} to signal abstention
      abstention_token: "[IDK]"

      # Reward for correct and incorrect answers
      correct_reward: 1.0
      incorrect_reward: 0.0

abstention_simple_agent:
  responses_api_agents:
    simple_agent:
      entrypoint: app.py
      resources_server:
        type: resources_servers
        name: abstention
      model_server:
        type: responses_api_models
        name: policy_model
      datasets:
        - name: hotpotqa_train
          type: train
          jsonl_fpath: resources_servers/abstention/data/hotpotqa_train.jsonl
          license: Creative Commons Attribution-ShareAlike 4.0 International
        - name: hotpotqa_val
          type: validation
          jsonl_fpath: resources_servers/abstention/data/hotpotqa_val.jsonl
          license: Creative Commons Attribution-ShareAlike 4.0 International
        - name: example
          type: example
          jsonl_fpath: resources_servers/abstention/data/example.jsonl