{
  "flow": {
    "nodes": [
      {
        "id": "cmt-a1a6fad3271f",
        "type": "comment",
        "position": {
          "x": -800,
          "y": -200
        },
        "data": {
          "title": "Execution-Grounded Code Eval",
          "text": "Six Python coding tasks. Models must generate working implementations.\nThe evaluator node runs each response against hidden unit tests and returns True/False.\nThis gives an objective pass@1 metric \u2014 no LLM judge subjectivity.",
          "color": "#d0e8ff"
        }
      },
      {
        "id": "tbl-d20b4a0ac785",
        "type": "table",
        "position": {
          "x": -800,
          "y": 100
        },
        "data": {
          "title": "Coding Tasks",
          "columns": [
            {
              "header": "task",
              "width": 120,
              "accessor": "c0"
            },
            {
              "header": "function_sig",
              "width": 320,
              "accessor": "c1"
            },
            {
              "header": "test_code",
              "width": 400,
              "accessor": "c2"
            }
          ],
          "rows": [
            {
              "task": "palindrome",
              "function_sig": "def is_palindrome(s: str) -> bool:\n    \"\"\"Return True if s is a palindrome. Handle None, empty string, mixed case.\"\"\"",
              "test_code": "assert is_palindrome('racecar')\nassert is_palindrome('Madam')\nassert not is_palindrome('hello')\nassert is_palindrome('')\nassert not is_palindrome(None) if is_palindrome(None) is False else True"
            },
            {
              "task": "flatten",
              "function_sig": "def flatten(lst) -> list:\n    \"\"\"Flatten a nested list of arbitrary depth into a flat list.\"\"\"",
              "test_code": "assert flatten([1,[2,[3,4]],5]) == [1,2,3,4,5]\nassert flatten([]) == []\nassert flatten([[1],[[2]],3]) == [1,2,3]"
            },
            {
              "task": "count_words",
              "function_sig": "def count_words(text: str) -> dict:\n    \"\"\"Return word frequency dict (case-insensitive). Empty string returns {}.\"\"\"",
              "test_code": "r = count_words('the cat sat on the mat')\nassert r['the'] == 2\nassert r['cat'] == 1\nassert count_words('') == {}"
            },
            {
              "task": "binary_search",
              "function_sig": "def binary_search(arr: list, target) -> int:\n    \"\"\"Return index of target in sorted arr, or -1 if not found.\"\"\"",
              "test_code": "assert binary_search([1,3,5,7,9], 5) == 2\nassert binary_search([1,3,5,7,9], 0) == -1\nassert binary_search([], 1) == -1"
            },
            {
              "task": "fizzbuzz",
              "function_sig": "def fizzbuzz(n: int) -> list:\n    \"\"\"Return list of FizzBuzz strings from 1 to n.\"\"\"",
              "test_code": "r = fizzbuzz(15)\nassert r[2] == 'Fizz'\nassert r[4] == 'Buzz'\nassert r[14] == 'FizzBuzz'\nassert r[0] == '1'"
            },
            {
              "task": "max_subarray",
              "function_sig": "def max_subarray(nums: list) -> int:\n    \"\"\"Return the maximum subarray sum (Kadane's algorithm).\"\"\"",
              "test_code": "assert max_subarray([-2,1,-3,4,-1,2,1,-5,4]) == 6\nassert max_subarray([1]) == 1\nassert max_subarray([-1,-2,-3]) == -1"
            }
          ]
        }
      },
      {
        "id": "pmt-e54b1e4ef197",
        "type": "prompt",
        "position": {
          "x": -200,
          "y": 100
        },
        "data": {
          "prompt": "Implement this Python function exactly as specified by the docstring:\n\n{function_sig}\n\nReturn ONLY the function code \u2014 no imports, no examples, no explanation.",
          "llms": [
            {
              "key": "hermes3-3a1df725f93e",
              "name": "hermes3",
              "emoji": "H",
              "model": "hermes3:latest",
              "base_model": "openai",
              "temp": 0.7,
              "settings": {
                "system_msg": "You are an expert Python programmer. Return ONLY the Python function implementation \u2014 no markdown fences, no explanation, nothing else.",
                "temperature": 0.7,
                "base_url": "http://localhost:11434/v1",
                "api_key": "ollama",
                "response_format": {
                  "type": "text"
                },
                "tools": [],
                "stop": [],
                "top_p": 1,
                "presence_penalty": 0,
                "frequency_penalty": 0
              },
              "formData": {
                "shortname": "hermes3",
                "model": "hermes3:latest",
                "system_msg": "You are an expert Python programmer. Return ONLY the Python function implementation \u2014 no markdown fences, no explanation, nothing else.",
                "temperature": 0.7,
                "base_url": "http://localhost:11434/v1",
                "api_key": "ollama",
                "response_format": "text",
                "tools": "",
                "stop": "",
                "top_p": 1,
                "presence_penalty": 0,
                "frequency_penalty": 0
              }
            },
            {
              "key": "qwen3-46d891b4466e",
              "name": "qwen3",
              "emoji": "Q",
              "model": "qwen3:14b",
              "base_model": "openai",
              "temp": 0.7,
              "settings": {
                "system_msg": "You are an expert Python programmer. Return ONLY the Python function implementation \u2014 no markdown fences, no explanation, nothing else.",
                "temperature": 0.7,
                "base_url": "http://localhost:11434/v1",
                "api_key": "ollama",
                "response_format": {
                  "type": "text"
                },
                "tools": [],
                "stop": [],
                "top_p": 1,
                "presence_penalty": 0,
                "frequency_penalty": 0
              },
              "formData": {
                "shortname": "qwen3",
                "model": "qwen3:14b",
                "system_msg": "You are an expert Python programmer. Return ONLY the Python function implementation \u2014 no markdown fences, no explanation, nothing else.",
                "temperature": 0.7,
                "base_url": "http://localhost:11434/v1",
                "api_key": "ollama",
                "response_format": "text",
                "tools": "",
                "stop": "",
                "top_p": 1,
                "presence_penalty": 0,
                "frequency_penalty": 0
              }
            },
            {
              "key": "phi4-0da188d36729",
              "name": "phi4",
              "emoji": "P",
              "model": "phi4:latest",
              "base_model": "openai",
              "temp": 0.7,
              "settings": {
                "system_msg": "You are an expert Python programmer. Return ONLY the Python function implementation \u2014 no markdown fences, no explanation, nothing else.",
                "temperature": 0.7,
                "base_url": "http://localhost:11434/v1",
                "api_key": "ollama",
                "response_format": {
                  "type": "text"
                },
                "tools": [],
                "stop": [],
                "top_p": 1,
                "presence_penalty": 0,
                "frequency_penalty": 0
              },
              "formData": {
                "shortname": "phi4",
                "model": "phi4:latest",
                "system_msg": "You are an expert Python programmer. Return ONLY the Python function implementation \u2014 no markdown fences, no explanation, nothing else.",
                "temperature": 0.7,
                "base_url": "http://localhost:11434/v1",
                "api_key": "ollama",
                "response_format": "text",
                "tools": "",
                "stop": "",
                "top_p": 1,
                "presence_penalty": 0,
                "frequency_penalty": 0
              }
            },
            {
              "key": "llama3.1-46b65e85729a",
              "name": "llama3.1",
              "emoji": "L",
              "model": "llama3.1:8b",
              "base_model": "openai",
              "temp": 0.7,
              "settings": {
                "system_msg": "You are an expert Python programmer. Return ONLY the Python function implementation \u2014 no markdown fences, no explanation, nothing else.",
                "temperature": 0.7,
                "base_url": "http://localhost:11434/v1",
                "api_key": "ollama",
                "response_format": {
                  "type": "text"
                },
                "tools": [],
                "stop": [],
                "top_p": 1,
                "presence_penalty": 0,
                "frequency_penalty": 0
              },
              "formData": {
                "shortname": "llama3.1",
                "model": "llama3.1:8b",
                "system_msg": "You are an expert Python programmer. Return ONLY the Python function implementation \u2014 no markdown fences, no explanation, nothing else.",
                "temperature": 0.7,
                "base_url": "http://localhost:11434/v1",
                "api_key": "ollama",
                "response_format": "text",
                "tools": "",
                "stop": "",
                "top_p": 1,
                "presence_penalty": 0,
                "frequency_penalty": 0
              }
            }
          ],
          "n": 1,
          "temperature": 0.7
        }
      },
      {
        "id": "eval-1635c5ee1d91",
        "type": "evaluator",
        "position": {
          "x": 420,
          "y": 100
        },
        "data": {
          "title": "Pass/Fail Test Runner",
          "language": "python",
          "sandbox": true,
          "refresh": false,
          "code": "def evaluate(r):\n    import re, textwrap\n    code = r.text\n    # strip markdown fences\n    code = re.sub(r'```[\\w]*\\n?', '', code).strip()\n    test = r.meta.get('test_code', '')\n    full = textwrap.dedent(code) + '\\n\\n' + textwrap.dedent(test)\n    try:\n        exec(full, {})\n        return True\n    except Exception as e:\n        print(f'FAIL [{r.meta.get(\"task\",\"?\")}]: {type(e).__name__}: {e}')\n        return False\n"
        }
      },
      {
        "id": "vis-c7ef9c11ebde",
        "type": "vis",
        "position": {
          "x": 900,
          "y": 100
        },
        "data": {}
      }
    ],
    "edges": [
      {
        "id": "e-be20511e2488-function_sig",
        "source": "tbl-d20b4a0ac785",
        "sourceHandle": "function_sig",
        "target": "pmt-e54b1e4ef197",
        "targetHandle": "function_sig",
        "markerEnd": {
          "type": "arrowclosed",
          "width": 22,
          "height": 22
        }
      },
      {
        "id": "e-ecc22c8fe5bf",
        "source": "pmt-e54b1e4ef197",
        "sourceHandle": "prompt",
        "target": "eval-1635c5ee1d91",
        "targetHandle": "responseBatch",
        "markerEnd": {
          "type": "arrowclosed",
          "width": 22,
          "height": 22
        }
      },
      {
        "id": "e-ad8e6fe84e9d",
        "source": "eval-1635c5ee1d91",
        "sourceHandle": "output",
        "target": "vis-c7ef9c11ebde",
        "targetHandle": "input",
        "markerEnd": {
          "type": "arrowclosed",
          "width": 22,
          "height": 22
        }
      }
    ]
  }
}