{
  "title": "Dynamic Service Generation",
  "short_description": "Generate IoT services dynamically from task descriptions and runtime context.",
  "long_description": "Given a task description and optional baseline context (e.g., existing schemas), generate a complete IoT service implementation at runtime. Submissions are evaluated using CodeBERTScore metrics (Precision, Recall, F1, F3) measuring semantic similarity between the generated service and a reference implementation. Efficiency is also assessed via token consumption.",
  "paper_link": "https://arxiv.org/abs/2502.00689",
  "metrics": [
    { 
      "name": "Precision", 
      "description": "CodeBERTScore precision measuring code correctness" 
    },
    { 
      "name": "Recall", 
      "description": "CodeBERTScore recall measuring code completeness" 
    },
    {
      "name": "F1",
      "description": "CodeBERTScore F1 measuring overall code similarity"
    },
    {
      "name": "F3",
      "description": "CodeBERTScore F3 emphasizing code completeness"
    },
    { 
      "name": "Tokens", 
      "description": "Total token usage for service generation" 
    }
  ],
  "entries": [
    {
      "name": "CodeQwen1.5-7B",
      "precision": 0.86,
      "recall": 0.79,
      "f1": 0.83,
      "f3": 0.80,
      "tokens": 3482.00,
      "date": "2025-02-02",
      "link": "https://github.com/sa4s-serc/SAS_llm_query"
    },
    {
      "name": "DeepSeek-V2.5",
      "precision": 0.91,
      "recall": 0.85,
      "f1": 0.88,
      "f3": 0.86,
      "tokens": 4376.25,
      "date": "2025-02-02",
      "link": "https://github.com/sa4s-serc/SAS_llm_query"
    },
    {
      "name": "GPT-4o-mini",
      "precision": 0.90,
      "recall": 0.85,
      "f1": 0.87,
      "f3": 0.85,
      "tokens": 2063.17,
      "date": "2025-02-02",
      "link": "https://github.com/sa4s-serc/SAS_llm_query"
    }
  ],
  "type": "dynamic",
  "token_limit": "8K",
  "response_time": "30 seconds",
  "example_available": true,
  "dataset_download": true,
  "dataset_link": "https://github.com/sa4s-serc/SAS_llm_query/blob/iot-prototype/dataset.csv"
}