{
  "title": "Architectural Component Generation",
  "short_description": "Generate complete serverless functions (FaaS) from specifications and codebase context.",
  "long_description": "Given a serverless function specification, codebase summary, and function description, generate a complete Functions-as-a-Service (FaaS) implementation. The dataset spans repositories in JavaScript, TypeScript, and Python from the Wonderless dataset. Submissions are evaluated in two settings: without human intervention (testing generated code directly) and with intervention (after human review and modification). Metrics include codebase-level and function-level test pass rates, code quality measures (SLOC, cyclomatic/cognitive complexity, Halstead volume), and CodeBLEU similarity to the original implementation.",
  "paper_link": "https://arxiv.org/abs/2502.02539",
  "metrics": [
    { 
      "name": "Codebase Tests",
      "description": "Percentage of test cases passed for the entire codebase" 
    },
    { 
      "name": "Function Tests",
      "description": "Percentage of test cases passed for individual serverless functions" 
    },
    {
      "name": "Source Lines of Code",
      "description": "The number of lines with source code in the serverless function"
    },
    {
      "name": "Halstead's Volume",
      "description": "A measure of the size of the serverless function as a collection of operators and operands"
    },
    {
      "name": "Cyclomatic Complexity",
      "description": "A measure of the complexity of the serverless functions control flow"
    },
    {
      "name": "Cognitive Complexity",
      "description": "A measure of the understandability of the serverless function"
    },
    {
      "name": "CodeBLEU",
      "description": "A measure of similarity between the original and generated serverless functions"
    }
  ],
  "entries": [
    {
      "name": "DeepSeek-Coder-V2", 
      "no_intervention": {
        "codebase_tests": 81,
        "function_tests": 13,
        "source_lines_of_code": 37.03,
        "cyclomatic_complexity": 7.72,
        "cognitive_complexity": 5.45,
        "halstead_volume": 1096.21,
        "codebleu": 0.37
      },
      "with_intervention": {
        "codebase_tests": 90,
        "function_tests": 71
      },
      "date": "2024-02-04",
      "link": "https://github.com/Meghanatedla/LLM-ComponentGen"
    },
    {
      "name": "GPT-4",
      "no_intervention": {
        "codebase_tests": 79,
        "function_tests": 10,
        "source_lines_of_code": 36.14,
        "cyclomatic_complexity": 7.54,
        "cognitive_complexity": 6.48,
        "halstead_volume": 1112.95,
        "codebleu": 0.35
      },
      "with_intervention": {
        "codebase_tests": 90,
        "function_tests": 50
      },
      "date": "2024-02-04",
      "link": "https://github.com/Meghanatedla/LLM-ComponentGen"
    },
    {
      "name": "GPT-3.5-Turbo",
      "no_intervention": {
        "codebase_tests": 68,
        "function_tests": 4,
        "source_lines_of_code": 28.52,
        "cyclomatic_complexity": 6.75,
        "cognitive_complexity": 4.66,
        "halstead_volume": 852.64,
        "codebleu": 0.32
      },
      "with_intervention": {
        "codebase_tests": 89,
        "function_tests": 64
      },
      "date": "2024-02-04",
      "link": "https://github.com/Meghanatedla/LLM-ComponentGen"
    },
    {
      "name": "CodeQwen1.5-7B-Chat",
      "no_intervention": {
        "codebase_tests": 69,
        "function_tests": 7,
        "source_lines_of_code": 36.28,
        "cyclomatic_complexity": 8.19,
        "cognitive_complexity": 6.31,
        "halstead_volume": 1042.10,
        "codebleu": 0.30
      },
      "with_intervention": {
        "codebase_tests": 80,
        "function_tests": 33
      },
      "date": "2024-02-04",
      "link": "https://github.com/Meghanatedla/LLM-ComponentGen"
    },
    {
      "name": "Artigenz-Coder-DS-6.7B",
      "no_intervention": {
        "codebase_tests": 67,
        "function_tests": 0,
        "source_lines_of_code": 31.69,
        "cyclomatic_complexity": 7.10,
        "cognitive_complexity": 5.86,
        "halstead_volume": 899.89,
        "codebleu": 0.26
      },
      "with_intervention": {
        "codebase_tests": 74,
        "function_tests": 0
      },
      "date": "2024-02-04",
      "link": "https://github.com/Meghanatedla/LLM-ComponentGen"
    }
  ],
  "type": "serverless",
  "test_cases": {
    "description": "Comprehensive test suite covering both function-level and codebase-level functionality, code quality metrics and CodeBLEU score between original and generated serverless function"
  },
  "intervention_details": "Human review and modification of generated code",
  "example_available": true,
  "dataset_download": true,
  "dataset_link": "https://github.com/Meghanatedla/LLM-ComponentGen/tree/main/data"
}