{
  "title": "Architectural Design Records Generation",
  "short_description": "Generate Architecture Decision Records (ADRs) from given decision contexts.",
  "long_description": "Given an architectural decision context, generate the corresponding Architecture Decision Record (ADR) capturing the decision made, its rationale, and consequences. Submissions are evaluated using text similarity metrics (ROUGE, BLEU, METEOR) and semantic similarity (BERTScore) against ground-truth ADRs. Approaches may include zero-shot, few-shot, or fine-tuned generation strategies.",
  "paper_link": "https://arxiv.org/abs/2403.01709",
  "metrics": [
    { 
      "name": "ROUGE-1", 
      "description": "Measures the overlap of unigrams between generated and reference text" 
    },
    { 
      "name": "BLEU", 
      "description": "Evaluates the quality of machine-generated text against reference text" 
    },
    { 
      "name": "METEOR", 
      "description": "Metric for evaluation of text with explicit ordering consideration" 
    },
    { 
      "name": "BERTScore-P", 
      "description": "BERTScore precision: Semantic similarity using BERT embeddings" 
    },
    { 
      "name": "BERTScore-R", 
      "description": "BERTScore recall: Semantic similarity using BERT embeddings" 
    },
    { 
      "name": "BERTScore-F1", 
      "description": "BERTScore F1: Harmonic mean of precision and recall, correlates well with human judgment" 
    }
  ],
  "entries": [
    {
      "name": "GPT-4 (0-shot)", 
      "rouge1": 0.259, 
      "bleu": 0.028, 
      "meteor": 0.219, 
      "bertscore_p": 0.847,
      "bertscore_r": 0.851,
      "bertscore_f1": 0.849,
      "date": "2024-03-04",
      "link": "https://github.com/sa4s-serc/ArchAI_ADR"
    },
    {
      "name": "GPT-3.5-davinci-003 (few-shot)", 
      "rouge1": 0.245, 
      "bleu": 0.028, 
      "meteor": 0.207, 
      "bertscore_p": 0.849,
      "bertscore_r": 0.851,
      "bertscore_f1": 0.849,
      "date": "2024-03-04",
      "link": "https://github.com/sa4s-serc/ArchAI_ADR"
    },
    {
      "name": "Flan-T5-base (fine-tuned)", 
      "rouge1": 0.231, 
      "bleu": 0.028, 
      "meteor": 0.171, 
      "bertscore_p": 0.842,
      "bertscore_r": 0.841,
      "bertscore_f1": 0.841,
      "date": "2024-03-04",
      "link": "https://github.com/sa4s-serc/ArchAI_ADR"
    },
    {
      "name": "T0-3b (0-shot)", 
      "rouge1": 0.187, 
      "bleu": 0.005, 
      "meteor": 0.122, 
      "bertscore_p": 0.856,
      "bertscore_r": 0.823,
      "bertscore_f1": 0.839,
      "date": "2024-03-04",
      "link": "https://github.com/sa4s-serc/ArchAI_ADR"
    }
  ],
  "type": "adr",
  "dataset_size": "1000",
  "time_limit": "2 hours",
  "example_available": true,
  "dataset_download": true,
  "dataset_link": "https://github.com/sa4s-serc/ArchAI_ADR/tree/main/data"

}