{ "disclosures": { "disclaimer": "Structured dataset metadata used for pre- and post-training IBM Granite 4 base and instruct models is being released as part of IBM's effort to provide the community with highly transparent open source model development. In this spirit, we are releasing metadata related to datasets directly ingested during training and metadata for datasets used to source seed data for IBM's synthetic data pipelines. This structured disclosure is provided in machine readable .JSON format as a governance artifact to inform risk-based controls for downstream model use. For more information on IBM's compliance with AI laws and regulations, go to: https://www.ibm.com/granite and https://www.ibm.com/granite/docs/resources/responsible-use-guide.pdf.", "timestamp": "2026-04-22T14:12:43Z", "model": { "name": "Granite-4.1-30b-base", "developer": "IBM", "family": "IBM Granite", "version": "4.1", "release_date": "04/29/2026", "parameter_count": 28865728512, "approx_parameter_count": 30000000000, "intended_use": "Designed to follow general instructions and can serve as foundation for AI assistants across diverse domains, including business applications, as well as for LLM agents equipped with tool-use capabilities.", "architecture": { "type": "Dense", "transformer_class": "GraniteForCausalLM", "layers": 64, "attention_heads": 32, "attention_head_dim": 128, "KV_heads": 8, "positional_embeddings": { "type": "RoPE" } }, "license": { "type": "Apache License 2.0", "url": "https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/apache-2.0.md" } }, "pre-training": { "stage": "initial training phase", "methods": [ { "category": "web", "datasets": [ { "name": "GneissWeb", "url": "https://huggingface.co/datasets/ibm-granite/GneissWeb", "date_obtained": "02/10/2025", "description": "GneissWeb is a large-scale dataset for LLM pretraining, consisting of around 10 trillion tokens. (Gohari, et al., 2025)", "size_MB": 30200000.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Web crawling", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Webhose", "url": "", "date_obtained": "05/04/2023", "description": "Unstructured web content in English converted into machine-readable data feeds acquired by IBM.", "size_MB": 170368.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Private non-publicly available datasets obtained from third-parties", "user_data": "no", "synthetic": "no", "seed_data": "no" } ] }, { "category": "code", "datasets": [ { "name": "Code Pile", "url": "", "date_obtained": "12/03/2024", "description": "Our code pile is sourced from a combination of publicly available datasets like Github Code Clean, StarCoderdata, and additional public code repositiories and issues from Github. We filter raw data to retain a list of 116 programming languages and only keep files with permissive licenses for model training.", "size_MB": 95559.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Granite Feedback", "url": "", "date_obtained": "06/01/2025", "description": "Dataset acquired based on feedback from the internal Granite feedback team. It includes: IBM enterprise, ITops, JCL data, ESG data, Angular data, ESQL data.", "size_MB": 1408851.0, "languages": "en, es, fr, uk, da, de, rw, pl, ja, la, eu, , un, bg, cs, crs, co, ceb, ca, bs, bi, ar, aa, zh, af, vo, zzp, cy, ro, nl, gn, eo, et, gl, hr, fa, fi, fy, gd, id, it, jw, iw, haw, ia, ie, fo, is, ln, hu, ht, ha, gv, ga, ko, lg, mfe, ms, na, mg, lb, ak, mt, no, oc, az, kha, nn, pt, ru, sco, sk, sa, rn, rm, be, ny, sv, ta, tk, ss, sr, sn, sl, tl, tlh, tn, tr, to, uz, war, br, zh-Hant, vi, xh, wo, bn, el, sm, ik, hi, ig, gu, so, kk, kl, ku, lt, ky, kn, mk, mn, mr, lv, ka, ne, ml, om, th, pa, or, qu, sq, tt, tg, sw, te, ug, ur, mi, yi, ba, sd, my, km, ps, su, xx-Bopo, yo, hy, xx-Tavt, fj, zu, ve, xx-Copt, ab, ts, za, hmn, am, lo, si,", "type": "text", "source": "third party", "data_provenance": "Web crawling", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "opc-annealing-corpus", "url": "https://huggingface.co/datasets/OpenCoder-LLM/opc-annealing-corpus", "date_obtained": "04/15/2025", "description": "Synthetic data and algorithmic corpus used for opencoder annealing.", "size_MB": 7083.0, "languages": "en, ru, es, zh, bg, it, ko, ja, uk, ar, de, hr, sh, ceb, vi, fr, ro, sq, sr, pt, no, tr, el, fa, gl, la, da, hu, cs, pl, kk, mk, pms, hi, be, ca, id, nl, ka, th, sv, km, he, ur, az, war, sl, si, fi, gu, mn, lt, ilo, wuu, jv, oc, nn, or, et, tt, mr, eo, af, dv, kn, ml, ia, tl, jbo, ta, sk, te, ms, kv, ku, cv, bn, eu, is, uz, hy, ne, lv, pa, sa, cy, bs, ky, tg, lo, ba, my, br, gn, sah,", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" } ] }, { "category": "domain", "datasets": [ { "name": "Free Law", "url": "https://huggingface.co/datasets/LLM360/TxT360/tree/main/data/freelaw", "date_obtained": "04/15/2025", "description": "Public-domain legal opinions from the US federal and state courts", "size_MB": 7673.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "USPTO", "url": "https://huggingface.co/datasets/LLM360/TxT360/tree/main/data/uspto", "date_obtained": "04/15/2025", "description": "Collection of US patents granted from 1975 to May 2023, excluding design patents", "size_MB": 7861.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "EDGAR Filings", "url": "", "date_obtained": "10/22/2024", "description": "Annual reports from all the publicly traded companies in the US spanning a period of more than 25 years", "size_MB": 10753.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "SEC Filings", "url": "", "date_obtained": "10/22/2024", "description": "10-K/Q filings from the US Securities and Exchange Commision (SEC) for the years 1934-2022.", "size_MB": 51278.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "FDIC", "url": "", "date_obtained": "10/22/2024", "description": "The data is from the annual submissions of the FDIC", "size_MB": 57.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Earnings Call Transcripts", "url": "", "date_obtained": "10/22/2024", "description": "Transcripts from the quarterly earnings call transcripts, the related stock prices, and the sector index.", "size_MB": 90.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Cybersecurity", "url": "", "date_obtained": "10/22/2024", "description": "Compendium of data crawled from different Web sources about cybersecurity-related topics", "size_MB": 21245.0, "languages": "en, ar, be, bg, ne, de, es, hy, co, bn, yi, sq, id, cy, uk, un, sw, mn, gu, my, ky, eu, is, vi, ur, th, ru, sk, sr, oc, la, ml, jw, hr, gl, fi, et, it, as, cs, dv, el, bs, hu, nl, bo, ca, da, lb, yo, pa, ug, ps, sv, lv, km, fy, pt, no, lt, kn, fr, iw, hi, mg, ka, ko, ja, mr, ro, eo, ta, lo, pl, mt, te, zh, ha, so, mk, ht, ku, su, si, tr, nn, sco, st, ms, af, zh-Hant, ia, sl, sd, uz, tl, fa, ga, gd, az,", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "IBM Documentation", "url": "", "description": "IBM redbooks and product documents, as well as publicly available documentation from Fortune 500 companies.", "size_MB": 892.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Web crawling", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "ibm.com data", "url": "", "date_obtained": "08/19/2024", "description": "Crawled data from ibm.com websites", "size_MB": 22153.0, "languages": "lt, zh, en, ru, no, pt, sl, zzp, fi, un, su, br, syr, fy, tr, haw, hr, ak, ceb, cy, nl, xx-Yiii, wo, sn, ja, lif, de, ia, crs, uk, ie, sk, ca, eu, zh-Hant, gn, gl, rw, yo, rn, es, mfe, cs, fo, gd, lg, sm, xx-Brah, el, hy, kha, ko, la, sa, so, zu, eo, om, st, tlh, ga, pl, it, ln, ms, sco, th, aa, kl, xh, sv, sr, co, war, da, bs, mi, vo, bg, id, ml, sw, uz, is, mt, ny, nn, xx-Xsux, iw, ar, ik, xx-Vaii, hu, af, rm, xx-Egyp, mn, tg, vi, fr, ht, ve, to, jw, sg, tk, az, be, bi, lb, ka, qu, sq, hmn, ay, gv, kk, ro, fj, mg, tl, xx-Phag, za, et, lv, oc, ts, xx-Nkoo, xx-Copt, tt, xx-Plrd, xx-Qaai, dv, mk, na, ss, tn,", "type": "text", "source": "IBM", "data_provenance": "Web crawling", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Top US Companies", "url": "", "date_obtained": "09/02/2024", "description": "Crawled internet data based on a list of top US companies", "size_MB": 1594201.0, "languages": "en, un, da, sk, et, de, es, tr, sv, it, cs, nl, fr, sl, no, pt, pl, ar, ja, ru, zh, wo, la, ga, ms, ko, sw, zh-Hant, el, sm, tn, crs, hr, id, af, xh, sco, ro, kk, oc, hi, uk, iw, fa, ml, kn, bn, ie, nn, lt, eu, zzp, ca, az, ak, mfe, ia, gl, rm, fi, fy, zu, tt, my, tl, vi, co, gd, sa, th, mr, km, sr, bh, mk, sq, gu, ta, lo, aa, hu, is, hy, lv, rn, am, ka, br, as, ne, vo, rw, bg, to, pa, te, hmn, ht, ur, ti, so, dz, or, bi, kha, syr, be, gn, uz, su, mg, xx-Copt, cy, yo, ss, xx-Runr, war, ps, mt, mn, ky, tg, ik, tk, bs, gv, st, fj, xx-Qaai, ku, ceb, tlh, qu, eo, haw, lg, ln, jw, iu, za, yi, si, mi, xx-Samr, fo, na, ig, om,", "type": "text", "source": "third party", "data_provenance": "Web crawling", "user_data": "no", "synthetic": "no", "seed_data": "no" } ] }, { "category": "academic", "datasets": [ { "name": "Txt360 Papers", "url": "https://huggingface.co/datasets/LLM360/TxT360", "date_obtained": "07/01/2025", "description": "Scientific and scholarly documents incorporated as one of the curated sources within TxT360, providing domain‑rich academic text for language‑model pretraining. These papers undergo rigorous extraction, filtering, and global deduplication to ensure coherent, information‑dense content suitable for advanced reasoning and knowledge‑intensive tasks.", "size_MB": 161446.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Financial Research Papers", "url": "", "date_obtained": "10/22/2024", "description": "Publicly available financial research paper corpus, curated and filtered by IBM.", "size_MB": 82.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Web crawling", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Papers With Code", "url": "", "date_obtained": "10/22/2024", "description": "Selection of publicly available academic papers sourced from the papers with code website.", "size_MB": 14389.0, "languages": "ur, hi, de, af, ja, bg, un, en, hu, tl, ca, fi, th, fr, pl, ka, fa, vi, uk, es, ga, pt, it, hr, ru, el, ko, ro, et, sv, no, zh, zh-Hant, nl, id, bs, tr, None, ar, cs, sq, eo", "type": "text", "source": "third party", "data_provenance": "Web crawling", "user_data": "no", "synthetic": "no", "seed_data": "no" } ] }, { "category": "technical", "datasets": [ { "name": "Wikipedia - Txt360", "url": "https://huggingface.co/datasets/LLM360/TxT360/tree/main/data/wikipedia", "date_obtained": "04/15/2025", "description": "Wikimedia dataset filtered for high quality content articles", "size_MB": 75666.0, "languages": "af, pl, cv, la, ti, de, sd, hif, kv, ik, os, ce, lmo, kw, bjn, shi, map-bms, ban, wa, gpe, sat, ca, lg, sg, sn, ami, kcg, mdf, fur, kg, pcm, ace, ja, ilo, iu, dz, nl, se, dsb, bo, awa, gan, gu, sah, is, min, bar, ht, koi, pdc, lad, nov, sc, ur, kn, sk, inh, trv, arz, tl, tum, jam, kbd, ml, mwl, mi, pwn, fa, bg, eo, war, cbk-zam, li, guw, nrm, mk, ig, wo, vi, gur, ar, smn, or, fo, av, gom, as, fy, su, nso, zh-min-nan, za, fat, so, vls, wuu, pfl, be-x-old, dag, lld, tet, gcr, ltg, dv, pa, ady, nap, pi, lb, szy, lez, xmf, te, yo, bs, sr, kbp, sq, tt, io, fj, bcl, frp, hy, ug, zea, ms, crh, uk, he, ia, be, nqo, pam, mai, zh, anp, zh-yue, srn, sl, oc, mr, haw, xh, bh, szl, ab, ga, simple, hi, sa, gl, nds, nia, mzn, hsb, ky, nn, ko, tcy, gor, azb, frr, fi, myv, zu, tr, cs, sm, mnw, jbo, tn, pap, bm, ay, pnb, lij, kk, krc, ckb, rw, csb, als, sv, new, eml, got, sco, vec, ln, pnt, pih, olo, rn, stq, sh, sw, shn, th, bpy, es, mrj, ty, to, atj, it, dty, pt, hak, ne, pag, tk, ss, my, rue, ku, din, xal, an, kaa, ast, ha, ta, fon, vep, ny, ve, el, tpi, pms, ks, arc, et, st, gv, br, cu, ceb, km, roa-tara, bn, jv, lt, gag, tg, fiu-vro, nah, blk, ext, mhr, cr, rmy, vo, ba, lbe, tay, nds-nl, lo, uz, skr, glk, ka, chy, id, qu, mg, nv, avk, bat-smg, az, en, hr, mni, ki, zh-classical, pcd, hyw, tyv, no, mad, co, diq, fr, gd, chr, alt, ee, lfn, lv, ksh, mt, am, ch, roa-rup, bi, ps, yi, bxr, ang, gn, scn, ie, cy, si, bug, hu, tly, da, kl, tw, udm, ru, mn, ro, cdo, ary, ff, eu, guc, ts, kab, om, rm", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Finance Textbooks", "url": "", "date_obtained": "10/22/2024", "description": "Finance textbooks obtained by crawling https://open.umn.edu/opentextbooks/subjects/finance", "size_MB": 5.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Web crawling", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Deepmind PG-19", "url": "", "date_obtained": "10/22/2024", "description": "Books crawled from Deepmind PG-19", "size_MB": 1611.0, "languages": "it, no, da, el, ru, es, uk, ar, la, fr, ja, nn, pt, sv, un, de, iw, ie, cs, nl, ga, pl, rw, sl, xx-Copt, en, None, tr, bg, sr,", "type": "text", "source": "third party", "data_provenance": "Web crawling", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "LOC-PD-Books", "url": "https://huggingface.co/datasets/storytracer/LoC-PD-Books", "date_obtained": "03/18/2024", "description": "This dataset contains more than 140,000 English books (~ 8 billion words) digitised by the Library of Congress (LoC) that are in the public domain in the United States.", "size_MB": 15673.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Cosmopedia V2", "url": "https://huggingface.co/datasets/HuggingFaceTB/smollm-corpus", "date_obtained": "03/25/2024", "description": "An enhanced version of Cosmopedia, the largest synthetic dataset for pre-training, consisting of over 39 million textbooks, blog posts, and stories generated by Mixtral-8x7B-Instruct-v0.1", "size_MB": 67600.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" } ] }, { "category": "math", "datasets": [ { "name": "DM Math", "url": "https://huggingface.co/datasets/LLM360/TxT360/tree/main/data/dm_maths", "date_obtained": "07/01/2025", "description": "DeepMind Math dataset with generated questions from various topics like algebra, calculus, geometry, etc. and converted to jsonl format.", "size_MB": 173629.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Stack-Exchange - Txt360", "url": "https://huggingface.co/datasets/LLM360/TxT360/tree/main/data/stackexchange", "date_obtained": "04/15/2025", "description": "One of the largest publicly available repositories for question-answer pairs, this dataset is a network of question-and-answer websites on various subjects including programming, science, mathematics, and more.", "size_MB": 35857.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "open-web-math", "url": "https://huggingface.co/datasets/open-web-math/open-web-math", "date_obtained": "03/18/2024", "description": "Contains the majority of the high-quality, mathematical text from the internet.", "size_MB": 17327.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Algebraic-Stack", "url": "https://huggingface.co/datasets/EleutherAI/proof-pile-2", "date_obtained": "03/04/2024", "description": "A new dataset of mathematical code composed of 11B tokens that includes numerical computing, computer algebra, and formal mathematics. This dataset is part of the Proof-Pile-2 collection (Paster et al., 2023).", "size_MB": 7250.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "opc-fineweb-math-corpus", "url": "https://huggingface.co/datasets/OpenCoder-LLM/opc-fineweb-math-corpus", "date_obtained": "12/03/2024", "description": "This math-related data from Fineweb was specifically used in OpenCoder pre-training. We employ fastText in three iterative rounds to recall a final dataset of 55B code and math-related data.", "size_MB": 6466.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "FineMath", "url": "https://huggingface.co/datasets/HuggingFaceTB/finemath", "date_obtained": "12/23/2024", "description": "FineMath consists of 34B tokens (FineMath-3+) and 54B tokens (FineMath-3+ with InfiMM-WebMath-3+) of mathematical educational content filtered from CommonCrawl.", "size_MB": 95494.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "MegaMath", "url": "https://huggingface.co/datasets/LLM360/MegaMath", "date_obtained": "04/01/2025", "description": "MegaMath is an open math pretraining dataset curated from diverse, math-focused sources, with over 300B tokens.", "size_MB": 281100.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Nemotron-MIND", "url": "https://huggingface.co/datasets/nvidia/Nemotron-MIND", "date_obtained": "06/15/2025", "description": "A compilation of pretraining data that supports improvements of math reasoning capabilities of the Nemotron5 series of models. Nemotron-MIND contains over 138 billion tokens of structured mathematical dialogues generated by Nemotron4-340B-Instruct.", "size_MB": 264.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" } ] }, { "category": "multilingual", "datasets": [ { "name": "MADLAD-400", "url": "https://huggingface.co/datasets/allenai/MADLAD-400", "date_obtained": "02/25/2025", "description": "MADLAD-400 (Multilingual Audited Dataset: Low-resource And Document-level) is a document-level multilingual dataset based on Common Crawl, covering 419 languages in total. This uses all snapshots of CommonCrawl available as of August 1, 2022. The primary advantage of this dataset over similar datasets is that it is more multilingual (419 languages), it is audited and more highly filtered, and it is document-level.", "size_MB": 844171.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "CulturaX", "url": "https://huggingface.co/datasets/uonlp/CulturaX", "date_obtained": "06/01/2025", "description": "A multilingual dataset with 6.3 trillion tokens in 167 languages, tailored for LLM development.", "size_MB": 900000.0, "languages": "bn, gu, hi, kn, ml, mr, or, pa, ta, te", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Fineweb2-edu-multilingual", "url": "", "description": "Internal multilingual dataset, based upon Fineweb2 dataset, filtered for highest quality documents.", "size_MB": 6500000.0, "languages": "en, ja, de, fr, it, ko, ar, zh", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Open-Platypus Translated", "url": "", "date_obtained": "12/18/2023", "description": "Translated Open Platypus dataset for improved multilingual support.", "size_MB": 49.0, "languages": "gu,ml,or,ta,kn,hi,mr,bn,te,pa", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Daring-Anteater Translated", "url": "", "date_obtained": "07/25/2024", "description": "Translated Daring Anteater dataset for improved multilingual support.", "size_MB": 1500.0, "languages": "gu,ml,or,ta,kn,hi,mr,bn,te,pa", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "EvolKit-20 Translated", "url": "", "date_obtained": "09/19/2024", "description": "Translated Evolkit-20k dataset for improved multilingual support.", "size_MB": 178.0, "languages": "gu,ml,or,ta,kn,hi,mr,bn,te,pa", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "HelpSteer2 Translated", "url": "", "date_obtained": "07/25/2024", "description": "Translated Helpsteeer2 dataset for improved multilingual support.", "size_MB": 12.0, "languages": "gu,ml,or,ta,kn,hi,mr,bn,te,pa", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Sangraha", "url": "https://huggingface.co/datasets/ai4bharat/sangraha", "date_obtained": "06/01/2025", "description": "Translated Sangraha dataset for improved multilingual support.", "size_MB": 450000.0, "languages": "bn, gu, hi, kn, ml, mr, or, pa, ta, te", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" } ] }, { "category": "instructions", "datasets": [ { "name": "DOLMino dataset mix - Flan partition", "url": "https://huggingface.co/datasets/allenai/dolmino-mix-1124", "date_obtained": "07/01/2025", "description": "Flan partition of the allenai dolmino mix 1124 dataset.", "size_MB": 30737.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" } ] }, { "category": "General English", "datasets": [ { "name": "Hermes Function-Calling V1", "url": "https://huggingface.co/datasets/NousResearch/hermes-function-calling-v1", "date_obtained": "09/02/2024", "description": "A structured output dataset of multi-turn function-calling conversations designed to train models to interpret natural language and generate correct function invocations with JSON-style responses. It includes diverse scenarios where assistants must decide which functions to call and format their arguments properly, making it useful for improving real dialogue-level tool-calling behavior.", "size_MB": 10.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" } ] }, { "category": "reasoning", "datasets": [ { "name": "CoT-Python Synthetic", "url": "", "date_obtained": "07/01/2025", "description": "Synthetic Python code reasoning SFT dataset with Chain of Thought reasoning, which is aimed to improve model's reasoning performance for Input-Ouput prediction task for Python code.", "size_MB": 51.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Temporal Reasoning Synthetic", "url": "", "date_obtained": "08/01/2025", "description": "A dataset of question-answer pairs designed to improve the temporal reasoning capabilities of large language models. The dataset specifically targets known weaknesses in understanding time-related concepts, such as historical era transitions (BC/AD), cyclical time calculations (day-of-week), and other complex temporal arithmetic problems such as duration and schedule. The data includes a step-by-step \"chain of thought\" reasoning for each problem, leading to a final answer.", "size_MB": 7.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic MagPie Logical Reasoning", "url": "", "date_obtained": "08/15/2025", "description": "Synthetic dataset of puzzles and solutions created by folllowing MagPie self generation technique", "size_MB": 167.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" } ] }, { "category": "pre-training", "datasets": [ { "name": "SAP Knowledge Base", "url": "", "date_obtained": "12/07/2023", "description": "Crawled data from sap.com knowledge support websites", "size_MB": 16100.0, "languages": "en,zh,es,ja,fr,de,pt,zh-Hant,un,ru,nl,hu,it,da,ko,fi,ro,sl,cs,pl,bg,sv,th,sk,no,ms,tr,sr,hr,ar,iw,el,lb,cy,id,gl,bs,rw,hi,vi,ca,la,et,kk,nn,ga,wo,crs,uk,xh,oc,war,co,so,fy,ss,", "type": "text", "source": "third party", "data_provenance": "Web crawling", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Nemotron-CC", "url": "https://data.commoncrawl.org/contrib/Nemotron/Nemotron-CC/index.html", "date_obtained": "01/09/2025", "description": "A 6.3T token dataset based on Common Crawl, consisting of 4.4T globally deduplicated original tokens and 1.9T synthetically generated tokens.", "size_MB": 11687104.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Concert Workflow Data", "url": "", "date_obtained": "04/01/2025", "description": "Data generated by, processed through, or used within the IBM Concert Workflows platform", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Concert Actions Data", "url": "", "date_obtained": "03/01/2025", "description": "Structured information associated with actions generated, recommended, or executed within the IBM Concert platform.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "OPA Data_Public", "url": "https://github.com/open-policy-agent/opa/tree/main", "date_obtained": "03/01/2025", "description": "Structured JSON data that provides external context or facts used by Open Policy Agent (OPA) policies.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "OPA Data_Docs", "url": "https://www.openpolicyagent.org/docs", "date_obtained": "03/01/2025", "description": "Core concepts in OPA's policy language (Rego).", "size_MB": 2.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Spider 1.0", "url": "https://yale-lily.github.io/spider", "date_obtained": "07/07/2023", "description": "Large-scale complex and cross-domain semantic parsing and text-to-SQL dataset.", "size_MB": 196.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "Kubectl data", "url": "", "date_obtained": "05/07/2025", "description": "Kubectl data relevant to SRE Agent.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "OpenMathReasoning", "url": "https://huggingface.co/datasets/nvidia/OpenMathReasoning", "date_obtained": "06/01/2025", "description": "Large-scale math reasoning dataset for training large language models (LLMs)", "size_MB": 20000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "NuminaMath-CoT", "url": "https://huggingface.co/datasets/AI-MO/NuminaMath-CoT", "date_obtained": "12/03/2024", "description": "Approximately 860k math problems, where each solution is formatted in a Chain of Thought (CoT) manner.", "size_MB": 800.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "data ablation_full59K", "url": "https://huggingface.co/datasets/simplescaling/data_ablation_full59K", "date_obtained": "03/05/2025", "description": "Approximately 59k high-quality mathematical problems and solutions designed for training reasoning-based AI models. Features detailed thinking trajectories and chain-of-thought (CoT) responses sourced from various competitive math benchmarks.", "size_MB": 5800.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "Llama-Nemotron Post Training", "url": "https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset", "date_obtained": "06/01/2025", "description": "Compilation of SFT and RL data that supports improvements of math, code, general reasoning, and instruction following capabilities of the original Llama instruct model, in support of NVIDIA’s release of Llama-3.1-Nemotron-Ultra-253B-v1, Llama-3.3-Nemotron-Super-49B-v1 and Llama-3.1-Nemotron-Nano-8B-v1.", "size_MB": 125000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "DeepScaleR Preview Dataset", "url": "https://huggingface.co/datasets/agentica-org/DeepScaleR-Preview-Dataset", "date_obtained": "06/01/2025", "description": "Approximately 40,000 unique mathematics problem-answer pairs compiled from: AIME, AMC, Omni-MATH dataset, and Still dataset.", "size_MB": 7.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "HARDMath", "url": "https://github.com/sarahmart/HARDMath", "date_obtained": "06/01/2025", "description": "A dataset of challenging, graduate-level problems in applied mathematics that can be used for language model evaluation. Unlike other popular mathematical datasets, HARDMath contains problems that require a combination of advanced problem-solving skills, approximation methods, and mathematical intuition. The dataset contains a large test set of 1,050 problems and a mini test set of 437 problems, divided across seven different problem types.", "size_MB": 5.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "Ncert Dataset", "url": "https://huggingface.co/datasets/KadamParth/Ncert_dataset", "date_obtained": "06/01/2025", "description": "A collection of educational content derived from NCERT textbooks for students in standards 6 to 12. NCERT books are textbooks published by the National Council of Educational Research and Training (NCERT), an Indian organization focused on improving school education.", "size_MB": 100.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "open-thoughts/OpenThoughts3-1.2M", "url": "https://huggingface.co/datasets/open-thoughts/OpenThoughts3-1.2M/tree/main", "date_obtained": "07/01/2025", "description": "The third iteration the our line of OpenThoughts datasets, building on previous OpenThoughts-114k and OpenThoughts2-1M. The dataset consists of 850,000 math questions, 250,000 code questions, and 100,000 science questions.", "size_MB": 10000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "open-r1/codeforces - questions only: HuggingFace Download", "url": "https://huggingface.co/datasets/open-r1/codeforces", "date_obtained": "06/01/2025", "description": "CodeForces is one of the most popular websites among competitive programmers, hosting regular contests where participants must solve challenging algorithmic optimization problems. The challenging nature of these problems makes them an interesting dataset to improve and test models’ code reasoning capabilities. This dataset includes more than 10k unique problems covering the very first contests all the way to 2025. Additionally, it includes generated and extensively validated custom checker code for problems with multiple solutions, and generated additional challenging test cases.", "size_MB": 2600.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "openCodeReasoning", "url": "https://huggingface.co/datasets/nvidia/OpenCodeReasoning", "date_obtained": "06/01/2025", "description": "Open Code Reasoning is the largest reasoning-based synthetic dataset to date for coding, comprising of 736,712 samples in Python across 28,904 unique competitive programming questions.", "size_MB": 3000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "/OpenCoder-LLM/opc-sft-stage2/viewer/educational_instruct", "url": "https://huggingface.co/datasets/OpenCoder-LLM/opc-sft-stage2/viewer/educational_instruct", "date_obtained": "12/03/2024", "description": "Contains generated (instruction, code, test case) triples, validated through a Python compiler. Notably, the inclusion of test cases provides a valuable signal for code RL.", "size_MB": 53.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "OpenCoder-LLM/opc-sft-stage2/viewer/package_instruct", "url": "https://huggingface.co/datasets/OpenCoder-LLM/opc-sft-stage2/viewer/package_instruct", "date_obtained": "12/03/2024", "description": "Used extracted common interface documentation from pydoc as a seed to generate Python package-related questions.", "size_MB": 286.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "Llama-Nemotron-Post-Training-Dataset", "url": "https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset", "date_obtained": "06/01/2025", "description": "A compilation of SFT and RL data that supports improvements of math, code, general reasoning, and instruction following capabilities of the original Llama instruct model, in support of NVIDIA’s release of Llama-3.1-Nemotron-Ultra-253B-v1, Llama-3.3-Nemotron-Super-49B-v1 and Llama-3.1-Nemotron-Nano-8B-v1.", "size_MB": 145000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "Multi-turn sequential slot filling and tool usage", "url": "", "date_obtained": "06/01/2023", "description": "A multi-step dataset derived from the Schema-Guided Dialogue (SGD) data that focuses on sequential slot filling and tool usage across multiple turns. It requires models to extract slots from user input, call the appropriate tools in order, and use tool responses to complete downstream actions correctly.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "GovReport-QS", "url": "https://gov-report-data.github.io/", "date_obtained": "05/01/2025", "description": "Multi turn conversations extracted from the GovReport datasets, specifically forcusing on the conversations on long documents.", "size_MB": 13.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "hotpot_qa", "url": "https://huggingface.co/datasets/hotpotqa/hotpot_qa", "date_obtained": "03/11/2024", "description": "Singe turn conversations extracted from the hotpotQA, specifically forcusing on the conversations on long documents and requiring reasoning.", "size_MB": 480.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "NarrativeQA", "url": "https://huggingface.co/datasets/meithnav/narrativeqa", "date_obtained": "05/01/2025", "description": "Singe turn conversations extracted from the NarrativeQA, specifically focusing on the conversations on long documents.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "TechQA", "url": "https://github.com/ibm/techqa", "date_obtained": "06/30/2020", "description": "Singe turn conversations sampled from the TechQA, specifically focusing on the conversations on techincal documents.", "size_MB": 4.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "FinQA", "url": "https://huggingface.co/datasets/ibm-research/finqa", "date_obtained": "05/15/2025", "description": "FinQA is a large-scale dataset with 2.8k financial reports for 8k Q&A pairs to study numerical reasoning with structured and unstructured evidence.", "size_MB": 3.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Indic Vault", "url": "https://huggingface.co/datasets/maya-research/IndicVault", "date_obtained": "07/01/2025", "description": "Instruction-tuned dataset that contains question-answer pairs in contemporary, everyday Indian languages", "size_MB": 60.0, "languages": "hi, te, en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "leetcode-solutions-en-2k", "url": "https://huggingface.co/datasets/TigerResearch/tigerbot-kaggle-leetcodesolutions-en-2k", "date_obtained": "07/22/2025", "description": "Dataset contains English descriptions and solutions for approximately 2,000 LeetCode problems.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "DCLM-baseline", "url": "https://huggingface.co/datasets/mlfoundations/dclm-baseline-1.0", "date_obtained": "08/13/2024", "description": "A 4T token / 3B document pretraining dataset that achieves strong performance on language model benchmarks.", "size_MB": 11400000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available dataset", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Synthetic syn-sql-2.5", "url": "", "date_obtained": "07/01/2025", "description": "IBM generated synthetic dataset mimics the manner in which syn-sql-2.5 dataset is created, but using gpt-oss-120b.", "size_MB": 843.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic MagPie Science QA", "url": "", "date_obtained": "08/18/2025", "description": "Dataset generated using MagPie self data generation technique. Using Phi-4 as a teacher model, the model generates both questions and answers to science questions.", "size_MB": 266.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic MagPie API", "url": "", "date_obtained": "08/01/2025", "description": "Synthetic dataset of questions and answers relating to API calling and its specifications, created by folllowing MagPie self generation technique", "size_MB": 275.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "APIGen function calling Synthetic", "url": "", "date_obtained": "08/01/2025", "description": "A function calling dataset using a modification of the APIgen pipeline where only gpt-oss-120b is used.", "size_MB": 41.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Open Policy Agent (OPA) policies Dataset", "url": "", "date_obtained": "03/01/2025", "description": "A synthetic dataset of OPA policies.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "NL2Opt Pyomo", "url": "", "date_obtained": "04/14/2025", "description": "Synthetically generated dataset containing natural language descriptions of mixed-integer linear optimization problems and formatted demonstrations showcasing how to solve the (synthetic) optimization problems using Pyomo.", "size_MB": 6.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "NL2GraphQL", "url": "", "date_obtained": "04/01/2025", "description": "A subset of relational tables from the WikiSQL and CoSQL datasets with a generated GraphQL schema", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "NL2OSLC", "url": "", "date_obtained": "04/29/2025", "description": "Training dataset for natural language questions to Maximo OSLC APIs translation.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "NL2Kubectl", "url": "", "date_obtained": "05/10/2025", "description": "Synthetic kubectl command samples.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "MuSR Synthetic", "url": "", "date_obtained": "05/15/2025", "description": "IBM generated dataset using the MuSR code shared by the original owners and Phi-4 model.", "size_MB": 26.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "PuzzleQA", "url": "https://huggingface.co/datasets/Jingmiao/PUZZLEQA", "date_obtained": "04/09/2025", "description": "A dataset of 558 NPR Sunday Puzzle games from 2007-2021.", "size_MB": 4.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "ACPBench- open ended questions", "url": "", "date_obtained": "05/01/2025", "description": "Data for training and evaluation of LLMs on reasonings tasks about action, change, and planning.\nThe dataset consists of open-ended questions that test reasoning abilities.", "size_MB": 4.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "ACPBench- boolean and multiple choice", "url": "", "date_obtained": "02/19/2025", "description": "Data for training and evaluation of LLMs on reasonings tasks about action, change, and planning.\nThe dataset consists of boolean and multiple choice questions that test reasoning abilities.", "size_MB": 16.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "MagPie Synthetic", "url": "", "date_obtained": "04/10/2025", "description": "Synthetic single turn instruction dataset created by following MagPie using two teacher language models.", "size_MB": 198.0, "languages": "gu,ml,or,ta,kn,hi,mr,bn,te,pa", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "NuminaMath-CoT Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Problems from NuminaMath-CoT dataset (https://huggingface.co/datasets/AI-MO/NuminaMath-CoT). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 1209.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "data_ablation_full59k Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Problems from data_ablation_full59k dataset (https://huggingface.co/datasets/simplescaling/data_ablation_full59K). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 155.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenMathReasoning Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Problems from OpenMathReasoning dataset (https://huggingface.co/datasets/nvidia/OpenMathReasoning) Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 8691.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Llama-Nemotron-Post-Training Synthetic", "url": "", "date_obtained": "05/18/2025", "description": "Problems from Llama-Nemotron-Post-Training dataset (https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 6860.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenThoughts2-1M Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Problems from OpenThoughts2-1M dataset (https://huggingface.co/datasets/open-thoughts/OpenThoughts2-1M). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 1073.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenThoughts2-1M", "url": "https://huggingface.co/datasets/open-thoughts/OpenThoughts2-1M", "date_obtained": "06/01/2025", "description": "Open synthetic reasoning dataset with 1M high-quality examples covering math, science, code, and puzzles.", "size_MB": 3000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "yes" }, { "name": "DeepScaleR-Preview Synthetic", "url": "", "date_obtained": "05/15/2025", "description": "Problems from DeepScaleR-Preview dataset (https://huggingface.co/datasets/agentica-org/DeepScaleR-Preview-Dataset). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 330.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "HARDMath Synthetic", "url": "", "date_obtained": "05/15/2025", "description": "Problems from HARDMath dataset (https://github.com/sarahmart/HARDMath). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Ncert Synthetic", "url": "", "date_obtained": "05/01/2025", "description": "Problems Ncert dataset (https://huggingface.co/collections/KadamParth/ncert-dataset). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 881.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Codeforces Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Synthetic dataset consisting of ~10K competitive coding problems from Codeforces websites (https://huggingface.co/datasets/open-r1/codeforces) all the way to 2025 with solutions generated using Phi-4 reasoning.", "size_MB": 8000.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenCodeReasoning Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Synthetic dataset consisting of 28,319 unique competitive programming questions sourced from APPS, TACO,CodeContests,Codeforces (https://huggingface.co/datasets/nvidia/OpenCodeReasoning) with responses generated using Phi-4-reasoning.", "size_MB": 8000.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "DeepCoder-PrimeIntellect Synthetic", "url": "", "date_obtained": "05/15/2025", "description": "Synthetic dataset consisting of 16K verified coding problems from PrimeIntellect’s SYNTHETIC-1 (https://huggingface.co/datasets/agentica-org/DeepCoder-Preview-Dataset/viewer/primeintellect) coding challenges with verifiable unit tests with responses generated using Phi-4-reasoning.", "size_MB": 8000.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenCodeInstruct Synthetic", "url": "", "date_obtained": "05/15/2025", "description": "Synthetic dataset consisting of 5M synthetically generated coding problems starting from seed problems from 10 datasets and Python functions from GitHub. (https://huggingface.co/datasets/nvidia/OpenCodeInstruct). Solutions generated using Phi-4-reasoning.", "size_MB": 8000.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Open Code Instruct", "url": "https://huggingface.co/datasets/nvidia/OpenCodeInstruct", "date_obtained": "06/01/2025", "description": "The largest open-access instruction tuning dataset (in python), comprising 5 million diverse samples.", "size_MB": 4200.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "yes" }, { "name": "Llama Nemotron post training Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Synthetic dataset consisting of 9M sythetically generated coding problems sourced from Llama-Nemotron-Post-Training dataset (https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset). Solutions generated using Phi-4-reasoning.", "size_MB": 8000.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Glaive-code-assistant", "url": "https://huggingface.co/datasets/glaiveai/glaive-code-assistant", "date_obtained": "04/04/2024", "description": "A synthetic dataset of real conversational examples formatted for function calling, where AI assistant interactions include explicit function call invocations for structured outputs.", "size_MB": 8.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "IBM Seq SGD Dataset", "url": "https://github.ibm.com/conversational-ai/API_integrated_LLMs/tree/main/data/SeqSGD", "date_obtained": "07/16/2024", "description": "A multi-turn, multi-domain task-oriented dialogue dataset with annotated schemas and turn-level labels for learning structured interactions with services and APIs. The data is filtered using an FC-Reflector–based approach to retain high-quality tool-calling and dialogue transitions.", "size_MB": 12.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Multi-turn slot filling with system instruction", "url": "", "description": "A dataset created from the Schema-Guided Dialogue (SGD) data, focused on slot-filling tasks where models must extract and populate correct slot values from multi-turn dialogue context. It emphasizes accurate slot prediction across turns and services, supporting structured information extraction for task-oriented dialogue systems.", "size_MB": 8.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Slot instruction", "url": "", "description": "A multi-turn tool-calling dataset focused on generating correct tool parameters from dialogue context. It provides slot-level instructions (type, description, and formatting rules like YYYY-MM-DD) to guide accurate argument filling and validation.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "When2Call", "url": "https://huggingface.co/datasets/nvidia/When2Call", "date_obtained": "07/27/2025", "description": "A dataset designed to teach and evaluate models on when to call a tool vs. when to answer directly or ask a follow-up question, improving decision-making for tool invocation. It includes multiple-choice and training splits to help models learn the correct tool-calling behavior.", "size_MB": 3.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Routing dataset for function calling", "url": "", "date_obtained": "08/02/2024", "description": "A routing dataset designed to train models to select the correct tool (or no tool) based on user intent from a set of possible tools. Each tool represents a distinct intent category, emphasizing accurate intent recognition, ambiguity handling, and correct route selection without argument generation.", "size_MB": 7.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "xlam-irrelevance-7.5k", "url": "https://huggingface.co/datasets/MadeAgents/xlam-irrelevance-7.5k", "date_obtained": "07/25/2024", "description": "A dataset of 7,500 examples created by removing the correct functions from tool lists so the model must recognize when available functions are irrelevant to the user’s request, helping improve irrelevance detection in tool-calling models.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "xlam_function_calling_60k - filtered", "url": "xlam_function_calling_60k", "date_obtained": "07/25/2024", "description": "We use xlam-function-calling-60k (https://huggingface.co/datasets/Salesforce/xlam-function-calling-60k), a synthetic dataset of 60,000 examples focused on converting user instructions into correct tool calls with realistic API schemas and structured arguments. We used ToolRM (https://arxiv.org/abs/2509.11963) filter to get best 75% data.", "size_MB": 11.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic data for Instana tools", "url": "", "description": "Synthetic data generated for Instana tools and filtered using FC-Reflector", "size_MB": 2.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic data for Apptio", "url": "", "description": "Synthetic data generated for Apptio tools and filtered using FC-Reflector", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic data - parallel and multiple tool-calling scenarios.", "url": "", "date_obtained": "10/03/2024", "description": "Synthetic data generated using an IBM-developed synthetic data generation pipeline, focused on parallel and multiple tool-calling scenarios. The data is filtered using IBM-developed synthetic filters and a ToolRM-based filter to ensure high quality.", "size_MB": 42.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Long-context-10K", "url": "", "date_obtained": "04/14/2025", "description": "10k subset from the dataset collection, where the tool library is inflated with additional tools to increase the context size, which are also filtered with Mistral-7B-RM.", "size_MB": 35.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "XLAM-robustness", "url": "", "date_obtained": "03/01/2025", "description": "A dataset built on top of xLAM function-calling data to evaluate robustness in tool-calling models under query rephrasing and changes in available tools. The data is generated by rewriting user queries and creating new function-calling candidates using expanded toolkits while filtering equivalent tools to ensure valid and challenging tool-calling scenarios.", "size_MB": 6.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "XLAM-masked", "url": "", "date_obtained": "04/10/2025", "description": "Subset of the XLAM-robustness data where the Function names and the Arguments are masked.", "size_MB": 5.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic single turn RAG dataset", "url": "", "date_obtained": "08/28/2024", "description": "Single turn synthetically generated RAG dataset. The dataset is filtered using the using MagPie filters on input quality, sample quality and difficulty.", "size_MB": 71.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic single turn RAG dataset - Wikipedia and government domains", "url": "", "description": "Single turn synthetically generated RAG dataset. The domains include wikipedia and goverment domains. The dataset is filtered using the using MagPie filters on input quality, sample quality and difficulty.", "size_MB": 47.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic multi-turn RAG dataset - Wikipedia and government domains", "url": "", "date_obtained": "08/28/2024", "description": "Multi turn synthetically generated RAG dataset. The number of turns range from 2-16. The domains include wikipedia and goverment domains. The dataset is filtered using the using MagPie filters on input quality, sample quality and difficulty.", "size_MB": 47.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Acquired multi-turn dataset; government domain", "url": "", "date_obtained": "04/07/2025", "description": "Multiturn conversation data created by the human annotators at Defined.ai, and reviewed for quality from human annotators. The data comes from government domains.", "size_MB": 25.0, "languages": "en", "type": "text", "source": "Third Party", "data_provenance": "Private non-publicly available datasets obtained from third-parties", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenCodeGeneticInstruct - mixtral 8x22b instruct subset", "url": "https://huggingface.co/datasets/nvidia/OpenCodeGeneticInstruct/tree/main/mixtral-8x22b-instruct", "date_obtained": "07/22/2025", "description": "OpenCodeGeneticInstruct comprises more than 15M coding instructions in python which is generated synthetically with the Genetic-Instruct approach. Only the Mixtral-8x22b-instruct subset of this dataset is used.", "size_MB": 16000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenMathInstruct-1", "url": "https://huggingface.co/datasets/nvidia/OpenMathInstruct-1", "date_obtained": "07/25/2024", "description": "A math instruction tuning dataset with 1.8M problem-solution pairs generated using permissively licensed Mixtral-8x7B model.", "size_MB": 2100.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Hermes 3", "url": "https://huggingface.co/datasets/NousResearch/Hermes-3-Dataset", "date_obtained": "07/22/2025", "description": "A diverse collection of instruction data, curated and generated to include a range of domains and use cases.", "size_MB": 494.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Hermes reasoning tool use", "url": "https://huggingface.co/datasets/mlx-community/hermes-reasoning-tool-use", "date_obtained": "07/27/2025", "description": "A 50,000-row dataset compilation focused on tool-use reasoning.", "size_MB": 64.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "json-mode-reasoning", "url": "https://huggingface.co/datasets/interstellarninja/json-mode-reasoning", "date_obtained": "07/27/2025", "description": "A structured output reasoning dataset. Output adheres to defined json schema.", "size_MB": 23.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Code CoT Synthetic", "url": "", "date_obtained": "04/30/2025", "description": "An IBM synthetic dataset generated with the objective of improving model reasoning.", "size_MB": 250.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Enhanced FIM dataset", "url": "", "date_obtained": "01/15/2024", "description": "Synthetic FIM dataset.", "size_MB": 200.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" } ] } ] }, "post_training": { "stage": "supervised fine-tuning and alignment", "methods": [ { "category": "multilingual", "datasets": [ { "name": "CulturaX", "url": "https://huggingface.co/datasets/uonlp/CulturaX", "date_obtained": "06/01/2025", "description": "A multilingual dataset with 6.3 trillion tokens in 167 languages, tailored for LLM development.", "size_MB": 900000.0, "languages": "bn, gu, hi, kn, ml, mr, or, pa, ta, te", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Open-Platypus Translated", "url": "", "date_obtained": "12/18/2023", "description": "Translated Open Platypus dataset for improved multilingual support.", "size_MB": 49.0, "languages": "gu,ml,or,ta,kn,hi,mr,bn,te,pa", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Daring-Anteater Translated", "url": "", "date_obtained": "07/25/2024", "description": "Translated Daring Anteater dataset for improved multilingual support.", "size_MB": 1500.0, "languages": "gu,ml,or,ta,kn,hi,mr,bn,te,pa", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "EvolKit-20 Translated", "url": "", "date_obtained": "09/19/2024", "description": "Translated Evolkit-20k dataset for improved multilingual support.", "size_MB": 178.0, "languages": "gu,ml,or,ta,kn,hi,mr,bn,te,pa", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "HelpSteer2 Translated", "url": "", "date_obtained": "07/25/2024", "description": "Translated Helpsteeer2 dataset for improved multilingual support.", "size_MB": 12.0, "languages": "gu,ml,or,ta,kn,hi,mr,bn,te,pa", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Sangraha", "url": "https://huggingface.co/datasets/ai4bharat/sangraha", "date_obtained": "06/01/2025", "description": "Translated Sangraha dataset for improved multilingual support.", "size_MB": 450000.0, "languages": "bn, gu, hi, kn, ml, mr, or, pa, ta, te", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" } ] }, { "category": "General English", "datasets": [ { "name": "Hermes Function-Calling V1", "url": "https://huggingface.co/datasets/NousResearch/hermes-function-calling-v1", "date_obtained": "09/02/2024", "description": "A structured output dataset of multi-turn function-calling conversations designed to train models to interpret natural language and generate correct function invocations with JSON-style responses. It includes diverse scenarios where assistants must decide which functions to call and format their arguments properly, making it useful for improving real dialogue-level tool-calling behavior.", "size_MB": 10.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" } ] }, { "category": "reasoning", "datasets": [ { "name": "CoT-Python Synthetic", "url": "", "date_obtained": "07/01/2025", "description": "Synthetic Python code reasoning SFT dataset with Chain of Thought reasoning, which is aimed to improve model's reasoning performance for Input-Ouput prediction task for Python code.", "size_MB": 51.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Temporal Reasoning Synthetic", "url": "", "date_obtained": "08/01/2025", "description": "A dataset of question-answer pairs designed to improve the temporal reasoning capabilities of large language models. The dataset specifically targets known weaknesses in understanding time-related concepts, such as historical era transitions (BC/AD), cyclical time calculations (day-of-week), and other complex temporal arithmetic problems such as duration and schedule. The data includes a step-by-step \"chain of thought\" reasoning for each problem, leading to a final answer.", "size_MB": 7.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic MagPie Logical Reasoning", "url": "", "date_obtained": "08/15/2025", "description": "Synthetic dataset of puzzles and solutions created by folllowing MagPie self generation technique", "size_MB": 167.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" } ] }, { "category": "post-training", "datasets": [ { "name": "Concert Workflow Data", "url": "", "date_obtained": "04/01/2025", "description": "Data generated by, processed through, or used within the IBM Concert Workflows platform", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Concert Actions Data", "url": "", "date_obtained": "03/01/2025", "description": "Structured information associated with actions generated, recommended, or executed within the IBM Concert platform.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "OPA Data_Public", "url": "https://github.com/open-policy-agent/opa/tree/main", "date_obtained": "03/01/2025", "description": "Structured JSON data that provides external context or facts used by Open Policy Agent (OPA) policies.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "OPA Data_Docs", "url": "https://www.openpolicyagent.org/docs", "date_obtained": "03/01/2025", "description": "Core concepts in OPA's policy language (Rego).", "size_MB": 2.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Spider 1.0", "url": "https://yale-lily.github.io/spider", "date_obtained": "07/07/2023", "description": "Large-scale complex and cross-domain semantic parsing and text-to-SQL dataset.", "size_MB": 196.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "Kubectl data", "url": "", "date_obtained": "05/07/2025", "description": "Kubectl data relevant to SRE Agent.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "OpenMathReasoning", "url": "https://huggingface.co/datasets/nvidia/OpenMathReasoning", "date_obtained": "06/01/2025", "description": "Large-scale math reasoning dataset for training large language models (LLMs)", "size_MB": 20000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "NuminaMath-CoT", "url": "https://huggingface.co/datasets/AI-MO/NuminaMath-CoT", "date_obtained": "12/03/2024", "description": "Approximately 860k math problems, where each solution is formatted in a Chain of Thought (CoT) manner.", "size_MB": 800.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "data ablation_full59K", "url": "https://huggingface.co/datasets/simplescaling/data_ablation_full59K", "date_obtained": "03/05/2025", "description": "Approximately 59k high-quality mathematical problems and solutions designed for training reasoning-based AI models. Features detailed thinking trajectories and chain-of-thought (CoT) responses sourced from various competitive math benchmarks.", "size_MB": 5800.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "Llama-Nemotron Post Training", "url": "https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset", "date_obtained": "06/01/2025", "description": "Compilation of SFT and RL data that supports improvements of math, code, general reasoning, and instruction following capabilities of the original Llama instruct model, in support of NVIDIA’s release of Llama-3.1-Nemotron-Ultra-253B-v1, Llama-3.3-Nemotron-Super-49B-v1 and Llama-3.1-Nemotron-Nano-8B-v1.", "size_MB": 125000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "DeepScaleR Preview Dataset", "url": "https://huggingface.co/datasets/agentica-org/DeepScaleR-Preview-Dataset", "date_obtained": "06/01/2025", "description": "Approximately 40,000 unique mathematics problem-answer pairs compiled from: AIME, AMC, Omni-MATH dataset, and Still dataset.", "size_MB": 7.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "HARDMath", "url": "https://github.com/sarahmart/HARDMath", "date_obtained": "06/01/2025", "description": "A dataset of challenging, graduate-level problems in applied mathematics that can be used for language model evaluation. Unlike other popular mathematical datasets, HARDMath contains problems that require a combination of advanced problem-solving skills, approximation methods, and mathematical intuition. The dataset contains a large test set of 1,050 problems and a mini test set of 437 problems, divided across seven different problem types.", "size_MB": 5.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "Ncert Dataset", "url": "https://huggingface.co/datasets/KadamParth/Ncert_dataset", "date_obtained": "06/01/2025", "description": "A collection of educational content derived from NCERT textbooks for students in standards 6 to 12. NCERT books are textbooks published by the National Council of Educational Research and Training (NCERT), an Indian organization focused on improving school education.", "size_MB": 100.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "open-thoughts/OpenThoughts3-1.2M", "url": "https://huggingface.co/datasets/open-thoughts/OpenThoughts3-1.2M/tree/main", "date_obtained": "07/01/2025", "description": "The third iteration the our line of OpenThoughts datasets, building on previous OpenThoughts-114k and OpenThoughts2-1M. The dataset consists of 850,000 math questions, 250,000 code questions, and 100,000 science questions.", "size_MB": 10000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "open-r1/codeforces - questions only: HuggingFace Download", "url": "https://huggingface.co/datasets/open-r1/codeforces", "date_obtained": "06/01/2025", "description": "CodeForces is one of the most popular websites among competitive programmers, hosting regular contests where participants must solve challenging algorithmic optimization problems. The challenging nature of these problems makes them an interesting dataset to improve and test models’ code reasoning capabilities. This dataset includes more than 10k unique problems covering the very first contests all the way to 2025. Additionally, it includes generated and extensively validated custom checker code for problems with multiple solutions, and generated additional challenging test cases.", "size_MB": 2600.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "openCodeReasoning", "url": "https://huggingface.co/datasets/nvidia/OpenCodeReasoning", "date_obtained": "06/01/2025", "description": "Open Code Reasoning is the largest reasoning-based synthetic dataset to date for coding, comprising of 736,712 samples in Python across 28,904 unique competitive programming questions.", "size_MB": 3000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "/OpenCoder-LLM/opc-sft-stage2/viewer/educational_instruct", "url": "https://huggingface.co/datasets/OpenCoder-LLM/opc-sft-stage2/viewer/educational_instruct", "date_obtained": "12/03/2024", "description": "Contains generated (instruction, code, test case) triples, validated through a Python compiler. Notably, the inclusion of test cases provides a valuable signal for code RL.", "size_MB": 53.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "OpenCoder-LLM/opc-sft-stage2/viewer/package_instruct", "url": "https://huggingface.co/datasets/OpenCoder-LLM/opc-sft-stage2/viewer/package_instruct", "date_obtained": "12/03/2024", "description": "Used extracted common interface documentation from pydoc as a seed to generate Python package-related questions.", "size_MB": 286.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "Llama-Nemotron-Post-Training-Dataset", "url": "https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset", "date_obtained": "06/01/2025", "description": "A compilation of SFT and RL data that supports improvements of math, code, general reasoning, and instruction following capabilities of the original Llama instruct model, in support of NVIDIA’s release of Llama-3.1-Nemotron-Ultra-253B-v1, Llama-3.3-Nemotron-Super-49B-v1 and Llama-3.1-Nemotron-Nano-8B-v1.", "size_MB": 145000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "yes" }, { "name": "Multi-turn sequential slot filling and tool usage", "url": "", "date_obtained": "06/01/2023", "description": "A multi-step dataset derived from the Schema-Guided Dialogue (SGD) data that focuses on sequential slot filling and tool usage across multiple turns. It requires models to extract slots from user input, call the appropriate tools in order, and use tool responses to complete downstream actions correctly.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "GovReport-QS", "url": "https://gov-report-data.github.io/", "date_obtained": "05/01/2025", "description": "Multi turn conversations extracted from the GovReport datasets, specifically forcusing on the conversations on long documents.", "size_MB": 13.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "hotpot_qa", "url": "https://huggingface.co/datasets/hotpotqa/hotpot_qa", "date_obtained": "03/11/2024", "description": "Singe turn conversations extracted from the hotpotQA, specifically forcusing on the conversations on long documents and requiring reasoning.", "size_MB": 480.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "NarrativeQA", "url": "https://huggingface.co/datasets/meithnav/narrativeqa", "date_obtained": "05/01/2025", "description": "Singe turn conversations extracted from the NarrativeQA, specifically focusing on the conversations on long documents.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "TechQA", "url": "https://github.com/ibm/techqa", "date_obtained": "06/30/2020", "description": "Singe turn conversations sampled from the TechQA, specifically focusing on the conversations on techincal documents.", "size_MB": 4.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "FinQA", "url": "https://huggingface.co/datasets/ibm-research/finqa", "date_obtained": "05/15/2025", "description": "FinQA is a large-scale dataset with 2.8k financial reports for 8k Q&A pairs to study numerical reasoning with structured and unstructured evidence.", "size_MB": 3.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Indic Vault", "url": "https://huggingface.co/datasets/maya-research/IndicVault", "date_obtained": "07/01/2025", "description": "Instruction-tuned dataset that contains question-answer pairs in contemporary, everyday Indian languages", "size_MB": 60.0, "languages": "hi, te, en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "leetcode-solutions-en-2k", "url": "https://huggingface.co/datasets/TigerResearch/tigerbot-kaggle-leetcodesolutions-en-2k", "date_obtained": "07/22/2025", "description": "Dataset contains English descriptions and solutions for approximately 2,000 LeetCode problems.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "no", "seed_data": "no" }, { "name": "Synthetic syn-sql-2.5", "url": "", "date_obtained": "07/01/2025", "description": "IBM generated synthetic dataset mimics the manner in which syn-sql-2.5 dataset is created, but using gpt-oss-120b.", "size_MB": 843.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic MagPie Science QA", "url": "", "date_obtained": "08/18/2025", "description": "Dataset generated using MagPie self data generation technique. Using Phi-4 as a teacher model, the model generates both questions and answers to science questions.", "size_MB": 266.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic MagPie API", "url": "", "date_obtained": "08/01/2025", "description": "Synthetic dataset of questions and answers relating to API calling and its specifications, created by folllowing MagPie self generation technique", "size_MB": 275.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "APIGen function calling Synthetic", "url": "", "date_obtained": "08/01/2025", "description": "A function calling dataset using a modification of the APIgen pipeline where only gpt-oss-120b is used.", "size_MB": 41.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Open Policy Agent (OPA) policies Dataset", "url": "", "date_obtained": "03/01/2025", "description": "A synthetic dataset of OPA policies.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "NL2Opt Pyomo", "url": "", "date_obtained": "04/14/2025", "description": "Synthetically generated dataset containing natural language descriptions of mixed-integer linear optimization problems and formatted demonstrations showcasing how to solve the (synthetic) optimization problems using Pyomo.", "size_MB": 6.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "NL2GraphQL", "url": "", "date_obtained": "04/01/2025", "description": "A subset of relational tables from the WikiSQL and CoSQL datasets with a generated GraphQL schema", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "NL2OSLC", "url": "", "date_obtained": "04/29/2025", "description": "Training dataset for natural language questions to Maximo OSLC APIs translation.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "NL2Kubectl", "url": "", "date_obtained": "05/10/2025", "description": "Synthetic kubectl command samples.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "MuSR Synthetic", "url": "", "date_obtained": "05/15/2025", "description": "IBM generated dataset using the MuSR code shared by the original owners and Phi-4 model.", "size_MB": 26.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "PuzzleQA", "url": "https://huggingface.co/datasets/Jingmiao/PUZZLEQA", "date_obtained": "04/09/2025", "description": "A dataset of 558 NPR Sunday Puzzle games from 2007-2021.", "size_MB": 4.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "ACPBench- open ended questions", "url": "", "date_obtained": "05/01/2025", "description": "Data for training and evaluation of LLMs on reasonings tasks about action, change, and planning.\nThe dataset consists of open-ended questions that test reasoning abilities.", "size_MB": 4.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "ACPBench- boolean and multiple choice", "url": "", "date_obtained": "02/19/2025", "description": "Data for training and evaluation of LLMs on reasonings tasks about action, change, and planning.\nThe dataset consists of boolean and multiple choice questions that test reasoning abilities.", "size_MB": 16.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "MagPie Synthetic", "url": "", "date_obtained": "04/10/2025", "description": "Synthetic single turn instruction dataset created by following MagPie using two teacher language models.", "size_MB": 198.0, "languages": "gu,ml,or,ta,kn,hi,mr,bn,te,pa", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "NuminaMath-CoT Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Problems from NuminaMath-CoT dataset (https://huggingface.co/datasets/AI-MO/NuminaMath-CoT). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 1209.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "data_ablation_full59k Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Problems from data_ablation_full59k dataset (https://huggingface.co/datasets/simplescaling/data_ablation_full59K). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 155.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenMathReasoning Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Problems from OpenMathReasoning dataset (https://huggingface.co/datasets/nvidia/OpenMathReasoning) Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 8691.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Llama-Nemotron-Post-Training Synthetic", "url": "", "date_obtained": "05/18/2025", "description": "Problems from Llama-Nemotron-Post-Training dataset (https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 6860.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenThoughts2-1M Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Problems from OpenThoughts2-1M dataset (https://huggingface.co/datasets/open-thoughts/OpenThoughts2-1M). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 1073.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenThoughts2-1M", "url": "https://huggingface.co/datasets/open-thoughts/OpenThoughts2-1M", "date_obtained": "06/01/2025", "description": "Open synthetic reasoning dataset with 1M high-quality examples covering math, science, code, and puzzles.", "size_MB": 3000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "yes" }, { "name": "DeepScaleR-Preview Synthetic", "url": "", "date_obtained": "05/15/2025", "description": "Problems from DeepScaleR-Preview dataset (https://huggingface.co/datasets/agentica-org/DeepScaleR-Preview-Dataset). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 330.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "HARDMath Synthetic", "url": "", "date_obtained": "05/15/2025", "description": "Problems from HARDMath dataset (https://github.com/sarahmart/HARDMath). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Ncert Synthetic", "url": "", "date_obtained": "05/01/2025", "description": "Problems Ncert dataset (https://huggingface.co/collections/KadamParth/ncert-dataset). Responses generated using Phi-4-reasoning and Mistral-Small-24B-Instruct-2501 as a judge to identify correct chain of thought.", "size_MB": 881.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Codeforces Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Synthetic dataset consisting of ~10K competitive coding problems from Codeforces websites (https://huggingface.co/datasets/open-r1/codeforces) all the way to 2025 with solutions generated using Phi-4 reasoning.", "size_MB": 8000.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenCodeReasoning Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Synthetic dataset consisting of 28,319 unique competitive programming questions sourced from APPS, TACO,CodeContests,Codeforces (https://huggingface.co/datasets/nvidia/OpenCodeReasoning) with responses generated using Phi-4-reasoning.", "size_MB": 8000.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "DeepCoder-PrimeIntellect Synthetic", "url": "", "date_obtained": "05/15/2025", "description": "Synthetic dataset consisting of 16K verified coding problems from PrimeIntellect’s SYNTHETIC-1 (https://huggingface.co/datasets/agentica-org/DeepCoder-Preview-Dataset/viewer/primeintellect) coding challenges with verifiable unit tests with responses generated using Phi-4-reasoning.", "size_MB": 8000.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenCodeInstruct Synthetic", "url": "", "date_obtained": "05/15/2025", "description": "Synthetic dataset consisting of 5M synthetically generated coding problems starting from seed problems from 10 datasets and Python functions from GitHub. (https://huggingface.co/datasets/nvidia/OpenCodeInstruct). Solutions generated using Phi-4-reasoning.", "size_MB": 8000.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Open Code Instruct", "url": "https://huggingface.co/datasets/nvidia/OpenCodeInstruct", "date_obtained": "06/01/2025", "description": "The largest open-access instruction tuning dataset (in python), comprising 5 million diverse samples.", "size_MB": 4200.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "yes" }, { "name": "Llama Nemotron post training Synthetic", "url": "", "date_obtained": "06/01/2025", "description": "Synthetic dataset consisting of 9M sythetically generated coding problems sourced from Llama-Nemotron-Post-Training dataset (https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset). Solutions generated using Phi-4-reasoning.", "size_MB": 8000.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Glaive-code-assistant", "url": "https://huggingface.co/datasets/glaiveai/glaive-code-assistant", "date_obtained": "04/04/2024", "description": "A synthetic dataset of real conversational examples formatted for function calling, where AI assistant interactions include explicit function call invocations for structured outputs.", "size_MB": 8.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "IBM Seq SGD Dataset", "url": "https://github.ibm.com/conversational-ai/API_integrated_LLMs/tree/main/data/SeqSGD", "date_obtained": "07/16/2024", "description": "A multi-turn, multi-domain task-oriented dialogue dataset with annotated schemas and turn-level labels for learning structured interactions with services and APIs. The data is filtered using an FC-Reflector–based approach to retain high-quality tool-calling and dialogue transitions.", "size_MB": 12.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Multi-turn slot filling with system instruction", "url": "", "description": "A dataset created from the Schema-Guided Dialogue (SGD) data, focused on slot-filling tasks where models must extract and populate correct slot values from multi-turn dialogue context. It emphasizes accurate slot prediction across turns and services, supporting structured information extraction for task-oriented dialogue systems.", "size_MB": 8.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Slot instruction", "url": "", "description": "A multi-turn tool-calling dataset focused on generating correct tool parameters from dialogue context. It provides slot-level instructions (type, description, and formatting rules like YYYY-MM-DD) to guide accurate argument filling and validation.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "When2Call", "url": "https://huggingface.co/datasets/nvidia/When2Call", "date_obtained": "07/27/2025", "description": "A dataset designed to teach and evaluate models on when to call a tool vs. when to answer directly or ask a follow-up question, improving decision-making for tool invocation. It includes multiple-choice and training splits to help models learn the correct tool-calling behavior.", "size_MB": 3.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Routing dataset for function calling", "url": "", "date_obtained": "08/02/2024", "description": "A routing dataset designed to train models to select the correct tool (or no tool) based on user intent from a set of possible tools. Each tool represents a distinct intent category, emphasizing accurate intent recognition, ambiguity handling, and correct route selection without argument generation.", "size_MB": 7.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "xlam-irrelevance-7.5k", "url": "https://huggingface.co/datasets/MadeAgents/xlam-irrelevance-7.5k", "date_obtained": "07/25/2024", "description": "A dataset of 7,500 examples created by removing the correct functions from tool lists so the model must recognize when available functions are irrelevant to the user’s request, helping improve irrelevance detection in tool-calling models.", "size_MB": 1.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "xlam_function_calling_60k - filtered", "url": "xlam_function_calling_60k", "date_obtained": "07/25/2024", "description": "We use xlam-function-calling-60k (https://huggingface.co/datasets/Salesforce/xlam-function-calling-60k), a synthetic dataset of 60,000 examples focused on converting user instructions into correct tool calls with realistic API schemas and structured arguments. We used ToolRM (https://arxiv.org/abs/2509.11963) filter to get best 75% data.", "size_MB": 11.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic data for Instana tools", "url": "", "description": "Synthetic data generated for Instana tools and filtered using FC-Reflector", "size_MB": 2.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic data for Apptio", "url": "", "description": "Synthetic data generated for Apptio tools and filtered using FC-Reflector", "size_MB": 1.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic data - parallel and multiple tool-calling scenarios.", "url": "", "date_obtained": "10/03/2024", "description": "Synthetic data generated using an IBM-developed synthetic data generation pipeline, focused on parallel and multiple tool-calling scenarios. The data is filtered using IBM-developed synthetic filters and a ToolRM-based filter to ensure high quality.", "size_MB": 42.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Long-context-10K", "url": "", "date_obtained": "04/14/2025", "description": "10k subset from the dataset collection, where the tool library is inflated with additional tools to increase the context size, which are also filtered with Mistral-7B-RM.", "size_MB": 35.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "XLAM-robustness", "url": "", "date_obtained": "03/01/2025", "description": "A dataset built on top of xLAM function-calling data to evaluate robustness in tool-calling models under query rephrasing and changes in available tools. The data is generated by rewriting user queries and creating new function-calling candidates using expanded toolkits while filtering equivalent tools to ensure valid and challenging tool-calling scenarios.", "size_MB": 6.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "XLAM-masked", "url": "", "date_obtained": "04/10/2025", "description": "Subset of the XLAM-robustness data where the Function names and the Arguments are masked.", "size_MB": 5.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic single turn RAG dataset", "url": "", "date_obtained": "08/28/2024", "description": "Single turn synthetically generated RAG dataset. The dataset is filtered using the using MagPie filters on input quality, sample quality and difficulty.", "size_MB": 71.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic single turn RAG dataset - Wikipedia and government domains", "url": "", "description": "Single turn synthetically generated RAG dataset. The domains include wikipedia and goverment domains. The dataset is filtered using the using MagPie filters on input quality, sample quality and difficulty.", "size_MB": 47.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Synthetic multi-turn RAG dataset - Wikipedia and government domains", "url": "", "date_obtained": "08/28/2024", "description": "Multi turn synthetically generated RAG dataset. The number of turns range from 2-16. The domains include wikipedia and goverment domains. The dataset is filtered using the using MagPie filters on input quality, sample quality and difficulty.", "size_MB": 47.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Acquired multi-turn dataset; government domain", "url": "", "date_obtained": "04/07/2025", "description": "Multiturn conversation data created by the human annotators at Defined.ai, and reviewed for quality from human annotators. The data comes from government domains.", "size_MB": 25.0, "languages": "en", "type": "text", "source": "Third Party", "data_provenance": "Private non-publicly available datasets obtained from third-parties", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenCodeGeneticInstruct - mixtral 8x22b instruct subset", "url": "https://huggingface.co/datasets/nvidia/OpenCodeGeneticInstruct/tree/main/mixtral-8x22b-instruct", "date_obtained": "07/22/2025", "description": "OpenCodeGeneticInstruct comprises more than 15M coding instructions in python which is generated synthetically with the Genetic-Instruct approach. Only the Mixtral-8x22b-instruct subset of this dataset is used.", "size_MB": 16000.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "OpenMathInstruct-1", "url": "https://huggingface.co/datasets/nvidia/OpenMathInstruct-1", "date_obtained": "07/25/2024", "description": "A math instruction tuning dataset with 1.8M problem-solution pairs generated using permissively licensed Mixtral-8x7B model.", "size_MB": 2100.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Hermes 3", "url": "https://huggingface.co/datasets/NousResearch/Hermes-3-Dataset", "date_obtained": "07/22/2025", "description": "A diverse collection of instruction data, curated and generated to include a range of domains and use cases.", "size_MB": 494.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Hermes reasoning tool use", "url": "https://huggingface.co/datasets/mlx-community/hermes-reasoning-tool-use", "date_obtained": "07/27/2025", "description": "A 50,000-row dataset compilation focused on tool-use reasoning.", "size_MB": 64.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Publicly available datasets", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "json-mode-reasoning", "url": "https://huggingface.co/datasets/interstellarninja/json-mode-reasoning", "date_obtained": "07/27/2025", "description": "A structured output reasoning dataset. Output adheres to defined json schema.", "size_MB": 23.0, "languages": "en", "type": "text", "source": "third party", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Code CoT Synthetic", "url": "", "date_obtained": "04/30/2025", "description": "An IBM synthetic dataset generated with the objective of improving model reasoning.", "size_MB": 250.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" }, { "name": "Enhanced FIM dataset", "url": "", "date_obtained": "01/15/2024", "description": "Synthetic FIM dataset.", "size_MB": 200.0, "languages": "en", "type": "text", "source": "IBM", "data_provenance": "Synthetic data that is not publicly accessible", "user_data": "no", "synthetic": "yes", "seed_data": "no" } ] } ] } } }