{
  "en-arxiv-abstracts": {
    "slug": "en-arxiv-abstracts",
    "source": {
      "type": "arxiv-api",
      "endpoint": "https://export.arxiv.org/api/query",
      "raw_responses": [
        {
          "file": "data/raw/arxiv/batch_00001.xml",
          "sha256": "7c8e7b5527d462a2b3d32be09f2db379f10372fc1f4fd7f013ce2fdbd55e0b43"
        },
        {
          "file": "data/raw/arxiv/batch_00101.xml",
          "sha256": "abc45c3e73e057d731fe94a445606bc59ba649f95c24cebc6667ade99864d5e0"
        },
        {
          "file": "data/raw/arxiv/batch_00201.xml",
          "sha256": "8515a7f0612a4c17c6e5580b8c0a1988e035e5e7b6e9d53d846a4a8d054a7734"
        },
        {
          "file": "data/raw/arxiv/batch_00301.xml",
          "sha256": "bf11d6a4ab64d60fd83ccbf9302a61e32d5c51042d6fabc962dd66aaf3d8fe96"
        },
        {
          "file": "data/raw/arxiv/batch_00401.xml",
          "sha256": "3beffd186e85ec03671e0fe590b656ff7d5ccba0b077c19f6456885952c1ceac"
        },
        {
          "file": "data/raw/arxiv/batch_00501.xml",
          "sha256": "34cafb5c830800d834f7d42e1f733889e1f9f516b2364b6cb64bf37c852bdb55"
        },
        {
          "file": "data/raw/arxiv/batch_00601.xml",
          "sha256": "11392f5e4bed55993938083f608a55d8a79d8f0ec34c4db72e6bbfdee2040f1f"
        },
        {
          "file": "data/raw/arxiv/batch_00701.xml",
          "sha256": "57ddd92e6a1d3e6803175624456fbe0ad4652c5cf59694ed47c6dfdb4842cb88"
        }
      ]
    },
    "selection": "sequential id_list 2401.00001..2401.02400 in batches of 100; keep abstracts with >=40 ws-words; stop at quota",
    "params": {
      "min_words": 40,
      "quota": 120000
    },
    "n_docs": 784,
    "n_words_ws": 126621,
    "n_chars_incl_spaces": 888253,
    "n_bytes_utf8": 888763,
    "file": "data/corpora/en-arxiv-abstracts.jsonl",
    "sha256_jsonl": "0727333b13f8b22d0d1a8b010c3bcdd8ee6298f3649b2bb758746130cb1cc743"
  },
  "en-latex-methods": {
    "slug": "en-latex-methods",
    "source": {
      "type": "hf-dataset",
      "repo": "EleutherAI/proof-pile-2",
      "revision": "901a9273a770e9d4138c5ddd91802f9c5c6cdc4b",
      "files": [
        {
          "path": "arxiv/test/arXiv_000.jsonl.zst",
          "sha256": "6cc63607ba350fe93763454a98ea1f6eae6f0b37986c0d01a76e8e969cf637a6"
        },
        {
          "path": "arxiv/test/arXiv_001.jsonl.zst",
          "sha256": "c9dccb7b2eac96107acad2c7293a8045331ccd94d0d3f362a26883af02464055"
        },
        {
          "path": "arxiv/test/arXiv_002.jsonl.zst",
          "sha256": "5e866b636cbeb978debced53c358c5b31d9036b3f1c3514407faa8fc313dc430"
        }
      ]
    },
    "selection": "arxiv/test shards in order; per paper keep \\section/\\subsection blocks whose title matches METHODY regex, >=80 words, capped 1500 words/paper; stop at quota",
    "params": {
      "section_regex": "(?i)\\b(method|approach|model|architecture|experiment|analysis|theory|proof|algorithm|implementation|setup|evaluation|formulation|preliminaries)\\b",
      "min_section_words": 80,
      "per_paper_word_cap": 1500,
      "quota": 120000
    },
    "n_docs": 219,
    "n_words_ws": 120168,
    "n_chars_incl_spaces": 967618,
    "n_bytes_utf8": 967666,
    "file": "data/corpora/en-latex-methods.jsonl",
    "sha256_jsonl": "c297c2f0e743db1406c22ffaa972d35e83c9b18b53f2372b2878e63c8bfb4514"
  },
  "en-python-code": {
    "slug": "en-python-code",
    "source": {
      "type": "hf-dataset",
      "repo": "codeparrot/codeparrot-clean-valid",
      "revision": "4db92d2ec0c1b4c41eeb439cfae16854511d9dcd",
      "files": [
        {
          "path": "file-000000000054.json.gz",
          "sha256_lfs": "7f8c20a737c9084779bcdb853325ad4774d0db52c74aa2a63fd658d6787eb35b",
          "jsonl_lines_consumed": 374
        }
      ]
    },
    "selection": ".py files in stream order; 300<=chars<=20000, ascii ratio>=0.9, truncated at 700 ws-words on line boundary; stop at quota",
    "params": {
      "min_words": 30,
      "per_doc_word_cap": 700,
      "quota": 120000
    },
    "n_docs": 331,
    "n_words_ws": 120489,
    "n_chars_incl_spaces": 1372236,
    "n_bytes_utf8": 1373355,
    "file": "data/corpora/en-python-code.jsonl",
    "sha256_jsonl": "46fa394fc57b5b719509dc92aa617b548fe841afc9d8c8df81d8e8848cd1684f"
  },
  "en-gsm8k": {
    "slug": "en-gsm8k",
    "source": {
      "type": "hf-dataset",
      "repo": "openai/gsm8k",
      "revision": "740312add88f781978c0658806c59bc2815b9866",
      "files": [
        {
          "path": "main/train-00000-of-00001.parquet",
          "sha256": "ea82612ea9582142387730c793eb67d3b12849002bc0b7fa6f8efafa7351419d"
        }
      ]
    },
    "selection": "all 'question' fields of main/train in row order (no quota cut; whole split kept)",
    "params": {},
    "n_docs": 7473,
    "n_words_ws": 336977,
    "n_chars_incl_spaces": 1747905,
    "n_bytes_utf8": 1748806,
    "file": "data/corpora/en-gsm8k.jsonl",
    "sha256_jsonl": "cd35932f6f060ea6f34ab500b76a4324d6ba942db57d83c1bd8db38211383241"
  },
  "pl-science-pes": {
    "slug": "pl-science-pes",
    "source": {
      "type": "hf-dataset",
      "repo": "speakleash/PES-2018-2022",
      "revision": "79ae158ceab8e4e083c8cfcacde56669407a94a2",
      "files": [
        {
          "path": "data/alergologia.csv",
          "sha256": "fde08e2e90578f90f576a615097f717a2b6a408aed6196263dd038eadbaa1f0b"
        },
        {
          "path": "data/anestezjologia.csv",
          "sha256": "7168cb6ca4dcc9bfb86b1d0db9439b5eeb137aa474e2abb9bc1496040b3aa52f"
        },
        {
          "path": "data/angiologia.csv",
          "sha256": "eddc3d71a043c90079b29cc185d8a4f6bbbe05ec2aa3f8dfcab6f8bedd944747"
        },
        {
          "path": "data/balneologia_i_medycyna_fizykalna.csv",
          "sha256": "7a9c6099364328ff4f600db23b24b9eb57a2a97a9768047e0031ed9efe0efb95"
        },
        {
          "path": "data/chirurgia_dziecieca.csv",
          "sha256": "48e23e19c2e3b4b3cf0e984204f3820edb0a442eead2887ee441f1eab84e4900"
        },
        {
          "path": "data/chirurgia_naczyniowa.csv",
          "sha256": "2c3ac49821fa77b2dd253a8c50a77295a519bcf7599fda96316fbbccefb58283"
        },
        {
          "path": "data/chirurgia_ogolna.csv",
          "sha256": "6fcfbceeb973f8aac908a0c3aed5fbfd0a133022df145a627a0274b56e5a0afe"
        },
        {
          "path": "data/chirurgia_onkologiczna.csv",
          "sha256": "52a0c7205c6f90ac21ebdeae738507450619ee45529fbe9e47485b6d668b34b9"
        },
        {
          "path": "data/chirurgia_stomatologiczna.csv",
          "sha256": "89e4bd11ae95716c87d0f54ddfa64ebbee2d4c95a509515d8949e71cd756e24c"
        },
        {
          "path": "data/chirurgia_szczekowo-twarzowa.csv",
          "sha256": "2c9bdd7ae560a7d2a2f12e59f8857264fd8eb6f47f172cb35e454a4625cc299c"
        },
        {
          "path": "data/choroby_pluc.csv",
          "sha256": "bbd450123a5b337271f5b3371a8f568605355de8ab5b258b7b258db093c6bd2d"
        },
        {
          "path": "data/choroby_pluc_dzieci.csv",
          "sha256": "29f9a501c535f4c618c1896bcd57c29b9a836822e8df4e6d3ff33cbe8d6812f5"
        },
        {
          "path": "data/choroby_wewnetrzne.csv",
          "sha256": "7d7ae14d05249fecd5983f30a1aa8a84b755a65613660424cabb886604c7b347"
        },
        {
          "path": "data/choroby_zakazne.csv",
          "sha256": "02f3120ef8fb545234b8ecf22cb7f5987c6174387cc294f43b8264ce6722f8f0"
        },
        {
          "path": "data/dermatologia_i_wenerologia.csv",
          "sha256": "235a58d50d260dcaef3d6b55fc20d209f0f8be723dc5c0791c0fff989c7225db"
        },
        {
          "path": "data/diabetologia.csv",
          "sha256": "bdb5078e4d872ee3345821b6ec1f83b69bd390950fa6604cf4b7b9435cf5e3c9"
        },
        {
          "path": "data/endokrynologia.csv",
          "sha256": "8551edf05cf1c79de719b4d5b364560168e7329e20dceb9a85a737d3195f7872"
        },
        {
          "path": "data/endokrynologia_ginekologiczna_i_rozrodczosc.csv",
          "sha256": "7317a7ab55cbd92cb5c3fe4b3a425b4d503746720a02f7dc65c2c2f0b9bc1ce4"
        },
        {
          "path": "data/endokrynologia_i_diabetologia_dziecieca.csv",
          "sha256": "da01b138446d2e5cd0735759aa2fb46f362092d29d6b23c0ba777955d28b1ce3"
        },
        {
          "path": "data/gastroenterologia.csv",
          "sha256": "ba5214a18807b7ff5efa72f5bd822c965f6df683be23c707c3fe124605cd5159"
        },
        {
          "path": "data/gastroenterologia_dziecieca.csv",
          "sha256": "93fea3c13b637fad9a9a0a8c8de210de7d808a00f4f1c8fc6971b68cef4c58dc"
        },
        {
          "path": "data/geriatria.csv",
          "sha256": "853af5a39c6b2fb1a7c060ee6d578b02a16bdc4cd87cc419bd28e456614ec4ba"
        },
        {
          "path": "data/ginekologia_onkologiczna.csv",
          "sha256": "388e2012ca53bbc3c6a96238efae56a90159b245a32d9e4fa8bfd1f35986ca42"
        },
        {
          "path": "data/hematologia.csv",
          "sha256": "649961bc4bf45bb83a13823db63407e887753aaf74353ce233072fe6466efc6a"
        },
        {
          "path": "data/hipertensjologia.csv",
          "sha256": "aad889627d3abd5bff39aed7ab142c01ef6145825b37c329b7528f105ef6c220"
        },
        {
          "path": "data/kardiochirurgia.csv",
          "sha256": "9588e1e67bdfcd618650200cdfbaa95c697d999b5188d267a098567e19b9b09e"
        },
        {
          "path": "data/kardiologia.csv",
          "sha256": "ceca021b1eb41e583b8c90c050bcaa368e4f604edfac3480fbeceaa965bdd008"
        },
        {
          "path": "data/medycyna_paliatywna.csv",
          "sha256": "7a90934239e416d82ae0d82947c37712905d2bb4f6c4430cba8e2962fcfb2501"
        },
        {
          "path": "data/medycyna_pracy.csv",
          "sha256": "2010ef22c1b75f63aa700d7dbff280beb16d0a444962dab667c0ad929b789223"
        },
        {
          "path": "data/medycyna_ratunkowa.csv",
          "sha256": "65ad25d94bc000f288b2eca8f427210bac98b0e50587e2dac33e0fde48455e2f"
        },
        {
          "path": "data/medycyna_rodzinna.csv",
          "sha256": "b841acd1a9ef85c6559d50382d74ee9ea5f013f1dcad684e1b8914ccb4210eef"
        },
        {
          "path": "data/medycyna_sportowa.csv",
          "sha256": "497ea734047f8db81ab074d96dea787afc12d79caff97b0e4e2f0259757d47bc"
        },
        {
          "path": "data/nefrologia.csv",
          "sha256": "81674d7cba0016405013adc5dbeeaa11d0b1770654656a68df7a0124e5bb288a"
        },
        {
          "path": "data/neonatologia.csv",
          "sha256": "1b909faf42f593ffddc0b87110fe4e08757a12b5db73b9900e9e63d3e12696d5"
        },
        {
          "path": "data/neurochirurgia.csv",
          "sha256": "9c91c266c499089b0302a4dc1519a2a190dadaccd375ba846356c87ccc34154e"
        },
        {
          "path": "data/neurologia.csv",
          "sha256": "aeea3e372cc174a57e2bb04df74b88f567628bf987e143c6d6c146fc09dc89ed"
        },
        {
          "path": "data/neurologia_dziecieca.csv",
          "sha256": "e45bcc9fe7cf4ddc300f2dad017c68c6f282a527a5d3e1fbcae3692f6afc06cf"
        },
        {
          "path": "data/okulistyka.csv",
          "sha256": "808ce29078e84b067f1df51b7307496ae2e595d66a51348dee765f86257b0a85"
        },
        {
          "path": "data/onkologia_kliniczna.csv",
          "sha256": "dda6d93868dd06268a23afcd70e3e0f97e78b66896933edefb4bdcd9bb86ec8c"
        },
        {
          "path": "data/ortodoncja.csv",
          "sha256": "9fee5ba2cc6017aad4305bb0b7a9ed7ee878ce0652a1f8e605cecd22a242fc8f"
        },
        {
          "path": "data/ortopedia.csv",
          "sha256": "794b8244095ed5e4b1a5c2c1b88de9095fbe3be1153304b6ba88104f6c394be1"
        },
        {
          "path": "data/otolaryngologia.csv",
          "sha256": "9c3ab26f9b428d79527cdabcbaf19f68ba6b70b2a66bb66c6ed63033df0155c0"
        },
        {
          "path": "data/patomorfologia.csv",
          "sha256": "19f30c31c2db6577d25b5f0878bbccb2f3f2427d2be29e6372b46f5ae64fdff7"
        },
        {
          "path": "data/pediatria.csv",
          "sha256": "3155d373c339cea289b3bc2902e1af63da8161e2187936ea67fc5261549c7342"
        },
        {
          "path": "data/perinatologia.csv",
          "sha256": "4e386243fc98b24c8a5ae6d1908dd0f8708fff07baeae4a6c2ee4a534e364519"
        },
        {
          "path": "data/periodontologia.csv",
          "sha256": "6235610f02a6df69788b27944dbb69fade6a300effb98a62dbe5aba5d4000d28"
        },
        {
          "path": "data/poloznictwo_i_ginekologia.csv",
          "sha256": "e74e86141099808166e5d3156404c5341d4d12b9f8bba2c600c57e28397b0328"
        },
        {
          "path": "data/protetyka_stomatologiczna.csv",
          "sha256": "ca9ddbe37c60beb0b420af4155607e3f121768ac643a94e5946fb1d3e4cd751a"
        },
        {
          "path": "data/psychiatria.csv",
          "sha256": "cfd748c6ef88e99e4513021242b39bc15199e8358195e98c83738b40dda7b0d5"
        },
        {
          "path": "data/psychiatria_dzieci_i_mlodziezy.csv",
          "sha256": "d397e2c935b1b77f524b9d2fecd95561f3f4c1104761532df6b10a58955da3bf"
        },
        {
          "path": "data/radiologia_i_diagnostyka_obrazowa.csv",
          "sha256": "f0623a02f56023e8dadd39f278dce818c65d76f46af88fa7b6f3ed0044a7a12e"
        },
        {
          "path": "data/radioterapia_onkologiczna.csv",
          "sha256": "3686c11a843fe26f49310f702144e5852a8b1fe985e4db3ca984da3fc39c387b"
        },
        {
          "path": "data/rehabilitacja_medyczna.csv",
          "sha256": "eda9d949229353f23c464d6b35043702540dd1d6c312cf0dca65c6006516e6ea"
        },
        {
          "path": "data/reumatologia.csv",
          "sha256": "3fd03508bdbcfb1294e45c7d90019e2d9ddc8ed8edc68fd6e3ca7c1c2b709d3b"
        },
        {
          "path": "data/stomatologia_dziecieca.csv",
          "sha256": "f415cf6fd033f641f1b53ca88c17a991407c3c7b5957f2fd054cd7e0c8ede7c7"
        },
        {
          "path": "data/stomatologia_zachowawcza.csv",
          "sha256": "df9078d3a1e4e3ecbc097cc9738947c42917d8a35c0e7efc509ba6cdcdad8e32"
        },
        {
          "path": "data/transplantologia_kliniczna.csv",
          "sha256": "920ea16a396bb32ad753b2a80290ed98acb52820feb313563382c06f0fcedf10"
        }
      ]
    },
    "selection": "round-robin over lexicographically sorted specialty CSVs, in-file row order; 'question' column (full text incl. options), >=25 ws-words; stop at quota",
    "params": {
      "min_words": 25,
      "quota": 120000
    },
    "n_docs": 1725,
    "n_words_ws": 120079,
    "n_chars_incl_spaces": 815783,
    "n_bytes_utf8": 853691,
    "file": "data/corpora/pl-science-pes.jsonl",
    "sha256_jsonl": "874761c42bc41caafb61f3a479c13fba6461e70f07df97a0541c1f73dd49e28c"
  },
  "pl-wiki-science": {
    "slug": "pl-wiki-science",
    "source": {
      "type": "hf-dataset",
      "repo": "wikimedia/wikipedia",
      "revision": "b04c8d1ceb2f5cd4588862100d08de323dccfbaa",
      "files": [
        {
          "path": "20231101.pl/train-00000-of-00006.parquet",
          "sha256": "e138c56e16acc8724e851c206f2b2f586e3cdf511105b1d8a81d90b6d374248c"
        }
      ]
    },
    "selection": "shards in order, rows in order; keep articles whose first 400 chars (lowercased) contain any WIKI_MARKERS entry and >=60 ws-words; truncate at 900 words on paragraph boundary; stop at quota",
    "params": {
      "markers": [
        "w matematyce",
        "w fizyce",
        "w chemii",
        "w biologii",
        "w informatyce",
        "w astronomii",
        "w medycynie",
        "w geologii",
        "w elektronice",
        "w mechanice",
        "dział fizyki",
        "dział chemii",
        "dział matematyki",
        "dział biologii",
        "dział informatyki",
        "dziedzina nauki",
        "dziedzina wiedzy",
        "związek chemiczny",
        "pierwiastek chemiczny",
        "związek organiczny",
        "związek nieorganiczny",
        "reakcja chemiczna",
        "cząstka elementarna",
        "jednostka miary",
        "twierdzenie",
        "równanie",
        "algorytm",
        "język programowania",
        "system operacyjny",
        "enzym",
        "białko",
        "hormon",
        "gatunek rośliny",
        "gatunek grzyba",
        "gatunek bakterii",
        "rodzaj roślin",
        "rodzaj grzybów",
        "choroba",
        "zespół chorobowy",
        "lek z grupy",
        "obiekt astronomiczny",
        "gwiazdozbiór",
        "planetoida",
        "galaktyka",
        "mgławica",
        "minerał",
        "skała",
        "teoria fizyczna",
        "proces technologiczny"
      ],
      "min_words": 60,
      "per_doc_word_cap": 900,
      "quota": 120000
    },
    "n_docs": 229,
    "n_words_ws": 120441,
    "n_chars_incl_spaces": 902747,
    "n_bytes_utf8": 950222,
    "file": "data/corpora/pl-wiki-science.jsonl",
    "sha256_jsonl": "6489e0f15906b7000e9d555bb150b0fdd65b6b427eae3eda24b1459ae9d91132"
  },
  "pl-informal": {
    "slug": "pl-informal",
    "source": {
      "type": "hf-dataset",
      "repo": "allegro/klej-allegro-reviews",
      "revision": "b9ca796de36c796d6d5b853c3f13ef461440d630",
      "files": [
        {
          "path": "train.csv",
          "sha256": "bc722eafd61ccec427cc3f6af112fbbfbc999bb619f8f02a7ad5027fec1c707e"
        }
      ]
    },
    "selection": "train.csv rows in order; 'text' column, >=15 ws-words; stop at quota",
    "params": {
      "min_words": 15,
      "quota": 120000
    },
    "n_docs": 1512,
    "n_words_ws": 120029,
    "n_chars_incl_spaces": 773098,
    "n_bytes_utf8": 815406,
    "file": "data/corpora/pl-informal.jsonl",
    "sha256_jsonl": "e3cb0d60de14b3dda0f56126757fef2f606c3d8e969c7ca75d841dd9fa279787"
  }
}