{
  "sci_latex": {
    "repo": "EleutherAI/proof-pile-2",
    "revision": "901a9273a770e9d4138c5ddd91802f9c5c6cdc4b",
    "shards_in_order": [
      "arxiv/train/arXiv_000.jsonl.zst"
    ],
    "quota": 71582788,
    "bytes": 71582536,
    "lines": 753418,
    "docs": 1255,
    "sha256": "baf588da429a519ed5c1aa9cda834a449e2c0f2a24af066f1152bb3c29b38c42",
    "max_line_bytes": 4000,
    "secs": 9.7
  },
  "pl": {
    "repo": "epfml/FineWeb2-HQ",
    "revision": "c0c06e94fd3a44ae9e802b2b0fc533817601eb5e",
    "config": "pol_Latn",
    "split": "train",
    "skip_first_docs": 2000,
    "wikipedia_url_filtered": 15497,
    "stream_docs_seen": 372187,
    "quota": 1073741824,
    "bytes": 1073741798,
    "lines": 4821824,
    "docs": 354690,
    "sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8",
    "max_line_bytes": 4000,
    "secs": 660.6
  },
  "sci_python": {
    "repo": "codeparrot/codeparrot-clean-train",
    "revision": "3e6ab65f2864931e041f6a82db9b5a6ec2b71ab4",
    "split": "train",
    "filters": "300<=chars<=20000, ascii_ratio>=0.9 (E1 en-python-code filters)",
    "stream_docs_seen": 15069,
    "docs_passing_filter": 13407,
    "quota": 71582788,
    "bytes": 71582760,
    "lines": 1734245,
    "docs": 13407,
    "sha256": "779923c8065875b4b3b53a10c8cb0c70cc3ce02b13a36470023146e3401d4966",
    "max_line_bytes": 4000,
    "secs": 12.9
  },
  "sci_mathweb": {
    "repo": "EleutherAI/proof-pile-2",
    "revision": "901a9273a770e9d4138c5ddd91802f9c5c6cdc4b",
    "shards_in_order": [
      "open-web-math/train/shard-0000.jsonl.zst"
    ],
    "quota": 71582788,
    "bytes": 71582730,
    "lines": 535527,
    "docs": 8642,
    "sha256": "36f49b00f3d299a30a2cec673c36b9c7b7d4291d5b80c5f63ef2cdb91e9ed3e5",
    "max_line_bytes": 4000,
    "secs": 9.2
  },
  "en_general": {
    "repo": "DKYoon/SlimPajama-6B",
    "revision": "b5f90f419b7489cdba26fdbc8c022fcb5562f968",
    "split": "train",
    "skip_first_docs": 2000,
    "stream_docs_seen": 52535,
    "quota": 214748365,
    "bytes": 214748283,
    "lines": 1410258,
    "docs": 50535,
    "sha256": "fae778ee956ef859ecc231b0c5264520af5ab6baad4fc3c9518c2e4d3744e4ed",
    "max_line_bytes": 4000,
    "secs": 19.1
  }
}