{
"e9-pl": {
"gates": {
"G1": {
"pass": true,
"shape_equal_apt4": true,
"layout_ok": true,
"max_piece_len": 16,
"ws_only_pieces": 0
},
"G4": {
"pass": true,
"n_lines": 10000,
"piece_mismatches": 0,
"decode_mismatches": 0,
"first_mismatch": null
},
"G5": {
"pass": true,
"expected_present": false,
"pieces_present": {
"U+00A0": false,
"U+202F": false,
"U+2212": false
}
}
},
"meta": {
"arm": "e9-pl",
"config": {
"mix": {
"pl": 1073741824
},
"fix": false,
"ws_pieces": false
},
"spm_flags": {
"model_type": "bpe",
"vocab_size": 32000,
"character_coverage": 0.9995,
"normalization_rule_name": "identity",
"remove_extra_whitespaces": false,
"add_dummy_prefix": true,
"byte_fallback": true,
"split_digits": true,
"max_sentencepiece_length": 16,
"allow_whitespace_only_pieces": false,
"shuffle_input_sentence": false,
"input_sentence_size": 0,
"max_sentence_length": 8192,
"num_threads": 8,
"unk_id": 0,
"bos_id": 1,
"eos_id": 2,
"pad_id": -1,
"control_symbols": [
"<|im_start|>",
"<|im_end|>",
"",
""
],
"user_defined_symbols": []
},
"spm_version": "0.2.1",
"input_file": "data/arm_inputs/e9-pl.txt",
"input_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8",
"input_sources": {
"pl": {
"bytes": 1073741824,
"source_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8"
}
},
"train_secs": 183.2,
"n_normal": 31737,
"n_merges": 60058,
"max_piece_len": 16,
"ws_only_pieces": 0
},
"files": {
"special_tokens_map.json": "96bdbb8504d9967606e5f661ccc7cbbac44a3661af863a7a58614670a0ccab33",
"spm.model": "eac085b68090be7a048f8dfcb80b1ea8be5d267b4b758e064c417dfa0026f6bf",
"spm.vocab": "25b8e27e518e086d54e944bb4bef87a8f04c4e5b676d1b4b91723b3a4049ef79",
"tokenizer.json": "f22aca2a7ccc0ce04e821d1b6c14a3f79c20b3a268fc4ea444636dc255950acc",
"tokenizer_config.json": "fa53599038cf703453fd2514f69938a5529eee2aa21b5e66f82fe33a75dcf82c",
"training_meta.json": "a731cf8f0c35ae8b41acc41f536a4b5c75a019bc5cd4591acf2a0f111b70d3fb"
}
},
"e9-pl-fix": {
"gates": {
"G1": {
"pass": true,
"shape_equal_apt4": true,
"layout_ok": true,
"max_piece_len": 16,
"ws_only_pieces": 0
},
"G4": {
"pass": true,
"n_lines": 10000,
"piece_mismatches": 0,
"decode_mismatches": 0,
"first_mismatch": null
},
"G5": {
"pass": true,
"expected_present": true,
"pieces_present": {
"U+00A0": true,
"U+202F": true,
"U+2212": true
}
}
},
"meta": {
"arm": "e9-pl-fix",
"config": {
"mix": {
"pl": 1073741824
},
"fix": true,
"ws_pieces": false
},
"spm_flags": {
"model_type": "bpe",
"vocab_size": 32000,
"character_coverage": 0.9995,
"normalization_rule_name": "identity",
"remove_extra_whitespaces": false,
"add_dummy_prefix": true,
"byte_fallback": true,
"split_digits": true,
"max_sentencepiece_length": 16,
"allow_whitespace_only_pieces": false,
"shuffle_input_sentence": false,
"input_sentence_size": 0,
"max_sentence_length": 8192,
"num_threads": 8,
"unk_id": 0,
"bos_id": 1,
"eos_id": 2,
"pad_id": -1,
"control_symbols": [
"<|im_start|>",
"<|im_end|>",
"",
""
],
"user_defined_symbols": [
"\u00a0",
"\u202f",
"\u2212"
]
},
"spm_version": "0.2.1",
"input_file": "data/arm_inputs/e9-pl-fix.txt",
"input_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8",
"input_sources": {
"pl": {
"bytes": 1073741824,
"source_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8"
}
},
"train_secs": 181.8,
"n_normal": 31734,
"n_merges": 60051,
"max_piece_len": 16,
"ws_only_pieces": 0
},
"files": {
"special_tokens_map.json": "96bdbb8504d9967606e5f661ccc7cbbac44a3661af863a7a58614670a0ccab33",
"spm.model": "49a2a0d8715397a0bbc7285eceab0fab61e42291dd2ac05f28f0c7371dea107e",
"spm.vocab": "59d0e6687faef75081e7fb6a9f8726b318ba777d7b7a99ef5feb652182e478e7",
"tokenizer.json": "23b1d93a66e5beccb197d55cdf6f500b57cc100c17f84114c32d525680f58d5e",
"tokenizer_config.json": "fa53599038cf703453fd2514f69938a5529eee2aa21b5e66f82fe33a75dcf82c",
"training_meta.json": "526a8f728d3646ef140beba8e48391b83bbd161f730698321f22788c8153c365"
}
},
"e9-sci": {
"gates": {
"G1": {
"pass": true,
"shape_equal_apt4": true,
"layout_ok": true,
"max_piece_len": 16,
"ws_only_pieces": 0
},
"G4": {
"pass": true,
"n_lines": 10000,
"piece_mismatches": 0,
"decode_mismatches": 0,
"first_mismatch": null
},
"G5": {
"pass": true,
"expected_present": true,
"pieces_present": {
"U+00A0": true,
"U+202F": true,
"U+2212": true
}
}
},
"meta": {
"arm": "e9-sci",
"config": {
"mix": {
"pl": 858993459,
"sci_latex": 71582788,
"sci_python": 71582788,
"sci_mathweb": 71582788
},
"fix": true,
"ws_pieces": false
},
"spm_flags": {
"model_type": "bpe",
"vocab_size": 32000,
"character_coverage": 0.9995,
"normalization_rule_name": "identity",
"remove_extra_whitespaces": false,
"add_dummy_prefix": true,
"byte_fallback": true,
"split_digits": true,
"max_sentencepiece_length": 16,
"allow_whitespace_only_pieces": false,
"shuffle_input_sentence": false,
"input_sentence_size": 0,
"max_sentence_length": 8192,
"num_threads": 8,
"unk_id": 0,
"bos_id": 1,
"eos_id": 2,
"pad_id": -1,
"control_symbols": [
"<|im_start|>",
"<|im_end|>",
"",
""
],
"user_defined_symbols": [
"\u00a0",
"\u202f",
"\u2212"
]
},
"spm_version": "0.2.1",
"input_file": "data/arm_inputs/e9-sci.txt",
"input_sha256": "849d247145f59ff22512fbc8d5c4e14db774bc12ec6d6f84d54dd751165c0edf",
"input_sources": {
"pl": {
"bytes": 858993459,
"source_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8"
},
"sci_latex": {
"bytes": 71582788,
"source_sha256": "baf588da429a519ed5c1aa9cda834a449e2c0f2a24af066f1152bb3c29b38c42"
},
"sci_python": {
"bytes": 71582788,
"source_sha256": "779923c8065875b4b3b53a10c8cb0c70cc3ce02b13a36470023146e3401d4966"
},
"sci_mathweb": {
"bytes": 71582788,
"source_sha256": "36f49b00f3d299a30a2cec673c36b9c7b7d4291d5b80c5f63ef2cdb91e9ed3e5"
}
},
"train_secs": 239.2,
"n_normal": 31734,
"n_merges": 64221,
"max_piece_len": 16,
"ws_only_pieces": 0
},
"files": {
"special_tokens_map.json": "96bdbb8504d9967606e5f661ccc7cbbac44a3661af863a7a58614670a0ccab33",
"spm.model": "2852497cba717a5af732252f5e2fda9d3cb8b4678c871c35eff7cc693e6a6705",
"spm.vocab": "90e84b0ccb977d1c72b62499b25e98371138c785e06a54e2c9780da5bf97bdef",
"tokenizer.json": "f93babbc4752a53ce56f6fee662943dbab026743536addead60dbd60cdb0fdfd",
"tokenizer_config.json": "fa53599038cf703453fd2514f69938a5529eee2aa21b5e66f82fe33a75dcf82c",
"training_meta.json": "22f441a88f061df84f32e15546291bbb2281adc74efebab832bbb9bd14434c9c"
}
},
"e9-sci-ws": {
"gates": {
"G1": {
"pass": true,
"shape_equal_apt4": true,
"layout_ok": true,
"max_piece_len": 16,
"ws_only_pieces": 13
},
"G4": {
"pass": true,
"n_lines": 10000,
"piece_mismatches": 0,
"decode_mismatches": 0,
"first_mismatch": null
},
"G5": {
"pass": true,
"expected_present": true,
"pieces_present": {
"U+00A0": true,
"U+202F": true,
"U+2212": true
}
}
},
"meta": {
"arm": "e9-sci-ws",
"config": {
"mix": {
"pl": 858993459,
"sci_latex": 71582788,
"sci_python": 71582788,
"sci_mathweb": 71582788
},
"fix": true,
"ws_pieces": true
},
"spm_flags": {
"model_type": "bpe",
"vocab_size": 32000,
"character_coverage": 0.9995,
"normalization_rule_name": "identity",
"remove_extra_whitespaces": false,
"add_dummy_prefix": true,
"byte_fallback": true,
"split_digits": true,
"max_sentencepiece_length": 16,
"allow_whitespace_only_pieces": true,
"shuffle_input_sentence": false,
"input_sentence_size": 0,
"max_sentence_length": 8192,
"num_threads": 8,
"unk_id": 0,
"bos_id": 1,
"eos_id": 2,
"pad_id": -1,
"control_symbols": [
"<|im_start|>",
"<|im_end|>",
"",
""
],
"user_defined_symbols": [
"\u00a0",
"\u202f",
"\u2212"
]
},
"spm_version": "0.2.1",
"input_file": "data/arm_inputs/e9-sci-ws.txt",
"input_sha256": "849d247145f59ff22512fbc8d5c4e14db774bc12ec6d6f84d54dd751165c0edf",
"input_sources": {
"pl": {
"bytes": 858993459,
"source_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8"
},
"sci_latex": {
"bytes": 71582788,
"source_sha256": "baf588da429a519ed5c1aa9cda834a449e2c0f2a24af066f1152bb3c29b38c42"
},
"sci_python": {
"bytes": 71582788,
"source_sha256": "779923c8065875b4b3b53a10c8cb0c70cc3ce02b13a36470023146e3401d4966"
},
"sci_mathweb": {
"bytes": 71582788,
"source_sha256": "36f49b00f3d299a30a2cec673c36b9c7b7d4291d5b80c5f63ef2cdb91e9ed3e5"
}
},
"train_secs": 248.3,
"n_normal": 31734,
"n_merges": 64314,
"max_piece_len": 16,
"ws_only_pieces": 13
},
"files": {
"special_tokens_map.json": "96bdbb8504d9967606e5f661ccc7cbbac44a3661af863a7a58614670a0ccab33",
"spm.model": "d8b7c9581fcbe01643f9d5efe31e28e74fe60dc2ddc1671955ea018f6c58a651",
"spm.vocab": "de8b524b3bc119ff16c6b2daafc678d053ff69617f0a41dd72b5e536e0959022",
"tokenizer.json": "2865e8236f0df659c48db8e63e9bd7dc484833ec03b50ad79be4a51f69414890",
"tokenizer_config.json": "fa53599038cf703453fd2514f69938a5529eee2aa21b5e66f82fe33a75dcf82c",
"training_meta.json": "62fa7f4df3b5983f64934e0b42f49bc2d81c22fc29cce2a6c16bfc428074594b"
}
},
"e9-bal": {
"gates": {
"G1": {
"pass": true,
"shape_equal_apt4": true,
"layout_ok": true,
"max_piece_len": 16,
"ws_only_pieces": 0
},
"G4": {
"pass": true,
"n_lines": 10000,
"piece_mismatches": 0,
"decode_mismatches": 0,
"first_mismatch": null
},
"G5": {
"pass": true,
"expected_present": true,
"pieces_present": {
"U+00A0": true,
"U+202F": true,
"U+2212": true
}
}
},
"meta": {
"arm": "e9-bal",
"config": {
"mix": {
"pl": 644245094,
"en_general": 214748365,
"sci_latex": 71582788,
"sci_python": 71582788,
"sci_mathweb": 71582788
},
"fix": true,
"ws_pieces": false
},
"spm_flags": {
"model_type": "bpe",
"vocab_size": 32000,
"character_coverage": 0.9995,
"normalization_rule_name": "identity",
"remove_extra_whitespaces": false,
"add_dummy_prefix": true,
"byte_fallback": true,
"split_digits": true,
"max_sentencepiece_length": 16,
"allow_whitespace_only_pieces": false,
"shuffle_input_sentence": false,
"input_sentence_size": 0,
"max_sentence_length": 8192,
"num_threads": 8,
"unk_id": 0,
"bos_id": 1,
"eos_id": 2,
"pad_id": -1,
"control_symbols": [
"<|im_start|>",
"<|im_end|>",
"",
""
],
"user_defined_symbols": [
"\u00a0",
"\u202f",
"\u2212"
]
},
"spm_version": "0.2.1",
"input_file": "data/arm_inputs/e9-bal.txt",
"input_sha256": "9cff8d2fe22b4495190028ca7bb24afb18a4f00510743d1e77e1e7d27d9970bd",
"input_sources": {
"pl": {
"bytes": 644245094,
"source_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8"
},
"en_general": {
"bytes": 214748365,
"source_sha256": "fae778ee956ef859ecc231b0c5264520af5ab6baad4fc3c9518c2e4d3744e4ed"
},
"sci_latex": {
"bytes": 71582788,
"source_sha256": "baf588da429a519ed5c1aa9cda834a449e2c0f2a24af066f1152bb3c29b38c42"
},
"sci_python": {
"bytes": 71582788,
"source_sha256": "779923c8065875b4b3b53a10c8cb0c70cc3ce02b13a36470023146e3401d4966"
},
"sci_mathweb": {
"bytes": 71582788,
"source_sha256": "36f49b00f3d299a30a2cec673c36b9c7b7d4291d5b80c5f63ef2cdb91e9ed3e5"
}
},
"train_secs": 260.6,
"n_normal": 31734,
"n_merges": 63620,
"max_piece_len": 16,
"ws_only_pieces": 0
},
"files": {
"special_tokens_map.json": "96bdbb8504d9967606e5f661ccc7cbbac44a3661af863a7a58614670a0ccab33",
"spm.model": "7eacfe79aae0140299afe2b958a4d0dfb7f6076fa48f4abef492d992dc907759",
"spm.vocab": "b59020b8ab7d354ebcb6573022158a84005cc9094155b4f1be242da0abac2600",
"tokenizer.json": "dd76a944cc776540c0a6f141148710d809781543ce6bf5184dc57a9e5007ff0b",
"tokenizer_config.json": "fa53599038cf703453fd2514f69938a5529eee2aa21b5e66f82fe33a75dcf82c",
"training_meta.json": "2b33dbbcc5bc04e575416c919485018934c0b2def3b26bdd68def49a1786e3b5"
}
},
"G3_determinism": {
"pass": true,
"model_content_identical": true,
"raw_model_bytes_identical": false,
"tokenizer_json_identical": true,
"note": "raw_model differs only in trainer_spec.model_prefix (the output path); content + tokenizer.json identical"
}
}