{ "e9-pl": { "gates": { "G1": { "pass": true, "shape_equal_apt4": true, "layout_ok": true, "max_piece_len": 16, "ws_only_pieces": 0 }, "G4": { "pass": true, "n_lines": 10000, "piece_mismatches": 0, "decode_mismatches": 0, "first_mismatch": null }, "G5": { "pass": true, "expected_present": false, "pieces_present": { "U+00A0": false, "U+202F": false, "U+2212": false } } }, "meta": { "arm": "e9-pl", "config": { "mix": { "pl": 1073741824 }, "fix": false, "ws_pieces": false }, "spm_flags": { "model_type": "bpe", "vocab_size": 32000, "character_coverage": 0.9995, "normalization_rule_name": "identity", "remove_extra_whitespaces": false, "add_dummy_prefix": true, "byte_fallback": true, "split_digits": true, "max_sentencepiece_length": 16, "allow_whitespace_only_pieces": false, "shuffle_input_sentence": false, "input_sentence_size": 0, "max_sentence_length": 8192, "num_threads": 8, "unk_id": 0, "bos_id": 1, "eos_id": 2, "pad_id": -1, "control_symbols": [ "<|im_start|>", "<|im_end|>", "", "" ], "user_defined_symbols": [] }, "spm_version": "0.2.1", "input_file": "data/arm_inputs/e9-pl.txt", "input_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8", "input_sources": { "pl": { "bytes": 1073741824, "source_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8" } }, "train_secs": 183.2, "n_normal": 31737, "n_merges": 60058, "max_piece_len": 16, "ws_only_pieces": 0 }, "files": { "special_tokens_map.json": "96bdbb8504d9967606e5f661ccc7cbbac44a3661af863a7a58614670a0ccab33", "spm.model": "eac085b68090be7a048f8dfcb80b1ea8be5d267b4b758e064c417dfa0026f6bf", "spm.vocab": "25b8e27e518e086d54e944bb4bef87a8f04c4e5b676d1b4b91723b3a4049ef79", "tokenizer.json": "f22aca2a7ccc0ce04e821d1b6c14a3f79c20b3a268fc4ea444636dc255950acc", "tokenizer_config.json": "fa53599038cf703453fd2514f69938a5529eee2aa21b5e66f82fe33a75dcf82c", "training_meta.json": "a731cf8f0c35ae8b41acc41f536a4b5c75a019bc5cd4591acf2a0f111b70d3fb" } }, "e9-pl-fix": { "gates": { "G1": { "pass": true, "shape_equal_apt4": true, "layout_ok": true, "max_piece_len": 16, "ws_only_pieces": 0 }, "G4": { "pass": true, "n_lines": 10000, "piece_mismatches": 0, "decode_mismatches": 0, "first_mismatch": null }, "G5": { "pass": true, "expected_present": true, "pieces_present": { "U+00A0": true, "U+202F": true, "U+2212": true } } }, "meta": { "arm": "e9-pl-fix", "config": { "mix": { "pl": 1073741824 }, "fix": true, "ws_pieces": false }, "spm_flags": { "model_type": "bpe", "vocab_size": 32000, "character_coverage": 0.9995, "normalization_rule_name": "identity", "remove_extra_whitespaces": false, "add_dummy_prefix": true, "byte_fallback": true, "split_digits": true, "max_sentencepiece_length": 16, "allow_whitespace_only_pieces": false, "shuffle_input_sentence": false, "input_sentence_size": 0, "max_sentence_length": 8192, "num_threads": 8, "unk_id": 0, "bos_id": 1, "eos_id": 2, "pad_id": -1, "control_symbols": [ "<|im_start|>", "<|im_end|>", "", "" ], "user_defined_symbols": [ "\u00a0", "\u202f", "\u2212" ] }, "spm_version": "0.2.1", "input_file": "data/arm_inputs/e9-pl-fix.txt", "input_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8", "input_sources": { "pl": { "bytes": 1073741824, "source_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8" } }, "train_secs": 181.8, "n_normal": 31734, "n_merges": 60051, "max_piece_len": 16, "ws_only_pieces": 0 }, "files": { "special_tokens_map.json": "96bdbb8504d9967606e5f661ccc7cbbac44a3661af863a7a58614670a0ccab33", "spm.model": "49a2a0d8715397a0bbc7285eceab0fab61e42291dd2ac05f28f0c7371dea107e", "spm.vocab": "59d0e6687faef75081e7fb6a9f8726b318ba777d7b7a99ef5feb652182e478e7", "tokenizer.json": "23b1d93a66e5beccb197d55cdf6f500b57cc100c17f84114c32d525680f58d5e", "tokenizer_config.json": "fa53599038cf703453fd2514f69938a5529eee2aa21b5e66f82fe33a75dcf82c", "training_meta.json": "526a8f728d3646ef140beba8e48391b83bbd161f730698321f22788c8153c365" } }, "e9-sci": { "gates": { "G1": { "pass": true, "shape_equal_apt4": true, "layout_ok": true, "max_piece_len": 16, "ws_only_pieces": 0 }, "G4": { "pass": true, "n_lines": 10000, "piece_mismatches": 0, "decode_mismatches": 0, "first_mismatch": null }, "G5": { "pass": true, "expected_present": true, "pieces_present": { "U+00A0": true, "U+202F": true, "U+2212": true } } }, "meta": { "arm": "e9-sci", "config": { "mix": { "pl": 858993459, "sci_latex": 71582788, "sci_python": 71582788, "sci_mathweb": 71582788 }, "fix": true, "ws_pieces": false }, "spm_flags": { "model_type": "bpe", "vocab_size": 32000, "character_coverage": 0.9995, "normalization_rule_name": "identity", "remove_extra_whitespaces": false, "add_dummy_prefix": true, "byte_fallback": true, "split_digits": true, "max_sentencepiece_length": 16, "allow_whitespace_only_pieces": false, "shuffle_input_sentence": false, "input_sentence_size": 0, "max_sentence_length": 8192, "num_threads": 8, "unk_id": 0, "bos_id": 1, "eos_id": 2, "pad_id": -1, "control_symbols": [ "<|im_start|>", "<|im_end|>", "", "" ], "user_defined_symbols": [ "\u00a0", "\u202f", "\u2212" ] }, "spm_version": "0.2.1", "input_file": "data/arm_inputs/e9-sci.txt", "input_sha256": "849d247145f59ff22512fbc8d5c4e14db774bc12ec6d6f84d54dd751165c0edf", "input_sources": { "pl": { "bytes": 858993459, "source_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8" }, "sci_latex": { "bytes": 71582788, "source_sha256": "baf588da429a519ed5c1aa9cda834a449e2c0f2a24af066f1152bb3c29b38c42" }, "sci_python": { "bytes": 71582788, "source_sha256": "779923c8065875b4b3b53a10c8cb0c70cc3ce02b13a36470023146e3401d4966" }, "sci_mathweb": { "bytes": 71582788, "source_sha256": "36f49b00f3d299a30a2cec673c36b9c7b7d4291d5b80c5f63ef2cdb91e9ed3e5" } }, "train_secs": 239.2, "n_normal": 31734, "n_merges": 64221, "max_piece_len": 16, "ws_only_pieces": 0 }, "files": { "special_tokens_map.json": "96bdbb8504d9967606e5f661ccc7cbbac44a3661af863a7a58614670a0ccab33", "spm.model": "2852497cba717a5af732252f5e2fda9d3cb8b4678c871c35eff7cc693e6a6705", "spm.vocab": "90e84b0ccb977d1c72b62499b25e98371138c785e06a54e2c9780da5bf97bdef", "tokenizer.json": "f93babbc4752a53ce56f6fee662943dbab026743536addead60dbd60cdb0fdfd", "tokenizer_config.json": "fa53599038cf703453fd2514f69938a5529eee2aa21b5e66f82fe33a75dcf82c", "training_meta.json": "22f441a88f061df84f32e15546291bbb2281adc74efebab832bbb9bd14434c9c" } }, "e9-sci-ws": { "gates": { "G1": { "pass": true, "shape_equal_apt4": true, "layout_ok": true, "max_piece_len": 16, "ws_only_pieces": 13 }, "G4": { "pass": true, "n_lines": 10000, "piece_mismatches": 0, "decode_mismatches": 0, "first_mismatch": null }, "G5": { "pass": true, "expected_present": true, "pieces_present": { "U+00A0": true, "U+202F": true, "U+2212": true } } }, "meta": { "arm": "e9-sci-ws", "config": { "mix": { "pl": 858993459, "sci_latex": 71582788, "sci_python": 71582788, "sci_mathweb": 71582788 }, "fix": true, "ws_pieces": true }, "spm_flags": { "model_type": "bpe", "vocab_size": 32000, "character_coverage": 0.9995, "normalization_rule_name": "identity", "remove_extra_whitespaces": false, "add_dummy_prefix": true, "byte_fallback": true, "split_digits": true, "max_sentencepiece_length": 16, "allow_whitespace_only_pieces": true, "shuffle_input_sentence": false, "input_sentence_size": 0, "max_sentence_length": 8192, "num_threads": 8, "unk_id": 0, "bos_id": 1, "eos_id": 2, "pad_id": -1, "control_symbols": [ "<|im_start|>", "<|im_end|>", "", "" ], "user_defined_symbols": [ "\u00a0", "\u202f", "\u2212" ] }, "spm_version": "0.2.1", "input_file": "data/arm_inputs/e9-sci-ws.txt", "input_sha256": "849d247145f59ff22512fbc8d5c4e14db774bc12ec6d6f84d54dd751165c0edf", "input_sources": { "pl": { "bytes": 858993459, "source_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8" }, "sci_latex": { "bytes": 71582788, "source_sha256": "baf588da429a519ed5c1aa9cda834a449e2c0f2a24af066f1152bb3c29b38c42" }, "sci_python": { "bytes": 71582788, "source_sha256": "779923c8065875b4b3b53a10c8cb0c70cc3ce02b13a36470023146e3401d4966" }, "sci_mathweb": { "bytes": 71582788, "source_sha256": "36f49b00f3d299a30a2cec673c36b9c7b7d4291d5b80c5f63ef2cdb91e9ed3e5" } }, "train_secs": 248.3, "n_normal": 31734, "n_merges": 64314, "max_piece_len": 16, "ws_only_pieces": 13 }, "files": { "special_tokens_map.json": "96bdbb8504d9967606e5f661ccc7cbbac44a3661af863a7a58614670a0ccab33", "spm.model": "d8b7c9581fcbe01643f9d5efe31e28e74fe60dc2ddc1671955ea018f6c58a651", "spm.vocab": "de8b524b3bc119ff16c6b2daafc678d053ff69617f0a41dd72b5e536e0959022", "tokenizer.json": "2865e8236f0df659c48db8e63e9bd7dc484833ec03b50ad79be4a51f69414890", "tokenizer_config.json": "fa53599038cf703453fd2514f69938a5529eee2aa21b5e66f82fe33a75dcf82c", "training_meta.json": "62fa7f4df3b5983f64934e0b42f49bc2d81c22fc29cce2a6c16bfc428074594b" } }, "e9-bal": { "gates": { "G1": { "pass": true, "shape_equal_apt4": true, "layout_ok": true, "max_piece_len": 16, "ws_only_pieces": 0 }, "G4": { "pass": true, "n_lines": 10000, "piece_mismatches": 0, "decode_mismatches": 0, "first_mismatch": null }, "G5": { "pass": true, "expected_present": true, "pieces_present": { "U+00A0": true, "U+202F": true, "U+2212": true } } }, "meta": { "arm": "e9-bal", "config": { "mix": { "pl": 644245094, "en_general": 214748365, "sci_latex": 71582788, "sci_python": 71582788, "sci_mathweb": 71582788 }, "fix": true, "ws_pieces": false }, "spm_flags": { "model_type": "bpe", "vocab_size": 32000, "character_coverage": 0.9995, "normalization_rule_name": "identity", "remove_extra_whitespaces": false, "add_dummy_prefix": true, "byte_fallback": true, "split_digits": true, "max_sentencepiece_length": 16, "allow_whitespace_only_pieces": false, "shuffle_input_sentence": false, "input_sentence_size": 0, "max_sentence_length": 8192, "num_threads": 8, "unk_id": 0, "bos_id": 1, "eos_id": 2, "pad_id": -1, "control_symbols": [ "<|im_start|>", "<|im_end|>", "", "" ], "user_defined_symbols": [ "\u00a0", "\u202f", "\u2212" ] }, "spm_version": "0.2.1", "input_file": "data/arm_inputs/e9-bal.txt", "input_sha256": "9cff8d2fe22b4495190028ca7bb24afb18a4f00510743d1e77e1e7d27d9970bd", "input_sources": { "pl": { "bytes": 644245094, "source_sha256": "5bd65b432c77180e3b05027d271bfb84c32a7891f971c029dc1d55ec338284b8" }, "en_general": { "bytes": 214748365, "source_sha256": "fae778ee956ef859ecc231b0c5264520af5ab6baad4fc3c9518c2e4d3744e4ed" }, "sci_latex": { "bytes": 71582788, "source_sha256": "baf588da429a519ed5c1aa9cda834a449e2c0f2a24af066f1152bb3c29b38c42" }, "sci_python": { "bytes": 71582788, "source_sha256": "779923c8065875b4b3b53a10c8cb0c70cc3ce02b13a36470023146e3401d4966" }, "sci_mathweb": { "bytes": 71582788, "source_sha256": "36f49b00f3d299a30a2cec673c36b9c7b7d4291d5b80c5f63ef2cdb91e9ed3e5" } }, "train_secs": 260.6, "n_normal": 31734, "n_merges": 63620, "max_piece_len": 16, "ws_only_pieces": 0 }, "files": { "special_tokens_map.json": "96bdbb8504d9967606e5f661ccc7cbbac44a3661af863a7a58614670a0ccab33", "spm.model": "7eacfe79aae0140299afe2b958a4d0dfb7f6076fa48f4abef492d992dc907759", "spm.vocab": "b59020b8ab7d354ebcb6573022158a84005cc9094155b4f1be242da0abac2600", "tokenizer.json": "dd76a944cc776540c0a6f141148710d809781543ce6bf5184dc57a9e5007ff0b", "tokenizer_config.json": "fa53599038cf703453fd2514f69938a5529eee2aa21b5e66f82fe33a75dcf82c", "training_meta.json": "2b33dbbcc5bc04e575416c919485018934c0b2def3b26bdd68def49a1786e3b5" } }, "G3_determinism": { "pass": true, "model_content_identical": true, "raw_model_bytes_identical": false, "tokenizer_json_identical": true, "note": "raw_model differs only in trainer_spec.model_prefix (the output path); content + tokenizer.json identical" } }