{ "convention": "E2 04_audit.py: carrier 'a '+s, no specials, span intersection", "tokenizers": { "e9-pl": { "vocab_digit_profile": { "n_digit_tokens": 10, "n_multi_digit_tokens": 0, "max_digit_run_in_vocab": 1 }, "classification": { "policy_class": "single-digit", "frac_single_digit_segmentation": 1.0, "frac_matches_3chunk_l2r": 0.0, "frac_matches_3chunk_r2l": 0.0, "place_value_boundary_coverage": 1.0, "n_classified": 5000 }, "sep_tpn_ge4digit": { "bare_ge4": 7.6392, "en_ge4": 9.0303, "pl_space_ge4": 9.0303, "pl_nbsp_ge4": 10.4214, "pl_nnbsp_ge4": 11.8124, "de_ge4": 9.0303, "bare_int_ge4": 6.966 }, "separator_char_tokens": { "NBSP": 2, "NNBSP": 3, "MINUS": 3, "space": 1, "comma": 1 }, "u2212_minus_tokens": { "s7-1-u2212": 4 }, "G2": { "pass": true } }, "e9-pl-fix": { "vocab_digit_profile": { "n_digit_tokens": 10, "n_multi_digit_tokens": 0, "max_digit_run_in_vocab": 1 }, "classification": { "policy_class": "single-digit", "frac_single_digit_segmentation": 1.0, "frac_matches_3chunk_l2r": 0.0, "frac_matches_3chunk_r2l": 0.0, "place_value_boundary_coverage": 1.0, "n_classified": 5000 }, "sep_tpn_ge4digit": { "bare_ge4": 7.6392, "en_ge4": 9.0303, "pl_space_ge4": 9.0303, "pl_nbsp_ge4": 9.0303, "pl_nnbsp_ge4": 9.0303, "de_ge4": 9.0303, "bare_int_ge4": 6.966 }, "separator_char_tokens": { "NBSP": 1, "NNBSP": 1, "MINUS": 1, "space": 1, "comma": 1 }, "u2212_minus_tokens": { "s7-1-u2212": 2 }, "G2": { "pass": true } }, "e9-sci": { "vocab_digit_profile": { "n_digit_tokens": 10, "n_multi_digit_tokens": 0, "max_digit_run_in_vocab": 1 }, "classification": { "policy_class": "single-digit", "frac_single_digit_segmentation": 1.0, "frac_matches_3chunk_l2r": 0.0, "frac_matches_3chunk_r2l": 0.0, "place_value_boundary_coverage": 1.0, "n_classified": 5000 }, "sep_tpn_ge4digit": { "bare_ge4": 7.6392, "en_ge4": 9.0303, "pl_space_ge4": 9.0303, "pl_nbsp_ge4": 9.0303, "pl_nnbsp_ge4": 9.0303, "de_ge4": 9.0303, "bare_int_ge4": 6.966 }, "separator_char_tokens": { "NBSP": 1, "NNBSP": 1, "MINUS": 1, "space": 1, "comma": 1 }, "u2212_minus_tokens": { "s7-1-u2212": 2 }, "G2": { "pass": true } }, "e9-sci-ws": { "vocab_digit_profile": { "n_digit_tokens": 10, "n_multi_digit_tokens": 0, "max_digit_run_in_vocab": 1 }, "classification": { "policy_class": "single-digit", "frac_single_digit_segmentation": 1.0, "frac_matches_3chunk_l2r": 0.0, "frac_matches_3chunk_r2l": 0.0, "place_value_boundary_coverage": 1.0, "n_classified": 5000 }, "sep_tpn_ge4digit": { "bare_ge4": 7.6392, "en_ge4": 9.0303, "pl_space_ge4": 9.0303, "pl_nbsp_ge4": 9.0303, "pl_nnbsp_ge4": 9.0303, "de_ge4": 9.0303, "bare_int_ge4": 6.966 }, "separator_char_tokens": { "NBSP": 1, "NNBSP": 1, "MINUS": 1, "space": 1, "comma": 1 }, "u2212_minus_tokens": { "s7-1-u2212": 2 }, "G2": { "pass": true } }, "e9-bal": { "vocab_digit_profile": { "n_digit_tokens": 10, "n_multi_digit_tokens": 0, "max_digit_run_in_vocab": 1 }, "classification": { "policy_class": "single-digit", "frac_single_digit_segmentation": 1.0, "frac_matches_3chunk_l2r": 0.0, "frac_matches_3chunk_r2l": 0.0, "place_value_boundary_coverage": 1.0, "n_classified": 5000 }, "sep_tpn_ge4digit": { "bare_ge4": 7.6392, "en_ge4": 9.0303, "pl_space_ge4": 9.0303, "pl_nbsp_ge4": 9.0303, "pl_nnbsp_ge4": 9.0303, "de_ge4": 9.0303, "bare_int_ge4": 6.966 }, "separator_char_tokens": { "NBSP": 1, "NNBSP": 1, "MINUS": 1, "space": 1, "comma": 1 }, "u2212_minus_tokens": { "s7-1-u2212": 2 }, "G2": { "pass": true } }, "apt4": { "vocab_digit_profile": { "n_digit_tokens": 10, "n_multi_digit_tokens": 0, "max_digit_run_in_vocab": 1 }, "classification": { "policy_class": "single-digit", "frac_single_digit_segmentation": 1.0, "frac_matches_3chunk_l2r": 0.0, "frac_matches_3chunk_r2l": 0.0, "place_value_boundary_coverage": 1.0, "n_classified": 5000 }, "sep_tpn_ge4digit": { "bare_ge4": 7.6392, "en_ge4": 9.0303, "pl_space_ge4": 9.0303, "pl_nbsp_ge4": 10.4214, "pl_nnbsp_ge4": 11.8124, "de_ge4": 9.0303, "bare_int_ge4": 6.966 }, "separator_char_tokens": { "NBSP": 2, "NNBSP": 3, "MINUS": 3, "space": 1, "comma": 1 }, "u2212_minus_tokens": { "s7-1-u2212": 4 } }, "bielik-v3-original": { "vocab_digit_profile": { "n_digit_tokens": 19, "n_multi_digit_tokens": 0, "max_digit_run_in_vocab": 1 }, "classification": { "policy_class": "single-digit", "frac_single_digit_segmentation": 1.0, "frac_matches_3chunk_l2r": 0.0, "frac_matches_3chunk_r2l": 0.0, "place_value_boundary_coverage": 1.0, "n_classified": 5000 }, "sep_tpn_ge4digit": { "bare_ge4": 7.6392, "en_ge4": 9.0303, "pl_space_ge4": 9.0303, "pl_nbsp_ge4": 9.0303, "pl_nnbsp_ge4": 11.8124, "de_ge4": 9.0303, "bare_int_ge4": 6.966 }, "separator_char_tokens": { "NBSP": 1, "NNBSP": 3, "MINUS": 1, "space": 1, "comma": 1 }, "u2212_minus_tokens": { "s7-1-u2212": 2 } } }, "e2_crosscheck": { "apt4": { "classification": true, "vocab_profile": true, "tokens_per_number": true }, "bielik-v3-original": { "classification": true, "vocab_profile": true, "tokens_per_number": true } } }