{ "dataset": { "source": "TIFs_PP2.csv", "n_transcripts_total": 7123, "n_transcripts_after_filtering": 6262, "filter": "transcripts with both an assigned 5'UTR and 3'UTR sequence", "target_regression": "TE3 (raw scale, not log-transformed)", "target_classification": "Low/Mid/High TE tertile-like category, quantile boundaries [0, 0.25, 0.75, 1] on TE3" }, "models": { "model1_regulatory_only": { "file": "model1_regulatory_only.pkl", "description": "Regulatory-features-only Random Forest classifier (Low/Mid/High TE). No sequence k-mer features.", "manuscript_figure": "Figure 3B (regulatory-feature-only comparison)", "task": "3-class classification", "feature_count": 48, "n_estimators": 350, "split": { "train_test_split": "60/20/20 (test_size=0.4 then 0.5, stratified)", "random_seed": 42, "split_index": 0 }, "preprocessing": "StandardScaler -> SimpleImputer(median) -> Kruskal-Wallis feature selection (p<0.5)", "preprocessing_note": "SimpleImputer(median) used in place of the original notebook's KNNImputer: verified NaN rate ~0.7%, so results are practically identical, and KNNImputer must retain the full training matrix to find neighbors at predict time (impractical file size for a small NaN rate).", "held_out_test_performance": { "accuracy": 0.6269968051118211, "roc_auc_macro_ovo": 0.7868590234325824, "f1_weighted": 0.6027746464439434 }, "manuscript_reported": "~0.80 ROC-AUC for this model type", "cv_documented": false, "cv_note": "Single canonical split reported (split 0); not independently re-verified across all 10 splits this pass." }, "model2_xgb_te_regressor": { "file": "model2_xgb_te_regressor.pkl", "description": "XGBoost regressor predicting continuous TE3 from combined regulatory + k-mer (both UTRs) features.", "manuscript_figure": "Figure 4B (TE regression, best-split result)", "task": "regression", "feature_count": 5440, "hyperparameters": { "n_estimators": 1200, "learning_rate": 0.12, "max_depth": 8, "gamma": 0.05 }, "split": { "train_test_split": "95/(~4.96)/(~0.04)% (test_size=0.05 then 0.0075, stratified)", "random_seed": 688, "split_index": 8 }, "preprocessing": "StandardScaler(X) -> Kruskal-Wallis feature selection (p<0.05) -> asymmetric y outlier trim (train_mean-1*std, train_mean+3*std) -> StandardScaler(y)", "outlier_trim": "keep (train_mean - 1*std, train_mean + 3*std) on raw y, using train stats for both train and test", "utr_scope": "5'UTR-only (3'UTR-pattern columns dropped from both regulatory numerics and k-mer matrix)", "held_out_test_performance": { "pearson_r": 0.6673802318198894 }, "manuscript_reported": "r=0.69 (paper's best-split result)", "reproduction_note": "Reproduced r=0.667 vs. paper's 0.694 on the identified split (split 8). Confirmed deterministic w.r.t. XGBoost random_state (30-seed sweep gave identical results); residual gap attributed to package-version differences between this environment and the original interactive session (Python 3.9/current sklearn+xgboost here vs. a base Python 3.12 conda env originally) rather than a methodological error - train-set correlation matches the original almost exactly (0.9926 vs 0.99273).", "cv_documented": false }, "model3_kmer_classifier": { "file": "model3_kmer_classifier.pkl", "description": "Pure k-mer (5'+3'UTR hexamer, 6 windows) Random Forest classifier. No regulatory numeric features.", "manuscript_figure": "Figure 3 (sequence-only k-mer model)", "task": "3-class classification", "feature_count": 3900, "n_estimators": 100, "split": { "train_test_split": "60/20/20 (test_size=0.4 then 0.5, stratified)", "random_seed": 42, "split_index": 0 }, "preprocessing": "StandardScaler -> SimpleImputer(median) -> Kruskal-Wallis feature selection (p<0.5)", "preprocessing_note": "SimpleImputer(median) in place of KNNImputer - verified 0% NaN in this feature set, so this is a complete no-op change.", "held_out_test_performance_canonical_split": { "accuracy": 0.6541533546325878, "roc_auc": 0.7933521488089769, "f1": 0.6454577396456138 }, "cv_documented": true, "cv_across_10_splits": { "accuracy_mean": 0.6476038338658148, "accuracy_std": 0.0133322530158756, "roc_auc_mean": 0.7859942515829837, "roc_auc_std": 0.00922746392626288, "f1_mean": 0.6337355311458344, "f1_std": 0.014257419994644476, "feature_count_range": [ 3811, 4168 ], "source_csv": "model3_cv_performance.csv" } }, "model4_bimodal_classifier": { "file": "model4_bimodal_classifier.pkl", "description": "Bi-modal Random Forest classifier: regulatory numeric features + k-mer (5'+3'UTR, 6 windows) features combined.", "manuscript_figure": "Figure 3/4 (bi-modal comparison)", "task": "3-class classification", "feature_count": 16074, "n_estimators": 100, "split": { "train_test_split": "60/20/20 (test_size=0.4 then 0.5, stratified)", "random_seed": 42, "split_index": 0 }, "preprocessing": "StandardScaler -> SimpleImputer(median) -> Kruskal-Wallis feature selection (p<0.5)", "retrained_fresh_note": "Retrained fresh under the current environment rather than restoring the original pickle, which requires sklearn 1.2.2 (incompatible with this repo's pinned 1.5.1).", "held_out_test_performance_canonical_split": { "accuracy": 0.6669329073482428, "roc_auc": 0.8271749226796231, "f1": 0.648983621067751 }, "cv_documented": true, "cv_across_10_splits": { "accuracy_mean": 0.6610223642172524, "accuracy_std": 0.012902101444166526, "roc_auc_mean": 0.8227317229599839, "roc_auc_std": 0.007009001730215918, "f1_mean": 0.6416703751501016, "f1_std": 0.013695454513195487, "feature_count_range": [ 15994, 16348 ], "source_csv": "model4_cv_performance.csv" } }, "model5_fig6_combo_classifier": { "file": "model5_fig6_combo/model5_fig6_combo_classifier.pkl", "auxiliary_files": { "selected_triple_features": "model5_fig6_combo/model5_selected_triple_features.pkl", "triple_fc_pvalues": "model5_fig6_combo/model5_triple_fc_pvalues.pkl" }, "description": "Random Forest classifier on 3-way multiplicative combinations of top-ranked 5'/3'UTR k-mers, used to rank Figure 6's motif-combination importances.", "manuscript_figure": "Figure 6 (motif combination arc-plot)", "task": "3-class classification", "feature_count": "n_features_in_ (varies by split, see split-specific selected-triple-features file)", "split": { "canonical_split_index": 1, "note": "Split 1 verified by tracing the source notebook's kernel execution order (exec_count), NOT split 0 or split 8, which were considered and ruled out during this analysis." }, "structural_sanity_check": { "n_estimators": 100, "n_features_in_matches_selected_list": true, "tree_depth_range": [ 42, 66 ], "tree_n_leaves_range": [ 609, 685 ], "feature_importances_sum": 1.0, "n_features_with_nonzero_importance": "2473/2477" }, "cv_documented": false, "cv_note": "Independent held-out CV reconstruction attempted and found NOT reliable: this model's 3-way multiplicative combination features are extremely sparse (a single test-set reconstruction attempt found ~7 nonzero cells out of ~7.6 million), so any small mismatch in the (partially externally-dependent, not fully recoverable) top-motif-pool selection or scaling pipeline collapses predictions to the majority class. Performance is reported as originally described in the manuscript (Figure 6 legend/Methods), not independently re-verified via held-out CV in this pass. See model5_cv_performance.csv for the attempted (unreliable) reconstruction, kept for transparency, not for citation.", "cv_attempted_but_unreliable": { "accuracy": 0.5, "note": "collapsed to majority-class prediction in every split - do not cite" }, "prediction_on_new_data_caveat": "This model cannot be straightforwardly applied to new sequences without reconstructing the exact top-motif pool used to build its 3-way combination features, which depended on external files not included in this repository. Provided for interpretability/reproducibility of the reported Figure 6 combination rankings, not for out-of-the-box prediction." } } }