mod common; use std::fs; use std::os::unix::fs::MetadataExt; use crate::common::*; use glean_core::metrics::*; use glean_core::CommonMetricData; use glean_core::Lifetime; use rkv::{Rkv, StoreOptions}; use uuid::uuid; static RKV_DATABASE: &[u8] = include_bytes!("77ca0472-5124-4f6b-971d-4a2a928fb158.safe.bin"); // This database is based on a submitted `metrics` ping from a real client. // Only metrics for that ping and a bare minimum of client info data is added, // a total of 61 metrics. // This is realistic enough to cause size differences in the non-vacuumed/vacuumed SQLite // databases after migration. static FILLED_RKV_DATABASE: &[u8] = include_bytes!("filled-rkv.data.safe.bin"); fn clientid_metric() -> UuidMetric { UuidMetric::new(CommonMetricData { name: "client_id".into(), category: "".into(), send_in_pings: vec!["glean_client_info".into()], lifetime: Lifetime::User, ..Default::default() }) } /// Copy of parts of `DatabaseMetrics` in `glean-core/src/internal_metrics.rs` for ease of testing. struct MigrationMetrics { metrics_in_sqlite: CounterMetric, migrated_metrics: CounterMetric, failed_metrics: CounterMetric, migration_duration: TimingDistributionMetric, migration_error: CounterMetric, } impl MigrationMetrics { fn new() -> Self { Self { migrated_metrics: CounterMetric::new(CommonMetricData { name: "migrated_metrics".into(), category: "glean.migration".into(), send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, ..Default::default() }), metrics_in_sqlite: CounterMetric::new(CommonMetricData { name: "metrics_in_sqlite".into(), category: "glean.migration".into(), send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, ..Default::default() }), failed_metrics: CounterMetric::new(CommonMetricData { name: "failed_metrics".into(), category: "glean.migration".into(), send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, ..Default::default() }), migration_duration: TimingDistributionMetric::new( CommonMetricData { name: "migration_duration".into(), category: "glean.migration".into(), send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, ..Default::default() }, TimeUnit::Millisecond, ), migration_error: CounterMetric::new(CommonMetricData { name: "error".into(), category: "glean.migration".into(), send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, ..Default::default() }), } } } #[test] fn migration_succeeds() { let temp = tempfile::tempdir().unwrap(); let db_path = temp.path().join("db"); fs::create_dir_all(&db_path).unwrap(); let safe_bin = db_path.join("data.safe.bin"); // File has been generated from essentially: // // ```rust // let tmpname = PathBuf::new("/tmp/glean-fc"); // let cfg = ConfigurationBuilder::new(true, tmpname.clone(), "glean-fc") // .with_server_endpoint("invalid-test-host") // .with_use_core_mps(false) // .build(); // glean::initialize(cfg, client_info); // glean::shutdown(); // ``` // // All ping-specific metrics have been removed. // Only client_info metrics are migrated, including the client ID. fs::write(safe_bin, RKV_DATABASE).unwrap(); let exp_client_id = uuid!("77ca0472-5124-4f6b-971d-4a2a928fb158"); let (glean, _temp) = new_glean(Some(temp)); let client_id = clientid_metric().get_value(&glean, None).unwrap(); assert_eq!(exp_client_id, client_id); let metrics = MigrationMetrics::new(); assert_eq!(Some(13), metrics.migrated_metrics.get_value(&glean, None)); assert_eq!(Some(13), metrics.metrics_in_sqlite.get_value(&glean, None)); assert_eq!(None, metrics.failed_metrics.get_value(&glean, None)); assert!(metrics.migration_duration.get_value(&glean, None).is_some()); assert_eq!(None, metrics.migration_error.get_value(&glean, None)); } #[test] fn migration_skipped_if_database_exists() { let (first_client_id, temp) = { let (glean, temp) = new_glean(None); let client_id = clientid_metric().get_value(&glean, None).unwrap(); drop(glean); (client_id, temp) }; let safe_bin = temp.path().join("db").join("data.safe.bin"); fs::write( &safe_bin, include_bytes!("77ca0472-5124-4f6b-971d-4a2a928fb158.safe.bin"), ) .unwrap(); let rkv_client_id = uuid!("77ca0472-5124-4f6b-971d-4a2a928fb158"); let (glean, _temp) = new_glean(Some(temp)); let client_id = clientid_metric().get_value(&glean, None).unwrap(); assert_eq!( first_client_id, client_id, "Client ID should be the one first generated" ); assert_ne!( rkv_client_id, client_id, "Client ID should not be one from the Rkv database" ); assert!(safe_bin.exists(), "Rkv file should not have been deleted"); let metrics = MigrationMetrics::new(); assert_eq!(None, metrics.migrated_metrics.get_value(&glean, None)); assert_eq!(None, metrics.metrics_in_sqlite.get_value(&glean, None)); assert_eq!(None, metrics.failed_metrics.get_value(&glean, None)); assert_eq!(None, metrics.migration_duration.get_value(&glean, None)); assert_eq!(None, metrics.migration_error.get_value(&glean, None)); } #[test] fn migration_succeeds_with_failures() { let temp = tempfile::tempdir().unwrap(); let db_path = temp.path().join("db"); fs::create_dir_all(&db_path).unwrap(); let safe_bin = db_path.join("data.safe.bin"); // Reusing the same database file from above. fs::write(safe_bin, RKV_DATABASE).unwrap(); let exp_client_id = uuid!("77ca0472-5124-4f6b-971d-4a2a928fb158"); // Modifying the database to force migration errors. { let rkv = Rkv::new::(&db_path).unwrap(); let ping_store = rkv .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) .unwrap(); let mut writer = rkv.write().unwrap(); let key = "metrics#a.broken.metric"; let value = rkv::Value::Blob(b"not a value"); ping_store.put(&mut writer, key, &value).unwrap(); let key = "baseline#second.broken.metric"; let value = rkv::Value::I64(31); ping_store.put(&mut writer, key, &value).unwrap(); writer.commit().unwrap(); } let (glean, _temp) = new_glean(Some(temp)); let client_id = clientid_metric().get_value(&glean, None).unwrap(); assert_eq!(exp_client_id, client_id); let metrics = MigrationMetrics::new(); assert_eq!(Some(13), metrics.migrated_metrics.get_value(&glean, None)); assert_eq!(Some(13), metrics.metrics_in_sqlite.get_value(&glean, None)); // We injected 2 broken metrics. assert_eq!(Some(2), metrics.failed_metrics.get_value(&glean, None)); assert!(metrics.migration_duration.get_value(&glean, None).is_some()); assert_eq!(None, metrics.migration_error.get_value(&glean, None)); } #[test] fn migration_fails() { let temp = tempfile::tempdir().unwrap(); let db_path = temp.path().join("db"); fs::create_dir_all(&db_path).unwrap(); let safe_bin = db_path.join("data.safe.bin"); fs::write(safe_bin, b"\0").unwrap(); let (glean, _temp) = new_glean(Some(temp)); let metrics = MigrationMetrics::new(); assert_eq!(Some(1), metrics.migration_error.get_value(&glean, None)); assert_eq!(None, metrics.migrated_metrics.get_value(&glean, None)); assert_eq!(None, metrics.metrics_in_sqlite.get_value(&glean, None)); assert_eq!(None, metrics.failed_metrics.get_value(&glean, None)); assert_eq!(None, metrics.migration_duration.get_value(&glean, None)); } #[test] fn migration_checkpoints() { let temp = tempfile::tempdir().unwrap(); let db_path = temp.path().join("db"); fs::create_dir_all(&db_path).unwrap(); let safe_bin = db_path.join("data.safe.bin"); // Reusing the same database file from above. fs::write(safe_bin, FILLED_RKV_DATABASE).unwrap(); let exp_client_id = uuid!("3114d9df-9ae3-43a7-83b0-3540c3eba886"); let (glean, _temp) = new_glean(Some(temp)); let client_id = clientid_metric().get_value(&glean, None).unwrap(); assert_eq!(exp_client_id, client_id); let metrics = MigrationMetrics::new(); assert_eq!(Some(61), metrics.migrated_metrics.get_value(&glean, None)); assert_eq!(Some(61), metrics.metrics_in_sqlite.get_value(&glean, None)); assert!(metrics.migration_duration.get_value(&glean, None).is_some()); assert_eq!(None, metrics.migration_error.get_value(&glean, None)); // Ensure we close the database connection. drop(glean); let db_file = db_path.join("glean.sqlite"); let db_file_size = fs::metadata(db_file).unwrap().size(); // This test is very vague, but it's hard to do better right now. // // Unvacuumed the database is _smaller_, around 20k bytes, because the migrated data is in the WAL file. // Vacuumed & checkpointed the WAL transactions are merged into the database. // As of writing that database is at least 9 pages big (9 * 4096 bytes = 36864 bytes). // This might grow if we add more metrics. // This might shrink if we remove metrics, in which case this test will break and needs adjustement. let vacuumed_database_size = 36864; assert!(db_file_size >= vacuumed_database_size); } #[test] fn migration_reapplied_after_not_finishing() { let temp = { let (glean, temp) = new_glean(None); drop(glean); let db_path = temp.path().join("db").join("glean.sqlite"); let conn = rusqlite::Connection::open(db_path).unwrap(); // Let's start with an empty database. conn.execute("DELETE FROM telemetry", []).unwrap(); // Ensure migration isn't marked as done. conn.execute("DELETE FROM migration", []).unwrap(); temp }; let db_path = temp.path().join("db"); let safe_bin = db_path.join("data.safe.bin"); // Reusing the same database file from above. fs::write(safe_bin, RKV_DATABASE).unwrap(); let exp_client_id = uuid!("77ca0472-5124-4f6b-971d-4a2a928fb158"); let (glean, _temp) = new_glean(Some(temp)); let client_id = clientid_metric().get_value(&glean, None).unwrap(); assert_eq!(exp_client_id, client_id); let metrics = MigrationMetrics::new(); assert_eq!(Some(13), metrics.migrated_metrics.get_value(&glean, None)); assert_eq!(Some(13), metrics.metrics_in_sqlite.get_value(&glean, None)); assert_eq!(None, metrics.failed_metrics.get_value(&glean, None)); assert!(metrics.migration_duration.get_value(&glean, None).is_some()); assert_eq!(None, metrics.migration_error.get_value(&glean, None)); } #[test] fn migration_not_reapplied_if_marked_as_finished() { let (first_client_id, temp) = { let (glean, temp) = new_glean(None); let client_id = clientid_metric().get_value(&glean, None).unwrap(); drop(glean); let db_path = temp.path().join("db").join("glean.sqlite"); let conn = rusqlite::Connection::open(db_path).unwrap(); // Ensure migration is marked as done. conn.execute("DELETE FROM migration", []).unwrap(); conn.execute("INSERT INTO migration (id, state) VALUES (1, 'done')", []) .unwrap(); (client_id, temp) }; let db_path = temp.path().join("db"); let safe_bin = db_path.join("data.safe.bin"); // Reusing the same database file from above. fs::write(safe_bin, RKV_DATABASE).unwrap(); let rkv_client_id = uuid!("77ca0472-5124-4f6b-971d-4a2a928fb158"); let (glean, _temp) = new_glean(Some(temp)); let client_id = clientid_metric().get_value(&glean, None).unwrap(); assert_ne!(rkv_client_id, client_id); assert_eq!(first_client_id, client_id); // No migration happened. let metrics = MigrationMetrics::new(); assert_eq!(None, metrics.migrated_metrics.get_value(&glean, None)); assert_eq!(None, metrics.metrics_in_sqlite.get_value(&glean, None)); assert_eq!(None, metrics.failed_metrics.get_value(&glean, None)); assert_eq!(None, metrics.migration_duration.get_value(&glean, None)); assert_eq!(None, metrics.migration_error.get_value(&glean, None)); }