/* * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ //! Structs to handle actions that mutate the History DB //! //! Places DB operations are complex and often involve several layers of changes which makes them //! hard to test. For example, a function that deletes visits in a date range needs to: //! - Calculate which visits to delete //! - Delete the visits //! - Insert visit tombstones //! - Update the frecency for non-orphaned affected pages //! - Delete orphaned pages //! - Insert page tombstones for deleted synced pages //! //! Test all of this functionality at once leads to ugly tests that are hard to reason about and //! hard to change. This is especially true since many steps have multiple branches which //! multiplies the complexity. //! //! This module is intended to split up operations to make testing simpler. It defines an enum //! whose variants encapsulate particular actions. We can use that enum to split operations into //! multiple parts, each which can be tested separately: code that calculates which actions to run //! and the code to run each action. //! //! Right now, only a couple function use this system, but hopefully we can use it more in the //! future. use super::{cleanup_pages, PageToClean}; use crate::error::Result; use crate::{PlacesDb, RowId}; use rusqlite::Row; use sql_support::ConnExt; use std::collections::HashSet; /// Enum whose variants describe a particular action on the DB #[derive(Debug, PartialEq, Eq)] pub(super) enum DbAction { /// Delete visit rows from the DB. DeleteVisitRows { visit_ids: HashSet }, /// Recalculate the moz_places data, including frecency, after changes to their visits. This /// also deletes orphaned pages (pages whose visits have all been deleted). RecalcPages { page_ids: HashSet }, /// Delete rows in pending temp tables. This should be done after any changes to the /// moz_places table. /// /// Deleting from these tables triggers changes to the `moz_origins` table. See /// `sql/create_shared_temp_tables.sql` and `sql/create_shared_triggers.sql` for details. DeleteFromPendingTempTables, } impl DbAction { pub(super) fn apply(self, db: &PlacesDb) -> Result<()> { match self { Self::DeleteVisitRows { visit_ids } => Self::delete_visit_rows(db, visit_ids), Self::RecalcPages { page_ids } => Self::recalc_pages(db, page_ids), Self::DeleteFromPendingTempTables => Self::delete_from_pending_temp_tables(db), } } pub(super) fn apply_all(db: &PlacesDb, actions: Vec) -> Result<()> { for action in actions { action.apply(db)?; } Ok(()) } fn delete_visit_rows(db: &PlacesDb, visit_ids: HashSet) -> Result<()> { sql_support::each_chunk(&Vec::from_iter(visit_ids), |chunk, _| -> Result<()> { let var_repeat = sql_support::repeat_sql_vars(chunk.len()); let params = rusqlite::params_from_iter(chunk); db.execute_cached( &format!( " INSERT OR IGNORE INTO moz_historyvisit_tombstones(place_id, visit_date) SELECT place_id, visit_date FROM moz_historyvisits WHERE id IN ({}) ", var_repeat, ), params.clone(), )?; db.execute_cached( &format!("DELETE FROM moz_historyvisits WHERE id IN ({})", var_repeat), params, )?; Ok(()) })?; Ok(()) } fn recalc_pages(db: &PlacesDb, page_ids: HashSet) -> Result<()> { let mut pages_to_clean: Vec = vec![]; sql_support::each_chunk(&Vec::from_iter(page_ids), |chunk, _| -> Result<()> { pages_to_clean.append(&mut db.query_rows_and_then_cached( &format!( "SELECT id, (foreign_count != 0) AS has_foreign, ((last_visit_date_local + last_visit_date_remote) != 0) as has_visits, sync_status FROM moz_places WHERE id IN ({})", sql_support::repeat_sql_vars(chunk.len()) ), rusqlite::params_from_iter(chunk), PageToClean::from_row, )?); Ok(()) })?; cleanup_pages(db, &pages_to_clean)?; Ok(()) } fn delete_from_pending_temp_tables(db: &PlacesDb) -> Result<()> { crate::storage::delete_pending_temp_tables(db) } } /// Stores a visit that we want to delete /// /// We build a Vec of these from queries against the `moz_historyvisits` table, then transform that /// into a `Vec`. #[derive(Debug, PartialEq, Eq, Hash)] pub(super) struct VisitToDelete { pub(super) visit_id: RowId, pub(super) page_id: RowId, } impl VisitToDelete { /// Create a VisitToDelete from a query row /// /// The query must that includes the `id` and `place_id` columns from `moz_historyvisits`. pub(super) fn from_row(row: &Row<'_>) -> Result { Ok(Self { visit_id: row.get("id")?, page_id: row.get("place_id")?, }) } } /// Create a Vec from a Vec pub(super) fn db_actions_from_visits_to_delete( visits_to_delete: Vec, ) -> Vec { let mut visit_ids = HashSet::::new(); let mut page_ids = HashSet::::new(); for visit_to_delete in visits_to_delete.into_iter() { visit_ids.insert(visit_to_delete.visit_id); page_ids.insert(visit_to_delete.page_id); } vec![ DbAction::DeleteVisitRows { visit_ids }, DbAction::RecalcPages { page_ids }, DbAction::DeleteFromPendingTempTables, ] } #[cfg(test)] mod tests { use super::*; use crate::observation::VisitObservation; use crate::storage::bookmarks::*; use crate::storage::history::apply_observation; use crate::types::VisitType; use crate::{frecency, ConnectionType, SyncStatus}; use rusqlite::params; use rusqlite::types::{FromSql, ToSql}; use std::time::Duration; use sync_guid::Guid; use types::Timestamp; use url::Url; fn query_vec(conn: &PlacesDb, sql: &str, params: &[&dyn ToSql]) -> Vec { conn.prepare(sql) .unwrap() .query_map(params, |row| row.get(0)) .unwrap() .collect::>>() .unwrap() } fn query_vec_pairs( conn: &PlacesDb, sql: &str, params: &[&dyn ToSql], ) -> Vec<(T, V)> { conn.prepare(sql) .unwrap() .query_map(params, |row| Ok((row.get(0)?, row.get(1)?))) .unwrap() .collect::>>() .unwrap() } fn query_visit_ids(conn: &PlacesDb) -> Vec { query_vec(conn, "SELECT id FROM moz_historyvisits ORDER BY id", &[]) } fn query_visit_tombstones(conn: &PlacesDb) -> Vec<(RowId, Timestamp)> { query_vec_pairs( conn, " SELECT place_id, visit_date FROM moz_historyvisit_tombstones ORDER BY place_id, visit_date ", &[], ) } fn query_page_ids(conn: &PlacesDb) -> Vec { query_vec(conn, "SELECT id FROM moz_places ORDER BY id", &[]) } fn query_page_tombstones(conn: &PlacesDb) -> Vec { query_vec( conn, "SELECT guid FROM moz_places_tombstones ORDER BY guid", &[], ) } struct TestPage { id: RowId, guid: Guid, url: Url, visit_ids: Vec, visit_dates: Vec, } impl TestPage { fn new(conn: &mut PlacesDb, url: &str, visit_dates: &[Timestamp]) -> Self { let url = Url::parse(url).unwrap(); let mut visit_ids = vec![]; for date in visit_dates { visit_ids.push( apply_observation( conn, VisitObservation::new(url.clone()) .with_visit_type(VisitType::Link) .with_at(*date), ) .unwrap() .unwrap(), ); } let (id, guid) = conn .query_row( " SELECT p.id, p.guid FROM moz_places p JOIN moz_historyvisits v ON p.id = v.place_id WHERE v.id = ?", [visit_ids[0]], |row| Ok((row.get(0)?, row.get(1)?)), ) .unwrap(); Self { id, guid, visit_ids, url, visit_dates: Vec::from_iter(visit_dates.iter().cloned()), } } fn set_sync_status(&self, conn: &PlacesDb, sync_status: SyncStatus) { conn.execute( "UPDATE moz_places SET sync_status = ? WHERE id = ?", params! {sync_status, self.id }, ) .unwrap(); } fn query_frecency(&self, conn: &PlacesDb) -> i32 { conn.query_row( "SELECT frecency FROM moz_places WHERE id = ?", [self.id], |row| row.get::(0), ) .unwrap() } fn calculate_frecency(&self, conn: &PlacesDb) -> i32 { frecency::calculate_frecency( conn, &frecency::DEFAULT_FRECENCY_SETTINGS, self.id.0, None, ) .unwrap() } fn bookmark(&self, conn: &PlacesDb, title: &str) { insert_bookmark( conn, InsertableBookmark { parent_guid: BookmarkRootGuid::Unfiled.into(), position: BookmarkPosition::Append, date_added: None, last_modified: None, guid: None, url: self.url.clone(), title: Some(title.to_owned()), } .into(), ) .unwrap(); } } #[test] fn test_delete_visit_rows() { let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap(); let yesterday = Timestamp::now() .checked_sub(Duration::from_secs(60 * 60 * 24)) .unwrap(); let page = TestPage::new( &mut conn, "http://example.com/", &[ Timestamp(yesterday.0 + 100), Timestamp(yesterday.0 + 200), Timestamp(yesterday.0 + 300), ], ); DbAction::DeleteVisitRows { visit_ids: HashSet::from_iter([page.visit_ids[0], page.visit_ids[1]]), } .apply(&conn) .unwrap(); assert_eq!(query_visit_ids(&conn), vec![page.visit_ids[2]]); assert_eq!( query_visit_tombstones(&conn), vec![ (page.id, page.visit_dates[0]), (page.id, page.visit_dates[1]), ] ); } #[test] fn test_recalc_pages() { let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap(); let yesterday = Timestamp::now() .checked_sub(Duration::from_secs(60 * 60 * 24)) .unwrap(); let page_with_visits_left = TestPage::new( &mut conn, "http://example.com/1", &[Timestamp(yesterday.0 + 100), Timestamp(yesterday.0 + 200)], ); let page_with_no_visits_unsynced = TestPage::new( &mut conn, "http://example.com/2", &[Timestamp(yesterday.0 + 300)], ); let page_with_no_visits_synced = TestPage::new( &mut conn, "http://example.com/2", &[Timestamp(yesterday.0 + 400)], ); let page_with_no_visits_bookmarked = TestPage::new( &mut conn, "http://example.com/3", &[Timestamp(yesterday.0 + 500)], ); page_with_no_visits_synced.set_sync_status(&conn, SyncStatus::Normal); page_with_no_visits_bookmarked.bookmark(&conn, "My Bookmark"); DbAction::DeleteVisitRows { visit_ids: HashSet::from_iter([ page_with_visits_left.visit_ids[0], page_with_no_visits_unsynced.visit_ids[0], page_with_no_visits_synced.visit_ids[0], page_with_no_visits_bookmarked.visit_ids[0], ]), } .apply(&conn) .unwrap(); DbAction::RecalcPages { page_ids: HashSet::from_iter([ page_with_visits_left.id, page_with_no_visits_unsynced.id, page_with_no_visits_synced.id, page_with_no_visits_bookmarked.id, ]), } .apply(&conn) .unwrap(); assert_eq!( query_page_ids(&conn), [page_with_visits_left.id, page_with_no_visits_bookmarked.id] ); assert_eq!( query_page_tombstones(&conn), [page_with_no_visits_synced.guid] ); assert_eq!( page_with_visits_left.query_frecency(&conn), page_with_visits_left.calculate_frecency(&conn) ); assert_eq!( page_with_no_visits_bookmarked.query_frecency(&conn), page_with_no_visits_bookmarked.calculate_frecency(&conn) ); } #[test] fn test_delete_from_pending_temp_tables() { let mut conn = PlacesDb::open_in_memory(ConnectionType::ReadWrite).unwrap(); let yesterday = Timestamp::now() .checked_sub(Duration::from_secs(60 * 60 * 24)) .unwrap(); let test_page = TestPage::new( &mut conn, "http://example.com/", &[ Timestamp(yesterday.0 + 100), Timestamp(yesterday.0 + 200), Timestamp(yesterday.0 + 300), ], ); DbAction::DeleteVisitRows { visit_ids: HashSet::from_iter([test_page.visit_ids[0]]), } .apply(&conn) .unwrap(); DbAction::RecalcPages { page_ids: HashSet::from_iter([test_page.id]), } .apply(&conn) .unwrap(); DbAction::DeleteFromPendingTempTables.apply(&conn).unwrap(); assert_eq!( conn.conn_ext_query_one::("SELECT COUNT(*) FROM moz_updateoriginsinsert_temp") .unwrap(), 0 ); assert_eq!( conn.conn_ext_query_one::("SELECT COUNT(*) FROM moz_updateoriginsupdate_temp") .unwrap(), 0 ); assert_eq!( conn.conn_ext_query_one::("SELECT COUNT(*) FROM moz_updateoriginsdelete_temp") .unwrap(), 0 ); } #[test] fn test_db_actions_from_visits_to_delete() { assert_eq!( db_actions_from_visits_to_delete(vec![ VisitToDelete { visit_id: RowId(1), page_id: RowId(1), }, VisitToDelete { visit_id: RowId(2), page_id: RowId(2), }, VisitToDelete { visit_id: RowId(3), page_id: RowId(2), }, ]), vec![ DbAction::DeleteVisitRows { visit_ids: HashSet::from_iter([RowId(1), RowId(2), RowId(3)]) }, DbAction::RecalcPages { page_ids: HashSet::from_iter([RowId(1), RowId(2)]) }, DbAction::DeleteFromPendingTempTables, ], ) } }