# SPDX-License-Identifier: GPL-3.0-or-later # Copyright (c) 2008-2026 Andrew Ziem. # # This work is licensed under the terms of the GNU GPL, version 3 or # later. See the COPYING file in the top-level directory. """ Test case for module DeepScan """ # standard imports import itertools import os import shutil import subprocess import unicodedata import unittest from unittest import mock from tests.common import pytest # first party imports from tests import common from tests import TestCleaner from tests.common import SPECIAL_TEST_STRINGS from bleachbit import IS_MAC, IS_WINDOWS, FS_CASE_SENSITIVE from bleachbit.Options import options from bleachbit.DeepScan import DeepScan, Search, normalized_walk from bleachbit.FileUtilities import openfiles, whitelisted if IS_WINDOWS: # pylint: disable-next=ungrouped-imports from tests.TestWindows import WindowsLinksMixIn else: WindowsLinksMixIn = object class DeepScanTestCase(common.BleachbitTestCase, WindowsLinksMixIn): """Test Case for module DeepScan""" def _test_encoding(self, fn): """Test encoding""" fullpath = self.write_file(fn + ".bak") # Add Unicode paths to encourage a crash. subdir = os.path.join(self.tempdir, 'ɡælɪk.dir') os.mkdir(subdir) subfile = os.path.join(subdir, 'ɡælɪk.file') common.touch_file(subfile) searches = {self.tempdir: []} searches[self.tempdir].append( Search(command='delete', regex=r'\.[Bb][Aa][Kk]$')) ds = DeepScan(searches) found = False for cmd in ds.scan(): if cmd is True: # True is used to yield to GTK+, but it is not # needed in this test. continue self.assertExists(cmd.path) if cmd.path == fullpath: found = True self.assertTrue(found, "Did not find '%s'" % fullpath) os.unlink(fullpath) self.assertNotExists(fullpath) shutil.rmtree(subdir) @pytest.mark.no_xdist def test_encoding(self): """Test encoding""" for test in SPECIAL_TEST_STRINGS: self._test_encoding(test) def test_DeepScan(self): """Unit test for class DeepScan on the kinds of files a home has""" # Not the real home: on CI it holds tool caches and the folders # other workers write to, so its size and contents vary root = self.mkdtemp() # Other tests scan the whole class tempdir. self.addCleanup(shutil.rmtree, root) sub = os.path.join(root, 'sub') deeper = os.path.join(sub, 'ɡælɪk.dir') os.makedirs(deeper) expected = { self.write_file(os.path.join(root, 'Makefile')), self.write_file(os.path.join(root, 'notes.txt~')), self.write_file(os.path.join(root, 'backup.bak')), self.write_file(os.path.join(sub, 'Thumbs.db')), self.write_file(os.path.join(deeper, 'ɡælɪk.bak')), } for near_miss in ('Makefile.am', '~notes.txt', 'bak.txt'): self.write_file(os.path.join(root, near_miss)) self.write_file(os.path.join(sub, 'Thumbs.db.old')) if not IS_WINDOWS: # Windows does not allow a colon in a file name expected.add(self.write_file( os.path.join(sub, 'Thumbs.db:encryptable'))) # Not followed, so the files under sub are reported once os.symlink(sub, os.path.join(root, 'link_to_sub')) # A dangling link still matches broken = os.path.join(root, 'broken.bak') os.symlink(os.path.join(root, 'missing'), broken) expected.add(broken) if os.geteuid() != 0: # A folder that cannot be listed is skipped locked = os.path.join(root, 'locked') os.mkdir(locked) self.write_file(os.path.join(locked, 'hidden.bak')) os.chmod(locked, 0) self.addCleanup(os.chmod, locked, 0o700) searches = {root: [ Search(command='delete', regex=regex) for regex in ('^Makefile$', '~$', 'bak$', '^Thumbs.db$', '^Thumbs.db:encryptable$')]} found = [] idle_yields = 0 with mock.patch('bleachbit.DeepScan.time') as mock_time: # A clock that jumps on every read makes each folder look slow, # so the scan also hands control to the GTK idle loop mock_time.time.side_effect = itertools.count() for cmd in DeepScan(searches).scan(): if cmd is True: idle_yields += 1 continue self.assertLExists(cmd.path) found.append(cmd.path) self.assertCountEqual(expected, found) self.assertGreater(idle_yields, 0) def test_compiled_search_path_prefix(self): """CompiledSearch.match should preserve root path semantics.""" from bleachbit.DeepScan import CompiledSearch, directory_prefix compiled = CompiledSearch( Search(command='delete', regex=r'\.txt$') ) for dirpath in (self.tempdir, os.path.abspath(os.sep)): with self.subTest(dirpath=dirpath): path_prefix = directory_prefix(dirpath) self.assertEqual( compiled.match(dirpath, 'foo.txt', path_prefix), os.path.join(dirpath, 'foo.txt'), ) def test_directory_prefix_unc_share_root(self): """Bare UNC share roots must insert a separator before the filename.""" import ntpath from bleachbit import DeepScan as deepscan_mod from bleachbit.DeepScan import CompiledSearch with mock.patch.object(deepscan_mod.os, 'path', ntpath): unc_root = r'\\server\share' prefix = deepscan_mod.directory_prefix(unc_root) # join(unc_root, '') alone omits the separator; concatenation # would otherwise produce \\server\sharefoo.txt. self.assertNotEqual(prefix, unc_root) self.assertTrue(prefix.endswith('\\')) self.assertEqual( prefix + 'foo.txt', ntpath.join(unc_root, 'foo.txt'), ) # Drive-relative paths must not gain a separator. self.assertEqual(deepscan_mod.directory_prefix('C:'), 'C:') self.assertEqual( deepscan_mod.directory_prefix('C:') + 'foo.txt', ntpath.join('C:', 'foo.txt'), ) compiled = CompiledSearch( Search(command='delete', regex=r'\.txt$') ) self.assertEqual( compiled.match(unc_root, 'foo.txt', prefix), ntpath.join(unc_root, 'foo.txt'), ) def test_skip_whitelisted_directory(self): """DeepScan should not search whitelisted directories.""" # Reminder: each test case gets a new options directory, # so we do not need to restore the old keep list (whitelist). keep_dir = self.mkdir('keep') search_dir = self.mkdir('search') keep_file = self.write_file(os.path.join(keep_dir, 'skip.bbtestbak')) search_file = self.write_file( os.path.join(search_dir, 'find.bbtestbak')) options.set_whitelist_paths([('folder', keep_dir)]) searches = { self.tempdir: [ Search(command='delete', regex=r'\.bbtestbak$') ] } paths = [ cmd.path for cmd in DeepScan(searches).scan() if cmd is not True ] self.assertNotIn(keep_file, paths) self.assertIn(search_file, paths) @common.skipIfWindows def test_scan_skip_open(self): """skip_open leaves files that a process holds open""" root = self.mkdtemp() open_fn = self.write_file(os.path.join(root, 'open.bbtestswp'), b'x') closed_fn = self.write_file( os.path.join(root, 'closed.bbtestswp'), b'x') searches = {root: [ Search(command='delete', regex=r'\.bbtestswp$', skip_open=True)]} # Open for reading so FreeBSD reports the path, as in test_open_files with open(open_fn, 'rb'): openfiles.scan() paths = [cmd.path for cmd in DeepScan( searches).scan() if cmd is not True] self.assertEqual(paths, [closed_fn]) @common.skipIfWindows def test_scan_skip_open_other_search(self): """An open file stays when a search without skip_open matches it too""" top = self.mkdtemp() # Other tests scan the whole class tempdir self.addCleanup(shutil.rmtree, top) subdir = self.mkdir(os.path.join(top, 'sub')) open_fn = self.write_file(os.path.join(subdir, 'open.bbtestswp'), b'x') closed_fn = self.write_file( os.path.join(subdir, 'closed.bbtestswp'), b'x') plain = Search(command='delete', regex=r'\.bbtestswp$') guard = Search(command='delete', regex=r'\.bbtestswp$', skip_open=True) cases = ( {top: [plain, guard]}, {top: [guard, plain]}, {subdir: [plain], top: [guard]}, {top: [guard], subdir: [plain]}, ) with open(open_fn, 'rb'): openfiles.scan() for searches in cases: with self.subTest(searches=searches): paths = [cmd.path for cmd in DeepScan( searches).scan() if cmd is not True] self.assertEqual(paths, [closed_fn]) @common.skipIfWindows def test_scan_skip_open_unlisted(self): """skip_open keeps files when open files cannot be listed""" root = self.mkdtemp() self.write_file(os.path.join(root, 'a.bbtestswp'), b'x') # A second match catches a failed listing cached as empty self.write_file(os.path.join(root, 'b.bbtestswp'), b'x') bak_fn = self.write_file(os.path.join(root, 'c.bbtestbak'), b'x') searches = {root: [ Search(command='delete', regex=r'\.bbtestswp$', skip_open=True), Search(command='delete', regex=r'\.bbtestbak$')]} for exc in (FileNotFoundError('lsof'), subprocess.CalledProcessError(1, 'lsof'), UnicodeDecodeError('utf-8', b'\xff', 0, 1, 'invalid')): with self.subTest(exc=exc), \ mock.patch('bleachbit.FileUtilities.open_files', side_effect=exc), \ mock.patch.object(openfiles, 'last_scan_time', None): paths = [cmd.path for cmd in DeepScan( searches).scan() if cmd is not True] self.assertEqual(paths, [bak_fn]) @common.skipIfWindows def test_keep_list_decomposed_name(self): """The macOS deep scan honors a kept file with a decomposed name""" search_dir = self.mkdtemp() # Other tests scan the whole class tempdir self.addCleanup(shutil.rmtree, search_dir) keep_file = self.write_file(os.path.join( search_dir, unicodedata.normalize('NFD', 'r\u00e9sum\u00e9.bbtestbak'))) options.set_whitelist_paths([('file', keep_file)]) searches = { search_dir: [ Search(command='delete', regex=r'\.bbtestbak$') ] } with mock.patch('bleachbit.DeepScan.IS_MAC', True), \ mock.patch('bleachbit.FileUtilities.IS_MAC', True): paths = [ cmd.path for cmd in DeepScan(searches).scan() if cmd is not True ] self.assertEqual(1, len(paths)) self.assertTrue(whitelisted(paths[0])) def test_scan_yields_each_file_once(self): """Overlapping searches and nested tops yield a file only once""" top = self.mkdtemp() # Other tests scan the whole class tempdir self.addCleanup(shutil.rmtree, top) subdir = self.mkdir(os.path.join(top, 'sub')) filename = self.write_file(os.path.join(subdir, 'foo.bbtestbak')) searches = { top: [ Search(command='delete', regex=r'\.bbtestbak$'), Search(command='delete', regex=r'^foo\.'), ], subdir: [Search(command='delete', regex=r'\.bbtestbak$')], } paths = [ cmd.path for cmd in DeepScan(searches).scan() if cmd is not True ] self.assertEqual(paths, [filename]) @common.skipUnlessWindows def test_scan_does_not_follow_junction(self): """DeepScan must not descend into a junction""" scan_root = self.mkdir('scanroot') victim = self.mkdir('victim') self.write_file(os.path.join(victim, 'target.bbtestbak')) self._create_win_junction(victim, os.path.join(scan_root, 'link')) searches = {scan_root: [ Search(command='delete', regex=r'\.bbtestbak$')]} paths = [cmd.path for cmd in DeepScan( searches).scan() if cmd is not True] self.assertEqual(paths, [], 'DeepScan followed a junction: %s' % paths) def _get_compiled_search(self, wholeregex_substr): """Return the CompiledSearch for the deepscan action whose wholeregex contains the given substring (e.g. 'node_modules', 'venv').""" import xml.etree.ElementTree as ET from bleachbit.DeepScan import CompiledSearch xml_path = os.path.join( os.path.dirname(__file__), '..', 'cleaners', 'deepscan.xml') root = ET.parse(xml_path).getroot() wholeregex = None nwholeregex = None for action in root.iter('action'): if action.get('wholeregex') and wholeregex_substr in action.get('wholeregex'): wholeregex = action.get('wholeregex') nwholeregex = action.get('nwholeregex') break self.assertIsNotNone(wholeregex) return CompiledSearch(Search(command='delete', wholeregex=wholeregex, nwholeregex=nwholeregex)) def test_angular_matches_only_cache_dir(self): """.angular option should match the cache folder but not config files.""" cs = self._get_compiled_search('angular') matched = [ ('/home/u/proj/.angular/cache/19.0.0/babel-webpack', 'x.json'), (r'C:\Users\u\proj\.angular\cache', 'x.json'), ] excluded = [ ('/home/u', '.angular-config.json'), ('/home/u/.config/angular', '.angular-config.json'), ('/home/u/proj', '.angular.json'), ('/home/u/proj', '.angular-cli.json'), (r'C:\Users\u', '.angular-config.json'), ] for dirpath, filename in matched: with self.subTest(dirpath=dirpath, filename=filename): self.assertIsNotNone( cs.match(dirpath, filename), f"Should match {dirpath}/{filename}") for dirpath, filename in excluded: with self.subTest(dirpath=dirpath, filename=filename): self.assertIsNone( cs.match(dirpath, filename), f"Should exclude {dirpath}/{filename}") def test_node_modules_excludes_dot_dirs(self): """node_modules option should exclude system/cache/editor directories.""" cs = self._get_compiled_search('node_modules') excluded = [ ('/home/neo/.nvm/versions/node/v22.22.3/lib/node_modules/corepack', 'README.md'), ('/home/trinity/.vscode/extensions/ms-python.python-2026.4.0-linux-x64/out/client/node_modules', 'sudo-prompt.js'), ('/home/morpheus/.npm/_npx/15b07286cbcc3329/node_modules', '.package-lock.json'), ('/home/smith/.cache/typescript/5.9/node_modules', '.package-lock.json'), # root's home when running as root, on Linux and macOS ('/root/.nvm/versions/node/v20.11.0/lib/node_modules/npm', 'index.js'), ('/root/.vscode-server/bin/abc/node_modules/foo', 'index.js'), ('/var/root/.nvm/versions/node/v20.11.0/lib/node_modules/npm', 'index.js'), (r'C:\Users\glados\.windsurf\extensions\ms-python.python-2026.4.0-universal\out\client\node_modules', 'unicode.js'), (r'C:\Users\glados\.cursor\projects\empty-window\canvases\node_modules', 'cursor.js'), (r'C:\Users\glados\AppData\Local\Programs\myapp\node_modules', 'file.js'), (r'C:\Users\glados\AppData\Local\ApertureScience\MainframeCore\node_modules\central-processing', 'index.js'), (r'C:\Users\Alice\AppData\Local\Programs\BeeperTexts\resources\app.asar.unpacked\node_modules\@beeper', 'beeper_client_sdk_napi_binding.node'), (r'C:\Users\bob\AppData\Local\Programs\Microsoft VS Code\7e7950df89\resources\app', 'node_modules.asar'), (r'C:\Users\mallory\AppData\Local\Programs\Microsoft VS Code\fcf604774b\resources\app', 'node_modules.asar'), ('/home/trent/projects/important/node_modules_backup', 'README.md'), ('/home/trent/projects/important', 'node_modules.asar'), (r'C:\Users\charlie\Documents\important', 'node_modules.asar'), (r'C:\Users\charlie\Documents\important\node_modules_backup', 'aes.js'), ] for dirpath, filename in excluded: with self.subTest(dirpath=dirpath, filename=filename): self.assertIsNone( cs.match(dirpath, filename), f"Should exclude {dirpath}/{filename}") def test_node_modules_matches_ordinary_dirs(self): """node_modules option should match ordinary user project directories.""" cs = self._get_compiled_search('node_modules') matched = [ ('/home/sfalken/games/wopr/node_modules/joshua/ai', 'core.js'), (r'C:\Users\esnowden\Documents\nsa\prism\node_modules\crypto-js', 'aes.js'), (r'C:\Users\chunkylover53\Documents\work-from-home\node_modules\click-button-automator\node_modules\is-buffer', 'index.js'), ('/home/chunkylover53/Downloads/donut-radar/node_modules/react-dom', 'bar.js'), ('/root/projects/app/node_modules/react', 'index.js'), ] for dirpath, filename in matched: with self.subTest(dirpath=dirpath, filename=filename): self.assertIsNotNone( cs.match(dirpath, filename), f"Should match {dirpath}/{filename}") def test_venv_excludes_system_and_editor_dirs(self): """venv option should exclude system/cache/editor directories and non-venv paths.""" cs = self._get_compiled_search('venv') excluded = [ # venv cannot be directly under home ('/home/username/.venv', 'pyvenv.cfg'), # Python standard library venv module (not a virtual environment) ('/home/username/.local/share/uv/python/cpython-3.12.11-linux-x86_64-gnu/lib/python3.12/venv', '__init__.py'), ('/home/username/.pyenv/versions/3.12.7/lib/python3.12/venv', '__init__.py'), ('/home/username/miniconda3/lib/python3.12/venv', '__init__.py'), ('/home/username/miniconda3/envs/foo/lib/python3.11/venv/scripts/common', 'activate'), ('/home/username/src/cpython/Lib/venv', '__init__.py'), (r'C:\Users\charlie\scoop\apps\python\3.12.4\Lib\venv', '__init__.py'), # tool-managed venvs in root's home ('/root/.local/share/uv/tools/ruff/.venv', 'pyvenv.cfg'), # pipx-managed venvs (managed by pipx, like npm cache) ('/home/username/.local/share/pipx/venvs/duplicity', 'pyvenv.cfg'), # typeshed stubs bundled with editor extensions ('/home/username/.vscode/extensions/ms-python.python-2026.4.0-linux-x64/python_files/lib/jedilsp/jedi/third_party/typeshed/stdlib/3/venv', '__init__.pyi'), # pyenv build patches ('/home/username/.pyenv/plugins/python-build/share/python-build/patches/3.5.2/Python-3.5.2', 'venv.patch'), # Windows: editor extensions and AppData (r'C:\Users\glados\.vscode\extensions\ms-python.python-2026.4.0-universal\python_files\lib\jedilsp\jedi\third_party\typeshed\stdlib\3\venv', '__init__.pyi'), (r'C:\Users\glados\AppData\Local\ApertureScience\MainframeCore\venv', 'pyvenv.cfg'), # Files/directories that contain venv (but not a venv directory) ('/home/aturing/npl/ace_project/tools/', 'bootstrap_ace_venv.sh'), ('/home/vonneumann/edvac/logs/', 'venv_build.log'), ('/home/aturing/npl/ace_project/venv_tools/', 'requirements_numerics.txt'), (r'C:\Users\charlie\Documents\important\venv_backup', 'pyvenv.cfg'), # .venv bundled inside an AppData\Local application directory (r'C:\Users\charlie\AppData\Local\AnkiProgramFiles\.venv\Lib\site-packages\PyQt6', 'QtCore.pyd'), # Python standard library venv module under uv-managed Python (Windows) (r'C:\Users\charlie\AppData\Roaming\uv\python\cpython-3.13.5-windows-x86_64-none\Lib\venv', '__main__.py'), ] if IS_WINDOWS: # Windows is case insensitive # venv cannot be directly under home excluded.append((r'c:\users\username\.VENV', 'pyvenv.cfg')) for dirpath, filename in excluded: with self.subTest(dirpath=dirpath, filename=filename): self.assertIsNone( cs.match(dirpath, filename), f"Should exclude {dirpath}/{filename}") def test_venv_matches_ordinary_dirs(self): """venv option should match ordinary user project virtual environments.""" cs = self._get_compiled_search('venv') matched = [ ('/home/genisys/t800/.venv/yolo-v666', 'pyvenv.cfg'), ('/home/stark/jarvis/venv/yolo14', 'pyvenv.cfg'), ('/root/projects/app/.venv', 'pyvenv.cfg'), ('/home/stark/jarvis/venv/lib/python3.12/site-packages/numpy', '__init__.py'), (r'C:\Users\esnowden\Documents\nsa\prism\venv', 'pyvenv.cfg'), ] if IS_WINDOWS: # case insensitive matched.append( (r'C:\USERS\esnowden\Documents\nsa\prism\.VENV', 'pyvenv.cfg')) matched.append( (r'c:\USERS\pinky\world_domination\VENV\site-packages\hypnosisis', '__init__.py')) for dirpath, filename in matched: with self.subTest(dirpath=dirpath, filename=filename): self.assertIsNotNone( cs.match(dirpath, filename), f"Should match {dirpath}/{filename}") def _test_delete(self, command): """Delete files in a test environment""" # make some files base = self.mkdtemp() f_del = self.write_file(os.path.join(base, 'foo.txt.bbtestbak')) f_keep = self.write_file(os.path.join(base, 'foo.txt')) subdir = os.path.join(base, 'sub') os.mkdir(subdir) f_del2 = self.write_file(os.path.join(subdir, 'bar.ini.bbtestbak')) f_del3 = self.write_file(os.path.join(subdir, 'bar.ini.bbtestBAK')) # sanity check self.assertExists(f_del) self.assertExists(f_keep) self.assertExists(f_del2) # run deep scan def run_deep_scan(regex): astr = ''.format( command, base, regex) cleaner = TestCleaner.action_to_cleaner(astr) from bleachbit.Worker import backends, Worker backends['test'] = cleaner operations = {'test': ['option1']} from bleachbit import CLI ui = CLI.CliCallback() worker = Worker(ui, True, operations) list(worker.run()) del backends['test'] # validate results; 'sub' as a path component, as a random # temporary directory name can contain it too run_deep_scan(r'regex="\.bbtestbak$" ' r'wholeregex="[/\\]sub[/\\][^/\\]*\.bbtestbak$"') self.assertExists(f_del) self.assertExists(f_keep) self.assertFalse(os.path.exists(f_del2)) if FS_CASE_SENSITIVE: self.assertExists(f_del3) else: self.assertFalse(os.path.exists(f_del3)) # validate results run_deep_scan(r'regex="\.bbtestbak$"') self.assertFalse(os.path.exists(f_del)) self.assertExists(f_keep) def test_delete(self): self._test_delete('delete') def test_shred(self): self._test_delete('shred') @unittest.skipUnless(IS_MAC, 'Not on Darwin') def test_normalized_walk_darwin(self): with mock.patch('os.walk') as mock_walk: mock_walk.return_value = [ ('/foo', ('bar',), ['ba\u0300z']), ('/foo/bar', (), ['spam', 'eggs']), ] self.assertEqual(list(normalized_walk('.')), [ ('/foo', ('bar',), ['b\u00e0z']), ('/foo/bar', (), ['spam', 'eggs']), ]) with mock.patch('os.walk') as mock_walk: expected = [ ('/foo', ('bar',), ['baz']), ('/foo/bar', (), ['spam', 'eggs']), ] mock_walk.return_value = expected self.assertEqual(list(normalized_walk('.')), expected)