/*
    This file is part of darktable,
    Copyright (C) 2014-2026 darktable developers.

    darktable is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    darktable is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with darktable.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <glib.h>
#include <glib/gstdio.h>
#include <sqlite3.h>
#include <stdio.h>
#include <string.h>

#include "common/collection.h"
#include "common/darktable.h"
#include "common/database.h"
#include "common/debug.h"
#include "common/history.h"
#include "common/film.h"
#include "common/image.h"
#include "control/conf.h"
#include "control/control.h"
#include "crawler.h"
#include "gui/gtk.h"
#include "gui/splash.h"

#ifdef GDK_WINDOWING_QUARTZ
#include "osx/osx.h"
#endif

// how many seconds may the sidecar file's timestamp differ from that recorded in the database?
#define MAX_TIME_SKEW 2

typedef enum dt_control_crawler_cols_t
{
  DT_CONTROL_CRAWLER_COL_ID = 0,
  DT_CONTROL_CRAWLER_COL_IMAGE_PATH,
  DT_CONTROL_CRAWLER_COL_XMP_PATH,
  DT_CONTROL_CRAWLER_COL_TS_XMP,
  DT_CONTROL_CRAWLER_COL_TS_DB,
  DT_CONTROL_CRAWLER_COL_TS_XMP_INT, // new timestamp to db
  DT_CONTROL_CRAWLER_COL_TS_DB_INT,
  DT_CONTROL_CRAWLER_COL_REPORT,
  DT_CONTROL_CRAWLER_COL_TIME_DELTA,
  DT_CONTROL_CRAWLER_NUM_COLS
} dt_control_crawler_cols_t;

typedef struct dt_control_crawler_result_t
{
  dt_imgid_t id;
  time_t timestamp_xmp;
  time_t timestamp_db;
  char *image_path, *xmp_path;
} dt_control_crawler_result_t;

static void _free_crawler_result(dt_control_crawler_result_t *entry)
{
  g_free(entry->image_path);
  g_free(entry->xmp_path);
  entry->image_path = entry->xmp_path = NULL;
}

static void _set_modification_time(char *filename,
                                   const time_t timestamp)
{
  GFile *gfile = g_file_new_for_path(filename);

  GFileInfo *info = g_file_query_info(
    gfile,
    G_FILE_ATTRIBUTE_TIME_MODIFIED "," G_FILE_ATTRIBUTE_TIME_MODIFIED_USEC,
    G_FILE_QUERY_INFO_NONE,
    NULL,
    NULL);

  // For reference, we could use the following lines but for some
  // reasons there is a deprecated message raised even though this
  // routine is not marked as deprecated in the documentation.
  //
  // GDateTime *datetime = g_date_time_new_from_unix_local(timestamp);
  // g_file_info_set_modification_date_time(info, datetime);

  if(info)
  {
    g_file_info_set_attribute_uint64
      (info,
       G_FILE_ATTRIBUTE_TIME_MODIFIED,
       timestamp);

    g_file_set_attributes_from_info(
      gfile,
      info,
      G_FILE_QUERY_INFO_NONE,
      NULL,
      NULL);
  }

  g_object_unref(gfile);
  if(info) g_clear_object(&info);
}

// progress update intervals in seconds
#define FAST_UPDATE 0.2
#define SLOW_UPDATE 1.0

// number of concurrent workers used to examine the filesystem.  the
// crawl is dominated by filesystem latency rather than cpu work --
// overwhelmingly so for libraries stored on a network share -- so we
// deliberately oversubscribe instead of scaling with the core count.
#define DEFAULT_CRAWLER_THREADS 16
#define MAX_CRAWLER_THREADS 64

// reported back to the caller while a scan is in progress.  the splash
// screen uses this during startup, the background job uses it to drive
// a progress bar in the gui.
typedef void (*dt_crawler_progress_cb)(const double fraction,
                                       const double elapsed,
                                       void *data);

// one image to be examined.  everything up to and including `flags' is
// filled in by the collecting pass, the remaining fields are written by
// the worker threads -- each worker touches only its own item, so no
// locking is required.
typedef struct dt_crawler_item_t
{
  dt_imgid_t id;
  time_t timestamp_db;
  int version;
  int flags;
  const char *folder;           // shared, owned by the directory list
  char *filename;               // owned, basename within `folder'

  int new_flags;
  char *xmp_path;               // only set when xmp_newer is TRUE
  time_t timestamp_xmp;
  gboolean missing;
  gboolean xmp_newer;
} dt_crawler_item_t;

// one directory holding a contiguous run of the items above.  a film
// roll is exactly one directory -- image filenames never contain a path
// separator -- so this is also the unit the background crawl works in.
typedef struct dt_crawler_dir_t
{
  char *folder;                 // owned
  int first;                    // index of its first item
  int count;
  GHashTable *entries;          // set of the names the directory holds
} dt_crawler_dir_t;

typedef struct dt_crawler_scan_t
{
  dt_crawler_item_t *items;
  int num_items;
  dt_crawler_dir_t *dirs;
  int num_dirs;
  gboolean look_for_xmp;
  gint next;                    // atomic: next index to hand out
  gint completed;               // atomic: items finished, drives progress
  const gint *abort;            // optional, checked atomically by the workers

  // the directory currently being examined, set by the driver before it
  // releases the workers on to that directory's range of items
  const dt_crawler_dir_t *dir;
} dt_crawler_scan_t;

static inline gboolean _crawler_aborted(const dt_crawler_scan_t *scan)
{
  return scan->abort && g_atomic_int_get(scan->abort);
}

/* Read the names a directory holds.
 *
 * Listing the directory once replaces the six filesystem probes per
 * image that this used to do -- an existence check, a stat() of the xmp
 * and four probes for .txt/.wav sidecars -- with a single pass plus one
 * stat() per xmp actually present.  On a library of ~85k images that is
 * 506k filesystem calls reduced to 85k, which matters most when the
 * images live on a network share.
 *
 * Returns NULL if the directory cannot be read, which is treated the
 * same way as it was before: every image in it counts as missing.
 */
static GHashTable *_crawler_read_dir(const char *folder)
{
  GError *error = NULL;
  GDir *dir = g_dir_open(folder, 0, &error);
  if(!dir)
  {
    if(error) g_error_free(error);
    return NULL;
  }

  GHashTable *entries = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
  const gchar *name;
  while((name = g_dir_read_name(dir)))
    g_hash_table_add(entries, g_strdup(name));

  g_dir_close(dir);
  return entries;
}

// examine a single image against the listing of the directory holding
// it.  this performs no database access and no gui access whatsoever so
// that it is safe to run from a worker thread.  the early returns mirror
// the `continue' statements of the original serial implementation -- in
// particular an image whose xmp is absent deliberately skips the
// .txt/.wav probe below.
static void _crawler_examine(dt_crawler_item_t *item,
                             GHashTable *entries,
                             const gboolean look_for_xmp)
{
  // if the image is missing we ignore it.
  if(!entries || !g_hash_table_contains(entries, item->filename))
  {
    item->missing = TRUE;
    return;
  }

  // no need to look for xmp files if none get written anyway.
  if(look_for_xmp)
  {
    // construct the xmp filename for this image
    gchar xmp_name[PATH_MAX] = { 0 };
    g_strlcpy(xmp_name, item->filename, sizeof(xmp_name));
    dt_image_path_append_version_no_db(item->version, xmp_name, sizeof(xmp_name));
    size_t len = strlen(xmp_name);
    if(len + 4 >= PATH_MAX) return;
    xmp_name[len++] = '.';
    xmp_name[len++] = 'x';
    xmp_name[len++] = 'm';
    xmp_name[len++] = 'p';
    xmp_name[len] = '\0';

    // the listing tells us whether it is there, so the only image we
    // still have to touch is the one that is
    if(!g_hash_table_contains(entries, xmp_name)) return;

    gchar *xmp_path = g_build_filename(item->folder, xmp_name, NULL);

    // on Windows the encoding might not be UTF8
    gchar *xmp_path_locale = dt_util_normalize_path(xmp_path);
    int stat_res = -1;
#ifdef _WIN32
    // UTF8 paths fail in this context, but converting to UTF16 works
    struct _stati64 statbuf;
    if(xmp_path_locale) // in Windows dt_util_normalize_path returns
                        // NULL if file does not exist
    {
      wchar_t *wfilename = g_utf8_to_utf16(xmp_path_locale, -1, NULL, NULL, NULL);
      stat_res = _wstati64(wfilename, &statbuf);
      g_free(wfilename);
    }
#else
    struct stat statbuf;
    stat_res = stat(xmp_path_locale, &statbuf);
#endif
    g_free(xmp_path_locale);
    if(stat_res)
    {
      g_free(xmp_path);
      return; // TODO: shall we report these?
    }

    // step 1: check if the xmp is newer than our db entry
    if(item->timestamp_db + MAX_TIME_SKEW < statbuf.st_mtime)
    {
      item->xmp_newer = TRUE;
      item->timestamp_xmp = statbuf.st_mtime;
      item->xmp_path = xmp_path;
    }
    else
      g_free(xmp_path);
    // older timestamps are the case for all images after the db
    // upgrade. better not report these
  }

  // step 2: check if the image has associated files (.txt, .wav).  these
  // are now answered out of the directory listing rather than by probing
  // the filesystem four more times.
  size_t len = strlen(item->filename);
  const char *c = item->filename + len;
  while((c > item->filename) && (*c != '.')) c--;
  len = c - item->filename + 1;

  char *extra_name = calloc(len + 3 + 1, sizeof(char));
  if(extra_name)
  {
    g_strlcpy(extra_name, item->filename, len + 1);

    extra_name[len] = 't';
    extra_name[len + 1] = 'x';
    extra_name[len + 2] = 't';
    gboolean has_txt = g_hash_table_contains(entries, extra_name);

    if(!has_txt)
    {
      extra_name[len] = 'T';
      extra_name[len + 1] = 'X';
      extra_name[len + 2] = 'T';
      has_txt = g_hash_table_contains(entries, extra_name);
    }

    extra_name[len] = 'w';
    extra_name[len + 1] = 'a';
    extra_name[len + 2] = 'v';
    gboolean has_wav = g_hash_table_contains(entries, extra_name);

    if(!has_wav)
    {
      extra_name[len] = 'W';
      extra_name[len + 1] = 'A';
      extra_name[len + 2] = 'V';
      has_wav = g_hash_table_contains(entries, extra_name);
    }

    // TODO: decide if we want to remove the flag for images that lost
    // their extra file. currently we do (the else cases)
    int new_flags = item->flags;
    if(has_txt)
      new_flags |= DT_IMAGE_HAS_TXT;
    else
      new_flags &= ~DT_IMAGE_HAS_TXT;
    if(has_wav)
      new_flags |= DT_IMAGE_HAS_WAV;
    else
      new_flags &= ~DT_IMAGE_HAS_WAV;
    item->new_flags = new_flags;

    free(extra_name);
  }
}

static gpointer _crawler_scan_thread(gpointer arg)
{
  dt_crawler_scan_t *scan = (dt_crawler_scan_t *)arg;
  const dt_crawler_dir_t *dir = scan->dir;
  const int last = dir->first + dir->count;

  while(!_crawler_aborted(scan))
  {
    const int idx = g_atomic_int_add(&scan->next, 1);
    if(idx >= last) break;
    _crawler_examine(&scan->items[idx], dir->entries, scan->look_for_xmp);
    g_atomic_int_inc(&scan->completed);
  }
  return NULL;
}

static int _crawler_num_threads(void)
{
  int num_threads = dt_conf_get_int("crawler_threads");
  if(num_threads < 1) num_threads = DEFAULT_CRAWLER_THREADS;
  return MIN(num_threads, MAX_CRAWLER_THREADS);
}

// examine the items of one directory, spreading them over the worker
// pool.  the directory listing itself is read by the calling thread.
static void _crawler_scan_dir(dt_crawler_scan_t *scan,
                              dt_crawler_dir_t *dir)
{
  dir->entries = _crawler_read_dir(dir->folder);

  scan->dir = dir;
  g_atomic_int_set(&scan->next, dir->first);

  const int num_threads = MIN(_crawler_num_threads(), MAX(dir->count, 1));

  if(num_threads <= 1)
  {
    // serial path, kept so that crawler_threads=1 reproduces the
    // original behaviour exactly for comparison purposes
    for(int i = dir->first; i < dir->first + dir->count && !_crawler_aborted(scan); i++)
    {
      _crawler_examine(&scan->items[i], dir->entries, scan->look_for_xmp);
      g_atomic_int_inc(&scan->completed);
    }
  }
  else
  {
    GThread **threads = calloc(num_threads, sizeof(GThread *));
    if(threads)
    {
      for(int t = 0; t < num_threads; t++)
        threads[t] = g_thread_new("crawler", _crawler_scan_thread, scan);
      for(int t = 0; t < num_threads; t++)
        if(threads[t]) g_thread_join(threads[t]);
      free(threads);
    }
    else
    {
      // out of memory: fall back to examining everything inline
      for(int i = dir->first; i < dir->first + dir->count; i++)
        _crawler_examine(&scan->items[i], dir->entries, scan->look_for_xmp);
      g_atomic_int_set(&scan->completed, dir->first + dir->count);
    }
  }

  if(dir->entries)
  {
    // the listing is only needed while its own directory is examined
    g_hash_table_destroy(dir->entries);
    dir->entries = NULL;
  }
}

// run the filesystem examination over all collected items, one directory
// at a time.  the calling thread stays responsible for progress
// reporting so that no gui call is ever made from a worker.
static void _crawler_scan_items(dt_crawler_scan_t *scan,
                                const dt_crawler_progress_cb progress,
                                void *progress_data)
{
  if(scan->num_items <= 0) return;

  const double start_time = dt_get_wtime();
  // set the "previous update" time to 10ms after a notional previous
  // update to ensure visibility of the first update (which might not
  // appear when done with zero delay) while minimizing the delay
  double last_time = start_time - (FAST_UPDATE - 0.01);

  for(int d = 0; d < scan->num_dirs && !_crawler_aborted(scan); d++)
  {
    _crawler_scan_dir(scan, &scan->dirs[d]);

    const double curr_time = dt_get_wtime();
    if(progress
       && curr_time >= last_time + ((curr_time - start_time > 4.0)
                                    ? SLOW_UPDATE : FAST_UPDATE))
    {
      progress(g_atomic_int_get(&scan->completed) / (double)scan->num_items,
               curr_time - start_time, progress_data);
      last_time = curr_time;
    }
  }
}

typedef struct _crawler_collect_t
{
  dt_crawler_item_t *items;
  int capacity;
  int count;
  dt_crawler_dir_t *dirs;
  int num_dirs;
  int dirs_capacity;
} _crawler_collect_t;

static gboolean _crawler_collect_row(const dt_image_sidecar_row_t *row, void *data)
{
  _crawler_collect_t *c = data;
  if(c->count >= c->capacity)
  {
    // the database changed underneath us, grow to fit
    const int new_capacity = c->capacity * 2;
    dt_crawler_item_t *grown =
      realloc(c->items, new_capacity * sizeof(dt_crawler_item_t));
    if(!grown) return FALSE;
    memset(grown + c->capacity, 0,
           (new_capacity - c->capacity) * sizeof(dt_crawler_item_t));
    c->items = grown;
    c->capacity = new_capacity;
  }

  if(!row->folder || !row->filename) return TRUE;

  // start a new directory whenever the folder changes
  if(c->num_dirs == 0 || strcmp(c->dirs[c->num_dirs - 1].folder, row->folder))
  {
    if(c->num_dirs >= c->dirs_capacity)
    {
      const int new_capacity = MAX(16, c->dirs_capacity * 2);
      dt_crawler_dir_t *grown =
        realloc(c->dirs, new_capacity * sizeof(dt_crawler_dir_t));
      if(!grown) return FALSE;
      memset(grown + c->dirs_capacity, 0,
             (new_capacity - c->dirs_capacity) * sizeof(dt_crawler_dir_t));
      c->dirs = grown;
      c->dirs_capacity = new_capacity;
    }
    c->dirs[c->num_dirs].folder = g_strdup(row->folder);
    c->dirs[c->num_dirs].first = c->count;
    c->dirs[c->num_dirs].count = 0;
    c->dirs[c->num_dirs].entries = NULL;
    c->num_dirs++;
  }

  dt_crawler_item_t *item = &c->items[c->count];
  item->id = row->id;
  item->timestamp_db = row->write_timestamp;
  item->version = row->version;
  item->folder = c->dirs[c->num_dirs - 1].folder;
  item->filename = g_strdup(row->filename);
  item->flags = row->flags;
  item->new_flags = item->flags;
  c->dirs[c->num_dirs - 1].count++;
  c->count++;
  return TRUE;
}

// collect the images to be examined.  when filmid is a valid film roll
// only that roll is collected, otherwise the whole library is.
static dt_crawler_item_t *_crawler_collect_items(const dt_filmid_t filmid,
                                                 int *num_items,
                                                 dt_crawler_dir_t **dirs_out,
                                                 int *num_dirs_out)
{
  // reserve based on the number of images we expect to see
  const int capacity = MAX(1, (int)(dt_is_valid_filmid(filmid)
                                    ? dt_film_count_images(filmid)
                                    : dt_image_count()));

  dt_crawler_item_t *items = calloc(capacity, sizeof(dt_crawler_item_t));
  if(!items)
  {
    *num_items = 0;
    *num_dirs_out = 0;
    *dirs_out = NULL;
    return NULL;
  }

  _crawler_collect_t c = { .items = items, .capacity = capacity };
  // ordered by film roll, so the images of one directory come in a
  // contiguous run, which is what lets them be examined a directory at a
  // time against a single listing of it
  dt_image_foreach_sidecar_row(filmid, _crawler_collect_row, &c);

  *num_items = c.count;
  *num_dirs_out = c.num_dirs;
  *dirs_out = c.dirs;
  return c.items;
}

static void _crawler_free_items(dt_crawler_item_t *items,
                                const int num_items,
                                dt_crawler_dir_t *dirs,
                                const int num_dirs)
{
  for(int i = 0; i < num_items; i++)
  {
    g_free(items[i].filename);
    g_free(items[i].xmp_path);
  }
  free(items);

  for(int d = 0; d < num_dirs; d++)
  {
    g_free(dirs[d].folder);
    if(dirs[d].entries) g_hash_table_destroy(dirs[d].entries);
  }
  free(dirs);
}

// apply the results gathered by the workers: write back changed flags
// and build the list of images whose xmp is newer than the database.
// this runs single threaded on the caller's thread and walks the items
// in collection order, so the resulting list is identical to the one
// produced by the original serial implementation.
static GList *_crawler_apply_results(dt_crawler_item_t *items,
                                     const int num_items,
                                     int *flags_changed)
{
  GList *result = NULL;
  int changed = 0;
  dt_imgid_t *changed_ids = g_new(dt_imgid_t, MAX(1, num_items));
  int *changed_flags = g_new(int, MAX(1, num_items));

  for(int i = 0; i < num_items; i++)
  {
    dt_crawler_item_t *item = &items[i];

    if(item->missing)
    {
      gchar *image_path = g_build_filename(item->folder, item->filename, NULL);
      dt_print(DT_DEBUG_CONTROL, "[crawler] `%s' (id: %d) is missing",
               image_path, item->id);
      g_free(image_path);
      continue;
    }

    if(item->xmp_newer)
    {
      dt_control_crawler_result_t *entry = malloc(sizeof(dt_control_crawler_result_t));
      if(entry)
      {
        entry->id = item->id;
        entry->timestamp_xmp = item->timestamp_xmp;
        entry->timestamp_db = item->timestamp_db;
        entry->image_path = g_build_filename(item->folder, item->filename, NULL);
        entry->xmp_path = g_strdup(item->xmp_path);

        result = g_list_prepend(result, entry);
        dt_print(DT_DEBUG_CONTROL,
                 "[crawler] `%s' (id: %d) is a newer XMP file",
                 item->xmp_path, item->id);
      }
    }

    if(item->new_flags != item->flags)
    {
      changed_ids[changed] = item->id;
      changed_flags[changed] = item->new_flags;
      changed++;
    }
  }

  dt_image_set_flags_list(changed_ids, changed_flags, changed);
  g_free(changed_ids);
  g_free(changed_flags);

  if(flags_changed) *flags_changed = changed;

  return g_list_reverse(result); // list was built in reverse order, so un-reverse it
}

static void _splash_progress(const double fraction,
                             const double elapsed,
                             void *data)
{
  dt_splash_screen_set_progress_percent(_("checking for updated sidecar files (%d%%)"),
                                        fraction, elapsed);
}

// examine one film roll (or the whole library when filmid is NO_FILMID)
// and return the list of images with a newer xmp file on disk.
static GList *_crawler_run_filmroll(const dt_filmid_t filmid,
                                    const dt_crawler_progress_cb progress,
                                    void *progress_data,
                                    const gint *abort,
                                    int *flags_changed)
{
  dt_crawler_scan_t scan = { 0 };
  scan.look_for_xmp = dt_image_get_xmp_mode() != DT_WRITE_XMP_NEVER;
  scan.abort = abort;
  scan.items = _crawler_collect_items(filmid, &scan.num_items,
                                      &scan.dirs, &scan.num_dirs);
  if(!scan.items) return NULL;

  const double start_time = dt_get_wtime();
  _crawler_scan_items(&scan, progress, progress_data);
  const double scan_time = dt_get_wtime() - start_time;

  GList *result = _crawler_apply_results(scan.items, scan.num_items, flags_changed);
  _crawler_free_items(scan.items, scan.num_items, scan.dirs, scan.num_dirs);

  if(dt_is_valid_filmid(filmid))
    dt_print(DT_DEBUG_CONTROL,
             "[crawler] film roll %d: examined %d images in %.2fs,"
             " %d updated XMP files found",
             filmid, scan.num_items, scan_time, g_list_length(result));
  else
    dt_print(DT_DEBUG_CONTROL,
             "[crawler] examined %d images in %d directories in %.2fs"
             " using %d threads, %d updated XMP files found",
             scan.num_items, scan.num_dirs, scan_time,
             MIN(_crawler_num_threads(), MAX(scan.num_items, 1)),
             g_list_length(result));

  return result;
}

GList *dt_control_crawler_run(void)
{
  return _crawler_run_filmroll(NO_FILMID, _splash_progress, NULL, NULL, NULL);
}

/******************** background crawling ********************/

static void _crawler_show_image_list(GList *images, const gboolean modal);
static void _crawler_ensure_current_collection(void);
static void _crawler_collection_changed(gpointer instance,
                                        dt_collection_change_t query_change,
                                        dt_collection_properties_t changed_property,
                                        gpointer imgs,
                                        const int next,
                                        gpointer user_data);

/* The crawl is split up per film roll so that its order can be changed
 * while it is running: opening a film roll that has not been examined
 * yet moves it to the front of the queue rather than making the user
 * wait for the rolls queued ahead of it.
 *
 * The queue is deliberately not persisted across restarts.  An XMP file
 * can be modified while darktable is not running, so every session has
 * to examine every image eventually -- the point of this queue is to
 * take that work off the critical path, not to skip it.
 */
typedef struct dt_crawler_bg_t
{
  GList *pending;       // film roll ids, head is examined next
  GList *conflicts;     // dt_control_crawler_result_t *, accumulated
  int num_rolls;
  int num_done;
  gint abort;           // read atomically by the scan workers
  gboolean running;

  /* Only one film roll may be examined at a time, by either the
   * background job or a caller waiting for a specific roll.  Besides
   * keeping the two from examining the same roll twice, this keeps all
   * of the crawler's database work on one thread at a time so that its
   * transactions are never interleaved with each other.
   */
  gboolean scanning;    // someone is inside a scan right now
  dt_filmid_t current;  // which film roll that is, NO_FILMID if none
} dt_crawler_bg_t;

// static storage, so these are zero-initialised as glib requires
static GMutex _crawler_bg_lock;
static GCond _crawler_bg_cond;
static dt_crawler_bg_t _crawler_bg = { 0 };

static void _crawler_free_result_full(gpointer data)
{
  dt_control_crawler_result_t *item = (dt_control_crawler_result_t *)data;
  _free_crawler_result(item);
  free(item);
}

// runs on the gui thread once the crawl has stopped, for whatever reason
static gboolean _crawler_bg_finished(gpointer data)
{
  GList *conflicts = (GList *)data;

  // nothing left to reprioritize
  DT_CONTROL_SIGNAL_DISCONNECT(_crawler_collection_changed, NULL);

  if(conflicts)
  {
    const guint count = g_list_length(conflicts);
    dt_control_log(ngettext("%u updated XMP sidecar file found",
                            "%u updated XMP sidecar files found", count), count);

    // shown non-modally: the crawl finishes long after startup, so this
    // must not interrupt whatever the user is currently doing
    _crawler_show_image_list(conflicts, FALSE);
  }
  return G_SOURCE_REMOVE;
}

/* Examine one film roll and fold the findings into the accumulated list.
 * The caller must already have claimed the scan (scanning == TRUE and
 * current == filmid) so that nothing else touches the database at the
 * same time.  Returns the conflicts found for this film roll alone,
 * still owned by the accumulated list unless `take' is TRUE.
 */
static GList *_crawler_scan_claimed_roll(const dt_filmid_t filmid,
                                         const gboolean take)
{
  int flags_changed = 0;
  GList *found = _crawler_run_filmroll(filmid, NULL, NULL,
                                       &_crawler_bg.abort, &flags_changed);

  g_mutex_lock(&_crawler_bg_lock);
  if(!take)
    _crawler_bg.conflicts = g_list_concat(_crawler_bg.conflicts, found);
  _crawler_bg.num_done++;
  _crawler_bg.scanning = FALSE;
  _crawler_bg.current = NO_FILMID;
  g_cond_broadcast(&_crawler_bg_cond);
  g_mutex_unlock(&_crawler_bg_lock);

  // the .txt/.wav flags are drawn as thumbnail overlays, so a change
  // needs to reach the screen.  this practically never fires.
  if(flags_changed) dt_control_queue_redraw_center();

  return take ? found : NULL;
}

static int32_t _crawler_bg_job_run(dt_job_t *job)
{
  dt_control_job_set_progress_message(job, "%s",
                                      _("checking for updated sidecar files"));

  while(TRUE)
  {
    g_mutex_lock(&_crawler_bg_lock);
    // let anyone waiting for a specific film roll go first
    while(_crawler_bg.scanning && !g_atomic_int_get(&_crawler_bg.abort))
      g_cond_wait(&_crawler_bg_cond, &_crawler_bg_lock);

    if(g_atomic_int_get(&_crawler_bg.abort) || !_crawler_bg.pending)
    {
      g_mutex_unlock(&_crawler_bg_lock);
      break;
    }
    const dt_filmid_t filmid = GPOINTER_TO_INT(_crawler_bg.pending->data);
    _crawler_bg.pending = g_list_delete_link(_crawler_bg.pending,
                                             _crawler_bg.pending);
    _crawler_bg.scanning = TRUE;
    _crawler_bg.current = filmid;
    const double fraction =
      (_crawler_bg.num_done + 1) / (double)MAX(_crawler_bg.num_rolls, 1);
    g_mutex_unlock(&_crawler_bg_lock);

    _crawler_scan_claimed_roll(filmid, FALSE);

    dt_control_job_set_progress(job, fraction);
  }

  g_mutex_lock(&_crawler_bg_lock);
  GList *conflicts = _crawler_bg.conflicts;
  _crawler_bg.conflicts = NULL;
  const gboolean aborted = g_atomic_int_get(&_crawler_bg.abort) != 0;
  const int num_done = _crawler_bg.num_done;
  const int num_rolls = _crawler_bg.num_rolls;
  _crawler_bg.running = FALSE;
  g_mutex_unlock(&_crawler_bg_lock);

  dt_print(DT_DEBUG_CONTROL,
           "[crawler] background crawl %s after %d of %d film rolls",
           aborted ? "cancelled" : "finished", num_done, num_rolls);

  if(aborted)
  {
    // darktable is shutting down: drop the findings rather than trying
    // to put a dialog on screen on the way out
    g_list_free_full(conflicts, _crawler_free_result_full);
    conflicts = NULL;
  }
  // not g_main_context_invoke(): while the gtk main loop is not running yet
  // (a small library can be crawled before dt_gui_gtk_run()) nobody owns the
  // default context, so glib would run the callback right here on the worker
  // thread and create the dialog off the main thread
  g_idle_add(_crawler_bg_finished, conflicts);

  return 0;
}

static dt_job_t *_crawler_bg_job_create(void)
{
  dt_job_t *job = dt_control_job_create(&_crawler_bg_job_run, "crawl for updated sidecar files");
  if(!job) return NULL;
  dt_control_job_set_params(job, NULL, NULL);
  return job;
}

void dt_control_crawler_start_background(void)
{
  if(!dt_conf_get_bool("run_crawler_on_start") || dt_gimpmode()) return;

  g_mutex_lock(&_crawler_bg_lock);
  if(_crawler_bg.running)
  {
    g_mutex_unlock(&_crawler_bg_lock);
    return;
  }

  // most recently opened film rolls first -- those are the ones the user
  // is most likely to look at in this session.  rolls that have never
  // been opened have a NULL access_timestamp and sort last.
  _crawler_bg.pending = dt_film_get_by_access();
  _crawler_bg.num_rolls = g_list_length(_crawler_bg.pending);
  _crawler_bg.num_done = 0;
  _crawler_bg.conflicts = NULL;
  g_atomic_int_set(&_crawler_bg.abort, 0);
  _crawler_bg.running = _crawler_bg.num_rolls > 0;
  const gboolean start = _crawler_bg.running;
  const int num_rolls = _crawler_bg.num_rolls;
  g_mutex_unlock(&_crawler_bg_lock);

  if(!start) return;

  dt_print(DT_DEBUG_CONTROL,
           "[crawler] queued %d film rolls for background crawling", num_rolls);

  // follow the user around the library while the crawl runs
  DT_CONTROL_SIGNAL_CONNECT(DT_SIGNAL_COLLECTION_CHANGED,
                            _crawler_collection_changed, NULL);

  // the collection restored at startup was set up before the signal above
  // was connected, and it is the one the user is looking at right now
  _crawler_ensure_current_collection();

  // note: dt_control_add_job() returns TRUE to report a *failure*
  if(dt_control_add_job(DT_JOB_QUEUE_SYSTEM_BG, _crawler_bg_job_create()))
  {
    // nothing will run, so do not leave the queue looking busy -- that
    // would make dt_control_crawler_stop() wait for a job that never was
    dt_print(DT_DEBUG_CONTROL, "[crawler] could not queue the background job");
    DT_CONTROL_SIGNAL_DISCONNECT(_crawler_collection_changed, NULL);
    g_mutex_lock(&_crawler_bg_lock);
    g_list_free(_crawler_bg.pending);
    _crawler_bg.pending = NULL;
    _crawler_bg.running = FALSE;
    g_mutex_unlock(&_crawler_bg_lock);
  }
}

/* Make sure a film roll has been examined before its images are shown.
 *
 * The images of a film roll must not be edited before its sidecar files
 * have been checked: darktable does not re-read an xmp when an image is
 * opened, and dt_image_write_sidecar_file() overwrites whatever is on
 * disk, so editing an image whose xmp was updated elsewhere would
 * silently discard those changes.  The blocking startup crawl this
 * replaces guaranteed that could not happen, and so must this.
 *
 * Examining a single film roll is cheap -- a few tens of milliseconds
 * for a typical roll -- so only the roll being opened is waited for,
 * rather than the whole library.
 */
static GList *_crawler_ensure_roll(const dt_filmid_t filmid)
{
  if(!dt_is_valid_filmid(filmid)) return NULL;

  g_mutex_lock(&_crawler_bg_lock);
  if(!_crawler_bg.running)
  {
    g_mutex_unlock(&_crawler_bg_lock);
    return NULL;
  }

  // if the background job is examining this very film roll, wait for it
  while(_crawler_bg.current == filmid
        && _crawler_bg.running
        && !g_atomic_int_get(&_crawler_bg.abort))
    g_cond_wait(&_crawler_bg_cond, &_crawler_bg_lock);

  // claim it, waiting for any other roll being examined to finish first
  while(_crawler_bg.scanning && !g_atomic_int_get(&_crawler_bg.abort))
    g_cond_wait(&_crawler_bg_cond, &_crawler_bg_lock);

  GList *link = g_list_find(_crawler_bg.pending, GINT_TO_POINTER(filmid));
  if(!link || g_atomic_int_get(&_crawler_bg.abort))
  {
    // already examined in this session, or we are shutting down
    g_mutex_unlock(&_crawler_bg_lock);
    return NULL;
  }
  _crawler_bg.pending = g_list_delete_link(_crawler_bg.pending, link);
  _crawler_bg.scanning = TRUE;
  _crawler_bg.current = filmid;
  g_mutex_unlock(&_crawler_bg_lock);

  const double start_time = dt_get_wtime();
  GList *found = _crawler_scan_claimed_roll(filmid, TRUE);
  dt_print(DT_DEBUG_CONTROL,
           "[crawler] film roll %d examined on demand in %.2fs, %d updated"
           " XMP files found",
           filmid, dt_get_wtime() - start_time, g_list_length(found));
  return found;
}

/* Report on demand findings straight away rather than leaving them for
 * the end of the crawl: the whole point of having waited is to tell the
 * user before they start editing these images.
 */
static void _crawler_report(GList *found)
{
  if(!found) return;

  const guint count = g_list_length(found);
  dt_control_log(ngettext("%u updated XMP sidecar file found",
                          "%u updated XMP sidecar files found", count), count);
  _crawler_show_image_list(found, FALSE);
}

void dt_control_crawler_ensure_filmroll(const dt_filmid_t filmid)
{
  _crawler_report(_crawler_ensure_roll(filmid));
}

/* Selecting a film roll in the collect module does not go through
 * dt_film_open(), it just changes the collection.  So follow the
 * collection as well, and for the same reason wait for it: the images it
 * contains are about to be shown and may be edited, so their sidecar
 * files have to have been checked first.
 *
 * Every film roll the collection covers is waited for, not a sample of
 * them, since the user can scroll to any of the images.  In the worst
 * case -- a collection spanning the whole library, opened immediately at
 * startup -- this costs the same as the blocking crawl it replaces, and
 * in every other case far less.  Once the background crawl has finished
 * this does nothing at all.
 */
static void _crawler_ensure_current_collection(void)
{
  g_mutex_lock(&_crawler_bg_lock);
  const gboolean running = _crawler_bg.running;
  g_mutex_unlock(&_crawler_bg_lock);
  if(!running) return;

  // the collection is materialised in memory.collected_images, so its
  // film rolls come out of a single query
  GList *rolls = NULL;
  sqlite3_stmt *stmt;
  // clang-format off
  DT_DEBUG_SQLITE3_PREPARE_V2(dt_database_get(darktable.db),
                              "SELECT i.film_id"
                              " FROM memory.collected_images AS c, main.images AS i"
                              " ON i.id = c.imgid"
                              " GROUP BY i.film_id"
                              " ORDER BY MIN(c.rowid)",
                              -1, &stmt, NULL);
  // clang-format on
  while(sqlite3_step(stmt) == SQLITE_ROW)
    rolls = g_list_prepend(rolls, GINT_TO_POINTER(sqlite3_column_int(stmt, 0)));
  sqlite3_finalize(stmt);
  rolls = g_list_reverse(rolls);

  GList *found = NULL;
  for(GList *iter = rolls; iter; iter = g_list_next(iter))
    found = g_list_concat(found,
                          _crawler_ensure_roll(GPOINTER_TO_INT(iter->data)));

  g_list_free(rolls);
  _crawler_report(found);
}

static void _crawler_collection_changed(gpointer instance,
                                        dt_collection_change_t query_change,
                                        dt_collection_properties_t changed_property,
                                        gpointer imgs,
                                        const int next,
                                        gpointer user_data)
{
  _crawler_ensure_current_collection();
}

void dt_control_crawler_stop(const gboolean wait)
{
  g_mutex_lock(&_crawler_bg_lock);
  const gboolean running = _crawler_bg.running;
  g_mutex_unlock(&_crawler_bg_lock);
  if(!running) return;

  g_atomic_int_set(&_crawler_bg.abort, 1);

  if(wait)
  {
    // the workers check the abort flag per image, so this returns
    // quickly unless a single stat() is stuck on an unresponsive mount
    for(int i = 0; i < 1000; i++)
    {
      g_mutex_lock(&_crawler_bg_lock);
      const gboolean still_running = _crawler_bg.running;
      g_mutex_unlock(&_crawler_bg_lock);
      if(!still_running) break;
      g_usleep(10000);
    }
  }
}


/********************* the gui stuff *********************/

typedef struct dt_control_crawler_gui_t
{
  GtkTreeView *tree;
  GtkTreeModel *model;
  GtkWidget *log;
  GtkWidget *spinner;
  GList *rows_to_remove;
} dt_control_crawler_gui_t;

// close the window and clean up
static void dt_control_crawler_response_callback(GtkWidget *dialog,
                                                 const gint response_id,
                                                 gpointer user_data)
{
  dt_control_crawler_gui_t *gui = (dt_control_crawler_gui_t *)user_data;
  g_object_unref(G_OBJECT(gui->model));
  gtk_widget_destroy(dialog);
  free(gui);
}


static void _delete_selected_rows(dt_control_crawler_gui_t *gui)
{
  GList *rr_list = gui->rows_to_remove;
  GtkTreeModel *model = gui->model;

  // Remove TreeView rows from rr_list. It needs to be populated before
  for(GList *node = rr_list; node != NULL; node = g_list_next(node))
  {
    GtkTreePath *path = gtk_tree_row_reference_get_path((GtkTreeRowReference*)node->data);

    if(path)
    {
      GtkTreeIter  iter;
      if(gtk_tree_model_get_iter(model, &iter, path))
        gtk_list_store_remove(GTK_LIST_STORE(model), &iter);
    }
  }

  // Cleanup the list of rows
  g_list_foreach(rr_list, (GFunc) gtk_tree_row_reference_free, NULL);
  g_list_free(rr_list);
}


static void _select_all_callback(GtkButton *button,
                                 gpointer user_data)
{
  dt_control_crawler_gui_t *gui = (dt_control_crawler_gui_t *)user_data;
  GtkTreeSelection *selection = gtk_tree_view_get_selection(gui->tree);
  gtk_tree_selection_select_all(selection);
}


static void _select_none_callback(GtkButton *button, gpointer user_data)
{
  dt_control_crawler_gui_t *gui = (dt_control_crawler_gui_t *)user_data;
  GtkTreeSelection *selection = gtk_tree_view_get_selection(gui->tree);
  gtk_tree_selection_unselect_all(selection);
}


static void _select_invert_callback(GtkButton *button, gpointer user_data)
{
  dt_control_crawler_gui_t *gui = (dt_control_crawler_gui_t *)user_data;
  GtkTreeSelection *selection = gtk_tree_view_get_selection(gui->tree);

  GtkTreeIter iter;
  gboolean valid = gtk_tree_model_get_iter_first(gui->model, &iter);
  while(valid)
  {
    if(gtk_tree_selection_iter_is_selected(selection, &iter))
      gtk_tree_selection_unselect_iter(selection, &iter);
    else
      gtk_tree_selection_select_iter(selection, &iter);

    valid = gtk_tree_model_iter_next(gui->model, &iter);
  }
}


static void _db_update_timestamp(const dt_imgid_t id, const time_t timestamp)
{
  // Update DB writing timestamp with XMP file timestamp
  dt_image_set_write_timestamp(id, timestamp);
}


static void _get_crawler_entry_from_model(GtkTreeModel *model,
                                          GtkTreeIter *iter,
                                          dt_control_crawler_result_t *entry)
{
  gtk_tree_model_get(model, iter,
                     DT_CONTROL_CRAWLER_COL_IMAGE_PATH, &entry->image_path,
                     DT_CONTROL_CRAWLER_COL_ID,         &entry->id,
                     DT_CONTROL_CRAWLER_COL_XMP_PATH,   &entry->xmp_path,
                     DT_CONTROL_CRAWLER_COL_TS_DB_INT,  &entry->timestamp_db,
                     DT_CONTROL_CRAWLER_COL_TS_XMP_INT, &entry->timestamp_xmp,
                     -1); // marks list end
}


static void _append_row_to_remove(GtkTreeModel *model,
                                  GtkTreePath *path,
                                  GList **rowref_list)
{
  // append TreeModel rows to the list to remove
  GtkTreeRowReference *rowref = gtk_tree_row_reference_new(model, path);
  *rowref_list = g_list_append(*rowref_list, rowref);
}

static void _log_synchronization(dt_control_crawler_gui_t *gui,
                                 gchar *pattern,
                                 gchar *filepath)
{
  gchar *message = g_markup_printf_escaped(pattern, filepath ? filepath : "");

  // add a new line in the log TreeView
  GtkTreeModel *model_log = gtk_tree_view_get_model(GTK_TREE_VIEW(gui->log));
  gtk_list_store_insert_with_values(GTK_LIST_STORE(model_log), NULL, -1, 0, message, -1);

  g_free(message);
}


static void sync_xmp_to_db(GtkTreeModel *model,
                           GtkTreePath *path,
                           GtkTreeIter *iter,
                           gpointer user_data)
{
  dt_control_crawler_gui_t *gui = (dt_control_crawler_gui_t *)user_data;
  dt_control_crawler_result_t entry = { NO_IMGID };
  _get_crawler_entry_from_model(model, iter, &entry);
  _db_update_timestamp(entry.id, entry.timestamp_xmp);

  const gboolean error = dt_history_load_and_apply(entry.id, entry.xmp_path, 0);

  if(error)
  {
    _log_synchronization(gui, _("ERROR: %s NOT synced XMP → DB"), entry.image_path);
    _log_synchronization(gui, _("ERROR: cannot write the database."
                                " the destination may be full, offline or read-only."),
                         NULL);
  }
  else
  {
    _append_row_to_remove(model, path, &gui->rows_to_remove);
    _log_synchronization(gui, _("SUCCESS: %s synced XMP → DB"), entry.image_path);
  }

  _free_crawler_result(&entry);
}


static void sync_db_to_xmp(GtkTreeModel *model,
                           GtkTreePath *path,
                           GtkTreeIter *iter,
                           gpointer user_data)
{
  dt_control_crawler_gui_t *gui = (dt_control_crawler_gui_t *)user_data;
  dt_control_crawler_result_t entry = { NO_IMGID };
  _get_crawler_entry_from_model(model, iter, &entry);

  // write the XMP and make sure it get the last modified timestamp of the db
  const gboolean error = dt_image_write_sidecar_file(entry.id);
  _set_modification_time(entry.xmp_path, entry.timestamp_db);

  if(error)
  {
    _log_synchronization(gui, _("ERROR: %s NOT synced DB → XMP"), entry.image_path);
    _log_synchronization(gui,
                         _("ERROR: cannot write %s \nthe destination may be full,"
                           " offline or read-only."), entry.xmp_path);
  }
  else
  {
    _append_row_to_remove(model, path, &gui->rows_to_remove);
    _log_synchronization(gui, _("SUCCESS: %s synced DB → XMP"), entry.image_path);
  }

  _free_crawler_result(&entry);
}

static void sync_newest_to_oldest(GtkTreeModel *model,
                                  GtkTreePath *path,
                                  GtkTreeIter *iter,
                                  gpointer user_data)
{
  dt_control_crawler_gui_t *gui = (dt_control_crawler_gui_t *)user_data;
  dt_control_crawler_result_t entry = { NO_IMGID };
  _get_crawler_entry_from_model(model, iter, &entry);

  gboolean error = FALSE;

  if(entry.timestamp_xmp > entry.timestamp_db)
  {
    // WRITE XMP in DB
    _db_update_timestamp(entry.id, entry.timestamp_xmp);
    error = dt_history_load_and_apply(entry.id, entry.xmp_path, 0);
    if(error)
    {
      _log_synchronization
        (gui,
         _("ERROR: %s NOT synced new (XMP) → old (DB)"), entry.image_path);
      _log_synchronization
        (gui,
         _("ERROR: cannot write the database. the destination may be full,"
           " offline or read-only."), NULL);
    }
    else
    {
      _log_synchronization
        (gui,
         _("SUCCESS: %s synced new (XMP) → old (DB)"), entry.image_path);
    }
  }
  else if(entry.timestamp_xmp < entry.timestamp_db)
  {
    // write the XMP and make sure it get the last modified timestamp of the db
    error = dt_image_write_sidecar_file(entry.id);
    _set_modification_time(entry.xmp_path, entry.timestamp_db);

    dt_print(DT_DEBUG_ALWAYS, "%s synced DB (new) → XMP (old)", entry.image_path);
    if(error)
    {
      _log_synchronization
        (gui,
         _("ERROR: %s NOT synced new (DB) → old (XMP)"), entry.image_path);
      _log_synchronization
        (gui,
         _("ERROR: cannot write %s \nthe destination may be full, offline or read-only."),
         entry.xmp_path);
    }
    else
    {
      _log_synchronization(gui, _("SUCCESS: %s synced new (DB) → old (XMP)"),
                           entry.image_path);
    }
  }
  else
  {
    // we should never reach that part of the code
    // if both timestamps are equal, they should not be in this list in the first place
    error = TRUE;
    _log_synchronization(gui, _("EXCEPTION: %s has inconsistent timestamps"),
                         entry.image_path);
  }

  if(!error) _append_row_to_remove(model, path, &gui->rows_to_remove);

  _free_crawler_result(&entry);
}


static void sync_oldest_to_newest(GtkTreeModel *model,
                                  GtkTreePath *path,
                                  GtkTreeIter *iter,
                                  gpointer user_data)
{
  dt_control_crawler_gui_t *gui = (dt_control_crawler_gui_t *)user_data;
  dt_control_crawler_result_t entry = { NO_IMGID };
  _get_crawler_entry_from_model(model, iter, &entry);
  gboolean error = FALSE;

  if(entry.timestamp_xmp < entry.timestamp_db)
  {
    // WRITE XMP in DB
    _db_update_timestamp(entry.id, entry.timestamp_xmp);
    error = dt_history_load_and_apply(entry.id, entry.xmp_path, 0);
    if(error)
    {
      _log_synchronization(gui,
                           _("ERROR: %s NOT synced old (XMP) → new (DB)"),
                           entry.image_path);
    _log_synchronization(gui,
                         _("ERROR: cannot write the database."
                           " the destination may be full, offline or read-only."), NULL);
    }
    else
    {
      _log_synchronization(gui,
                           _("SUCCESS: %s synced old (XMP) → new (DB)"),
                           entry.image_path);
    }
  }
  else if(entry.timestamp_xmp > entry.timestamp_db)
  {
    // WRITE DB in XMP
    error = dt_image_write_sidecar_file(entry.id);
    _set_modification_time(entry.xmp_path, entry.timestamp_db);
    if(error)
    {
      _log_synchronization(gui,
                           _("ERROR: %s NOT synced old (DB) → new (XMP)"),
                           entry.image_path);
      _log_synchronization(gui,
                           _("ERROR: cannot write %s \nthe destination may be full,"
                             " offline or read-only."), entry.xmp_path);
    }
    else
    {
      _log_synchronization(gui,
                           _("SUCCESS: %s synced old (DB) → new (XMP)"),
                           entry.image_path);
    }
  }
  else
  {
    // we should never reach that part of the code
    // if both timestamps are equal, they should not be in this list in the first place
    error = TRUE;
    _log_synchronization(gui,
                         _("EXCEPTION: %s has inconsistent timestamps"),
                         entry.image_path);
  }

  if(!error)
    _append_row_to_remove(model, path, &gui->rows_to_remove);

  _free_crawler_result(&entry);
}

// overwrite database with xmp
static void _reload_button_clicked(GtkButton *button, gpointer user_data)
{
  dt_control_crawler_gui_t *gui = (dt_control_crawler_gui_t *)user_data;
  GtkTreeSelection *selection = gtk_tree_view_get_selection(gui->tree);
  gui->rows_to_remove = NULL;
  gtk_spinner_start(GTK_SPINNER(gui->spinner));
  gtk_tree_selection_selected_foreach(selection, sync_xmp_to_db, gui);
  _delete_selected_rows(gui);
  gtk_spinner_stop(GTK_SPINNER(gui->spinner));
}

// overwrite xmp with database
static void _overwrite_button_clicked(GtkButton *button, gpointer user_data)
{
  dt_control_crawler_gui_t *gui = (dt_control_crawler_gui_t *)user_data;
  GtkTreeSelection *selection = gtk_tree_view_get_selection(gui->tree);
  gui->rows_to_remove = NULL;
  gtk_spinner_start(GTK_SPINNER(gui->spinner));
  gtk_tree_selection_selected_foreach(selection, sync_db_to_xmp, gui);
  _delete_selected_rows(gui);
  gtk_spinner_stop(GTK_SPINNER(gui->spinner));
}

// overwrite the oldest with the newest
static void _newest_button_clicked(GtkButton *button, gpointer user_data)
{
  dt_control_crawler_gui_t *gui = (dt_control_crawler_gui_t *)user_data;
  GtkTreeSelection *selection = gtk_tree_view_get_selection(gui->tree);
  gui->rows_to_remove = NULL;
  gtk_spinner_start(GTK_SPINNER(gui->spinner));
  gtk_tree_selection_selected_foreach(selection, sync_newest_to_oldest, gui);
  _delete_selected_rows(gui);
  gtk_spinner_stop(GTK_SPINNER(gui->spinner));
}

// overwrite the newest with the oldest
static void _oldest_button_clicked(GtkButton *button, gpointer user_data)
{
  dt_control_crawler_gui_t *gui = (dt_control_crawler_gui_t *)user_data;
  GtkTreeSelection *selection = gtk_tree_view_get_selection(gui->tree);
  gui->rows_to_remove = NULL;
  gtk_spinner_start(GTK_SPINNER(gui->spinner));
  gtk_tree_selection_selected_foreach(selection, sync_oldest_to_newest, gui);
  _delete_selected_rows(gui);
  gtk_spinner_stop(GTK_SPINNER(gui->spinner));
}

static gchar* str_time_delta(const int time_delta)
{
  // display the time difference as a legible string
  int seconds = time_delta;

  int minutes = seconds / 60;
  seconds -= 60 * minutes;

  int hours = minutes / 60;
  minutes -= 60 * hours;

  const int days = hours / 24;
  hours -= 24 * days;

  return g_strdup_printf(_("%id %02dh %02dm %02ds"), days, hours, minutes, seconds);
}

// show a popup window with a list of updated images/xmp files and allow the user to tell dt what to do about them
static void _crawler_show_image_list(GList *images, const gboolean modal)
{
  if(!images) return;

  dt_control_crawler_gui_t *gui = malloc(sizeof(dt_control_crawler_gui_t));

  // a list with all the images
  GtkTreeViewColumn *column;
  GtkListStore *store = gtk_list_store_new(DT_CONTROL_CRAWLER_NUM_COLS,
                                           G_TYPE_INT,    // id
                                           G_TYPE_STRING, // image path
                                           G_TYPE_STRING, // xmp path
                                           G_TYPE_STRING, // timestamp from xmp
                                           G_TYPE_STRING, // timestamp from db
                                           G_TYPE_INT,    // timestamp to db
                                           G_TYPE_INT,
                                           G_TYPE_STRING, // report: newer version
                                           G_TYPE_STRING);// time delta

  gui->model = GTK_TREE_MODEL(store);

  for(GList *list_iter = images; list_iter; list_iter = g_list_next(list_iter))
  {
    dt_control_crawler_result_t *item = list_iter->data;
    char timestamp_db[64], timestamp_xmp[64];
    struct tm tm_stamp;
    strftime(timestamp_db, sizeof(timestamp_db),
             "%c", localtime_r(&item->timestamp_db, &tm_stamp));
    strftime(timestamp_xmp, sizeof(timestamp_xmp),
             "%c", localtime_r(&item->timestamp_xmp, &tm_stamp));

    const time_t time_delta = llabs(item->timestamp_db - item->timestamp_xmp);
    gchar *timestamp_delta = str_time_delta(time_delta);

    gtk_list_store_insert_with_values(store, NULL, -1,
       DT_CONTROL_CRAWLER_COL_ID, item->id,
       DT_CONTROL_CRAWLER_COL_IMAGE_PATH, item->image_path,
       DT_CONTROL_CRAWLER_COL_XMP_PATH, item->xmp_path,
       DT_CONTROL_CRAWLER_COL_TS_XMP, timestamp_xmp,
       DT_CONTROL_CRAWLER_COL_TS_DB, timestamp_db,
       DT_CONTROL_CRAWLER_COL_TS_XMP_INT, item->timestamp_xmp,
       DT_CONTROL_CRAWLER_COL_TS_DB_INT, item->timestamp_db,
       DT_CONTROL_CRAWLER_COL_REPORT, (item->timestamp_xmp > item->timestamp_db)
                                      ? _("XMP")
                                      : _("database"),
       DT_CONTROL_CRAWLER_COL_TIME_DELTA, timestamp_delta,
       -1);
    _free_crawler_result(item);
    g_free(timestamp_delta);
  }
  g_list_free_full(images, g_free);

  GtkWidget *tree = gtk_tree_view_new_with_model(GTK_TREE_MODEL(store));
  GtkTreeSelection *selection = gtk_tree_view_get_selection(GTK_TREE_VIEW(tree));
  gtk_tree_selection_set_mode(selection, GTK_SELECTION_MULTIPLE);

  gui->tree = GTK_TREE_VIEW(tree); // FIXME: do we need to free that later ?

  GtkCellRenderer *renderer_text = gtk_cell_renderer_text_new();
  column = gtk_tree_view_column_new_with_attributes
    (_("path"), renderer_text, "text",
     DT_CONTROL_CRAWLER_COL_IMAGE_PATH, NULL);
  gtk_tree_view_append_column(GTK_TREE_VIEW(tree), column);
  gtk_tree_view_column_set_expand(column, TRUE);
  gtk_tree_view_column_set_resizable(column, TRUE);
  gtk_tree_view_column_set_min_width(column, DT_PIXEL_APPLY_DPI(200));
  g_object_set(renderer_text, "ellipsize", PANGO_ELLIPSIZE_MIDDLE, NULL);

  column = gtk_tree_view_column_new_with_attributes
    (_("XMP timestamp"), gtk_cell_renderer_text_new(), "text",
     DT_CONTROL_CRAWLER_COL_TS_XMP, NULL);
  gtk_tree_view_append_column(GTK_TREE_VIEW(tree), column);

  column = gtk_tree_view_column_new_with_attributes
    (_("database timestamp"), gtk_cell_renderer_text_new(), "text",
     DT_CONTROL_CRAWLER_COL_TS_DB, NULL);
  gtk_tree_view_append_column(GTK_TREE_VIEW(tree), column);

  column = gtk_tree_view_column_new_with_attributes
    (_("newest"), gtk_cell_renderer_text_new(), "text",
     DT_CONTROL_CRAWLER_COL_REPORT, NULL);
  gtk_tree_view_append_column(GTK_TREE_VIEW(tree), column);

  GtkCellRenderer *renderer_date = gtk_cell_renderer_text_new();
  column = gtk_tree_view_column_new_with_attributes
    (_("time difference"), renderer_date, "text",
     DT_CONTROL_CRAWLER_COL_TIME_DELTA, NULL);
  g_object_set(renderer_date, "xalign", 1., NULL);
  gtk_tree_view_append_column(GTK_TREE_VIEW(tree), column);

  GtkWidget *scroll = dt_gui_scroll_wrap(tree);
  gtk_scrolled_window_set_policy(GTK_SCROLLED_WINDOW(scroll),
                                 GTK_POLICY_NEVER, GTK_POLICY_AUTOMATIC);

  // build a dialog window that contains the list of images
  GtkWidget *win = dt_ui_main_window(darktable.gui->ui);
  GtkWidget *dialog = gtk_dialog_new_with_buttons
    (_("updated XMP sidecar files found"), GTK_WINDOW(win),
     GTK_DIALOG_DESTROY_WITH_PARENT | (modal ? GTK_DIALOG_MODAL : 0), _("_close"),
     GTK_RESPONSE_CLOSE, NULL);

#ifdef GDK_WINDOWING_QUARTZ
  dt_osx_disallow_fullscreen(dialog);
#endif
  gtk_widget_set_size_request(dialog, -1, DT_PIXEL_APPLY_DPI(400));
  gtk_window_set_transient_for(GTK_WINDOW(dialog), GTK_WINDOW(win));


  GtkWidget *select_all = gtk_button_new_with_label(_("select all"));
  GtkWidget *select_none = gtk_button_new_with_label(_("select none"));
  GtkWidget *select_invert = gtk_button_new_with_label(_("invert selection"));
  g_signal_connect(select_all, "clicked", G_CALLBACK(_select_all_callback), gui);
  g_signal_connect(select_none, "clicked", G_CALLBACK(_select_none_callback), gui);
  g_signal_connect(select_invert, "clicked", G_CALLBACK(_select_invert_callback), gui);

  GtkWidget *label = gtk_label_new_with_mnemonic(_("on the selection:"));
  GtkWidget *reload_button = gtk_button_new_with_label(_("keep the XMP edit"));
  GtkWidget *overwrite_button = gtk_button_new_with_label(_("keep the database edit"));
  GtkWidget *newest_button = gtk_button_new_with_label(_("keep the newest edit"));
  GtkWidget *oldest_button = gtk_button_new_with_label(_("keep the oldest edit"));
  g_signal_connect(reload_button, "clicked", G_CALLBACK(_reload_button_clicked), gui);
  g_signal_connect(overwrite_button, "clicked", G_CALLBACK(_overwrite_button_clicked), gui);
  g_signal_connect(newest_button, "clicked", G_CALLBACK(_newest_button_clicked), gui);
  g_signal_connect(oldest_button, "clicked", G_CALLBACK(_oldest_button_clicked), gui);

  /* Feedback spinner in case synch happens over network and stales */
  gui->spinner = gtk_spinner_new();

  /* Log report */
  gui->log = gtk_tree_view_new();

  gtk_tree_view_insert_column_with_attributes
    (GTK_TREE_VIEW(gui->log), -1,
     _("synchronization log"), renderer_text,
     "markup", 0, NULL);

  GtkListStore *store_log = gtk_list_store_new (1, G_TYPE_STRING);
  GtkTreeModel *model_log = GTK_TREE_MODEL(store_log);
  gtk_tree_view_set_model(GTK_TREE_VIEW(gui->log), model_log);
  g_object_unref(model_log);

  dt_gui_dialog_add(GTK_DIALOG(dialog),
    dt_gui_hbox(select_all, select_none, select_invert),
    scroll,
    dt_gui_hbox(label, reload_button, overwrite_button, newest_button, oldest_button, gui->spinner),
    dt_gui_scroll_wrap(gui->log));
  gtk_widget_show_all(dialog);

  g_signal_connect(dialog, "response",
                   G_CALLBACK(dt_control_crawler_response_callback), gui);
}

void dt_control_crawler_show_image_list(GList *images)
{
  // the synchronous startup crawl blocks the ui anyway, so keep it modal
  _crawler_show_image_list(images, TRUE);
}

/* backthumb crawler */

static inline gboolean _lighttable_silent(void)
{
  return dt_view_get_current() == DT_VIEW_LIGHTTABLE
          && dt_get_wtime() > darktable.backthumbs.time;
}

static inline gboolean _valid_mip(dt_mipmap_size_t mip)
{
  return mip > DT_MIPMAP_0 && mip < DT_MIPMAP_LDR_MAX;
}

static inline gboolean _still_thumbing(void)
{
  return darktable.backthumbs.state == DT_JOB_STATE_RUNNING
      && _lighttable_silent();
}

static int _update_img_thumbs(const dt_imgid_t imgid,
                              const dt_mipmap_size_t max_mip,
                              const int64_t stamp)
{
  dt_backthumb_t *bt = &darktable.backthumbs;

  /* as generating the thumb might take some time watch out for a non-running state
      and possibly return asap without updating the database.
  */
  for(dt_mipmap_size_t k = max_mip; k >= DT_MIPMAP_1 && bt->state == DT_JOB_STATE_RUNNING; k--)
  {
    dt_mipmap_buffer_t buf;
    dt_mipmap_cache_get(&buf, imgid, k, DT_MIPMAP_BLOCKING, 'r');
    dt_mipmap_cache_release(&buf);
  }

  if(bt->state != DT_JOB_STATE_RUNNING)
    return 0;

  // we have written all thumbs and are in running state so it's safe to write timestamp, hash and mipsize
  dt_image_set_thumb_state(imgid, max_mip, stamp);

  dt_mipmap_cache_evict(imgid);
  dt_history_hash_set_mipmap(imgid);
  return 1;
}

static int _update_all_thumbs(const dt_mipmap_size_t max_mip)
{
  int missed = 0;
  int updated = 0;

  GList *imgs = dt_image_get_outdated_thumbs(max_mip);
  for(const GList *l = imgs; l && _still_thumbing(); l = g_list_next(l))
  {
    const dt_image_thumb_stamp_t *t = l->data;
    const dt_imgid_t imgid = t->id;
    const int64_t stamp = t->stamp;

    char path[PATH_MAX] = { 0 };
    dt_image_full_path(imgid, path, sizeof(path), NULL);
    const gboolean available = dt_util_test_image_file(path);

    if(available)
      updated += _update_img_thumbs(imgid, max_mip, stamp);
    else
    {
      missed++;
      dt_print(DT_DEBUG_CACHE, "[thumb crawler] '%s' ID=%d NOT available", path, imgid);
    }
  }
  g_list_free_full(imgs, g_free);

  if(updated)
    dt_print(DT_DEBUG_CACHE,
      "[thumb crawler] max_mip=%d, %d thumbs updated, %d not found, %s",
      max_mip, updated, missed,
      _still_thumbing() ? "all done" : "interrupted by user activity");

  return updated;
}

static void _reinitialize_thumbs_database(void)
{
  dt_print(DT_DEBUG_CACHE, "[thumb crawler] initialize database");
  dt_image_reset_thumb_state();
}

/* public */
void dt_set_backthumb_time(const double next)
{
  dt_backthumb_t *bt = &darktable.backthumbs;
  if(next > 0.5)
    bt->time = dt_get_wtime() + next;
  else
    bt->time = fmax(bt->time, dt_get_wtime() + bt->idle);
}

void dt_update_thumbs_thread(void *p)
{
  dt_pthread_setname("thumbs_update");
  dt_backthumb_t *bt = &darktable.backthumbs;

  bt->idle = (double)dt_conf_get_float("backthumbs_inactivity");
  const dt_mipmap_size_t mipsize = dt_mipmap_cache_get_min_mip_from_pref(dt_conf_get_string_const("backthumbs_mipsize"));
  const gboolean dwriting = dt_conf_get_bool("cache_disk_backend");
  const gboolean service = dt_conf_get_bool("backthumbs_initialize");

  bt->state = DT_JOB_STATE_FINISHED;

  if(!dwriting || !_valid_mip(mipsize))
  {
    dt_print(DT_DEBUG_CACHE, "[thumb crawler] closing due to preferences setting");
    return;
  }

  // return if any thumbcache dir is not writable
  for(dt_mipmap_size_t k = DT_MIPMAP_1; k <= DT_MIPMAP_LDR_MAX-1; k++)
  {
    char dirname[PATH_MAX] = { 0 };
    snprintf(dirname, sizeof(dirname), "%s.d/%d", darktable.mipmap_cache->cachedir, k);
    if(g_mkdir_with_parents(dirname, 0750))
    {
      dt_print(DT_DEBUG_CACHE, "[thumb crawler] can't create mipmap dir '%s'", dirname);
      return;
    }
  }

  dt_print(DT_DEBUG_CACHE, "[thumb crawler] started");
  bt->state = DT_JOB_STATE_RUNNING;
  int updated = 0;

  if(service)
  {
    _reinitialize_thumbs_database();
    dt_conf_set_bool("backthumbs_initialize", FALSE);
  }

  dt_set_backthumb_time(1.0);
  while(bt->state == DT_JOB_STATE_RUNNING)
  {
    for(int i = 0; i < 12 && bt->state == DT_JOB_STATE_RUNNING; i++)
      g_usleep(250000);

    if(bt->state != DT_JOB_STATE_RUNNING)
      break;

    if(_lighttable_silent())
      updated += _update_all_thumbs(mipsize);
  }

  dt_print(DT_DEBUG_CACHE, "[thumb crawler] closing, %d mipmaps updated", updated);
  bt->state = DT_JOB_STATE_FINISHED;
}

// clang-format off
// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
// vim: shiftwidth=2 expandtab tabstop=2 cindent
// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
// clang-format on