Add song fingerprinting and tracking

Fixes #296
2021-04-25 21:16:44 +02:00
parent a883508eca
commit f8ed2afef1
40 changed files with 826 additions and 266 deletions
--- a/src/collection/collectionwatcher.cpp
+++ b/src/collection/collectionwatcher.cpp
@@ -47,13 +47,18 @@

 #include "core/filesystemwatcherinterface.h"
 #include "core/logging.h"
+#include "core/timeconstants.h"
 #include "core/tagreaderclient.h"
 #include "core/taskmanager.h"
+#include "core/imageutils.h"
 #include "directory.h"
 #include "collectionbackend.h"
 #include "collectionwatcher.h"
 #include "playlistparsers/cueparser.h"
 #include "settings/collectionsettingspage.h"
+#ifdef HAVE_SONGFINGERPRINTING
+#  include "engine/chromaprinter.h"
+#endif

 // This is defined by one of the windows headers that is included by taglib.
 #ifdef RemoveDirectory
@@ -73,25 +78,40 @@ CollectionWatcher::CollectionWatcher(Song::Source source, QObject *parent)
      backend_(nullptr),
      task_manager_(nullptr),
      fs_watcher_(FileSystemWatcherInterface::Create(this)),
+      original_thread_(nullptr),
      scan_on_startup_(true),
      monitor_(true),
-      mark_songs_unavailable_(false),
+      song_tracking_(true),
+      mark_songs_unavailable_(true),
+      expire_unavailable_songs_days_(60),
      stop_requested_(false),
      rescan_in_progress_(false),
      rescan_timer_(new QTimer(this)),
+      periodic_scan_timer_(new QTimer(this)),
      rescan_paused_(false),
      total_watches_(0),
      cue_parser_(new CueParser(backend_, this)),
-      original_thread_(nullptr) {
+      last_scan_time_(0) {

  original_thread_ = thread();

-  rescan_timer_->setInterval(1000);
+  rescan_timer_->setInterval(2000);
  rescan_timer_->setSingleShot(true);

+  periodic_scan_timer_->setInterval(86400 * kMsecPerSec);
+  periodic_scan_timer_->setSingleShot(false);
+
+  QStringList image_formats = ImageUtils::SupportedImageFormats();
+  for (const QString &format : image_formats) {
+    if (!sValidImages.contains(format)) {
+      sValidImages.append(format);
+    }
+  }
+
  ReloadSettings();

  QObject::connect(rescan_timer_, &QTimer::timeout, this, &CollectionWatcher::RescanPathsNow);
+  QObject::connect(periodic_scan_timer_, &QTimer::timeout, this, &CollectionWatcher::IncrementalScanCheck);

 }

@@ -123,8 +143,10 @@ void CollectionWatcher::ReloadSettings() {
  s.beginGroup(CollectionSettingsPage::kSettingsGroup);
  scan_on_startup_ = s.value("startup_scan", true).toBool();
  monitor_ = s.value("monitor", true).toBool();
-  mark_songs_unavailable_ = s.value("mark_songs_unavailable", false).toBool();
  QStringList filters = s.value("cover_art_patterns", QStringList() << "front" << "cover").toStringList();
+  song_tracking_ = s.value("song_tracking", false).toBool();
+  mark_songs_unavailable_ = song_tracking_ ? true : s.value("mark_songs_unavailable", true).toBool();
+  expire_unavailable_songs_days_ = s.value("expire_unavailable_songs", 60).toInt();
  s.endGroup();

  best_image_filters_.clear();
@@ -147,6 +169,13 @@ void CollectionWatcher::ReloadSettings() {
    }
  }

+  if (mark_songs_unavailable_ && !periodic_scan_timer_->isActive()) {
+    periodic_scan_timer_->start();
+  }
+  else if (!mark_songs_unavailable_ && periodic_scan_timer_->isActive()) {
+    periodic_scan_timer_->stop();
+  }
+
 }

 CollectionWatcher::ScanTransaction::ScanTransaction(CollectionWatcher *watcher, const int dir, const bool incremental, const bool ignores_mtime, const bool mark_songs_unavailable)
@@ -156,8 +185,10 @@ CollectionWatcher::ScanTransaction::ScanTransaction(CollectionWatcher *watcher,
      incremental_(incremental),
      ignores_mtime_(ignores_mtime),
      mark_songs_unavailable_(mark_songs_unavailable),
+      expire_unavailable_songs_days_(60),
      watcher_(watcher),
      cached_songs_dirty_(true),
+      cached_songs_missing_fingerprint_dirty_(true),
      known_subdirs_dirty_(true) {

  QString description;
@@ -185,14 +216,14 @@ CollectionWatcher::ScanTransaction::~ScanTransaction() {

 }

-void CollectionWatcher::ScanTransaction::AddToProgress(int n) {
+void CollectionWatcher::ScanTransaction::AddToProgress(const quint64 n) {

  progress_ += n;
  watcher_->task_manager_->SetTaskProgress(task_id_, progress_, progress_max_);

 }

-void CollectionWatcher::ScanTransaction::AddToProgressMax(int n) {
+void CollectionWatcher::ScanTransaction::AddToProgressMax(const quint64 n) {

  progress_max_ += n;
  watcher_->task_manager_->SetTaskProgress(task_id_, progress_, progress_max_);
@@ -201,16 +232,6 @@ void CollectionWatcher::ScanTransaction::AddToProgressMax(int n) {

 void CollectionWatcher::ScanTransaction::CommitNewOrUpdatedSongs() {

-  if (!new_songs.isEmpty()) {
-    emit watcher_->NewOrUpdatedSongs(new_songs);
-    new_songs.clear();
-  }
-
-  if (!touched_songs.isEmpty()) {
-    emit watcher_->SongsMTimeUpdated(touched_songs);
-    touched_songs.clear();
-  }
-
  if (!deleted_songs.isEmpty()) {
    if (mark_songs_unavailable_) {
      emit watcher_->SongsUnavailable(deleted_songs);
@@ -221,6 +242,16 @@ void CollectionWatcher::ScanTransaction::CommitNewOrUpdatedSongs() {
    deleted_songs.clear();
  }

+  if (!new_songs.isEmpty()) {
+    emit watcher_->NewOrUpdatedSongs(new_songs);
+    new_songs.clear();
+  }
+
+  if (!touched_songs.isEmpty()) {
+    emit watcher_->SongsMTimeUpdated(touched_songs);
+    touched_songs.clear();
+  }
+
  if (!readded_songs.isEmpty()) {
    emit watcher_->SongsReadded(readded_songs);
    readded_songs.clear();
@@ -252,22 +283,43 @@ void CollectionWatcher::ScanTransaction::CommitNewOrUpdatedSongs() {
  }
  new_subdirs.clear();

+  if (incremental_ || ignores_mtime_) {
+    emit watcher_->UpdateLastSeen(dir_, expire_unavailable_songs_days_);
+  }
+
 }


 SongList CollectionWatcher::ScanTransaction::FindSongsInSubdirectory(const QString &path) {

  if (cached_songs_dirty_) {
-    cached_songs_ = watcher_->backend_->FindSongsInDirectory(dir_);
+    const SongList songs = watcher_->backend_->FindSongsInDirectory(dir_);
+    for (const Song &song : songs) {
+      const QString p = song.url().toLocalFile().section('/', 0, -2);
+      cached_songs_.insert(p, song);
+    }
    cached_songs_dirty_ = false;
  }

-  // TODO: Make this faster
-  SongList ret;
-  for (const Song &song : cached_songs_) {
-    if (song.url().toLocalFile().section('/', 0, -2) == path) ret << song;
+  if (cached_songs_.contains(path)) {
+    return cached_songs_.values(path);
  }
-  return ret;
+  else return SongList();
+
+}
+
+bool CollectionWatcher::ScanTransaction::HasSongsWithMissingFingerprint(const QString &path) {
+
+  if (cached_songs_missing_fingerprint_dirty_) {
+    const SongList songs = watcher_->backend_->SongsWithMissingFingerprint(dir_);
+    for (const Song &song : songs) {
+      const QString p = song.url().toLocalFile().section('/', 0, -2);
+      cached_songs_missing_fingerprint_.insert(p, song);
+    }
+    cached_songs_missing_fingerprint_dirty_ = false;
+  }
+
+  return cached_songs_missing_fingerprint_.contains(path);

 }

@@ -292,8 +344,9 @@ bool CollectionWatcher::ScanTransaction::HasSeenSubdir(const QString &path) {

 SubdirectoryList CollectionWatcher::ScanTransaction::GetImmediateSubdirs(const QString &path) {

-  if (known_subdirs_dirty_)
+  if (known_subdirs_dirty_) {
    SetKnownSubdirs(watcher_->backend_->SubdirsInDirectory(dir_));
+  }

  SubdirectoryList ret;
  for (const Subdirectory &subdir : known_subdirs_) {
@@ -308,9 +361,12 @@ SubdirectoryList CollectionWatcher::ScanTransaction::GetImmediateSubdirs(const Q

 SubdirectoryList CollectionWatcher::ScanTransaction::GetAllSubdirs() {

-  if (known_subdirs_dirty_)
+  if (known_subdirs_dirty_) {
    SetKnownSubdirs(watcher_->backend_->SubdirsInDirectory(dir_));
+  }
+
  return known_subdirs_;
+
 }

 void CollectionWatcher::AddDirectory(const Directory &dir, const SubdirectoryList &subdirs) {
@@ -320,29 +376,36 @@ void CollectionWatcher::AddDirectory(const Directory &dir, const SubdirectoryLis
  if (subdirs.isEmpty()) {
    // This is a new directory that we've never seen before. Scan it fully.
    ScanTransaction transaction(this, dir.id, false, false, mark_songs_unavailable_);
+    const quint64 files_count = FilesCountForPath(&transaction, dir.path);
    transaction.SetKnownSubdirs(subdirs);
-    transaction.AddToProgressMax(1);
-    ScanSubdirectory(dir.path, Subdirectory(), &transaction);
+    transaction.AddToProgressMax(files_count);
+    ScanSubdirectory(dir.path, Subdirectory(), files_count, &transaction);
+    last_scan_time_ = QDateTime::currentDateTime().toSecsSinceEpoch();
  }
  else {
    // We can do an incremental scan - looking at the mtimes of each subdirectory and only rescan if the directory has changed.
    ScanTransaction transaction(this, dir.id, true, false, mark_songs_unavailable_);
+    QMap<QString, quint64> subdir_files_count;
+    const quint64 files_count = FilesCountForSubdirs(&transaction, subdirs, subdir_files_count);
    transaction.SetKnownSubdirs(subdirs);
-    transaction.AddToProgressMax(subdirs.count());
+    transaction.AddToProgressMax(files_count);
    for (const Subdirectory &subdir : subdirs) {
      if (stop_requested_) break;

-      if (scan_on_startup_) ScanSubdirectory(subdir.path, subdir, &transaction);
+      if (scan_on_startup_) ScanSubdirectory(subdir.path, subdir, subdir_files_count[subdir.path], &transaction);

      if (monitor_) AddWatch(dir, subdir.path);
    }
+
+    last_scan_time_ = QDateTime::currentDateTime().toSecsSinceEpoch();
+
  }

  emit CompilationsNeedUpdating();

 }

-void CollectionWatcher::ScanSubdirectory(const QString &path, const Subdirectory &subdir, ScanTransaction *t, bool force_noincremental) {
+void CollectionWatcher::ScanSubdirectory(const QString &path, const Subdirectory &subdir, const quint64 files_count, ScanTransaction *t, const bool force_noincremental) {

  QFileInfo path_info(path);
  QDir path_dir(path);
@@ -352,7 +415,6 @@ void CollectionWatcher::ScanSubdirectory(const QString &path, const Subdirectory
    QString real_path = path_info.symLinkTarget();
    for (const Directory &dir : qAsConst(watched_dirs_)) {
      if (real_path.startsWith(dir.path)) {
-        t->AddToProgress(1);
        return;
      }
    }
@@ -360,13 +422,19 @@ void CollectionWatcher::ScanSubdirectory(const QString &path, const Subdirectory

  // Do not scan directories containing a .nomedia or .nomusic file
  if (path_dir.exists(kNoMediaFile) || path_dir.exists(kNoMusicFile)) {
-    t->AddToProgress(1);
    return;
  }

-  if (!t->ignores_mtime() && !force_noincremental && t->is_incremental() && subdir.mtime == path_info.lastModified().toSecsSinceEpoch()) {
+  bool songs_missing_fingerprint = false;
+#ifdef HAVE_SONGFINGERPRINTING
+  if (song_tracking_) {
+    songs_missing_fingerprint = t->HasSongsWithMissingFingerprint(path);
+  }
+#endif
+
+  if (!t->ignores_mtime() && !force_noincremental && t->is_incremental() && subdir.mtime == path_info.lastModified().toSecsSinceEpoch() && !songs_missing_fingerprint) {
    // The directory hasn't changed since last time
-    t->AddToProgress(1);
+    t->AddToProgress(files_count);
    return;
  }

@@ -379,8 +447,7 @@ void CollectionWatcher::ScanSubdirectory(const QString &path, const Subdirectory
  SubdirectoryList previous_subdirs = t->GetImmediateSubdirs(path);
  for (const Subdirectory &prev_subdir : previous_subdirs) {
    if (!QFile::exists(prev_subdir.path) && prev_subdir.path != path) {
-      t->AddToProgressMax(1);
-      ScanSubdirectory(prev_subdir.path, prev_subdir, t, true);
+      ScanSubdirectory(prev_subdir.path, prev_subdir, 0, t, true);
    }
  }

@@ -402,15 +469,21 @@ void CollectionWatcher::ScanSubdirectory(const QString &path, const Subdirectory
        new_subdir.mtime = child_info.lastModified().toSecsSinceEpoch();
        my_new_subdirs << new_subdir;
      }
+      t->AddToProgress(1);
    }
    else {
      QString ext_part(ExtensionPart(child));
      QString dir_part(DirectoryPart(child));
-
-      if (sValidImages.contains(ext_part))
+      if (sValidImages.contains(ext_part)) {
        album_art[dir_part] << child;
-      else if (!child_info.isHidden())
+        t->AddToProgress(1);
+      }
+      else if (TagReaderClient::Instance()->IsMediaFileBlocking(child)) {
        files_on_disk << child;
+      }
+      else {
+        t->AddToProgress(1);
+      }
    }
  }

@@ -422,15 +495,18 @@ void CollectionWatcher::ScanSubdirectory(const QString &path, const Subdirectory
  QSet<QString> cues_processed;

  // Now compare the list from the database with the list of files on disk
-  for (const QString &file : files_on_disk) {
+  QStringList files_on_disk_copy = files_on_disk;
+  for (const QString &file : files_on_disk_copy) {
+
    if (stop_requested_) return;

-    // Associated cue
-    QString matching_cue = NoExtensionPart(file) + ".cue";
+    // Associated CUE
+    QString new_cue = NoExtensionPart(file) + ".cue";

-    Song matching_song(source_);
-    if (FindSongByPath(songs_in_db, file, &matching_song)) {
-      qint64 matching_cue_mtime = GetMtimeForCue(matching_cue);
+    SongList matching_songs;
+    if (FindSongsByPath(songs_in_db, file, &matching_songs)) {  // Found matching song in DB by path.
+
+      Song matching_song = matching_songs.first();

      // The song is in the database and still on disk.
      // Check the mtime to see if it's been changed since it was added.
@@ -439,18 +515,21 @@ void CollectionWatcher::ScanSubdirectory(const QString &path, const Subdirectory
      if (!file_info.exists()) {
        // Partially fixes race condition - if file was removed between being added to the list and now.
        files_on_disk.removeAll(file);
+        t->AddToProgress(1);
        continue;
      }

-      // CUE sheet's path from collection (if any)
-      QString song_cue = matching_song.cue_path();
-      qint64 song_cue_mtime = GetMtimeForCue(song_cue);
+      // CUE sheet's path from collection (if any).
+      qint64 matching_song_cue_mtime = GetMtimeForCue(matching_song.cue_path());

-      bool cue_deleted = song_cue_mtime == 0 && matching_song.has_cue();
-      bool cue_added = matching_cue_mtime != 0 && !matching_song.has_cue();
+      // CUE sheet's path from this file (if any).
+      qint64 new_cue_mtime = GetMtimeForCue(new_cue);
+
+      bool cue_added = new_cue_mtime != 0 && !matching_song.has_cue();
+      bool cue_deleted = matching_song_cue_mtime == 0 && matching_song.has_cue();

      // Watch out for CUE songs which have their mtime equal to qMax(media_file_mtime, cue_sheet_mtime)
-      bool changed = (matching_song.mtime() != qMax(file_info.lastModified().toSecsSinceEpoch(), song_cue_mtime)) || cue_deleted || cue_added;
+      bool changed = (matching_song.mtime() != qMax(file_info.lastModified().toSecsSinceEpoch(), matching_song_cue_mtime)) || cue_deleted || cue_added;

      // Also want to look to see whether the album art has changed
      QUrl image = ImageForSong(file, album_art);
@@ -458,53 +537,132 @@ void CollectionWatcher::ScanSubdirectory(const QString &path, const Subdirectory
        changed = true;
      }

-      // The song's changed - reread the metadata from file
-      if (t->ignores_mtime() || changed) {
-        qLog(Debug) << file << "changed";
+      bool missing_fingerprint = false;
+#ifdef HAVE_SONGFINGERPRINTING
+      if (song_tracking_ && matching_song.fingerprint().isEmpty()) {
+        missing_fingerprint = true;
+      }
+#endif
+
+      if (changed) {
+        qLog(Debug) << file << "has changed.";
+      }
+      else if (missing_fingerprint) {
+        qLog(Debug) << file << "is missing fingerprint.";
+      }
+
+      // The song's changed or missing fingerprint - create fingerprint and reread the metadata from file.
+      if (t->ignores_mtime() || changed || missing_fingerprint) {
+
+        QString fingerprint;
+#ifdef HAVE_SONGFINGERPRINTING
+        if (song_tracking_) {
+          Chromaprinter chromaprinter(file);
+          fingerprint = chromaprinter.CreateFingerprint();
+          if (fingerprint.isEmpty()) {
+            fingerprint = "NONE";
+          }
+        }
+#endif

        if (!cue_deleted && (matching_song.has_cue() || cue_added)) {  // If CUE associated.
-          UpdateCueAssociatedSongs(file, path, matching_cue, image, t);
+          UpdateCueAssociatedSongs(file, path, fingerprint, new_cue, image, matching_songs, t);
        }
        else {  // If no CUE or it's about to lose it.
-          UpdateNonCueAssociatedSong(file, matching_song, image, cue_deleted, t);
+          UpdateNonCueAssociatedSong(file, fingerprint, matching_songs, image, cue_deleted, t);
        }
      }

      // Nothing has changed - mark the song available without re-scanning
-      if (matching_song.is_unavailable()) {
-        if (matching_song.has_cue()) {
-          t->readded_songs << backend_->GetSongsByUrl(QUrl::fromLocalFile(file), true);
-        }
-        else {
-          t->readded_songs << matching_song;
-        }
+      else if (matching_song.is_unavailable()) {
+        t->readded_songs << matching_songs;
      }

    }
-    else {
-      // The song is on disk but not in the DB
-      SongList song_list = ScanNewFile(file, path, matching_cue, &cues_processed);
-
-      if (song_list.isEmpty()) {
-        continue;
+    else { // Search the DB by fingerprint.
+      QString fingerprint;
+#ifdef HAVE_SONGFINGERPRINTING
+      if (song_tracking_) {
+        Chromaprinter chromaprinter(file);
+        fingerprint = chromaprinter.CreateFingerprint();
+        if (fingerprint.isEmpty()) {
+          fingerprint = "NONE";
+        }
      }
+#endif
+      if (song_tracking_ && !fingerprint.isEmpty() && fingerprint != "NONE" && FindSongsByFingerprint(file, fingerprint, &matching_songs)) {

-      qLog(Debug) << file << "created";
-      // Choose an image for the song(s)
-      QUrl image = ImageForSong(file, album_art);
+        // The song is in the database and still on disk.
+        // Check the mtime to see if it's been changed since it was added.
+        QFileInfo file_info(file);
+        if (!file_info.exists()) {
+          // Partially fixes race condition - if file was removed between being added to the list and now.
+          files_on_disk.removeAll(file);
+          t->AddToProgress(1);
+          continue;
+        }

-      for (Song song : song_list) {
-        song.set_directory_id(t->dir());
-        if (song.art_automatic().isEmpty()) song.set_art_automatic(image);
-        t->new_songs << song;
+        // Make sure the songs aren't deleted, as they still exist elsewhere with a different fingerprint.
+        bool matching_songs_has_cue = false;
+        for (const Song &matching_song : matching_songs) {
+          QString matching_filename = matching_song.url().toLocalFile();
+          if (!t->files_changed_path_.contains(matching_filename)) {
+            t->files_changed_path_ << matching_filename;
+            qLog(Debug) << matching_filename << "has changed path to" << file;
+          }
+          if (t->deleted_songs.contains(matching_song)) {
+            t->deleted_songs.removeAll(matching_song);
+          }
+          if (matching_song.has_cue()) {
+            matching_songs_has_cue = true;
+          }
+        }
+
+        // CUE sheet's path from this file (if any).
+        const qint64 new_cue_mtime = GetMtimeForCue(new_cue);
+
+        const bool cue_deleted = new_cue_mtime == 0 && matching_songs_has_cue;
+        const bool cue_added = new_cue_mtime != 0 && !matching_songs_has_cue;
+
+        // Get new album art
+        QUrl image = ImageForSong(file, album_art);
+
+        if (!cue_deleted && (matching_songs_has_cue || cue_added)) {  // CUE associated.
+          UpdateCueAssociatedSongs(file, path, fingerprint, new_cue, image, matching_songs, t);
+        }
+        else {  // If no CUE or it's about to lose it.
+          UpdateNonCueAssociatedSong(file, fingerprint, matching_songs, image, cue_deleted, t);
+        }
+
+      }
+      else {  // The song is on disk but not in the DB
+
+        SongList songs = ScanNewFile(file, path, fingerprint, new_cue, &cues_processed);
+        if (songs.isEmpty()) {
+          t->AddToProgress(1);
+          continue;
+        }
+
+        qLog(Debug) << file << "is new.";
+
+        // Choose an image for the song(s)
+        QUrl image = ImageForSong(file, album_art);
+
+        for (Song song : songs) {
+          song.set_directory_id(t->dir());
+          if (song.art_automatic().isEmpty()) song.set_art_automatic(image);
+          t->new_songs << song;
+        }
      }
    }
+    t->AddToProgress(1);
  }

  // Look for deleted songs
  for (const Song &song : songs_in_db) {
-    if (!song.is_unavailable() && !files_on_disk.contains(song.url().toLocalFile())) {
-      qLog(Debug) << "Song deleted from disk:" << song.url().toLocalFile();
+    QString file = song.url().toLocalFile();
+    if (!song.is_unavailable() && !files_on_disk.contains(file) && !t->files_changed_path_.contains(file)) {
+      qLog(Debug) << "Song deleted from disk:" << file;
      t->deleted_songs << song;
    }
  }
@@ -515,162 +673,178 @@ void CollectionWatcher::ScanSubdirectory(const QString &path, const Subdirectory
  updated_subdir.mtime = path_info.exists() ? path_info.lastModified().toSecsSinceEpoch() : 0;
  updated_subdir.path = path;

-  if (subdir.directory_id == -1)
+  if (subdir.directory_id == -1) {
    t->new_subdirs << updated_subdir;
-  else
+  }
+  else {
    t->touched_subdirs << updated_subdir;
+  }

  if (updated_subdir.mtime == 0) { // Subdirectory deleted, mark it for removal from the watcher.
    t->deleted_subdirs << updated_subdir;
  }

-  t->AddToProgress(1);
-
  // Recurse into the new subdirs that we found
-  t->AddToProgressMax(my_new_subdirs.count());
  for (const Subdirectory &my_new_subdir : my_new_subdirs) {
    if (stop_requested_) return;
-    ScanSubdirectory(my_new_subdir.path, my_new_subdir, t, true);
+    ScanSubdirectory(my_new_subdir.path, my_new_subdir, 0, t, true);
  }

 }

-void CollectionWatcher::UpdateCueAssociatedSongs(const QString &file, const QString &path, const QString &matching_cue, const QUrl &image, ScanTransaction *t) {
-
-  QFile cue(matching_cue);
-  cue.open(QIODevice::ReadOnly);
-
-  SongList old_sections = backend_->GetSongsByUrl(QUrl::fromLocalFile(file));
+void CollectionWatcher::UpdateCueAssociatedSongs(const QString &file,
+                                                 const QString &path,
+                                                 const QString &fingerprint,
+                                                 const QString &matching_cue,
+                                                 const QUrl &image,
+                                                 const SongList &old_cue_songs,
+                                                 ScanTransaction *t) {

  QHash<quint64, Song> sections_map;
-  for (const Song &song : old_sections) {
-    sections_map[song.beginning_nanosec()] = song;
+  for (const Song &song : old_cue_songs) {
+    sections_map.insert(song.beginning_nanosec(), song);
  }

+  // Load new CUE songs
+  QFile cue_file(matching_cue);
+  if (!cue_file.exists() || !cue_file.open(QIODevice::ReadOnly)) return;
+  const SongList songs = cue_parser_->Load(&cue_file, matching_cue, path, false);
+  cue_file.close();
+
+  // Update every song that's in the CUE and collection
  QSet<int> used_ids;
+  for (Song new_cue_song : songs) {
+    new_cue_song.set_source(source_);
+    new_cue_song.set_directory_id(t->dir());
+    new_cue_song.set_fingerprint(fingerprint);

-  // Update every song that's in the cue and collection
-  for (Song cue_song : cue_parser_->Load(&cue, matching_cue, path)) {
-    cue_song.set_source(source_);
-    cue_song.set_directory_id(t->dir());
-
-    Song matching = sections_map[cue_song.beginning_nanosec()];
-    // A new section
-    if (!matching.is_valid()) {
-      t->new_songs << cue_song;
-      // changed section
+    if (sections_map.contains(new_cue_song.beginning_nanosec())) {  // Changed section
+      const Song matching_cue_song = sections_map[new_cue_song.beginning_nanosec()];
+      new_cue_song.set_id(matching_cue_song.id());
+      if (!new_cue_song.has_embedded_cover()) new_cue_song.set_art_automatic(image);
+      new_cue_song.MergeUserSetData(matching_cue_song);
+      AddChangedSong(file, matching_cue_song, new_cue_song, t);
+      used_ids.insert(matching_cue_song.id());
    }
-    else {
-      PreserveUserSetData(file, image, matching, &cue_song, t);
-      used_ids.insert(matching.id());
+    else {  // A new section
+      t->new_songs << new_cue_song;
    }
  }

  // Sections that are now missing
-  for (const Song &matching : old_sections) {
-    if (!used_ids.contains(matching.id())) {
-      t->deleted_songs << matching;
+  for (const Song &old_cue : old_cue_songs) {
+    if (!used_ids.contains(old_cue.id())) {
+      t->deleted_songs << old_cue;
    }
  }

 }

-void CollectionWatcher::UpdateNonCueAssociatedSong(const QString &file, const Song &matching_song, const QUrl &image, bool cue_deleted, ScanTransaction *t) {
+void CollectionWatcher::UpdateNonCueAssociatedSong(const QString &file,
+                                                   const QString &fingerprint,
+                                                   const SongList &matching_songs,
+                                                   const QUrl &image,
+                                                   const bool cue_deleted,
+                                                   ScanTransaction *t) {

  // If a CUE got deleted, we turn it's first section into the new 'raw' (cueless) song and we just remove the rest of the sections from the collection
+  const Song &matching_song = matching_songs.first();
  if (cue_deleted) {
-    for (const Song &song : backend_->GetSongsByUrl(QUrl::fromLocalFile(file))) {
-      if (!song.IsMetadataAndArtEqual(matching_song)) {
+    for (const Song &song : matching_songs) {
+      if (!song.IsMetadataAndMoreEqual(matching_song)) {
        t->deleted_songs << song;
      }
    }
  }

  Song song_on_disk(source_);
-  song_on_disk.set_directory_id(t->dir());
  TagReaderClient::Instance()->ReadFileBlocking(file, &song_on_disk);
-
  if (song_on_disk.is_valid()) {
-    PreserveUserSetData(file, image, matching_song, &song_on_disk, t);
+    song_on_disk.set_source(source_);
+    song_on_disk.set_directory_id(t->dir());
+    song_on_disk.set_id(matching_song.id());
+    song_on_disk.set_fingerprint(fingerprint);
+    if (!song_on_disk.has_embedded_cover()) song_on_disk.set_art_automatic(image);
+    song_on_disk.MergeUserSetData(matching_song);
+    AddChangedSong(file, matching_song, song_on_disk, t);
  }

 }

-SongList CollectionWatcher::ScanNewFile(const QString &file, const QString &path, const QString &matching_cue, QSet<QString> *cues_processed) {
+SongList CollectionWatcher::ScanNewFile(const QString &file, const QString &path, const QString &fingerprint, const QString &matching_cue, QSet<QString> *cues_processed) {

-  SongList song_list;
+  SongList songs;

  quint64 matching_cue_mtime = GetMtimeForCue(matching_cue);
  if (matching_cue_mtime) {  // If it's a CUE - create virtual tracks
-    // Don't process the same cue many times
-    if (cues_processed->contains(matching_cue)) return song_list;

-    QFile cue(matching_cue);
-    cue.open(QIODevice::ReadOnly);
+    // Don't process the same CUE many times
+    if (cues_processed->contains(matching_cue)) return songs;
+
+    QFile cue_file(matching_cue);
+    if (!cue_file.exists() || !cue_file.open(QIODevice::ReadOnly)) return songs;

    // Ignore FILEs pointing to other media files.
    // Also, watch out for incorrect media files.
    // Playlist parser for CUEs considers every entry in sheet valid and we don't want invalid media getting into collection!
    QString file_nfd = file.normalized(QString::NormalizationForm_D);
-    for (Song &cue_song : cue_parser_->Load(&cue, matching_cue, path)) {
+    SongList cue_congs = cue_parser_->Load(&cue_file, matching_cue, path, false);
+    for (Song &cue_song : cue_congs) {
      cue_song.set_source(source_);
+      cue_song.set_fingerprint(fingerprint);
      if (cue_song.url().toLocalFile().normalized(QString::NormalizationForm_D) == file_nfd) {
-        if (TagReaderClient::Instance()->IsMediaFileBlocking(file)) {
-          song_list << cue_song;
-        }
+        songs << cue_song;
      }
    }
-
-    if (!song_list.isEmpty()) {
+    if (!songs.isEmpty()) {
      *cues_processed << matching_cue;
    }
-
  }
  else {  // It's a normal media file
    Song song(source_);
    TagReaderClient::Instance()->ReadFileBlocking(file, &song);
    if (song.is_valid()) {
      song.set_source(source_);
-      song_list << song;
+      song.set_fingerprint(fingerprint);
+      songs << song;
    }
  }

-  return song_list;
+  return songs;

 }

-void CollectionWatcher::PreserveUserSetData(const QString &file, const QUrl &image, const Song &matching_song, Song *out, ScanTransaction *t) {
+void CollectionWatcher::AddChangedSong(const QString &file, const Song &matching_song, const Song &new_song, ScanTransaction *t) {

-  out->set_id(matching_song.id());
-
-  // Previous versions of Clementine incorrectly overwrote this and stored it in the DB,
-  // so we can't rely on matching_song to know if it has embedded artwork or not, but we can check here.
-  if (!out->has_embedded_cover()) out->set_art_automatic(image);
-
-  out->MergeUserSetData(matching_song);
-
-  // The song was deleted from the database (e.g. due to an unmounted filesystem), but has been restored.
  if (matching_song.is_unavailable()) {
-    qLog(Debug) << file << " unavailable song restored";
-
-    t->new_songs << *out;
+    qLog(Debug) << file << "unavailable song restored.";
+    t->new_songs << new_song;
  }
-  else if (!matching_song.IsMetadataAndArtEqual(*out)) {
-    qLog(Debug) << file << "metadata changed";
-
-    // Update the song in the DB
-    t->new_songs << *out;
+  else if (!matching_song.IsMetadataEqual(new_song)) {
+    qLog(Debug) << file << "metadata changed.";
+    t->new_songs << new_song;
+  }
+  else if (matching_song.fingerprint() != new_song.fingerprint()) {
+    qLog(Debug) << file << "fingerprint changed.";
+    t->new_songs << new_song;
+  }
+  else if (matching_song.art_automatic() != new_song.art_automatic() || matching_song.art_manual() != new_song.art_manual()) {
+    qLog(Debug) << file << "art changed.";
+    t->new_songs << new_song;
+  }
+  else if (matching_song.mtime() != new_song.mtime()) {
+    qLog(Debug) << file << "mtime changed.";
+    t->touched_songs << new_song;
  }
  else {
-    // Only the mtime's changed
-    t->touched_songs << *out;
+    qLog(Debug) << file << "unchanged.";
+    t->touched_songs << new_song;
  }

 }

 quint64 CollectionWatcher::GetMtimeForCue(const QString &cue_path) {

-  // Slight optimisation
  if (cue_path.isEmpty()) {
    return 0;
  }
@@ -721,16 +895,46 @@ void CollectionWatcher::RemoveDirectory(const Directory &dir) {

 }

-bool CollectionWatcher::FindSongByPath(const SongList &list, const QString &path, Song *out) {
+bool CollectionWatcher::FindSongsByPath(const SongList &songs, const QString &path, SongList *out) {

-  // TODO: Make this faster
-  for (const Song &song : list) {
+  for (const Song &song : songs) {
    if (song.url().toLocalFile() == path) {
-      *out = song;
+      *out << song;
+    }
+  }
+
+  return !out->isEmpty();
+
+}
+
+bool CollectionWatcher::FindSongsByFingerprint(const QString &file, const QString &fingerprint, SongList *out) {
+
+  SongList songs = backend_->GetSongsByFingerprint(fingerprint);
+  for (const Song &song : songs) {
+    QString filename = song.url().toLocalFile();
+    QFileInfo info(filename);
+    // Allow mulitiple songs in different directories with the same fingerprint.
+    // Only use the matching song by fingerprint if it doesn't already exist in a different path.
+    if (file == filename || !info.exists()) {
+      *out << song;
+    }
+  }
+
+  return !out->isEmpty();
+
+}
+
+bool CollectionWatcher::FindSongsByFingerprint(const QString &file, const SongList &songs, const QString &fingerprint, SongList *out) {
+
+  for (const Song &song : songs) {
+    QString filename = song.url().toLocalFile();
+    if (song.fingerprint() == fingerprint && (file == filename || !QFileInfo(filename).exists())) {
+      *out << song;
      return true;
    }
  }
-  return false;
+
+  return !out->isEmpty();

 }

@@ -758,7 +962,13 @@ void CollectionWatcher::RescanPathsNow() {
  for (const int dir : dirs) {
    if (stop_requested_) break;
    ScanTransaction transaction(this, dir, false, false, mark_songs_unavailable_);
-    transaction.AddToProgressMax(rescan_queue_[dir].count());
+
+    QMap<QString, quint64> subdir_files_count;
+    for (const QString &path : rescan_queue_[dir]) {
+      quint64 files_count = FilesCountForPath(&transaction, path);
+      subdir_files_count[path] = files_count;
+      transaction.AddToProgressMax(files_count);
+    }

    for (const QString &path : rescan_queue_[dir]) {
      if (stop_requested_) break;
@@ -766,7 +976,7 @@ void CollectionWatcher::RescanPathsNow() {
      subdir.directory_id = dir;
      subdir.mtime = 0;
      subdir.path = path;
-      ScanSubdirectory(path, subdir, &transaction);
+      ScanSubdirectory(path, subdir, subdir_files_count[path], &transaction);
    }
  }

@@ -877,6 +1087,16 @@ void CollectionWatcher::RescanTracksAsync(const SongList &songs) {

 }

+void CollectionWatcher::IncrementalScanCheck() {
+
+  qint64 duration = QDateTime::currentDateTime().toSecsSinceEpoch() - last_scan_time_;
+  if (duration >= 86400) {
+    qLog(Debug) << "Performing periodic incremental scan.";
+    IncrementalScanNow();
+  }
+
+}
+
 void CollectionWatcher::IncrementalScanNow() { PerformScan(true, false); }

 void CollectionWatcher::FullScanNow() { PerformScan(false, true); }
@@ -895,7 +1115,8 @@ void CollectionWatcher::RescanTracksNow() {
    if (!scanned_dirs.contains(songdir)) {
      qLog(Debug) << "Song" << song.title() << "dir id" << song.directory_id() << "dir" << songdir;
      ScanTransaction transaction(this, song.directory_id(), false, false, mark_songs_unavailable_);
-      ScanSubdirectory(songdir, Subdirectory(), &transaction);
+      quint64 files_count = FilesCountForPath(&transaction, songdir);
+      ScanSubdirectory(songdir, Subdirectory(), files_count, &transaction);
      scanned_dirs << songdir;
      emit CompilationsNeedUpdating();
    }
@@ -908,7 +1129,7 @@ void CollectionWatcher::RescanTracksNow() {

 }

-void CollectionWatcher::PerformScan(bool incremental, bool ignore_mtimes) {
+void CollectionWatcher::PerformScan(const bool incremental, const bool ignore_mtimes) {

  stop_requested_ = false;

@@ -928,15 +1149,72 @@ void CollectionWatcher::PerformScan(bool incremental, bool ignore_mtimes) {
      subdirs << subdir;
    }

-    transaction.AddToProgressMax(subdirs.count());
+    QMap<QString, quint64> subdir_files_count;
+    quint64 files_count = FilesCountForSubdirs(&transaction, subdirs, subdir_files_count);
+    transaction.AddToProgressMax(files_count);

    for (const Subdirectory &subdir : subdirs) {
      if (stop_requested_) break;
-
-      ScanSubdirectory(subdir.path, subdir, &transaction);
+      ScanSubdirectory(subdir.path, subdir, subdir_files_count[subdir.path], &transaction);
    }
+
  }

+  last_scan_time_ = QDateTime::currentDateTime().toSecsSinceEpoch();
+
  emit CompilationsNeedUpdating();

 }
+
+quint64 CollectionWatcher::FilesCountForPath(ScanTransaction *t, const QString &path) {
+
+  quint64 i = 0;
+  QDirIterator it(path, QDir::Dirs | QDir::Files | QDir::Hidden | QDir::NoDotAndDotDot);
+  while (it.hasNext()) {
+
+    if (stop_requested_) break;
+
+    QString child = it.next();
+    QFileInfo path_info(child);
+
+    if (path_info.isDir()) {
+      if (path_info.exists(kNoMediaFile) || path_info.exists(kNoMusicFile)) {
+        continue;
+      }
+      if (path_info.isSymLink()) {
+        QString real_path = path_info.symLinkTarget();
+        for (const Directory &dir : qAsConst(watched_dirs_)) {
+          if (real_path.startsWith(dir.path)) {
+            continue;
+          }
+        }
+      }
+
+      if (!t->HasSeenSubdir(child) && !path_info.isHidden()) {
+        // We haven't seen this subdirectory before, so we need to include the file count for this directory too.
+        i += FilesCountForPath(t, child);
+      }
+
+    }
+
+    ++i;
+
+  }
+
+  return i;
+
+}
+
+quint64 CollectionWatcher::FilesCountForSubdirs(ScanTransaction *t, const SubdirectoryList &subdirs, QMap<QString, quint64> &subdir_files_count) {
+
+  quint64 i = 0;
+  for (const Subdirectory &subdir : subdirs) {
+    if (stop_requested_) break;
+    const quint64 files_count = FilesCountForPath(t, subdir.path);
+    subdir_files_count[subdir.path] = files_count;
+    i += files_count;
+  }
+
+  return i;
+
+}