From 9083c578cc8e99c9ff15298f47ee534bbc4a20c3 Mon Sep 17 00:00:00 2001 From: Jonas Kvinge Date: Sun, 30 Jun 2019 21:06:07 +0200 Subject: [PATCH] Add live scanning (#199) --- src/collection/collection.cpp | 9 ++ src/collection/collection.h | 2 + src/collection/collectionview.cpp | 14 +++ src/collection/collectionview.h | 2 + src/collection/collectionwatcher.cpp | 151 ++++++++++++++++++------ src/collection/collectionwatcher.h | 20 +++- src/core/mainwindow.cpp | 26 ++++ src/core/mainwindow.h | 1 + src/core/mainwindow.ui | 17 ++- src/settings/collectionsettingspage.cpp | 4 + src/settings/collectionsettingspage.ui | 14 +++ 11 files changed, 216 insertions(+), 44 deletions(-) diff --git a/src/collection/collection.cpp b/src/collection/collection.cpp index 28f86e204..5364d9f9f 100644 --- a/src/collection/collection.cpp +++ b/src/collection/collection.cpp @@ -103,6 +103,15 @@ void SCollection::IncrementalScan() { watcher_->IncrementalScanAsync(); } void SCollection::FullScan() { watcher_->FullScanAsync(); } +void SCollection::AbortScan() { watcher_->Stop(); } + +void SCollection::Rescan(const SongList &songs) { + + qLog(Debug) << "Rescan" << songs.size() << "songs"; + if (!songs.isEmpty()) watcher_->RescanTracksAsync(songs); + +} + void SCollection::PauseWatcher() { watcher_->SetRescanPausedAsync(true); } void SCollection::ResumeWatcher() { watcher_->SetRescanPausedAsync(false); } diff --git a/src/collection/collection.h b/src/collection/collection.h index 6ef900f4e..54a0c695d 100644 --- a/src/collection/collection.h +++ b/src/collection/collection.h @@ -66,6 +66,8 @@ class SCollection : public QObject { void ResumeWatcher(); void FullScan(); + void AbortScan(); + void Rescan(const SongList &songs); private slots: void IncrementalScan(); diff --git a/src/collection/collectionview.cpp b/src/collection/collectionview.cpp index a79e8bf12..df1c0e59b 100644 --- a/src/collection/collectionview.cpp +++ b/src/collection/collectionview.cpp @@ -60,6 +60,7 @@ #include "core/iconloader.h" #include "core/mimedata.h" #include "core/utilities.h" +#include "collection.h" #include "collectionbackend.h" #include "collectiondirectorymodel.h" #include "collectionfilterwidget.h" @@ -349,6 +350,10 @@ void CollectionView::contextMenuEvent(QContextMenuEvent *e) { edit_tracks_ = context_menu_->addAction(IconLoader::Load("edit-rename"), tr("Edit tracks information..."), this, SLOT(EditTracks())); show_in_browser_ = context_menu_->addAction(IconLoader::Load("document-open-folder"), tr("Show in file browser..."), this, SLOT(ShowInBrowser())); + context_menu_->addSeparator(); + + rescan_songs_ = context_menu_->addAction(tr("Rescan song(s)"), this, SLOT(RescanSongs())); + context_menu_->addSeparator(); show_in_various_ = context_menu_->addAction( tr("Show in various artists"), this, SLOT(ShowInVarious())); no_show_in_various_ = context_menu_->addAction( tr("Don't show in various artists"), this, SLOT(NoShowInVarious())); @@ -395,6 +400,9 @@ void CollectionView::contextMenuEvent(QContextMenuEvent *e) { edit_track_->setVisible(regular_editable <= 1); edit_track_->setEnabled(regular_editable == 1); + rescan_songs_->setVisible(edit_track_->isVisible()); + rescan_songs_->setEnabled(true); + organise_->setVisible(regular_elements_only); #ifndef Q_OS_WIN copy_to_device_->setVisible(regular_elements_only); @@ -561,6 +569,12 @@ void CollectionView::EditTagError(const QString &message) { emit Error(message); } +void CollectionView::RescanSongs() { + + app_->collection()->Rescan(GetSelectedSongs()); + +} + void CollectionView::CopyToDevice() { #ifndef Q_OS_WIN if (!organise_dialog_) diff --git a/src/collection/collectionview.h b/src/collection/collectionview.h index d4b019dc7..570e58a70 100644 --- a/src/collection/collectionview.h +++ b/src/collection/collectionview.h @@ -114,6 +114,7 @@ class CollectionView : public AutoExpandingTreeView { void Organise(); void CopyToDevice(); void EditTracks(); + void RescanSongs(); void ShowInBrowser(); void ShowInVarious(); void NoShowInVarious(); @@ -148,6 +149,7 @@ class CollectionView : public AutoExpandingTreeView { QAction *delete_; QAction *edit_track_; QAction *edit_tracks_; + QAction *rescan_songs_; QAction *show_in_browser_; QAction *show_in_various_; QAction *no_show_in_various_; diff --git a/src/collection/collectionwatcher.cpp b/src/collection/collectionwatcher.cpp index 90701ce0f..aad789c20 100644 --- a/src/collection/collectionwatcher.cpp +++ b/src/collection/collectionwatcher.cpp @@ -70,9 +70,12 @@ CollectionWatcher::CollectionWatcher(Song::Source source, QObject *parent) backend_(nullptr), task_manager_(nullptr), fs_watcher_(FileSystemWatcherInterface::Create(this)), - stop_requested_(false), scan_on_startup_(true), monitor_(true), + live_scanning_(false), + prevent_delete_(false), + stop_requested_(false), + rescan_in_progress_(false), rescan_timer_(new QTimer(this)), rescan_paused_(false), total_watches_(0), @@ -90,15 +93,17 @@ CollectionWatcher::CollectionWatcher(Song::Source source, QObject *parent) connect(rescan_timer_, SIGNAL(timeout()), SLOT(RescanPathsNow())); } -CollectionWatcher::ScanTransaction::ScanTransaction(CollectionWatcher *watcher, int dir, bool incremental, bool ignores_mtime) +CollectionWatcher::ScanTransaction::ScanTransaction(CollectionWatcher *watcher, const int dir, const bool incremental, const bool ignores_mtime, const bool prevent_delete) : progress_(0), progress_max_(0), dir_(dir), incremental_(incremental), ignores_mtime_(ignores_mtime), + prevent_delete_(prevent_delete), watcher_(watcher), cached_songs_dirty_(true), - known_subdirs_dirty_(true) { + known_subdirs_dirty_(true) + { QString description; @@ -115,30 +120,20 @@ CollectionWatcher::ScanTransaction::ScanTransaction(CollectionWatcher *watcher, CollectionWatcher::ScanTransaction::~ScanTransaction() { // If we're stopping then don't commit the transaction - if (watcher_->stop_requested_) return; + if (!watcher_->stop_requested_) { - if (!new_songs.isEmpty()) emit watcher_->NewOrUpdatedSongs(new_songs); + CommitNewOrUpdatedSongs(); - if (!touched_songs.isEmpty()) emit watcher_->SongsMTimeUpdated(touched_songs); - - if (!deleted_songs.isEmpty()) emit watcher_->SongsDeleted(deleted_songs); - - if (!readded_songs.isEmpty()) emit watcher_->SongsReadded(readded_songs); - - if (!new_subdirs.isEmpty()) emit watcher_->SubdirsDiscovered(new_subdirs); - - if (!touched_subdirs.isEmpty()) - emit watcher_->SubdirsMTimeUpdated(touched_subdirs); - - watcher_->task_manager_->SetTaskFinished(task_id_); - - if (watcher_->monitor_) { - // Watch the new subdirectories - for (const Subdirectory &subdir : new_subdirs) { - watcher_->AddWatch(watcher_->watched_dirs_[dir_], subdir.path); + if (watcher_->monitor_) { + // Watch the new subdirectories + for (const Subdirectory &subdir : new_subdirs) { + watcher_->AddWatch(watcher_->watched_dirs_[dir_], subdir.path); + } } } + watcher_->task_manager_->SetTaskFinished(task_id_); + } void CollectionWatcher::ScanTransaction::AddToProgress(int n) { @@ -155,6 +150,41 @@ void CollectionWatcher::ScanTransaction::AddToProgressMax(int n) { } +void CollectionWatcher::ScanTransaction::CommitNewOrUpdatedSongs() { + + if (!new_songs.isEmpty()) { + emit watcher_->NewOrUpdatedSongs(new_songs); + new_songs.clear(); + } + + if (!touched_songs.isEmpty()) { + emit watcher_->SongsMTimeUpdated(touched_songs); + touched_songs.clear(); + } + + if (!deleted_songs.isEmpty() && !prevent_delete_) { + emit watcher_->SongsDeleted(deleted_songs); + deleted_songs.clear(); + } + + if (!readded_songs.isEmpty()) { + emit watcher_->SongsReadded(readded_songs); + readded_songs.clear(); + } + + if (!new_subdirs.isEmpty()) { + emit watcher_->SubdirsDiscovered(new_subdirs); + new_subdirs.clear(); + } + + if (!touched_subdirs.isEmpty()) { + emit watcher_->SubdirsMTimeUpdated(touched_subdirs); + touched_subdirs.clear(); + } + +} + + SongList CollectionWatcher::ScanTransaction::FindSongsInSubdirectory(const QString &path) { if (cached_songs_dirty_) { @@ -219,18 +249,18 @@ void CollectionWatcher::AddDirectory(const Directory &dir, const SubdirectoryLis if (subdirs.isEmpty()) { // This is a new directory that we've never seen before. Scan it fully. - ScanTransaction transaction(this, dir.id, false); + ScanTransaction transaction(this, dir.id, false, false, prevent_delete_); transaction.SetKnownSubdirs(subdirs); transaction.AddToProgressMax(1); ScanSubdirectory(dir.path, Subdirectory(), &transaction); } else { // We can do an incremental scan - looking at the mtimes of each subdirectory and only rescan if the directory has changed. - ScanTransaction transaction(this, dir.id, true); + ScanTransaction transaction(this, dir.id, true, false, prevent_delete_); transaction.SetKnownSubdirs(subdirs); transaction.AddToProgressMax(subdirs.count()); for (const Subdirectory &subdir : subdirs) { - if (stop_requested_) return; + if (stop_requested_) break; if (scan_on_startup_) ScanSubdirectory(subdir.path, subdir, &transaction); @@ -388,7 +418,6 @@ void CollectionWatcher::ScanSubdirectory(const QString &path, const Subdirectory QString image = ImageForSong(file, album_art); for (Song song : song_list) { - song.set_source(source_); song.set_directory_id(t->dir()); if (song.art_automatic().isEmpty()) song.set_art_automatic(image); t->new_songs << song; @@ -417,6 +446,8 @@ void CollectionWatcher::ScanSubdirectory(const QString &path, const Subdirectory t->AddToProgress(1); + if (live_scanning_) t->CommitNewOrUpdatedSongs(); + // Recurse into the new subdirs that we found t->AddToProgressMax(my_new_subdirs.count()); for (const Subdirectory &my_new_subdir : my_new_subdirs) { @@ -477,8 +508,7 @@ void CollectionWatcher::UpdateNonCueAssociatedSong(const QString &file, const So } } - Song song_on_disk; - song_on_disk.set_source(source_); + Song song_on_disk(source_); song_on_disk.set_directory_id(t->dir()); TagReaderClient::Instance()->ReadFileBlocking(file, &song_on_disk); @@ -520,9 +550,8 @@ SongList CollectionWatcher::ScanNewFile(const QString &file, const QString &path // it's a normal media file } else { - Song song; + Song song(source_); TagReaderClient::Instance()->ReadFileBlocking(file, &song); - if (song.is_valid()) { song_list << song; } @@ -636,11 +665,11 @@ void CollectionWatcher::RescanPathsNow() { for (int dir : rescan_queue_.keys()) { if (stop_requested_) return; - ScanTransaction transaction(this, dir, false); + ScanTransaction transaction(this, dir, false, false, prevent_delete_); transaction.AddToProgressMax(rescan_queue_[dir].count()); for (const QString &path : rescan_queue_[dir]) { - if (stop_requested_) return; + if (stop_requested_) break; Subdirectory subdir; subdir.directory_id = dir; subdir.mtime = 0; @@ -685,7 +714,7 @@ QString CollectionWatcher::PickBestImage(const QStringList &images) { QString biggest_path; for (const QString &path : filtered) { - if (stop_requested_) return QString(); + if (stop_requested_) break; QImage image(path); if (image.isNull()) continue; @@ -728,16 +757,18 @@ void CollectionWatcher::ReloadSettings() { const bool was_monitoring_before = monitor_; QSettings s; - s.beginGroup(CollectionSettingsPage::kSettingsGroup); scan_on_startup_ = s.value("startup_scan", true).toBool(); monitor_ = s.value("monitor", true).toBool(); + live_scanning_ = s.value("live_scanning", false).toBool(); + prevent_delete_ = s.value("prevent_delete", false).toBool(); + QStringList filters = s.value("cover_art_patterns", QStringList() << "front" << "cover").toStringList(); + s.endGroup(); best_image_filters_.clear(); - QStringList filters = s.value("cover_art_patterns", QStringList() << "front" << "cover").toStringList(); for (const QString &filter : filters) { - QString s = filter.trimmed(); - if (!s.isEmpty()) best_image_filters_ << s; + QString str = filter.trimmed(); + if (!str.isEmpty()) best_image_filters_ << str; } if (!monitor_ && was_monitoring_before) { @@ -780,19 +811,61 @@ void CollectionWatcher::FullScanAsync() { } +void CollectionWatcher::RescanTracksAsync(const SongList &songs) { + + // Is List thread safe? if not, this may crash. + song_rescan_queue_.append(songs); + + // Call only if it's not already running + if (!rescan_in_progress_) + QMetaObject::invokeMethod(this, "RescanTracksNow", Qt::QueuedConnection); + +} + void CollectionWatcher::IncrementalScanNow() { PerformScan(true, false); } void CollectionWatcher::FullScanNow() { PerformScan(false, true); } +void CollectionWatcher::RescanTracksNow() { + + Q_ASSERT(!rescan_in_progress_); + stop_requested_ = false; + + // Currently we are too stupid to rescan one file at a time, so we'll just scan the full directiories + QStringList scanned_dirs; // To avoid double scans + while (!song_rescan_queue_.isEmpty()) { + if (stop_requested_) break; + Song song = song_rescan_queue_.takeFirst(); + QString songdir = song.url().toLocalFile().section('/', 0, -2); + if (!scanned_dirs.contains(songdir)) { + qLog(Debug) << "Song" << song.title() << "dir id" << song.directory_id() << "dir" << songdir; + ScanTransaction transaction(this, song.directory_id(), false, false, prevent_delete_); + ScanSubdirectory(songdir, Subdirectory(), &transaction); + scanned_dirs << songdir; + emit CompilationsNeedUpdating(); + } + else { + qLog(Debug) << "Directory" << songdir << "already scanned - skipping."; + } + } + Q_ASSERT(song_rescan_queue_.isEmpty()); + rescan_in_progress_ = false; + +} + void CollectionWatcher::PerformScan(bool incremental, bool ignore_mtimes) { + stop_requested_ = false; + for (const Directory &dir : watched_dirs_.values()) { - ScanTransaction transaction(this, dir.id, incremental, ignore_mtimes); + + if (stop_requested_) break; + ScanTransaction transaction(this, dir.id, incremental, ignore_mtimes, prevent_delete_); SubdirectoryList subdirs(transaction.GetAllSubdirs()); transaction.AddToProgressMax(subdirs.count()); for (const Subdirectory &subdir : subdirs) { - if (stop_requested_) return; + if (stop_requested_) break; ScanSubdirectory(subdir.path, subdir, &transaction); } diff --git a/src/collection/collectionwatcher.h b/src/collection/collectionwatcher.h index fdbc4cd30..4c09895fe 100644 --- a/src/collection/collectionwatcher.h +++ b/src/collection/collectionwatcher.h @@ -54,6 +54,7 @@ class CollectionWatcher : public QObject { void IncrementalScanAsync(); void FullScanAsync(); + void RescanTracksAsync(const SongList &songs); void SetRescanPausedAsync(bool pause); void ReloadSettingsAsync(); @@ -85,7 +86,7 @@ signals: // Multiple calls to FindSongsInSubdirectory during one transaction will only result in one call to CollectionBackend::FindSongsInDirectory. class ScanTransaction { public: - ScanTransaction(CollectionWatcher *watcher, int dir, bool incremental, bool ignores_mtime = false); + ScanTransaction(CollectionWatcher *watcher, const int dir, const bool incremental, const bool ignores_mtime, const bool prevent_delete); ~ScanTransaction(); SongList FindSongsInSubdirectory(const QString &path); @@ -97,6 +98,9 @@ signals: void AddToProgress(int n = 1); void AddToProgressMax(int n); + // Emits the signals for new & deleted songs etc and clears the lists. This causes the new stuff to be updated on UI. + void CommitNewOrUpdatedSongs(); + int dir() const { return dir_; } bool is_incremental() const { return incremental_; } bool ignores_mtime() const { return ignores_mtime_; } @@ -124,6 +128,10 @@ signals: // Also, since it's ignoring mtimes on folders too, it will go as deep in the folder hierarchy as it's possible. bool ignores_mtime_; + // Set this to true to prevent deleting missing files from database. + // Useful for unstable network connections. + bool prevent_delete_; + CollectionWatcher *watcher_; SongList cached_songs_; @@ -137,6 +145,7 @@ signals: void DirectoryChanged(const QString &path); void IncrementalScanNow(); void FullScanNow(); + void RescanTracksNow(); void RescanPathsNow(); void ScanSubdirectory(const QString &path, const Subdirectory &subdir, ScanTransaction *t, bool force_noincremental = false); @@ -174,9 +183,13 @@ signals: // e.g. using ["front", "cover"] would identify front.jpg and exclude back.jpg. QStringList best_image_filters_; - bool stop_requested_; bool scan_on_startup_; bool monitor_; + bool live_scanning_; + bool prevent_delete_; + + bool stop_requested_; + bool rescan_in_progress_; // True if RescanTracksNow() has been called and is working. QMap watched_dirs_; QTimer *rescan_timer_; @@ -188,6 +201,9 @@ signals: CueParser *cue_parser_; static QStringList sValidImages; + + SongList song_rescan_queue_; // Set by ui thread + }; inline QString CollectionWatcher::NoExtensionPart(const QString& fileName) { diff --git a/src/core/mainwindow.cpp b/src/core/mainwindow.cpp index c95ab2522..587a5f467 100644 --- a/src/core/mainwindow.cpp +++ b/src/core/mainwindow.cpp @@ -410,6 +410,7 @@ MainWindow::MainWindow(Application *app, SystemTrayIcon *tray_icon, OSD *osd, co connect(ui_->action_remove_unavailable, SIGNAL(triggered()), app_->playlist_manager(), SLOT(RemoveUnavailableCurrent())); connect(ui_->action_remove_from_playlist, SIGNAL(triggered()), SLOT(PlaylistRemoveCurrent())); connect(ui_->action_edit_track, SIGNAL(triggered()), SLOT(EditTracks())); + connect(ui_->action_rescan_songs, SIGNAL(triggered()), SLOT(RescanSongs())); connect(ui_->action_renumber_tracks, SIGNAL(triggered()), SLOT(RenumberTracks())); connect(ui_->action_selection_set_value, SIGNAL(triggered()), SLOT(SelectionSetValue())); connect(ui_->action_edit_value, SIGNAL(triggered()), SLOT(EditValue())); @@ -434,6 +435,7 @@ MainWindow::MainWindow(Application *app, SystemTrayIcon *tray_icon, OSD *osd, co connect(ui_->action_jump, SIGNAL(triggered()), ui_->playlist->view(), SLOT(JumpToCurrentlyPlayingTrack())); connect(ui_->action_update_collection, SIGNAL(triggered()), app_->collection(), SLOT(IncrementalScan())); connect(ui_->action_full_collection_scan, SIGNAL(triggered()), app_->collection(), SLOT(FullScan())); + connect(ui_->action_abort_collection_scan, SIGNAL(triggered()), app_->collection(), SLOT(AbortScan())); #if defined(HAVE_GSTREAMER) connect(ui_->action_add_files_to_transcoder, SIGNAL(triggered()), SLOT(AddFilesToTranscoder())); #else @@ -614,6 +616,7 @@ MainWindow::MainWindow(Application *app, SystemTrayIcon *tray_icon, OSD *osd, co playlist_menu_->addAction(ui_->action_renumber_tracks); playlist_menu_->addAction(ui_->action_selection_set_value); playlist_menu_->addAction(ui_->action_auto_complete_tags); + playlist_menu_->addAction(ui_->action_rescan_songs); #ifdef HAVE_GSTREAMER playlist_menu_->addAction(ui_->action_add_files_to_transcoder); #endif @@ -1512,6 +1515,10 @@ void MainWindow::PlaylistRightClick(const QPoint &global_pos, const QModelIndex ui_->action_auto_complete_tags->setEnabled(false); ui_->action_auto_complete_tags->setVisible(false); #endif + + ui_->action_rescan_songs->setEnabled(editable); + ui_->action_rescan_songs->setVisible(editable); + // the rest of the read / write actions work only when there are no CUEs involved if (cue_selected) editable = 0; @@ -1659,6 +1666,25 @@ void MainWindow::PlaylistStopAfter() { app_->playlist_manager()->current()->StopAfter(playlist_menu_index_.row()); } +void MainWindow::RescanSongs() { + + SongList songs; + PlaylistItemList items; + + for (const QModelIndex& index : ui_->playlist->view()->selectionModel()->selection().indexes()) { + if (index.column() != 0) continue; + int row = app_->playlist_manager()->current()->proxy()->mapToSource(index).row(); + PlaylistItemPtr item(app_->playlist_manager()->current()->item_at(row)); + Song song = item->Metadata(); + + songs << song; + items << item; + } + + app_->collection()->Rescan(songs); + +} + void MainWindow::EditTracks() { SongList songs; diff --git a/src/core/mainwindow.h b/src/core/mainwindow.h index 94ab4d503..fd4929f5d 100644 --- a/src/core/mainwindow.h +++ b/src/core/mainwindow.h @@ -154,6 +154,7 @@ signals: void PlaylistSkip(); void PlaylistRemoveCurrent(); void PlaylistEditFinished(const QModelIndex& index); + void RescanSongs(); void EditTracks(); void EditTagDialogAccepted(); void RenumberTracks(); diff --git a/src/core/mainwindow.ui b/src/core/mainwindow.ui index 2dc57359a..f683e9542 100644 --- a/src/core/mainwindow.ui +++ b/src/core/mainwindow.ui @@ -442,7 +442,7 @@ 0 0 1131 - 24 + 21 @@ -501,6 +501,7 @@ + @@ -512,7 +513,7 @@ - &Previous track + Previous track F5 @@ -520,7 +521,7 @@ - P&lay + &Play F6 @@ -775,6 +776,11 @@ &Do a full collection rescan + + + Abort collection scan + + @@ -816,6 +822,11 @@ Ctrl+Shift+T + + + Rescan songs(s) + + diff --git a/src/settings/collectionsettingspage.cpp b/src/settings/collectionsettingspage.cpp index 781a06c1a..575a1adec 100644 --- a/src/settings/collectionsettingspage.cpp +++ b/src/settings/collectionsettingspage.cpp @@ -111,6 +111,8 @@ void CollectionSettingsPage::Load() { ui_->show_dividers->setChecked(s.value("show_dividers", true).toBool()); ui_->startup_scan->setChecked(s.value("startup_scan", true).toBool()); ui_->monitor->setChecked(s.value("monitor", true).toBool()); + ui_->live_scanning->setChecked(s.value("live_scanning", false).toBool()); + ui_->prevent_delete->setChecked(s.value("prevent_delete", false).toBool()); QStringList filters = s.value("cover_art_patterns", QStringList() << "front" << "cover").toStringList(); ui_->cover_art_patterns->setText(filters.join(",")); @@ -141,6 +143,8 @@ void CollectionSettingsPage::Save() { s.setValue("show_dividers", ui_->show_dividers->isChecked()); s.setValue("startup_scan", ui_->startup_scan->isChecked()); s.setValue("monitor", ui_->monitor->isChecked()); + s.setValue("live_scanning", ui_->live_scanning->isChecked()); + s.setValue("prevent_delete", ui_->prevent_delete->isChecked()); QString filter_text = ui_->cover_art_patterns->text(); QStringList filters = filter_text.split(',', QString::SkipEmptyParts); diff --git a/src/settings/collectionsettingspage.ui b/src/settings/collectionsettingspage.ui index e3ee3f41e..140cff03e 100644 --- a/src/settings/collectionsettingspage.ui +++ b/src/settings/collectionsettingspage.ui @@ -92,6 +92,20 @@ + + + + Use live scanning + + + + + + + Never delete songs from the collection + + +