Add Last.fm import

Fixes #247
2020-08-30 18:09:13 +02:00
parent 82d10dd7cb
commit 5aaa5231b8
17 changed files with 1204 additions and 4 deletions
--- a/src/scrobbler/lastfmimport.cpp
+++ b/src/scrobbler/lastfmimport.cpp
@@ -0,0 +1,588 @@
+/*
+ * Strawberry Music Player
+ * Copyright 2020, Jonas Kvinge <jonas@jkvinge.net>
+ *
+ * Strawberry is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Strawberry is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Strawberry.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "config.h"
+
+#include <algorithm>
+
+#include <QtGlobal>
+#include <QApplication>
+#include <QLocale>
+#include <QVariant>
+#include <QByteArray>
+#include <QString>
+#include <QUrl>
+#include <QUrlQuery>
+#include <QDateTime>
+#include <QTimer>
+#include <QSettings>
+#include <QNetworkRequest>
+#include <QNetworkReply>
+#include <QJsonDocument>
+#include <QJsonObject>
+#include <QJsonArray>
+#include <QJsonValue>
+
+#include "core/network.h"
+#include "core/timeconstants.h"
+#include "core/logging.h"
+
+#include "lastfmimport.h"
+
+#include "scrobblingapi20.h"
+#include "lastfmscrobbler.h"
+
+const int LastFMImport::kRequestsDelay = 2000;
+
+LastFMImport::LastFMImport(QObject *parent) :
+  QObject(parent),
+  network_(new NetworkAccessManager(this)),
+  timer_flush_requests_(new QTimer(this)),
+  lastplayed_(false),
+  playcount_(false),
+  playcount_total_(0),
+  lastplayed_total_(0),
+  playcount_received_(0),
+  lastplayed_received_(0) {
+
+  timer_flush_requests_->setInterval(kRequestsDelay);
+  timer_flush_requests_->setSingleShot(false);
+  connect(timer_flush_requests_, SIGNAL(timeout()), this, SLOT(FlushRequests()));
+
+}
+
+LastFMImport::~LastFMImport() {
+
+  AbortAll();
+
+}
+
+void LastFMImport::AbortAll() {
+
+  while (!replies_.isEmpty()) {
+    QNetworkReply *reply = replies_.takeFirst();
+    disconnect(reply, nullptr, this, nullptr);
+    reply->abort();
+    reply->deleteLater();
+  }
+
+  playcount_total_ = 0;
+  lastplayed_total_ = 0;
+  playcount_received_ = 0;
+  lastplayed_received_ = 0;
+
+  recent_tracks_requests_.clear();
+  top_tracks_requests_.clear();
+  timer_flush_requests_->stop();
+
+}
+
+void LastFMImport::ReloadSettings() {
+
+  QSettings s;
+  s.beginGroup(LastFMScrobbler::kSettingsGroup);
+  username_ = s.value("username").toString();
+  s.endGroup();
+
+}
+
+QNetworkReply *LastFMImport::CreateRequest(const ParamList &request_params) {
+
+  ParamList params = ParamList()
+    << Param("api_key", ScrobblingAPI20::kApiKey)
+    << Param("user", username_)
+    << Param("lang", QLocale().name().left(2).toLower())
+    << Param("format", "json")
+    << request_params;
+
+  std::sort(params.begin(), params.end());
+
+  QUrlQuery url_query;
+  for (const Param &param : params) {
+    url_query.addQueryItem(QUrl::toPercentEncoding(param.first), QUrl::toPercentEncoding(param.second));
+  }
+
+  QUrl url(LastFMScrobbler::kApiUrl);
+  url.setQuery(url_query);
+  QNetworkRequest req(url);
+#if QT_VERSION >= QT_VERSION_CHECK(5, 9, 0)
+  req.setAttribute(QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::NoLessSafeRedirectPolicy);
+#else
+  req.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true);
+#endif
+  req.setHeader(QNetworkRequest::ContentTypeHeader, "application/x-www-form-urlencoded");
+
+  QNetworkReply *reply = network_->get(req);
+  replies_ << reply;
+
+  //qLog(Debug) << "Sending request" << url_query.toString(QUrl::FullyDecoded);
+
+  return reply;
+
+}
+
+QByteArray LastFMImport::GetReplyData(QNetworkReply *reply) {
+
+  QByteArray data;
+
+  if (reply->error() == QNetworkReply::NoError && reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() == 200) {
+    data = reply->readAll();
+  }
+  else {
+    if (reply->error() != QNetworkReply::NoError && reply->error() < 200) {
+      // This is a network error, there is nothing more to do.
+      Error(QString("%1 (%2)").arg(reply->errorString()).arg(reply->error()));
+    }
+    else {
+      QString error;
+      // See if there is Json data containing "error" and "message" - then use that instead.
+      data = reply->readAll();
+      QJsonParseError json_error;
+      QJsonDocument json_doc = QJsonDocument::fromJson(data, &json_error);
+      int error_code = -1;
+      if (json_error.error == QJsonParseError::NoError && !json_doc.isNull() && !json_doc.isEmpty() && json_doc.isObject()) {
+        QJsonObject json_obj = json_doc.object();
+        if (json_obj.contains("error") && json_obj.contains("message")) {
+          error_code = json_obj["error"].toInt();
+          QString error_message = json_obj["message"].toString();
+          error = QString("%1 (%2)").arg(error_message).arg(error_code);
+        }
+      }
+      if (error.isEmpty()) {
+        if (reply->error() != QNetworkReply::NoError) {
+          error = QString("%1 (%2)").arg(reply->errorString()).arg(reply->error());
+        }
+        else {
+          error = QString("Received HTTP code %1").arg(reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt());
+        }
+      }
+      Error(error);
+    }
+    return QByteArray();
+  }
+
+  return data;
+
+}
+
+QJsonObject LastFMImport::ExtractJsonObj(const QByteArray &data) {
+
+  QJsonParseError error;
+  QJsonDocument json_doc = QJsonDocument::fromJson(data, &error);
+
+  if (error.error != QJsonParseError::NoError) {
+    Error("Reply from server missing Json data.", data);
+    return QJsonObject();
+  }
+  if (json_doc.isNull() || json_doc.isEmpty()) {
+    Error("Received empty Json document.", json_doc);
+    return QJsonObject();
+  }
+  if (!json_doc.isObject()) {
+    Error("Json document is not an object.", json_doc);
+    return QJsonObject();
+  }
+  QJsonObject json_obj = json_doc.object();
+  if (json_obj.isEmpty()) {
+    Error("Received empty Json object.", json_doc);
+    return QJsonObject();
+  }
+
+  return json_obj;
+
+}
+
+void LastFMImport::ImportData(const bool lastplayed, const bool playcount) {
+
+  if (!lastplayed && !playcount) return;
+
+  ReloadSettings();
+
+  if (username_.isEmpty()) {
+    Error(tr("Missing username, please login to last.fm first!"));
+    return;
+  }
+
+  AbortAll();
+
+  lastplayed_ = lastplayed;
+  playcount_ = playcount;
+
+  if (lastplayed) AddGetRecentTracksRequest(0);
+  if (playcount) AddGetTopTracksRequest(0);
+
+}
+
+void LastFMImport::FlushRequests() {
+
+  if (!recent_tracks_requests_.isEmpty()) {
+    SendGetRecentTracksRequest(recent_tracks_requests_.dequeue());
+    return;
+  }
+
+  if (!top_tracks_requests_.isEmpty()) {
+    SendGetTopTracksRequest(top_tracks_requests_.dequeue());
+    return;
+  }
+
+  timer_flush_requests_->stop();
+
+}
+
+void LastFMImport::AddGetRecentTracksRequest(const int page) {
+
+  recent_tracks_requests_.enqueue(GetRecentTracksRequest(page));
+
+  if (!timer_flush_requests_->isActive()) {
+    timer_flush_requests_->start();
+  }
+
+}
+
+void LastFMImport::SendGetRecentTracksRequest(GetRecentTracksRequest request) {
+
+  ParamList params = ParamList() << Param("method", "user.getRecentTracks");
+
+  if (request.page == 0) {
+    params << Param("page", "1");
+    params << Param("limit", "1");
+  }
+  else {
+    params << Param("page", QString::number(request.page));
+    params << Param("limit", "500");
+  }
+
+  QNetworkReply *reply = CreateRequest(params);
+  connect(reply, &QNetworkReply::finished, [=] { GetRecentTracksRequestFinished(reply, request.page); });
+
+}
+
+void LastFMImport::GetRecentTracksRequestFinished(QNetworkReply *reply, const int page) {
+
+  if (!replies_.contains(reply)) return;
+  replies_.removeAll(reply);
+  disconnect(reply, nullptr, this, nullptr);
+  reply->deleteLater();
+
+  QByteArray data = GetReplyData(reply);
+  if (data.isEmpty()) {
+    return;
+  }
+
+  QJsonObject json_obj = ExtractJsonObj(data);
+  if (json_obj.isEmpty()) {
+    return;
+  }
+
+  if (json_obj.contains("error") && json_obj.contains("message")) {
+    int error_code = json_obj["error"].toInt();
+    QString error_message = json_obj["message"].toString();
+    QString error_reason = QString("%1 (%2)").arg(error_message).arg(error_code);
+    Error(error_reason);
+    return;
+  }
+
+  if (!json_obj.contains("recenttracks")) {
+    Error("JSON reply from server is missing recenttracks.", json_obj);
+    return;
+  }
+
+  if (!json_obj["recenttracks"].isObject()) {
+    Error("Failed to pase JSON: recenttracks is not an object!", json_obj);
+    return;
+  }
+  json_obj = json_obj["recenttracks"].toObject();
+
+  if (!json_obj.contains("@attr")) {
+    Error("JSON reply from server is missing @attr.", json_obj);
+    return;
+  }
+
+  if (!json_obj.contains("track")) {
+    Error("JSON reply from server is missing track.", json_obj);
+    return;
+  }
+
+  if (!json_obj["@attr"].isObject()) {
+    Error("Failed to pase JSON: @attr is not an object.", json_obj);
+    return;
+  }
+
+  if (!json_obj["track"].isArray()) {
+    Error("Failed to pase JSON: track is not an object.", json_obj);
+    return;
+  }
+
+  QJsonObject obj_attr = json_obj["@attr"].toObject();
+
+  if (!obj_attr.contains("page")) {
+    Error("Failed to pase JSON: attr object is missing page.", json_obj);
+    return;
+  }
+  if (!obj_attr.contains("totalPages")) {
+    Error("Failed to pase JSON: attr object is missing totalPages.", json_obj);
+    return;
+  }
+  if (!obj_attr.contains("total")) {
+    Error("Failed to pase JSON: attr object is missing total.", json_obj);
+    return;
+  }
+
+  int total = obj_attr["total"].toString().toInt();
+  int pages = obj_attr["totalPages"].toString().toInt();
+
+  if (page == 0) {
+    lastplayed_total_ = total;
+    UpdateTotal();
+    AddGetRecentTracksRequest(1);
+  }
+  else {
+
+    QJsonArray array_track = json_obj["track"].toArray();
+
+    for (const QJsonValue &value_track : array_track) {
+
+      ++lastplayed_received_;
+
+      if (!value_track.isObject()) {
+        continue;
+      }
+      QJsonObject obj_track = value_track.toObject();
+      if (!obj_track.contains("artist") ||
+          !obj_track.contains("album") ||
+          !obj_track.contains("name") ||
+          !obj_track.contains("date") ||
+          !obj_track["artist"].isObject() ||
+          !obj_track["album"].isObject() ||
+          !obj_track["date"].isObject()
+      ) {
+        continue;
+      }
+
+      QJsonObject obj_artist = obj_track["artist"].toObject();
+      QJsonObject obj_album = obj_track["album"].toObject();
+      QJsonObject obj_date = obj_track["date"].toObject();
+
+      if (!obj_artist.contains("#text") || !obj_album.contains("#text") || !obj_date.contains("#text")) {
+        continue;
+      }
+
+      QString artist = obj_artist["#text"].toString();
+      QString album = obj_album["#text"].toString();
+      QString date = obj_date["#text"].toString();
+      QString title = obj_track["name"].toString();
+      QDateTime datetime = QDateTime::fromString(date, "dd MMM yyyy, hh:mm");
+
+      emit UpdateLastPlayed(artist, album, title, datetime.toSecsSinceEpoch());
+      UpdateProgress();
+
+    }
+
+    if (page == 1) {
+      for (int i = 2 ; i <= pages ; ++i) {
+        AddGetRecentTracksRequest(i);
+      }
+    }
+
+  }
+
+  FinishCheck();
+
+}
+
+void LastFMImport::AddGetTopTracksRequest(const int page) {
+
+  top_tracks_requests_.enqueue(GetTopTracksRequest(page));
+
+  if (!timer_flush_requests_->isActive()) {
+    timer_flush_requests_->start();
+  }
+
+}
+
+void LastFMImport::SendGetTopTracksRequest(GetTopTracksRequest request) {
+
+  ParamList params = ParamList() << Param("method", "user.getTopTracks");
+
+  if (request.page == 0) {
+    params << Param("page", "1");
+    params << Param("limit", "1");
+  }
+  else {
+    params << Param("page", QString::number(request.page));
+    params << Param("limit", "500");
+  }
+
+  QNetworkReply *reply = CreateRequest(params);
+  connect(reply, &QNetworkReply::finished, [=] { GetTopTracksRequestFinished(reply, request.page); });
+
+}
+
+void LastFMImport::GetTopTracksRequestFinished(QNetworkReply *reply, const int page) {
+
+  if (!replies_.contains(reply)) return;
+  replies_.removeAll(reply);
+  disconnect(reply, nullptr, this, nullptr);
+  reply->deleteLater();
+
+  QByteArray data = GetReplyData(reply);
+  if (data.isEmpty()) {
+    return;
+  }
+
+  QJsonObject json_obj = ExtractJsonObj(data);
+  if (json_obj.isEmpty()) {
+    return;
+  }
+
+  if (json_obj.contains("error") && json_obj.contains("message")) {
+    int error_code = json_obj["error"].toInt();
+    QString error_message = json_obj["message"].toString();
+    QString error_reason = QString("%1 (%2)").arg(error_message).arg(error_code);
+    Error(error_reason);
+    return;
+  }
+
+  if (!json_obj.contains("toptracks")) {
+    Error("JSON reply from server is missing toptracks.", json_obj);
+    return;
+  }
+
+  if (!json_obj["toptracks"].isObject()) {
+    Error("Failed to pase JSON: toptracks is not an object!", json_obj);
+    return;
+  }
+  json_obj = json_obj["toptracks"].toObject();
+
+  if (!json_obj.contains("@attr")) {
+    Error("JSON reply from server is missing @attr.", json_obj);
+    return;
+  }
+
+  if (!json_obj.contains("track")) {
+    Error("JSON reply from server is missing track.", json_obj);
+    return;
+  }
+
+  if (!json_obj["@attr"].isObject()) {
+    Error("Failed to pase JSON: @attr is not an object.", json_obj);
+    return;
+  }
+
+  if (!json_obj["track"].isArray()) {
+    Error("Failed to pase JSON: track is not an object.", json_obj);
+    return;
+  }
+
+  QJsonObject obj_attr = json_obj["@attr"].toObject();
+
+  if (!obj_attr.contains("page")) {
+    Error("Failed to pase JSON: attr object is missing page.", json_obj);
+    return;
+  }
+  if (!obj_attr.contains("totalPages")) {
+    Error("Failed to pase JSON: attr object is missing page.", json_obj);
+    return;
+  }
+  if (!obj_attr.contains("total")) {
+    Error("Failed to pase JSON: attr object is missing total.", json_obj);
+    return;
+  }
+
+  int pages = obj_attr["totalPages"].toString().toInt();
+  int total = obj_attr["total"].toString().toInt();
+
+  if (page == 0) {
+    playcount_total_ = total;
+    UpdateTotal();
+    AddGetTopTracksRequest(1);
+  }
+  else {
+
+    QJsonArray array_track = json_obj["track"].toArray();
+    for (const QJsonValue &value_track : array_track) {
+
+      ++playcount_received_;
+
+      if (!value_track.isObject()) {
+        continue;
+      }
+
+      QJsonObject obj_track = value_track.toObject();
+      if (!obj_track.contains("artist") ||
+          !obj_track.contains("name") ||
+          !obj_track.contains("playcount") ||
+          !obj_track["artist"].isObject()
+      ) {
+        continue;
+      }
+
+      QJsonObject obj_artist = obj_track["artist"].toObject();
+      if (!obj_artist.contains("name")) {
+        continue;
+      }
+
+      QString artist = obj_artist["name"].toString();
+      QString title = obj_track["name"].toString();
+      int playcount = obj_track["playcount"].toString().toInt();
+
+      if (playcount <= 0) continue;
+
+      emit UpdatePlayCount(artist, title, playcount);
+      UpdateProgress();
+
+    }
+
+    if (page == 1) {
+      for (int i = 2 ; i <= pages ; ++i) {
+        AddGetTopTracksRequest(i);
+      }
+    }
+
+  }
+
+  FinishCheck();
+
+}
+
+void LastFMImport::UpdateTotal() {
+
+  if ((!playcount_ || playcount_total_ > 0) && (!lastplayed_ || lastplayed_total_ > 0))
+    emit UpdateTotal(lastplayed_total_, playcount_total_);
+
+}
+
+void LastFMImport::UpdateProgress() {
+  emit UpdateProgress(lastplayed_received_, playcount_received_);
+}
+
+void LastFMImport::FinishCheck() {
+  if (replies_.isEmpty() && recent_tracks_requests_.isEmpty() && top_tracks_requests_.isEmpty()) emit Finished();
+}
+
+void LastFMImport::Error(const QString &error, const QVariant &debug) {
+
+  qLog(Error) << error;
+  if (debug.isValid()) qLog(Debug) << debug;
+
+  emit FinishedWithError(error);
+
+  AbortAll();
+
+}
--- a/src/scrobbler/lastfmimport.h
+++ b/src/scrobbler/lastfmimport.h
@@ -0,0 +1,113 @@
+/*
+ * Strawberry Music Player
+ * Copyright 2020, Jonas Kvinge <jonas@jkvinge.net>
+ *
+ * Strawberry is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Strawberry is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Strawberry.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef LASTFMIMPORT_H
+#define LASTFMIMPORT_H
+
+#include "config.h"
+
+#include <QtGlobal>
+#include <QObject>
+#include <QList>
+#include <QVariant>
+#include <QByteArray>
+#include <QString>
+#include <QQueue>
+#include <QDateTime>
+
+class QTimer;
+class QNetworkReply;
+
+class NetworkAccessManager;
+
+class LastFMImport : public QObject {
+  Q_OBJECT
+
+ public:
+  explicit LastFMImport(QObject *parent = nullptr);
+  ~LastFMImport() override;
+
+  void ReloadSettings();
+  void ImportData(const bool lastplayed = true, const bool playcount = true);
+  void AbortAll();
+
+ private:
+  typedef QPair<QString, QString> Param;
+  typedef QList<Param> ParamList;
+
+  struct GetRecentTracksRequest {
+    explicit GetRecentTracksRequest(const int _page) : page(_page) {}
+    int page;
+  };
+  struct GetTopTracksRequest {
+    explicit GetTopTracksRequest(const int _page) : page(_page) {}
+    int page;
+  };
+
+ private:
+  QNetworkReply *CreateRequest(const ParamList &request_params);
+  QByteArray GetReplyData(QNetworkReply *reply);
+  QJsonObject ExtractJsonObj(const QByteArray &data);
+
+  void AddGetRecentTracksRequest(const int page = 0);
+  void AddGetTopTracksRequest(const int page = 0);
+
+  void SendGetRecentTracksRequest(GetRecentTracksRequest request);
+  void SendGetTopTracksRequest(GetTopTracksRequest request);
+
+  void Error(const QString &error, const QVariant &debug = QVariant());
+
+  void UpdateTotal();
+  void UpdateProgress();
+
+  void FinishCheck();
+
+ signals:
+  void UpdatePlayCount(QString, QString, int);
+  void UpdateLastPlayed(QString, QString, QString, int);
+  void UpdateTotal(int, int);
+  void UpdateProgress(int, int);
+  void Finished();
+  void FinishedWithError(QString);
+
+ private slots:
+  void FlushRequests();
+  void GetRecentTracksRequestFinished(QNetworkReply *reply, const int page);
+  void GetTopTracksRequestFinished(QNetworkReply *reply, const int page);
+
+ private:
+  static const int kRequestsDelay;
+
+  NetworkAccessManager *network_;
+  QTimer *timer_flush_requests_;
+
+  QString username_;
+  bool lastplayed_;
+  bool playcount_;
+  int playcount_total_;
+  int lastplayed_total_;
+  int playcount_received_;
+  int lastplayed_received_;
+  QQueue<GetRecentTracksRequest> recent_tracks_requests_;
+  QQueue<GetTopTracksRequest> top_tracks_requests_;
+  QList<QNetworkReply*> replies_;
+
+};
+
+#endif  // LASTFMIMPORT_H
--- a/src/scrobbler/lastfmscrobbler.h
+++ b/src/scrobbler/lastfmscrobbler.h
@@ -42,13 +42,13 @@ class LastFMScrobbler : public ScrobblingAPI20 {

  static const char *kName;
  static const char *kSettingsGroup;
+  static const char *kApiUrl;

  NetworkAccessManager *network() const override { return network_; }
  ScrobblerCache *cache() const override { return cache_; }

 private:
  static const char *kAuthUrl;
-  static const char *kApiUrl;
  static const char *kCacheFile;

  QString settings_group_;
--- a/src/scrobbler/scrobblingapi20.cpp
+++ b/src/scrobbler/scrobblingapi20.cpp
@@ -1013,7 +1013,7 @@ void ScrobblingAPI20::Error(const QString &error, const QVariant &debug) {

 }

-QString ScrobblingAPI20::ErrorString(const ScrobbleErrorCode error) const {
+QString ScrobblingAPI20::ErrorString(const ScrobbleErrorCode error) {

  switch (error) {
    case ScrobbleErrorCode::NoError:
--- a/src/scrobbler/scrobblingapi20.h
+++ b/src/scrobbler/scrobblingapi20.h
@@ -47,6 +47,7 @@ class ScrobblingAPI20 : public ScrobblerService {
  explicit ScrobblingAPI20(const QString &name, const QString &settings_group, const QString &auth_url, const QString &api_url, const bool batch, Application *app, QObject *parent = nullptr);
  ~ScrobblingAPI20() override;

+  static const char *kApiKey;
  static const char *kRedirectUrl;

  void ReloadSettings() override;
@@ -118,7 +119,6 @@ class ScrobblingAPI20 : public ScrobblerService {
    RateLimitExceeded = 29,
  };

-  static const char *kApiKey;
  static const char *kSecret;
  static const int kScrobblesPerRequest;

@@ -129,7 +129,7 @@ class ScrobblingAPI20 : public ScrobblerService {
  void AuthError(const QString &error);
  void SendSingleScrobble(ScrobblerCacheItemPtr item);
  void Error(const QString &error, const QVariant &debug = QVariant()) override;
-  QString ErrorString(const ScrobbleErrorCode error) const;
+  static QString ErrorString(const ScrobbleErrorCode error);
  void DoSubmit() override;
  void CheckScrobblePrevSong();