dryark/strawberry
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite collection model and search

Browse Source 
Fixes #392
This commit is contained in:
Jonas Kvinge
2021-06-27 22:54:08 +02:00
parent ea1e4541c0
commit e477449cd4
52 changed files with 2321 additions and 2637 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
src/collection/collectionquery.cpp
View File

@@ -38,78 +38,13 @@
#include "collectionfilteroptions.h"
#include "utilities/searchparserutils.h"
CollectionQuery::CollectionQuery(const QSqlDatabase &db, const QString &songs_table, const QString &fts_table, const CollectionFilterOptions &filter_options)
CollectionQuery::CollectionQuery(const QSqlDatabase &db, const QString &songs_table, const CollectionFilterOptions &filter_options)
: SqlQuery(db),
songs_table_(songs_table),
fts_table_(fts_table),
include_unavailable_(false),
join_with_fts_(false),
duplicates_only_(false),
limit_(-1) {
if (!filter_options.filter_text().isEmpty()) {
// We need to munge the filter text a little bit to get it to work as expected with sqlite's FTS5:
// 1) Append * to all tokens.
// 2) Prefix "fts" to column names.
// 3) Remove colons which don't correspond to column names.
// Split on whitespace
QString filter_text = filter_options.filter_text().replace(QRegularExpression(QStringLiteral(":\\s+")), QStringLiteral(":"));
#if QT_VERSION >= QT_VERSION_CHECK(5, 14, 0)
QStringList tokens(filter_text.split(QRegularExpression(QStringLiteral("\\s+")), Qt::SkipEmptyParts));
#else
QStringList tokens(filter_text.split(QRegularExpression(QStringLiteral("\\s+")), QString::SkipEmptyParts));
#endif
QString query;
for (QString token : tokens) {
token.remove(QLatin1Char('('))
.remove(QLatin1Char(')'))
.remove(QLatin1Char('"'))
.replace(QLatin1Char('-'), QLatin1Char(' '));
if (token.contains(QLatin1Char(':'))) {
const QString columntoken = token.section(QLatin1Char(':'), 0, 0);
QString subtoken = token.section(QLatin1Char(':'), 1, -1).replace(QLatin1String(":"), QLatin1String(" ")).trimmed();
if (subtoken.isEmpty()) continue;
if (Song::kFtsColumns.contains(QLatin1String("fts") + columntoken, Qt::CaseInsensitive)) {
if (!query.isEmpty()) query.append(QLatin1String(" "));
query += QStringLiteral("fts") + columntoken + QStringLiteral(":\"") + subtoken + QStringLiteral("\"*");
}
else if (Song::kNumericalColumns.contains(columntoken, Qt::CaseInsensitive)) {
QString comparator = RemoveSqlOperator(subtoken);
if (columntoken.compare(QLatin1String("rating"), Qt::CaseInsensitive) == 0) {
AddWhereRating(subtoken, comparator);
}
else if (columntoken.compare(QLatin1String("length"), Qt::CaseInsensitive) == 0) {
// Time is saved in nanoseconds, so add 9 0's
QString parsedTime = QString::number(Utilities::ParseSearchTime(subtoken)) + QStringLiteral("000000000");
AddWhere(columntoken, parsedTime, comparator);
}
else {
AddWhere(columntoken, subtoken, comparator);
}
}
// Not a valid filter, remove
else {
token = token.replace(QLatin1String(":"), QLatin1String(" ")).trimmed();
if (!token.isEmpty()) {
if (!query.isEmpty()) query.append(QLatin1Char(' '));
query += QLatin1Char('\"') + token + QStringLiteral("\"*");
}
}
}
else {
if (!query.isEmpty()) query.append(QLatin1Char(' '));
query += QLatin1Char('\"') + token + QStringLiteral("\"*");
}
}
if (!query.isEmpty()) {
where_clauses_ << QStringLiteral("fts.%fts_table_noprefix MATCH ?");
bound_values_ << query;
join_with_fts_ = true;
}
}
if (filter_options.max_age() != -1) {
qint64 cutoff = QDateTime::currentDateTime().toSecsSinceEpoch() - filter_options.max_age();
@@ -117,12 +52,6 @@ CollectionQuery::CollectionQuery(const QSqlDatabase &db, const QString &songs_ta
bound_values_ << cutoff;
}
// TODO: Currently you cannot use any FilterMode other than All and FTS at the same time.
// Joining songs, duplicated_songs and songs_fts all together takes a huge amount of time.
// The query takes about 20 seconds on my machine then. Why?
// Untagged mode could work with additional filtering but I'm disabling it just to be consistent
// this way filtering is available only in the All mode.
// Remember though that when you fix the Duplicates + FTS cooperation, enable the filtering in both Duplicates and Untagged modes.
duplicates_only_ = filter_options.filter_mode() == CollectionFilterOptions::FilterMode::Duplicates;
if (filter_options.filter_mode() == CollectionFilterOptions::FilterMode::Untagged) {
@@ -131,28 +60,10 @@ CollectionQuery::CollectionQuery(const QSqlDatabase &db, const QString &songs_ta
}
QString CollectionQuery::RemoveSqlOperator(QString &token) {
QString op = QStringLiteral("=");
static QRegularExpression rxOp(QStringLiteral("^(=|<[>=]?|>=?|!=)"));
QRegularExpressionMatch match = rxOp.match(token);
if (match.hasMatch()) {
op = match.captured(0);
}
token.remove(rxOp);
if (op == QStringLiteral("!=")) {
op = QStringLiteral("<>");
}
return op;
}
void CollectionQuery::AddWhere(const QString &column, const QVariant &value, const QString &op) {
// Ignore 'literal' for IN
if (op.compare(QLatin1String("IN"), Qt::CaseInsensitive) == 0) {
if (op.compare(QStringLiteral("IN"), Qt::CaseInsensitive) == 0) {
QStringList values = value.toStringList();
QStringList final_values;
final_values.reserve(values.count());
@@ -161,7 +72,7 @@ void CollectionQuery::AddWhere(const QString &column, const QVariant &value, con
bound_values_ << single_value;
}
where_clauses_ << QStringLiteral("%1 IN (%2)").arg(column, final_values.join(QStringLiteral(",")));
where_clauses_ << QStringLiteral("%1 IN (%2)").arg(column, final_values.join(QLatin1Char(',')));
}
else {
// Do integers inline - sqlite seems to get confused when you pass integers to bound parameters
@@ -190,49 +101,8 @@ void CollectionQuery::AddWhere(const QString &column, const QVariant &value, con
}
void CollectionQuery::AddWhereArtist(const QVariant &value) {
where_clauses_ << QStringLiteral("((artist = ? AND albumartist = '') OR albumartist = ?)");
bound_values_ << value;
bound_values_ << value;
}
void CollectionQuery::AddWhereRating(const QVariant &value, const QString &op) {
float parsed_rating = Utilities::ParseSearchRating(value.toString());
// You can't query the database for a float, due to float precision errors,
// So we have to use a certain tolerance, so that the searched value is definetly included.
const float tolerance = 0.001F;
if (op == QStringLiteral("<")) {
AddWhere(QStringLiteral("rating"), parsed_rating-tolerance, QStringLiteral("<"));
}
else if (op == QStringLiteral(">")) {
AddWhere(QStringLiteral("rating"), parsed_rating+tolerance, QStringLiteral(">"));
}
else if (op == QStringLiteral("<=")) {
AddWhere(QStringLiteral("rating"), parsed_rating+tolerance, QStringLiteral("<="));
}
else if (op == QStringLiteral(">=")) {
AddWhere(QStringLiteral("rating"), parsed_rating-tolerance, QStringLiteral(">="));
}
else if (op == QStringLiteral("<>")) {
where_clauses_ << QStringLiteral("(rating<? OR rating>?)");
bound_values_ << parsed_rating - tolerance;
bound_values_ << parsed_rating + tolerance;
}
else /* (op == "=") */ {
AddWhere(QStringLiteral("rating"), parsed_rating+tolerance, QStringLiteral("<"));
AddWhere(QStringLiteral("rating"), parsed_rating-tolerance, QStringLiteral(">"));
}
}
void CollectionQuery::AddCompilationRequirement(const bool compilation) {
// The unary + is added to prevent sqlite from using the index idx_comp_artist.
// When joining with fts, sqlite 3.8 has a tendency to use this index and thereby nesting the tables in an order which gives very poor performance
where_clauses_ << QStringLiteral("+compilation_effective = %1").arg(compilation ? 1 : 0);
}
@@ -248,14 +118,7 @@ QString CollectionQuery::GetInnerQuery() const {
bool CollectionQuery::Exec() {
QString sql;
if (join_with_fts_) {
sql = QStringLiteral("SELECT %1 FROM %2 INNER JOIN %3 AS fts ON %2.ROWID = fts.ROWID").arg(column_spec_, songs_table_, fts_table_);
}
else {
sql = QStringLiteral("SELECT %1 FROM %2 %3").arg(column_spec_, songs_table_, GetInnerQuery());
}
QString sql = QStringLiteral("SELECT %1 FROM %2 %3").arg(column_spec_, songs_table_, GetInnerQuery());
QStringList where_clauses(where_clauses_);
if (!include_unavailable_) {
@@ -268,9 +131,7 @@ bool CollectionQuery::Exec() {
if (limit_ != -1) sql += QStringLiteral(" LIMIT ") + QString::number(limit_);
sql.replace(QLatin1String("%songs_table"), songs_table_);
sql.replace(QLatin1String("%fts_table_noprefix"), fts_table_.section(QLatin1Char('.'), -1, -1));
sql.replace(QLatin1String("%fts_table"), fts_table_);
sql.replace(QStringLiteral("%songs_table"), songs_table_);
if (!QSqlQuery::prepare(sql)) return false;