From bafcb97fa1b778ae2b6d25896f57c3bd8533d223 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Tue, 27 Jun 2023 05:02:02 +0300 Subject: [PATCH] Implement `EBUR128Analysis` The most juicy bit! This is based on Song Fingerprint Analysis, but here we must know the actual song, and not just the file. The library supports only interleaved S16/S32/F32/F64, so we must be sure we insert `audioconvert` into pipeline. One point of contention here for me, is whether we should feed the frames to the library the moment we get them in `NewBufferCallback`, or collect them in a buffer and pass them all at once. I've gone with the former, because it seems like that is not the worst choice: https://github.com/strawberrymusicplayer/strawberry/pull/1216#issuecomment-1610075876 In principle, the analysis *could* fail, so we want to handle that gracefully. --- src/CMakeLists.txt | 5 + src/engine/ebur128analysis.cpp | 382 +++++++++++++++++++++++++++++++++ src/engine/ebur128analysis.h | 41 ++++ src/engine/ebur128measures.h | 30 +++ 4 files changed, 458 insertions(+) create mode 100644 src/engine/ebur128analysis.cpp create mode 100644 src/engine/ebur128analysis.h create mode 100644 src/engine/ebur128measures.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a8a4e0895..af46c17be 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -927,6 +927,11 @@ optional_source(HAVE_MOODBAR settings/moodbarsettingspage.ui ) +# EBU R 128 +optional_source(HAVE_EBUR128 + SOURCES engine/ebur128analysis.cpp +) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/version.h.in ${CMAKE_CURRENT_BINARY_DIR}/version.h) diff --git a/src/engine/ebur128analysis.cpp b/src/engine/ebur128analysis.cpp new file mode 100644 index 000000000..89555128a --- /dev/null +++ b/src/engine/ebur128analysis.cpp @@ -0,0 +1,382 @@ +/* + * Strawberry Music Player + * Copyright 2023 Roman Lebedev + * + * Strawberry is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Strawberry is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Strawberry. If not, see . + * + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "core/logging.h" +#include "core/signalchecker.h" + +#include "ebur128analysis.h" + +static const int kTimeoutSecs = 60; + +namespace { + +struct ebur128_state_deleter { + void operator()(ebur128_state *p) const { ebur128_destroy(&p); }; +}; + +struct GstSampleDeleter { + void operator()(GstSample *s) const { gst_sample_unref(s); }; +}; + +struct FrameFormat { + enum class DataFormat { + S16, + S32, + FP32, + FP64 + }; + + int channels; + int samplerate; + DataFormat format; + + explicit FrameFormat(GstCaps *caps); +}; + +class EBUR128State { + public: + EBUR128State() = delete; + EBUR128State(const EBUR128State&) = delete; + EBUR128State(EBUR128State&&) = delete; + + EBUR128State &operator=(const EBUR128State&) = delete; + EBUR128State &operator=(EBUR128State&&) = delete; + + explicit EBUR128State(FrameFormat dsc_); + const FrameFormat dsc; + + void AddFrames(const char *data, size_t size); + + static std::optional Finalize(EBUR128State &&state); + + private: + std::unique_ptr st; +}; + +class EBUR128AnalysisImpl { + EBUR128AnalysisImpl() = default; + + public: + static std::optional Compute(const Song &song); + + private: + GstElement *convert_element_ = nullptr; + + std::optional state; + + static void NewPadCallback(GstElement *elt, GstPad *pad, gpointer data); + static GstFlowReturn NewBufferCallback(GstAppSink *app_sink, gpointer self); +}; + +FrameFormat::FrameFormat(GstCaps *caps) { + + GstStructure *structure = gst_caps_get_structure(caps, 0); + QString format_str = gst_structure_get_string(structure, "format"); + gst_structure_get_int(structure, "rate", &samplerate); + gst_structure_get_int(structure, "channels", &channels); + + if (format_str == "S16LE") { + format = DataFormat::S16; + } + else if (format_str == "S32LE") { + format = DataFormat::S32; + } + else if (format_str == "F32LE") { + format = DataFormat::FP32; + } + else if (format_str == "F64LE") { + format = DataFormat::FP64; + } + else { + qLog(Error) << "EBUR128AnalysisImpl: got unexpected format " << format_str; + Q_ASSERT(false && "Unexpected format. How did you get here?"); + } + +} + +bool operator==(const FrameFormat &lhs, const FrameFormat &rhs) { + + return std::tie(lhs.channels, lhs.samplerate, lhs.format) == std::tie(rhs.channels, rhs.samplerate, rhs.format); + +} +bool operator!=(const FrameFormat &lhs, const FrameFormat &rhs) { + + return !(lhs == rhs); + +} + +EBUR128State::EBUR128State(FrameFormat dsc_) : dsc(dsc_) { + + st.reset(ebur128_init(dsc.channels, dsc.samplerate, EBUR128_MODE_I | EBUR128_MODE_LRA)); + Q_ASSERT(st); + +}; + +void EBUR128State::AddFrames(const char *data, size_t size) { + + Q_ASSERT(st); + + int bytes_per_sample = -1; + switch (dsc.format) { + case FrameFormat::DataFormat::S16: + bytes_per_sample = sizeof(int16_t); + break; + case FrameFormat::DataFormat::S32: + bytes_per_sample = sizeof(int32_t); + break; + case FrameFormat::DataFormat::FP32: + bytes_per_sample = sizeof(float); + break; + case FrameFormat::DataFormat::FP64: + bytes_per_sample = sizeof(double); + break; + } + + int bytes_per_frame = dsc.channels * bytes_per_sample; + Q_ASSERT(size % bytes_per_frame == 0); + auto num_frames = size / bytes_per_frame; + + int ebur_error; + switch (dsc.format) { + case FrameFormat::DataFormat::S16: + ebur_error = ebur128_add_frames_short(&*st, reinterpret_cast(data), num_frames); + break; + case FrameFormat::DataFormat::S32: + ebur_error = ebur128_add_frames_int(&*st, reinterpret_cast(data), num_frames); + break; + case FrameFormat::DataFormat::FP32: + ebur_error = ebur128_add_frames_float(&*st, reinterpret_cast(data), num_frames); + break; + case FrameFormat::DataFormat::FP64: + ebur_error = ebur128_add_frames_double(&*st, reinterpret_cast(data), num_frames); + break; + } + Q_ASSERT(ebur_error == EBUR128_SUCCESS); + +} + +std::optional EBUR128State::Finalize(EBUR128State&& state) { + + ebur128_state *ebur128 = &*state.st; + + EBUR128Measures result; + + double out = NAN; + int ebur_error = ebur128_loudness_global(ebur128, &out); + Q_ASSERT(ebur_error == EBUR128_SUCCESS); + result.loudness_lufs = out; + + out = NAN; + ebur_error = ebur128_loudness_range(ebur128, &out); + Q_ASSERT(ebur_error == EBUR128_SUCCESS); + result.range_lu = out; + + return result; + +} + +void EBUR128AnalysisImpl::NewPadCallback(GstElement *elt, GstPad *pad, gpointer data) { + + Q_UNUSED(elt); + + EBUR128AnalysisImpl *me = reinterpret_cast(data); + GstPad *const audiopad = gst_element_get_static_pad(me->convert_element_, "sink"); + + if (GST_PAD_IS_LINKED(audiopad)) { + qLog(Warning) << "audiopad is already linked, unlinking old pad"; + gst_pad_unlink(audiopad, GST_PAD_PEER(audiopad)); + } + + gst_pad_link(pad, audiopad); + gst_object_unref(audiopad); + +} + +GstFlowReturn EBUR128AnalysisImpl::NewBufferCallback(GstAppSink *app_sink, gpointer self) { + + EBUR128AnalysisImpl *me = reinterpret_cast(self); + + std::unique_ptr sample(gst_app_sink_pull_sample(app_sink)); + if (!sample) return GST_FLOW_ERROR; + + const FrameFormat dsc(gst_sample_get_caps(&*sample)); + if (!me->state) { + me->state.emplace(dsc); + } + else if (me->state->dsc != dsc) { + return GST_FLOW_ERROR; + } + + GstBuffer *buffer = gst_sample_get_buffer(&*sample); + if (buffer) { + GstMapInfo map; + if (gst_buffer_map(buffer, &map, GST_MAP_READ)) { + me->state->AddFrames(reinterpret_cast(map.data), static_cast(map.size)); + gst_buffer_unmap(buffer, &map); + } + } + + return GST_FLOW_OK; + +} + +GstElement *CreateElement(const QString &factory_name, GstElement *bin) { + + GstElement *ret = gst_element_factory_make(factory_name.toLatin1().constData(), factory_name.toLatin1().constData()); + + if (ret && bin) gst_bin_add(GST_BIN(bin), ret); + + if (!ret) { + qLog(Warning) << "Couldn't create the gstreamer element" << factory_name; + } + + return ret; + +} + +std::optional EBUR128AnalysisImpl::Compute(const Song &song) { + + EBUR128AnalysisImpl impl; + + GstElement *pipeline = gst_pipeline_new("pipeline"); + if (!pipeline) { + return std::nullopt; + } + + GstElement *src = CreateElement("filesrc", pipeline); + GstElement *decode = CreateElement("decodebin", pipeline); + GstElement *convert = CreateElement("audioconvert", pipeline); + GstElement *sink = CreateElement("appsink", pipeline); + + if (!src || !decode || !convert || !sink) { + gst_object_unref(pipeline); + return std::nullopt; + } + + impl.convert_element_ = convert; + + // Connect the elements + gst_element_link_many(src, decode, nullptr); + + GstStaticCaps static_caps = GST_STATIC_CAPS( + "audio/x-raw," + "format = (string) { S16LE, S32LE, F32LE, F64LE }," + "layout = (string) interleaved"); + + GstCaps *caps = gst_static_caps_get(&static_caps); + gst_element_link_filtered(convert, sink, caps); + gst_caps_unref(caps); + + GstAppSinkCallbacks callbacks; + memset(&callbacks, 0, sizeof(callbacks)); + callbacks.new_sample = NewBufferCallback; + gst_app_sink_set_callbacks(reinterpret_cast(sink), &callbacks, &impl, nullptr); + g_object_set(G_OBJECT(sink), "sync", FALSE, nullptr); + g_object_set(G_OBJECT(sink), "emit-signals", TRUE, nullptr); + + // Set the filename + g_object_set(src, "location", song.url().toLocalFile().toUtf8().constData(), nullptr); + + // Connect signals + GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline)); + CHECKED_GCONNECT(decode, "pad-added", &NewPadCallback, &impl); + + // Play only the specified song! + gst_element_set_state(pipeline, GST_STATE_PAUSED); + // wait for state change before seeking + gst_element_get_state(pipeline, nullptr, nullptr, kTimeoutSecs * GST_SECOND); + gst_element_seek(pipeline, 1.0, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH, GST_SEEK_TYPE_SET, song.beginning_nanosec() * GST_NSECOND, GST_SEEK_TYPE_SET, song.end_nanosec() * GST_NSECOND); + + QElapsedTimer time; + time.start(); + + // Start playing + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + // Wait until EOS or error + bool hadError = false; + GstMessage *msg = gst_bus_timed_pop_filtered(bus, kTimeoutSecs * GST_SECOND, static_cast(GST_MESSAGE_EOS | GST_MESSAGE_ERROR)); + if (msg) { + if (msg->type == GST_MESSAGE_ERROR) { + hadError = true; + // Report error + GError *error = nullptr; + gchar *debugs = nullptr; + gst_message_parse_error(msg, &error, &debugs); + if (error) { + QString message = QString::fromLocal8Bit(error->message); + g_error_free(error); + qLog(Debug) << "Error processing " << song.url() << ":" << message; + } + if (debugs) free(debugs); + } + gst_message_unref(msg); + } + + const qint64 decode_time = time.restart(); + + std::optional result; + if (!hadError && impl.state) { + // Generate loudness characteristics from sampled data. + result = EBUR128State::Finalize(std::move(impl.state.value())); + + const qint64 finalize_time = time.elapsed(); + + qLog(Debug) << "Decode time:" << decode_time << "Finalization time:" << finalize_time; + } + + // Cleanup + callbacks.new_sample = nullptr; + gst_object_unref(bus); + gst_element_set_state(pipeline, GST_STATE_NULL); + gst_object_unref(pipeline); + + return result; + +} + +}; // namespace + +std::optional EBUR128Analysis::Compute(const Song &song) { + + Q_ASSERT(QThread::currentThread() != qApp->thread()); + + return EBUR128AnalysisImpl::Compute(song); + +} diff --git a/src/engine/ebur128analysis.h b/src/engine/ebur128analysis.h new file mode 100644 index 000000000..644d3ae3a --- /dev/null +++ b/src/engine/ebur128analysis.h @@ -0,0 +1,41 @@ +/* + * Strawberry Music Player + * Copyright 2023 Roman Lebedev + * + * Strawberry is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Strawberry is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Strawberry. If not, see . + * + */ + +#ifndef EBUR128ANALYSIS_H +#define EBUR128ANALYSIS_H + +#include "config.h" + +#include + +#include "core/song.h" +#include "ebur128measures.h" + +class EBUR128Analysis { + public: + ~EBUR128Analysis() = delete; // Do not construct variables of this class. + + // Performs an EBU R 128 analysis on the given song. + // Returns `std::nullopt` if the analysis fails. + // + // This method is blocking, so you want to call it in another thread. + static std::optional Compute(const Song &song); +}; + +#endif // EBUR128ANALYSIS_H diff --git a/src/engine/ebur128measures.h b/src/engine/ebur128measures.h new file mode 100644 index 000000000..b5b2a668b --- /dev/null +++ b/src/engine/ebur128measures.h @@ -0,0 +1,30 @@ +/* + * Strawberry Music Player + * Copyright 2023 Roman Lebedev + * + * Strawberry is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Strawberry is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Strawberry. If not, see . + * + */ + +#ifndef EBUR128MEASURES_H +#define EBUR128MEASURES_H + +#include + +struct EBUR128Measures { + std::optional loudness_lufs; // Global integrated loudness + std::optional range_lu; // Loudness Range +}; + +#endif // EBUR128MEASURES_H