From ca5f6104ddd79801ce800c98706647369777f9f5 Mon Sep 17 00:00:00 2001 From: mesonium Date: Fri, 25 Nov 2022 17:54:55 +0100 Subject: feat: Preview: Add audio preview controls and visualization * Remove the delay when starting a playback * Don't scrub during seek when dragging the slider * Don't open audio stream connections before playing * Determine the duration early to prevent missing duration info + Show total duration + Add option to display remaining play time + Add possibility to seek by clicking into the seekbar + Add possibility to seek by scrolling with the mouse wheel + Add fast forward and rewind buttons + Add buttons to change the playback speed + Add automatic restoration of the audio player settings in a session + Add automatic stop of other streams when starting to play + Add a visualization of the RMS peaks --- gajim/common/preview.py | 56 ++- gajim/common/text_helpers.py | 23 ++ gajim/data/gui/preview_audio.ui | 378 +++++++++++++++++++++ gajim/data/style/gajim.css | 2 + gajim/gtk/builder.pyi | 23 +- gajim/gtk/preview_audio.py | 620 +++++++++++++++++++++++++++++----- gajim/gtk/preview_audio_analyzer.py | 167 +++++++++ gajim/gtk/preview_audio_visualizer.py | 255 ++++++++++++++ pyproject.toml | 2 + test/no_gui/test_text_helpers.py | 25 ++ 10 files changed, 1455 insertions(+), 96 deletions(-) create mode 100644 gajim/data/gui/preview_audio.ui create mode 100644 gajim/gtk/preview_audio_analyzer.py create mode 100644 gajim/gtk/preview_audio_visualizer.py diff --git a/gajim/common/preview.py b/gajim/common/preview.py index 0c2734b84..5feffc884 100644 --- a/gajim/common/preview.py +++ b/gajim/common/preview.py @@ -12,7 +12,11 @@ # You should have received a copy of the GNU General Public License # along with Gajim. If not, see . +from __future__ import annotations + +from dataclasses import dataclass, field from typing import Any +from typing import Callable from typing import Optional import logging @@ -54,6 +58,19 @@ mime_types = set(MIME_TYPES) # Merge both: if it’s a previewable image, it should be allowed ALLOWED_MIME_TYPES = mime_types.union(PREVIEWABLE_MIME_TYPES) +AudioSampleT = list[tuple[float, float]] + + +@dataclass +class AudioPreviewState: + duration: float = 0.0 + position: float = 0.0 + is_eos: bool = False + speed: float = 1.0 + is_timestamp_positive: bool = True + samples: AudioSampleT = field(default_factory=list) + is_audio_analyzed = False + class Preview: def __init__(self, @@ -198,20 +215,55 @@ class PreviewManager: self._orig_dir = Path(configpaths.get('MY_DATA')) / 'downloads' self._thumb_dir = Path(configpaths.get('MY_CACHE')) / 'downloads.thumb' - self._previews: dict[str, Preview] = {} - if GLib.mkdir_with_parents(str(self._orig_dir), 0o700) != 0: log.error('Failed to create: %s', self._orig_dir) if GLib.mkdir_with_parents(str(self._thumb_dir), 0o700) != 0: log.error('Failed to create: %s', self._thumb_dir) + self._previews: dict[str, Preview] = {} + + # Holds active audio preview sessions + # for resuming after switching chats + self._audio_sessions: dict[int, AudioPreviewState] = {} + + # References a stop function for each audio preview, which allows us + # to stop previews by preview_id, see stop_audio_except(preview_id) + self._audio_stop_functions: dict[int, Callable[..., None]] = {} + def get_preview(self, preview_id: str) -> Optional[Preview]: return self._previews.get(preview_id) def clear_previews(self) -> None: self._previews.clear() + def get_audio_state(self, + preview_id: int + ) -> AudioPreviewState: + + state = self._audio_sessions.get(preview_id) + if state is not None: + return state + self._audio_sessions[preview_id] = AudioPreviewState() + return self._audio_sessions[preview_id] + + def register_audio_stop_func(self, + preview_id: int, + stop_func: Callable[..., None] + ) -> None: + + self._audio_stop_functions[preview_id] = stop_func + + def unregister_audio_stop_func(self, preview_id: int) -> None: + self._audio_stop_functions.pop(preview_id, None) + + def stop_audio_except(self, preview_id: int) -> None: + # Stops playback of all audio previews except of for preview_id. + # This makes sure that only one preview is played at the time. + for id_, stop_func in self._audio_stop_functions.items(): + if id_ != preview_id: + stop_func() + def _get_session(self, account: str) -> Soup.Session: if account not in self._sessions: self._sessions[account] = self._create_session(account) diff --git a/gajim/common/text_helpers.py b/gajim/common/text_helpers.py index b133913e9..96481b663 100644 --- a/gajim/common/text_helpers.py +++ b/gajim/common/text_helpers.py @@ -36,3 +36,26 @@ def escape_iri_query(s: str) -> str: def jid_to_iri(jid: str) -> str: return 'xmpp:' + escape_iri_path(jid) + + +def format_duration(ns: float, total_ns: float) -> str: + seconds = ns / 1e9 + minutes = seconds / 60 + hours = minutes / 60 + + total_minutes = total_ns / 1e9 / 60 + total_hours = total_minutes / 60 + + i_seconds = int(seconds) % 60 + i_minutes = int(minutes) % 60 + i_hours = int(hours) + + if total_hours >= 1: + width = len(str(int(total_hours))) + return (f'%0{width}d' % i_hours) + f':{i_minutes:02d}:{i_seconds:02d}' + + if total_minutes >= 1: + width = len(str(int(total_minutes))) + return (f'%0{width}d' % i_minutes) + f':{i_seconds:02d}' + + return f'0:{i_seconds:02d}' diff --git a/gajim/data/gui/preview_audio.ui b/gajim/data/gui/preview_audio.ui new file mode 100644 index 000000000..93028cb1f --- /dev/null +++ b/gajim/data/gui/preview_audio.ui @@ -0,0 +1,378 @@ + + + + + + 100 + 1 + 1 + + + 0.25 + 2 + 1 + 0.25 + 0.25 + + + True + False + vertical + 12 + + + + True + False + 6 + + + 300 + 30 + True + False + center + vertical + + + + + + 0 + 0 + + + + + True + True + False + GDK_STRUCTURE_MASK | GDK_SCROLL_MASK + + + + 300 + -1 + True + True + GDK_KEY_PRESS_MASK | GDK_SCROLL_MASK + seek_bar_adj + on + on + False + 0 + 2 + False + bottom + + + + + + + + + + + 0 + 1 + + + + + True + False + + + + + progressbar_label + True + False + Click to change time display + end + center + -0:00/0:00 + fill + True + + + + + + 1 + 1 + + + + + + + + False + True + 0 + + + + + True + False + 6 + 10 + + + True + False + True + + + True + True + True + Rewind 10 seconds + + + + True + False + media-seek-backward + + + + + False + True + 0 + + + + + True + True + True + + + + True + False + Start/stop playback + media-playback-start + + + + + False + True + 1 + + + + + True + True + True + + + + True + False + Forward 10 seconds + media-seek-forward + + + + + False + True + 2 + + + + + + False + True + 0 + + + + + True + False + + + True + True + True + Decrease playback speed + + + + True + False + center + center + list-remove + + + + + False + True + 0 + + + + + True + True + False + True + Select playback speed + center + center + speed_popover + + + True + False + 1 + + + True + False + center + center + 2 + 1.00x + True + + + False + True + 1 + + + + + True + False + center + center + 2 + go-down + + + False + True + 2 + + + + + + + + False + True + 1 + + + + + True + True + True + Increase playback speed + + + + True + False + center + center + list-add + + + + + False + True + 2 + + + + + + False + True + end + 3 + + + + + False + True + 3 + + + + + False + speed_menubutton + bottom + + + True + False + vertical + + + True + False + + + 225 + -1 + True + True + 3 + 3 + speed_bar_adj + False + 1 + 0 + 2 + False + False + right + + + + + + + False + True + 0 + + + + + + + diff --git a/gajim/data/style/gajim.css b/gajim/data/style/gajim.css index 5d3daad48..298ad021a 100644 --- a/gajim/data/style/gajim.css +++ b/gajim/data/style/gajim.css @@ -33,6 +33,8 @@ treeview.space { padding: 6px; } /* GtkLinkButton style */ button.flat.link { padding: 0; border: 0; } +.tabular-digits { font-feature-settings: "tnum"; } + messagedialog.confirmation-dialog > box { margin-top: 18px; } /* InfoBar */ diff --git a/gajim/gtk/builder.pyi b/gajim/gtk/builder.pyi index 7f5efdfd7..8855c5c29 100644 --- a/gajim/gtk/builder.pyi +++ b/gajim/gtk/builder.pyi @@ -652,7 +652,6 @@ class PreferencesBuilder(Builder): status_message: Gtk.Grid automatic_status: Gtk.Grid themes: Gtk.Grid - emoji: Gtk.Grid av_info_bar: Gtk.InfoBar button1: Gtk.Button av_info_bar_label: Gtk.Label @@ -687,6 +686,26 @@ class PreviewBuilder(Builder): info_message: Gtk.Label +class PreviewAudioBuilder(Builder): + seek_bar_adj: Gtk.Adjustment + speed_bar_adj: Gtk.Adjustment + preview_box: Gtk.Box + drawing_box: Gtk.Box + seek_bar: Gtk.Scale + progress_label: Gtk.Label + control_box: Gtk.Box + rewind_button: Gtk.Button + play_pause_button: Gtk.Button + play_icon: Gtk.Image + forward_button: Gtk.Button + speed_dec_button: Gtk.Button + speed_menubutton: Gtk.MenuButton + speed_label: Gtk.Label + speed_inc_button: Gtk.Button + speed_popover: Gtk.Popover + speed_bar: Gtk.Scale + + class ProfileBuilder(Builder): privacy_popover: Gtk.Popover avatar_nick_access: Gtk.Switch @@ -1011,6 +1030,8 @@ def get_builder(file_name: Literal['preferences.ui'], widgets: list[str] = ...) @overload def get_builder(file_name: Literal['preview.ui'], widgets: list[str] = ...) -> PreviewBuilder: ... @overload +def get_builder(file_name: Literal['preview_audio.ui'], widgets: list[str] = ...) -> PreviewAudioBuilder: ... +@overload def get_builder(file_name: Literal['profile.ui'], widgets: list[str] = ...) -> ProfileBuilder: ... @overload def get_builder(file_name: Literal['roster.ui'], widgets: list[str] = ...) -> RosterBuilder: ... diff --git a/gajim/gtk/preview_audio.py b/gajim/gtk/preview_audio.py index 6052a0ce8..ad892a252 100644 --- a/gajim/gtk/preview_audio.py +++ b/gajim/gtk/preview_audio.py @@ -14,171 +14,605 @@ from __future__ import annotations -from typing import Optional +from typing import Any import logging from pathlib import Path +from gi.repository import Gdk from gi.repository import GLib from gi.repository import Gtk + try: from gi.repository import Gst except Exception: pass +from gajim.common import app from gajim.common.i18n import _ +from gajim.common.text_helpers import format_duration +from gajim.common.preview import AudioSampleT +from .builder import get_builder +from .preview_audio_analyzer import AudioAnalyzer +from .preview_audio_visualizer import AudioVisualizerWidget from .util import get_cursor log = logging.getLogger('gajim.gui.preview_audio') +# Padding to align the visualizer drawing with the seekbar +SEEK_BAR_PADDING = 11 + class AudioWidget(Gtk.Box): def __init__(self, file_path: Path) -> None: + Gtk.Box.__init__(self, orientation=Gtk.Orientation.HORIZONTAL, spacing=6) - self._playbin: Optional[Gst.Element] = Gst.ElementFactory.make( - 'playbin', 'bin') + + self._playbin = Gst.ElementFactory.make('playbin', 'bin') + self._bus_watch_id: int = 0 + self._timeout_id: int = -1 + if self._playbin is None: + log.warning('Could not create GST playbin') label = Gtk.Label(label=_('Audio preview is not available')) self.add(label) - log.debug('Could not create GST playbin') + self.show_all() return - self._query: Gst.Query = Gst.Query.new_position(Gst.Format.TIME) - self._has_timeout: bool = False + self._file_path = file_path + self._id = hash(self._file_path) - self._build_audio_widget() + app.preview_manager.register_audio_stop_func(self._id, self._set_ready) + + self._seek_pos = -1.0 + self._offset_backward = -10e9 # in ns + self._offset_forward = 10e9 + self._pause_seek = False + self._is_ready = True + self._next_state_is_playing = False + + # Constants which define player's behaviour + self._speed_min = 0.25 + self._speed_max = 2.00 + self._speed_inc_step = 0.25 + self._speed_dec_step = 0.25 + + self._query = Gst.Query.new_position(Gst.Format.TIME) self._setup_audio_player(file_path) - def _build_audio_widget(self) -> None: - play_button = Gtk.Button() - play_button.get_style_context().add_class('flat') - play_button.get_style_context().add_class('border') - play_button.set_tooltip_text(_('Start/stop playback')) - self._play_icon = Gtk.Image.new_from_icon_name( - 'media-playback-start-symbolic', - Gtk.IconSize.BUTTON) - play_button.add(self._play_icon) - play_button.connect('clicked', self._on_play_clicked) - event_box = Gtk.EventBox() - event_box.connect('realize', self._on_realize) - event_box.add(play_button) - self.add(event_box) - - self._seek_bar = Gtk.Scale( - orientation=Gtk.Orientation.HORIZONTAL) - self._seek_bar.set_range(0.0, 1.0) - self._seek_bar.set_size_request(300, -1) - self._seek_bar.set_value_pos(Gtk.PositionType.RIGHT) - self._seek_bar.connect('change-value', self._on_seek) - self._seek_bar.connect( - 'format-value', self._format_audio_timestamp) - event_box = Gtk.EventBox() - event_box.connect('realize', self._on_realize) - event_box.add(self._seek_bar) - self.add(event_box) + self._ui = get_builder('preview_audio.ui') + self._is_LTR = \ + (self._ui.seek_bar.get_direction() == Gtk.TextDirection.LTR) + self._enable_controls(False) + + self._ui.connect_signals(self) + self.add(self._ui.preview_box) + self._setup_audio_visualizer() + + # Initialize with restored audio state or defaults + self._state = app.preview_manager.get_audio_state(self._id) + self._audio_analyzer = None + + if not self._state.is_audio_analyzed: + # Analyze the audio to determine samples and duration, + # calls self._update_audio_data when done. + self._audio_analyzer = AudioAnalyzer( + file_path, self._update_duration, self._update_samples) + else: + self._update_ui() + + self._ui.speed_bar_adj.configure( + value=self._state.speed, + lower=self._speed_min, + upper=self._speed_max, + step_increment=self._speed_inc_step, + page_increment=self._speed_inc_step, + page_size=0 + ) + + self._ui.speed_bar.set_value(self._state.speed) + self._set_playback_speed(self._state.speed) + + self._ui.speed_bar.add_mark(0.25, Gtk.PositionType.BOTTOM, '0.25') + self._ui.speed_bar.add_mark(0.5, Gtk.PositionType.BOTTOM, '') + self._ui.speed_bar.add_mark(0.75, Gtk.PositionType.BOTTOM, '') + self._ui.speed_bar.add_mark(1, Gtk.PositionType.BOTTOM, '1.0') + self._ui.speed_bar.add_mark(1.25, Gtk.PositionType.BOTTOM, '') + self._ui.speed_bar.add_mark(1.5, Gtk.PositionType.BOTTOM, '1.5') + self._ui.speed_bar.add_mark(1.75, Gtk.PositionType.BOTTOM, '') + self._ui.speed_bar.add_mark(2, Gtk.PositionType.BOTTOM, '2') + + if self._is_LTR: + self._ui.progress_label.set_xalign(1.0) + else: + self._ui.progress_label.set_xalign(0.0) self.connect('destroy', self._on_destroy) self.show_all() + def _enable_controls(self, status: bool) -> None: + self._ui.seek_bar.set_sensitive(status) + self._ui.progress_label.set_sensitive(status) + self._ui.play_pause_button.set_sensitive(status) + self._ui.rewind_button.set_sensitive(status) + self._ui.forward_button.set_sensitive(status) + self._ui.speed_dec_button.set_sensitive(status) + self._ui.speed_inc_button.set_sensitive(status) + self._ui.speed_menubutton.set_sensitive(status) + + def _update_ui(self) -> None: + if not self._state.duration > 0: + log.debug('Could not successfully load audio. Duration is zero.') + return + + self._enable_controls(True) + + if len(self._state.samples) > 0: + self._audio_visualizer.update_params( + self._state.samples, + self._state.position / self._state.duration) + self._audio_visualizer.draw_graph( + self._state.position / self._state.duration) + + self._ui.seek_bar_adj.configure( + value=self._state.position, + lower=0.0, + upper=self._state.duration, + step_increment=5e9, # for hold+position left/right + page_increment=0, # determines scrolling and click behaviour + page_size=0 + ) + + # Calculate max string length to prevent timestamp label from jumping + formatted = format_duration(0.0, self._state.duration) + self._ui.progress_label.set_width_chars( + len(f'-{formatted}/{formatted}')) + self._update_timestamp_label() + + def _update_samples(self, + samples: AudioSampleT, + ) -> None: + self._state.samples = samples + self._state.is_audio_analyzed = True + self._update_ui() + + def _update_duration(self, duration: float): + self._state.duration = duration + self._update_ui() + def _setup_audio_player(self, file_path: Path) -> None: assert self._playbin is not None + + # Set up the whole pipline + # For reference see + # https://gstreamer.freedesktop.org/ + # documentation/audiofx/scaletempo.html + audio_sink = Gst.Bin.new('audiosink') + audioconvert = Gst.ElementFactory.make('audioconvert', 'audioconvert') + scaletempo = Gst.ElementFactory.make('scaletempo', 'scaletempo') + audioresample = Gst.ElementFactory.make( + 'audioresample', 'audioresample') + autoaudiosink = Gst.ElementFactory.make( + 'autoaudiosink', 'autoaudiosink') + + pipeline_elements = [ + audio_sink, audioconvert, scaletempo, audioresample, autoaudiosink] + if any(element is None for element in pipeline_elements): + # If it fails there will be + # * a delay until playback starts + # * a chipmunk effect when speeding up the playback + log.warning('Could not set up full audio preview pipeline.') + else: + assert autoaudiosink is not None + assert audioconvert is not None + assert scaletempo is not None + assert audioresample is not None + autoaudiosink.set_property('sync', False) + + audio_sink.add(audioconvert) + audio_sink.add(scaletempo) + audio_sink.add(audioresample) + audio_sink.add(autoaudiosink) + + audioconvert.link(scaletempo) + scaletempo.link(audioresample) + audioresample.link(autoaudiosink) + + sink_pad = audioconvert.get_static_pad('sink') + assert sink_pad is not None + ghost_pad = Gst.GhostPad.new('sink', sink_pad) + assert ghost_pad is not None + audio_sink.add_pad(ghost_pad) + self._playbin.set_property('audio-sink', audio_sink) + self._playbin.set_property('uri', file_path.as_uri()) - state_return = self._playbin.set_state(Gst.State.PAUSED) + self._playbin.no_more_pads() + + state_return = self._playbin.set_state(Gst.State.READY) if state_return == Gst.StateChangeReturn.FAILURE: log.debug('Could not setup GST playbin') return - bus: Optional[Gst.Bus] = self._playbin.get_bus() + bus = self._playbin.get_bus() if bus is None: log.debug('Could not get GST Bus') return + bus.add_signal_watch() - bus.connect('message', self._on_bus_message) + self._bus_watch_id = bus.connect('message', self._on_bus_message) - def _on_bus_message(self, _bus: Gst.Bus, message: Gst.Message) -> None: - assert self._playbin is not None - if message.type == Gst.MessageType.EOS: - self._set_pause(True) - self._playbin.seek_simple( - Gst.Format.TIME, Gst.SeekFlags.FLUSH, 0) - elif message.type == Gst.MessageType.STATE_CHANGED: - success, duration = self._playbin.query_duration( - Gst.Format.TIME) - if not success: - return - assert duration is not None - if duration > 0: - self._seek_bar.set_range(0.0, duration) + def _setup_audio_visualizer(self) -> None: + width, height = self._ui.seek_bar.get_size_request() - is_paused = self._get_paused() - if (duration > 0 and not is_paused and - not self._has_timeout): - GLib.timeout_add(50, self._update_seek_bar) - self._has_timeout = True + if width == -1: + width = self._ui.seek_bar.get_preferred_width()[1] - def _on_seek(self, - _range: Gtk.Range, - _scroll: Gtk.ScrollType, - value: float - ) -> bool: + if height == -1: + height = self._ui.seek_bar.get_preferred_height()[1] + + if width is None or height is None: + return + + width -= 2 * SEEK_BAR_PADDING + seek_bar_rgba = self._ui.seek_bar.get_style_context().get_color( + Gtk.StateFlags.LINK) + self._audio_visualizer = AudioVisualizerWidget( + width, + height, + SEEK_BAR_PADDING, + seek_bar_rgba) + + self._ui.drawing_box.add(self._audio_visualizer) + + def _update_timestamp_label(self) -> None: + cur = self._state.position + dur = self._state.duration + + dur_str = format_duration(dur, dur) + ltr_char = u'\u202B' + pop_char = u'\u202C' + + if self._state.is_timestamp_positive: + cur_str = f'{format_duration(cur, dur)}' + self._ui.progress_label.set_text(f'{cur_str}/{dur_str}') + else: + cur_str = f'{format_duration(dur - cur, dur)}' + if self._is_LTR: + self._ui.progress_label.set_text(f'-{cur_str}/{dur_str}') + else: + self._ui.progress_label.set_text( + f'{cur_str}{ltr_char}/-{pop_char}{dur_str}' + ) + + def _update_seek_bar_and_visualisation(self) -> bool: assert self._playbin is not None - self._playbin.seek_simple( - Gst.Format.TIME, Gst.SeekFlags.FLUSH, int(value)) - return False + if self._playbin.query(self._query): + _fmt, position = self._query.parse_position() - def _on_play_clicked(self, _button: Gtk.Button) -> None: - self._set_pause(not self._get_paused()) + if position is None: + self._timeout_id = -1 + return False + + if not self._pause_seek: + self._state.position = position + self._ui.seek_bar.set_value(self._state.position) + + if not self._get_ready() and not self._get_paused(): + self._audio_visualizer.draw_graph( + position / self._state.duration, + self._seek_pos / self._state.duration) + return True + + def _add_seek_bar_update_idle(self) -> None: + if self._timeout_id != -1: + return + + self._timeout_id = \ + GLib.timeout_add(80, self._update_seek_bar_and_visualisation) + + def _remove_seek_bar_update_idle(self) -> None: + if self._timeout_id != -1: + GLib.source_remove(self._timeout_id) + self._timeout_id = -1 + + def _get_constrained_position(self, pos: float) -> float: + if pos >= self._state.duration: + return self._state.duration + if pos < 0: + return 0.0 + return pos + + def _get_constrained_speed(self, speed: float) -> tuple[bool, float]: + if self._speed_min <= speed <= self._speed_max: + return True, speed + return False, self._state.speed + + def _set_playback_speed(self, speed: float) -> bool: + success, self._state.speed = self._get_constrained_speed(speed) + if not success: + return False + + self._ui.speed_label.set_text(f'{self._state.speed:.2f}x') - def _on_destroy(self, _widget: Gtk.Widget) -> None: assert self._playbin is not None - self._playbin.set_state(Gst.State.NULL) + self._playbin.seek(self._state.speed, + Gst.Format.TIME, + Gst.SeekFlags.FLUSH, + Gst.SeekType.SET, + int(self._state.position), + Gst.SeekType.NONE, + 0) + return True def _get_paused(self) -> bool: assert self._playbin is not None - _, state, _ = self._playbin.get_state(20) + _, state, _ = self._playbin.get_state(timeout=40) return state == Gst.State.PAUSED + def _get_ready(self) -> bool: + assert self._playbin is not None + _, state, _ = self._playbin.get_state(timeout=40) + return state == Gst.State.READY + def _set_pause(self, paused: bool) -> None: assert self._playbin is not None if paused: self._playbin.set_state(Gst.State.PAUSED) - self._play_icon.set_from_icon_name( + self._remove_seek_bar_update_idle() + self._ui.play_icon.set_from_icon_name( 'media-playback-start-symbolic', Gtk.IconSize.BUTTON) else: self._playbin.set_state(Gst.State.PLAYING) - self._play_icon.set_from_icon_name( + self._add_seek_bar_update_idle() + self._ui.play_icon.set_from_icon_name( 'media-playback-pause-symbolic', Gtk.IconSize.BUTTON) - def _update_seek_bar(self) -> bool: - if self._get_paused(): - self._has_timeout = False - return False + def _set_ready(self) -> None: + assert self._playbin is not None + self._playbin.set_state(Gst.State.READY) + self._is_ready = True + + # State order is READY -> PAUSE -> PLAYING + # I.e. we need to pause first, but keep in mind, that we want to + # go a state further + self._next_state_is_playing = True + self._ui.play_icon.set_from_icon_name( + 'media-playback-start-symbolic', + Gtk.IconSize.BUTTON) + + def _seek(self, position: float) -> None: + ''' + Used in: + * _on_seek: When the slider is dragged + * _on_seek_bar_button_released: + * _on_play_clicked + ''' assert self._playbin is not None - if self._playbin.query(self._query): - _fmt, cur_pos = self._query.parse_position() - if cur_pos is not None: - self._seek_bar.set_value(cur_pos) - return True - @staticmethod - def _format_audio_timestamp(_widget: Gtk.Scale, ns: float) -> str: - seconds = ns / 1000000000 - minutes = seconds / 60 - hours = minutes / 60 - - i_seconds = int(seconds) % 60 - i_minutes = int(minutes) % 60 - i_hours = int(hours) - - if i_hours > 0: - return f'{i_hours:d}:{i_minutes:02d}:{i_seconds:02d}' - return f'{i_minutes:d}:{i_seconds:02d}' + self._state.position = self._get_constrained_position(position) + self._state.is_eos = self._state.position >= self._state.duration + + if self._pause_seek: + return + + self._playbin.seek(self._state.speed, + Gst.Format.TIME, + Gst.SeekFlags.FLUSH, + Gst.SeekType.SET, + int(self._state.position), Gst.SeekType.NONE, 0 + ) + + if self._state.position >= self._state.duration: + self._playbin.send_event(Gst.Event.new_eos()) + + def _seek_unconditionally(self, position: float) -> None: + ''' + Used in: + * _on_seek_bar_button_pressed + * _on_rewind_clicked + * _on_forward_clicked + ''' + assert self._playbin is not None + + self._state.position = self._get_constrained_position(position) + self._state.is_eos = self._state.position >= self._state.duration + + if not self._is_ready: + self._playbin.seek(self._state.speed, + Gst.Format.TIME, + Gst.SeekFlags.FLUSH, + Gst.SeekType.SET, + int(self._state.position), + Gst.SeekType.NONE, + 0) + + self._ui.seek_bar.set_value(self._state.position) + self._audio_visualizer.draw_graph( + self._state.position / self._state.duration) + + def _on_bus_message(self, _bus: Gst.Bus, message: Gst.Message) -> None: + assert self._playbin is not None + + if message.type == Gst.MessageType.EOS: + self._state.is_eos = True + self._set_pause(True) + self._ui.seek_bar.set_value(self._state.duration) + self._audio_visualizer.draw_graph(1.0) + + elif message.type == Gst.MessageType.STATE_CHANGED: + is_paused = self._get_paused() + + if self._is_ready and is_paused: + # State changed from READY --> PAUSED + self._is_ready = False + self._playbin.seek(self._state.speed, + Gst.Format.TIME, + Gst.SeekFlags.FLUSH, + Gst.SeekType.SET, + int(self._state.position), + Gst.SeekType.NONE, + 0) + self._ui.seek_bar.set_value(self._state.position) + + if self._next_state_is_playing: + # Continue from state PAUSED --> PLAYING + self._set_pause(False) + self._next_state_is_playing = False + + return + + def _on_speed_change(self, + _range: Gtk.Range, + _scroll: Gtk.ScrollType, + value: float + ) -> None: + + self._set_playback_speed(value) + + def _on_speed_inc_clicked(self, _button: Gtk.Button) -> None: + speed = self._state.speed + self._speed_inc_step + if self._set_playback_speed(speed): + self._ui.speed_bar.set_value(speed) + + def _on_speed_dec_clicked(self, _button: Gtk.Button) -> None: + speed = self._state.speed - self._speed_inc_step + if self._set_playback_speed(speed): + self._ui.speed_bar.set_value(speed) + else: + log.debug('Could not set speed!') + + def _on_seek_bar_moved(self, _scake: Gtk.Scale) -> None: + self._update_timestamp_label() + + def _on_timestamp_label_clicked(self, + _label: Gtk.Label, + *args: Any + ) -> None: + + self._state.is_timestamp_positive = \ + not self._state.is_timestamp_positive + self._update_timestamp_label() + + def _on_seek_bar_button_pressed(self, + _scale: Gtk.Scale, + event: Gdk.EventButton, + ) -> None: + assert self._cursor_pos is not None + # There are two cases when the user clicks on the seek bar: + # 1) Press and immediately release: Jump to the new position and + # continue playing + # 2) Start of dragging the slider + # In case of 2) pause active seeking to prevent audio scrubbing and + # instead continue playing + self._pause_seek = True + width = self._ui.seek_bar.get_allocation().width - 2 * SEEK_BAR_PADDING + new_pos = self._state.duration * self._cursor_pos / width + self._seek_unconditionally(new_pos) + + def _on_seek_bar_button_released(self, + _scale: Gtk.Scale, + event: Gdk.EventButton, + ) -> None: + self._pause_seek = False + self._seek(self._state.position) + + # Set the seek position to -1 to indicate that the user isn't about + # to change the position + self._seek_pos = -1 + + def _on_seek_bar_cursor_move(self, + _scale: Gtk.Scale, + event: Gdk.EventMotion + ) -> None: + # Used to determine the click position on the seekbar + if self._is_LTR: + self._cursor_pos = event.x - SEEK_BAR_PADDING + else: + width = self._ui.seek_bar.get_allocation().width + self._cursor_pos = width - (event.x + SEEK_BAR_PADDING) + + def _on_seek_bar_scrolled(self, + _scale: Gtk.Scale, + event: Gdk.EventMotion + ) -> None: + _is_smooth, _delta_x, delta_y = event.get_scroll_deltas() + if delta_y > 0: + new_pos = self._state.position + self._offset_backward + else: + new_pos = self._state.position + self._offset_forward + self._seek_unconditionally(new_pos) + + def _on_seek(self, + _scale: Gtk.Scale, + _scroll: Gtk.ScrollType, + value: float + ) -> None: + self._seek(value) + + if not (self._get_paused() or self._get_ready()): + self._seek_pos = value + else: + self._audio_visualizer.draw_graph( + self._state.position / self._state.duration) + + def _on_play_clicked(self, _button: Gtk.Button) -> None: + app.preview_manager.stop_audio_except(self._id) + if self._get_ready(): + # The order is always READY -> PAUSE -> PLAYING + self._set_pause(True) + self._next_state_is_playing = True + + if self._state.is_eos: + new_pos = 0.0 + else: + new_pos = self._state.position + + self._seek(new_pos) + self._query.set_position(Gst.Format.TIME, int(new_pos)) + self._ui.seek_bar.set_value(new_pos) + return + + if self._state.is_eos and self._get_paused(): + self._seek(0.0) + self._ui.seek_bar.set_value(0.0) + + self._set_pause(not self._get_paused()) + + def _on_rewind_clicked(self, _button: Gtk.Button) -> None: + new_pos = self._get_constrained_position( + self._state.position + self._offset_backward) + self._seek_unconditionally(new_pos) + + def _on_forward_clicked(self, _button: Gtk.Button) -> None: + new_pos = self._get_constrained_position( + self._state.position + self._offset_forward) + self._seek_unconditionally(new_pos) + + def _on_destroy(self, _widget: Gtk.Widget) -> None: + if self._playbin is not None: + self._playbin.set_state(Gst.State.NULL) + bus = self._playbin.get_bus() + + if bus is not None: + bus.remove_signal_watch() + bus.disconnect(self._bus_watch_id) + + if self._audio_analyzer is not None: + self._audio_analyzer.destroy() + + if self._ui.speed_popover is not None: + self._ui.speed_popover.destroy() + + self._remove_seek_bar_update_idle() + + app.preview_manager.unregister_audio_stop_func(self._id) + app.check_finalize(self) @staticmethod def _on_realize(event_box: Gtk.EventBox) -> None: diff --git a/gajim/gtk/preview_audio_analyzer.py b/gajim/gtk/preview_audio_analyzer.py new file mode 100644 index 000000000..cc7cb10bc --- /dev/null +++ b/gajim/gtk/preview_audio_analyzer.py @@ -0,0 +1,167 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see . + +from __future__ import annotations + +from typing import Callable +from typing import cast +from typing import Optional + +import logging +import math +from pathlib import Path + +try: + from gi.repository import Gst +except Exception: + pass + +from gajim.common import app +from gajim.common.preview import AudioSampleT + +log = logging.getLogger('gajim.gui.preview_audio_analyzer') + + +class AudioAnalyzer: + def __init__(self, + filepath: Path, + duration_callback: Callable[[float], None], + samples_callback: Callable[[AudioSampleT], None] + ) -> None: + + self._playbin = Gst.ElementFactory.make('playbin', 'bin') + + if self._playbin is None: + log.debug('Could not create GST playbin for AudioAnalyzer') + return + + self._duration_callback = duration_callback + self._duration_updated = False + self._samples_callback = samples_callback + self._query = Gst.Query.new_position(Gst.Format.TIME) + self._duration = Gst.CLOCK_TIME_NONE # in ns + self._num_channels = 1 + self._samples: list[tuple[float, float]] = [] + self._level: Optional[Gst.Element] = None + self._bus_watch_id: int = 0 + + self._setup_audio_analyzer(filepath) + + def _setup_audio_analyzer(self, file_path: Path) -> None: + assert isinstance(self._playbin, Gst.Bin) + + audio_sink = Gst.Bin.new('audiosink') + audioconvert = Gst.ElementFactory.make('audioconvert', 'audioconvert') + self._level = Gst.ElementFactory.make('level', 'level') + fakesink = Gst.ElementFactory.make('fakesink', 'fakesink') + + pipeline_elements = [audio_sink, audioconvert, self._level, fakesink] + if any(element is None for element in pipeline_elements): + log.error('Could not set up pipeline for AudioAnalyzer') + return + + assert audioconvert is not None + assert self._level is not None + assert fakesink is not None + + audio_sink.add(audioconvert) + audio_sink.add(self._level) + audio_sink.add(fakesink) + + audioconvert.link(self._level) + self._level.link(fakesink) + + sink_pad = audioconvert.get_static_pad('sink') + assert sink_pad is not None + ghost_pad = Gst.GhostPad.new('sink', sink_pad) + assert ghost_pad is not None + audio_sink.add_pad(ghost_pad) + + self._playbin.set_property('audio-sink', audio_sink) + file_uri = file_path.as_uri() + self._playbin.set_property('uri', file_uri) + self._playbin.no_more_pads() + + self._level.set_property('message', True) + fakesink.set_property('sync', False) + + state_return = self._playbin.set_state(Gst.State.PLAYING) + if state_return == Gst.StateChangeReturn.FAILURE: + log.warning('Could not set up GST playbin') + return + + self._level_element = self._playbin.get_by_name('level') + bus = self._playbin.get_bus() + if bus is None: + log.debug('Could not get GST Bus') + return + + bus.add_signal_watch() + self._bus_watch_id = bus.connect('message', self._on_bus_message) + + def _on_bus_message(self, _bus: Gst.Bus, message: Gst.Message) -> None: + assert self._playbin is not None + + if message.type == Gst.MessageType.EOS: + self._samples_callback(self._samples) + self._playbin.set_state(Gst.State.NULL) + return + + if (message.type in (Gst.MessageType.STATE_CHANGED, + Gst.MessageType.DURATION_CHANGED)): + _success, self._duration = self._playbin.query_duration( + Gst.Format.TIME) + if not self._duration_updated: + if _success: + assert self._duration is not None + self._duration_callback(float(self._duration)) + self._duration_updated = True + return + + if message.src is self._level: + structure = message.get_structure() + if structure is None or structure.get_name() != 'level': + return + + if not structure.has_field('rms'): + return + + # RMS: Root Mean Square = Average Power + rms_values = cast(list[float], structure.get_value('rms')) + assert rms_values is not None + self._num_channels = min(2, len(rms_values)) + + # Convert from dB to a linear scale. + # The sound pressure level L is defined as + # L = 10 log_10((p/p_0)^2) dB, where p is the RMS value + # of the sound pressure. + if self._num_channels == 1: + lin_val = math.pow(10, rms_values[0] / 10 / 2) + self._samples.append((lin_val, lin_val)) + else: + lin_val1 = math.pow(10, rms_values[0] / 10 / 2) + lin_val2 = math.pow(10, rms_values[1] / 10 / 2) + self._samples.append((lin_val1, lin_val2)) + + def destroy(self) -> None: + if self._playbin is not None: + self._playbin.set_state(Gst.State.NULL) + bus = self._playbin.get_bus() + + if bus is not None: + bus.remove_signal_watch() + bus.disconnect(self._bus_watch_id) + + del self._duration_callback, self._samples_callback + app.check_finalize(self) diff --git a/gajim/gtk/preview_audio_visualizer.py b/gajim/gtk/preview_audio_visualizer.py new file mode 100644 index 000000000..37d108c4c --- /dev/null +++ b/gajim/gtk/preview_audio_visualizer.py @@ -0,0 +1,255 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see . + +from __future__ import annotations + +from typing import Optional + +import logging +import cairo +import math +from statistics import mean + +from gi.repository import Gdk +from gi.repository import Gtk + +from gajim.common.preview import AudioSampleT + +from .util import rgba_to_float + + +log = logging.getLogger('gajim.gui.preview_audio_visualizer') + + +class AudioVisualizerWidget(Gtk.DrawingArea): + def __init__(self, + width: int, + height: int, + x_offset: int, + seek_bar_color: Gdk.RGBA + ) -> None: + + Gtk.DrawingArea.__init__(self) + + self._x_offset = x_offset + self._width = width + self._height = height + self._is_LTR = (self.get_direction() == Gtk.TextDirection.LTR) + self._amplitude_width = 2 + + self._num_samples = 0 + self._samples: AudioSampleT = [] + self._seek_position = -1.0 + self._position = 0.0 + + self._default_color = (0.6, 0.6, 0.6) # gray + self._progress_color = rgba_to_float(seek_bar_color) + + # Use a 50% lighter color to the seeking indicator + self._seek_color = tuple( + min(1.0, color * 1.5) for color in self._progress_color + ) + + self._surface: Optional[cairo.ImageSurface] = None + + self.set_size_request(self._width, self._height) + + self.connect('draw', self._on_drawingarea_draw) + self.connect('configure-event', self._on_drawingarea_changed) + + self._setup_surface() + + def update_params(self, + samples: AudioSampleT, + position: float + ) -> None: + + self._samples = samples + self._position = position + self._process_samples() + self._num_samples = len(self._samples) + + def draw_graph(self, position: float, seek_position: float = -1.0) -> None: + if self._num_samples == 0: + return + + self._position = position + self._seek_position = seek_position + + self.queue_draw() + + def _setup_surface(self) -> None: + if self._surface: + self._surface.finish() + + self._surface = cairo.ImageSurface( + cairo.FORMAT_ARGB32, + self._width, + self._height) + ctx: cairo.Context[cairo.ImageSurface] = cairo.Context(self._surface) + + if self._num_samples != 0: + self._draw_surface(ctx) + + def _process_samples(self) -> None: + # Pick a subset of all samples and average over three samples + w = int(self._width / (self._amplitude_width * 2)) + n = math.floor(len(self._samples) / w) + 1 + + if n >= 2: + samples: AudioSampleT = [] + samples1, samples2 = zip(*self._samples) + for i in range(2, int(len(self._samples) - n / 2), n): + index = int(i + n / 2) + avg1 = mean(samples1[index - 1:index + 1]) + avg2 = mean(samples2[index - 1:index + 1]) + samples.append((avg1, avg2)) + else: + samples = self._samples + + # Normalize both channels using the same scale + max_elem = max(max(samples)) + self._samples = [ + (val1 / max_elem, val2 / max_elem) for val1, val2 in samples + ] + + def _draw_surface(self, ctx: cairo.Context[cairo.ImageSurface]) -> None: + # First draw the progress part from left + end = min(round(self._position * self._num_samples), self._num_samples) + self._draw_rms_amplitudes(ctx, + start=0, + end=end, + color=self._progress_color) + + if self._seek_position >= 0: + # If state is PLAYING and the user seeks, determine whether the + # seek position lies in the past, i.e. on the left side of current + # position or in the future, i.e. on the right side. + # Highlight the skipped area from seek + # to the current position on the determined site. + play_pos = min(round( + self._position * self._num_samples), self._num_samples) + seek_pos = min(round( + self._seek_position * self._num_samples), self._num_samples) + if play_pos > seek_pos: + start = seek_pos + end = play_pos + else: + start = play_pos + end = seek_pos + + self._draw_rms_amplitudes(ctx, + start=start, + end=end, + color=self._seek_color) + + # Draw the default amplitudes for the rest of the timeline + play_pos = min(round( + self._position * self._num_samples), self._num_samples) + seek_pos = min(round( + self._seek_position * self._num_samples), self._num_samples) + start_default = max(play_pos, seek_pos) + + self._draw_rms_amplitudes(ctx, + start=start_default, + end=self._num_samples, + color=self._default_color) + + def _draw_rms_amplitudes(self, + ctx: cairo.Context[cairo.ImageSurface], + start: int, + end: int, + color: tuple[float, float, float] + ) -> None: + + ctx.set_source_rgb(*color) + + w = self._amplitude_width + + # determines the spacing between the amplitudes + o = (self._width - 2 * w) / self._num_samples + y = self._height / 2 + r = 1 # radius of the arcs on top and bottom of the amplitudes + s = self._height / 2 - r # amplitude scaling factor + + if self._is_LTR: + o = (self._width - 2 * w) / self._num_samples + x = self._x_offset + o * (start + 1) + else: + o = -(self._width - 2 * w) / self._num_samples + x = -o * (self._num_samples - start) + self._x_offset + + for i in range(start, end): + # Ensure that no empty area is drawn, + # thus use a minimum sample value + + if self._is_LTR: + sample1 = max(0.05, self._samples[i][0]) + sample2 = max(0.05, self._samples[i][1]) + else: + sample1 = max(0.05, self._samples[i][1]) + sample2 = max(0.05, self._samples[i][0]) + + self._draw_rounded_rec2(ctx, x, y, w, sample1 * s, sample2 * s, r) + ctx.fill() + x += o + + def _on_drawingarea_draw(self, + _drawing_area: Gtk.DrawingArea, + ctx: cairo.Context[cairo.ImageSurface], + ) -> None: + + if self._num_samples != 0: + self._draw_surface(ctx) + + def _on_drawingarea_changed(self, + _drawing_area: Gtk.DrawingArea, + _event: Gdk.EventConfigure, + ) -> None: + + self._is_LTR = (self.get_direction() == Gtk.TextDirection.LTR) + self._setup_surface() + self.queue_draw() + + def _draw_rounded_rec2(self, + context: cairo.Context[cairo.ImageSurface], + x: float, + y: float, + w: float, + h1: float, + h2: float, + r: float + ) -> None: + + ''' + Draws a rectangle of width w and total height of h1+h2 + The top and bottom edges are curved to the outside + ''' + m = w / 2 + + if not self._is_LTR: + m = -m + w = -w + + context.curve_to(x, y + h1, + x + m, y + h1 + r, + x + w, y + h1) + + context.line_to(x + w, y - h2) + + context.curve_to(x + w, y - h2, + x + m, y - h2 - r, + x, y - h2) + + context.line_to(x, y + h1) diff --git a/pyproject.toml b/pyproject.toml index ea2b8d296..2160a1b8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,6 +125,8 @@ include = [ "gajim/gtk/plugins.py", "gajim/gtk/preview.py", "gajim/gtk/preview_audio.py", + "gajim/gtk/preview_audio_analyzer.py", + "gajim/gtk/preview_audio_visualizer.py", "gajim/gtk/proxies.py", "gajim/gtk/remove_account.py", "gajim/gtk/resource_selector.py", diff --git a/test/no_gui/test_text_helpers.py b/test/no_gui/test_text_helpers.py index 86897829c..c71b8ee8e 100644 --- a/test/no_gui/test_text_helpers.py +++ b/test/no_gui/test_text_helpers.py @@ -2,6 +2,7 @@ import unittest from gajim.common.text_helpers import escape_iri_path_segment from gajim.common.text_helpers import jid_to_iri +from gajim.common.text_helpers import format_duration class Test(unittest.TestCase): @@ -30,6 +31,30 @@ class Test(unittest.TestCase): self.assertEqual(jid_to_iri(jid), r'xmpp:my%5C20self@%5B::1%5D/home', jid) + def test_format_duration_width(self): + def do(total_seconds, expected): + self.assertEqual(format_duration(0.0, total_seconds*1e9), expected) + + do( 0, '0:00') + do( 60, '0:00') + do( 10*60, '00:00') + do( 60*60, '0:00:00') + do( 10*60*60, '00:00:00') + do(100*60*60, '000:00:00') + + def test_format_duration(self): + def do(duration, expected): + self.assertEqual(format_duration(duration, 100*60*60*1e9), expected) + + do( 1.0, '000:00:00') + do( 999999999.0, '000:00:00') + do( 1000000000.0, '000:00:01') + do( 59999999999.0, '000:00:59') + do( 60000000000.0, '000:01:00') + do( 3599999999999.0, '000:59:59') + do( 3600000000000.0, '001:00:00') + do(3599999999999999.0, '999:59:59') + if __name__ == '__main__': unittest.main() -- cgit v1.2.3