diff options
author | Philipp Hörist <philipp@hoerist.com> | 2023-06-11 22:48:45 +0300 |
---|---|---|
committer | Philipp Hörist <philipp@hoerist.com> | 2023-11-18 12:55:41 +0300 |
commit | d5e8951bbf0a70cb05c0b4594092c83f493e1d55 (patch) | |
tree | b85144061f418316d9787603cc8b5038b6248c25 | |
parent | 29b7736708c18b9e0caea7a12647e47a352f6aab (diff) |
refactor: Rewrite archive database code
64 files changed, 5702 insertions, 3046 deletions
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 948ccd6ba..59806345c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,3 +84,14 @@ While developing this command is useful to preview the manpage To convert the markdown $ pandoc gajim.1.md -s -t man -o gajim.1 + +# Database + +The database layout is defined in `archive.dbml`. +Use the python package `dbml-sqlite` to transform it to SQLite. + +```python +from dbml_sqlite import toSQLite +from pathlib import Path +Path("./archive.sql").write_text(toSQLite('archive.dbml', tableExists=False, indexExists=False)) +``` @@ -20,7 +20,7 @@ - [GLib](https://gitlab.gnome.org/GNOME/glib) (>=2.66.0) - [GtkSourceView](https://gitlab.gnome.org/GNOME/gtksourceview) - [Pango](https://gitlab.gnome.org/GNOME/pango) (>=1.50.0) -- [sqlite](https://www.sqlite.org/) (>=3.33.0) +- [sqlite](https://www.sqlite.org/) (>=3.35.0) - [omemo-dr](https://dev.gajim.org/gajim/omemo-dr) (>=1.0.0) - [qrcode](https://pypi.org/project/qrcode/) (>=7.3.1) - [winsdk](https://pypi.org/project/winsdk/) (Only on Windows) diff --git a/archive.dbml b/archive.dbml new file mode 100644 index 000000000..4ae59cde3 --- /dev/null +++ b/archive.dbml @@ -0,0 +1,243 @@ +Table account { + entitykey INTEGER [pk, increment] + jid TEXT +} + +Table jid { + entitykey INTEGER [pk, increment] + jid TEXT +} + +Table occupant { + entitykey INTEGER [pk, increment] + fk_account_ek INTEGER [not null] + fk_jid_ek INTEGER [not null, note: 'remote jid'] + timestamp REAL [not null] + id TEXT [not null] + fk_real_jid_ek TEXT + nickname TEXT + avatar_sha TEXT + + indexes { + (id, fk_jid_ek, fk_account_ek) [unique, name: 'idx_occupant'] + } + +} + +Ref: occupant.fk_account_ek > account.entitykey [delete: cascade] +Ref: occupant.fk_jid_ek > jid.entitykey +Ref: occupant.fk_real_jid_ek > jid.entitykey + +Table message { + entitykey INTEGER [pk, increment] + fk_account_ek INTEGER [not null] + fk_jid_ek INTEGER [not null, note: 'remote jid'] + resource TEXT + m_type INTEGER [not null] + direction INTEGER [not null] + timestamp REAL [not null] + message_id TEXT [not null] + stanza_id TEXT + stable_id INTEGER [not null] + fk_occupant_ek INTEGER + message TEXT [not null] + user_delay_ts REAL + fk_encryption_ek INTEGER + fk_securitylabel_ek INTEGER + + indexes { + (fk_account_ek, fk_jid_ek, timestamp) [name: 'idx_message', note: 'timestamp DESC'] + (message_id, fk_jid_ek, fk_account_ek, direction) [unique, name: 'idx_message_dedup'] + } + +} + +Ref: message.fk_account_ek > account.entitykey [delete: cascade] +Ref: message.fk_jid_ek > jid.entitykey +Ref: message.fk_occupant_ek > occupant.entitykey +Ref: message.fk_securitylabel_ek > securitylabel.entitykey +Ref: message.fk_encryption_ek > encryption.entitykey + +Table call { + entitykey INTEGER [pk] + sid TEXT +} + +Ref: call.entitykey - message.entitykey [delete: cascade] + +Table filetransfer { + entitykey INTEGER [pk] + sid TEXT +} + +Ref: filetransfer.entitykey - message.entitykey [delete: cascade] + +Table encryption { + entitykey INTEGER [pk, increment] + protocol INTEGER [not null] + key TEXT [not null] + trust INTEGER [not null] + + indexes { + (protocol, key, trust) [name: 'idx_encryption'] + } + +} + +Table oob { + entitykey INTEGER [pk] + url TEXT [not null] + description TEXT +} + +Ref: oob.entitykey - message.entitykey [delete: cascade] + +Table reply { + entitykey INTEGER [pk] + fallback_end INTEGER + quoted_jid TEXT [not null] + quoted_id TEXT [not null] +} + +Ref: reply.entitykey - message.entitykey [delete: cascade] + +Table securitylabel { + entitykey INTEGER [pk, increment] + fk_account_ek INTEGER [not null] + fk_jid_ek INTEGER [not null, note: 'remote jid'] + timestamp REAL [not null] + label_hash TEXT [not null] + displaymarking TEXT [not null] + fgcolor TEXT [not null] + bgcolor TEXT [not null] + + indexes { + (label_hash, fk_jid_ek, fk_account_ek) [unique, name: 'idx_securitylabel'] + } + +} + +Ref: securitylabel.fk_account_ek > account.entitykey [delete: cascade] +Ref: securitylabel.fk_jid_ek > jid.entitykey + + +Table error { + entitykey INTEGER [pk, increment] + fk_account_ek INTEGER [not null] + fk_jid_ek INTEGER [not null, note: 'remote jid'] + error_id TEXT + by TEXT + e_type TEXT [not null] + text TEXT + condition TEXT [not null] + condition_text TEXT + + indexes { + (error_id, fk_jid_ek, fk_account_ek) [unique, name: 'idx_error'] + } + +} + +Ref: error.fk_account_ek > account.entitykey [delete: cascade] +Ref: error.fk_jid_ek > jid.entitykey + +Table reaction { + entitykey INTEGER [pk, increment] + fk_account_ek INTEGER [not null] + fk_jid_ek INTEGER [not null, note: 'remote jid'] + direction INTEGER [not null] + timestamp REAL [not null] + fk_occupant_ek INTEGER [not null] + reaction_id TEXT + emojis TEXT [not null] + + indexes { + (reaction_id, fk_jid_ek, fk_account_ek, fk_occupant_ek) [unique, name: 'idx_reaction'] + } + +} + +Ref: reaction.fk_account_ek > account.entitykey [delete: cascade] +Ref: reaction.fk_jid_ek > jid.entitykey + +Table retraction { + entitykey INTEGER [pk, increment] + fk_account_ek INTEGER [not null] + fk_jid_ek INTEGER [not null, note: 'remote jid'] + direction INTEGER [not null] + timestamp REAL [not null] + fk_occupant_ek INTEGER [not null] + retraction_id TEXT [not null] + + indexes { + (retraction_id, fk_jid_ek, fk_account_ek, fk_occupant_ek, direction) [unique, name: 'idx_retraction'] + } + +} + +Ref: retraction.fk_account_ek > account.entitykey [delete: cascade] +Ref: retraction.fk_jid_ek > jid.entitykey + +Table moderation { + entitykey INTEGER [pk, increment] + fk_account_ek INTEGER [not null] + fk_jid_ek INTEGER [not null, note: 'remote jid'] + timestamp REAL [not null] + moderation_id TEXT [not null] + fk_occupant_ek INTEGER + by TEXT [not null] + reason TEXT + + indexes { + (moderation_id, fk_jid_ek, fk_account_ek) [unique, name: 'idx_moderation'] + } + +} + +Ref: moderation.fk_account_ek > account.entitykey [delete: cascade] +Ref: moderation.fk_jid_ek > jid.entitykey +Ref: moderation.fk_occupant_ek > occupant.entitykey + +Table correction { + entitykey INTEGER [pk, increment] + fk_account_ek INTEGER [not null] + fk_jid_ek INTEGER [not null, note: 'remote jid'] + resource TEXT + direction INTEGER [not null] + timestamp INTEGER [not null] + message_id TEXT + fk_occupant_ek INTEGER + correction_id TEXT [not null] + corrected_message TEXT [not null] + fk_encryption_ek INTEGER + + indexes { + (correction_id, fk_jid_ek, fk_account_ek, direction, fk_occupant_ek, timestamp) [name: 'idx_correction', note: 'timestamp DESC'] + (message_id, fk_jid_ek, fk_account_ek, direction) [unique, name: 'idx_correction_dedup'] + } + +} + +Ref: correction.fk_account_ek > account.entitykey [delete: cascade] +Ref: correction.fk_jid_ek > jid.entitykey +Ref: correction.fk_occupant_ek > occupant.entitykey +Ref: correction.fk_encryption_ek > encryption.entitykey + +Table marker { + entitykey INTEGER [pk, increment] + fk_account_ek INTEGER [not null] + fk_jid_ek INTEGER [not null, note: 'remote jid'] + fk_occupant_ek INTEGER [not null] + marker_id TEXT [not null] + received_ts INTEGER + displayed_ts INTEGER + acknowledged_ts INTEGER + + indexes { + (marker_id, fk_jid_ek, fk_account_ek, fk_occupant_ek) [unique, name: 'idx_marker'] + } + +} + +Ref: marker.fk_account_ek > account.entitykey [delete: cascade] +Ref: marker.fk_jid_ek > jid.entitykey diff --git a/debian/control b/debian/control index c349e7334..046775bbd 100644 --- a/debian/control +++ b/debian/control @@ -20,6 +20,7 @@ Build-Depends: python3-pil (>=9.1.0), gir1.2-gtk-3.0 (>=3.24.30), gir1.2-gtksource-4, + sqlite3 (>=3.35.0), Rules-Requires-Root: no Standards-Version: 4.1.4 Homepage: https://gajim.org/ @@ -40,6 +41,7 @@ Depends: gir1.2-pango-1.0 (>= 1.50.0), gir1.2-gtk-3.0 (>= 3.24.30), gir1.2-gtksource-4, + sqlite3 (>=3.35.0), Recommends: aspell-en | aspell-dictionary, ca-certificates, diff --git a/gajim/common/app.py b/gajim/common/app.py index d0c694d23..e287dbd06 100644 --- a/gajim/common/app.py +++ b/gajim/common/app.py @@ -30,6 +30,7 @@ import typing from typing import Any from typing import cast +import functools import gc import logging import os @@ -56,7 +57,7 @@ if typing.TYPE_CHECKING: from gajim.common.cert_store import CertificateStore from gajim.common.commands import ChatCommands # noqa: F401 from gajim.common.preview import PreviewManager - from gajim.common.storage.archive import MessageArchiveStorage + from gajim.common.storage.archive.storage import MessageArchiveStorage from gajim.common.storage.cache import CacheStorage from gajim.common.storage.draft import DraftStorage from gajim.common.storage.events import EventStorage @@ -467,6 +468,7 @@ def jid_is_transport(jid: str) -> bool: return False +functools.lru_cache(None) def get_jid_from_account(account_name: str) -> str: ''' Return the jid we use in the given account diff --git a/gajim/common/application.py b/gajim/common/application.py index b9a09d924..5e7b0cf14 100644 --- a/gajim/common/application.py +++ b/gajim/common/application.py @@ -52,7 +52,7 @@ from gajim.common.helpers import from_one_line from gajim.common.helpers import get_random_string from gajim.common.settings import LegacyConfig from gajim.common.settings import Settings -from gajim.common.storage.archive import MessageArchiveStorage +from gajim.common.storage.archive.storage import MessageArchiveStorage from gajim.common.storage.cache import CacheStorage from gajim.common.storage.draft import DraftStorage from gajim.common.storage.events import EventStorage @@ -421,11 +421,10 @@ class CoreApplication(ged.EventHelper): passwords.delete_password(account) app.settings.remove_account(account) app.app.remove_account_actions(account) + app.storage.archive.remove_all_from_account(account) def _on_signed_in(self, event: SignedIn) -> None: client = app.get_client(event.account) - app.storage.archive.insert_jid(client.get_own_jid().bare) - if client.get_module('MAM').available: client.get_module('MAM').request_archive_on_signin() diff --git a/gajim/common/call_manager.py b/gajim/common/call_manager.py index 96c3d73f3..50e46e259 100644 --- a/gajim/common/call_manager.py +++ b/gajim/common/call_manager.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging import time +import uuid from nbxmpp.namespaces import Namespace from nbxmpp.protocol import JID @@ -13,14 +14,17 @@ from gajim.common import sound from gajim.common import types from gajim.common.const import CallType from gajim.common.const import JingleState -from gajim.common.const import KindConstant from gajim.common.ged import EventHelper -from gajim.common.helpers import AdditionalDataDict from gajim.common.helpers import play_sound from gajim.common.i18n import _ from gajim.common.jingle_rtp import JingleAudio from gajim.common.jingle_session import JingleSession from gajim.common.modules.contacts import BareContact +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbInsertCallRowData +from gajim.common.storage.archive.structs import DbInsertMessageRowData log = logging.getLogger('gajim.c.call_manager') @@ -418,11 +422,22 @@ class CallManager(EventHelper): @staticmethod def _store_outgoing_call(account: str, jid: JID, sid: str) -> None: - additional_data = AdditionalDataDict() - additional_data.set_value('gajim', 'sid', sid) - app.storage.archive.insert_into_logs( - account, - jid, - time.time(), - KindConstant.CALL_OUTGOING, - additional_data=additional_data) + call_data = DbInsertCallRowData( + sid=sid, + state=0, # TODO + ) + + message_data = DbInsertMessageRowData( + account=account, + remote_jid=jid, + m_type=MessageType.CHAT, + direction=ChatDirection.OUTGOING, + timestamp=time.time(), + state=MessageState.ACKNOWLEDGED, + message_id=str(uuid.uuid4()), + ) + + app.storage.archive.insert_row( + message_data, + [call_data], + ) diff --git a/gajim/common/client.py b/gajim/common/client.py index bfef7a628..d104f3a43 100644 --- a/gajim/common/client.py +++ b/gajim/common/client.py @@ -504,21 +504,17 @@ class Client(Observable, ClientModules): message.set_sent_timestamp() message.message_id = self.send_stanza(message.stanza) - log_line_id = None - if not message.is_groupchat: - log_line_id = self.get_module('Message').log_message(message) + if message.message is None: + return + + entitykey = self.get_module('Message').store_outgoing_message(message) + if entitykey is None: + return app.ged.raise_event( MessageSent(jid=message.jid, account=message.account, - message=message.message, - chatstate=message.chatstate, - timestamp=message.timestamp, - additional_data=message.additional_data, - label=message.label, - correct_id=message.correct_id, - message_id=message.message_id, - msg_log_id=log_line_id, + entitykey=entitykey, play_sound=message.play_sound)) def connect( diff --git a/gajim/common/const.py b/gajim/common/const.py index dd1e6b3c1..4b33139c9 100644 --- a/gajim/common/const.py +++ b/gajim/common/const.py @@ -17,6 +17,8 @@ from __future__ import annotations from typing import Any from typing import NamedTuple +from datetime import datetime +from datetime import timezone from enum import Enum from enum import IntEnum from enum import unique @@ -83,9 +85,9 @@ class AvatarSize(IntEnum): PUBLISH = 200 -class ArchiveState(IntEnum): - NEVER = 0 - ALL = 1 +class ArchiveState: + NEVER = None + ALL = datetime.fromtimestamp(0, timezone.utc) @unique diff --git a/gajim/common/dbus/remote_control.py b/gajim/common/dbus/remote_control.py index d9bce8354..d564f126e 100644 --- a/gajim/common/dbus/remote_control.py +++ b/gajim/common/dbus/remote_control.py @@ -187,9 +187,6 @@ class GajimRemote(Server): app.ged.register_event_handler('presence-received', ged.POSTGUI, self._on_presence_received) - app.ged.register_event_handler('gc-message-received', - ged.POSTGUI, - self._on_gc_message_received) app.ged.register_event_handler('message-received', ged.POSTGUI, self._on_message_received) @@ -201,9 +198,10 @@ class GajimRemote(Server): self._on_message_sent) def _on_message_sent(self, event: events.MessageSent) -> None: + joined_data = event.joined_data self.raise_signal('MessageSent', ( event.account, [event.jid, - event.message])) + joined_data.message])) def _on_presence_received(self, event: events.PresenceReceived) -> None: self.raise_signal('ContactPresence', (event.account, [ @@ -212,28 +210,16 @@ class GajimRemote(Server): event.show, event.status])) - def _on_gc_message_received(self, event: events.GcMessageReceived) -> None: - self.raise_signal('GCMessage', ( - event.conn.name, [event.fjid, - event.msgtxt, - event.properties.timestamp, - event.delayed, - event.displaymarking])) - - def _on_message_received(self, - event: events.MessageReceived) -> None: - - event_type = event.properties.type.value - if event.properties.is_muc_pm: - event_type = 'pm' + def _on_message_received(self, event: events.MessageReceived) -> None: + joined_data = event.joined_data self.raise_signal('NewMessage', ( - event.conn.name, [event.fjid, - event.msgtxt, - event.properties.timestamp, - event_type, - event.properties.subject, - event.msg_log_id, - event.properties.nickname])) + event.account, [ + joined_data.remote_jid, + joined_data.resource, + joined_data.m_type, + joined_data.timestamp, + joined_data.message, + ])) def _on_our_status(self, event: events.ShowChanged) -> None: self.raise_signal('AccountPresence', (event.show, event.account)) diff --git a/gajim/common/events.py b/gajim/common/events.py index 68fd21220..c447d22f8 100644 --- a/gajim/common/events.py +++ b/gajim/common/events.py @@ -21,26 +21,26 @@ from typing import Union from collections.abc import Callable from dataclasses import dataclass from dataclasses import field +from functools import cached_property from nbxmpp.const import Affiliation from nbxmpp.const import InviteType from nbxmpp.const import Role from nbxmpp.const import StatusCode from nbxmpp.modules.security_labels import Catalog -from nbxmpp.modules.security_labels import Displaymarking -from nbxmpp.modules.security_labels import SecurityLabel from nbxmpp.protocol import JID from nbxmpp.structs import HTTPAuthData from nbxmpp.structs import LocationData -from nbxmpp.structs import ModerationData from nbxmpp.structs import RosterItem from nbxmpp.structs import TuneData +from gajim.common import app from gajim.common.const import EncryptionInfoMsg from gajim.common.const import JingleState -from gajim.common.const import KindConstant from gajim.common.file_props import FileProp -from gajim.common.helpers import AdditionalDataDict +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbConversationJoinedData +from gajim.common.storage.archive.structs import DbInsertModerationRowData if typing.TYPE_CHECKING: from gajim.common.client import Client @@ -48,12 +48,11 @@ if typing.TYPE_CHECKING: ChatListEventT = Union[ 'MessageReceived', - 'MamMessageReceived', - 'GcMessageReceived', - 'MessageUpdated', + 'MessageCorrected', 'MessageModerated', 'PresenceReceived', 'MessageSent', + 'MessageDeleted', 'JingleRequestReceived', 'FileRequestReceivedEvent' ] @@ -166,15 +165,12 @@ class MessageSent(ApplicationEvent): name: str = field(init=False, default='message-sent') account: str jid: JID - message: str - message_id: str - msg_log_id: int | None - chatstate: str | None - timestamp: float - additional_data: AdditionalDataDict - label: SecurityLabel | None - correct_id: str | None - play_sound: bool + entitykey: int + play_sound: bool = False + + @cached_property + def joined_data(self) -> DbConversationJoinedData: + return app.storage.archive.get_message_with_entitykey(self.entitykey) @dataclass @@ -188,6 +184,14 @@ class MessageNotSent(ApplicationEvent): @dataclass +class MessageDeleted(ApplicationEvent): + name: str = field(init=False, default='message-deleted') + account: str + jid: JID + entitykey: int + + +@dataclass class FileProgress(ApplicationEvent): name: str = field(init=False, default='file-progress') file_props: FileProp @@ -368,14 +372,15 @@ class ArchivingIntervalFinished(ApplicationEvent): @dataclass -class MessageUpdated(ApplicationEvent): - name: str = field(init=False, default='message-updated') +class MessageCorrected(ApplicationEvent): + name: str = field(init=False, default='message-corrected') account: str jid: JID - msgtxt: str - nickname: str | None - properties: Any - correct_id: str + entitykey: int + + @cached_property + def joined_data(self) -> DbConversationJoinedData: + return app.storage.archive.get_message_with_entitykey(self.entitykey) @dataclass @@ -383,52 +388,21 @@ class MessageModerated(ApplicationEvent): name: str = field(init=False, default='message-moderated') account: str jid: JID - moderation: ModerationData - - -@dataclass -class MamMessageReceived(ApplicationEvent): - name: str = field(init=False, default='mam-message-received') - account: str - jid: JID - msgtxt: str - properties: Any - additional_data: AdditionalDataDict - unique_id: str - stanza_id: str - archive_jid: str - kind: KindConstant - occupant_id: str | None - real_jid: JID | None - msg_log_id: int | None - displaymarking: Displaymarking | None + moderation: DbInsertModerationRowData @dataclass class MessageReceived(ApplicationEvent): name: str = field(init=False, default='message-received') - conn: 'Client' - stanza: Any account: str jid: JID - msgtxt: str - properties: Any - additional_data: AdditionalDataDict - unique_id: str - stanza_id: str - fjid: str - resource: str | None - delayed: float | None - msg_log_id: int | None - displaymarking: Displaymarking | None + m_type: MessageType + from_mam: bool + entitykey: int - -@dataclass -class GcMessageReceived(MessageReceived): - name: str = field(init=False, default='gc-message-received') - room_jid: str - real_jid: JID | None - occupant_id: str | None + @cached_property + def joined_data(self) -> DbConversationJoinedData: + return app.storage.archive.get_message_with_entitykey(self.entitykey) @dataclass @@ -436,7 +410,6 @@ class MessageError(ApplicationEvent): name: str = field(init=False, default='message-error') account: str jid: JID - room_jid: str message_id: str error: Any diff --git a/gajim/common/jingle_ft.py b/gajim/common/jingle_ft.py index a5ebb0009..ff72e06df 100644 --- a/gajim/common/jingle_ft.py +++ b/gajim/common/jingle_ft.py @@ -24,6 +24,7 @@ from typing import TYPE_CHECKING import logging import threading import time +import uuid from enum import IntEnum from enum import unique @@ -33,11 +34,9 @@ from nbxmpp.namespaces import Namespace from gajim.common import app from gajim.common import helpers -from gajim.common.const import KindConstant from gajim.common.events import FileRequestReceivedEvent from gajim.common.file_props import FileProp from gajim.common.file_props import FilesProp -from gajim.common.helpers import AdditionalDataDict from gajim.common.jingle_content import contents from gajim.common.jingle_content import JingleContent from gajim.common.jingle_ftstates import StateCandReceived @@ -48,6 +47,12 @@ from gajim.common.jingle_ftstates import StateTransfering from gajim.common.jingle_ftstates import StateTransportReplace from gajim.common.jingle_transport import JingleTransportSocks5 from gajim.common.jingle_transport import TransportType +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import FiletransferSourceType +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbInsertFiletransferRowData +from gajim.common.storage.archive.structs import DbInsertMessageRowData if TYPE_CHECKING: from gajim.common.jingle_session import JingleSession @@ -159,16 +164,27 @@ class JingleFileTransfer(JingleContent): stanza) jid = JID.from_string(jid) sid = stanza.getTag('jingle').getAttr('sid') - timestamp = time.time() - additional_data = AdditionalDataDict() - additional_data.set_value('gajim', 'type', 'jingle') - additional_data.set_value('gajim', 'sid', sid) - app.storage.archive.insert_into_logs( - account, - jid.bare, - timestamp, - KindConstant.FILE_TRANSFER_INCOMING, - additional_data=additional_data) + assert sid is not None + + message_data = DbInsertMessageRowData( + account=account, + remote_jid=jid.new_as_bare(), + m_type=MessageType.CHAT, + direction=ChatDirection.INCOMING, + timestamp=time.time(), + state=MessageState.ACKNOWLEDGED, + message_id=str(uuid.uuid4()), + ) + + message_ek = app.storage.archive.insert_row(message_data) + + filetransfer_data = DbInsertFiletransferRowData( + fk_message_ek=message_ek, + source_type=FiletransferSourceType.JINGLE, + source=sid, + state=1, + ) + app.storage.archive.insert_row(filetransfer_data) if self.session.request: # accept the request diff --git a/gajim/common/jingle_session.py b/gajim/common/jingle_session.py index 75d66ff97..81db50a7e 100644 --- a/gajim/common/jingle_session.py +++ b/gajim/common/jingle_session.py @@ -33,6 +33,7 @@ from typing import TYPE_CHECKING import logging import time +import uuid from collections.abc import Callable from enum import Enum from enum import unique @@ -46,15 +47,18 @@ from nbxmpp.util import generate_id from gajim.common import app from gajim.common import events from gajim.common.client import Client -from gajim.common.const import KindConstant from gajim.common.file_props import FilesProp -from gajim.common.helpers import AdditionalDataDict from gajim.common.jingle_content import get_jingle_content from gajim.common.jingle_content import JingleContent from gajim.common.jingle_content import JingleContentSetupException from gajim.common.jingle_ft import State from gajim.common.jingle_transport import get_jingle_transport from gajim.common.jingle_transport import JingleTransportIBB +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbInsertCallRowData +from gajim.common.storage.archive.structs import DbInsertMessageRowData if TYPE_CHECKING: from gajim.common.jingle_transport import JingleTransport @@ -699,15 +703,26 @@ class JingleSession: account = self.connection.name jid = JID.from_string(self.peerjid) - timestamp = time.time() - additional_data = AdditionalDataDict() - additional_data.set_value('gajim', 'sid', self.sid) - app.storage.archive.insert_into_logs( - account, - jid.bare, - timestamp, - KindConstant.CALL_INCOMING, - additional_data=additional_data) + + call_data = DbInsertCallRowData( + sid=self.sid, + state=0, # TODO + ) + + message_data = DbInsertMessageRowData( + account=account, + remote_jid=jid.new_as_bare(), + m_type=MessageType.CHAT, + direction=ChatDirection.INCOMING, + timestamp=time.time(), + state=MessageState.ACKNOWLEDGED, + message_id=str(uuid.uuid4()), + ) + + app.storage.archive.insert_row( + message_data, + [call_data], + ) def __broadcast(self, stanza: nbxmpp.Node, diff --git a/gajim/common/modules/chat_markers.py b/gajim/common/modules/chat_markers.py index a574df3d0..9d84225de 100644 --- a/gajim/common/modules/chat_markers.py +++ b/gajim/common/modules/chat_markers.py @@ -29,6 +29,7 @@ from gajim.common.events import ReadStateSync from gajim.common.modules.base import BaseModule from gajim.common.modules.contacts import BareContact from gajim.common.modules.contacts import GroupchatContact +from gajim.common.storage.archive.structs import DbUpsertMarkerRowData from gajim.common.structs import OutgoingMessage @@ -96,19 +97,23 @@ class ChatMarkers(BaseModule): properties.jid, properties.marker.id) - jid = properties.jid if not properties.is_muc_pm and not properties.type.is_groupchat: - jid = properties.jid.bare - - app.storage.archive.set_marker( - app.get_jid_from_account(self._account), - jid, - properties.marker.id, - 'displayed') + if properties.is_mam_message: + timestamp = properties.mam.timestamp + else: + timestamp = properties.timestamp + + marker_data = DbUpsertMarkerRowData( + account=self._account, + remote_jid=properties.remote_jid, + fk_occupant_ek=None, + marker_id=properties.marker.id, + displayed_ts=timestamp) + app.storage.archive.upsert_row(marker_data) app.ged.raise_event( DisplayedReceived(account=self._account, - jid=jid, + jid=properties.remote_jid, properties=properties, type=properties.type, is_muc_pm=properties.is_muc_pm, diff --git a/gajim/common/modules/mam.py b/gajim/common/modules/mam.py index c5df5f402..c7b9d241c 100644 --- a/gajim/common/modules/mam.py +++ b/gajim/common/modules/mam.py @@ -16,9 +16,6 @@ from __future__ import annotations -from typing import Any - -import time from datetime import datetime from datetime import timedelta from datetime import timezone @@ -42,19 +39,13 @@ from gajim.common import app from gajim.common import types from gajim.common.const import ArchiveState from gajim.common.const import ClientState -from gajim.common.const import KindConstant from gajim.common.const import SyncThreshold from gajim.common.events import ArchivingIntervalFinished from gajim.common.events import FeatureDiscovered -from gajim.common.events import MamMessageReceived from gajim.common.events import RawMamMessageReceived -from gajim.common.helpers import AdditionalDataDict -from gajim.common.helpers import get_retraction_text from gajim.common.modules.base import BaseModule -from gajim.common.modules.misc import parse_oob from gajim.common.modules.util import as_task -from gajim.common.modules.util import check_if_message_correction -from gajim.common.modules.util import get_eme_message +from gajim.common.storage.archive.structs import DbUpsertMamArchiveStateRowData class MAM(BaseModule): @@ -75,7 +66,7 @@ class MAM(BaseModule): priority=41), StanzaHandler(name='message', callback=self._mam_message_received, - priority=51), + priority=49), ] self.available = False @@ -133,25 +124,6 @@ class MAM(BaseModule): return properties.mam.archive.bare_match(expected_archive) - def _get_unique_id(self, - properties: MessageProperties - ) -> tuple[str | None, str | None]: - if properties.type.is_groupchat: - return properties.mam.id, None - - if properties.is_self_message: - return None, properties.id - - if properties.is_muc_pm: - return properties.mam.id, properties.id - - if self._con.get_own_jid().bare_match(properties.from_): - # message we sent - return properties.mam.id, properties.id - - # A message we received - return properties.mam.id, None - @staticmethod def _get_stanza_id(properties: MessageProperties, archive_jid: str @@ -202,16 +174,24 @@ class MAM(BaseModule): if not self.is_catch_up_finished(archive_jid): return - app.storage.archive.set_archive_infos( - archive_jid, - last_mam_id=stanza_id.id, - last_muc_timestamp=timestamp) + if timestamp is not None: + timestamp = datetime.fromtimestamp(timestamp, timezone.utc) + + app.storage.archive.upsert_row( + DbUpsertMamArchiveStateRowData( + account=self._account, + remote_jid=archive_jid, + to_stanza_id=stanza_id.id, + to_stanza_ts=timestamp, + ) + ) def _mam_message_received(self, _con: types.xmppClient, stanza: Message, properties: MessageProperties ) -> None: + if not properties.is_mam_message: return @@ -233,140 +213,6 @@ class MAM(BaseModule): self._log.debug(stanza) raise nbxmpp.NodeProcessed - is_groupchat = properties.type.is_groupchat - if is_groupchat: - kind = KindConstant.GC_MSG - else: - if properties.from_.bare_match(self._con.get_own_jid()): - kind = KindConstant.CHAT_MSG_SENT - else: - kind = KindConstant.CHAT_MSG_RECV - - stanza_id, message_id = self._get_unique_id(properties) - - # Search for duplicates - if app.storage.archive.find_stanza_id(self._account, - str(properties.mam.archive), - stanza_id, - message_id, - groupchat=is_groupchat): - self._log.info('Found duplicate with stanza-id: %s, ' - 'message-id: %s', stanza_id, message_id) - raise nbxmpp.NodeProcessed - - additional_data = AdditionalDataDict() - if properties.has_user_delay: - # Record it as a user timestamp - additional_data.set_value( - 'gajim', 'user_timestamp', properties.user_timestamp) - - parse_oob(properties, additional_data) - - msgtxt = properties.body - - if properties.is_encrypted: - additional_data['encrypted'] = properties.encrypted.additional_data - else: - if properties.eme is not None: - msgtxt = get_eme_message(properties.eme) - - if properties.is_moderation: - additional_data.set_value( - 'retracted', 'by', properties.moderation.moderator_jid) - additional_data.set_value( - 'retracted', 'timestamp', properties.moderation.timestamp) - additional_data.set_value( - 'retracted', 'reason', properties.moderation.reason) - - msgtxt = get_retraction_text( - properties.moderation.moderator_jid, - properties.moderation.reason) - - if not msgtxt: - # For example Chatstates, Receipts, Chatmarkers - self._log.debug(stanza.getProperties()) - return - - jid = properties.jid.new_as_bare() - if properties.is_muc_pm: - jid = properties.jid - - if properties.is_self_message: - # Self messages can only be deduped with origin-id - if message_id is None: - self._log.warning('Self message without origin-id found') - return - stanza_id = message_id - - occupant_id = self._get_occupant_id(properties) - real_jid = self._get_real_jid(properties) - - displaymarking = None - if properties.has_security_label: - displaymarking = properties.security_label.displaymarking - - event_attr: dict[str, Any] = { - 'account': self._account, - 'jid': jid, - 'msgtxt': properties.body, - 'properties': properties, - 'additional_data': additional_data, - 'unique_id': properties.id, - 'stanza_id': stanza_id, - 'archive_jid': properties.mam.archive, - 'kind': kind, - 'occupant_id': occupant_id, - 'real_jid': real_jid, - 'displaymarking': displaymarking - } - - if check_if_message_correction(properties, - self._account, - properties.jid, - properties.body, - kind, - properties.mam.timestamp, - self._log): - return - - event_attr['msg_log_id'] = app.storage.archive.insert_into_logs( - self._account, - jid, - properties.mam.timestamp, - kind, - message=msgtxt, - contact_name=properties.muc_nickname, - additional_data=additional_data, - stanza_id=stanza_id, - message_id=properties.id, - occupant_id=occupant_id, - real_jid=real_jid, - ) - - app.ged.raise_event(MamMessageReceived(**event_attr)) - - def _get_real_jid(self, properties: MessageProperties) -> JID | None: - if not properties.type.is_groupchat: - return None - - if properties.muc_user is None: - return None - - return properties.muc_user.jid - - def _get_occupant_id(self, properties: MessageProperties) -> str | None: - if not properties.type.is_groupchat: - return None - - if properties.occupant_id is None: - return None - - contact = self._client.get_module('Contacts').get_contact( - properties.jid.bare, groupchat=True) - if contact.supports(Namespace.OCCUPANT_ID): - return properties.occupant_id - return None - def _is_valid_request(self, properties: MessageProperties) -> bool: valid_id = self._mam_query_ids.get(properties.mam.archive, None) return valid_id == properties.mam.query_id @@ -378,11 +224,12 @@ class MAM(BaseModule): def _get_query_params(self) -> tuple[str | None, datetime | None]: own_jid = self._con.get_own_jid().bare - archive = app.storage.archive.get_archive_infos(own_jid) + archive = app.storage.archive.get_mam_archive_state( + self._account, own_jid) mam_id = None if archive is not None: - mam_id = archive.last_mam_id + mam_id = archive.to_stanza_id start_date = None if mam_id: @@ -401,12 +248,12 @@ class MAM(BaseModule): threshold: SyncThreshold ) -> tuple[str | None, datetime | None]: - archive = app.storage.archive.get_archive_infos(jid) + archive = app.storage.archive.get_mam_archive_state(self._account, jid) mam_id = None start_date = None now = datetime.now(timezone.utc) - if archive is None or archive.last_mam_id is None: + if archive is None or archive.to_stanza_id is None: # First join start_date = now - timedelta(days=1) self._log.info('Request archive: %s, after date %s', @@ -415,19 +262,18 @@ class MAM(BaseModule): elif threshold == SyncThreshold.NO_THRESHOLD: # Not our first join and no threshold set - mam_id = archive.last_mam_id + mam_id = archive.to_stanza_id self._log.info('Request archive: %s, after mam-id %s', - jid, archive.last_mam_id) + jid, archive.to_stanza_id) else: # Not our first join, check how much time elapsed since our # last join and check against threshold - last_timestamp = archive.last_muc_timestamp - if last_timestamp is None: + last = archive.to_stanza_ts + if last is None: self._log.info('No last muc timestamp found: %s', jid) - last_timestamp = 0 + last = datetime.fromtimestamp(0, timezone.utc) - last = datetime.fromtimestamp(float(last_timestamp), timezone.utc) if now - last > timedelta(days=threshold): # To much time has elapsed since last join, apply threshold start_date = now - timedelta(days=threshold) @@ -437,9 +283,9 @@ class MAM(BaseModule): else: # Request from last mam-id - mam_id = archive.last_mam_id + mam_id = archive.to_stanza_id self._log.info('Request archive: %s, after mam-id %s:', - jid, archive.last_mam_id) + jid, archive.to_stanza_id) return mam_id, start_date @@ -461,7 +307,8 @@ class MAM(BaseModule): self._log.warning(result) return - app.storage.archive.reset_archive_infos(result.jid) + app.storage.archive.reset_mam_archive_state( + self._account, result.jid) _, start_date = self._get_query_params() result = yield self._execute_query(result.jid, None, start_date) if is_error(result): @@ -472,18 +319,26 @@ class MAM(BaseModule): # <last> is not provided if the requested page was empty # so this means we did not get anything hence we only need # to update the archive info if <last> is present - app.storage.archive.set_archive_infos( - result.jid, - last_mam_id=result.rsm.last, - last_muc_timestamp=time.time()) + app.storage.archive.upsert_row( + DbUpsertMamArchiveStateRowData( + account=self._account, + remote_jid=result.jid, + to_stanza_id=result.rsm.last, + to_stanza_ts=datetime.now(timezone.utc), + ) + ) if start_date is not None: # Record the earliest timestamp we request from # the account archive. For the account archive we only # set start_date at the very first request. - app.storage.archive.set_archive_infos( - result.jid, - oldest_mam_timestamp=start_date.timestamp()) + app.storage.archive.upsert_row( + DbUpsertMamArchiveStateRowData( + account=self._account, + remote_jid=result.jid, + from_stanza_ts=start_date + ) + ) @as_task def request_archive_on_muc_join(self, jid: JID): @@ -509,7 +364,8 @@ class MAM(BaseModule): contact.notify('mam-sync-error', result.get_text()) return - app.storage.archive.reset_archive_infos(result.jid) + app.storage.archive.reset_mam_archive_state( + self._account, result.jid) _, start_date = self._get_muc_query_params(jid, threshold) result = yield self._execute_query(result.jid, None, start_date) if is_error(result): @@ -521,10 +377,14 @@ class MAM(BaseModule): # <last> is not provided if the requested page was empty # so this means we did not get anything hence we only need # to update the archive info if <last> is present - app.storage.archive.set_archive_infos( - result.jid, - last_mam_id=result.rsm.last, - last_muc_timestamp=time.time()) + app.storage.archive.upsert_row( + DbUpsertMamArchiveStateRowData( + account=self._account, + remote_jid=result.jid, + to_stanza_id=result.rsm.last, + to_stanza_ts=datetime.now(timezone.utc) + ) + ) contact.notify('mam-sync-finished') @@ -551,8 +411,14 @@ class MAM(BaseModule): raise_if_error(result) while not result.complete: - app.storage.archive.set_archive_infos(result.jid, - last_mam_id=result.rsm.last) + app.storage.archive.upsert_row( + DbUpsertMamArchiveStateRowData( + account=self._account, + remote_jid=result.jid, + to_stanza_id=result.rsm.last, + ) + ) + queryid = self._get_query_id(result.jid) result = yield self.make_query(result.jid, @@ -609,15 +475,21 @@ class MAM(BaseModule): self._remove_query_id(result.jid) if start_date: - timestamp = start_date.timestamp() + timestamp = start_date else: timestamp = ArchiveState.ALL if result.complete: self._log.info('Request finished: %s, last mam id: %s', result.jid, result.rsm.last) - app.storage.archive.set_archive_infos( - result.jid, oldest_mam_timestamp=timestamp) + app.storage.archive.upsert_row( + DbUpsertMamArchiveStateRowData( + account=self._account, + remote_jid=result.jid, + from_stanza_ts=timestamp, + ) + ) + app.ged.raise_event(ArchivingIntervalFinished( account=self._account, query_id=queryid)) diff --git a/gajim/common/modules/message.py b/gajim/common/modules/message.py index c94214f10..ebd3c9e65 100644 --- a/gajim/common/modules/message.py +++ b/gajim/common/modules/message.py @@ -16,8 +16,6 @@ from __future__ import annotations -from typing import Any - import time import nbxmpp @@ -29,18 +27,28 @@ from nbxmpp.util import generate_id from gajim.common import app from gajim.common import types -from gajim.common.const import KindConstant -from gajim.common.events import GcMessageReceived +from gajim.common.events import MessageCorrected +from gajim.common.events import MessageDeleted from gajim.common.events import MessageError from gajim.common.events import MessageReceived from gajim.common.events import RawMessageReceived -from gajim.common.helpers import AdditionalDataDict from gajim.common.modules.base import BaseModule from gajim.common.modules.contacts import GroupchatParticipant from gajim.common.modules.misc import parse_oob -from gajim.common.modules.misc import parse_xhtml -from gajim.common.modules.util import check_if_message_correction +from gajim.common.modules.util import get_chat_type_and_direction from gajim.common.modules.util import get_eme_message +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbInsertCorrectionRowData +from gajim.common.storage.archive.structs import DbInsertEncryptionRowData +from gajim.common.storage.archive.structs import DbInsertErrorsRowData +from gajim.common.storage.archive.structs import DbInsertMessageRowData +from gajim.common.storage.archive.structs import DbInsertOOBRowData +from gajim.common.storage.archive.structs import DbInsertRowDataBase +from gajim.common.storage.archive.structs import DbUpsertOccupantRowData +from gajim.common.storage.archive.structs import DbUpsertSecurityLabelRowData +from gajim.common.storage.base import VALUE_MISSING from gajim.common.structs import OutgoingMessage @@ -94,10 +102,11 @@ class Message(BaseModule): stanza: nbxmpp.Message, properties: MessageProperties ) -> None: - if (properties.is_mam_message or - properties.is_pubsub or + + if (properties.is_pubsub or properties.type.is_error): return + # Check if a child of the message contains any # namespaces that we handle in other modules. # nbxmpp executes less common handlers last @@ -117,217 +126,243 @@ class Message(BaseModule): # TODO: Check where in Gajim and plugins we depend on that behavior stanza.setFrom(stanza.getTo()) - from_ = stanza.getFrom() - fjid = str(from_) - jid = from_.bare - resource = from_.resource + m_type, direction = get_chat_type_and_direction( + self._client.get_own_jid(), properties) + timestamp = self._get_message_timestamp(properties) + remote_jid = properties.remote_jid + assert remote_jid is not None - type_ = properties.type + fk_occupant_ek = None + if m_type == MessageType.GROUPCHAT: + # Delete pending message when we receive the reflection + entitykey = app.storage.archive.delete_pending_message( + self._account, remote_jid, properties.id) - stanza_id, message_id = self._get_unique_id(properties) + if entitykey is not None: + app.ged.raise_event(MessageDeleted(account=self._account, + jid=remote_jid, + entitykey=entitykey)) - if (properties.is_self_message or properties.is_muc_pm): - archive_jid = self._con.get_own_jid().bare - if app.storage.archive.find_stanza_id( - self._account, - archive_jid, - stanza_id, - message_id, - properties.type.is_groupchat): - return + fk_occupant_ek = self._store_occupant_info( + remote_jid, timestamp, properties) - msgtxt = properties.body + stanza_id = self._get_stanza_id(properties) - additional_data = AdditionalDataDict() + message_text = properties.body - if properties.has_user_delay: - additional_data.set_value( - 'gajim', 'user_timestamp', properties.user_timestamp) + dependent_row_data: list[DbInsertRowDataBase] = [] - parse_oob(properties, additional_data) - parse_xhtml(properties, additional_data) + oob_data = parse_oob(properties) + if oob_data is not None: + dependent_row_data.append(oob_data) + encryption_ek = None if properties.is_encrypted: - additional_data['encrypted'] = properties.encrypted.additional_data - else: - if properties.eme is not None: - msgtxt = get_eme_message(properties.eme) + enc_data = properties.encrypted.additional_data + encryption_data = DbInsertEncryptionRowData( + protocol=enc_data['name'], + trust=enc_data['trust'], + key=enc_data.get('fingerprint'), + ) + encryption_ek = app.storage.archive.insert_row( + encryption_data, raise_on_conflict=False) + + elif properties.eme is not None: + message_text = get_eme_message(properties.eme) + + if not message_text: + return - displaymarking = None - if properties.has_security_label: - displaymarking = properties.security_label.displaymarking + if properties.correction: + correction_data = DbInsertCorrectionRowData( + account=self._account, + remote_jid=remote_jid, + resource=properties.jid.resource, + direction=direction, + timestamp=timestamp, + message_id=properties.id, + fk_occupant_ek=fk_occupant_ek, + correction_id=properties.correction.id, + corrected_message=message_text, + fk_encryption_ek=encryption_ek, + ) + + entitykey = app.storage.archive.insert_row( + correction_data, ignore_on_conflict=True) + if entitykey == -1: + # Duplicated correction + return - event_attr: dict[str, Any] = { - 'conn': self._con, - 'stanza': stanza, - 'account': self._account, - 'additional_data': additional_data, - 'fjid': fjid, - 'jid': from_ if properties.is_muc_pm else from_.new_as_bare(), - 'resource': resource, - 'stanza_id': stanza_id, - 'unique_id': stanza_id or message_id, - 'msgtxt': msgtxt, - 'delayed': properties.user_timestamp is not None, - 'msg_log_id': None, - 'displaymarking': displaymarking, - 'properties': properties, - } - - if type_.is_groupchat: - kind = KindConstant.GC_MSG - elif properties.is_sent_carbon: - kind = KindConstant.CHAT_MSG_SENT - else: - kind = KindConstant.CHAT_MSG_RECV - - if check_if_message_correction(properties, - self._account, - from_, - msgtxt, - kind, - properties.timestamp, - self._log): + message_ek = app.storage.archive.get_corrected_message_entitykey( + m_type, + correction_data) + if message_ek is None: + return + + event = MessageCorrected(account=self._account, + jid=remote_jid, + entitykey=message_ek) + app.ged.raise_event(event) return - if type_.is_groupchat: - if not msgtxt: - return + fk_securitylabel_ek = None + if properties.has_security_label: + assert properties.security_label is not None + displaymarking = properties.security_label.displaymarking + if displaymarking is not None: + securitylabel = DbUpsertSecurityLabelRowData( + account=self._account, + remote_jid=remote_jid, + timestamp=timestamp, + label_hash=properties.security_label.get_label_hash(), + displaymarking=displaymarking.name, + fgcolor=displaymarking.fgcolor, + bgcolor=displaymarking.bgcolor, + ) + fk_securitylabel_ek = app.storage.archive.upsert_row( + securitylabel) + + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=remote_jid, + m_type=m_type, + direction=direction, + timestamp=timestamp, + state=MessageState.ACKNOWLEDGED, + resource=properties.jid.resource, + message=message_text, + message_id=properties.id, + stanza_id=stanza_id, + stable_id=properties.origin_id is not None, + fk_occupant_ek=fk_occupant_ek, + user_delay_ts=properties.user_timestamp, + fk_securitylabel_ek=fk_securitylabel_ek, + fk_encryption_ek=encryption_ek, + ) + + entitykey = app.storage.archive.insert_row( + message_data, + dependent_row_data, + ignore_on_conflict=True + ) + if entitykey == -1: + # Duplicated message + return - occupant_id = None - group_contact = self._client.get_module('Contacts').get_contact( - jid, groupchat=True) - if group_contact.supports(Namespace.OCCUPANT_ID): - # Only store occupant-id if MUC announces support - occupant_id = properties.occupant_id + app.ged.raise_event(MessageReceived(account=self._account, + jid=remote_jid, + m_type=m_type, + from_mam=properties.is_mam_message, + entitykey=entitykey)) - real_jid = self._get_real_jid(properties) + def _get_message_timestamp(self, properties: MessageProperties) -> float: + if properties.is_mam_message: + return properties.mam.timestamp + return properties.timestamp - event_attr.update({ - 'room_jid': jid, - 'real_jid': real_jid, - 'occupant_id': occupant_id, - }) + def _get_real_jid(self, properties: MessageProperties) -> JID | None: + if properties.is_mam_message: + if properties.muc_user is None: + return None + return properties.muc_user.jid - event = GcMessageReceived(**event_attr) + if properties.jid.is_bare: + return None - msg_log_id = self._log_muc_message(event) - event.msg_log_id = msg_log_id - app.ged.raise_event(event) - return + occupant_contact = self._client.get_module('Contacts').get_contact( + properties.jid, groupchat=True) + assert isinstance(occupant_contact, GroupchatParticipant) + return occupant_contact.real_jid + + def _store_occupant_info( + self, + remote_jid: JID, + timestamp: float, + properties: MessageProperties + ) -> int | None: + + real_jid = self._get_real_jid(properties) + occupant_id = self._get_occupant_id(properties) or real_jid + if occupant_id is None: + return None - event = MessageReceived(**event_attr) - if not msgtxt: - app.ged.raise_event(event) - return + occupant_data = DbUpsertOccupantRowData( + account=self._account, + remote_jid=remote_jid, + id=str(occupant_id), + timestamp=timestamp, + nickname=properties.jid.resource or VALUE_MISSING, + real_jid=real_jid or VALUE_MISSING + ) + return app.storage.archive.upsert_row(occupant_data) + + def _get_occupant_id(self, properties: MessageProperties) -> str | None: + if not properties.type.is_groupchat: + return None - msg_log_id = app.storage.archive.insert_into_logs( - self._account, - fjid if properties.is_muc_pm else jid, - properties.timestamp, - kind, - message=msgtxt, - subject=properties.subject, - additional_data=additional_data, - stanza_id=stanza_id or message_id, - message_id=properties.id) + if properties.occupant_id is None: + return None - event.msg_log_id = msg_log_id - app.ged.raise_event(event) + contact = self._client.get_module('Contacts').get_contact( + properties.remote_jid, groupchat=True) + if contact.supports(Namespace.OCCUPANT_ID): + return properties.occupant_id + return None def _message_error_received(self, _con: types.xmppClient, _stanza: nbxmpp.Message, properties: MessageProperties ) -> None: - jid = properties.jid - if not properties.is_muc_pm: - jid = jid.new_as_bare() - self._log.info(properties.error) - - app.storage.archive.set_message_error( - app.get_jid_from_account(self._account), - jid, - properties.id, - properties.error) + remote_jid = properties.remote_jid + message_id = properties.id + + error_data = DbInsertErrorsRowData( + account=self._account, + remote_jid=remote_jid, + error_id=message_id, + by=properties.error.by, + e_type=properties.error.type, + text=properties.error.get_text() or None, + condition=properties.error.condition, + condition_text=properties.error.condition_data, + ) + app.storage.archive.insert_row(error_data) app.ged.raise_event( MessageError(account=self._account, - jid=jid, - room_jid=jid, - message_id=properties.id, + jid=remote_jid, + message_id=message_id, error=properties.error)) - def _log_muc_message(self, event: GcMessageReceived) -> int | None: - if not event.properties.muc_nickname: - return None - - if not event.msgtxt: - return None - - self._check_for_mam_compliance(event.room_jid, event.stanza_id) - - return app.storage.archive.insert_into_logs( - self._account, - event.jid, - event.properties.timestamp, - KindConstant.GC_MSG, - message=event.msgtxt, - contact_name=event.properties.muc_nickname, - additional_data=event.additional_data, - stanza_id=event.stanza_id, - message_id=event.properties.id, - occupant_id=event.occupant_id, - real_Jid=event.real_jid) - - def _check_for_mam_compliance(self, room_jid: str, stanza_id: str) -> None: - disco_info = app.storage.cache.get_last_disco_info(room_jid) - if stanza_id is None and disco_info.mam_namespace == Namespace.MAM_2: - self._log.warning('%s announces mam:2 without stanza-id', room_jid) - - def _get_real_jid(self, properties: MessageProperties) -> JID | None: - if not properties.type.is_groupchat: - return None - - if not properties.jid.is_full: - return None - - participant = self._client.get_module('Contacts').get_contact( - properties.jid, groupchat=True) - assert isinstance(participant, GroupchatParticipant) - return participant.real_jid - - def _get_unique_id(self, + def _get_stanza_id(self, properties: MessageProperties - ) -> tuple[str | None, str | None]: - if properties.is_self_message: - # Deduplicate self message with message-id - return None, properties.id + ) -> str | None: + + if properties.is_mam_message: + return properties.mam.id if not properties.stanza_ids: - return None, None + return None if properties.type.is_groupchat: - disco_info = app.storage.cache.get_last_disco_info( - properties.jid.bare) - - if disco_info.mam_namespace != Namespace.MAM_2: - return None, None + archive = properties.remote_jid + disco_info = app.storage.cache.get_last_disco_info(archive) + if not disco_info.supports(Namespace.SID): + return None - archive = properties.jid else: if not self._con.get_module('MAM').available: - return None, None + return None - archive = self._con.get_own_jid() + archive = self._con.get_own_jid().new_as_bare() for stanza_id in properties.stanza_ids: + # Check if message is from expected archive if archive.bare_match(stanza_id.by): - return stanza_id.id, None - - # stanza-id not added by the archive, ignore it. - return None, None + return stanza_id.id + return None def build_message_stanza(self, message: OutgoingMessage) -> nbxmpp.Message: own_jid = self._con.get_own_jid() @@ -335,8 +370,7 @@ class Message(BaseModule): stanza = nbxmpp.Message(to=message.jid, body=message.message, typ=message.type_, - subject=message.subject, - xhtml=message.xhtml) + subject=message.subject) if message.correct_id: stanza.setTag('replace', attrs={'id': message.correct_id}, @@ -403,32 +437,101 @@ class Message(BaseModule): return stanza - def log_message(self, message: OutgoingMessage) -> int | None: - if not message.is_loggable: - return None - - if message.message is None: - return None + def store_outgoing_message(self, message: OutgoingMessage) -> int | None: + direction = ChatDirection.OUTGOING + remote_jid = message.jid + message_text = message.message + assert message_text is not None + timestamp = message.timestamp + assert timestamp is not None + m_type = message.message_type + + state = MessageState.ACKNOWLEDGED + if m_type == MessageType.GROUPCHAT: + state = MessageState.PENDING + + encryption_ek = None + if message.is_encrypted: + enc_data = message.additional_data['encrypted'] + encryption_data = DbInsertEncryptionRowData( + protocol=enc_data['name'], + trust=enc_data['trust'], + key='Unknown', + ) + encryption_ek = app.storage.archive.insert_row( + encryption_data, raise_on_conflict=False) + + fk_securitylabel_ek = None + if message.label is not None: + displaymarking = message.label.displaymarking + if displaymarking is not None: + securitylabel = DbUpsertSecurityLabelRowData( + account=self._account, + remote_jid=remote_jid, + timestamp=timestamp, + label_hash=message.label.get_label_hash(), + displaymarking=displaymarking.name, + fgcolor=displaymarking.fgcolor, + bgcolor=displaymarking.bgcolor, + ) + fk_securitylabel_ek = app.storage.archive.upsert_row( + securitylabel) if message.correct_id is not None: - app.storage.archive.try_message_correction( - self._account, - message.jid, - None, - message.message, - message.correct_id, - KindConstant.CHAT_MSG_SENT, - message.timestamp) - return None + correction_data = DbInsertCorrectionRowData( + account=self._account, + remote_jid=remote_jid, + resource=None, + direction=direction, + timestamp=timestamp, + message_id=message.message_id, + fk_occupant_ek=None, + correction_id=message.correct_id, + corrected_message=message_text, + fk_encryption_ek=encryption_ek, + ) + app.storage.archive.insert_row(correction_data) + + message_ek = app.storage.archive.get_corrected_message_entitykey( + m_type, + correction_data) + if message_ek is None: + return + + event = MessageCorrected(account=self._account, + jid=remote_jid, + entitykey=message_ek) + app.ged.raise_event(event) + return + + dependent_row_data: list[DbInsertRowDataBase] = [] - msg_log_id = app.storage.archive.insert_into_logs( - self._account, - message.jid, - message.timestamp, - message.kind, - message=message.message, - subject=message.subject, - additional_data=message.additional_data, + if message.oob_url is not None: + dependent_row_data.append( + DbInsertOOBRowData(message.oob_url, None) + ) + + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=remote_jid, + m_type=m_type, + direction=direction, + timestamp=timestamp, + state=state, + resource=None, + message=message_text, message_id=message.message_id, - stanza_id=message.message_id) - return msg_log_id + stanza_id=None, + stable_id=True, + fk_occupant_ek=None, + user_delay_ts=None, + fk_encryption_ek=encryption_ek, + fk_securitylabel_ek=fk_securitylabel_ek, + ) + + entitykey = app.storage.archive.insert_row( + message_data, + dependent_row_data + ) + + return entitykey diff --git a/gajim/common/modules/misc.py b/gajim/common/modules/misc.py index 5993bbeb8..0600827e5 100644 --- a/gajim/common/modules/misc.py +++ b/gajim/common/modules/misc.py @@ -19,41 +19,19 @@ import logging from nbxmpp.structs import MessageProperties -from gajim.common.helpers import AdditionalDataDict -from gajim.common.i18n import get_rfc5646_lang +from gajim.common.storage.archive.structs import DbInsertOOBRowData log = logging.getLogger('gajim.c.m.misc') # XEP-0066: Out of Band Data -def parse_oob(properties: MessageProperties, - additional_data: AdditionalDataDict - ) -> None: +def parse_oob(properties: MessageProperties) -> DbInsertOOBRowData | None: if not properties.is_oob: return assert properties.oob is not None - additional_data.set_value('gajim', 'oob_url', properties.oob.url) - if properties.oob.desc is not None: - additional_data.set_value('gajim', 'oob_desc', - properties.oob.desc) - -# XEP-0308: Last Message Correction -def parse_correction(properties: MessageProperties) -> str | None: - if not properties.is_correction: - return None - assert properties.correction is not None - return properties.correction.id - - -# XEP-0071: XHTML-IM -def parse_xhtml(properties: MessageProperties, - additional_data: AdditionalDataDict - ) -> None: - if not properties.has_xhtml: - return - - assert properties.xhtml is not None - body = properties.xhtml.get_body(get_rfc5646_lang()) - additional_data.set_value('gajim', 'xhtml', body) + return DbInsertOOBRowData( + properties.oob.url, + properties.oob.desc + ) diff --git a/gajim/common/modules/moderations.py b/gajim/common/modules/moderations.py new file mode 100644 index 000000000..56cc24476 --- /dev/null +++ b/gajim/common/modules/moderations.py @@ -0,0 +1,222 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see <http://www.gnu.org/licenses/>. + +# XEP-0425: Message Moderation + +from __future__ import annotations + +import sqlite3 + +from nbxmpp import NodeProcessed +from nbxmpp.namespaces import Namespace +from nbxmpp.protocol import Message +from nbxmpp.structs import MessageProperties +from nbxmpp.structs import StanzaHandler + +from gajim.common import app +from gajim.common import types +from gajim.common.events import MessageModerated +from gajim.common.events import MessageReceived +from gajim.common.i18n import _ +from gajim.common.modules.base import BaseModule +from gajim.common.modules.util import get_chat_type_and_direction +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbInsertMessageRowData +from gajim.common.storage.archive.structs import DbInsertModerationRowData +from gajim.common.storage.archive.structs import DbUpsertOccupantRowData +from gajim.common.storage.base import VALUE_MISSING + +UNKNOWN_MESSAGE = _('Message content unknown') + +class Moderations(BaseModule): + def __init__(self, client: types.Client) -> None: + BaseModule.__init__(self, client) + + self.handlers = [ + StanzaHandler(name='message', + callback=self._process_fasten_message, + typ='groupchat', + ns=Namespace.FASTEN, + priority=48), + StanzaHandler(name='message', + callback=self._process_message_moderated_tombstone, + typ='groupchat', + ns=Namespace.MESSAGE_MODERATE, + priority=48), + ] + + def _process_message_moderated_tombstone( + self, + _client: types.xmppClient, + stanza: Message, + properties: MessageProperties + ) -> None: + + if not properties.is_moderation: + return + + if not properties.is_mam_message: + return + + assert properties.moderation is not None + + if not properties.moderation.is_tombstone: + return + + is_occupant_id_supported = self._is_occupant_id_supported(properties) + + self._insert_tombstone(properties, is_occupant_id_supported) + self._insert_moderation_message(properties, is_occupant_id_supported) + + raise NodeProcessed + + def _process_fasten_message( + self, + _client: types.xmppClient, + stanza: Message, + properties: MessageProperties + ) -> None: + + if not properties.is_moderation: + return + + assert properties.moderation is not None + + is_occupant_id_supported = self._is_occupant_id_supported(properties) + + self._insert_moderation_message(properties, is_occupant_id_supported) + + raise NodeProcessed + + def _insert_moderation_message( + self, + properties: MessageProperties, + is_occupant_id_supported: bool + ) -> None: + + assert properties.moderation is not None + + moderator_nickname = self._get_moderator_nickname(properties) + + moderator_occupant_id = None + if is_occupant_id_supported: + moderator_occupant_id = properties.moderation.occupant_id + + remote_jid = properties.remote_jid + assert remote_jid is not None + + fk_occupant_ek = None + if moderator_occupant_id is not None: + occupant_data = DbUpsertOccupantRowData( + account=self._account, + remote_jid=remote_jid, + timestamp=properties.moderation.stamp, + id=moderator_occupant_id, + nickname=moderator_nickname or VALUE_MISSING, + ) + fk_occupant_ek = app.storage.archive.upsert_row(occupant_data) + + moderation_data = DbInsertModerationRowData( + account=self._account, + remote_jid=remote_jid, + timestamp=properties.moderation.stamp, + moderation_id=properties.moderation.stanza_id, + by=moderator_nickname, + fk_occupant_ek=fk_occupant_ek, + reason=properties.moderation.reason + ) + + try: + app.storage.archive.insert_row(moderation_data) + except sqlite3.IntegrityError as error: + self._log.info('Failed to insert moderation request: %s', error) + return + + app.ged.raise_event( + MessageModerated( + account=self._account, + jid=remote_jid, + moderation=moderation_data)) + + def _insert_tombstone( + self, + properties: MessageProperties, + is_occupant_id_supported: bool + ) -> None: + + assert properties.mam is not None + + message_occupant_id = None + if is_occupant_id_supported: + message_occupant_id = properties.occupant_id + + remote_jid = properties.remote_jid + assert remote_jid is not None + assert properties.jid is not None + + fk_occupant_ek = None + if message_occupant_id is not None: + occupant_data = DbUpsertOccupantRowData( + account=self._account, + remote_jid=remote_jid, + timestamp=properties.mam.timestamp, + id=message_occupant_id, + nickname=properties.jid.resource or VALUE_MISSING, + ) + fk_occupant_ek = app.storage.archive.upsert_row(occupant_data) + + m_type, direction = get_chat_type_and_direction( + self._client.get_own_jid(), properties) + + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=remote_jid, + m_type=m_type, + direction=direction, + timestamp=properties.mam.timestamp, + state=MessageState.ACKNOWLEDGED, + resource=properties.jid.resource, + message=UNKNOWN_MESSAGE, + message_id=properties.id, + stanza_id=properties.mam.id, + stable_id=properties.origin_id is not None, + fk_occupant_ek=fk_occupant_ek, + ) + + entitykey = app.storage.archive.insert_row(message_data) + + app.ged.raise_event( + MessageReceived( + account=self._account, + jid=remote_jid, + m_type=MessageType.GROUPCHAT, + from_mam=True, + entitykey=entitykey)) + + def _is_occupant_id_supported(self, properties: MessageProperties) -> bool: + assert properties.remote_jid is not None + contact = self._client.get_module('Contacts').get_contact( + properties.remote_jid, groupchat=True) + return contact.supports(Namespace.OCCUPANT_ID) + + def _get_moderator_nickname( + self, + properties: MessageProperties + ) -> str | None: + + assert properties.moderation is not None + if properties.moderation.by is None: + return None + return properties.moderation.by.resource diff --git a/gajim/common/modules/muc.py b/gajim/common/modules/muc.py index 7c8ede649..3111f90d6 100644 --- a/gajim/common/modules/muc.py +++ b/gajim/common/modules/muc.py @@ -47,7 +47,6 @@ from gajim.common import helpers from gajim.common import types from gajim.common.const import ClientState from gajim.common.const import MUCJoinedState -from gajim.common.events import MessageModerated from gajim.common.events import MucAdded from gajim.common.events import MucDecline from gajim.common.events import MucInvitation @@ -57,6 +56,8 @@ from gajim.common.modules.base import BaseModule from gajim.common.modules.bits_of_binary import store_bob_data from gajim.common.modules.contacts import GroupchatContact from gajim.common.modules.contacts import GroupchatParticipant +from gajim.common.storage.archive.structs import DbUpsertOccupantRowData +from gajim.common.storage.base import VALUE_MISSING from gajim.common.structs import MUCData from gajim.common.structs import MUCPresenceData @@ -108,11 +109,6 @@ class MUC(BaseModule): typ='groupchat', priority=49), StanzaHandler(name='message', - callback=self._on_moderation, - ns=Namespace.FASTEN, - typ='groupchat', - priority=49), - StanzaHandler(name='message', callback=self._on_config_change, ns=Namespace.MUC_USER, priority=49), @@ -568,6 +564,8 @@ class MUC(BaseModule): occupant = self._get_contact(properties.jid, groupchat=True) room = self._get_contact(properties.jid.bare) + self._store_occupant_info(room, properties) + if properties.is_muc_destroyed: self._log.info('MUC destroyed: %s', room_jid) self._set_muc_state(room_jid, MUCJoinedState.NOT_JOINED) @@ -676,6 +674,27 @@ class MUC(BaseModule): self._process_occupant_presence_change(properties, presence, occupant) + def _store_occupant_info( + self, + room_contact: GroupchatContact, + properties: PresenceProperties + ) -> None: + + assert properties.muc_user is not None + real_jid = properties.muc_user.jid + + occupant_id = properties.occupant_id or real_jid + + occupant_data = DbUpsertOccupantRowData( + account=self._account, + remote_jid=room_contact.jid, + id=str(occupant_id), + timestamp=properties.timestamp, + nickname=properties.jid.resource or VALUE_MISSING, + real_jid=real_jid or VALUE_MISSING + ) + app.storage.archive.upsert_row(occupant_data) + def _process_occupant_presence_change( self, properties: PresenceProperties, @@ -847,24 +866,6 @@ class MUC(BaseModule): raise nbxmpp.NodeProcessed - def _on_moderation(self, - _con: types.xmppClient, - _stanza: Message, - properties: MessageProperties - ) -> None: - if not properties.is_moderation: - return - - app.storage.archive.update_additional_data( - self._account, properties.moderation.stanza_id, properties) - - app.ged.raise_event( - MessageModerated(account=self._account, - jid=properties.jid, - moderation=properties.moderation)) - - raise nbxmpp.NodeProcessed - def cancel_password_request(self, room_jid: JID) -> None: self._set_muc_state(room_jid, MUCJoinedState.NOT_JOINED) diff --git a/gajim/common/modules/omemo.py b/gajim/common/modules/omemo.py index 44d0b2ff5..68cff0fe6 100644 --- a/gajim/common/modules/omemo.py +++ b/gajim/common/modules/omemo.py @@ -288,7 +288,6 @@ class OMEMO(BaseModule): if event.is_groupchat: self._muc_temp_store[omemo_message.payload] = event.message else: - event.xhtml = None event.additional_data['encrypted'] = { 'name': 'OMEMO', 'trust': GajimTrust[OMEMOTrust.VERIFIED.name]} diff --git a/gajim/common/modules/receipts.py b/gajim/common/modules/receipts.py index b783b059a..7f1114cfc 100644 --- a/gajim/common/modules/receipts.py +++ b/gajim/common/modules/receipts.py @@ -28,6 +28,7 @@ from gajim.common import app from gajim.common import types from gajim.common.events import ReceiptReceived from gajim.common.modules.base import BaseModule +from gajim.common.storage.archive.structs import DbUpsertMarkerRowData class Receipts(BaseModule): @@ -46,6 +47,7 @@ class Receipts(BaseModule): stanza: Message, properties: MessageProperties ) -> None: + if not properties.is_receipt: return @@ -58,7 +60,6 @@ class Receipts(BaseModule): if (properties.type.is_groupchat or properties.is_self_message or - properties.is_mam_message or properties.is_carbon_message and properties.carbon.is_sent): if properties.receipt.is_received: @@ -66,7 +67,7 @@ class Receipts(BaseModule): raise nbxmpp.NodeProcessed return - if properties.receipt.is_request: + if properties.receipt.is_request and not properties.is_mam_message: if not app.settings.get_account_setting(self._account, 'answer_receipts'): return @@ -87,20 +88,23 @@ class Receipts(BaseModule): properties.jid, properties.receipt.id) - jid = properties.jid - if not properties.is_muc_pm: - jid = jid.new_as_bare() + if properties.is_mam_message: + timestamp = properties.mam.timestamp + else: + timestamp = properties.timestamp - app.storage.archive.set_marker( - app.get_jid_from_account(self._account), - jid, - properties.receipt.id, - 'received') + marker_data = DbUpsertMarkerRowData( + account=self._account, + remote_jid=properties.remote_jid, + fk_occupant_ek=None, + marker_id=properties.receipt.id, + received_ts=timestamp) + app.storage.archive.upsert_row(marker_data) app.ged.raise_event( ReceiptReceived( account=self._account, - jid=jid, + jid=properties.remote_jid, receipt_id=properties.receipt.id)) raise nbxmpp.NodeProcessed diff --git a/gajim/common/modules/util.py b/gajim/common/modules/util.py index b8a00bb27..557e9547e 100644 --- a/gajim/common/modules/util.py +++ b/gajim/common/modules/util.py @@ -18,14 +18,12 @@ from __future__ import annotations from typing import Any -import logging from collections.abc import Callable from functools import partial from functools import wraps from logging import LoggerAdapter import nbxmpp -from nbxmpp.const import MessageType from nbxmpp.namespaces import Namespace from nbxmpp.protocol import JID from nbxmpp.protocol import Message @@ -36,9 +34,8 @@ from nbxmpp.task import Task from gajim.common import app from gajim.common import types from gajim.common.const import EME_MESSAGES -from gajim.common.const import KindConstant -from gajim.common.events import MessageUpdated -from gajim.common.modules.misc import parse_correction +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageType def from_xs_boolean(value: str | bool) -> bool: @@ -131,62 +128,6 @@ def as_task(func: Any) -> Any: return func_wrapper -def check_if_message_correction(properties: MessageProperties, - account: str, - jid: JID, - msgtxt: str, - kind: KindConstant, - timestamp: float, - logger: LoggerAdapter[logging.Logger]) -> bool: - - correct_id = parse_correction(properties) - if correct_id is None: - return False - - if properties.type not in (MessageType.GROUPCHAT, MessageType.CHAT): - logger.warning('Ignore correction with message type: %s', - properties.type) - return False - - nickname = None - if properties.type.is_groupchat: - if jid.is_bare: - logger.warning( - 'Ignore correction from bare groupchat jid: %s', jid) - return False - - nickname = jid.resource - jid = jid.new_as_bare() - - elif not properties.is_muc_pm: - jid = jid.new_as_bare() - - successful = app.storage.archive.try_message_correction( - account, - jid, - nickname, - msgtxt, - correct_id, - kind, - timestamp) - - if not successful: - logger.info('Message correction not successful') - return False - - nickname = properties.muc_nickname or properties.nickname - - event = MessageUpdated(account=account, - jid=jid, - msgtxt=msgtxt, - nickname=nickname, - properties=properties, - correct_id=correct_id) - - app.ged.raise_event(event) - return True - - def prepare_stanza(stanza: Message, plaintext: str) -> None: delete_nodes(stanza, 'encrypted', Namespace.OMEMO_TEMP) delete_nodes(stanza, 'body') @@ -201,3 +142,25 @@ def delete_nodes(stanza: Message, nodes = stanza.getTags(name, namespace=namespace) for node in nodes: stanza.delChild(node) + + +def get_chat_type_and_direction( + own_jid: JID, properties: MessageProperties +) -> tuple[MessageType, ChatDirection]: + + if properties.type.is_groupchat: + return MessageType.GROUPCHAT, ChatDirection.INCOMING + + assert properties.from_ is not None + if properties.from_.bare_match(own_jid): + direction = ChatDirection.OUTGOING + else: + direction = ChatDirection.INCOMING + + if properties.is_muc_pm: + return MessageType.PM, direction + + if not properties.type.is_chat: + raise ValueError('Invalid message type', properties.type) + + return MessageType.CHAT, direction diff --git a/gajim/common/preview.py b/gajim/common/preview.py index 3d8db8430..c18553393 100644 --- a/gajim/common/preview.py +++ b/gajim/common/preview.py @@ -38,7 +38,6 @@ from gajim.common import app from gajim.common import configpaths from gajim.common import regex from gajim.common.const import MIME_TYPES -from gajim.common.helpers import AdditionalDataDict from gajim.common.helpers import get_tls_error_phrases from gajim.common.helpers import load_file_async from gajim.common.helpers import write_file_async @@ -52,6 +51,7 @@ from gajim.common.preview_helpers import guess_mime_type from gajim.common.preview_helpers import parse_fragment from gajim.common.preview_helpers import pixbuf_from_data from gajim.common.preview_helpers import split_geo_uri +from gajim.common.storage.archive.structs import DbOOBRowData from gajim.common.types import GdkPixbufType from gajim.common.util.http import create_http_request @@ -263,8 +263,8 @@ class PreviewManager: @staticmethod def _accept_uri(urlparts: ParseResult, uri: str, - additional_data: AdditionalDataDict) -> bool: - oob_url = additional_data.get_value('gajim', 'oob_url') + oob_url: str | None + ) -> bool: # geo if urlparts.scheme == 'geo': @@ -306,11 +306,9 @@ class PreviewManager: def is_previewable(self, text: str, - additional_data: AdditionalDataDict | None + oob_data: DbOOBRowData | None ) -> bool: - if additional_data is None: - return False if not IRI_RX.fullmatch(text): # urlparse removes whitespace (and who knows what else) from URLs, @@ -323,7 +321,8 @@ class PreviewManager: except Exception: return False - if not self._accept_uri(urlparts, uri, additional_data): + oob_url = None if oob_data is None else oob_data.url + if not self._accept_uri(urlparts, uri, oob_url): return False if urlparts.scheme == 'geo': diff --git a/gajim/common/storage/archive.py b/gajim/common/storage/archive.py deleted file mode 100644 index c6d583dc7..000000000 --- a/gajim/common/storage/archive.py +++ /dev/null @@ -1,1480 +0,0 @@ -# Copyright (C) 2003-2014 Yann Leboulanger <asterix AT lagaule.org> -# Copyright (C) 2004-2005 Vincent Hanquez <tab AT snarc.org> -# Copyright (C) 2005-2006 Nikos Kouremenos <kourem AT gmail.com> -# Copyright (C) 2006 Dimitur Kirov <dkirov AT gmail.com> -# Copyright (C) 2006-2008 Jean-Marie Traissard <jim AT lapin.org> -# Copyright (C) 2007 Tomasz Melcer <liori AT exroot.org> -# Julien Pivotto <roidelapluie AT gmail.com> -# Copyright (C) 2018 Philipp Hörist <philipp AT hoerist.com> -# -# This file is part of Gajim. -# -# Gajim is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published -# by the Free Software Foundation; version 3 only. -# -# Gajim is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Gajim. If not, see <http://www.gnu.org/licenses/>. - -from typing import Any -from typing import Literal -from typing import NamedTuple - -import calendar -import datetime -import json -import logging -import sqlite3 as sqlite -import time -from collections import namedtuple -from collections.abc import Iterator -from collections.abc import KeysView - -from nbxmpp import JID -from nbxmpp.structs import CommonError -from nbxmpp.structs import MessageProperties - -from gajim.common import app -from gajim.common import configpaths -from gajim.common.const import JIDConstant -from gajim.common.const import KindConstant -from gajim.common.const import MAX_MESSAGE_CORRECTION_DELAY -from gajim.common.helpers import AdditionalDataDict -from gajim.common.modules.contacts import GroupchatContact -from gajim.common.storage.base import SqliteStorage -from gajim.common.storage.base import timeit - -CURRENT_USER_VERSION = 7 - -ARCHIVE_SQL_STATEMENT = ''' - CREATE TABLE jids( - jid_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE, - jid TEXT UNIQUE, - type INTEGER - ); - CREATE TABLE logs( - log_line_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE, - account_id INTEGER, - jid_id INTEGER, - contact_name TEXT, - occupant_id TEXT, - real_jid TEXT, - time INTEGER, - kind INTEGER, - show INTEGER, - message TEXT, - error TEXT, - subject TEXT, - additional_data TEXT, - stanza_id TEXT, - message_id TEXT, - encryption TEXT, - encryption_state TEXT, - marker INTEGER - ); - CREATE TABLE last_archive_message( - jid_id INTEGER PRIMARY KEY UNIQUE, - last_mam_id TEXT, - oldest_mam_timestamp TEXT, - last_muc_timestamp TEXT - ); - CREATE INDEX idx_logs_jid_id_time ON logs (jid_id, time DESC); - CREATE INDEX idx_logs_stanza_id ON logs (stanza_id); - CREATE INDEX idx_logs_message_id ON logs (message_id); - PRAGMA user_version=%s; - ''' % CURRENT_USER_VERSION - -log = logging.getLogger('gajim.c.storage.archive') - - -class JidsTableRow(NamedTuple): - jid_id: int - jid: JID - type: JIDConstant - - -class ConversationRow(NamedTuple): - log_line_id: int - contact_name: str | None - occupant_id: str | None - real_jid: JID | None - time: float - kind: int - message: str - error: CommonError - additional_data: AdditionalDataDict | None - stanza_id: str - message_id: str - marker: str - - -class LastConversationRow(NamedTuple): - contact_name: str - time: float - kind: int - message: str | None - additional_data: AdditionalDataDict | None - message_id: str - stanza_id: str - - -class SearchLogRow(NamedTuple): - account_id: int - jid_id: int - jid: JID - contact_name: str - time: float - kind: int - show: int - message: str - subject: str - additional_data: AdditionalDataDict - log_line_id: int - - -class HistoryDayRow(NamedTuple): - day: int - - -class MessageMetaRow(NamedTuple): - log_line_id: int - time: float - - -class LastArchiveMessageRow(NamedTuple): - id: int - last_mam_id: str - oldest_mam_timestamp: str | None - last_muc_timestamp: str - - -class MessageExportRow(NamedTuple): - jid: str - contact_name: str - time: float - kind: int - message: str - - -class MessageArchiveStorage(SqliteStorage): - def __init__(self, in_memory: bool = False): - path = None if in_memory else configpaths.get('LOG_DB') - SqliteStorage.__init__(self, - log, - path, - ARCHIVE_SQL_STATEMENT) - - self._jid_ids: dict[JID, JidsTableRow] = {} - self._jid_ids_reversed: dict[int, JidsTableRow] = {} - - def init(self, **kwargs: Any) -> None: - SqliteStorage.init(self, - detect_types=sqlite.PARSE_COLNAMES) - - self._set_journal_mode('WAL') - self._enable_secure_delete() - - self._con.row_factory = self._namedtuple_factory - - self._con.create_function('like', 1, self._like) - - self._get_jid_ids_from_db() - - def _namedtuple_factory(self, - cursor: sqlite.Cursor, - row: tuple[Any, ...]) -> NamedTuple: - - assert cursor.description is not None - fields = [col[0] for col in cursor.description] - Row = namedtuple('Row', fields) # pyright: ignore - named_row = Row(*row) - if 'additional_data' in fields: - _dict = json.loads(named_row.additional_data or '{}') - named_row = named_row._replace( - additional_data=AdditionalDataDict(_dict)) - - # if an alias `account` for the field `account_id` is used for the - # query, the account_id is converted to the account jid - if 'account' in fields: - if named_row.account: - jid = self._jid_ids_reversed[named_row.account].jid - named_row = named_row._replace(account=jid) - return named_row - - def _migrate(self) -> None: - user_version = self.user_version - if user_version == 0: - # All migrations from 0.16.9 until 1.0.0 - statements = [ - 'ALTER TABLE logs ADD COLUMN "account_id" INTEGER', - 'ALTER TABLE logs ADD COLUMN "stanza_id" TEXT', - 'ALTER TABLE logs ADD COLUMN "encryption" TEXT', - 'ALTER TABLE logs ADD COLUMN "encryption_state" TEXT', - 'ALTER TABLE logs ADD COLUMN "marker" INTEGER', - 'ALTER TABLE logs ADD COLUMN "additional_data" TEXT', - '''CREATE TABLE IF NOT EXISTS last_archive_message( - jid_id INTEGER PRIMARY KEY UNIQUE, - last_mam_id TEXT, - oldest_mam_timestamp TEXT, - last_muc_timestamp TEXT - )''', - - '''CREATE INDEX IF NOT EXISTS idx_logs_stanza_id - ON logs(stanza_id)''', - 'PRAGMA user_version=1' - ] - - self._execute_multiple(statements) - - if user_version < 2: - statements = [ - ('ALTER TABLE last_archive_message ' - 'ADD COLUMN "sync_threshold" INTEGER'), - 'PRAGMA user_version=2' - ] - self._execute_multiple(statements) - - if user_version < 3: - statements = [ - 'ALTER TABLE logs ADD COLUMN "message_id" TEXT', - 'PRAGMA user_version=3' - ] - self._execute_multiple(statements) - - if user_version < 4: - statements = [ - 'ALTER TABLE logs ADD COLUMN "error" TEXT', - 'PRAGMA user_version=4' - ] - self._execute_multiple(statements) - - if user_version < 5: - statements = [ - '''CREATE INDEX IF NOT EXISTS idx_logs_message_id - ON logs (message_id)''', - 'PRAGMA user_version=5' - ] - self._execute_multiple(statements) - - if user_version < 7: - statements = [ - 'ALTER TABLE logs ADD COLUMN "real_jid" TEXT', - 'ALTER TABLE logs ADD COLUMN "occupant_id" TEXT', - 'PRAGMA user_version=7' - ] - self._execute_multiple(statements) - - @staticmethod - def _like(search_str: str) -> str: - return f'%{search_str}%' - - @timeit - def _get_jid_ids_from_db(self) -> None: - ''' - Load all jid/jid_id tuples into a dict for faster access - ''' - rows = self._con.execute( - 'SELECT jid_id, jid, type FROM jids').fetchall() - for row in rows: - self._jid_ids[row.jid] = row - self._jid_ids_reversed[row.jid_id] = row - - def get_jids_in_db(self) -> KeysView[JID]: - return self._jid_ids.keys() - - def get_account_id(self, - account: str, - type_: JIDConstant = JIDConstant.NORMAL_TYPE - ) -> int: - jid = app.get_jid_from_account(account) - return self.get_jid_id(jid, type_=type_) - - def get_active_account_ids(self) -> list[int]: - account_ids: list[int] = [] - for account in app.settings.get_active_accounts(): - account_ids.append(self.get_account_id(account)) - return account_ids - - @timeit - def get_jid_id(self, - jid: JID, - kind: KindConstant | None = None, - type_: JIDConstant | None = None - ) -> int: - ''' - Get the jid id from a jid. - In case the jid id is not found create a new one. - - :param jid: The JID - - :param kind: The KindConstant - - :param type_: The JIDConstant - - return the jid id - ''' - - if kind in (KindConstant.GC_MSG, KindConstant.GCSTATUS): - type_ = JIDConstant.ROOM_TYPE - elif kind is not None: - type_ = JIDConstant.NORMAL_TYPE - - result = self._jid_ids.get(jid, None) - if result is not None: - return result.jid_id - - sql = 'SELECT jid_id, jid, type FROM jids WHERE jid = ?' - row = self._con.execute(sql, [jid]).fetchone() - if row is not None: - self._jid_ids[jid] = row - return row.jid_id - - if type_ is None: - raise ValueError( - 'Unable to insert new JID because type is missing') - - sql = 'INSERT INTO jids (jid, type) VALUES (?, ?)' - lastrowid = self._con.execute(sql, (jid, type_)).lastrowid - assert lastrowid is not None - self._jid_ids[jid] = JidsTableRow(jid_id=lastrowid, - jid=jid, - type=type_) - self._delayed_commit() - return lastrowid - - @timeit - def get_conversation_before_after(self, - account: str, - jid: JID, - before: bool, - timestamp: float, - n_lines: int - ) -> list[ConversationRow]: - ''' - Load n_lines lines of conversation with jid before or after timestamp - - :param account: The account - - :param jid: The jid for which we request the conversation - - :param before: bool for direction (before or after timestamp) - - :param timestamp: timestamp - - returns a list of namedtuples - ''' - jids = [jid] - account_id = self.get_account_id(account) - kinds = map(str, [KindConstant.ERROR, - KindConstant.STATUS]) - - if before: - time_order = 'AND time < ? ORDER BY time DESC, log_line_id DESC' - else: - time_order = 'AND time > ? ORDER BY time ASC, log_line_id ASC' - - sql = ''' - SELECT - log_line_id, - contact_name, - occupant_id, - real_jid as "real_jid [jid]", - time, - kind, - message, - error as "error [common_error]", - additional_data, - stanza_id, - message_id, - marker as "marker [marker]" - FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) - AND account_id = {account_id} - AND kind NOT IN ({kinds}) - {time_order} - LIMIT ? - '''.format(jids=', '.join('?' * len(jids)), - account_id=account_id, - kinds=', '.join(kinds), - time_order=time_order) - - return self._con.execute( - sql, - tuple(jids) + (timestamp, n_lines)).fetchall() - - @timeit - def get_last_conversation_line(self, - account: str, - jid: JID - ) -> LastConversationRow | None: - ''' - Load the last line of a conversation with jid for account. - Loads messages, but no status messages or error messages. - - :param account: The account - - :param jid: The jid for which we request the conversation - - returns a list of namedtuples - ''' - jids = [jid] - account_id = self.get_account_id(account) - - kinds = map(str, [KindConstant.STATUS, - KindConstant.GCSTATUS, - KindConstant.ERROR]) - - sql = ''' - SELECT contact_name, time, kind, message, stanza_id, message_id, - additional_data - FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) - AND account_id = {account_id} - AND kind NOT IN ({kinds}) - ORDER BY time DESC - '''.format(jids=', '.join('?' * len(jids)), - account_id=account_id, - kinds=', '.join(kinds)) - - return self._con.execute(sql, tuple(jids)).fetchone() - - @timeit - def get_conversation_around(self, - account: str, - jid: JID, - timestamp: float - ) -> tuple[list[ConversationRow], - list[ConversationRow]]: - ''' - Load all lines of conversation with jid around a specific timestamp - - :param account: The account - - :param jid: The jid for which we request the conversation - - :param timestamp: Timestamp around which to fetch messages - - returns a list of namedtuples - ''' - jids = [jid] - account_id = self.get_account_id(account) - kinds = map(str, [KindConstant.ERROR]) - n_lines = 50 - - sql_before = ''' - SELECT - log_line_id, - contact_name, - occupant_id, - real_jid as "real_jid [jid]", - time, - kind, - message, - error as "error [common_error]", - additional_data, - stanza_id, - message_id, - marker as "marker [marker]" - FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) - AND account_id = {account_id} - AND kind NOT IN ({kinds}) - AND time < ? - ORDER BY time DESC, log_line_id DESC - LIMIT ? - '''.format(jids=', '.join('?' * len(jids)), - account_id=account_id, - kinds=', '.join(kinds)) - sql_at_after = ''' - SELECT - log_line_id, - contact_name, - occupant_id, - real_jid as "real_jid [jid]", - time, - kind, - message, - error as "error [common_error]", - additional_data, - stanza_id, - message_id, - marker as "marker [marker]" - FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) - AND account_id = {account_id} - AND kind NOT IN ({kinds}) - AND time >= ? - ORDER BY time ASC, log_line_id ASC - LIMIT ? - '''.format(jids=', '.join('?' * len(jids)), - account_id=account_id, - kinds=', '.join(kinds)) - before = self._con.execute( - sql_before, - tuple(jids) + (timestamp, n_lines)).fetchall() - at_after = self._con.execute( - sql_at_after, - tuple(jids) + (timestamp, n_lines)).fetchall() - return before, at_after - - @timeit - def get_conversation_between(self, - account: str, - jid: str, - before: float, - after: float) -> list[ConversationRow]: - ''' - Load all lines of conversation with jid between two timestamps - - :param account: The account - - :param jid: The jid for which we request the conversation - - :param before: latest timestamp - - :param after: earliest timestamp - - returns a list of namedtuples - ''' - jids = [jid] - account_id = self.get_account_id(account) - kinds = map(str, [KindConstant.ERROR]) - - sql = ''' - SELECT - log_line_id, - contact_name, - occupant_id, - real_jid as "real_jid [jid]", - time, - kind, - message, - error as "error [common_error]", - additional_data, - stanza_id, - message_id, - marker as "marker [marker]" - FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) - AND account_id = {account_id} - AND kind NOT IN ({kinds}) - AND time < ? AND time >= ? - ORDER BY time DESC, log_line_id DESC - '''.format(jids=', '.join('?' * len(jids)), - account_id=account_id, - kinds=', '.join(kinds)) - - return self._con.execute( - sql, - tuple(jids) + (before, after)).fetchall() - - @timeit - def search_log(self, - account: str, - jid: JID, - query: str, - from_users: list[str] | None = None, - before: datetime.datetime | None = None, - after: datetime.datetime | None = None - ) -> Iterator[SearchLogRow]: - ''' - Search the conversation log for messages containing the `query` string. - - :param account: The account - - :param jid: The jid for which we request the conversation - - :param query: A search string - - :param from_users: A list of usernames or None - - :param before: A datetime.datetime instance or None - - :param after: A datetime.datetime instance or None - - returns a list of namedtuples - ''' - account_id = self.get_account_id(account) - jids = [jid] - - kinds = map(str, [KindConstant.STATUS, - KindConstant.GCSTATUS]) - - if before is None: - before_ts = datetime.datetime.now().timestamp() - else: - before_ts = before.timestamp() - - after_ts = 0 - if after is not None: - after_ts = after.timestamp() - - if from_users is None: - users_query_string = '' - else: - users_query_string = 'AND UPPER(contact_name) IN (?)' - - sql = ''' - SELECT - account_id, - jid_id, - jid as "jid [jid]", - contact_name, - time, - kind, - show, - message, - subject, - additional_data, - log_line_id - FROM logs NATURAL JOIN jids - WHERE jid IN ({jids}) - AND account_id = {account_id} - AND message LIKE like(?) - AND kind NOT IN ({kinds}) - {users_query} - AND time BETWEEN ? AND ? - ORDER BY time DESC, log_line_id - '''.format(jids=', '.join('?' * len(jids)), - account_id=account_id, - kinds=', '.join(kinds), - users_query=users_query_string) - - if from_users is None: - - cursor = self._con.execute( - sql, tuple(jids) + (query, after_ts, before_ts)) - while True: - results = cursor.fetchmany(25) - if not results: - break - for result in results: - yield result - return - - users = ','.join([user.upper() for user in from_users]) - - cursor = self._con.execute( - sql, tuple(jids) + (query, users, after_ts, before_ts)) - while True: - results = cursor.fetchmany(25) - if not results: - break - for result in results: - yield result - - @timeit - def search_all_logs(self, - query: str, - from_users: list[str] | None = None, - before: datetime.datetime | None = None, - after: datetime.datetime | None = None - ) -> Iterator[SearchLogRow]: - ''' - Search all conversation logs for messages containing the `query` - string. - - :param query: A search string - - :param from_users: A list of usernames or None - - :param before: A datetime.datetime instance or None - - :param after: A datetime.datetime instance or None - - returns a list of namedtuples - ''' - account_ids = self.get_active_account_ids() - kinds = map(str, [KindConstant.STATUS, - KindConstant.GCSTATUS]) - - if before is None: - before_ts = datetime.datetime.now().timestamp() - else: - before_ts = before.timestamp() - - after_ts = 0 - if after is not None: - after_ts = after.timestamp() - - if from_users is None: - users_query_string = '' - else: - users_query_string = 'AND UPPER(contact_name) IN (?)' - - sql = ''' - SELECT - account_id, - jid_id, - jid as "jid [jid]", - contact_name, - time, - kind, - show, - message, - subject, - additional_data, - log_line_id - FROM logs NATURAL JOIN jids - WHERE message LIKE like(?) - AND account_id IN ({account_ids}) - AND kind NOT IN ({kinds}) - {users_query} - AND time BETWEEN ? AND ? - ORDER BY time DESC, log_line_id - '''.format(account_ids=', '.join(map(str, account_ids)), - kinds=', '.join(kinds), - users_query=users_query_string) - - if from_users is None: - cursor = self._con.execute(sql, (query, after_ts, before_ts)) - while True: - results = cursor.fetchmany(25) - if not results: - break - for result in results: - yield result - return - - users = ','.join([user.upper() for user in from_users]) - cursor = self._con.execute(sql, (query, users, after_ts, before_ts)) - while True: - results = cursor.fetchmany(25) - if not results: - break - for result in results: - yield result - - @timeit - def get_days_with_history(self, - account: str, - jid: str, - year: int, - month: int - ) -> list[HistoryDayRow]: - ''' - Get days in month where messages for 'jid' exist - ''' - jids = [jid] - account_id = self.get_account_id(account) - kinds = map(str, [KindConstant.STATUS, - KindConstant.GCSTATUS]) - - # Calculate the start and end datetime of the month - date = datetime.datetime(year, month, 1) - days = calendar.monthrange(year, month)[1] - 1 - delta = datetime.timedelta( - days=days, hours=23, minutes=59, seconds=59, microseconds=999999) - - sql = ''' - SELECT DISTINCT - CAST(strftime('%d', time, 'unixepoch', 'localtime') AS INTEGER) - AS day FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) - AND account_id = {account_id} - AND time BETWEEN ? AND ? - AND kind NOT IN ({kinds}) - ORDER BY time - '''.format(jids=', '.join('?' * len(jids)), - account_id=account_id, - kinds=', '.join(kinds)) - - return self._con.execute( - sql, - tuple(jids) + (date.timestamp(), - (date + delta).timestamp())).fetchall() - - @timeit - def get_last_history_timestamp(self, - account: str, - jid: str - ) -> float | None: - ''' - Get the timestamp of the last message we received for the jid - ''' - jids = [jid] - account_id = self.get_account_id(account) - kinds = map(str, [KindConstant.STATUS, - KindConstant.GCSTATUS]) - - sql = ''' - SELECT MAX(time) as time FROM logs - NATURAL JOIN jids WHERE jid IN ({jids}) - AND account_id = {account_id} - AND kind NOT IN ({kinds}) - '''.format(account_id=account_id, - jids=', '.join('?' * len(jids)), - kinds=', '.join(kinds)) - - # fetchone() returns always at least one Row with all - # attributes set to None because of the MAX() function - return self._con.execute(sql, tuple(jids)).fetchone().time - - @timeit - def get_first_history_timestamp(self, - account: str, - jid: str - ) -> float | None: - ''' - Get the timestamp of the first message we received for the jid - ''' - jids = [jid] - account_id = self.get_account_id(account) - kinds = map(str, [KindConstant.STATUS, - KindConstant.GCSTATUS]) - - sql = ''' - SELECT MIN(time) as time FROM logs - NATURAL JOIN jids WHERE jid IN ({jids}) - AND account_id = {account_id} - AND kind NOT IN ({kinds}) - '''.format(jids=', '.join('?' * len(jids)), - account_id=account_id, - kinds=', '.join(kinds)) - - # fetchone() returns always at least one Row with all - # attributes set to None because of the MIN() function - return self._con.execute(sql, tuple(jids)).fetchone().time - - @timeit - def date_has_history(self, - account: str, - jid: str, - date: datetime.datetime - ) -> float | None: - ''' - Get a single timestamp of a message for 'jid' - in time range of one day - ''' - jids = [jid] - account_id = self.get_account_id(account) - delta = datetime.timedelta( - hours=23, minutes=59, seconds=59, microseconds=999999) - - start = date.timestamp() - end = (date + delta).timestamp() - - sql = ''' - SELECT time - FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) - AND account_id = {account_id} - AND time BETWEEN ? AND ? - '''.format(jids=', '.join('?' * len(jids)), - account_id=account_id) - - return self._con.execute( - sql, tuple(jids) + (start, end)).fetchone() - - @timeit - def get_first_message_meta_for_date(self, - account: str, - jid: str, - date: datetime.datetime - ) -> MessageMetaRow | None: - ''' - Load meta data for the first message of a specific date - ''' - jids = [jid] - account_id = self.get_account_id(account) - - delta = datetime.timedelta( - hours=23, minutes=59, seconds=59, microseconds=999999) - date_ts = date.timestamp() - delta_ts = (date + delta).timestamp() - - sql = ''' - SELECT time, log_line_id - FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) - AND account_id = {account_id} - AND time BETWEEN ? AND ? - ORDER BY time ASC, log_line_id ASC - '''.format(jids=', '.join('?' * len(jids)), - account_id=account_id) - - return self._con.execute( - sql, - tuple(jids) + (date_ts, delta_ts)).fetchone() - - @timeit - def get_recent_muc_nicks(self, contact: GroupchatContact) -> list[str]: - ''' - Queries the last 50 message rows and gathers nicknames in a list - ''' - jids = [contact.jid] - account_id = self.get_account_id(contact.account) - kinds = map(str, [KindConstant.GC_MSG]) - - sql = ''' - SELECT contact_name - FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) - AND account_id = {account_id} - AND kind IN ({kinds}) - ORDER BY time DESC - '''.format(jids=', '.join('?' * len(jids)), - account_id=account_id, - kinds=', '.join(kinds)) - - - result = self._con.execute(sql, (tuple(jids))).fetchmany(50) - - nicknames: list[str] = [] - for row in result: - if (row.contact_name not in nicknames and - row.contact_name is not None): - nicknames.append(row.contact_name) - - return nicknames - - @timeit - def find_stanza_id(self, - account: str, - archive_jid: str, - stanza_id: str | None, - origin_id: str | None = None, - groupchat: bool = False - ) -> bool: - ''' - Checks if a stanza-id is already in the `logs` table - - :param account: The account - - :param archive_jid: The jid of the archive the stanza-id belongs to - only used if groupchat=True - - :param stanza_id: The stanza-id - - :param origin_id: The origin-id - - :param groupchat: stanza-id is from a groupchat - - return True if the stanza-id was found - ''' - ids: list[str] = [] - if stanza_id is not None: - ids.append(stanza_id) - if origin_id is not None: - ids.append(origin_id) - - if not ids: - return False - - type_ = JIDConstant.NORMAL_TYPE - if groupchat: - type_ = JIDConstant.ROOM_TYPE - - archive_id = self.get_jid_id(archive_jid, type_=type_) - account_id = self.get_account_id(account) - - if groupchat: - # Stanza ID is only unique within a specific archive. - # So a Stanza ID could be repeated in different MUCs, so we - # filter also for the archive JID which is the bare MUC jid. - - # Use Unary-"+" operator for "jid_id", otherwise the - # idx_logs_jid_id_time index is used instead of the much better - # idx_logs_stanza_id index - sql = ''' - SELECT stanza_id FROM logs - WHERE stanza_id IN ({values}) - AND +jid_id = ? AND account_id = ? LIMIT 1 - '''.format(values=', '.join('?' * len(ids))) - result = self._con.execute( - sql, tuple(ids) + (archive_id, account_id)).fetchone() - else: - sql = ''' - SELECT stanza_id FROM logs - WHERE stanza_id IN ({values}) AND account_id = ? AND - kind != ? LIMIT 1 - '''.format(values=', '.join('?' * len(ids))) - result = self._con.execute( - sql, tuple(ids) + (account_id, KindConstant.GC_MSG)).fetchone() - - if result is not None: - log.info('Found duplicated message, stanza-id: %s, origin-id: %s, ' - 'archive-jid: %s, account: %s', stanza_id, origin_id, - archive_jid, account_id) - return True - return False - - @timeit - def get_last_correctable_message(self, - account: str, - jid: JID, - message_id: str - ) -> LastConversationRow | None: - ''' - Load the last correctable message of a conversation by message_id. - Conditions: max 5 min old - ''' - jids = [jid] - account_id = self.get_account_id(account) - min_time = time.time() - MAX_MESSAGE_CORRECTION_DELAY - - sql = ''' - SELECT contact_name, time, kind, message, stanza_id, message_id, - additional_data - FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) - AND account_id = {account_id} - AND message_id = ? - AND time > ? - '''.format(jids=', '.join('?' * len(jids)), - account_id=account_id) - - return self._con.execute( - sql, - tuple(jids) + (message_id, min_time)).fetchone() - - @timeit - def try_message_correction(self, - account: str, - jid: JID, - nickname: str | None, - corrected_text: str, - correct_id: str, - kind: KindConstant, - timestamp: float) -> bool: - - '''Try to correct a message - - :param jid: This can be a full jid or bare jid. A full jid only if the - message is a MUC PM, otherwise a bare jid needs to be - passed. `nickname` should only be passed if the message - is a group chat message. - ''' - - account_id = self.get_account_id(account) - max_timestamp = timestamp - MAX_MESSAGE_CORRECTION_DELAY - - self._log.debug( - 'Check if message is correctable, parameters: %s %s %s %s %s', - jid, account_id, nickname, correct_id, max_timestamp) - - sql = '''SELECT log_line_id, message, additional_data - FROM logs - NATURAL JOIN jids jid_id - WHERE +jid = ? - AND account_id = ? - AND contact_name IS ? - AND message_id = ? - AND kind = ? - AND time > ? - ''' - - rows = self._con.execute(sql, (jid, - account_id, - nickname, - correct_id, - kind, - max_timestamp - )).fetchall() - - if not rows: - self._log.debug('No correctable messages found') - return False - - if len(rows) != 1: - self._log.warning('More than one correctable message found') - return False - - row = rows[0] - - if row.additional_data is None: - additional_data = AdditionalDataDict() - else: - additional_data = row.additional_data - - original_text = additional_data.get_value( - 'corrected', 'original_text') - if original_text is None: - # Only set original_text for the first correction - additional_data.set_value( - 'corrected', 'original_text', row.message) - serialized_dict = json.dumps(additional_data.data) - - sql = ''' - UPDATE logs SET message = ?, additional_data = ? - WHERE log_line_id = ? - ''' - self._con.execute( - sql, (corrected_text, serialized_dict, row.log_line_id)) - - return True - - @timeit - def update_additional_data(self, - account: str, - stanza_id: str, - properties: MessageProperties) -> None: - is_groupchat = properties.type.is_groupchat - type_ = JIDConstant.NORMAL_TYPE - if is_groupchat: - type_ = JIDConstant.ROOM_TYPE - - assert properties.jid is not None - archive_id = self.get_jid_id(properties.jid.bare, type_=type_) - account_id = self.get_account_id(account) - - if is_groupchat: - # Stanza ID is only unique within a specific archive. - # So a Stanza ID could be repeated in different MUCs, so we - # filter also for the archive JID which is the bare MUC jid. - - # Use Unary-"+" operator for "jid_id", otherwise the - # idx_logs_jid_id_time index is used instead of the much better - # idx_logs_stanza_id index - sql = ''' - SELECT additional_data FROM logs - WHERE stanza_id = ? - AND +jid_id = ? - AND account_id = ? - LIMIT 1 - ''' - result = self._con.execute( - sql, (stanza_id, archive_id, account_id)).fetchone() - else: - sql = ''' - SELECT additional_data FROM logs - WHERE stanza_id = ? - AND account_id = ? - AND kind != ? - LIMIT 1 - ''' - result = self._con.execute( - sql, (stanza_id, account_id, KindConstant.GC_MSG)).fetchone() - - if result is None: - return - - if result.additional_data is None: - additional_data = AdditionalDataDict() - else: - additional_data = result.additional_data - - if properties.is_moderation: - assert properties.moderation is not None - additional_data.set_value( - 'retracted', 'by', properties.moderation.moderator_jid) - additional_data.set_value( - 'retracted', 'timestamp', properties.moderation.timestamp) - additional_data.set_value( - 'retracted', 'reason', properties.moderation.reason) - serialized_dict = json.dumps(additional_data.data) - - if is_groupchat: - sql = ''' - UPDATE logs SET additional_data = ? - WHERE stanza_id = ? - AND account_id = ? - AND +jid_id = ? - ''' - self._con.execute( - sql, (serialized_dict, stanza_id, account_id, archive_id)) - else: - sql = ''' - UPDATE logs SET additional_data = ? - WHERE stanza_id = ? - AND account_id = ? - AND kind != ? - ''' - self._con.execute( - sql, (serialized_dict, stanza_id, account_id, - KindConstant.GC_MSG)) - - def insert_jid(self, - jid: str, - kind: KindConstant | None = None, - type_: JIDConstant = JIDConstant.NORMAL_TYPE - ) -> int: - ''' - Insert a new jid into the `jids` table. - This is an alias of get_jid_id() for better readablility. - - :param jid: The jid as string - - :param kind: A KindConstant - - :param type_: A JIDConstant - ''' - return self.get_jid_id(jid, kind, type_) - - @timeit - def insert_into_logs(self, - account: str, - jid: str, - time_: float, - kind: KindConstant, - **kwargs: Any - ) -> int: - ''' - Insert a new message into the `logs` table - - :param jid: The jid as string - - :param time_: The timestamp in UTC epoch - - :param kind: A KindConstant - - :param unread: If True the message is added to the`unread_messages` - table. Only if kind == CHAT_MSG_RECV - - :param kwargs: Every additional named argument must correspond to - a field in the `logs` table - ''' - jid_id = self.get_jid_id(jid, kind=kind) - account_id = self.get_account_id(account) - - if 'additional_data' in kwargs: - if not kwargs['additional_data']: - del kwargs['additional_data'] - else: - serialized_dict = json.dumps(kwargs['additional_data'].data) - kwargs['additional_data'] = serialized_dict - - sql = ''' - INSERT INTO logs (account_id, jid_id, time, kind, {columns}) - VALUES (?, ?, ?, ?, {values}) - '''.format(columns=', '.join(kwargs.keys()), - values=', '.join('?' * len(kwargs))) - - lastrowid = self._con.execute( - sql, (account_id, - jid_id, time_, - kind) + tuple(kwargs.values())).lastrowid - assert lastrowid is not None - - log.info('Insert into DB: jid: %s, time: %s, kind: %s, stanza_id: %s', - jid, time_, kind, kwargs.get('stanza_id', None)) - - self._delayed_commit() - - return lastrowid - - @timeit - def delete_message_from_logs(self, log_line_id: int) -> None: - ''' - Delete a message from the `logs` table - - :param log_line_id: The message's log_line_id - ''' - sql = 'DELETE FROM logs WHERE log_line_id = ?' - self._con.execute(sql, (log_line_id, )) - - self._delayed_commit() - log.info('Deleted message with log_line_id %s', log_line_id) - - @timeit - def set_message_error(self, - account_jid: str, - jid: JID, - message_id: str, - error: str - ) -> None: - ''' - Update the corresponding message with the error - - :param account_jid: The jid of the account - - :param jid: The jid that belongs to the avatar - - :param message_id: The id of the message - - :param error: The error stanza as string - - ''' - - account_id = self.get_jid_id(account_jid) - try: - jid_id = self.get_jid_id(str(jid)) - except ValueError: - # Unknown JID - return - - sql = ''' - UPDATE logs SET error = ? - WHERE account_id = ? AND jid_id = ? AND message_id = ? - ''' - self._con.execute(sql, (error, account_id, jid_id, message_id)) - self._delayed_commit() - - @timeit - def set_marker(self, - account_jid: str, - jid: str, - message_id: str, - state: Literal['received', 'displayed'] - ) -> None: - ''' - Update the marker state of the corresponding message - ''' - - if state not in ('received', 'displayed'): - raise ValueError('Invalid marker state') - - account_id = self.get_jid_id(account_jid) - try: - jid_id = self.get_jid_id(str(jid)) - except ValueError: - # Unknown JID - return - - state_int = 0 if state == 'received' else 1 - - sql = ''' - UPDATE logs SET marker = ? - WHERE account_id = ? AND jid_id = ? AND message_id = ? - ''' - self._con.execute(sql, (state_int, account_id, jid_id, message_id)) - self._delayed_commit() - - @timeit - def get_archive_infos(self, jid: str) -> LastArchiveMessageRow | None: - ''' - Get the archive infos - - :param jid: The jid that belongs to the avatar - - ''' - jid_id = self.get_jid_id(jid, type_=JIDConstant.ROOM_TYPE) - sql = '''SELECT * FROM last_archive_message WHERE jid_id = ?''' - return self._con.execute(sql, (jid_id,)).fetchone() - - @timeit - def set_archive_infos(self, jid: str, **kwargs: Any) -> None: - ''' - Set archive infos - - :param jid: The jid that belongs to the avatar - - :param last_mam_id: The last MAM result id - - :param oldest_mam_timestamp: The oldest date we requested MAM - history for - - :param last_muc_timestamp: The timestamp of the last message we - received in a MUC - - :param sync_threshold: The max days that we request from a - MUC archive - - ''' - jid_id = self.get_jid_id(jid) - exists = self.get_archive_infos(jid) - if not exists: - sql = '''INSERT INTO last_archive_message - (jid_id, last_mam_id, oldest_mam_timestamp, - last_muc_timestamp) - VALUES (?, ?, ?, ?)''' - self._con.execute(sql, ( - jid_id, - kwargs.get('last_mam_id', None), - kwargs.get('oldest_mam_timestamp', None), - kwargs.get('last_muc_timestamp', None), - )) - else: - for key, value in list(kwargs.items()): - if value is None: - del kwargs[key] - - args = ' = ?, '.join(kwargs.keys()) + ' = ?' - sql = '''UPDATE last_archive_message SET {} - WHERE jid_id = ?'''.format(args) # noqa: UP032 - self._con.execute(sql, tuple(kwargs.values()) + (jid_id,)) - log.info('Set message archive info: %s %s', jid, kwargs) - self._delayed_commit() - - @timeit - def reset_archive_infos(self, jid: str) -> None: - ''' - Set archive infos - - :param jid: The jid of the archive - - ''' - jid_id = self.get_jid_id(jid) - sql = '''UPDATE last_archive_message - SET last_mam_id = NULL, oldest_mam_timestamp = NULL, - last_muc_timestamp = NULL - WHERE jid_id = ?''' - self._con.execute(sql, (jid_id,)) - log.info('Reset message archive info: %s', jid) - self._delayed_commit() - - def get_conversation_jids(self, account: str) -> list[JID]: - account_id = self.get_account_id(account) - sql = '''SELECT DISTINCT jid as "jid [jid]" - FROM logs - NATURAL JOIN jids jid_id - WHERE account_id = ?''' - rows = self._con.execute(sql, (account_id, )).fetchall() - return [row.jid for row in rows] - - def get_messages_for_export(self, - account: str, - jid: JID - ) -> Iterator[MessageExportRow]: - - kinds = map(str, [KindConstant.CHAT_MSG_RECV, - KindConstant.SINGLE_MSG_SENT, - KindConstant.CHAT_MSG_SENT, - KindConstant.GC_MSG]) - - account_id = self.get_account_id(account) - sql = '''SELECT jid, time, kind, message, contact_name - FROM logs - NATURAL JOIN jids jid_id - WHERE account_id = ? AND kind in ({kinds}) AND jid = ? - ORDER BY time'''.format(kinds=', '.join(kinds)) - - cursor = self._con.execute(sql, (account_id, jid)) - while True: - results = cursor.fetchmany(10) - if not results: - break - yield from results - - def remove_history(self, account: str, jid: JID) -> None: - ''' - Remove history for a specific chat. - If it's a group chat, remove last MAM ID as well. - ''' - account_id = self.get_account_id(account) - try: - jid_id = self.get_jid_id(jid) - except ValueError: - log.info('No history entries for: %s', jid) - return - sql = 'DELETE FROM logs WHERE account_id = ? AND jid_id = ?' - self._con.execute(sql, (account_id, jid_id)) - - self._delayed_commit() - log.info('Removed history for: %s', jid) - - def remove_all_history(self) -> None: - ''' - Remove all messages for all accounts - ''' - statements = [ - 'DELETE FROM logs', - 'DELETE FROM jids', - 'DELETE FROM last_archive_message' - ] - self._execute_multiple(statements) - log.info('Removed all chat history') - - def cleanup_chat_history(self) -> None: - ''' - Remove messages from account where messages are older than max_age - ''' - for account in app.settings.get_accounts(): - max_age = app.settings.get_account_setting( - account, 'chat_history_max_age') - if max_age == -1: - continue - account_id = self.get_account_id(account) - now = time.time() - point_in_time = now - int(max_age) - - sql = 'DELETE FROM logs WHERE account_id = ? AND time < ?' - - cursor = self._con.execute(sql, (account_id, point_in_time)) - self._delayed_commit() - log.info('Removed %s old messages for %s', cursor.rowcount, account) diff --git a/gajim/common/storage/archive/__init__.py b/gajim/common/storage/archive/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/gajim/common/storage/archive/__init__.py diff --git a/gajim/common/storage/archive/const.py b/gajim/common/storage/archive/const.py new file mode 100644 index 000000000..73021c4f5 --- /dev/null +++ b/gajim/common/storage/archive/const.py @@ -0,0 +1,36 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see <http://www.gnu.org/licenses/>. + +from enum import IntEnum + + +class MessageState(IntEnum): + PENDING = 1 + ACKNOWLEDGED = 2 + + +class MessageType(IntEnum): + CHAT = 1 + GROUPCHAT = 2 + PM = 3 + + +class ChatDirection(IntEnum): + INCOMING = 1 + OUTGOING = 2 + + +class FiletransferSourceType(IntEnum): + JINGLE = 1 + URL = 2 diff --git a/gajim/common/storage/archive/migration_v8.py b/gajim/common/storage/archive/migration_v8.py new file mode 100644 index 000000000..f5e080e37 --- /dev/null +++ b/gajim/common/storage/archive/migration_v8.py @@ -0,0 +1,439 @@ +from typing import Any + +import json +import sqlite3 + +from nbxmpp.structs import CommonError + +from gajim.common.const import Trust +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType + + +class MigrationV8: + def __init__(self, connection: sqlite3.Connection) -> None: + self._con = connection + self._account_eks: dict[str, int] = {} + self._jid_eks: dict[str, int] = {} + self._encryption_eks: dict[Any, int] = {} + + def _get_type_and_direction(self, kind: int) -> tuple[int, int]: + match kind: + case 2: + return MessageType.GROUPCHAT, ChatDirection.INCOMING + case 4: + return MessageType.CHAT, ChatDirection.INCOMING + case 6: + return MessageType.CHAT, ChatDirection.OUTGOING + case _: + raise ValueError('Unknown kind: %s' % kind) + + def _get_account_ek(self, row: Any) -> int: + account_ek = self._account_eks.get(row.account_jid) + if account_ek is not None: + return account_ek + + account_ek = self._con.execute( + 'INSERT INTO account(jid) VALUES(?)', row.account_jid + ).lastrowid + assert account_ek is not None + self._account_eks[row.account_jid] = account_ek + return account_ek + + def _get_jid_ek(self, row: Any) -> int: + jid_ek = self._jid_eks.get(row.remote_jid) + if jid_ek is not None: + return jid_ek + + jid_ek = self._con.execute( + 'INSERT INTO jid(jid) VALUES(?)', row.remote_jid + ).lastrowid + assert jid_ek is not None + self._jid_eks[row.account_jid] = jid_ek + return jid_ek + + def _get_message_data(self, row: Any) -> dict[str, Any] | None: + timestamp = row.time + if timestamp is None: + return None + + message = row.message + if message is None: + return None + + account_ek = self._get_account_ek(row) + jid_ek = self._get_jid_ek(row) + m_type, direction = self._get_type_and_direction(row.kind) + data = { + 'account_ek': account_ek, + 'jid_ek': jid_ek, + 'm_type': m_type, + 'direction': direction, + 'stanza_id': row.stanza_id, + 'message_id': row.message_id, + 'message': message, + 'marker': row.marker, + 'error': row.error, + 'timestamp': timestamp, + 'resource': row.contact_name, + 'state': MessageState.ACKNOWLEDGED, + 'additional_data': json.loads(row.additional_data or '{}'), + } + return data + + def _migrate_correction( + self, + message_data: dict[str, Any], + encryption_ek: int | None, + ) -> str | None: + additional_data = message_data['additional_data'] + if additional_data is None: + return + + corrected = additional_data.get('corrected') + if corrected is None: + return None + + original_text = corrected.get('original_text') + + if message_data['message_id'] is None: + return + + corrected_message = message_data['message'] + + stmt = ''' + INSERT INTO correction( + fk_account_ek, + fk_jid_ek, + resource, + direction, + timestamp, + correction_id, + corrected_message, + fk_encryption_ek + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ''' + + self._con.execute( + stmt, + ( + message_data['account_ek'], + message_data['jid_ek'], + message_data['resource'], + message_data['direction'], + message_data['timestamp'] + 0.1, + message_data['message_id'], + corrected_message, + encryption_ek, + ), + ) + + return original_text + + def _migrate_message( + self, + message_data: dict[str, Any], + original_text: str | None, + encryption_ek: int | None, + ) -> int: + user_timestamp = None + additional_data = message_data['additional_data'] + if additional_data is not None: + gajim_data = additional_data.get('gajim') + if gajim_data is not None: + user_timestamp = gajim_data.get('user_timestamp') + + message = message_data['message'] + if original_text is not None: + message = original_text + + stmt = ''' + INSERT INTO message( + fk_account_ek, + fk_jid_ek, + resource, + m_type, + direction, + timestamp, + state, + message_id, + stanza_id, + message, + user_delay_ts, + fk_encryption_ek + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ''' + + entitykey = self._con.execute( + stmt, + ( + message_data['account_ek'], + message_data['jid_ek'], + message_data['resource'], + message_data['m_type'], + message_data['direction'], + message_data['timestamp'], + message_data['state'], + message_data['message_id'], + message_data['stanza_id'], + message, + user_timestamp, + encryption_ek, + ), + ).lastrowid + + assert entitykey is not None + return entitykey + + def _migrate_oob( + self, + entitykey: int, + message_data: dict[str, Any], + ) -> None: + additional_data = message_data['additional_data'] + if additional_data is None: + return + + gajim_data = additional_data.get('gajim') + if gajim_data is None: + return None + + url = gajim_data.get('oob_url') + description = gajim_data.get('oob_desc') + + if url is None: + return None + + self._con.execute( + 'INSERT INTO oob(entitykey, url, description) VALUES (?, ?, ?)', + (entitykey, url, description), + ) + + def _migrate_errors( + self, + message_data: dict[str, Any], + ) -> None: + error_node_serialized = message_data['error'] + if error_node_serialized is None: + return None + + message_id = message_data['message_id'] + if message_id is None: + return None + + try: + error = CommonError.from_string(error_node_serialized) + except Exception: + return None + + by = error.by + e_type = error.type # pyright: ignore + text = error.get_text() or None # pyright: ignore + condition = error.condition # pyright: ignore + condition_text = error.condition_data or None # pyright: ignore + + if e_type is None or condition is None: + return None + + stmt = ''' + INSERT INTO error( + fk_account_ek, + fk_jid_ek, + message_id, + by, + e_type, + text, + condition, + condition_text + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ''' + + try: + self._con.execute( + stmt, + ( + message_data['account_ek'], + message_data['jid_ek'], + message_id, + by, + e_type, + text, + condition, + condition_text, + ), # pyright: ignore + ) + except sqlite3.IntegrityError: + pass + + def _migrate_moderations( + self, + message_data: dict[str, Any], + ) -> None: + additional_data = message_data['additional_data'] + if additional_data is None: + return None + + stanza_id = message_data['stanza_id'] + if stanza_id is None: + return None + + retracted = additional_data.get('retracted') + if retracted is None: + return None + + by = retracted.get('by') + timestamp = retracted.get('timestamp') + reason = retracted.get('reason') + + if timestamp is None: + timestamp = message_data['timestamp'] + + stmt = ''' + INSERT INTO moderation( + fk_account_ek, + fk_jid_ek, + timestamp, + stanza_id, + by, + reason) + VALUES (?, ?, ?, ?, ?, ?) + ''' + + try: + self._con.execute( + stmt, + ( + message_data['account_ek'], + message_data['jid_ek'], + timestamp, + stanza_id, + by, + reason, + ), + ) + except sqlite3.IntegrityError: + pass + + def _migrate_markers( + self, + message_data: dict[str, Any], + ) -> None: + marker = message_data['marker'] + if marker is None: + return + + message_id = message_data['message_id'] + if message_id is None: + return + + received_ts = 0 + displayed_ts = 0 if marker == 1 else None + + stmt = ''' + INSERT INTO marker( + fk_account_ek, + fk_jid_ek, + marker_id, + received_ts, + displayed_ts + ) VALUES (?, ?, ?, ?, ?) + ''' + + try: + self._con.execute( + stmt, + ( + message_data['account_ek'], + message_data['jid_ek'], + message_id, + received_ts, + displayed_ts, + ), + ) + except sqlite3.IntegrityError: + pass + + def _migrate_encryption(self, message_data: dict[str, Any]) -> int | None: + additional_data = message_data['additional_data'] + if additional_data is None: + return None + + encrypted = additional_data.get('encrypted') + if encrypted is None: + return None + + protocol = encrypted.get('name') + key = encrypted.get('fingerprint') + trust = encrypted.get('trust') + + if protocol not in ('OpenPGP', 'OMEMO', 'PGP'): + return None + + if key is None: + key = 'Unknown' + + if trust is None: + trust = Trust.VERIFIED + + else: + try: + trust = Trust(trust) + except Exception: + trust = Trust.UNTRUSTED + + encryption_ek = self._encryption_eks.get((protocol, key, trust)) + if encryption_ek is not None: + return encryption_ek + + stmt = 'INSERT INTO encryption(protocol, key, trust) VALUES (?, ?, ?)' + + encryption_ek = self._con.execute( + stmt, (protocol, key, trust)).lastrowid + assert encryption_ek is not None + return encryption_ek + + def run(self) -> None: + stmt = ''' + SELECT + account.jid as account_jid, + jids.jid as remote_jid, + account_id, + contact_name, + time, + kind, + message, + error, + additional_data, + stanza_id, + message_id, + marker + FROM logs + LEFT OUTER JOIN jids AS account ON logs.account_id = account.jid_id + LEFT OUTER JOIN jids ON jids.jid_id = logs.jid_id + WHERE kind IN (2, 4, 6) + ''' + + rows = self._con.execute(stmt).fetchall() + for row in rows: + message_data = self._get_message_data(row) + if message_data is None: + continue + + encryption_ek = self._migrate_encryption(message_data) + original_text = self._migrate_correction( + message_data, + encryption_ek, + ) + + message_entity_key = self._migrate_message( + message_data, + original_text, + encryption_ek, + ) + + assert message_entity_key is not None + self._migrate_oob(message_entity_key, message_data) + self._migrate_errors(message_data) + self._migrate_moderations(message_data) + self._migrate_markers(message_data) diff --git a/gajim/common/storage/archive/statements.py b/gajim/common/storage/archive/statements.py new file mode 100644 index 000000000..3e1d4698c --- /dev/null +++ b/gajim/common/storage/archive/statements.py @@ -0,0 +1,619 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see <http://www.gnu.org/licenses/>. + + +ACCOUNT_COLUMNS = ( + 'jid', +) + +JID_COLUMNS = ( + 'jid', +) + +OCCUPANT_COLUMNS = ( + 'fk_account_ek', + 'fk_jid_ek', + 'timestamp', + 'id', + 'fk_real_jid_ek', + 'nickname', + 'avatar_sha', +) + +MESSAGE_COLUMNS = ( + 'fk_account_ek', + 'fk_jid_ek', + 'resource', + 'm_type', + 'direction', + 'timestamp', + 'state', + 'message_id', + 'stanza_id', + 'stable_id', + 'fk_occupant_ek', + 'message', + 'user_delay_ts', + 'fk_securitylabel_ek', + 'fk_encryption_ek', +) + +CALL_COLUMNS = ( + 'entitykey', + 'sid', + 'duration', + 'state', +) + +FILETRANSFER_COLUMNS = ( + 'fk_message_ek', + 'source_type', + 'source', + 'date', + 'desc', + 'height', + 'width', + 'length', + 'size', + 'name', + 'media_type', + 'thumb_path', + 'path', + 'state', +) + +ENCRYPTION_COLUMNS = ( + 'protocol', + 'key', + 'trust', +) + +OOB_COLUMNS = ( + 'entitykey', + 'url', + 'description', +) + +REPLY_COLUMNS = ( + 'entitykey', + 'fallback_end', + 'quoted_jid', + 'quoted_id', +) + +ERROR_COLUMNS = ( + 'fk_account_ek', + 'fk_jid_ek', + 'error_id', + 'by', + 'e_type', + 'text', + 'condition', + 'condition_text', +) + +SECURITYLABEL_COLUMNS = ( + 'fk_account_ek', + 'fk_jid_ek', + 'timestamp', + 'label_hash', + 'displaymarking', + 'fgcolor', + 'bgcolor', +) + +REACTION_COLUMNS = ( + 'fk_account_ek', + 'fk_jid_ek', + 'direction', + 'timestamp', + 'fk_occupant_ek', + 'reaction_id', + 'emojis', +) + +RETRACTION_COLUMNS = ( + 'fk_account_ek', + 'fk_jid_ek', + 'direction', + 'timestamp', + 'fk_occupant_ek', + 'retraction_id', +) + +MODERATION_COLUMNS = ( + 'fk_account_ek', + 'fk_jid_ek', + 'timestamp', + 'moderation_id', + 'fk_occupant_ek', + 'by', + 'reason', +) + +CORRECTION_COLUMNS = ( + 'fk_account_ek', + 'fk_jid_ek', + 'resource', + 'direction', + 'timestamp', + 'message_id', + 'fk_occupant_ek', + 'correction_id', + 'corrected_message', + 'fk_encryption_ek', +) + +MARKER_COLUMNS = ( + 'fk_account_ek', + 'fk_jid_ek', + 'fk_occupant_ek', + 'marker_id', + 'received_ts', + 'displayed_ts', + 'acknowledged_ts', +) + +MAM_ARCHIVE_STATE_COLUMNS = ( + 'fk_account_ek', + 'fk_jid_ek', + 'from_stanza_id', + 'from_stanza_ts', + 'to_stanza_id', + 'to_stanza_ts', +) + +ARCHIVE_CREATE_STMT = ''' + CREATE TABLE account ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + jid TEXT + ); + CREATE TABLE jid ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + jid TEXT + ); + CREATE TABLE occupant ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + fk_account_ek INTEGER NOT NULL, + fk_jid_ek INTEGER NOT NULL, + timestamp REAL, + id TEXT NOT NULL, + fk_real_jid_ek TEXT, + nickname TEXT, + avatar_sha TEXT, + FOREIGN KEY(fk_account_ek) REFERENCES account(entitykey) ON DELETE CASCADE, + FOREIGN KEY(fk_jid_ek) REFERENCES jid(entitykey), + FOREIGN KEY(fk_real_jid_ek) REFERENCES jid(entitykey) + ); + CREATE TABLE message ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + fk_account_ek INTEGER NOT NULL, + fk_jid_ek INTEGER NOT NULL, + resource TEXT, + m_type INTEGER NOT NULL, + direction INTEGER NOT NULL, + timestamp REAL NOT NULL, + state INTEGER NOT NULL, + message_id TEXT NOT NULL, + stanza_id TEXT, + stable_id INTEGER NOT NULL, + fk_occupant_ek INTEGER, + message TEXT, + user_delay_ts REAL, + fk_encryption_ek INTEGER, + fk_securitylabel_ek INTEGER, + FOREIGN KEY(fk_account_ek) REFERENCES account(entitykey) ON DELETE CASCADE, + FOREIGN KEY(fk_jid_ek) REFERENCES jid(entitykey), + FOREIGN KEY(fk_occupant_ek) REFERENCES occupant(entitykey), + FOREIGN KEY(fk_securitylabel_ek) REFERENCES securitylabel(entitykey), + FOREIGN KEY(fk_encryption_ek) REFERENCES encryption(entitykey) + ); + CREATE TABLE call ( + entitykey INTEGER PRIMARY KEY, + sid TEXT, + duration INTEGER, + state INTEGER, + FOREIGN KEY(entitykey) REFERENCES message(entitykey) ON DELETE CASCADE + ); + CREATE TABLE filetransfer ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + fk_message_ek INTEGER NOT NULL, + source_type INTEGER NOT NULL, + source TEXT NOT NULL, + date TEXT, + desc TEXT, + height INTEGER, + width INTEGER, + length INTEGER, + size INTEGER, + name TEXT, + media_type TEXT, + thumb_path TEXT, + path TEXT, + state INTEGER, + FOREIGN KEY(fk_message_ek) REFERENCES message(entitykey) ON DELETE CASCADE + ); + CREATE TABLE encryption ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + protocol INTEGER NOT NULL, + key TEXT NOT NULL, + trust INTEGER NOT NULL + ); + CREATE TABLE oob ( + entitykey INTEGER PRIMARY KEY, + url TEXT NOT NULL, + description TEXT, + FOREIGN KEY(entitykey) REFERENCES message(entitykey) ON DELETE CASCADE + ); + CREATE TABLE reply ( + entitykey INTEGER PRIMARY KEY, + fallback_end INTEGER, + quoted_jid TEXT NOT NULL, + quoted_id TEXT NOT NULL, + FOREIGN KEY(entitykey) REFERENCES message(entitykey) ON DELETE CASCADE + ); + CREATE TABLE securitylabel ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + fk_account_ek INTEGER NOT NULL, + fk_jid_ek INTEGER NOT NULL, + timestamp REAL NOT NULL, + label_hash TEXT NOT NULL, + displaymarking TEXT NOT NULL, + fgcolor TEXT NOT NULL, + bgcolor TEXT NOT NULL, + FOREIGN KEY(fk_account_ek) REFERENCES account(entitykey) ON DELETE CASCADE, + FOREIGN KEY(fk_jid_ek) REFERENCES jid(entitykey) + ); + CREATE TABLE error ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + fk_account_ek INTEGER NOT NULL, + fk_jid_ek INTEGER NOT NULL, + error_id TEXT, + by TEXT, + e_type TEXT NOT NULL, + text TEXT, + condition TEXT NOT NULL, + condition_text TEXT, + FOREIGN KEY(fk_account_ek) REFERENCES account(entitykey) ON DELETE CASCADE, + FOREIGN KEY(fk_jid_ek) REFERENCES jid(entitykey) + ); + CREATE TABLE reaction ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + fk_account_ek INTEGER NOT NULL, + fk_jid_ek INTEGER NOT NULL, + direction INTEGER NOT NULL, + timestamp REAL NOT NULL, + fk_occupant_ek INTEGER, + reaction_id TEXT, + emojis TEXT NOT NULL, + FOREIGN KEY(fk_account_ek) REFERENCES account(entitykey) ON DELETE CASCADE, + FOREIGN KEY(fk_jid_ek) REFERENCES jid(entitykey) + ); + CREATE TABLE retraction ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + fk_account_ek INTEGER NOT NULL, + fk_jid_ek INTEGER NOT NULL, + direction INTEGER NOT NULL, + timestamp REAL NOT NULL, + fk_occupant_ek INTEGER NOT NULL, + retraction_id TEXT NOT NULL, + FOREIGN KEY(fk_account_ek) REFERENCES account(entitykey) ON DELETE CASCADE, + FOREIGN KEY(fk_jid_ek) REFERENCES jid(entitykey) + ); + CREATE TABLE moderation ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + fk_account_ek INTEGER NOT NULL, + fk_jid_ek INTEGER NOT NULL, + timestamp REAL NOT NULL, + moderation_id TEXT NOT NULL, + fk_occupant_ek INTEGER, + by TEXT, + reason TEXT, + FOREIGN KEY(fk_account_ek) REFERENCES account(entitykey) ON DELETE CASCADE, + FOREIGN KEY(fk_jid_ek) REFERENCES jid(entitykey), + FOREIGN KEY(fk_occupant_ek) REFERENCES occupant(entitykey) + ); + CREATE TABLE correction ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + fk_account_ek INTEGER NOT NULL, + fk_jid_ek INTEGER NOT NULL, + resource TEXT, + direction INTEGER NOT NULL, + timestamp INTEGER NOT NULL, + message_id TEXT, + fk_occupant_ek INTEGER, + correction_id TEXT NOT NULL, + corrected_message TEXT NOT NULL, + fk_encryption_ek INTEGER, + FOREIGN KEY(fk_account_ek) REFERENCES account(entitykey) ON DELETE CASCADE, + FOREIGN KEY(fk_jid_ek) REFERENCES jid(entitykey), + FOREIGN KEY(fk_occupant_ek) REFERENCES occupant(entitykey), + FOREIGN KEY(fk_encryption_ek) REFERENCES encryption(entitykey) + ); + CREATE TABLE marker ( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + fk_account_ek INTEGER NOT NULL, + fk_jid_ek INTEGER NOT NULL, + fk_occupant_ek INTEGER NOT NULL, + marker_id TEXT NOT NULL, + received_ts INTEGER, + displayed_ts INTEGER, + acknowledged_ts INTEGER, + FOREIGN KEY(fk_account_ek) REFERENCES account(entitykey) ON DELETE CASCADE, + FOREIGN KEY(fk_jid_ek) REFERENCES jid(entitykey) + ); + CREATE TABLE mam_archive_state( + entitykey INTEGER PRIMARY KEY AUTOINCREMENT, + fk_account_ek INTEGER NOT NULL, + fk_jid_ek INTEGER NOT NULL, + from_stanza_id TEXT, + from_stanza_ts INTEGER, + to_stanza_id TEXT, + to_stanza_ts INTEGER, + FOREIGN KEY(fk_account_ek) REFERENCES account(entitykey) ON DELETE CASCADE, + FOREIGN KEY(fk_jid_ek) REFERENCES jid(entitykey) + ); + CREATE UNIQUE INDEX idx_account ON account (jid); + CREATE UNIQUE INDEX idx_occupant ON occupant (id, fk_jid_ek, fk_account_ek); + CREATE INDEX idx_message ON message (fk_jid_ek, fk_account_ek, timestamp DESC); + CREATE UNIQUE INDEX idx_message_dedup ON message (message_id, fk_jid_ek, fk_account_ek, direction); + CREATE UNIQUE INDEX idx_encryption ON encryption (protocol, key, trust); + CREATE UNIQUE INDEX idx_securitylabel ON securitylabel (label_hash, fk_jid_ek, fk_account_ek); + CREATE UNIQUE INDEX idx_error ON error (error_id, fk_jid_ek, fk_account_ek); + CREATE UNIQUE INDEX idx_reaction ON reaction (reaction_id, fk_jid_ek, fk_account_ek, fk_occupant_ek); + CREATE UNIQUE INDEX idx_retraction ON retraction (retraction_id, fk_jid_ek, fk_account_ek, fk_occupant_ek, direction); + CREATE UNIQUE INDEX idx_moderation ON moderation (moderation_id, fk_jid_ek, fk_account_ek); + CREATE INDEX idx_correction ON correction (correction_id, fk_jid_ek, fk_account_ek, direction, fk_occupant_ek, timestamp DESC); + CREATE UNIQUE INDEX idx_correction_dedup ON correction (message_id, fk_jid_ek, fk_account_ek, direction); + CREATE UNIQUE INDEX idx_marker ON marker (marker_id, fk_jid_ek, fk_account_ek, fk_occupant_ek); + CREATE UNIQUE INDEX idx_mam_archive_state ON mam_archive_state (fk_jid_ek, fk_account_ek); + + PRAGMA user_version=%s; +''' + +GET_CONVERSATION_STMT = ''' + SELECT + message.entitykey AS entitykey, + account.jid AS account_jid, + account.entitykey AS account_ek, + jid.jid AS "remote_jid [jid]", + jid.entitykey AS remote_ek, + message.resource AS resource, + message.m_type AS m_type, + message.direction AS direction, + message.timestamp AS timestamp, + message.state AS state, + message.message_id AS message_id, + message.stanza_id AS stanza_id, + message.stable_id AS stable_id, + message.message AS message, + message.user_delay_ts AS user_delay_ts, + encryption.protocol AS t_encryption_protocol, + encryption.key AS t_encryption_key, + encryption.trust AS t_encryption_trust, + error.by AS t_error_by, + error.e_type AS t_error_e_type, + error.text AS t_error_text, + error.condition AS t_error_condition, + error.condition_text AS t_error_condition_text, + call.sid AS t_call_sid, + call.duration AS t_call_duration, + call.state AS t_call_state, + CASE WHEN filetransfer.entitykey THEN 1 ELSE 0 END has_filetransfers, + oob.url AS t_oob_url, + oob.description AS t_oob_description, + reply.fallback_end AS t_reply_fallback_end, + reply.quoted_jid AS t_reply_quoted_jid, + reply.quoted_id AS t_reply_quoted_id, + securitylabel.displaymarking AS t_securitylabel_displaymarking, + securitylabel.fgcolor AS t_securitylabel_fgcolor, + securitylabel.bgcolor AS t_securitylabel_bgcolor, + CASE WHEN marker.entitykey THEN 1 ELSE 0 END has_markers, + marker.received_ts AS t_marker_received_ts, + marker.displayed_ts AS t_marker_displayed_ts, + moderation.fk_occupant_ek AS t_moderation_occupant_ek, + moderation.timestamp AS t_moderation_timestamp, + moderation.by AS t_moderation_by, + moderation.reason AS t_moderation_reason, + CASE WHEN retraction.entitykey THEN 1 ELSE 0 END is_retracted, + CASE WHEN reaction.entitykey THEN 1 ELSE 0 END has_reactions, + correction.entitykey AS t_correction_entitykey, + correction.corrected_message AS t_correction_message, + correction.timestamp AS t_correction_timestamp, + correction_encryption.protocol AS t_correction_encryption_protocol, + correction_encryption.key AS t_correction_encryption_key, + correction_encryption.trust AS t_correction_encryption_trust, + occupant.entitykey AS t_occupant_entitykey, + occupant.id AS t_occupant_id, + occupant.nickname AS t_occupant_nickname, + occupant_jid.jid AS "t_occupant_real_jid [jid]" + FROM message + LEFT OUTER JOIN account ON account.entitykey = message.fk_account_ek + LEFT OUTER JOIN jid ON jid.entitykey = message.fk_jid_ek + LEFT OUTER JOIN occupant ON occupant.entitykey = message.fk_occupant_ek + LEFT OUTER JOIN jid occupant_jid ON occupant_jid.entitykey = occupant.fk_real_jid_ek + LEFT OUTER JOIN call ON call.entitykey = message.entitykey + LEFT OUTER JOIN filetransfer ON filetransfer.fk_message_ek = (SELECT entitykey + FROM filetransfer WHERE + filetransfer.fk_message_ek = message.entitykey + LIMIT 1 + ) + LEFT OUTER JOIN oob ON oob.entitykey = message.entitykey + LEFT OUTER JOIN encryption ON encryption.entitykey = message.fk_encryption_ek + LEFT OUTER JOIN reply ON reply.entitykey = message.entitykey + LEFT OUTER JOIN securitylabel ON securitylabel.entitykey = message.fk_securitylabel_ek + LEFT OUTER JOIN error ON + error.error_id = message.message_id AND + error.fk_jid_ek = message.fk_jid_ek AND + error.fk_account_ek = message.fk_account_ek + LEFT OUTER JOIN marker ON marker.entitykey = (SELECT entitykey + FROM marker WHERE + marker.marker_id = message.message_id AND + marker.fk_jid_ek = message.fk_jid_ek AND + marker.fk_account_ek = message.fk_account_ek + LIMIT 1 + ) + LEFT OUTER JOIN moderation ON + moderation.moderation_id = message.stanza_id AND + moderation.fk_jid_ek = message.fk_jid_ek AND + moderation.fk_account_ek = message.fk_account_ek + LEFT OUTER JOIN retraction ON + retraction.retraction_id = message.message_id AND + retraction.fk_jid_ek = message.fk_jid_ek AND + retraction.fk_account_ek = message.fk_account_ek AND + retraction.fk_occupant_ek IS message.fk_occupant_ek AND + retraction.direction = message.direction + LEFT OUTER JOIN correction ON correction.entitykey = (SELECT entitykey + FROM correction WHERE + correction.correction_id = message.message_id AND + correction.fk_jid_ek = message.fk_jid_ek AND + correction.fk_account_ek = message.fk_account_ek AND + correction.direction = message.direction AND + correction.fk_occupant_ek IS message.fk_occupant_ek AND + CASE WHEN message.m_type = 2 AND message.fk_occupant_ek IS NULL THEN + correction.resource = message.resource + ELSE 1 + END + ORDER BY correction.timestamp DESC + LIMIT 1 + ) + LEFT OUTER JOIN encryption AS correction_encryption ON correction_encryption.entitykey = correction.fk_encryption_ek + LEFT OUTER JOIN reaction ON reaction.entitykey = (SELECT entitykey + FROM reaction WHERE + reaction.reaction_id = message.message_id AND + reaction.fk_jid_ek = message.fk_jid_ek AND + reaction.fk_account_ek = message.fk_account_ek + LIMIT 1 + ) + WHERE +''' + +GET_DAYS_CONTAINING_MESSAGES_STMT = ''' + SELECT DISTINCT + CAST(strftime('%d', message.timestamp, 'unixepoch', 'localtime') AS INTEGER) AS day + FROM message + WHERE +''' + +GET_FIRST_MESSAGE_TS_STMT = ''' + SELECT MIN(message.timestamp) as timestamp + FROM message + WHERE +''' + +GET_LAST_MESSAGE_TS_STMT = ''' + SELECT MAX(message.timestamp) as timestamp + FROM message + WHERE +''' + +GET_MESSAGE_META_STMT = ''' + SELECT + message.entitykey, + message.timestamp + FROM message + WHERE +''' + +FIND_CORRECTED_MSG_BASE_STMT = ''' + SELECT + entitykey + FROM message + WHERE + message_id = :correction_id AND %s + fk_occupant_ek IS :fk_occupant_ek AND + fk_jid_ek = :fk_jid_ek AND + fk_account_ek = :fk_account_ek AND + direction = :direction +''' + +FIND_CORRECTED_MSG_STMT = FIND_CORRECTED_MSG_BASE_STMT % '' +FIND_CORRECTED_MSG_WITH_RES_STMT = FIND_CORRECTED_MSG_BASE_STMT % 'resource = :resource AND' + + +GET_CORRECTIONS_BASE_STMT = ''' + SELECT + correction.entitykey, + correction.corrected_message AS message, + correction.timestamp, + encryption.protocol AS encryption_protocol, + encryption.key AS encryption_key, + encryption.trust AS encryption_trust + FROM correction + LEFT OUTER JOIN encryption ON encryption.entitykey = correction.fk_encryption_ek + WHERE + correction_id = :message_id AND %s + fk_occupant_ek IS :fk_occupant_ek AND + fk_jid_ek = :fk_jid_ek AND + fk_account_ek = :fk_account_ek AND + direction = :direction +''' + +GET_CORRECTIONS_STMT = GET_CORRECTIONS_BASE_STMT % '' +GET_CORRECTIONS_WITH_RES_STMT = GET_CORRECTIONS_BASE_STMT % 'resource = :resource AND' + + +GET_CONVERATION_JIDS_STMT = ''' + SELECT DISTINCT + jid.jid as "remote_jid [jid]" + FROM message + LEFT OUTER JOIN jid ON jid.entitykey = message.fk_jid_ek + WHERE + fk_account_ek = ? +''' + +GET_MAM_ARCHIVE_STATE_STMT = ''' + SELECT + to_stanza_id, + to_stanza_ts as "to_stanza_ts [datetime]", + from_stanza_id, + from_stanza_ts as "from_stanza_ts [datetime]" + FROM mam_archive_state + WHERE + fk_jid_ek = ? AND fk_account_ek = ? +''' + +RESET_ARCHIVE_STATE_STMT = ''' + DELETE FROM mam_archive_state + WHERE + fk_jid_ek = ? AND fk_account_ek = ? +''' + +def _create_insert_stmt(table: str, columns: tuple[str, ...]) -> str: + values = [f':{col}' for col in columns] + values = ', '.join(values) + cols = ', '.join(columns) + if table not in ('message', 'correction'): + # Default to -1 if fk_occupant_ek is NULL + # This is necessary for UNIQUE indexes + values = values.replace(':fk_occupant_ek', 'COALESCE(:fk_occupant_ek, -1)') + return f'INSERT INTO {table}({cols}) VALUES ({values})' + +ACCOUNT_INSERT_STMT = _create_insert_stmt('account', ACCOUNT_COLUMNS) +JID_INSERT_STMT = _create_insert_stmt('jid', JID_COLUMNS) +OCCUPANT_INSERT_STMT = _create_insert_stmt('occupant', OCCUPANT_COLUMNS) +MESSAGE_INSERT_STMT = _create_insert_stmt('message', MESSAGE_COLUMNS) +CALL_INSERT_STMT = _create_insert_stmt('call', CALL_COLUMNS) +FILETRANSFER_INSERT_STMT = _create_insert_stmt('filetransfer', FILETRANSFER_COLUMNS) +ENCRYPTION_INSERT_STMT = _create_insert_stmt('encryption', ENCRYPTION_COLUMNS) +OOB_INSERT_STMT = _create_insert_stmt('oob', OOB_COLUMNS) +REPLY_INSERT_STMT = _create_insert_stmt('reply', REPLY_COLUMNS) +SECURITYLABEL_INSERT_STMT = _create_insert_stmt('securitylabel', SECURITYLABEL_COLUMNS) +ERROR_INSERT_STMT = _create_insert_stmt('error', ERROR_COLUMNS) +REACTION_INSERT_STMT = _create_insert_stmt('reaction', REACTION_COLUMNS) +RETRACTION_INSERT_STMT = _create_insert_stmt('retraction', RETRACTION_COLUMNS) +MODERATION_INSERT_STMT = _create_insert_stmt('moderation', MODERATION_COLUMNS) +CORRECTION_INSERT_STMT = _create_insert_stmt('correction', CORRECTION_COLUMNS) +MARKER_INSERT_STMT = _create_insert_stmt('marker', MARKER_COLUMNS) +MAM_ARCHIVE_STATE_INSERT_STMT = _create_insert_stmt('mam_archive_state', MAM_ARCHIVE_STATE_COLUMNS) diff --git a/gajim/common/storage/archive/storage.py b/gajim/common/storage/archive/storage.py new file mode 100644 index 000000000..aee23936c --- /dev/null +++ b/gajim/common/storage/archive/storage.py @@ -0,0 +1,963 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import annotations + +from typing import Any + +import calendar +import datetime +import logging +import sqlite3 as sqlite +import time +from collections import namedtuple +from collections.abc import Iterator +from functools import partial + +from nbxmpp import JID + +from gajim.common import app +from gajim.common import configpaths +from gajim.common.const import MAX_MESSAGE_CORRECTION_DELAY +from gajim.common.modules.contacts import GroupchatContact +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +# from gajim.common.storage.archive.migration_v8 import MigrationV8 +from gajim.common.storage.archive.statements import ARCHIVE_CREATE_STMT +from gajim.common.storage.archive.statements import FIND_CORRECTED_MSG_STMT +from gajim.common.storage.archive.statements import \ + FIND_CORRECTED_MSG_WITH_RES_STMT +from gajim.common.storage.archive.statements import GET_CONVERATION_JIDS_STMT +from gajim.common.storage.archive.statements import GET_CONVERSATION_STMT +from gajim.common.storage.archive.statements import GET_CORRECTIONS_STMT +from gajim.common.storage.archive.statements import \ + GET_CORRECTIONS_WITH_RES_STMT +from gajim.common.storage.archive.statements import \ + GET_DAYS_CONTAINING_MESSAGES_STMT +from gajim.common.storage.archive.statements import GET_FIRST_MESSAGE_TS_STMT +from gajim.common.storage.archive.statements import GET_LAST_MESSAGE_TS_STMT +from gajim.common.storage.archive.statements import GET_MAM_ARCHIVE_STATE_STMT +from gajim.common.storage.archive.statements import GET_MESSAGE_META_STMT +from gajim.common.storage.archive.statements import JID_INSERT_STMT +from gajim.common.storage.archive.statements import RESET_ARCHIVE_STATE_STMT +from gajim.common.storage.archive.structs import DbChatJoinedData +from gajim.common.storage.archive.structs import DbConversationJoinedData +from gajim.common.storage.archive.structs import DbConversationJoinedRowData +from gajim.common.storage.archive.structs import DbCorrectionRowData +from gajim.common.storage.archive.structs import DbFiletransferRowData +from gajim.common.storage.archive.structs import DbGroupchatJoinedData +from gajim.common.storage.archive.structs import DbInsertCorrectionRowData +from gajim.common.storage.archive.structs import DbInsertRowDataBase +from gajim.common.storage.archive.structs import DbMamArchiveStateRowData +from gajim.common.storage.archive.structs import DbPrivateMessageJoinedData +from gajim.common.storage.archive.structs import DbUpsertRowDataBase +from gajim.common.storage.base import SqliteStorage +from gajim.common.storage.base import timeit +from gajim.common.storage.base import VALUE_MISSING + +CURRENT_USER_VERSION = 7 + +log = logging.getLogger('gajim.c.storage.archive') + + +def _messages_factory(cursor: sqlite.Cursor, row: tuple[Any, ...]) -> Any: + fields = [col[0] for col in cursor.description] + + match row[6]: + case MessageType.CHAT: + return DbChatJoinedData.from_row(fields, row) + case MessageType.GROUPCHAT: + return DbGroupchatJoinedData.from_row(fields, row) + case MessageType.PM: + return DbPrivateMessageJoinedData.from_row(fields, row) + case _: + raise ValueError + + +def _row_factory( + row_class: Any, + cursor: sqlite.Cursor, + row: tuple[Any, ...] +) -> Any: + + fields = [col[0] for col in cursor.description] + kv = dict(zip(fields, row, strict=True)) + return row_class.from_query(kv) + + +def namedtuple_row(cursor: sqlite.Cursor, row: tuple[Any, ...]) -> Any: + fields = [column[0] for column in cursor.description] # pyright: ignore + cls = namedtuple('Row', fields) # pyright: ignore + return cls._make(row) + + +class MessageArchiveStorage(SqliteStorage): + def __init__(self, in_memory: bool = False): + path = None if in_memory else configpaths.get('LOG_DB') + SqliteStorage.__init__(self, + log, + path, + ARCHIVE_CREATE_STMT % CURRENT_USER_VERSION) + + self._jid_eks: dict[JID | str, int] = {} + self._accounts_eks: dict[str, int] = {} + + def init(self, **kwargs: Any) -> None: + SqliteStorage.init(self, + detect_types=sqlite.PARSE_COLNAMES) + + self._set_journal_mode('WAL') + self._enable_foreign_keys() + self._enable_secure_delete() + + self._con.row_factory = namedtuple_row + + self._con.create_function('like', 1, self._like) + + self._load_jids() + + def _migrate(self) -> None: + user_version = self.user_version + if user_version == 0: + # All migrations from 0.16.9 until 1.0.0 + statements = [ + 'ALTER TABLE logs ADD COLUMN "account_id" INTEGER', + 'ALTER TABLE logs ADD COLUMN "stanza_id" TEXT', + 'ALTER TABLE logs ADD COLUMN "encryption" TEXT', + 'ALTER TABLE logs ADD COLUMN "encryption_state" TEXT', + 'ALTER TABLE logs ADD COLUMN "marker" INTEGER', + 'ALTER TABLE logs ADD COLUMN "additional_data" TEXT', + '''CREATE TABLE IF NOT EXISTS last_archive_message( + jid_id INTEGER PRIMARY KEY UNIQUE, + last_mam_id TEXT, + oldest_mam_timestamp TEXT, + last_muc_timestamp TEXT + )''', + + '''CREATE INDEX IF NOT EXISTS idx_logs_stanza_id + ON logs(stanza_id)''', + 'PRAGMA user_version=1' + ] + + self._execute_multiple(statements) + + if user_version < 2: + statements = [ + ('ALTER TABLE last_archive_message ' + 'ADD COLUMN "sync_threshold" INTEGER'), + 'PRAGMA user_version=2' + ] + self._execute_multiple(statements) + + if user_version < 3: + statements = [ + 'ALTER TABLE logs ADD COLUMN "message_id" TEXT', + 'PRAGMA user_version=3' + ] + self._execute_multiple(statements) + + if user_version < 4: + statements = [ + 'ALTER TABLE logs ADD COLUMN "error" TEXT', + 'PRAGMA user_version=4' + ] + self._execute_multiple(statements) + + if user_version < 5: + statements = [ + '''CREATE INDEX IF NOT EXISTS idx_logs_message_id + ON logs (message_id)''', + 'PRAGMA user_version=5' + ] + self._execute_multiple(statements) + + if user_version < 7: + statements = [ + 'ALTER TABLE logs ADD COLUMN "real_jid" TEXT', + 'ALTER TABLE logs ADD COLUMN "occupant_id" TEXT', + 'PRAGMA user_version=7' + ] + self._execute_multiple(statements) + + # if user_version < 8: + # MigrationV8(self._con).run() + + @staticmethod + def _like(search_str: str) -> str: + return f'%{search_str}%' + + def _get_cursor( + self, + factory: Any = None, + row_class: Any = None + ) -> sqlite.Cursor: + + cursor = self._con.cursor() + + if row_class is not None: + factory = partial(factory, row_class) + cursor.row_factory = factory + return cursor + + @timeit + def _load_jids(self) -> None: + rows = self._con.execute('SELECT entitykey, jid FROM jid').fetchall() + for row in rows: + self._jid_eks[row.jid] = row.entitykey + + def _get_account_ek(self, account: str) -> int: + account_ek = self._accounts_eks.get(account) + if account_ek is not None: + return account_ek + + jid = app.get_jid_from_account(account) + + select_stmt = 'SELECT entitykey FROM account WHERE jid = ?' + result = self._con.execute(select_stmt, (jid,)).fetchone() + if result is not None: + self._accounts_eks[account] = result.entitykey + return result.entitykey + + insert_stmt = 'INSERT INTO account(jid) VALUES (?)' + entitykey = self._con.execute(insert_stmt, (jid,)).lastrowid + assert entitykey is not None + self._accounts_eks[account] = entitykey + self._delayed_commit() + + return entitykey + + def _get_jid_ek(self, jid: JID | str) -> int: + entitykey = self._jid_eks.get(jid) + if entitykey is not None: + return entitykey + + entitykey = self._con.execute(JID_INSERT_STMT, {'jid': jid}).lastrowid + assert entitykey is not None + self._jid_eks[jid] = entitykey + return entitykey + + def _get_occupant_ek(self, + account_ek: int, + jid_ek: int, + occupant_id: str) -> int: + # TODO cache calls? + stmt = ''' + SELECT entitykey FROM occupant WHERE + fk_accounts_ek = ? AND + fk_jids_ek = ? AND + id = ? + ''' + + row = self._con.execute( + stmt, (account_ek, jid_ek, occupant_id)).fetchone() + assert row is not None + return row.entitykey + + def _get_foreign_keys( + self, + db_row_data: Any + ) -> dict[str, int | None]: + + account = getattr(db_row_data, 'account', None) + if account is None: + account_ek = None + else: + account_ek = self._get_account_ek(account) + + remote_jid = getattr(db_row_data, 'remote_jid', None) + if remote_jid is None: + jid_ek = None + else: + jid_ek = self._get_jid_ek(remote_jid) + + real_jid_ek = None + real_jid = getattr(db_row_data, 'real_jid', None) + if real_jid not in (None, VALUE_MISSING): + real_jid_ek = self._get_jid_ek(real_jid) + + return { + 'fk_account_ek': account_ek, + 'fk_jid_ek': jid_ek, + 'fk_real_jid_ek': real_jid_ek, + } + + def get_conversation_jids(self, account: str) -> list[JID]: + account_ek = self._get_account_ek(account) + + rows = self._con.execute( + GET_CONVERATION_JIDS_STMT, (account_ek, )).fetchall() + return [row.remote_jid for row in rows] + + @timeit + def get_conversation_before_after(self, + account: str, + jid: JID, + before: bool, + timestamp: float, + n_lines: int + ) -> list[DbConversationJoinedData]: + ''' + Load n_lines lines of conversation with jid before or after timestamp + + :param account: The account + + :param jid: The jid for which we request the conversation + + :param before: bool for direction (before or after timestamp) + + :param timestamp: timestamp + + :param nlines: number of rows to get + + returns a list of namedtuples + ''' + + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + + if before: + time_order = ('message.timestamp < ? ORDER BY ' + 'message.timestamp DESC, message.entitykey DESC') + else: + time_order = ('message.timestamp > ? ORDER BY ' + 'message.timestamp ASC, message.entitykey ASC') + + stmt = f''' + {GET_CONVERSATION_STMT} + message.fk_jid_ek = ? AND + message.fk_account_ek = ? AND + {time_order} + LIMIT ? + ''' + cursor = self._get_cursor(_messages_factory) + return cursor.execute( + stmt, (jid_ek, account_ek, timestamp, n_lines)).fetchall() + + @timeit + def get_message_with_entitykey( + self, + entitykey: int + ) -> DbConversationJoinedData: + + stmt = f''' + {GET_CONVERSATION_STMT} + message.entitykey = ? + ''' + cursor = self._get_cursor(_messages_factory) + return cursor.execute(stmt, (entitykey,)).fetchone() + + def get_row_with_entitykey(self, table: str, entitykey: int) -> Any: + stmt = f'SELECT * FROM {table} WHERE entitykey=?' + return self._con.execute(stmt, (entitykey,)).fetchone() + + @timeit + def get_last_conversation_row(self, + account: str, + jid: JID + ) -> DbConversationJoinedData | None: + ''' + Load the last line of a conversation with jid for account. + Loads messages, but no status messages or error messages. + + :param account: The account + + :param jid: The jid for which we request the conversation + + returns a namedtuple or None + ''' + + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + + stmt = f''' + {GET_CONVERSATION_STMT} + message.fk_jid_ek = ? AND + message.fk_account_ek = ? + ORDER BY message.timestamp DESC + ''' + cursor = self._get_cursor(_messages_factory) + return cursor.execute(stmt, (jid_ek, account_ek)).fetchone() + + @timeit + def get_last_correctable_message(self, + account: str, + jid: JID, + message_id: str + ) -> DbConversationJoinedData | None: + ''' + Load the last correctable message of a conversation by message_id. + Conditions: max 5 min old + ''' + min_time = time.time() - MAX_MESSAGE_CORRECTION_DELAY + + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + + # TODO there is no index for message_id + + stmt = f''' + {GET_CONVERSATION_STMT} + message.fk_jid_ek = ? AND + message.fk_account_ek = ? AND + message.message_id = ? AND + message.timestamp > ? + ORDER BY message.timestamp DESC + ''' + cursor = self._get_cursor(_messages_factory) + return cursor.execute( + stmt, (jid_ek, account_ek, message_id, min_time)).fetchone() + + def get_corrected_message_entitykey( + self, + m_type: int, + corrected_data: DbInsertCorrectionRowData + ) -> int | None: + + fk_account_ek = self._get_account_ek(corrected_data.account) + fk_jid_ek = self._get_jid_ek(corrected_data.remote_jid) + + params = corrected_data.asdict() + params['fk_jid_ek'] = fk_jid_ek + params['fk_account_ek'] = fk_account_ek + + stmt = FIND_CORRECTED_MSG_STMT + if (m_type == MessageType.GROUPCHAT and + corrected_data.fk_occupant_ek is None): + stmt = FIND_CORRECTED_MSG_WITH_RES_STMT + + cursor = self._get_cursor() + result = cursor.execute(stmt, params).fetchone() + if result is None: + return None + return result[0] + + def get_corrections( + self, + joined_data: DbConversationJoinedRowData + ) -> list[DbCorrectionRowData]: + + occupant_ek = None + if joined_data.occupants is not None: + occupant_ek = joined_data.occupants.entitykey + + resource = None + if (joined_data.m_type == MessageType.GROUPCHAT and + joined_data.occupants is None): + resource = joined_data.resource + + params = { + 'fk_jid_ek': joined_data.remote_ek, + 'fk_account_ek': joined_data.account_ek, + 'message_id': joined_data.message_id, + 'resource': resource, + 'fk_occupant_ek': occupant_ek, + 'direction': joined_data.direction + } + + stmt = GET_CORRECTIONS_STMT + if resource is not None: + stmt = GET_CORRECTIONS_WITH_RES_STMT + + cursor = self._get_cursor(_row_factory, row_class=DbCorrectionRowData) + return cursor.execute(stmt, params).fetchall() + + + def get_filetransfers(self, message_ek: int) -> list[DbFiletransferRowData]: + cursor = self._get_cursor(_row_factory, row_class=DbFiletransferRowData) + stmt = DbFiletransferRowData.get_select_stmt() + return cursor.execute(stmt, (message_ek,)).fetchall() + + + @timeit + def search_archive(self, + account: str | None, + jid: str | None, + query: str, + from_users: list[str] | None = None, + before: datetime.datetime | None = None, + after: datetime.datetime | None = None + ) -> Iterator[DbConversationJoinedData]: + ''' + Search the conversation log for messages containing the `query` string. + + The search can either span the complete log for the given + `account` and `jid` or be restricted to a single day by + specifying `date`. + + :param account: The account + + :param jid: The jid for which we request the conversation + + :param query: A search string + + :param from_users: A list of usernames or None + + :param before: A datetime.datetime instance or None + + :param after: A datetime.datetime instance or None + + returns a list of namedtuples + ''' + if before is None: + before_ts = datetime.datetime.now().timestamp() + else: + before_ts = before.timestamp() + + after_ts = 0 + if after is not None: + after_ts = after.timestamp() + + contact_stmt = '' + if account is not None and jid is not None: + contact_stmt = ''' + message.fk_jid_ek = ? AND + message.fk_account_ek = ? AND''' + + if from_users is None: + users_query_stmt = '' + else: + users_query_stmt = 'UPPER(message.resource) IN (?) AND' + + stmt = f''' + {GET_CONVERSATION_STMT} + {contact_stmt} + {users_query_stmt} + IFNULL(correction.corrected_message, message.message) + LIKE like(?) AND + message.timestamp BETWEEN ? AND ? + ORDER BY message.timestamp DESC, message.entitykey + ''' + + cursor = self._get_cursor(_messages_factory) + if from_users is None: + if contact_stmt: + assert account is not None + assert jid is not None + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + cursor.execute( + stmt, (jid_ek, account_ek, query, after_ts, before_ts)) + else: + cursor.execute( + stmt, (query, after_ts, before_ts)) + while True: + results = cursor.fetchmany(25) + if not results: + break + for result in results: + yield result + return + + users = ','.join([user.upper() for user in from_users]) + if contact_stmt: + assert account is not None + assert jid is not None + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + cursor.execute( + stmt, (jid_ek, account_ek, users, query, after_ts, before_ts)) + else: + cursor.execute( + stmt, (users, query, after_ts, before_ts)) + while True: + results = cursor.fetchmany(25) + if not results: + break + for result in results: + yield result + + @timeit + def get_days_containing_messages(self, + account: str, + jid: str, + year: int, + month: int + ) -> list[int]: + ''' + Get days in month of year where messages for account/jid exist + ''' + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + + # Calculate the start and end datetime of the month + date = datetime.datetime(year, month, 1) + days = calendar.monthrange(year, month)[1] - 1 + delta = datetime.timedelta( + days=days, hours=23, minutes=59, seconds=59, microseconds=999999) + start_ts = date.timestamp() + end_ts = (date + delta).timestamp() + + stmt = f''' + {GET_DAYS_CONTAINING_MESSAGES_STMT} + message.fk_jid_ek = ? AND + message.fk_account_ek = ? AND + message.timestamp BETWEEN ? AND ? + ORDER BY message.timestamp + ''' + return [row.day for row in self._con.execute( + stmt, (jid_ek, account_ek, start_ts, end_ts)).fetchall()] + + @timeit + def get_last_history_ts(self, + account: str, + jid: str + ) -> float | None: + ''' + Get the timestamp of the last message we received for the jid + ''' + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + stmt = f''' + {GET_LAST_MESSAGE_TS_STMT} + message.fk_jid_ek = ? AND + message.fk_account_ek = ? + ''' + # fetchone() returns always at least one Row with all + # attributes set to None because of the MAX() function + return self._con.execute( + stmt, (jid_ek, account_ek)).fetchone().timestamp + + @timeit + def get_first_history_ts(self, + account: str, + jid: str + ) -> float | None: + ''' + Get the timestamp of the first message we received for the jid + ''' + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + stmt = f''' + {GET_FIRST_MESSAGE_TS_STMT} + message.fk_jid_ek = ? AND + message.fk_account_ek = ? + ''' + # fetchone() returns always at least one Row with all + # attributes set to None because of the MIN() function + return self._con.execute( + stmt, (jid_ek, account_ek)).fetchone().timestamp + + @timeit + def get_first_message_meta_for_date(self, + account: str, + jid: str, + date: datetime.datetime + ) -> tuple[int, float] | None: + ''' + Load meta data (entitykey, timestamp) for the first message of + a specific date + ''' + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + delta = datetime.timedelta( + hours=23, minutes=59, seconds=59, microseconds=999999) + date_ts = date.timestamp() + delta_ts = (date + delta).timestamp() + stmt = f''' + {GET_MESSAGE_META_STMT} + message.fk_jid_ek = ? AND + message.fk_account_ek = ? AND + message.timestamp BETWEEN ? AND ? + ORDER BY message.timestamp ASC, message.entitykey ASC + ''' + return self._con.execute( + stmt, (jid_ek, account_ek, date_ts, delta_ts)).fetchone() + + @timeit + def date_has_history(self, + account: str, + jid: str, + date: datetime.datetime + ) -> float | None: + ''' + Get a single meta row of a message for 'jid' + in time range of one day + ''' + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + delta = datetime.timedelta( + hours=23, minutes=59, seconds=59, microseconds=999999) + start_ts = date.timestamp() + end_ts = (date + delta).timestamp() + stmt = f''' + {GET_MESSAGE_META_STMT} + message.fk_jid_ek = ? AND + message.fk_account_ek = ? AND + message.timestamp BETWEEN ? AND ? + ''' + return self._con.execute( + stmt, (jid_ek, account_ek, start_ts, end_ts)).fetchone() + + def _insert_row( + self, + row_data: DbInsertRowDataBase, + with_entitykey: int | None = None, + raise_on_conflict: bool = True, + ignore_on_conflict: bool = False, + ) -> int: + + foreign_keys = self._get_foreign_keys(row_data) + + values = row_data.asdict() + values.update(foreign_keys) + if with_entitykey is not None: + values['entitykey'] = with_entitykey + + log.info('Insert into %s: %s', row_data.get_table_name(), values) + try: + cursor = self._con.execute(row_data.get_insert_stmt(), values) + except sqlite.IntegrityError as error: + if ignore_on_conflict: + self._log.debug('Ignore failed insert because of contraint') + return -1 + + if raise_on_conflict: + raise error + + row = self._con.execute( + row_data.get_select_stmt(), values).fetchone() + return row.entitykey + + entitykey = cursor.lastrowid + assert entitykey is not None + return entitykey + + def insert_row( + self, + row_data: DbInsertRowDataBase, + dependent_row_data: list[DbInsertRowDataBase] | None = None, + raise_on_conflict: bool = True, + ignore_on_conflict: bool = False, + ) -> int: + + if dependent_row_data is None: + dependent_row_data = [] + + entitykey = self._insert_row( + row_data, + raise_on_conflict=raise_on_conflict, + ignore_on_conflict=ignore_on_conflict) + if entitykey == -1: + return -1 + + for row_data in dependent_row_data: + self._insert_row(row_data, with_entitykey=entitykey) + + self._delayed_commit() + return entitykey + + def upsert_row(self, row_data: DbUpsertRowDataBase) -> int: + foreign_keys = self._get_foreign_keys(row_data) + + values = row_data.asdict() + values.update(foreign_keys) + + log.info('Upsert into %s: %s', row_data.get_table_name(), values) + row = self._con.execute(row_data.get_upsert_stmt(), values).fetchone() + if row is None: + row = self._con.execute( + row_data.get_select_stmt(), values).fetchone() + + self._delayed_commit() + return row.entitykey + + @timeit + def get_recent_muc_nicks(self, contact: GroupchatContact) -> set[str]: + account_ek = self._get_account_ek(contact.account) + jid_ek = self._get_jid_ek(contact.jid) + + sql = ''' + SELECT resource + FROM message WHERE + fk_jid_ek = ? AND + fk_account_ek = ? + ORDER BY timestamp DESC''' + + results = self._con.execute(sql, (jid_ek, account_ek)).fetchmany(50) + + nicknames: set[str] = set() + for row in results: + if row.resource is None: + continue + nicknames.add(row.resource) + + return nicknames + + def get_mam_archive_state( + self, + account: str, + jid: JID + ) -> DbMamArchiveStateRowData | None: + + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + + cursor = self._get_cursor(_row_factory, DbMamArchiveStateRowData) + return cursor.execute( + GET_MAM_ARCHIVE_STATE_STMT, (jid_ek, account_ek)).fetchone() + + def reset_mam_archive_state(self, account: str, jid: str) -> None: + + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + + self._con.execute(RESET_ARCHIVE_STATE_STMT, (jid_ek, account_ek)) + self._delayed_commit() + + def _delete_messages(self, find_stmt: str, params: Any) -> None: + delete_stmts = [ + f'DELETE FROM message WHERE entitykey IN ({find_stmt})', + ] + + for stmt in delete_stmts: + self._con.execute(stmt, params) + + def delete_pending_message( + self, + account: str, + remote_jid: JID, + message_id: str + ) -> int | None: + + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(remote_jid) + + delete_stmt = ''' + DELETE FROM message + WHERE + state = ? AND + message_id = ? AND + fk_jid_ek = ? AND + fk_account_ek = ? + RETURNING entitykey + ''' + + results = self._con.execute(delete_stmt, ( + MessageState.PENDING, + message_id, + jid_ek, + account_ek)).fetchall() + + if results: + assert len(results) == 1 + self._delayed_commit() + return results[0].entitykey + + def delete_message(self, entitykey: int) -> None: + + # TODO Delete Corrections + delete_stmts = [ + 'DELETE FROM message WHERE entitykey = ?', + ] + + for stmt in delete_stmts: + self._con.execute(stmt, (entitykey,)) + self._commit() + + def remove_history(self, account: str, jid: JID) -> None: + ''' + Remove history for a specific chat. + ''' + + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(jid) + + statements = [ + 'DELETE FROM message WHERE fk_jid_ek = ? and fk_account_ek = ?', + 'DELETE FROM error WHERE fk_jid_ek = ? and fk_account_ek = ?', + 'DELETE FROM reaction WHERE fk_jid_ek = ? and fk_account_ek = ?', + 'DELETE FROM retraction WHERE fk_jid_ek = ? and fk_account_ek = ?', + 'DELETE FROM moderation WHERE fk_jid_ek = ? and fk_account_ek = ?', + 'DELETE FROM correction WHERE fk_jid_ek = ? and fk_account_ek = ?', + 'DELETE FROM marker WHERE fk_jid_ek = ? and fk_account_ek = ?', + ] + + for stmt in statements: + self._con.execute(stmt, (jid_ek, account_ek)) + + self._commit() + + log.info('Removed history for: %s', jid) + + def remove_all_history(self) -> None: + ''' + Remove all messages for all accounts + ''' + statements = [ + 'DELETE FROM error', + 'DELETE FROM reaction', + 'DELETE FROM retraction', + 'DELETE FROM moderation', + 'DELETE FROM correction', + 'DELETE FROM marker', + 'DELETE FROM message', + ] + self._execute_multiple(statements) + log.info('Removed all chat history') + + def remove_all_from_account(self, account: str) -> None: + account_ek = self._get_account_ek(account) + self._con.execute( + 'DELETE FROM account WHERE entitykey = ?', (account_ek,)) + self._commit() + + def cleanup_chat_history(self) -> None: + ''' + Remove messages from account where messages are older than max_age + ''' + for account in app.settings.get_accounts(): + max_age = app.settings.get_account_setting( + account, 'chat_history_max_age') + if max_age == -1: + continue + + account_ek = self._get_account_ek(account) + now = time.time() + point_in_time = now - int(max_age) + + find_stmt = ''' + SELECT entitykey FROM message + WHERE fk_account_ek = ? and timestamp < ? + ''' + + results = self._con.execute( + find_stmt, (account_ek, point_in_time)).fetchall() + if not results: + continue + + self._delete_messages(find_stmt, (account_ek, point_in_time)) + self._commit() + log.info('Removed %s old messages for %s', len(results), account) + + def get_messages_for_export(self, + account: str, + remote_jid: JID + ) -> Iterator[DbConversationJoinedData]: + + account_ek = self._get_account_ek(account) + jid_ek = self._get_jid_ek(remote_jid) + + stmt = f''' + {GET_CONVERSATION_STMT} + message.fk_jid_ek = ? AND + message.fk_account_ek = ? + ORDER BY message.timestamp ASC + ''' + + cursor = self._con.execute(stmt, (jid_ek, account_ek)) + while True: + results = cursor.fetchmany(10) + if not results: + break + yield from results diff --git a/gajim/common/storage/archive/structs.py b/gajim/common/storage/archive/structs.py new file mode 100644 index 000000000..8eb5d6496 --- /dev/null +++ b/gajim/common/storage/archive/structs.py @@ -0,0 +1,662 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import annotations + +import typing +from typing import Any + +import dataclasses +import pprint +from datetime import datetime + +from nbxmpp.protocol import JID + +from gajim.common import app +from gajim.common.storage.archive.statements import CALL_INSERT_STMT +from gajim.common.storage.archive.statements import CORRECTION_INSERT_STMT +from gajim.common.storage.archive.statements import ENCRYPTION_INSERT_STMT +from gajim.common.storage.archive.statements import ERROR_INSERT_STMT +from gajim.common.storage.archive.statements import FILETRANSFER_INSERT_STMT +from gajim.common.storage.archive.statements import \ + MAM_ARCHIVE_STATE_INSERT_STMT +from gajim.common.storage.archive.statements import MARKER_INSERT_STMT +from gajim.common.storage.archive.statements import MESSAGE_INSERT_STMT +from gajim.common.storage.archive.statements import MODERATION_INSERT_STMT +from gajim.common.storage.archive.statements import OCCUPANT_INSERT_STMT +from gajim.common.storage.archive.statements import OOB_INSERT_STMT +from gajim.common.storage.archive.statements import REACTION_INSERT_STMT +from gajim.common.storage.archive.statements import REPLY_INSERT_STMT +from gajim.common.storage.archive.statements import RETRACTION_INSERT_STMT +from gajim.common.storage.archive.statements import SECURITYLABEL_INSERT_STMT +from gajim.common.storage.base import VALUE_MISSING +from gajim.common.storage.base import ValueMissingT + + +@dataclasses.dataclass +class DbBaseRowData: + @classmethod + def from_query(cls, key_value_dict: dict[str, Any]): + return cls(**key_value_dict) + + +@dataclasses.dataclass +class DbOOBRowData(DbBaseRowData): + url: str + description: str | None = None + + +@dataclasses.dataclass +class DbEncryptionRowData(DbBaseRowData): + protocol: str + trust: int + key: str + + +@dataclasses.dataclass +class DbReplyRowData(DbBaseRowData): + quoted_id: str + quoted_jid: str | None = None + fallback_end: int | None = None + + +@dataclasses.dataclass +class DbCallRowData(DbBaseRowData): + sid: str + state: int + duration: int | None = None + + +@dataclasses.dataclass +class DbFiletransferRowData(DbBaseRowData): + source_type: int + source: str + date: str + desc: str + height: int + width: int + length: int + size: int + name: str + media_type: str + thumb_path: str + path: str + state: int + + @classmethod + def get_select_stmt(cls) -> str: + cols = [f.name for f in dataclasses.fields(cls)] + cols = ', '.join(cols) + return 'SELECT %s FROM filetransfer WHERE fk_message_ek = ?' % cols + + +@dataclasses.dataclass +class DbCorrectionRowData(DbBaseRowData): + entitykey: int + message: str + timestamp: float + encryption: DbEncryptionRowData | None = None + + @classmethod + def from_query(cls, key_value_dict: dict[str, Any]) -> DbCorrectionRowData: + encryption_values: dict[str, Any] = {} + other_values: dict[str, Any] = {} + for key, value in key_value_dict.items(): + if key.startswith('encryption_'): + field_name = key.removeprefix('encryption_') + encryption_values[field_name] = value + else: + other_values[key] = value + + if encryption_values: + other_values['encryption'] = DbEncryptionRowData( + **encryption_values) + + return cls(**other_values) + + +@dataclasses.dataclass +class DbMarkerRowData(DbBaseRowData): + acknowledged_ts: float | None = None + displayed_ts: float | None = None + received_ts: float | None = None + + +@dataclasses.dataclass +class DbErrorRowData(DbBaseRowData): + e_type: str + condition: str + by: str | None = None + text: str | None = None + condition_text: str | None = None + + +@dataclasses.dataclass +class DbSecurityLabelRowData(DbBaseRowData): + displaymarking: str | None + fgcolor: str | None + bgcolor: str | None + + +@dataclasses.dataclass +class DbModerationRowData(DbBaseRowData): + timestamp: float + by: str | None = None + reason: str | None = None + fk_occupant_ek: int | None = None + + +@dataclasses.dataclass +class DbOccupantRowData(DbBaseRowData): + entitykey: int + id: str + nickname: str | None = None + real_jid: JID | None = None + avatar_sha: str | None = None + + +@dataclasses.dataclass +class DbMamArchiveStateRowData(DbBaseRowData): + to_stanza_id: str | None + to_stanza_ts: datetime | None + from_stanza_id: str | None + from_stanza_ts: datetime | None + + +@dataclasses.dataclass +class DbInsertRowDataBase: + _table_name: typing.ClassVar[str] = '' + _create_stmt: typing.ClassVar[str] = '' + _conflict_columns: typing.ClassVar[list[str]] = [] + + def get_insert_stmt(self) -> str: + return self._create_stmt + + def get_select_stmt(self) -> str: + conditions: list[str] = [] + for col in self._conflict_columns: + conditions.append(f'{col} IS :{col}') + + cond_stmt = ' AND '.join(conditions) + return f''' + SELECT entitykey FROM {self._table_name} WHERE {cond_stmt} + ''' + + def get_table_name(self) -> str: + return self._table_name + + def asdict(self) -> dict[str, Any]: + return {field.name:getattr(self, field.name) for + field in dataclasses.fields(self)} + + +@dataclasses.dataclass +class DbInsertCorrectionRowData(DbInsertRowDataBase): + _table_name: typing.ClassVar[str] = 'corrections' + _create_stmt: typing.ClassVar[str] = CORRECTION_INSERT_STMT + + account: str + remote_jid: JID + resource: str | None + direction: int + timestamp: float + message_id: str | None + fk_occupant_ek: int | None + correction_id: str + corrected_message: str + fk_encryption_ek: int | None = None + + +@dataclasses.dataclass +class DbInsertEncryptionRowData(DbInsertRowDataBase): + _table_name: typing.ClassVar[str] = 'encryption' + _create_stmt: typing.ClassVar[str] = ENCRYPTION_INSERT_STMT + _conflict_columns: typing.ClassVar[list[str]] = [ + 'protocol', + 'trust', + 'key', + ] + + protocol: str + trust: int + key: str | None + + +@dataclasses.dataclass +class DbInsertErrorsRowData(DbInsertRowDataBase): + _table_name: typing.ClassVar[str] = 'error' + _create_stmt: typing.ClassVar[str] = ERROR_INSERT_STMT + + account: str + remote_jid: JID + error_id: str + by: str | None + e_type: str + text: str | None + condition: str + condition_text: str | None + + +@dataclasses.dataclass +class DbInsertMessageRowData(DbInsertRowDataBase): + _table_name: typing.ClassVar[str] = 'message' + _create_stmt: typing.ClassVar[str] = MESSAGE_INSERT_STMT + + account: str + remote_jid: JID + m_type: int + direction: int + timestamp: float + state: int + resource: str | None = None + message: str | None = None + message_id: str | None = None + stanza_id: str | None = None + stable_id: bool = False + fk_occupant_ek: int | None = None + user_delay_ts: float | None = None + fk_securitylabel_ek: int | None = None + fk_encryption_ek: int | None = None + + +@dataclasses.dataclass +class DbInsertModerationRowData(DbInsertRowDataBase): + _table_name: typing.ClassVar[str] = 'moderation' + _create_stmt: typing.ClassVar[str] = MODERATION_INSERT_STMT + + account: str + remote_jid: JID + timestamp: float + moderation_id: str + by: str | None + fk_occupant_ek: int | None + reason: str | None + + +@dataclasses.dataclass +class DbInsertCallRowData(DbInsertRowDataBase): + _table_name: typing.ClassVar[str] = 'call' + _create_stmt: typing.ClassVar[str] = CALL_INSERT_STMT + + sid: str + state: int + duration: int | None = None + + +@dataclasses.dataclass +class DbInsertFiletransferRowData(DbInsertRowDataBase): + _table_name: typing.ClassVar[str] = 'filetransfer' + _create_stmt: typing.ClassVar[str] = FILETRANSFER_INSERT_STMT + + fk_message_ek: int + source_type: int + source: str + state: int + date: str | None = None + desc: str | None = None + height: int | None = None + width: int | None = None + length: int | None = None + size: int | None = None + name: str | None = None + media_type: str | None = None + thumb_path: str | None = None + path: str | None = None + + +@dataclasses.dataclass +class DbInsertOOBRowData(DbInsertRowDataBase): + _table_name: typing.ClassVar[str] = 'oob' + _create_stmt: typing.ClassVar[str] = OOB_INSERT_STMT + + url: str + description: str | None + + +@dataclasses.dataclass +class DbInsertReactionRowData(DbInsertRowDataBase): + _table_name: typing.ClassVar[str] = 'reaction' + _create_stmt: typing.ClassVar[str] = REACTION_INSERT_STMT + + account: str + remote_jid: JID + direction: int + timestamp: float + fk_occupant_ek: int | None + reaction_id: str + emojis: list[str] + + +@dataclasses.dataclass +class DbInsertReplyRowData(DbInsertRowDataBase): + _table_name: typing.ClassVar[str] = 'reply' + _create_stmt: typing.ClassVar[str] = REPLY_INSERT_STMT + + quoted_id: str + quoted_jid: str + fallback_end: int | None = None + + +@dataclasses.dataclass +class DbInsertRetractionRowData(DbInsertRowDataBase): + _table_name: typing.ClassVar[str] = 'retraction' + _create_stmt: typing.ClassVar[str] = RETRACTION_INSERT_STMT + + account: str + remote_jid: JID + direction: int + timestamp: float + fk_occupant_ek: int | None + retraction_id: str + + +@dataclasses.dataclass +class DbUpsertRowDataBase: + _table_name: typing.ClassVar[str] = '' + _create_stmt: typing.ClassVar[str] = '' + _update_set_stmt: typing.ClassVar[str] = '{col}=:{col}' + _conflict_columns: typing.ClassVar[list[str]] = [] + _updateable_columns: typing.ClassVar[list[str | tuple[str, str]]] = [] + + def _get_updated_column_stmt(self) -> str: + update_stmts: list[str] = [] + for col in self._updateable_columns: + if isinstance(col, tuple): + col, attr_name = col + else: + attr_name = col + value = getattr(self, attr_name) + if value is VALUE_MISSING: + continue + + update_set_stmt = self._update_set_stmt.format(col=col) + update_stmts.append(update_set_stmt) + + return ', '.join(update_stmts) + + def get_upsert_stmt(self) -> str: + combined_update_stmt = self._get_updated_column_stmt() + conflict_cols = ', '.join(self._conflict_columns) + + return f''' + {self._create_stmt} + ON CONFLICT({conflict_cols}) DO UPDATE SET + {combined_update_stmt} + WHERE excluded.timestamp > {self._table_name}.timestamp + RETURNING entitykey + ''' + + def get_select_stmt(self) -> str: + conditions: list[str] = [] + for col in self._conflict_columns: + conditions.append(f'{col} IS :{col}') + + cond = ' AND '.join(conditions) + + return f''' + SELECT entitykey FROM {self._table_name} WHERE {cond} + ''' + + def get_table_name(self) -> str: + return self._table_name + + def asdict(self) -> dict[str, Any]: + return {field.name:getattr(self, field.name) for + field in dataclasses.fields(self)} + + +@dataclasses.dataclass +class DbUpsertMarkerRowData(DbUpsertRowDataBase): + ''' + Timestamps are only updated once. + ''' + _table_name: typing.ClassVar[str] = 'marker' + _create_stmt: typing.ClassVar[str] = MARKER_INSERT_STMT + _update_set_stmt: typing.ClassVar[str] = '{col}=COALESCE({col}, :{col})' + _conflict_columns: typing.ClassVar[list[str]] = [ + 'fk_account_ek', + 'fk_jid_ek', + 'fk_occupant_ek', + 'marker_id', + ] + _updateable_columns: typing.ClassVar[list[str | tuple[str, str]]] = [ + 'received_ts', + 'displayed_ts', + 'acknowledged_ts', + ] + + account: str + remote_jid: JID + fk_occupant_ek: int | None + marker_id: str + received_ts: float | None | ValueMissingT = VALUE_MISSING + displayed_ts: float | None | ValueMissingT = VALUE_MISSING + acknowledged_ts : float | None | ValueMissingT = VALUE_MISSING + + def get_upsert_stmt(self) -> str: + combined_update_stmt = self._get_updated_column_stmt() + conflict_cols = ', '.join(self._conflict_columns) + + return f''' + {self._create_stmt} + ON CONFLICT({conflict_cols}) DO UPDATE SET + {combined_update_stmt} + RETURNING entitykey + ''' + +@dataclasses.dataclass +class DbUpsertOccupantRowData(DbUpsertRowDataBase): + _table_name: typing.ClassVar = 'occupant' + _create_stmt: typing.ClassVar = OCCUPANT_INSERT_STMT + _conflict_columns: typing.ClassVar[list[str]] = [ + 'fk_account_ek', + 'fk_jid_ek', + 'id', + ] + _updateable_columns: typing.ClassVar[list[str | tuple[str, str]]] = [ + 'timestamp', + ('fk_real_jid_ek', 'real_jid'), + 'nickname', + 'avatar_sha', + ] + + account: str + remote_jid: JID + timestamp: float + id: str + nickname: str | None | ValueMissingT = VALUE_MISSING + real_jid: JID | None | ValueMissingT = VALUE_MISSING + avatar_sha: str | None | ValueMissingT = VALUE_MISSING + + +@dataclasses.dataclass +class DbUpsertSecurityLabelRowData(DbUpsertRowDataBase): + _table_name: typing.ClassVar = 'securitylabel' + _create_stmt: typing.ClassVar = SECURITYLABEL_INSERT_STMT + _conflict_columns: typing.ClassVar[list[str]] = [ + 'fk_account_ek', + 'fk_jid_ek', + 'label_hash', + ] + _updateable_columns: typing.ClassVar[list[str | tuple[str, str]]] = [ + 'timestamp', + 'displaymarking', + 'fgcolor', + 'bgcolor', + ] + + account: str + remote_jid: JID + timestamp: float + label_hash: str + displaymarking: str + fgcolor: str + bgcolor: str + + +@dataclasses.dataclass +class DbUpsertMamArchiveStateRowData(DbUpsertRowDataBase): + _table_name: typing.ClassVar = 'mam_archive_state' + _create_stmt: typing.ClassVar = MAM_ARCHIVE_STATE_INSERT_STMT + _conflict_columns: typing.ClassVar[list[str]] = [ + 'fk_account_ek', + 'fk_jid_ek', + ] + _updateable_columns: typing.ClassVar[list[str | tuple[str, str]]] = [ + 'to_stanza_id', + 'to_stanza_ts', + 'from_stanza_id', + 'from_stanza_ts', + ] + + account: str + remote_jid: JID + to_stanza_id: str | None | ValueMissingT = VALUE_MISSING + to_stanza_ts: datetime | None | ValueMissingT = VALUE_MISSING + from_stanza_id: str | None | ValueMissingT = VALUE_MISSING + from_stanza_ts: datetime | None | ValueMissingT = VALUE_MISSING + + def get_upsert_stmt(self) -> str: + combined_update_stmt = self._get_updated_column_stmt() + conflict_cols = ', '.join(self._conflict_columns) + + return f''' + {self._create_stmt} + ON CONFLICT({conflict_cols}) DO UPDATE SET + {combined_update_stmt} + RETURNING entitykey + ''' + +@dataclasses.dataclass +class DbBaseJoinedRowData: + + _table_classes: typing.ClassVar = { # noqa: RUF008 + 'correction': DbCorrectionRowData, + 'marker': DbMarkerRowData, + 'call': DbCallRowData, + 'oob': DbOOBRowData, + 'encryption': DbEncryptionRowData, + 'reply': DbReplyRowData, + 'error': DbErrorRowData, + 'securitylabel': DbSecurityLabelRowData, + 'moderation': DbModerationRowData, + 'occupant': DbOccupantRowData, + } + + @classmethod + def from_row( + cls, + column_names: list[str], + row: tuple[Any] + ): + + tkv: dict[str, Any] = {} + fields = dataclasses.fields(cls) + field_names = {f.name for f in fields} + + for key, value in zip(column_names, row, strict=True): + if key.startswith('t_'): + key = key.removeprefix('t_') + table, _, column = key.partition('_') + if table not in field_names: + continue + if value is None: + continue + if tkv.get(table) is None: + tkv[table] = {} + tkv[table][column] = value + + elif key in field_names: + tkv[key] = value + + + for field in fields: + name = field.name + if field.type == 'bool': + # Convert integer values to bool + tkv[name] = bool(tkv[name]) + continue + + if name not in cls._table_classes: + continue + + values = tkv.get(name) + if values is None: + tkv[name] = None + continue + + tkv[name] = cls._table_classes[name].from_query(values) + + return cls(**tkv) + + def __str__(self) -> str: + return pprint.pformat(self, width=20) + + +@dataclasses.dataclass +class DbConversationJoinedRowData(DbBaseJoinedRowData): + entitykey: int + account_jid: str + account_ek: int + remote_jid: str + remote_ek: int + resource: str | None + m_type: int + direction: int + timestamp: float + state: int + message_id: str | None + stanza_id: str | None + stable_id: bool + message: str | None + user_delay_ts: float | None + has_markers: bool + is_retracted: bool + has_reactions: bool + has_filetransfers: bool + call: DbCallRowData | None + oob: DbOOBRowData | None + encryption: DbEncryptionRowData | None + reply: DbReplyRowData | None + error: DbErrorRowData | None + securitylabel: DbSecurityLabelRowData | None + correction: DbCorrectionRowData | None + marker: DbMarkerRowData | None + occupants: DbOccupantRowData | None = None + + def get_corrections(self) -> list[DbCorrectionRowData]: + return app.storage.archive.get_corrections(self) + + def get_filetransfers(self) -> list[DbFiletransferRowData]: + return app.storage.archive.get_filetransfers(self.entitykey) + + +@dataclasses.dataclass +class DbChatJoinedData(DbConversationJoinedRowData): + pass + + +@dataclasses.dataclass +class DbPrivateMessageJoinedData(DbConversationJoinedRowData): + pass + + +@dataclasses.dataclass +class DbGroupchatJoinedData(DbConversationJoinedRowData): + moderations: DbModerationRowData | None = None + + +DbConversationJoinedData = (DbChatJoinedData | + DbGroupchatJoinedData | + DbPrivateMessageJoinedData) diff --git a/gajim/common/storage/base.py b/gajim/common/storage/base.py index 107c1b65d..b0ed9ede5 100644 --- a/gajim/common/storage/base.py +++ b/gajim/common/storage/base.py @@ -25,6 +25,7 @@ import sqlite3 import sys import time from collections.abc import Callable +from datetime import datetime from pathlib import Path import nbxmpp.const @@ -41,6 +42,11 @@ from nbxmpp.structs import RosterItem _T = TypeVar('_T') +class ValueMissingT: + pass + +VALUE_MISSING = ValueMissingT() + def timeit(func: Callable[..., _T]) -> Callable[..., _T]: def func_wrapper(self: Any, *args: Any, **kwargs: Any) -> _T: @@ -106,6 +112,15 @@ def _convert_json(json_string: bytes) -> dict[str, Any]: sqlite3.register_converter('JSON', _convert_json) +def _datetime_converter(data: bytes) -> datetime: + return datetime.fromisoformat(data.decode()) + +sqlite3.register_converter('datetime', _datetime_converter) + + +sqlite3.register_adapter(ValueMissingT, lambda _val: None) + + class Encoder(json.JSONEncoder): def default(self, o: Any) -> Any: if isinstance(o, set): @@ -178,15 +193,26 @@ class SqliteStorage: self._migrate_storage() + def get_connection(self) -> sqlite3.Connection: + # Use this only for unittests + return self._con + + def _enable_foreign_keys(self) -> None: + self._con.execute('PRAGMA foreign_keys=ON') + def _set_journal_mode(self, mode: str) -> None: self._con.execute(f'PRAGMA journal_mode={mode}') def _set_synchronous(self, mode: str) -> None: self._con.execute(f'PRAGMA synchronous={mode}') - def _enable_secure_delete(self): + def _enable_secure_delete(self) -> None: self._con.execute('PRAGMA secure_delete=1') + def _run_analyze(self) -> None: + self._con.execute('PRAGMA analysis_limit=400') + self._con.execute('PRAGMA optimize') + @property def user_version(self) -> int: return self._con.execute('PRAGMA user_version').fetchone()[0] @@ -267,5 +293,6 @@ class SqliteStorage: GLib.source_remove(self._commit_source_id) self._commit() + self._run_analyze() self._con.close() del self._con diff --git a/gajim/common/structs.py b/gajim/common/structs.py index 0706d5492..cf96e5f65 100644 --- a/gajim/common/structs.py +++ b/gajim/common/structs.py @@ -39,6 +39,7 @@ from gajim.common.const import MUCJoinedState from gajim.common.const import PresenceShowExt from gajim.common.const import URIType from gajim.common.util.datetime import convert_epoch_to_local_datetime +from gajim.common.storage.archive.const import MessageType _T = TypeVar('_T') @@ -101,7 +102,6 @@ class OutgoingMessage: attention: bool | None = None, correct_id: str | None = None, oob_url: str | None = None, - xhtml: str | None = None, nodes: Any | None = None, play_sound: bool = True ) -> None: @@ -126,9 +126,6 @@ class OutgoingMessage: else: raise ValueError('Unknown message type') - from gajim.common.helpers import AdditionalDataDict - self.additional_data = AdditionalDataDict() - self.subject = subject self.chatstate = chatstate self.marker = marker @@ -138,37 +135,18 @@ class OutgoingMessage: self.control = control self.attention = attention self.correct_id = correct_id - self.oob_url = oob_url - - if oob_url is not None: - self.additional_data.set_value('gajim', 'oob_url', oob_url) - - self.xhtml = xhtml - - if xhtml is not None: - self.additional_data.set_value('gajim', 'xhtml', xhtml) - self.nodes = nodes self.play_sound = play_sound + from gajim.common.helpers import AdditionalDataDict + self.additional_data = AdditionalDataDict() + self.timestamp = None self.message_id = None self.stanza = None self.delayed = None # TODO never set - self.is_loggable = True - - def copy(self): - message = OutgoingMessage(self.account, - self.contact, - self.message, - self.type_) - for name, value in vars(self).items(): - setattr(message, name, value) - message.additional_data = self.additional_data.copy() - return message - @property def jid(self) -> JID: return self.contact.jid @@ -185,9 +163,25 @@ class OutgoingMessage: def is_normal(self) -> bool: return self.type_ == 'normal' + @property + def is_pm(self) -> bool: + from gajim.common.modules.contacts import GroupchatParticipant + return isinstance(self.contact, GroupchatParticipant) + + @property + def message_type(self) -> MessageType: + if self.is_pm: + return MessageType.PM + + if self.type_ == 'chat': + return MessageType.CHAT + + if self.type_ == 'groupchat': + return MessageType.GROUPCHAT + + raise ValueError + def set_sent_timestamp(self) -> None: - if self.is_groupchat: - return self.timestamp = time.time() @property diff --git a/gajim/gtk/chat_banner.py b/gajim/gtk/chat_banner.py index eba89d574..2670de7dd 100644 --- a/gajim/gtk/chat_banner.py +++ b/gajim/gtk/chat_banner.py @@ -36,6 +36,8 @@ from gajim.common.i18n import _ from gajim.common.modules.contacts import BareContact from gajim.common.modules.contacts import GroupchatContact from gajim.common.modules.contacts import GroupchatParticipant +from gajim.common.modules.util import ChatDirection +from gajim.common.storage.archive.const import MessageType from gajim.gtk.builder import get_builder from gajim.gtk.groupchat_voice_requests_button import VoiceRequestsButton @@ -231,14 +233,21 @@ class ChatBanner(Gtk.Box, EventHelper): self._update_account_badge() def _on_message_received(self, event: MessageReceived) -> None: - if (not isinstance(self._contact, BareContact) or - event.jid != self._contact.jid or - not event.msgtxt or - event.properties.is_sent_carbon or - event.resource is None): + assert self._contact is not None + if event.jid != self._contact.jid: + return + + if event.from_mam or event.m_type != MessageType.CHAT: return - resource_contact = self._contact.get_resource(event.resource) + joined_data = event.joined_data + if joined_data.direction == ChatDirection.OUTGOING: + return + + assert joined_data.resource is not None + assert isinstance(self._contact, BareContact) + + resource_contact = self._contact.get_resource(joined_data.resource) if resource_contact.is_phone: self._last_message_from_phone.add(self._contact) else: diff --git a/gajim/gtk/chat_list.py b/gajim/gtk/chat_list.py index 9ef5de286..f03678309 100644 --- a/gajim/gtk/chat_list.py +++ b/gajim/gtk/chat_list.py @@ -34,17 +34,16 @@ from gajim.common.const import RowHeaderType from gajim.common.helpers import get_group_chat_nick from gajim.common.helpers import get_retraction_text from gajim.common.i18n import _ +from gajim.common.modules.util import ChatDirection from gajim.common.setting_values import OpenChatsSettingT +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbConversationJoinedData from gajim.gtk.chat_list_row import ChatListRow from gajim.gtk.util import EventHelper log = logging.getLogger('gajim.gtk.chatlist') -MessageEventT = (events.MessageReceived | - events.GcMessageReceived | - events.MamMessageReceived) - class ChatList(Gtk.ListBox, EventHelper): @@ -314,12 +313,12 @@ class ChatList(Gtk.ListBox, EventHelper): return self._chats.get((account, jid)) is not None def process_event(self, event: events.ChatListEventT) -> None: - if isinstance(event, (events.MessageReceived | - events.MamMessageReceived | - events.GcMessageReceived)): + if isinstance(event, events.MessageReceived): self._on_message_received(event) - elif isinstance(event, events.MessageUpdated): - self._on_message_updated(event) + elif isinstance(event, events.MessageDeleted): + self._on_message_deleted(event) + elif isinstance(event, events.MessageCorrected): + self._on_message_corrected(event) elif isinstance(event, events.MessageModerated): self._on_message_moderated(event) elif isinstance(event, events.PresenceReceived): @@ -560,33 +559,34 @@ class ChatList(Gtk.ListBox, EventHelper): app.app.activate_action('start-chat', GLib.Variant('as', ['', ''])) @staticmethod - def _get_nick_for_received_message(event: MessageEventT) -> str: + def _get_nick_for_received_message( + account: str, + data: DbConversationJoinedData + ) -> str: + nick = _('Me') - if event.properties.type.is_groupchat: - event_nick = event.properties.muc_nickname - our_nick = get_group_chat_nick(event.account, event.jid) + if data.m_type == MessageType.GROUPCHAT: + event_nick = data.resource + assert event_nick is not None + our_nick = get_group_chat_nick(account, data.remote_jid) if event_nick != our_nick: nick = event_nick - else: - con = app.get_client(event.account) - own_jid = con.get_own_jid() - if not own_jid.bare_match(event.properties.from_): - nick = '' + + elif data.direction == ChatDirection.INCOMING: + nick = '' + return nick @staticmethod - def _add_unread(row: ChatListRow, event: MessageEventT) -> None: - if event.properties.is_carbon_message: - if event.properties.carbon.is_sent: - return - - if event.properties.is_from_us(): + def _add_unread(row: ChatListRow, event: events.MessageReceived) -> None: + joined_data = event.joined_data + if joined_data.direction == ChatDirection.OUTGOING: # Last message was from us (1:1), reset counter row.reset_unread() return our_nick = get_group_chat_nick(event.account, event.jid) - if event.properties.muc_nickname == our_nick: + if joined_data.resource == our_nick: # Last message was from us (MUC), reset counter row.reset_unread() return @@ -597,74 +597,74 @@ class ChatList(Gtk.ListBox, EventHelper): control.get_autoscroll()): return - row.add_unread(event.msgtxt) + assert joined_data.message is not None + row.add_unread(joined_data.message) - def _on_message_received(self, event: MessageEventT) -> None: - if not event.msgtxt: - return + def _on_message_received(self, event: events.MessageReceived) -> None: row = self._chats.get((event.account, JID.from_string(event.jid))) if row is None: return - nick = self._get_nick_for_received_message(event) - row.set_nick(nick) - if event.name == 'mam-message-received': - row.set_timestamp(event.properties.mam.timestamp) - else: - row.set_timestamp(event.properties.timestamp) - row.set_stanza_id(event.stanza_id) - row.set_message_id(event.properties.id) + joined_data = event.joined_data + if joined_data.message is None: + return + + nick = self._get_nick_for_received_message(event.account, joined_data) + row.set_nick(nick) + row.set_timestamp(joined_data.timestamp) + row.set_stanza_id(joined_data.stanza_id) + row.set_message_id(joined_data.message_id) row.set_message_text( - event.msgtxt, + joined_data.message, nickname=nick, - additional_data=event.additional_data) + oob=joined_data.oob) self._add_unread(row, event) row.changed() - def _on_message_updated(self, event: events.MessageUpdated) -> None: + def _on_message_deleted(self, event: events.MessageDeleted) -> None: + # TODO + pass + + def _on_message_corrected(self, event: events.MessageCorrected) -> None: row = self._chats.get((event.account, JID.from_string(event.jid))) if row is None: return - if event.correct_id == row.message_id: - row.set_message_text(event.msgtxt, event.nickname) + joined_data = event.joined_data + assert joined_data.correction is not None + if joined_data.message_id == row.message_id: + row.set_message_text( + joined_data.correction.message, + self._get_nick_for_received_message( + event.account, joined_data)) def _on_message_moderated(self, event: events.MessageModerated) -> None: row = self._chats.get((event.account, event.jid)) if row is None: return - if event.moderation.stanza_id == row.stanza_id: + if event.moderation.moderation_id == row.stanza_id: text = get_retraction_text( - event.moderation.moderator_jid, + event.moderation.by, event.moderation.reason) row.set_message_text(text) def _on_message_sent(self, event: events.MessageSent) -> None: - msgtext = event.message - if not msgtext: - return - row = self._chats.get((event.account, JID.from_string(event.jid))) if row is None: return - client = app.get_client(event.account) - own_jid = client.get_own_jid() - - if own_jid.bare_match(event.jid): - nick = '' - else: - nick = _('Me') - row.set_nick(nick) + joined_data = event.joined_data + if joined_data.message is None: + return - # Set timestamp if it's None (outgoing MUC messages) - row.set_timestamp(event.timestamp or time.time()) + row.set_nick(_('Me')) + row.set_timestamp(joined_data.timestamp) row.set_message_text( - event.message, + joined_data.message, nickname=app.nicks[event.account], - additional_data=event.additional_data) + oob=joined_data.oob) row.changed() def _on_presence_received(self, event: events.PresenceReceived) -> None: diff --git a/gajim/gtk/chat_list_row.py b/gajim/gtk/chat_list_row.py index f8b709550..5f1cb1f90 100644 --- a/gajim/gtk/chat_list_row.py +++ b/gajim/gtk/chat_list_row.py @@ -30,9 +30,7 @@ from nbxmpp import JID from gajim.common import app from gajim.common.const import AvatarSize -from gajim.common.const import KindConstant from gajim.common.const import RowHeaderType -from gajim.common.helpers import AdditionalDataDict from gajim.common.helpers import get_group_chat_nick from gajim.common.helpers import get_groupchat_name from gajim.common.helpers import get_retraction_text @@ -42,10 +40,14 @@ from gajim.common.i18n import _ from gajim.common.modules.contacts import BareContact from gajim.common.modules.contacts import GroupchatContact from gajim.common.modules.contacts import GroupchatParticipant +from gajim.common.modules.util import ChatDirection +from gajim.common.preview import DbOOBRowData from gajim.common.preview_helpers import filename_from_uri from gajim.common.preview_helpers import format_geo_coords from gajim.common.preview_helpers import guess_simple_file_type from gajim.common.preview_helpers import split_geo_uri +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbGroupchatJoinedData from gajim.common.storage.draft import DraftStorage from gajim.common.types import ChatContactT @@ -147,64 +149,71 @@ class ChatListRow(Gtk.ListBoxRow): self._ui.unread_label.get_style_context().add_class( 'unread-counter-silent') - self._display_last_conversation_line() + self._display_last_conversation_row() - def _display_last_conversation_line(self) -> None: - line = app.storage.archive.get_last_conversation_line( + def _display_last_conversation_row(self) -> None: + db_row = app.storage.archive.get_last_conversation_row( self.contact.account, self.contact.jid) - if line is None: + if db_row is None: self.show_all() return - if line.message is not None: - message_text = line.message + assert isinstance( + self.contact, + BareContact | GroupchatContact | GroupchatParticipant) + + if db_row.message is not None: + message_text = db_row.message - if line.additional_data is not None: - retracted_by = line.additional_data.get_value( - 'retracted', 'by') - if retracted_by is not None: - reason = line.additional_data.get_value( - 'retracted', 'reason') - message_text = get_retraction_text( - retracted_by, reason) + if db_row.correction is not None: + message_text = db_row.correction.message + + if db_row.is_retracted: + # TODO: message retraction + message_text = _('Message has been retracted') + + if (isinstance(db_row, DbGroupchatJoinedData) and + db_row.moderations is not None): + message_text = get_retraction_text( + db_row.moderations.by, db_row.moderations.reason) me_nickname = None - if line.kind in (KindConstant.CHAT_MSG_SENT, - KindConstant.SINGLE_MSG_SENT): + if (db_row.m_type == MessageType.CHAT and + db_row.direction == ChatDirection.OUTGOING): self.set_nick(_('Me')) me_nickname = app.nicks[self.contact.account] - if line.kind == KindConstant.GC_MSG: + if db_row.m_type == MessageType.GROUPCHAT: + # TODO: not joined when starting (no groupchat nick) our_nick = get_group_chat_nick( self.contact.account, self.contact.jid) - if line.contact_name == our_nick: + + if db_row.resource == our_nick: self.set_nick(_('Me')) me_nickname = our_nick else: - self.set_nick(line.contact_name) - me_nickname = line.contact_name + self.set_nick(db_row.resource or '') + me_nickname = db_row.resource self.set_message_text( message_text, nickname=me_nickname, - additional_data=line.additional_data) + oob=db_row.oob) - self.set_timestamp(line.time) + self.set_timestamp(db_row.timestamp) - self.stanza_id = line.stanza_id - self.message_id = line.message_id + self.stanza_id = db_row.stanza_id + self.message_id = db_row.message_id - if line.kind in (KindConstant.FILE_TRANSFER_INCOMING, - KindConstant.FILE_TRANSFER_OUTGOING): + if db_row.has_filetransfers: self.set_message_text( _('File'), icon_name='text-x-generic-symbolic') - self.set_timestamp(line.time) + self.set_timestamp(db_row.timestamp) - if line.kind in (KindConstant.CALL_INCOMING, - KindConstant.CALL_OUTGOING): + if db_row.call is not None: self.set_message_text( _('Call'), icon_name='call-start-symbolic') - self.set_timestamp(line.time) + self.set_timestamp(db_row.timestamp) self.show_all() @@ -260,7 +269,7 @@ class ChatListRow(Gtk.ListBoxRow): text: str, nickname: str | None = None, icon_name: str | None = None, - additional_data: AdditionalDataDict | None = None + oob: DbOOBRowData | None = None ) -> None: assert isinstance( @@ -277,8 +286,8 @@ class ChatListRow(Gtk.ListBoxRow): icon = None if icon_name is not None: icon = Gio.Icon.new_for_string(icon_name) - if additional_data is not None: - if app.preview_manager.is_previewable(text, additional_data): + if oob is not None: + if app.preview_manager.is_previewable(text, oob): scheme = urlparse(text).scheme if scheme == 'geo': location = split_geo_uri(text) @@ -429,7 +438,7 @@ class ChatListRow(Gtk.ListBoxRow): def _show_draft(self, draft: str | None) -> None: if not draft: self._ui.message_label.get_style_context().remove_class('draft') - self._display_last_conversation_line() + self._display_last_conversation_row() return self.set_nick('') diff --git a/gajim/gtk/chat_list_stack.py b/gajim/gtk/chat_list_stack.py index 2888d1ae4..37c5488f2 100644 --- a/gajim/gtk/chat_list_stack.py +++ b/gajim/gtk/chat_list_stack.py @@ -79,12 +79,11 @@ class ChatListStack(Gtk.Stack, EventHelper): self.register_events([ ('message-received', ged.GUI2, self._on_event), - ('mam-message-received', ged.GUI2, self._on_event), - ('gc-message-received', ged.GUI2, self._on_event), - ('message-updated', ged.GUI2, self._on_event), + ('message-corrected', ged.GUI2, self._on_event), ('message-moderated', ged.GUI2, self._on_event), ('presence-received', ged.GUI2, self._on_event), ('message-sent', ged.GUI2, self._on_event), + ('message-deleted', ged.GUI2, self._on_event), ('file-request-received', ged.GUI2, self._on_event), ('jingle-request-received', ged.GUI2, self._on_event), ]) diff --git a/gajim/gtk/chat_stack.py b/gajim/gtk/chat_stack.py index 7ef7bf57b..bad526099 100644 --- a/gajim/gtk/chat_stack.py +++ b/gajim/gtk/chat_stack.py @@ -16,7 +16,6 @@ from __future__ import annotations import logging import sys -import time from urllib.parse import urlparse from gi.repository import Gdk @@ -37,6 +36,9 @@ from gajim.common.modules.contacts import GroupchatContact from gajim.common.modules.contacts import GroupchatParticipant from gajim.common.preview_helpers import format_geo_coords from gajim.common.preview_helpers import split_geo_uri +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbConversationJoinedData from gajim.common.structs import OutgoingMessage from gajim.common.types import ChatContactT from gajim.common.util.text import remove_invalid_xml_chars @@ -151,7 +153,6 @@ class ChatStack(Gtk.Stack, EventHelper): self.register_events([ ('message-received', 85, self._on_message_received), - ('gc-message-received', 85, self._on_message_received), ('muc-disco-update', 85, self._on_muc_disco_update), ('account-connected', 85, self._on_account_state), ('account-disconnected', 85, self._on_account_state), @@ -373,35 +374,46 @@ class ChatStack(Gtk.Stack, EventHelper): self._update_chat_actions(self._current_contact) def _on_message_received(self, event: events.MessageReceived) -> None: - if not event.msgtxt or event.properties.is_sent_carbon: + if event.from_mam: return - client = app.get_client(event.account) - contact = client.get_module('Contacts').get_contact(event.jid) - if isinstance(contact, GroupchatContact): + if app.window.is_chat_active(event.account, event.jid): + return + + joined_data = event.joined_data + if joined_data.direction == ChatDirection.OUTGOING: + return + + if event.m_type == MessageType.GROUPCHAT: + client = app.get_client(event.account) + contact = client.get_module('Contacts').get_contact( + event.jid, groupchat=True) + assert isinstance(contact, GroupchatContact) # MUC messages may be received after some delay, so make sure we # don't issue notifications for our own messages. self_contact = contact.get_self() if (self_contact is not None and - self_contact.name == event.properties.muc_nickname): + self_contact.name == joined_data.resource): return - if app.window.is_chat_active(event.account, event.jid): - return + self._issue_notification(event.account, joined_data) - self._issue_notification(event) + def _issue_notification( + self, + account: str, + data: DbConversationJoinedData + ) -> None: - def _issue_notification(self, event: events.MessageReceived) -> None: - text = event.msgtxt - tim = event.properties.timestamp - additional_data = event.additional_data - client = app.get_client(event.account) - contact = client.get_module('Contacts').get_contact(event.jid) + text = data.message + assert text is not None + + client = app.get_client(account) + contact = client.get_module('Contacts').get_contact(data.remote_jid) title = _('New message from') is_previewable = app.preview_manager.is_previewable( - text, additional_data) + text, data.oob) if is_previewable: scheme = urlparse(text).scheme if scheme == 'geo': @@ -423,7 +435,7 @@ class ChatStack(Gtk.Stack, EventHelper): if isinstance(contact, GroupchatContact): msg_type = 'group-chat-message' - title += f' {event.resource} ({contact.name})' + title += f' {data.resource} ({contact.name})' assert contact.nickname is not None needs_highlight = helpers.message_needs_highlight( text, contact.nickname, client.get_own_jid().bare) @@ -443,17 +455,13 @@ class ChatStack(Gtk.Stack, EventHelper): title += f' {contact.name} (private in {contact.room.name})' sound = 'first_message_received' - # Is it a history message? Don't want sound-floods when we join. - if tim is not None and time.mktime(time.localtime()) - tim > 1: - sound = None - assert isinstance( contact, BareContact | GroupchatContact | GroupchatParticipant) if app.settings.get('notification_preview_message'): if text.startswith('/me'): name = contact.name if isinstance(contact, GroupchatContact): - name = event.properties.muc_nickname + name = data.resource text = f'* {name} {text[3:]}' else: text = '' diff --git a/gajim/gtk/const.py b/gajim/gtk/const.py index dd17857a0..b2bd9d6e7 100644 --- a/gajim/gtk/const.py +++ b/gajim/gtk/const.py @@ -218,7 +218,7 @@ MAIN_WIN_ACTIONS = [ ('input-clear', None, True), ('show-emoji-chooser', None, True), ('activate-message-selection', 'u', True), - ('delete-message-locally', 'u', True), + ('delete-message-locally', 'a{sv}', True), ('correct-message', None, False), ('copy-message', 's', True), ('retract-message', 'a{sv}', False), diff --git a/gajim/gtk/control.py b/gajim/gtk/control.py index ef2ddbbce..45267049a 100644 --- a/gajim/gtk/control.py +++ b/gajim/gtk/control.py @@ -25,7 +25,6 @@ from gi.repository import GLib from gi.repository import Gtk from nbxmpp import JID from nbxmpp.const import StatusCode -from nbxmpp.modules.security_labels import Displaymarking from nbxmpp.structs import MucSubject from gajim.common import app @@ -33,17 +32,16 @@ from gajim.common import events from gajim.common import ged from gajim.common import helpers from gajim.common import types -from gajim.common.const import KindConstant from gajim.common.const import XmppUriQuery from gajim.common.ged import EventHelper -from gajim.common.helpers import AdditionalDataDict from gajim.common.helpers import get_retraction_text from gajim.common.i18n import _ from gajim.common.modules.contacts import BareContact from gajim.common.modules.contacts import GroupchatContact from gajim.common.modules.contacts import GroupchatParticipant from gajim.common.modules.httpupload import HTTPFileTransfer -from gajim.common.storage.archive import ConversationRow +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.structs import DbConversationJoinedData from gajim.gtk.builder import get_builder from gajim.gtk.conversation.jump_to_end_button import JumpToEndButton @@ -52,7 +50,7 @@ from gajim.gtk.conversation.view import ConversationView from gajim.gtk.groupchat_roster import GroupchatRoster from gajim.gtk.groupchat_state import GroupchatState -HistoryRowT = events.ApplicationEvent | ConversationRow +HistoryRowT = events.ApplicationEvent | DbConversationJoinedData REQUEST_LINES_COUNT = 20 @@ -189,10 +187,14 @@ class ChatControl(EventHelper): # Clear view and reload conversation around timestamp self._scrolled_view.reset() self._scrolled_view.block_signals(True) - before, at_after = app.storage.archive.get_conversation_around( - self.contact.account, self.contact.jid, timestamp) - self._add_messages(before) - self._add_messages(at_after) + messages: list[DbConversationJoinedData] = [] + messages.append(app.storage.archive.get_message_with_entitykey( + log_line_id)) + messages.extend(app.storage.archive.get_conversation_before_after( + self.contact.account, self.contact.jid, True, timestamp, 50)) + messages.extend(app.storage.archive.get_conversation_before_after( + self.contact.account, self.contact.jid, False, timestamp, 50)) + self._add_messages(messages) self._scrolled_view.set_history_complete(False, False) GLib.idle_add(self._scrolled_view.block_signals, False) @@ -274,10 +276,9 @@ class ChatControl(EventHelper): self.register_events([ ('presence-received', ged.GUI2, self._on_presence_received), ('message-sent', ged.GUI2, self._on_message_sent), + ('message-deleted', ged.GUI2, self._on_message_deleted), ('message-received', ged.GUI2, self._on_message_received), - ('mam-message-received', ged.GUI2, self._on_mam_message_received), - ('gc-message-received', ged.GUI2, self._on_gc_message_received), - ('message-updated', ged.GUI2, self._on_message_updated), + ('message-corrected', ged.GUI2, self._on_message_corrected), ('message-moderated', ged.GUI2, self._on_message_moderated), ('receipt-received', ged.GUI2, self._on_receipt_received), ('displayed-received', ged.GUI2, self._on_displayed_received), @@ -322,144 +323,43 @@ class ChatControl(EventHelper): if not self._is_event_processable(event): return - if not event.message: + joined_data = event.joined_data + if joined_data.message is None: return - if self.contact.is_groupchat: + if joined_data.correction is not None: + self._scrolled_view.correct_message(joined_data) return - message_id = event.message_id + self._add_message(joined_data) - if event.label: - displaymarking = event.label.displaymarking - else: - displaymarking = None - - if event.correct_id: - self._scrolled_view.correct_message( - event.correct_id, event.message, self._get_our_nick()) - return - - name = self._get_our_nick() - - self._add_message(text=event.message, - kind='outgoing', - name=name, - timestamp=event.timestamp, - displaymarking=displaymarking, - msg_log_id=event.msg_log_id, - message_id=message_id, - stanza_id=None, - additional_data=event.additional_data) - - def _on_message_received(self, event: events.MessageReceived) -> None: + def _on_message_deleted(self, event: events.MessageDeleted) -> None: if not self._is_event_processable(event): return - if self.is_groupchat: - return - - if not event.msgtxt: - return - - kind = 'incoming' - name = self.contact.name - if event.properties.is_sent_carbon: - kind = 'outgoing' - name = self._get_our_nick() - - self._add_message(text=event.msgtxt, - kind=kind, - name=name, - timestamp=event.properties.timestamp, - displaymarking=event.displaymarking, - msg_log_id=event.msg_log_id, - message_id=event.properties.id, - stanza_id=event.stanza_id, - additional_data=event.additional_data) - - def _on_mam_message_received(self, - event: events.MamMessageReceived) -> None: - - if not self._is_event_processable(event): - return - - if isinstance(self.contact, GroupchatContact): - - if not event.properties.type.is_groupchat: - return - if event.archive_jid != self.contact.jid: - return - - nickname = event.properties.muc_nickname - if nickname == self.contact.nickname: - kind = 'outgoing' - else: - kind = 'incoming' - - else: + self.remove_message(event.entitykey) - if event.properties.is_muc_pm: - if not event.properties.jid == self.contact.jid: - return - else: - if not event.properties.jid.bare_match(self.contact.jid): - return - - kind = 'incoming' - nickname = self.contact.name - if event.kind == KindConstant.CHAT_MSG_SENT: - kind = 'outgoing' - nickname = self._get_our_nick() - - self._add_message(text=event.msgtxt, - kind=kind, - name=nickname, - timestamp=event.properties.mam.timestamp, - displaymarking=event.displaymarking, - msg_log_id=event.msg_log_id, - message_id=event.properties.id, - stanza_id=event.stanza_id, - additional_data=event.additional_data) - - def _on_gc_message_received(self, event: events.GcMessageReceived) -> None: + def _on_message_received(self, event: events.MessageReceived) -> None: if not self._is_event_processable(event): return - assert isinstance(self.contact, GroupchatContact) + self._add_message(event.joined_data) - nickname = event.properties.muc_nickname - if nickname == self.contact.nickname: - kind = 'outgoing' - else: - kind = 'incoming' - - self._add_message(text=event.msgtxt, - kind=kind, - name=nickname, - timestamp=event.properties.timestamp, - displaymarking=event.displaymarking, - msg_log_id=event.msg_log_id, - message_id=event.properties.id, - stanza_id=event.stanza_id, - additional_data=event.additional_data) - - def _on_message_updated(self, event: events.MessageUpdated) -> None: + def _on_message_corrected(self, event: events.MessageCorrected) -> None: if not self._is_event_processable(event): return - self._scrolled_view.correct_message( - event.correct_id, event.msgtxt, event.nickname) + self._scrolled_view.correct_message(event.joined_data) def _on_message_moderated(self, event: events.MessageModerated) -> None: if not self._is_event_processable(event): return text = get_retraction_text( - event.moderation.moderator_jid, + event.moderation.by, event.moderation.reason) self._scrolled_view.show_message_retraction( - event.moderation.stanza_id, text) + event.moderation.moderation_id, text) def _on_receipt_received(self, event: events.ReceiptReceived) -> None: if not self._is_event_processable(event): @@ -582,104 +482,35 @@ class ChatControl(EventHelper): if self._allow_add_message(): self._scrolled_view.add_call_message(event=event) - def _add_message(self, - *, - text: str, - kind: str, - name: str, - timestamp: float, - displaymarking: Displaymarking | None, - msg_log_id: int | None, - message_id: str | None, - stanza_id: str | None, - additional_data: AdditionalDataDict | None - ) -> None: - - if additional_data is None: - additional_data = AdditionalDataDict() - + def _add_message(self, db_row: DbConversationJoinedData) -> None: + # TODO: Unify with _add_db_row() if self._allow_add_message(): - self._scrolled_view.add_message( - text, - kind, - name, - timestamp, - display_marking=displaymarking, - message_id=message_id, - stanza_id=stanza_id, - log_line_id=msg_log_id, - additional_data=additional_data) + self._scrolled_view.add_message_from_db(db_row) if not self._scrolled_view.get_autoscroll(): - if kind == 'outgoing': + if db_row.direction == ChatDirection.OUTGOING: self._scrolled_view.scroll_to_end() else: self._jump_to_end_button.add_unread_count() else: self._jump_to_end_button.add_unread_count() - def _add_messages(self, messages: list[ConversationRow]): + def _add_db_row(self, db_row: DbConversationJoinedData): + if db_row.has_filetransfers: + self._scrolled_view.add_jingle_file_transfer(db_row=db_row) + return + + if db_row.call is not None: + self._scrolled_view.add_call_message(db_row=db_row) + return + + self._scrolled_view.add_message_from_db(db_row) + + def _add_messages(self, messages: list[DbConversationJoinedData]): for msg in messages: - if msg.kind in (KindConstant.FILE_TRANSFER_INCOMING, - KindConstant.FILE_TRANSFER_OUTGOING): - assert msg.additional_data is not None - if msg.additional_data.get_value('gajim', 'type') == 'jingle': - self._scrolled_view.add_jingle_file_transfer( - db_message=msg) - continue - - if msg.kind in (KindConstant.CALL_INCOMING, - KindConstant.CALL_OUTGOING): - self._scrolled_view.add_call_message(db_message=msg) - continue - - if not msg.message: - continue - - message_text = msg.message - - contact_name = msg.contact_name - kind = 'incoming' - if msg.kind in ( - KindConstant.SINGLE_MSG_RECV, KindConstant.CHAT_MSG_RECV): - kind = 'incoming' - contact_name = self.contact.name - elif msg.kind == KindConstant.GC_MSG: - kind = 'incoming' - if contact_name is None: - # Fall back to MUC name if contact name is None - # (may be the case for service messages from the MUC) - contact_name = self.contact.name - elif msg.kind in ( - KindConstant.SINGLE_MSG_SENT, KindConstant.CHAT_MSG_SENT): - kind = 'outgoing' - contact_name = self._get_our_nick() - else: - log.warning('kind attribute could not be processed' - 'while adding message') - - assert contact_name is not None - - if msg.additional_data is not None: - retracted_by = msg.additional_data.get_value('retracted', 'by') - if retracted_by is not None: - reason = msg.additional_data.get_value( - 'retracted', 'reason') - message_text = get_retraction_text(retracted_by, reason) - - self._scrolled_view.add_message( - message_text, - kind, - contact_name, - msg.time, - additional_data=msg.additional_data, - message_id=msg.message_id, - stanza_id=msg.stanza_id, - log_line_id=msg.log_line_id, - marker=msg.marker, - error=msg.error) - - def _request_messages(self, before: bool) -> list[ConversationRow]: + self._add_db_row(msg) + + def _request_messages(self, before: bool) -> list[DbConversationJoinedData]: if before: row = self._scrolled_view.get_first_message_row() else: @@ -772,15 +603,13 @@ class ChatControl(EventHelper): self._scrolled_view.block_signals(False) @staticmethod - def _sort_request_rows(messages: list[ConversationRow], + def _sort_request_rows(messages: list[DbConversationJoinedData], event_rows: list[events.ApplicationEvent], before: bool ) -> list[HistoryRowT]: def sort_func(obj: HistoryRowT) -> float: - if isinstance(obj, events.ApplicationEvent): - return obj.timestamp # pyright: ignore - return obj.time + return obj.timestamp # pyright: ignore rows = messages + event_rows rows.sort(key=sort_func, reverse=before) diff --git a/gajim/gtk/conversation/rows/base.py b/gajim/gtk/conversation/rows/base.py index b7a3f26dc..09b33c58c 100644 --- a/gajim/gtk/conversation/rows/base.py +++ b/gajim/gtk/conversation/rows/base.py @@ -20,6 +20,7 @@ from gi.repository import Gtk from gi.repository import Pango from gajim.common import app +from gajim.common.storage.archive.const import ChatDirection class BaseRow(Gtk.ListBoxRow): @@ -27,9 +28,11 @@ class BaseRow(Gtk.ListBoxRow): Gtk.ListBoxRow.__init__(self) self._account = account self._client = app.get_client(account) + self.type: str = '' - self.timestamp: datetime = datetime.fromtimestamp(0) + self.timestamp = datetime.fromtimestamp(0) self.kind: str = '' + self.direction = ChatDirection.INCOMING self.name: str = '' self.message_id: str | None = None self.log_line_id: int | None = None diff --git a/gajim/gtk/conversation/rows/call.py b/gajim/gtk/conversation/rows/call.py index a6cda74f0..8bd41085b 100644 --- a/gajim/gtk/conversation/rows/call.py +++ b/gajim/gtk/conversation/rows/call.py @@ -23,12 +23,12 @@ from gi.repository import Gtk from gajim.common import app from gajim.common import types from gajim.common.const import AvatarSize -from gajim.common.const import KindConstant from gajim.common.events import JingleRequestReceived from gajim.common.i18n import _ from gajim.common.jingle_session import JingleSession from gajim.common.modules.contacts import BareContact -from gajim.common.storage.archive import ConversationRow +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.structs import DbConversationJoinedData from gajim.gtk.conversation.rows.base import BaseRow from gajim.gtk.conversation.rows.widgets import DateTimeLabel @@ -41,7 +41,7 @@ class CallRow(BaseRow): account: str, contact: types.BareContact, event: JingleRequestReceived | None = None, - db_message: ConversationRow | None = None + db_row: DbConversationJoinedData | None = None ) -> None: BaseRow.__init__(self, account) @@ -49,8 +49,8 @@ class CallRow(BaseRow): self._client = app.get_client(account) - if db_message is not None: - timestamp = db_message.time + if db_row is not None: + timestamp = db_row.timestamp else: timestamp = time.time() self.timestamp = datetime.fromtimestamp(timestamp) @@ -58,17 +58,15 @@ class CallRow(BaseRow): self._contact = contact self._event = event - self._db_message = db_message + self._db_row = db_row self._session: JingleSession | None = None - if db_message is not None: - assert db_message.additional_data is not None - sid = db_message.additional_data.get_value('gajim', 'sid') + if db_row is not None and db_row.call is not None: module = self._client.get_module('Jingle') self._session = module.get_jingle_session( - str(self._contact.jid), sid) - self.log_line_id = db_message.log_line_id + str(self._contact.jid), db_row.call.sid) + self.log_line_id = db_row.entitykey self._avatar_placeholder = Gtk.Box() self._avatar_placeholder.set_size_request(AvatarSize.ROSTER, -1) @@ -122,8 +120,8 @@ class CallRow(BaseRow): assert isinstance(contact, BareContact) is_self = True - if self._db_message is not None: - if self._db_message.kind == KindConstant.CALL_INCOMING: + if self._db_row is not None: + if self._db_row.direction == ChatDirection.INCOMING: contact = self._contact is_self = True else: diff --git a/gajim/gtk/conversation/rows/file_transfer_jingle.py b/gajim/gtk/conversation/rows/file_transfer_jingle.py index 94ae7e601..ab84bdb66 100644 --- a/gajim/gtk/conversation/rows/file_transfer_jingle.py +++ b/gajim/gtk/conversation/rows/file_transfer_jingle.py @@ -26,7 +26,6 @@ from gi.repository import Gtk from gajim.common import app from gajim.common import ged from gajim.common.const import AvatarSize -from gajim.common.const import KindConstant from gajim.common.events import FileCompleted from gajim.common.events import FileError from gajim.common.events import FileHashError @@ -43,7 +42,8 @@ from gajim.common.helpers import open_file from gajim.common.helpers import show_in_folder from gajim.common.i18n import _ from gajim.common.modules.contacts import BareContact -from gajim.common.storage.archive import ConversationRow +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.structs import DbConversationJoinedData from gajim.gtk.builder import get_builder from gajim.gtk.conversation.rows.base import BaseRow @@ -61,14 +61,14 @@ class FileTransferJingleRow(BaseRow): account: str, contact: BareContact, event: TransferEventT | None = None, - db_message: ConversationRow | None = None + db_row: DbConversationJoinedData | None = None ) -> None: BaseRow.__init__(self, account) self.type = 'file-transfer' - if db_message is not None: - timestamp = db_message.time + if db_row is not None: + timestamp = db_row.timestamp else: timestamp = time.time() self.timestamp = datetime.fromtimestamp(timestamp) @@ -76,14 +76,14 @@ class FileTransferJingleRow(BaseRow): self._contact = contact - if db_message is not None: - assert db_message.additional_data is not None - sid = db_message.additional_data.get_value('gajim', 'sid') - assert sid is not None - self._file_props = FilesProp.getFilePropBySid(sid) + if db_row is not None and db_row.has_filetransfers: + filetransfers = db_row.get_filetransfers() + file_transfer = filetransfers[0] # TODO: Proper processing + self._file_props = FilesProp.getFilePropBySid(file_transfer.source) if self._file_props is None: - log.debug('File prop not found for SID: %s', sid) - self.log_line_id = db_message.log_line_id + log.debug( + 'File prop not found for SID: %s', file_transfer.source) + self.log_line_id = db_row.entitykey else: assert event is not None self._file_props = event.file_props @@ -99,8 +99,8 @@ class FileTransferJingleRow(BaseRow): avatar_placeholder.set_valign(Gtk.Align.START) self.grid.attach(avatar_placeholder, 0, 0, 1, 1) - if db_message is not None: - if db_message.kind == KindConstant.FILE_TRANSFER_INCOMING: + if db_row is not None: + if db_row.direction == ChatDirection.INCOMING: contact = self._contact is_self = True else: @@ -148,7 +148,7 @@ class FileTransferJingleRow(BaseRow): self.show_all() - if db_message is not None: + if db_row is not None: self._reconstruct_transfer() else: assert event is not None diff --git a/gajim/gtk/conversation/rows/message.py b/gajim/gtk/conversation/rows/message.py index 5dba1c9c6..ac0b236bb 100644 --- a/gajim/gtk/conversation/rows/message.py +++ b/gajim/gtk/conversation/rows/message.py @@ -19,27 +19,29 @@ from datetime import datetime from datetime import timedelta import cairo -from gi.repository import GdkPixbuf from gi.repository import GLib from gi.repository import Gtk -from gi.repository import Pango -from nbxmpp.errors import StanzaError -from nbxmpp.modules.security_labels import Displaymarking -from nbxmpp.structs import CommonError from gajim.common import app from gajim.common.const import AvatarSize from gajim.common.const import Trust from gajim.common.const import TRUST_SYMBOL_DATA -from gajim.common.helpers import AdditionalDataDict from gajim.common.helpers import get_group_chat_nick +from gajim.common.helpers import get_retraction_text from gajim.common.helpers import message_needs_highlight -from gajim.common.helpers import to_user_string from gajim.common.i18n import _ from gajim.common.i18n import is_rtl_text +from gajim.common.modules.contacts import BareContact from gajim.common.modules.contacts import GroupchatContact from gajim.common.modules.contacts import GroupchatParticipant from gajim.common.modules.contacts import ResourceContact +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbConversationJoinedData +from gajim.common.storage.archive.structs import DbEncryptionRowData +from gajim.common.storage.archive.structs import DbGroupchatJoinedData +from gajim.common.storage.archive.structs import DbSecurityLabelRowData from gajim.common.types import ChatContactT from gajim.gtk.conversation.message_widget import MessageWidget @@ -59,136 +61,179 @@ MERGE_TIMEFRAME = timedelta(seconds=120) class MessageRow(BaseRow): def __init__(self, - account: str, contact: ChatContactT, - message_id: str | None, - stanza_id: str | None, - timestamp: float, - kind: str, - name: str, - text: str, - additional_data: AdditionalDataDict | None = None, - display_marking: Displaymarking | None = None, - marker: str | None = None, - error: CommonError | StanzaError | None = None, - log_line_id: int | None = None) -> None: - - BaseRow.__init__(self, account) - self.type = 'chat' - self.timestamp = datetime.fromtimestamp(timestamp) - self.db_timestamp = timestamp - self.message_id = message_id - self.stanza_id = stanza_id - self.log_line_id = log_line_id - self.kind = kind - self.name = name - self.text = text - self.additional_data = additional_data - self.display_marking = display_marking + db_row: DbConversationJoinedData + ) -> None: + BaseRow.__init__(self, contact.account) self.set_selectable(True) - - self._account = account + self.type = 'chat' self._contact = contact + self._db_row = db_row - self._is_groupchat: bool = False - if contact.is_groupchat: - self._is_groupchat = True + self._avatar_box = AvatarBox(contact) - self._has_receipt: bool = marker == 'received' - self._has_displayed: bool = marker == 'displayed' + self._meta_box = Gtk.Box(spacing=6) + self._meta_box.set_hexpand(True) - # Keep original text for message correction - self._original_text: str = text + self._bottom_box = Gtk.Box(spacing=6) - if self._is_groupchat: - our_nick = get_group_chat_nick(self._account, self._contact.jid) - from_us = name == our_nick - else: - from_us = kind == 'outgoing' - - if app.preview_manager.is_previewable(text, additional_data): - muc_context = None - if isinstance(self._contact, - GroupchatContact | GroupchatParticipant): - muc_context = self._contact.muc_context - self._message_widget = PreviewWidget(account) - app.preview_manager.create_preview( - text, self._message_widget, from_us, muc_context) - else: - self._message_widget = MessageWidget(account) - self._message_widget.add_with_styling(text, nickname=name) - if self._is_groupchat: - our_nick = get_group_chat_nick( - self._account, self._contact.jid) - if name != our_nick: - self._check_for_highlight(text) + self.grid.attach(self._avatar_box, 0, 0, 1, 2) + self.grid.attach(self._meta_box, 1, 0, 1, 1) + self.grid.attach(self._bottom_box, 1, 1, 1, 1) - if self._contact.jid == self._client.get_own_jid().bare: - name = _('Me') + self.update_with_content(db_row) - name_widget = NicknameLabel(name, from_us) + @classmethod + def from_db_row(cls, + contact: ChatContactT, + db_row: DbConversationJoinedData + ) -> MessageRow: - self._meta_box = Gtk.Box(spacing=6) - self._meta_box.set_hexpand(True) - self._meta_box.pack_start(name_widget, False, True, 0) - timestamp_label = DateTimeLabel(self.timestamp) - self._meta_box.pack_start(timestamp_label, False, True, 0) + return cls(contact, db_row) - if additional_data is not None: - encryption_img = self._get_encryption_image(additional_data) - if encryption_img: - self._meta_box.pack_start(encryption_img, False, True, 0) + def update_with_content(self, db_row: DbConversationJoinedData) -> None: + self._db_row = db_row - self._add_security_label(display_marking) + self.set_merged(False) + self.get_style_context().remove_class('retracted-message') + self.get_style_context().remove_class('gajim-mention-highlight') - self._message_icons = MessageIcons() + for widget in self._meta_box.get_children(): + widget.destroy() + + for widget in self._bottom_box.get_children(): + widget.destroy() + + self.timestamp = datetime.fromtimestamp(db_row.timestamp) + self.db_timestamp = db_row.timestamp + self.message_id = db_row.message_id + self.stanza_id = db_row.stanza_id + self.log_line_id = db_row.entitykey + self.direction = ChatDirection(db_row.direction) + self.encryption = db_row.encryption + self.securitylabel = db_row.securitylabel + self.text = db_row.message or '' + self._original_text = self.text # Message corrections + + self.name = self._get_contact_name(db_row, self._contact) - if additional_data is not None: - if additional_data.get_value('retracted', 'by') is not None: - self.get_style_context().add_class('retracted-message') - - correction_original = additional_data.get_value( - 'corrected', 'original_text') - if correction_original is not None: - self._original_text = correction_original - self._message_icons.set_correction_icon_visible(True) - original_text = textwrap.fill(correction_original, - width=150, - max_lines=10, - placeholder='…') - self._message_icons.set_correction_tooltip( - _('Message corrected. Original message:' - '\n%s') % original_text) - - if error is not None: - self.set_error(to_user_string(error)) - - if marker is not None: - if marker in ('received', 'displayed'): - self.set_receipt() + avatar = self._get_avatar(self.direction, self.name) + self._avatar_box.set_from_surface(avatar) + self._avatar_box.set_name(self.name) + + self._meta_box.pack_start(NicknameLabel( + self.name, self._message_from_us), False, True, 0) + self._meta_box.pack_start( + DateTimeLabel(self.timestamp), False, True, 0) + self._message_icons = MessageIcons() self._meta_box.pack_start(self._message_icons, False, True, 0) - avatar = self._get_avatar(kind, name) - self._avatar_box = AvatarBox(self._contact, name, avatar) + if app.preview_manager.is_previewable(self.text, db_row.oob): + self._message_widget = PreviewWidget(self._contact.account) + app.preview_manager.create_preview( + self.text, + self._message_widget, + self._message_from_us, + self._muc_context) + else: + self._message_widget = MessageWidget(self._contact.account) + self._message_widget.add_with_styling(self.text, nickname=self.name) + if self._contact.is_groupchat and not self._message_from_us: + self._apply_highlight(self.text) - self._bottom_box = Gtk.Box(spacing=6) - self._bottom_box.add(self._message_widget) + self._bottom_box.pack_start(self._message_widget, True, True, 0) - if is_rtl_text(text): - self._bottom_box.set_halign(Gtk.Align.END) - self._message_widget.set_direction(Gtk.TextDirection.RTL) + self._set_text_direction(self.text) more_menu_button = MoreMenuButton(self._on_more_menu_button_clicked) self._bottom_box.pack_end(more_menu_button, False, True, 0) - self.grid.attach(self._avatar_box, 0, 0, 1, 2) - self.grid.attach(self._meta_box, 1, 0, 1, 1) - self.grid.attach(self._bottom_box, 1, 1, 1, 1) + if db_row.correction is not None: + self.set_correction(db_row.correction.message, self.name) + + if (isinstance(db_row, DbGroupchatJoinedData) and + db_row.moderations is not None): + self.set_retracted(get_retraction_text( + db_row.moderations.by, db_row.moderations.reason)) + + encryption_data = self._get_encryption_data(db_row.encryption) + if encryption_data is not None: + self._message_icons.set_encrytion_icon_data(*encryption_data) + self._message_icons.set_encryption_icon_visible(True) + + sec_label_data = self._get_security_labels_data(db_row.securitylabel) + if sec_label_data is not None: + self._message_icons.set_security_label_data(*sec_label_data) + self._message_icons.set_security_label_visible(True) + + if (not self._contact.is_groupchat and + db_row.direction == ChatDirection.OUTGOING and + db_row.marker is not None and + db_row.marker.received_ts is not None): + self.show_receipt(True) + + if (self._contact.is_groupchat and + db_row.direction == ChatDirection.OUTGOING): + self.show_group_chat_message_state(MessageState(db_row.state)) + + if db_row.error is not None: + if db_row.error.text is not None: + error_text = f'{db_row.error.text} ({db_row.error.condition})' + else: + error_text = db_row.error.condition + self.show_error(error_text) self.show_all() + def _set_text_direction(self, text: str) -> None: + if is_rtl_text(text): + self._bottom_box.set_halign(Gtk.Align.END) + self._message_widget.set_direction(Gtk.TextDirection.RTL) + else: + self._bottom_box.set_halign(Gtk.Align.FILL) + self._message_widget.set_direction(Gtk.TextDirection.LTR) + + @staticmethod + def _get_contact_name( + db_row: DbConversationJoinedData, + contact: ChatContactT + ) -> str: + + if isinstance(contact, BareContact) and contact.is_self: + return _('Me') + + if db_row.m_type == MessageType.CHAT: + if db_row.direction == ChatDirection.INCOMING: + return contact.name + return app.nicks[contact.account] + + elif db_row.m_type == MessageType.GROUPCHAT: + resource = db_row.resource + if resource is None: + # Fall back to MUC name if contact name is None + # (may be the case for service messages from the MUC) + return contact.name + return resource + + else: + raise ValueError + + @property + def _muc_context(self) -> str | None: + if isinstance(self._contact, + GroupchatContact | GroupchatParticipant): + return self._contact.muc_context + return None + + @property + def _message_from_us(self) -> bool: + if self._contact.is_groupchat: + our_nick = get_group_chat_nick(self._account, self._contact.jid) + return self.name == our_nick + return self.direction == ChatDirection.OUTGOING + def _on_more_menu_button_clicked(self, button: Gtk.Button) -> None: menu = get_chat_row_menu( self._contact, @@ -210,51 +255,49 @@ class MessageRow(BaseRow): if isinstance(self._message_widget, MessageWidget): self._message_widget.set_selectable(True) - def _add_security_label(self, - display_marking: Displaymarking | None - ) -> None: + def _get_security_labels_data( + self, + security_labels: DbSecurityLabelRowData | None + ) -> tuple[str, str] | None: - if display_marking is None: - return + if (security_labels is None or security_labels.displaymarking is None): + return None if not app.settings.get_account_setting(self._account, 'enable_security_labels'): - return + return None - label_text = GLib.markup_escape_text(display_marking.name) - if label_text: - display_marking_label = Gtk.Label() - display_marking_label.set_ellipsize(Pango.EllipsizeMode.END) - display_marking_label.set_max_width_chars(30) - display_marking_label.set_tooltip_text(label_text) - bgcolor = display_marking.bgcolor - fgcolor = display_marking.fgcolor - label_text = ( + displaymarking = GLib.markup_escape_text(security_labels.displaymarking) + if displaymarking: + bgcolor = security_labels.bgcolor + fgcolor = security_labels.fgcolor + markup = ( f'<span size="small" bgcolor="{bgcolor}" ' - f'fgcolor="{fgcolor}"><tt>{label_text}</tt></span>') - display_marking_label.set_markup(label_text) - self._meta_box.add(display_marking_label) + f'fgcolor="{fgcolor}"><tt>{displaymarking}</tt></span>') + return displaymarking, markup + return None - def _check_for_highlight(self, text: str) -> None: + def _apply_highlight(self, text: str) -> None: assert isinstance(self._contact, GroupchatContact) if self._contact.nickname is None: return - needs_highlight = message_needs_highlight( - text, - self._contact.nickname, - self._client.get_own_jid().bare) - if needs_highlight: + if message_needs_highlight( + text, self._contact.nickname, self._client.get_own_jid().bare): self.get_style_context().add_class( 'gajim-mention-highlight') - def _get_avatar(self, kind: str, name: str) -> cairo.ImageSurface | None: + def _get_avatar(self, + direction: ChatDirection, + name: str + ) -> cairo.ImageSurface | None: + scale = self.get_scale_factor() if isinstance(self._contact, GroupchatContact): contact = self._contact.get_resource(name) return contact.get_avatar(AvatarSize.ROSTER, scale, add_show=False) - if kind == 'outgoing': + if direction == ChatDirection.OUTGOING: contact = self._client.get_module('Contacts').get_contact( str(self._client.get_own_jid().bare)) else: @@ -262,39 +305,30 @@ class MessageRow(BaseRow): assert not isinstance(contact, GroupchatContact | ResourceContact) avatar = contact.get_avatar(AvatarSize.ROSTER, scale, add_show=False) - assert not isinstance(avatar, GdkPixbuf.Pixbuf) return avatar def is_same_sender(self, message: MessageRow) -> bool: return message.name == self.name def is_same_encryption(self, message: MessageRow) -> bool: - m_add_data = message.additional_data - if m_add_data is None: - m_add_data = AdditionalDataDict() - s_add_data = self.additional_data - if s_add_data is None: - s_add_data = AdditionalDataDict() - - message_details = self._get_encryption_details(m_add_data) - own_details = self._get_encryption_details(s_add_data) - if message_details is None and own_details is None: + c_enc = message.encryption + o_enc = self.encryption + if c_enc is None and o_enc is None: return True - if message_details is not None and own_details is not None: - # *_details contains encryption method's name, fingerprint, trust - m_name, _, m_trust = message_details - o_name, _, o_trust = own_details - if m_name == o_name and m_trust == o_trust: + if c_enc is not None and o_enc is not None: + if c_enc.protocol == o_enc.protocol and c_enc.trust == o_enc.trust: return True + return False - def is_same_display_marking(self, message: MessageRow) -> bool: - if message.display_marking == self.display_marking: + def is_same_securitylabels(self, message: MessageRow) -> bool: + if message.securitylabel == self.securitylabel: return True - if (message.display_marking is not None and - self.display_marking is not None): - if message.display_marking.name == self.display_marking.name: + if (message.securitylabel is not None and + self.securitylabel is not None): + if (message.securitylabel.displaymarking == + self.securitylabel.displaymarking): return True return False @@ -305,105 +339,66 @@ class MessageRow(BaseRow): return False if not self.is_same_encryption(message): return False - if not self.is_same_display_marking(message): + if not self.is_same_securitylabels(message): + return False + if self._db_row.correction is not None: return False return abs(message.timestamp - self.timestamp) < MERGE_TIMEFRAME def get_text(self) -> str: return self._message_widget.get_text() - def _get_encryption_image(self, - additional_data: AdditionalDataDict, - ) -> Gtk.Image | None: + def _get_encryption_data(self, + encryption_data: DbEncryptionRowData | None, + ) -> tuple[str, str, str] | None: - details = self._get_encryption_details(additional_data) - if details is None: - # Message was not encrypted - if not self._contact.settings.get('encryption'): + contact_encryption = self._contact.settings.get('encryption') + if encryption_data is None: + if not contact_encryption: return None + icon = 'channel-insecure-symbolic' color = 'unencrypted-color' tooltip = _('Not encrypted') else: - name, fingerprint, trust = details - tooltip = _('Encrypted (%s)') % (name) - if trust is None: - # The encryption plugin did not pass trust information - icon = 'channel-secure-symbolic' - color = 'encrypted-color' - else: - icon, trust_tooltip, color = TRUST_SYMBOL_DATA[Trust(trust)] - tooltip = f'{tooltip}\n{trust_tooltip}' - if fingerprint is not None: - fingerprint = format_fingerprint(fingerprint) + tooltip = _('Encrypted (%s)') % (encryption_data.protocol) + icon, trust_tooltip, color = TRUST_SYMBOL_DATA[ + Trust(encryption_data.trust)] + tooltip = f'{tooltip}\n{trust_tooltip}' + if encryption_data.key != 'Unknown': + fingerprint = format_fingerprint(encryption_data.key) tooltip = f'{tooltip}\n<tt>{fingerprint}</tt>' - image = Gtk.Image.new_from_icon_name(icon, Gtk.IconSize.MENU) - image.set_tooltip_markup(tooltip) - image.get_style_context().add_class(color) - image.show() - return image - - @staticmethod - def _get_encryption_details( - additional_data: AdditionalDataDict - ) -> tuple[str, str | None, Trust | None] | None: - - name = additional_data.get_value('encrypted', 'name') - if name is None: - return None + return icon, color, tooltip - fingerprint = additional_data.get_value('encrypted', 'fingerprint') - trust_data = additional_data.get_value('encrypted', 'trust') + def show_receipt(self, show: bool) -> None: + self._message_icons.set_receipt_icon_visible(show) - if trust_data is not None: - trust_data = Trust(trust_data) - return name, fingerprint, trust_data + def show_group_chat_message_state(self, state: MessageState) -> None: + self._message_icons.set_group_chat_message_state_icon(state) - @property - def has_receipt(self) -> bool: - return self._has_receipt - - @property - def has_displayed(self) -> bool: - return self._has_displayed - - def set_receipt(self) -> None: - self._has_receipt = True - self._message_icons.set_receipt_icon_visible(True) - - def set_displayed(self) -> None: - self._has_displayed = True + def show_error(self, tooltip: str) -> None: + self._message_icons.set_error_icon_visible(True) + self._message_icons.set_error_tooltip(tooltip) def set_retracted(self, text: str) -> None: + self.text = text + if isinstance(self._message_widget, PreviewWidget): self._message_widget.destroy() self._message_widget = MessageWidget(self._account) self._bottom_box.pack_start(self._message_widget, True, True, 0) - if is_rtl_text(text): - self._bottom_box.set_halign(Gtk.Align.END) - self._message_widget.set_direction(Gtk.TextDirection.RTL) - else: - self._bottom_box.set_halign(Gtk.Align.FILL) - self._message_widget.set_direction(Gtk.TextDirection.LTR) + self._set_text_direction(text) self._message_widget.add_with_styling(text) self.get_style_context().add_class('retracted-message') def set_correction(self, text: str, nickname: str | None) -> None: + self.text = text + self.show_receipt(False) if not isinstance(self._message_widget, PreviewWidget): self._message_widget.add_with_styling(text, nickname) - - if is_rtl_text(text): - self._bottom_box.set_halign(Gtk.Align.END) - self._message_widget.set_direction(Gtk.TextDirection.RTL) - else: - self._bottom_box.set_halign(Gtk.Align.FILL) - self._message_widget.set_direction(Gtk.TextDirection.LTR) - - self._has_receipt = False - self._message_icons.set_receipt_icon_visible(False) - self._message_icons.set_correction_icon_visible(True) + self._set_text_direction(text) original_text = textwrap.fill(self._original_text, width=150, @@ -411,13 +406,10 @@ class MessageRow(BaseRow): placeholder='…') self._message_icons.set_correction_tooltip( _('Message corrected. Original message:\n%s') % original_text) - - def set_error(self, tooltip: str) -> None: - self._message_icons.set_error_icon_visible(True) - self._message_icons.set_error_tooltip(tooltip) + self._message_icons.set_correction_icon_visible(True) def update_avatar(self) -> None: - avatar = self._get_avatar(self.kind, self.name) + avatar = self._get_avatar(self.direction, self.name) self._avatar_box.set_from_surface(avatar) def set_merged(self, merged: bool) -> None: diff --git a/gajim/gtk/conversation/rows/widgets.py b/gajim/gtk/conversation/rows/widgets.py index 99f556998..983b49855 100644 --- a/gajim/gtk/conversation/rows/widgets.py +++ b/gajim/gtk/conversation/rows/widgets.py @@ -27,8 +27,11 @@ from gi.repository import Pango from gajim.common import app from gajim.common.const import AvatarSize +from gajim.common.const import TRUST_SYMBOL_DATA +from gajim.common.i18n import _ from gajim.common.i18n import p_ from gajim.common.modules.contacts import GroupchatContact +from gajim.common.storage.archive.const import MessageState from gajim.common.types import ChatContactT from gajim.gtk.menus import get_groupchat_participant_menu @@ -49,7 +52,6 @@ class SimpleLabel(Gtk.Label): @wrap_with_event_box class MoreMenuButton(Gtk.Button): def __init__(self, on_click_handler: Callable[[Gtk.Button], Any]) -> None: - Gtk.Button.__init__(self) self.set_valign(Gtk.Align.START) self.set_halign(Gtk.Align.END) @@ -61,7 +63,12 @@ class MoreMenuButton(Gtk.Button): image = Gtk.Image.new_from_icon_name( 'feather-more-horizontal-symbolic', Gtk.IconSize.BUTTON) self.add(image) - self.connect('clicked', on_click_handler) + + self._click_handler_id = self.connect('clicked', on_click_handler) + self.connect('destroy', self._on_destroy) + + def _on_destroy(self, _buton: MoreMenuButton) -> None: + self.disconnect(self._click_handler_id) class DateTimeLabel(Gtk.Label): @@ -103,34 +110,89 @@ class NicknameLabel(Gtk.Label): class MessageIcons(Gtk.Box): def __init__(self) -> None: - Gtk.Box.__init__(self, orientation=Gtk.Orientation.HORIZONTAL) + Gtk.Box.__init__(self, + orientation=Gtk.Orientation.HORIZONTAL, + spacing=3) + + self._encryption_image = Gtk.Image() + self._encryption_image.set_no_show_all(True) + self._encryption_image.set_margin_end(6) + + self._security_label = Gtk.Label() + self._security_label.set_no_show_all(True) + self._security_label.set_margin_end(6) + self._security_label.set_ellipsize(Pango.EllipsizeMode.END) + self._security_label.set_max_width_chars(20) self._correction_image = Gtk.Image.new_from_icon_name( 'document-edit-symbolic', Gtk.IconSize.MENU) self._correction_image.set_no_show_all(True) self._correction_image.get_style_context().add_class('dim-label') - self._marker_image = Gtk.Image() + self._group_chat_message_state_image = Gtk.Image() + self._group_chat_message_state_image.set_no_show_all(True) + self._group_chat_message_state_image.get_style_context().add_class( + 'dim-label') + + self._marker_image = Gtk.Image.new_from_icon_name( + 'feather-check-symbolic', Gtk.IconSize.MENU) self._marker_image.set_no_show_all(True) self._marker_image.get_style_context().add_class('dim-label') + self._marker_image.set_tooltip_text(p_('Message state', 'Received')) self._error_image = Gtk.Image.new_from_icon_name( 'dialog-warning-symbolic', Gtk.IconSize.MENU) self._error_image.get_style_context().add_class('warning-color') self._error_image.set_no_show_all(True) + self.add(self._encryption_image) + self.add(self._security_label) self.add(self._correction_image) + self.add(self._group_chat_message_state_image) self.add(self._marker_image) self.add(self._error_image) self.show_all() + def set_encryption_icon_visible(self, visible: bool) -> None: + self._encryption_image.set_visible(visible) + + def set_encrytion_icon_data(self, + icon: str, + color: str, + tooltip: str + ) -> None: + + context = self._encryption_image.get_style_context() + for trust_data in TRUST_SYMBOL_DATA.values(): + context.remove_class(trust_data[2]) + + context.add_class(color) + self._encryption_image.set_from_icon_name(icon, Gtk.IconSize.MENU) + self._encryption_image.set_tooltip_markup(tooltip) + + def set_security_label_visible(self, visible: bool) -> None: + self._security_label.set_visible(visible) + + def set_security_label_data(self, tooltip: str, markup: str) -> None: + self._security_label.set_tooltip_text(tooltip) + self._security_label.set_markup(markup) + def set_receipt_icon_visible(self, visible: bool) -> None: if not app.settings.get('positive_184_ack'): return self._marker_image.set_visible(visible) - self._marker_image.set_from_icon_name( - 'feather-check-symbolic', Gtk.IconSize.MENU) - self._marker_image.set_tooltip_text(p_('Message state', 'Received')) + + def set_group_chat_message_state_icon(self, state: MessageState) -> None: + if state == MessageState.PENDING: + icon_name = 'feather-clock-symbolic' + tooltip_text = _('Pending') + else: + icon_name = 'feather-check-symbolic' + tooltip_text = _('Received') + self._group_chat_message_state_image.set_from_icon_name( + icon_name, Gtk.IconSize.MENU) + self._group_chat_message_state_image.set_tooltip_text(tooltip_text) + self._group_chat_message_state_image.show() def set_correction_icon_visible(self, visible: bool) -> None: self._correction_image.set_visible(visible) @@ -146,31 +208,28 @@ class MessageIcons(Gtk.Box): class AvatarBox(Gtk.EventBox): - def __init__(self, - contact: ChatContactT, - name: str, - avatar: cairo.ImageSurface | None, - ) -> None: - + def __init__(self, contact: ChatContactT) -> None: Gtk.EventBox.__init__(self) - self.set_size_request(AvatarSize.ROSTER, -1) self.set_valign(Gtk.Align.START) self._contact = contact + self._name = '' - self._image = Gtk.Image.new_from_surface(avatar) + self._image = Gtk.Image() self.add(self._image) if self._contact.is_groupchat: self.connect('realize', self._on_realize) - self.connect('button-press-event', - self._on_avatar_clicked, name) + self.connect('button-press-event', self._on_avatar_clicked) def set_from_surface(self, surface: cairo.ImageSurface | None) -> None: self._image.set_from_surface(surface) + def set_name(self, name: str) -> None: + self._name = name + def set_merged(self, merged: bool) -> None: self._image.set_no_show_all(merged) self._image.set_visible(not merged) @@ -184,7 +243,6 @@ class AvatarBox(Gtk.EventBox): def _on_avatar_clicked(self, _widget: Gtk.Widget, event: Gdk.EventButton, - name: str ) -> int: if event.type == Gdk.EventType.BUTTON_PRESS: @@ -192,9 +250,10 @@ class AvatarBox(Gtk.EventBox): return Gdk.EVENT_STOP if event.button == Gdk.BUTTON_PRIMARY: - app.window.activate_action('mention', GLib.Variant('s', name)) + app.window.activate_action( + 'mention', GLib.Variant('s', self._name)) elif event.button == Gdk.BUTTON_SECONDARY: - self._show_participant_menu(name, event) + self._show_participant_menu(self._name, event) return Gdk.EVENT_STOP diff --git a/gajim/gtk/conversation/view.py b/gajim/gtk/conversation/view.py index fdbe85e4b..89808fa67 100644 --- a/gajim/gtk/conversation/view.py +++ b/gajim/gtk/conversation/view.py @@ -19,7 +19,6 @@ from typing import cast from typing import Literal import logging -import time from collections.abc import Generator from datetime import datetime from datetime import timedelta @@ -29,22 +28,20 @@ from gi.repository import Gio from gi.repository import GObject from gi.repository import Gtk from nbxmpp.errors import StanzaError -from nbxmpp.modules.security_labels import Displaymarking from nbxmpp.protocol import JID -from nbxmpp.structs import CommonError from nbxmpp.structs import MucSubject from gajim.common import app from gajim.common import events from gajim.common import types from gajim.common.const import Direction -from gajim.common.helpers import AdditionalDataDict from gajim.common.helpers import get_start_of_day from gajim.common.helpers import to_user_string from gajim.common.modules.contacts import BareContact from gajim.common.modules.contacts import GroupchatContact from gajim.common.modules.httpupload import HTTPFileTransfer -from gajim.common.storage.archive import ConversationRow +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.structs import DbConversationJoinedData from gajim.common.types import ChatContactT from gajim.gtk.conversation.rows.base import BaseRow @@ -384,16 +381,17 @@ class ConversationView(Gtk.ScrolledWindow): assert row is not None return cast(BaseRow, row) - def get_first_message_row(self) -> MessageRow | None: + def get_first_message_row(self + ) -> MessageRow | CallRow | FileTransferJingleRow | None: for row in self._list_box.get_children(): - if isinstance(row, MessageRow): + if isinstance(row, MessageRow | CallRow | FileTransferJingleRow): return row return None - def get_last_message_row(self) -> MessageRow | None: + def get_last_message_row(self) -> MessageRow | CallRow | FileTransferJingleRow | None: children = reversed(self._list_box.get_children()) for row in children: - if isinstance(row, MessageRow): + if isinstance(row, MessageRow | CallRow | FileTransferJingleRow): return row return None @@ -472,7 +470,7 @@ class ConversationView(Gtk.ScrolledWindow): event: (events.FileRequestReceivedEvent | events.FileRequestSent | None) = None, - db_message: ConversationRow | None = None + db_row: DbConversationJoinedData | None = None ) -> None: assert isinstance(self._contact, BareContact) @@ -480,7 +478,7 @@ class ConversationView(Gtk.ScrolledWindow): self._contact.account, self._contact, event=event, - db_message=db_message) + db_row=db_row) self._insert_message(jingle_transfer_row) def add_encryption_info(self, event: events.EncryptionInfo) -> None: @@ -489,14 +487,14 @@ class ConversationView(Gtk.ScrolledWindow): def add_call_message(self, event: events.JingleRequestReceived | None = None, - db_message: ConversationRow | None = None + db_row: DbConversationJoinedData | None = None ) -> None: assert isinstance(self._contact, BareContact) call_row = CallRow( self._contact.account, self._contact, event=event, - db_message=db_message) + db_row=db_row) self._insert_message(call_row) def add_command_output(self, text: str, is_error: bool) -> None: @@ -504,46 +502,21 @@ class ConversationView(Gtk.ScrolledWindow): self.contact.account, text, is_error) self._insert_message(command_output_row) - def add_message(self, - text: str, - kind: str, - name: str, - timestamp: float, - log_line_id: int | None = None, - message_id: str | None = None, - stanza_id: str | None = None, - display_marking: Displaymarking | None = None, - additional_data: AdditionalDataDict | None = None, - marker: str | None = None, - error: CommonError | StanzaError | None = None - ) -> None: - - if not timestamp: - timestamp = time.time() - - message_row = MessageRow( - self.contact.account, - self.contact, - message_id, - stanza_id, - timestamp, - kind, - name, - text, - additional_data=additional_data, - display_marking=display_marking, - marker=marker, - error=error, - log_line_id=log_line_id) + def add_message_from_db(self, db_row: DbConversationJoinedData) -> None: + message_row = MessageRow.from_db_row(self.contact, db_row) + + message_id = db_row.message_id if message_id is not None: self._message_id_row_map[message_id] = message_row - if kind == 'incoming': + if db_row.direction == ChatDirection.INCOMING: assert self._read_marker_row is not None self._read_marker_row.set_last_incoming_timestamp( message_row.timestamp) - if (marker is not None and marker == 'displayed' and + + if (db_row.marker is not None and + db_row.marker.displayed_ts is not None and message_id is not None): self.set_read_marker(message_id) @@ -555,7 +528,7 @@ class ConversationView(Gtk.ScrolledWindow): self._check_for_merge(message) assert self._read_marker_row is not None - if message.kind == 'incoming': + if message.direction == ChatDirection.INCOMING: if message.timestamp > self._read_marker_row.timestamp: self._read_marker_row.hide() @@ -789,7 +762,6 @@ class ConversationView(Gtk.ScrolledWindow): if row is None: return - row.set_displayed() assert self._read_marker_row is not None timestamp = row.timestamp + timedelta(microseconds=1) if self._read_marker_row.timestamp > timestamp: @@ -802,16 +774,11 @@ class ConversationView(Gtk.ScrolledWindow): if isinstance(row, MessageRow): row.update_avatar() - def correct_message(self, - correct_id: str, - text: str, - nickname: str | None - ) -> None: - - message_row = self._get_row_by_message_id(correct_id) + def correct_message(self, db_row: DbConversationJoinedData) -> None: + assert db_row.message_id is not None + message_row = self._get_row_by_message_id(db_row.message_id) if message_row is not None: - message_row.set_correction(text, nickname) - message_row.set_merged(False) + message_row.update_with_content(db_row) def show_message_retraction(self, stanza_id: str, text: str) -> None: message_row = self.get_row_by_stanza_id(stanza_id) @@ -821,12 +788,12 @@ class ConversationView(Gtk.ScrolledWindow): def show_receipt(self, id_: str) -> None: message_row = self._get_row_by_message_id(id_) if message_row is not None: - message_row.set_receipt() + message_row.show_receipt(True) def show_error(self, id_: str, error: StanzaError) -> None: message_row = self._get_row_by_message_id(id_) if message_row is not None: - message_row.set_error(to_user_string(error)) + message_row.show_error(to_user_string(error)) message_row.set_merged(False) def _on_contact_setting_changed(self, diff --git a/gajim/gtk/filetransfer.py b/gajim/gtk/filetransfer.py index d3c7ab012..71106af1b 100644 --- a/gajim/gtk/filetransfer.py +++ b/gajim/gtk/filetransfer.py @@ -22,6 +22,7 @@ from __future__ import annotations import logging import os import time +import uuid from datetime import datetime from datetime import timezone from enum import IntEnum @@ -39,12 +40,10 @@ from gajim.common import app from gajim.common import ged from gajim.common import helpers from gajim.common import types -from gajim.common.const import KindConstant from gajim.common.events import FileRequestSent from gajim.common.events import Notification from gajim.common.file_props import FileProp from gajim.common.file_props import FilesProp -from gajim.common.helpers import AdditionalDataDict from gajim.common.helpers import file_is_locked from gajim.common.helpers import show_in_folder from gajim.common.i18n import _ @@ -52,6 +51,12 @@ from gajim.common.modules.bytestream import is_transfer_active from gajim.common.modules.bytestream import is_transfer_paused from gajim.common.modules.bytestream import is_transfer_stopped from gajim.common.modules.contacts import BareContact +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import FiletransferSourceType +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbInsertFiletransferRowData +from gajim.common.storage.archive.structs import DbInsertMessageRowData from gajim.gtk.builder import get_builder from gajim.gtk.dialogs import ConfirmationDialog @@ -427,16 +432,25 @@ class FileTransfersWindow: if file_props is None: return False - # Insert file request into DB - additional_data = AdditionalDataDict() - additional_data.set_value('gajim', 'type', 'jingle') - additional_data.set_value('gajim', 'sid', file_props.sid) - app.storage.archive.insert_into_logs( - account, - contact.jid.bare, - time.time(), - KindConstant.FILE_TRANSFER_OUTGOING, - additional_data=additional_data) + message_data = DbInsertMessageRowData( + account=account, + remote_jid=contact.jid.new_as_bare(), + m_type=MessageType.CHAT, + direction=ChatDirection.OUTGOING, + timestamp=time.time(), + state=MessageState.ACKNOWLEDGED, + message_id=str(uuid.uuid4()), + ) + + message_ek = app.storage.archive.insert_row(message_data) + + filetransfer_data = DbInsertFiletransferRowData( + fk_message_ek=message_ek, + source_type=FiletransferSourceType.JINGLE, + source=file_props.sid, + state=1, + ) + app.storage.archive.insert_row(filetransfer_data) client = app.get_client(account) client.get_module('Jingle').start_file_transfer( diff --git a/gajim/gtk/groupchat_nick_completion.py b/gajim/gtk/groupchat_nick_completion.py index 33672f14c..08c1cc378 100644 --- a/gajim/gtk/groupchat_nick_completion.py +++ b/gajim/gtk/groupchat_nick_completion.py @@ -21,7 +21,7 @@ from gi.repository import Gtk from gajim.common import app from gajim.common import ged -from gajim.common.events import GcMessageReceived +from gajim.common.events import MessageReceived from gajim.common.ged import EventHelper from gajim.common.helpers import jid_is_blocked from gajim.common.modules.contacts import GroupchatContact @@ -39,7 +39,7 @@ class GroupChatNickCompletion(EventHelper): self._last_key_tab = False self.register_event( - 'gc-message-received', ged.GUI2, self._on_gc_message_received) + 'message-received', ged.GUI2, self._on_message_received) def switch_contact(self, contact: GroupchatContact) -> None: self._suggestions.clear() @@ -157,11 +157,11 @@ class GroupChatNickCompletion(EventHelper): return matches + other_nicks - def _on_gc_message_received(self, event: GcMessageReceived) -> None: + def _on_message_received(self, event: MessageReceived) -> None: if self._contact is None: return - if event.room_jid != self._contact.jid: + if event.jid != self._contact.jid: return if not self._last_key_tab: diff --git a/gajim/gtk/history_export.py b/gajim/gtk/history_export.py index 434b2c52e..6b7316f42 100644 --- a/gajim/gtk/history_export.py +++ b/gajim/gtk/history_export.py @@ -27,11 +27,12 @@ from gi.repository import Gtk from gajim.common import app from gajim.common import configpaths -from gajim.common.const import KindConstant from gajim.common.helpers import filesystem_path_from_uri from gajim.common.helpers import make_path_from_jid from gajim.common.i18n import _ -from gajim.common.storage.archive import MessageExportRow +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbConversationJoinedData from gajim.gtk.assistant import Assistant from gajim.gtk.assistant import ErrorPage @@ -142,30 +143,34 @@ class HistoryExport(Assistant): with open(file_path / 'history.txt', 'w', encoding='utf-8') as file: file.write(f'History for {jid}\n\n') for message in messages: + if message.call is not None: + continue + if message.has_filetransfers: + continue file.write(self._get_export_line(message)) self.show_page('success', Gtk.StackTransitionType.SLIDE_LEFT) - def _get_export_line(self, message: MessageExportRow) -> str: - if message.kind in (KindConstant.SINGLE_MSG_RECV, - KindConstant.CHAT_MSG_RECV): - name = message.jid - elif message.kind in (KindConstant.SINGLE_MSG_SENT, - KindConstant.CHAT_MSG_SENT): - name = _('You') - elif message.kind == KindConstant.GC_MSG: - name = message.contact_name - else: - raise ValueError('Unknown kind: %s' % message.kind) + def _get_nickname(self, joined_data: DbConversationJoinedData) -> str: + if joined_data.m_type == MessageType.GROUPCHAT: + assert joined_data.resource is not None + return joined_data.resource + + if joined_data.direction == ChatDirection.INCOMING: + return joined_data.remote_jid + + return _('You') + + def _get_export_line(self, joined_data: DbConversationJoinedData) -> str: + name = self._get_nickname(joined_data) + timestamp = time.strftime( + '%Y-%m-%d %H:%M:%S', time.localtime(joined_data.timestamp)) - timestamp = '' - try: - timestamp = time.strftime( - '%Y-%m-%d %H:%M:%S', time.localtime(message.time)) - except ValueError: - pass + message = joined_data.message + if joined_data.correction is not None: + message = joined_data.correction.message - return f'{timestamp} {name}: {message.message}\n' + return f'{timestamp} {name}: {message}\n' class SelectAccountDir(Page): diff --git a/gajim/gtk/history_sync.py b/gajim/gtk/history_sync.py index 9f605c1e0..94f3cac7d 100644 --- a/gajim/gtk/history_sync.py +++ b/gajim/gtk/history_sync.py @@ -61,19 +61,17 @@ class HistorySyncAssistant(Assistant, EventHelper): self._start: datetime | None = None self._end: datetime | None = None - mam_start = ArchiveState.NEVER - archive = app.storage.archive.get_archive_infos( - self._client.get_own_jid().bare) - if archive is not None and archive.oldest_mam_timestamp is not None: - mam_start = int(float(archive.oldest_mam_timestamp)) + mam_start = None + archive = app.storage.archive.get_mam_archive_state( + account, self._client.get_own_jid().new_as_bare()) - if mam_start == ArchiveState.NEVER: + if archive is not None and archive.from_stanza_ts is not None: + mam_start = archive.from_stanza_ts + + if mam_start is None: self._current_start = self._now - elif mam_start == ArchiveState.ALL: - self._current_start = datetime.fromtimestamp(0, timezone.utc) else: - self._current_start = datetime.fromtimestamp(mam_start, - timezone.utc) + self._current_start = mam_start self.add_button('synchronize', _('Synchronize'), 'suggested-action') self.add_button('close', _('Close')) diff --git a/gajim/gtk/main.py b/gajim/gtk/main.py index 4dfe3e089..88397b2fe 100644 --- a/gajim/gtk/main.py +++ b/gajim/gtk/main.py @@ -46,6 +46,7 @@ from gajim.common.modules.bytestream import is_transfer_active from gajim.common.modules.contacts import BareContact from gajim.common.modules.contacts import GroupchatContact from gajim.common.modules.contacts import GroupchatParticipant +from gajim.common.storage.archive.const import MessageType from gajim.plugins.manifest import PluginManifest from gajim.plugins.repository import PluginRepository @@ -68,6 +69,7 @@ from gajim.gtk.structs import AccountJidParam from gajim.gtk.structs import actionmethod from gajim.gtk.structs import AddChatActionParams from gajim.gtk.structs import ChatListEntryParam +from gajim.gtk.structs import DeleteMessageParam from gajim.gtk.structs import RetractMessageParam from gajim.gtk.util import get_app_window from gajim.gtk.util import open_window @@ -729,16 +731,18 @@ class MainWindow(Gtk.ApplicationWindow, EventHelper): input_str=_('Spam'), transient_for=app.window).show() + @actionmethod def _on_delete_message_locally(self, _action: Gio.SimpleAction, - param: GLib.Variant + params: DeleteMessageParam ) -> None: def _on_delete() -> None: - log_line_id = param.get_uint32() - app.storage.archive.delete_message_from_logs(log_line_id) - control = self.get_control() - control.remove_message(log_line_id) + app.storage.archive.delete_message(params.entitykey) + app.ged.raise_event( + events.MessageDeleted(account=params.account, + jid=params.jid, + entitykey=params.entitykey)) ConfirmationDialog( _('Delete Message'), @@ -1175,15 +1179,18 @@ class MainWindow(Gtk.ApplicationWindow, EventHelper): # Read marker must be sent only once return - last_message = app.storage.archive.get_last_conversation_line( + last_message = app.storage.archive.get_last_conversation_row( account, jid) - if last_message is None: + if last_message is None or last_message.message_id is None: return + assert last_message.message_id is not None + client = app.get_client(account) contact = client.get_module('Contacts').get_contact(jid) assert isinstance( contact, BareContact | GroupchatContact | GroupchatParticipant) + client.get_module('ChatMarkers').send_displayed_marker( contact, last_message.message_id, @@ -1347,21 +1354,18 @@ class MainWindow(Gtk.ApplicationWindow, EventHelper): self._set_startup_finished() def _on_message_received(self, event: events.MessageReceived) -> None: - if not self.chat_exists(event.account, event.jid): - if not event.properties.body: - # Don’t open control on chatstate etc. - return + if self.chat_exists(event.account, event.jid): + return - if event.properties.is_muc_pm: - self.add_private_chat(event.account, - event.properties.jid) + if event.m_type == MessageType.PM: + self.add_private_chat(event.account, + event.jid) - else: - jid = event.properties.jid.new_as_bare() - self.add_chat(event.account, jid, 'contact') + else: + self.add_chat(event.account, event.jid, 'contact') def _on_read_state_sync(self, event: events.ReadStateSync) -> None: - last_message = app.storage.archive.get_last_conversation_line( + last_message = app.storage.archive.get_last_conversation_row( event.account, event.jid) if last_message is None: diff --git a/gajim/gtk/menus.py b/gajim/gtk/menus.py index 9fb0693d1..dd0cccc5c 100644 --- a/gajim/gtk/menus.py +++ b/gajim/gtk/menus.py @@ -54,6 +54,7 @@ from gajim.gtk.const import MuteState from gajim.gtk.structs import AccountJidParam from gajim.gtk.structs import AddChatActionParams from gajim.gtk.structs import ChatListEntryParam +from gajim.gtk.structs import DeleteMessageParam from gajim.gtk.structs import MuteContactParam from gajim.gtk.structs import RemoveHistoryActionParams from gajim.gtk.structs import RetractMessageParam @@ -683,7 +684,7 @@ def get_chat_row_menu(contact: types.ChatContactT, timestamp: datetime, message_id: str | None, stanza_id: str | None, - log_line_id: int | None + entitykey: int | None ) -> GajimMenu: menu_items: MenuItemListT = [] @@ -720,7 +721,7 @@ def get_chat_row_menu(contact: types.ChatContactT, menu_items.append( (p_('Message row action', 'Select Messages…'), 'win.activate-message-selection', - GLib.Variant('u', log_line_id or 0))) + GLib.Variant('u', entitykey or 0))) show_correction = False if message_id is not None: @@ -752,11 +753,17 @@ def get_chat_row_menu(contact: types.ChatContactT, 'win.retract-message', param)) - if log_line_id is not None: + if entitykey is not None: + param = DeleteMessageParam( + account=contact.account, + jid=contact.jid, + entitykey=entitykey) + menu_items.append( - (p_('Message row action', 'Delete Message Locally…'), - 'win.delete-message-locally', - GLib.Variant('u', log_line_id or 0))) + (p_('Message row action', + 'Delete Message Locally…'), + 'win.delete-message-locally', + param)) return GajimMenu.from_list(menu_items) diff --git a/gajim/gtk/message_input.py b/gajim/gtk/message_input.py index 4b682498b..e6dea3117 100644 --- a/gajim/gtk/message_input.py +++ b/gajim/gtk/message_input.py @@ -140,9 +140,12 @@ class MessageInputTextView(Gtk.TextView, EventHelper): if message_row is None or message_row.message is None: return + text = message_row.message + if message_row.correction is not None: + text = message_row.correction.message self._set_correcting(True) self.get_style_context().add_class('gajim-msg-correcting') - self.insert_text(message_row.message) + self.insert_text(text) def try_message_correction(self, message: str) -> str | None: assert self._contact is not None @@ -169,18 +172,21 @@ class MessageInputTextView(Gtk.TextView, EventHelper): self._correcting[self._contact] = state def _on_message_sent(self, event: MessageSent) -> None: - if not event.message: + joined_data = event.joined_data + if not joined_data.message: return - if event.correct_id is None: - # This wasn't a corrected message - assert self._contact is not None - oob_url = event.additional_data.get_value('gajim', 'oob_url') - if oob_url == event.message: - # Don't allow to correct HTTP Upload file transfer URLs - self._last_message_id[self._contact] = None - else: - self._last_message_id[self._contact] = event.message_id + if joined_data.correction is not None: + return + + assert self._contact is not None + + if (joined_data.oob is not None and + joined_data.oob.url == joined_data.message): + # Don't allow to correct HTTP Upload file transfer URLs + self._last_message_id[self._contact] = None + else: + self._last_message_id[self._contact] = joined_data.message_id def _init_spell_checker(self) -> int: if not app.is_installed('GSPELL'): diff --git a/gajim/gtk/search_view.py b/gajim/gtk/search_view.py index 6ab3e5654..e4b0d4bea 100644 --- a/gajim/gtk/search_view.py +++ b/gajim/gtk/search_view.py @@ -31,14 +31,16 @@ from nbxmpp import JID from gajim.common import app from gajim.common import ged +from gajim.common.client import Client from gajim.common.const import AvatarSize from gajim.common.const import Direction -from gajim.common.const import KindConstant from gajim.common.modules.contacts import BareContact from gajim.common.modules.contacts import GroupchatContact from gajim.common.modules.contacts import GroupchatParticipant from gajim.common.modules.contacts import ResourceContact -from gajim.common.storage.archive import SearchLogRow +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.structs import DbConversationJoinedData from gajim.gtk.builder import get_builder from gajim.gtk.conversation.message_widget import MessageWidget @@ -62,7 +64,7 @@ class SearchView(Gtk.Box): self._account: str | None = None self._jid: JID | None = None - self._results_iterator: Iterator[SearchLogRow] | None = None + self._results_iterator: Iterator[DbConversationJoinedData] | None = None self._first_date: datetime | None = None self._last_date: datetime | None = None @@ -87,14 +89,17 @@ class SearchView(Gtk.Box): @staticmethod def _header_func(row: ResultRow, before: ResultRow | None) -> None: if before is None: - row.set_header(RowHeader(row.account, row.jid, row.time)) + row.set_header(RowHeader( + row.account, row.remote_jid, row.timestamp)) else: - date1 = time.strftime('%x', time.localtime(row.time)) - date2 = time.strftime('%x', time.localtime(before.time)) + date1 = time.strftime('%x', time.localtime(row.timestamp)) + date2 = time.strftime('%x', time.localtime(before.timestamp)) if before.jid != row.jid: - row.set_header(RowHeader(row.account, row.jid, row.time)) + row.set_header(RowHeader( + row.account, row.remote_jid, row.timestamp)) elif date1 != date2: - row.set_header(RowHeader(row.account, row.jid, row.time)) + row.set_header(RowHeader( + row.account, row.remote_jid, row.timestamp)) else: row.set_header(None) @@ -162,15 +167,15 @@ class SearchView(Gtk.Box): self._ui.search_checkbutton.set_active(True) if not context or everywhere: - self._results_iterator = app.storage.archive.search_all_logs( - text, - from_users=from_filters, - before=before_filters, - after=after_filters) + account = None + jid = None else: - self._results_iterator = app.storage.archive.search_log( - self._account, - self._jid, + account = self._account + jid = self._jid + + self._results_iterator = app.storage.archive.search_archive( + account, + jid, text, from_users=from_filters, before=before_filters, @@ -193,14 +198,9 @@ class SearchView(Gtk.Box): return new_text, filters or None def _add_results(self) -> None: - accounts = self._get_accounts() assert self._results_iterator is not None - for msg in itertools.islice(self._results_iterator, 25): - result_row = ResultRow( - msg, - accounts[msg.account_id], - msg.jid) - + for db_row in itertools.islice(self._results_iterator, 25): + result_row = ResultRow(db_row) self._ui.results_listbox.add(result_row) def _on_edge_reached(self, @@ -218,12 +218,12 @@ class SearchView(Gtk.Box): self._ui.calendar_button.set_sensitive(True) - first_log = app.storage.archive.get_first_history_timestamp( + first_log = app.storage.archive.get_first_history_ts( self._account, self._jid) if first_log is None: return self._first_date = self._get_date_from_timestamp(first_log) - last_log = app.storage.archive.get_last_history_timestamp( + last_log = app.storage.archive.get_last_history_ts( self._account, self._jid) if last_log is None: return @@ -242,10 +242,10 @@ class SearchView(Gtk.Box): calendar.clear_marks() month = python_month(month) - history_days = app.storage.archive.get_days_with_history( + history_days = app.storage.archive.get_days_containing_messages( self._account, self._jid, year, month) - for date in history_days: - calendar.mark_day(date.day) + for day in history_days: + calendar.mark_day(day) def _on_date_selected(self, calendar: Gtk.Calendar) -> None: year, month, day = calendar.get_date() @@ -310,13 +310,12 @@ class SearchView(Gtk.Box): if not control.has_active_chat(): return if control.contact.jid == self._jid: - meta_row = app.storage.archive.get_first_message_meta_for_date( + meta = app.storage.archive.get_first_message_meta_for_date( self._account, self._jid, date) - if meta_row is None: + if meta is None: return - control.scroll_to_message( - meta_row.log_line_id, - meta_row.time) + entitykey, timestamp = meta + control.scroll_to_message(entitykey, timestamp) @staticmethod def _get_date_from_timestamp(timestamp: float) -> datetime: @@ -326,27 +325,24 @@ class SearchView(Gtk.Box): return date @staticmethod - def _get_accounts() -> dict[int, str]: - accounts: dict[int, str] = {} - for account in app.settings.get_accounts(): - account_id = app.storage.archive.get_account_id(account) - accounts[account_id] = account - return accounts - - @staticmethod def _on_row_activated(_listbox: SearchView, row: ResultRow) -> None: control = app.window.get_control() if control.has_active_chat(): - if control.contact.jid == row.jid: - control.scroll_to_message(row.log_line_id, row.timestamp) + if control.contact.jid == row.remote_jid: + control.scroll_to_message(row.entitykey, row.timestamp) return # Other chat or no control opened - jid = JID.from_string(row.jid) - app.window.add_chat(row.account, jid, row.type, select=True) + jid = row.remote_jid + chat_type = 'chat' + if row.type == MessageType.GROUPCHAT: + chat_type = 'groupchat' + elif row.type == MessageType.PM: + chat_type = 'pm' + app.window.add_chat(row.account, jid, chat_type, select=True) control = app.window.get_control() if control.has_active_chat(): - control.scroll_to_message(row.log_line_id, row.timestamp) + control.scroll_to_message(row.entitykey, row.timestamp) def set_focus(self) -> None: self._ui.search_entry.grab_focus() @@ -381,25 +377,27 @@ class RowHeader(Gtk.Box): class ResultRow(Gtk.ListBoxRow): - def __init__(self, msg: SearchLogRow, account: str, jid: JID) -> None: + def __init__(self, db_row: DbConversationJoinedData) -> None: Gtk.ListBoxRow.__init__(self) - self.account = account - self.jid = jid - self.time = msg.time - self._client = app.get_client(account) - self.log_line_id = msg.log_line_id - self.timestamp = msg.time - self.kind = msg.kind + print(db_row) + self._client = self._get_client(db_row.account_jid) + self.account = self._client.account - self.type = 'contact' - if msg.kind == KindConstant.GC_MSG: - self.type = 'groupchat' - if jid.is_full: - self.type = 'pm' + self.remote_jid = JID.from_string(db_row.remote_jid) + self.direction = ChatDirection(db_row.direction) + + self.jid = JID.from_string(db_row.remote_jid) + if (db_row.direction == ChatDirection.OUTGOING): + self.jid = JID.from_string(self._client.get_own_jid().bare) + + self.entitykey = db_row.entitykey + self.timestamp = db_row.timestamp + + self.type = MessageType(db_row.m_type) self.contact = self._client.get_module('Contacts').get_contact( - jid, groupchat=self.type == 'groupchat') + self.jid, groupchat=self.type == MessageType.GROUPCHAT) assert isinstance( self.contact, BareContact | GroupchatContact | GroupchatParticipant) @@ -408,36 +406,41 @@ class ResultRow(Gtk.ListBoxRow): self._ui = get_builder('search_view.ui') self.add(self._ui.result_row_grid) - kind = 'status' - contact_name = msg.contact_name - if msg.kind in ( - KindConstant.SINGLE_MSG_RECV, KindConstant.CHAT_MSG_RECV): - kind = 'incoming' - contact_name = self.contact.name - elif msg.kind == KindConstant.GC_MSG: - kind = 'incoming' - elif msg.kind in ( - KindConstant.SINGLE_MSG_SENT, KindConstant.CHAT_MSG_SENT): - kind = 'outgoing' - contact_name = app.nicks[account] + contact_name = self.contact.name + if self.type == MessageType.GROUPCHAT: + contact_name = db_row.resource + assert contact_name is not None + self._ui.row_name_label.set_text(contact_name) - avatar = self._get_avatar(kind, contact_name) + avatar = self._get_avatar(self.direction, contact_name) self._ui.row_avatar.set_from_surface(avatar) - local_time = time.localtime(msg.time) + local_time = time.localtime(self.timestamp) format_string = app.settings.get('time_format') date = time.strftime(format_string, local_time) self._ui.row_time_label.set_label(date) - message_widget = MessageWidget(account, selectable=False) - message_widget.add_with_styling(msg.message, nickname=contact_name) + assert db_row.message is not None + message = db_row.message + if db_row.correction is not None: + message = db_row.correction.message + + message_widget = MessageWidget(self.account, selectable=False) + message_widget.add_with_styling(message, nickname=contact_name) self._ui.result_row_grid.attach(message_widget, 1, 1, 2, 1) self.show_all() + def _get_client(self, account_jid: str) -> Client: + for client in app.get_clients(): + if client.is_own_jid(account_jid): + return client + + raise ValueError('Unable to find account: %s' % account_jid) + def _get_avatar(self, - kind: str, + direction: ChatDirection, name: str) -> cairo.ImageSurface | None: scale = self.get_scale_factor() @@ -445,9 +448,9 @@ class ResultRow(Gtk.ListBoxRow): contact = self.contact.get_resource(name) return contact.get_avatar(AvatarSize.ROSTER, scale, add_show=False) - if kind == 'outgoing': + if direction == ChatDirection.OUTGOING: contact = self._client.get_module('Contacts').get_contact( - str(self._client.get_own_jid().bare)) + self._client.get_own_jid().bare) else: contact = self.contact diff --git a/gajim/gtk/structs.py b/gajim/gtk/structs.py index 91f3aad56..9313c7353 100644 --- a/gajim/gtk/structs.py +++ b/gajim/gtk/structs.py @@ -81,6 +81,13 @@ class RetractMessageParam(VariantMixin): stanza_id: str +@dataclass +class DeleteMessageParam(VariantMixin): + account: str + jid: JID + entitykey: int + + def get_params_class(func: Callable[..., Any]) -> Any: module = sys.modules[__name__] params = inspect.signature(func).parameters diff --git a/gajim/main.py b/gajim/main.py index 5c7d0ab79..0b547d24f 100644 --- a/gajim/main.py +++ b/gajim/main.py @@ -35,7 +35,7 @@ _MIN_CAIRO_VER = '1.16.0' _MIN_PYGOBJECT_VER = '3.42.0' _MIN_GLIB_VER = '2.66.0' _MIN_PANGO_VER = '1.50.0' -_MIN_SQLITE_VER = '3.33.0' +_MIN_SQLITE_VER = '3.35.0' def check_version(dep_name: str, current_ver: str, min_ver: str) -> None: diff --git a/pyproject.toml b/pyproject.toml index b1523399f..72ee3d0c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,6 +137,7 @@ include = [ "gajim/common/modules/register.py", "gajim/common/modules/vcard4.py", "gajim/common/modules/vcard_temp.py", + "gajim/common/modules/moderations.py", "gajim/common/passwords.py", "gajim/common/preview_helpers.py", "gajim/common/regex.py", @@ -285,6 +286,7 @@ target-version = "py310" [tool.ruff.per-file-ignores] "gajim/common/iana.py" = ["E501"] "gajim/common/storage/omemo.py" = ["N803"] +"gajim/common/storage/archive/statements.py" = ["E501"] "test/*" = ["E402"] "test/common/test_styling.py" = ["RUF001", "E501"] "test/common/test_regex.py" = ["RUF001"] diff --git a/test/database/__init__.py b/test/database/__init__.py new file mode 100644 index 000000000..be5142643 --- /dev/null +++ b/test/database/__init__.py @@ -0,0 +1,13 @@ +import gi + + +def require_versions(): + gi.require_versions({'Gdk': '3.0', + 'GLib': '2.0', + 'Gio': '2.0', + 'Gtk': '3.0', + 'GtkSource': '4', + 'GObject': '2.0', + 'Pango': '1.0'}) + +require_versions() diff --git a/test/database/test_call.py b/test/database/test_call.py new file mode 100644 index 000000000..7fe1b613b --- /dev/null +++ b/test/database/test_call.py @@ -0,0 +1,71 @@ + +import time +import unittest + +from nbxmpp.protocol import JID + +from gajim.common import app +from gajim.common.settings import Settings +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.storage import MessageArchiveStorage +from gajim.common.storage.archive.structs import DbInsertCallRowData +from gajim.common.storage.archive.structs import DbInsertMessageRowData + + +class CallTest(unittest.TestCase): + + def setUp(self) -> None: + self._archive = MessageArchiveStorage(in_memory=True) + self._archive.init() + app.storage.archive = self._archive + + self._account = 'testacc1' + self._remote_jid = JID.from_string('remote@jid.org') + self._occupant_id = 'occupantid1' + self._init_settings() + + def _init_settings(self) -> None: + app.settings = Settings(in_memory=True) + app.settings.init() + app.settings.add_account('testacc1') + app.settings.set_account_setting('testacc1', 'name', 'user') + app.settings.set_account_setting('testacc1', 'hostname', 'domain.org') + + def test_call_insert(self) -> None: + + call_data = DbInsertCallRowData( + sid='sid123', + state=0, + duration=123 + ) + + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=self._remote_jid, + m_type=MessageType.CHAT, + direction=ChatDirection.INCOMING, + timestamp=time.time(), + state=MessageState.ACKNOWLEDGED, + resource='someres1', + message_id='1', + message='message', + ) + + entity_key = self._archive.insert_row( + message_data, + [call_data] + ) + + joined_data = self._archive.get_message_with_entitykey(entity_key) + + self.assertIsNotNone(joined_data.call) + assert joined_data.call is not None + self.assertEqual(joined_data.call.state, 0) + self.assertEqual(joined_data.call.sid, 'sid123') + self.assertEqual(joined_data.call.duration, 123) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/database/test_corrections.py b/test/database/test_corrections.py new file mode 100644 index 000000000..35f862c2e --- /dev/null +++ b/test/database/test_corrections.py @@ -0,0 +1,427 @@ + +import unittest + +from nbxmpp.protocol import JID + +from gajim.common import app +from gajim.common.settings import Settings +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.storage import MessageArchiveStorage +from gajim.common.storage.archive.structs import DbInsertCorrectionRowData +from gajim.common.storage.archive.structs import DbInsertEncryptionRowData +from gajim.common.storage.archive.structs import DbInsertMessageRowData +from gajim.common.storage.archive.structs import DbUpsertOccupantRowData + + +class CorrectionsTest(unittest.TestCase): + + def setUp(self) -> None: + + self._archive = MessageArchiveStorage(in_memory=True) + self._archive.init() + app.storage.archive = self._archive + + self._account = 'testacc1' + self._remote_jid = JID.from_string('remote@jid.org') + self._occupant_id = 'occupantid1' + self._occupant_id2 = 'occupantid2' + self._init_settings() + + def _init_settings(self) -> None: + app.settings = Settings(in_memory=True) + app.settings.init() + app.settings.add_account('testacc1') + app.settings.set_account_setting('testacc1', 'name', 'user') + app.settings.set_account_setting('testacc1', 'hostname', 'domain.org') + + def test_correction_chat(self) -> None: + occupant_data = DbUpsertOccupantRowData( + account=self._account, + remote_jid=self._remote_jid, + id=self._occupant_id, + timestamp=0, + nickname='nickname1', + ) + fk_occupant_ek = self._archive.upsert_row(occupant_data) + + encryption_data1 = DbInsertEncryptionRowData('OMEMO', 1, 'Abcd') + encryption_ek1 = self._archive.insert_row(encryption_data1, + raise_on_conflict=False) + + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=self._remote_jid, + m_type=MessageType.CHAT, + direction=ChatDirection.INCOMING, + state=MessageState.ACKNOWLEDGED, + timestamp=0, + resource='res1', + message='Some Message', + message_id='messageid1', + stanza_id='1a', + stable_id=True, + fk_occupant_ek=None, + user_delay_ts=None, + fk_securitylabel_ek=None, + fk_encryption_ek=encryption_ek1, + ) + + message_ek = self._archive.insert_row(message_data) + + correction_data1 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource=None, + direction=ChatDirection.INCOMING, + timestamp=1, + message_id='message_id1', + fk_occupant_ek=None, + correction_id='messageid1', + corrected_message='corrected message 1', + fk_encryption_ek=encryption_ek1 + ) + + self._archive.insert_row(correction_data1) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + self.assertIsNotNone(conversation_row.correction) + assert conversation_row.correction is not None + self.assertEqual( + conversation_row.correction.message, 'corrected message 1') + self.assertEqual(conversation_row.correction.timestamp, 1) + self.assertIsNotNone(conversation_row.correction.encryption) + assert conversation_row.correction.encryption is not None + self.assertEqual( + conversation_row.correction.encryption.protocol, 'OMEMO') + self.assertEqual(conversation_row.correction.encryption.trust, 1) + self.assertEqual(conversation_row.correction.encryption.key, 'Abcd') + + # Correct follow up correction + correction_data2 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource=None, + direction=ChatDirection.INCOMING, + timestamp=2, + message_id='message_id2', + fk_occupant_ek=None, + correction_id='messageid1', + corrected_message='corrected message 2', + fk_encryption_ek=encryption_ek1 + ) + + self._archive.insert_row(correction_data2) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + self.assertIsNotNone(conversation_row.correction) + assert conversation_row.correction is not None + self.assertEqual( + conversation_row.correction.message, 'corrected message 2') + self.assertEqual(conversation_row.correction.timestamp, 2) + + # Direction different, should not be joined + correction_data3 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource=None, + direction=ChatDirection.OUTGOING, + timestamp=99, + message_id='message_id3', + fk_occupant_ek=None, + correction_id='messageid1', + corrected_message='corrected message 3', + fk_encryption_ek=encryption_ek1 + ) + + self._archive.insert_row(correction_data3) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + self.assertIsNotNone(conversation_row.correction) + assert conversation_row.correction is not None + self.assertEqual( + conversation_row.correction.message, 'corrected message 2') + + # message id different, should not be joined + correction_data4 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource=None, + direction=ChatDirection.INCOMING, + timestamp=99, + message_id='message_id4', + fk_occupant_ek=None, + correction_id='messageid2', + corrected_message='corrected message 4', + fk_encryption_ek=encryption_ek1 + ) + + self._archive.insert_row(correction_data4) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + self.assertIsNotNone(conversation_row.correction) + assert conversation_row.correction is not None + self.assertEqual( + conversation_row.correction.message, 'corrected message 2') + + # has occupant key, should not be joined + correction_data5 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource=None, + direction=ChatDirection.INCOMING, + timestamp=99, + message_id='message_id5', + fk_occupant_ek=fk_occupant_ek, + correction_id='messageid1', + corrected_message='corrected message 5', + fk_encryption_ek=encryption_ek1 + ) + + self._archive.insert_row(correction_data5) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + self.assertIsNotNone(conversation_row.correction) + assert conversation_row.correction is not None + self.assertEqual( + conversation_row.correction.message, 'corrected message 2') + + # resource is ignored for message type chat + correction_data6 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource='res2', + direction=ChatDirection.INCOMING, + timestamp=99, + message_id='message_id6', + fk_occupant_ek=None, + correction_id='messageid1', + corrected_message='corrected message 6', + fk_encryption_ek=encryption_ek1 + ) + + self._archive.insert_row(correction_data6) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + self.assertIsNotNone(conversation_row.correction) + assert conversation_row.correction is not None + self.assertEqual( + conversation_row.correction.message, 'corrected message 6') + + def test_correction_groupchat(self) -> None: + occupant_data = DbUpsertOccupantRowData( + account=self._account, + remote_jid=self._remote_jid, + id=self._occupant_id, + timestamp=0, + nickname='nickname1', + ) + fk_occupant_ek = self._archive.upsert_row(occupant_data) + + occupant_data = DbUpsertOccupantRowData( + account=self._account, + remote_jid=self._remote_jid, + id=self._occupant_id2, + timestamp=0, + nickname='nickname2', + ) + fk_occupant_ek2 = self._archive.upsert_row(occupant_data) + + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=self._remote_jid, + m_type=MessageType.GROUPCHAT, + direction=ChatDirection.INCOMING, + timestamp=0, + state=MessageState.ACKNOWLEDGED, + resource='res1', + message='Some Message', + message_id='messageid1', + stanza_id='1a', + stable_id=True, + fk_occupant_ek=fk_occupant_ek, + ) + + message_ek = self._archive.insert_row(message_data) + + # Correction with occupant ek set + correction_data1 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource=None, + direction=ChatDirection.INCOMING, + timestamp=1, + message_id='message_id1', + fk_occupant_ek=fk_occupant_ek, + correction_id='messageid1', + corrected_message='corrected message 1' + ) + + self._archive.insert_row(correction_data1) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + self.assertIsNotNone(conversation_row.correction) + assert conversation_row.correction is not None + self.assertEqual( + conversation_row.correction.message, 'corrected message 1') + self.assertEqual(conversation_row.correction.timestamp, 1) + + # Correction with occupant ek set but different resource + # occupant ek is prefered and correction is allowed + correction_data3 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource='res2', + direction=ChatDirection.INCOMING, + timestamp=3, + message_id='message_id3', + fk_occupant_ek=fk_occupant_ek, + correction_id='messageid1', + corrected_message='corrected message 3' + ) + + self._archive.insert_row(correction_data3) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + self.assertIsNotNone(conversation_row.correction) + assert conversation_row.correction is not None + self.assertEqual( + conversation_row.correction.message, 'corrected message 3') + + # Different occupant ek, message should not be joined + correction_data4 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource='res2', + direction=ChatDirection.INCOMING, + timestamp=4, + message_id='message_id4', + fk_occupant_ek=fk_occupant_ek2, + correction_id='messageid1', + corrected_message='corrected message 4' + ) + + self._archive.insert_row(correction_data4) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + self.assertIsNotNone(conversation_row.correction) + assert conversation_row.correction is not None + self.assertEqual( + conversation_row.correction.message, 'corrected message 3') + + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=self._remote_jid, + m_type=MessageType.GROUPCHAT, + direction=ChatDirection.INCOMING, + timestamp=0, + state=MessageState.ACKNOWLEDGED, + resource='res2', + message='Some Message', + message_id='messageid2', + stanza_id='2a', + stable_id=True, + ) + + message_ek2 = self._archive.insert_row(message_data) + + # Correction with only resource, should be joined + correction_data5 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource='res2', + direction=ChatDirection.INCOMING, + timestamp=4, + message_id='message_id5', + fk_occupant_ek=None, + correction_id='messageid2', + corrected_message='corrected message 5' + ) + + self._archive.insert_row(correction_data5) + + conversation_row = self._archive.get_message_with_entitykey(message_ek2) + + self.assertIsNotNone(conversation_row.correction) + assert conversation_row.correction is not None + self.assertEqual( + conversation_row.correction.message, 'corrected message 5') + + def test_get_corrections(self) -> None: + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=self._remote_jid, + m_type=MessageType.CHAT, + direction=ChatDirection.INCOMING, + timestamp=0, + state=MessageState.ACKNOWLEDGED, + resource='res1', + message='Some Message', + message_id='messageid1', + stanza_id='1a', + stable_id=True, + fk_occupant_ek=None, + ) + + message_ek = self._archive.insert_row(message_data) + + correction_data1 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource=None, + direction=ChatDirection.INCOMING, + timestamp=1, + message_id='message_id1', + fk_occupant_ek=None, + correction_id='messageid1', + corrected_message='corrected message 1' + ) + + correction_ek1 = self._archive.insert_row(correction_data1) + + correction_data2 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource=None, + direction=ChatDirection.INCOMING, + timestamp=2, + message_id='message_id2', + fk_occupant_ek=None, + correction_id='messageid1', + corrected_message='corrected message 2' + ) + + correction_ek2 = self._archive.insert_row(correction_data2) + + correction_data3 = DbInsertCorrectionRowData( + account=self._account, + remote_jid=self._remote_jid, + resource=None, + direction=ChatDirection.INCOMING, + timestamp=3, + message_id='message_id3', + fk_occupant_ek=None, + correction_id='messageid1', + corrected_message='corrected message 3' + ) + + correction_ek3 = self._archive.insert_row(correction_data3) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + corrections = conversation_row.get_corrections() + eks = {c.entitykey for c in corrections} + self.assertEqual(eks, {correction_ek1, correction_ek2, correction_ek3}) + +if __name__ == '__main__': + unittest.main() diff --git a/test/database/test_filetransfers.py b/test/database/test_filetransfers.py new file mode 100644 index 000000000..a2ed6f4d1 --- /dev/null +++ b/test/database/test_filetransfers.py @@ -0,0 +1,86 @@ + +import time +import unittest + +from nbxmpp.protocol import JID + +from gajim.common import app +from gajim.common.settings import Settings +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.storage import MessageArchiveStorage +from gajim.common.storage.archive.structs import DbInsertFiletransferRowData +from gajim.common.storage.archive.structs import DbInsertMessageRowData + + +class FiletransferTest(unittest.TestCase): + + def setUp(self) -> None: + self._archive = MessageArchiveStorage(in_memory=True) + self._archive.init() + app.storage.archive = self._archive + + self._account = 'testacc1' + self._remote_jid = JID.from_string('remote@jid.org') + self._occupant_id = 'occupantid1' + self._init_settings() + + def _init_settings(self) -> None: + app.settings = Settings(in_memory=True) + app.settings.init() + app.settings.add_account('testacc1') + app.settings.set_account_setting('testacc1', 'name', 'user') + app.settings.set_account_setting('testacc1', 'hostname', 'domain.org') + + def test_insert(self) -> None: + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=self._remote_jid, + m_type=MessageType.CHAT, + direction=ChatDirection.INCOMING, + timestamp=time.time(), + state=MessageState.ACKNOWLEDGED, + resource='someres1', + message_id='1', + message='message', + ) + + entity_key = self._archive.insert_row(message_data) + + filetransfer_data1 = DbInsertFiletransferRowData( + fk_message_ek=entity_key, + source_type=1, + source='1', + state=1 + ) + + filetransfer_data2 = DbInsertFiletransferRowData( + fk_message_ek=entity_key, + source_type=2, + source='2', + state=2 + ) + + self._archive.insert_row(filetransfer_data1) + self._archive.insert_row(filetransfer_data2) + + joined_data = self._archive.get_message_with_entitykey(entity_key) + + self.assertTrue(joined_data.has_filetransfers) + + filetransfers = joined_data.get_filetransfers() + ft1 = filetransfers[0] + ft2 = filetransfers[1] + + self.assertEqual(ft1.source_type, 1) + self.assertEqual(ft1.source, '1') + self.assertEqual(ft1.state, 1) + + self.assertEqual(ft2.source_type, 2) + self.assertEqual(ft2.source, '2') + self.assertEqual(ft2.state, 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/database/test_foreign_keys.py b/test/database/test_foreign_keys.py new file mode 100644 index 000000000..2faaa946d --- /dev/null +++ b/test/database/test_foreign_keys.py @@ -0,0 +1,111 @@ + +import time +import unittest + +from nbxmpp.protocol import JID + +from gajim.common import app +from gajim.common.settings import Settings +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.storage import MessageArchiveStorage +from gajim.common.storage.archive.structs import DbInsertCallRowData +from gajim.common.storage.archive.structs import DbInsertFiletransferRowData +from gajim.common.storage.archive.structs import DbInsertMessageRowData +from gajim.common.storage.archive.structs import DbInsertOOBRowData +from gajim.common.storage.archive.structs import DbInsertReplyRowData + + +class ForeignKeyTest(unittest.TestCase): + + def setUp(self) -> None: + self._archive = MessageArchiveStorage(in_memory=True) + self._archive.init() + + self._account = 'testacc1' + self._remote_jid = JID.from_string('remote@jid.org') + self._occupant_id = 'occupantid1' + self._init_settings() + + def _init_settings(self) -> None: + app.settings = Settings(in_memory=True) + app.settings.init() + app.settings.add_account('testacc1') + app.settings.set_account_setting('testacc1', 'name', 'user') + app.settings.set_account_setting('testacc1', 'hostname', 'domain.org') + + def test_message_delete_cascade(self) -> None: + oob_data = DbInsertOOBRowData( + url='https://www.test.com', + description='somedesc' + ) + + call_data = DbInsertCallRowData( + sid='123', + state=0, + duration=123 + ) + + reply_data = DbInsertReplyRowData( + quoted_id='123', + quoted_jid='somejid@jid.com', + fallback_end=5 + ) + + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=self._remote_jid, + m_type=MessageType.CHAT, + direction=ChatDirection.INCOMING, + timestamp=time.time(), + state=MessageState.ACKNOWLEDGED, + resource='someres1', + message_id='1', + message='message', + ) + + entity_key = self._archive.insert_row( + message_data, + [call_data, reply_data, oob_data], + ) + + filetransfer_data = DbInsertFiletransferRowData( + fk_message_ek=entity_key, + source_type=1, + source='123', + state=1 + ) + + self._archive.insert_row(filetransfer_data) + self._archive.insert_row(filetransfer_data) + + joined_data = self._archive.get_message_with_entitykey(entity_key) + + assert joined_data.reply is not None + assert joined_data.oob is not None + assert joined_data.call is not None + + self.assertEqual(joined_data.reply.quoted_id, '123') + self.assertEqual(joined_data.oob.description, 'somedesc') + self.assertEqual(joined_data.call.sid, '123') + self.assertTrue(joined_data.has_filetransfers) + + self._archive.delete_message(entity_key) + + connection = self._archive.get_connection() + + row = connection.execute('SELECT * FROM call').fetchone() + self.assertIsNone(row) + row = connection.execute('SELECT * FROM filetransfer').fetchone() + self.assertIsNone(row) + row = connection.execute('SELECT * FROM oob').fetchone() + self.assertIsNone(row) + row = connection.execute('SELECT * FROM reply').fetchone() + self.assertIsNone(row) + row = connection.execute('SELECT * FROM message').fetchone() + self.assertIsNone(row) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/database/test_markers.py b/test/database/test_markers.py new file mode 100644 index 000000000..77a851078 --- /dev/null +++ b/test/database/test_markers.py @@ -0,0 +1,211 @@ + +import unittest + +from nbxmpp.protocol import JID + +from gajim.common import app +from gajim.common.settings import Settings +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.storage import MessageArchiveStorage +from gajim.common.storage.archive.structs import DbInsertMessageRowData +from gajim.common.storage.archive.structs import DbUpsertMarkerRowData +from gajim.common.storage.archive.structs import DbUpsertOccupantRowData + + +class MarkersTest(unittest.TestCase): + + def setUp(self) -> None: + self._archive = MessageArchiveStorage(in_memory=True) + self._archive.init() + + self._account = 'testacc1' + self._remote_jid = JID.from_string('remote@jid.org') + self._occupant_id = 'occupantid1' + self._init_settings() + + def _init_settings(self) -> None: + app.settings = Settings(in_memory=True) + app.settings.init() + app.settings.add_account('testacc1') + app.settings.set_account_setting('testacc1', 'name', 'user') + app.settings.set_account_setting('testacc1', 'hostname', 'domain.org') + + def test_markers_join_chat(self): + marker_data1 = DbUpsertMarkerRowData( + account=self._account, + remote_jid=self._remote_jid, + fk_occupant_ek=None, + marker_id='messageid1', + received_ts=1, + displayed_ts=None, + acknowledged_ts=None, + ) + + marker_data2 = DbUpsertMarkerRowData( + account=self._account, + remote_jid=self._remote_jid, + fk_occupant_ek=None, + marker_id='messageid1', + received_ts=None, + displayed_ts=2, + acknowledged_ts=None, + ) + + marker_ek1 = self._archive.upsert_row(marker_data1) + marker_ek2 = self._archive.upsert_row(marker_data2) + + self.assertEqual(marker_ek1, marker_ek2) + + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=self._remote_jid, + m_type=MessageType.CHAT, + direction=ChatDirection.OUTGOING, + timestamp=0, + state=MessageState.ACKNOWLEDGED, + resource='res', + message='Some Message', + message_id='messageid1', + stanza_id='1a', + stable_id=True, + fk_occupant_ek=None, + user_delay_ts=None, + fk_securitylabel_ek=None, + fk_encryption_ek=None, + ) + + message_ek = self._archive.insert_row(message_data) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + self.assertTrue(conversation_row.has_markers) + self.assertIsNotNone(conversation_row.marker) + assert conversation_row.marker is not None + self.assertEqual(conversation_row.marker.received_ts, 1) + self.assertEqual(conversation_row.marker.displayed_ts, 2) + + row = self._archive.get_row_with_entitykey('marker', marker_ek1) + self.assertEqual(row.fk_occupant_ek, -1) + + def test_markers_join_groupchat(self): + occupant_data = DbUpsertOccupantRowData( + account=self._account, + remote_jid=self._remote_jid, + id=self._occupant_id, + timestamp=0, + nickname='nickname1', + ) + fk_occupant_ek = self._archive.upsert_row(occupant_data) + + marker_data1 = DbUpsertMarkerRowData( + account=self._account, + remote_jid=self._remote_jid, + fk_occupant_ek=fk_occupant_ek, + marker_id='messageid1', + received_ts=1, + displayed_ts=None, + acknowledged_ts=None, + ) + + marker_data2 = DbUpsertMarkerRowData( + account=self._account, + remote_jid=self._remote_jid, + fk_occupant_ek=fk_occupant_ek, + marker_id='messageid1', + received_ts=None, + displayed_ts=2, + acknowledged_ts=None, + ) + + marker_ek1 = self._archive.upsert_row(marker_data1) + marker_ek2 = self._archive.upsert_row(marker_data2) + + self.assertEqual(marker_ek1, marker_ek2) + + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=self._remote_jid, + m_type=MessageType.CHAT, + direction=ChatDirection.INCOMING, + timestamp=0, + state=MessageState.ACKNOWLEDGED, + resource='res', + message='Some Message', + message_id='messageid1', + stanza_id='1a', + stable_id=True, + fk_occupant_ek=fk_occupant_ek, + user_delay_ts=None, + fk_securitylabel_ek=None, + fk_encryption_ek=None, + ) + + message_ek = self._archive.insert_row(message_data) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + self.assertTrue(conversation_row.has_markers) + self.assertIsNotNone(conversation_row.marker) + assert conversation_row.marker is not None + self.assertEqual(conversation_row.marker.received_ts, 1) + self.assertEqual(conversation_row.marker.displayed_ts, 2) + + row = self._archive.get_row_with_entitykey('marker', marker_ek1) + self.assertEqual(row.fk_occupant_ek, fk_occupant_ek) + + def test_markers_update(self): + marker_data1 = DbUpsertMarkerRowData( + account=self._account, + remote_jid=self._remote_jid, + fk_occupant_ek=None, + marker_id='messageid1', + received_ts=1, + displayed_ts=None, + acknowledged_ts=None, + ) + + marker_data2 = DbUpsertMarkerRowData( + account=self._account, + remote_jid=self._remote_jid, + fk_occupant_ek=None, + marker_id='messageid1', + received_ts=2, + displayed_ts=None, + acknowledged_ts=None, + ) + + marker_data3 = DbUpsertMarkerRowData( + account=self._account, + remote_jid=self._remote_jid, + fk_occupant_ek=None, + marker_id='messageid1', + received_ts=None, + displayed_ts=3, + acknowledged_ts=None, + ) + + marker_ek1 = self._archive.upsert_row(marker_data1) + + row = self._archive.get_row_with_entitykey('marker', marker_ek1) + self.assertEqual(row.received_ts, 1) + + marker_ek2 = self._archive.upsert_row(marker_data2) + + row = self._archive.get_row_with_entitykey('marker', marker_ek1) + self.assertEqual(row.received_ts, 1) + + self.assertEqual(marker_ek1, marker_ek2) + + marker_ek3 = self._archive.upsert_row(marker_data3) + + self.assertEqual(marker_ek1, marker_ek3) + + row = self._archive.get_row_with_entitykey('marker', marker_ek3) + self.assertEqual(row.received_ts, 1) + self.assertEqual(row.displayed_ts, 3) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/database/test_security_labels.py b/test/database/test_security_labels.py new file mode 100644 index 000000000..c7fd8cefc --- /dev/null +++ b/test/database/test_security_labels.py @@ -0,0 +1,141 @@ + +import unittest + +from nbxmpp.modules.security_labels import SecurityLabel +from nbxmpp.protocol import JID +from nbxmpp.simplexml import Node + +from gajim.common import app +from gajim.common.settings import Settings +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.storage import MessageArchiveStorage +from gajim.common.storage.archive.structs import DbInsertMessageRowData +from gajim.common.storage.archive.structs import DbUpsertSecurityLabelRowData + + +class SecurityLabelsTest(unittest.TestCase): + + def setUp(self) -> None: + self._archive = MessageArchiveStorage(in_memory=True) + self._archive.init() + xml1 = ''' + <securitylabel xmlns='urn:xmpp:sec-label:0'> + <displaymarking fgcolor='black' bgcolor='red'>SECRET</displaymarking> + <label> + <icismlabel xmlns='http://example.gov/IC-ISM/0' classification='S' ownerProducer='USA'/> + </label> + </securitylabel> + ''' # noqa: E501 + + xml2 = ''' + <securitylabel xmlns='urn:xmpp:sec-label:0'> + <displaymarking fgcolor='white' bgcolor='blue'>NOT SECRET</displaymarking> + <label> + <icismlabel xmlns='http://example.gov/IC-ISM/0' classification='S' ownerProducer='USA'/> + </label> + </securitylabel> + ''' # noqa: E501 + + self._account = 'testacc1' + self._remote_jid = JID.from_string('remote@jid.org') + self._sec_label1 = SecurityLabel.from_node(Node(node=xml1)) # pyright: ignore # noqa: E501 + self._sec_label2 = SecurityLabel.from_node(Node(node=xml2)) # pyright: ignore # noqa: E501 + self._init_settings() + + def _init_settings(self) -> None: + app.settings = Settings(in_memory=True) + app.settings.init() + app.settings.add_account('testacc1') + app.settings.set_account_setting('testacc1', 'name', 'user') + app.settings.set_account_setting('testacc1', 'hostname', 'domain.org') + + def test_security_labels_join(self): + + displaymarking = self._sec_label1.displaymarking + assert displaymarking is not None + + sec_data = DbUpsertSecurityLabelRowData( + account=self._account, + remote_jid=self._remote_jid, + timestamp=0, + label_hash=self._sec_label1.get_label_hash(), + displaymarking=displaymarking.name, + fgcolor=displaymarking.fgcolor, + bgcolor=displaymarking.bgcolor, + ) + + fk_securitylabel_ek = self._archive.upsert_row(sec_data) + + message_data = DbInsertMessageRowData( + account=self._account, + remote_jid=self._remote_jid, + m_type=MessageType.CHAT, + direction=ChatDirection.INCOMING, + timestamp=0, + state=MessageState.ACKNOWLEDGED, + resource='res', + message='Some Message', + message_id='1', + stanza_id='1a', + stable_id=True, + fk_occupant_ek=None, + user_delay_ts=None, + fk_securitylabel_ek=fk_securitylabel_ek, + fk_encryption_ek=None, + ) + + message_ek = self._archive.insert_row(message_data) + + conversation_row = self._archive.get_message_with_entitykey(message_ek) + + self.assertIsNotNone(conversation_row.securitylabel) + assert conversation_row.securitylabel is not None + self.assertEqual( + conversation_row.securitylabel.displaymarking, 'SECRET') + self.assertEqual(conversation_row.securitylabel.fgcolor, 'black') + self.assertEqual(conversation_row.securitylabel.bgcolor, 'red') + + def test_security_labels_update(self): + + displaymarking1 = self._sec_label1.displaymarking + assert displaymarking1 is not None + + sec_data1 = DbUpsertSecurityLabelRowData( + account=self._account, + remote_jid=self._remote_jid, + timestamp=0, + label_hash=self._sec_label1.get_label_hash(), + displaymarking=displaymarking1.name, + fgcolor=displaymarking1.fgcolor, + bgcolor=displaymarking1.bgcolor, + ) + + displaymarking2 = self._sec_label2.displaymarking + assert displaymarking2 is not None + + sec_data2 = DbUpsertSecurityLabelRowData( + account=self._account, + remote_jid=self._remote_jid, + timestamp=1, + label_hash=self._sec_label2.get_label_hash(), + displaymarking=displaymarking2.name, + fgcolor=displaymarking2.fgcolor, + bgcolor=displaymarking2.bgcolor, + ) + + fk_securitylabel_ek1 = self._archive.upsert_row(sec_data1) + fk_securitylabel_ek2 = self._archive.upsert_row(sec_data2) + + self.assertEqual(fk_securitylabel_ek1, fk_securitylabel_ek2) + + row = self._archive.get_row_with_entitykey( + 'securitylabel', fk_securitylabel_ek2) + self.assertEqual(row.displaymarking, 'NOT SECRET') + self.assertEqual(row.fgcolor, 'white') + self.assertEqual(row.bgcolor, 'blue') + self.assertEqual(row.timestamp, 1) + +if __name__ == '__main__': + unittest.main() diff --git a/test/dialogs/conversation_view.py b/test/dialogs/conversation_view.py index 40b8595f7..4e001bc9b 100644 --- a/test/dialogs/conversation_view.py +++ b/test/dialogs/conversation_view.py @@ -8,12 +8,15 @@ from nbxmpp.protocol import JID from gajim.common import app from gajim.common import configpaths from gajim.common.const import AvatarSize -from gajim.common.const import KindConstant from gajim.common.modules.contacts import BareContact from gajim.common.modules.contacts import ContactSettings from gajim.common.preview import PreviewManager from gajim.common.settings import Settings -from gajim.common.storage.archive import MessageArchiveStorage +from gajim.common.storage.archive.const import ChatDirection +from gajim.common.storage.archive.const import MessageState +from gajim.common.storage.archive.const import MessageType +from gajim.common.storage.archive.storage import MessageArchiveStorage +from gajim.common.storage.archive.structs import DbInsertMessageRowData from gajim.common.storage.events import EventStorage from gajim.gtk.avatar import generate_default_avatar @@ -82,16 +85,24 @@ class ConversationViewTest(Gtk.ApplicationWindow): def add_archive_messages() -> None: + remote_jid = JID.from_string(FROM_JID) timestamp = BASE_TIMESTAMP for num in range(1000): - app.storage.archive.insert_into_logs( - ACCOUNT, - FROM_JID, - timestamp, - KindConstant.CHAT_MSG_RECV, - message=num, - stanza_id=num, - message_id=num) + message_data = DbInsertMessageRowData( + account=ACCOUNT, + remote_jid=remote_jid, + m_type=MessageType.CHAT, + direction=ChatDirection.INCOMING, + timestamp=timestamp, + state=MessageState.ACKNOWLEDGED, + resource=None, + message=str(num), + message_id=str(num), + stanza_id=str(num), + ) + + app.storage.archive.insert_row(message_data) + timestamp += 1 |