iartamonov/mautrix-telegram
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add command to backfill room history from Telegram

Browse Source 
Currently supports backfilling one room at a time and backfills
everything after the last bridged message.
This commit is contained in:
Tulir Asokan
2020-02-04 22:41:49 +02:00
parent 07edcc4867
commit 2fbee75453
6 changed files with 85 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mautrix_telegram/portal/__init__.pyi
+3 -3
View File
@@ -1,8 +1,8 @@
from typing import Union
from .base import BasePortal
from .portal_matrix import PortalMatrix
from .portal_metadata import PortalMetadata
from .portal_telegram import PortalTelegram
from .matrix import PortalMatrix
from .metadata import PortalMetadata
from .telegram import PortalTelegram
from ..context import Context
Portal = Union[BasePortal, PortalMatrix, PortalMetadata, PortalTelegram]
mautrix_telegram/portal/base.py
+14 -5
View File
@@ -13,7 +13,7 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, TYPE_CHECKING
from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, Set, TYPE_CHECKING
from abc import ABC, abstractmethod
import asyncio
import logging
@@ -35,7 +35,7 @@ from mautrix.util.simple_template import SimpleTemplate
from ..types import TelegramID
from ..context import Context
from ..db import Portal as DBPortal
from ..db import Portal as DBPortal, Message as DBMessage
from .. import puppet as p, user as u, util
from .deduplication import PortalDedup
from .send_lock import PortalSendLock
@@ -86,6 +86,8 @@ class BasePortal(ABC):
photo_id: Optional[str]
local_config: Dict[str, Any]
deleted: bool
backfilling: bool
backfill_leave: Optional[Set[IntentAPI]]
log: logging.Logger
alias: Optional[RoomAlias]
@@ -115,6 +117,8 @@ class BasePortal(ABC):
self._main_intent = None
self.deleted = False
self.log = self.base_log.getChild(self.tgid_log if self.tgid else self.mxid)
self.backfilling = True
self.backfill_leave = None
self.dedup = PortalDedup(self)
self.send_lock = PortalSendLock()
@@ -273,8 +277,8 @@ class BasePortal(ABC):
authenticated.append(user)
return authenticated
@staticmethod
async def cleanup_room(intent: IntentAPI, room_id: RoomID, message: str,
@classmethod
async def cleanup_room(cls, intent: IntentAPI, room_id: RoomID, message: str,
puppets_only: bool = False) -> None:
try:
members = await intent.get_room_members(room_id)
@@ -293,7 +297,7 @@ class BasePortal(ABC):
try:
await intent.leave_room(room_id)
except (MatrixRequestError, IntentError):
self.log.warning("Failed to leave room when cleaning up room", exc_info=True)
cls.log.warning(f"Failed to leave room {room_id} when cleaning up room", exc_info=True)
async def cleanup_portal(self, message: str, puppets_only: bool = False) -> None:
if self.username:
@@ -342,6 +346,7 @@ class BasePortal(ABC):
pass
if self._db_instance:
self._db_instance.delete()
DBMessage.delete_all(self.mxid)
self.deleted = True
@classmethod
@@ -491,6 +496,10 @@ class BasePortal(ABC):
old_levels: Dict[UserID, int]) -> Awaitable[None]:
pass
@abstractmethod
def backfill(self, source: 'AbstractUser') -> Awaitable[None]:
pass
# endregion
mautrix_telegram/portal/telegram.py
+35 -5
View File
@@ -14,7 +14,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import Awaitable, Dict, List, Optional, Tuple, Union, NamedTuple, TYPE_CHECKING
from html import escape as escape_html
from abc import ABC
import random
import mimetypes
@@ -30,7 +29,7 @@ from telethon.tl.types import (
MessageMediaPoll, MessageActionChannelCreate, MessageActionChatAddUser,
MessageActionChatCreate, MessageActionChatDeletePhoto, MessageActionChatDeleteUser,
MessageActionChatEditPhoto, MessageActionChatEditTitle, MessageActionChatJoinedByLink,
MessageActionChatMigrateTo, MessageActionPinMessage, MessageActionGameScore,
MessageActionChatMigrateTo, MessageActionChannelMigrateFrom, MessageActionGameScore,
MessageMediaDocument, MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported,
MessageMediaGame, PeerUser, PhotoCachedSize, TypeChannelParticipant, TypeChatParticipant,
TypeDocumentAttribute, TypeMessageAction, TypePhotoSize, PhotoSize, UpdateChatUserTyping,
@@ -356,6 +355,31 @@ class PortalTelegram(BasePortal, ABC):
edit_index=prev_edit_msg.edit_index + 1).insert()
DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=event_id)
async def backfill(self, source: 'AbstractUser') -> None:
last = DBMessage.find_last(self.mxid, (source.tgid if self.peer_type != "channel"
else self.tgid))
min_id = last.tgid if last else 0
self.backfilling = True
self.backfill_leave = set()
max_file_size = min(config["bridge.max_document_size"], 1500) * 1024 * 1024
async with source.client.takeout(files=True, megagroups=self.megagroup,
chats=self.peer_type == "chat",
users=self.peer_type == "user",
channels=(self.peer_type == "channel"
and not self.megagroup),
max_file_size=max_file_size
) as takeout_client:
async for message in takeout_client.iter_messages(await self.get_input_entity(source),
reverse=True, min_id=min_id):
sender = p.Puppet.get(message.sender_id)
# if isinstance(message, MessageService):
# await self.handle_telegram_action(source, sender, message)
await self.handle_telegram_message(source, sender, message)
for intent in self.backfill_leave:
await intent.leave_room(self.mxid)
self.backfilling = False
self.backfill_leave = None
async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet,
evt: Message) -> None:
if not self.mxid:
@@ -383,7 +407,7 @@ class PortalTelegram(BasePortal, ABC):
tg_space=tg_space, edit_index=0).insert()
return
if self.dedup.pre_db_check and self.peer_type == "channel":
if self.backfilling or (self.dedup.pre_db_check and self.peer_type == "channel"):
msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
if msg:
self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already"
@@ -402,7 +426,13 @@ class PortalTelegram(BasePortal, ABC):
MessageMediaGame, MessageMediaPoll, MessageMediaUnsupported)
media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
allowed_media) else None
intent = sender.intent_for(self) if sender else self.main_intent
if sender:
intent = sender.intent_for(self)
if self.backfilling and intent != sender.default_mxid_intent:
intent = sender.default_mxid_intent
self.backfill_leave.add(intent)
else:
intent = self.main_intent
if not media and evt.message:
is_bot = sender.is_bot if sender else False
event_id = await self.handle_telegram_text(source, intent, is_bot, evt)
@@ -502,7 +532,7 @@ class PortalTelegram(BasePortal, ABC):
await self.main_intent.set_power_levels(self.mxid, levels)
async def receive_telegram_pin_id(self, msg_id: TelegramID, receiver: TelegramID) -> None:
tg_space = receiver if self.peer_type != "channel" else self.tgid
tg_space = receiver if self.peer_type != "channel" else self.tgid
message = DBMessage.get_one_by_tgid(msg_id, tg_space) if msg_id != 0 else None
if message:
await self.main_intent.set_pinned_messages(self.mxid, [message.mxid])