Bump version to 0.5.1

This reverts commit 1da1133934 and a part of 2cf9dcafd9

.gitignore

@@ -1,6 +1,7 @@
 .idea/
 
 .venv
+env/
 pip-selfcheck.json
 *.pyc
 __pycache__

Dockerfile

@@ -15,18 +15,23 @@ RUN apk add --no-cache \
       py3-sqlalchemy \
       py3-markdown \
       py3-psycopg2 \
+      # Not yet in stable repos:
+      #py3-ruamel \
       # Indirect dependencies
-      py3-numpy \
+      #commonmark
-      py3-asn1crypto \
+        py3-future \
-      py3-future \
+      #alembic
-      py3-markupsafe \
+        py3-mako \
-      py3-mako \
+        py3-dateutil \
-      py3-decorator \
+        py3-markupsafe \
-      py3-dateutil \
+      #moviepy
-      py3-idna \
+        py3-decorator \
-      py3-six \
+        #py3-tqdm \
-      py3-asn1 \
+        py3-requests \
-      py3-rsa \
+        #imageio
+          py3-numpy \
+      #telethon
+        py3-rsa \
       # Other dependencies
       python3-dev \
       build-base \

ROADMAP.md

@@ -21,6 +21,10 @@
     * [ ] ‡ Changes to displayname/avatar
 * Telegram → Matrix
   * [x] Message content (text, formatting, files, etc..)
+  * [ ] Advanced message content/media
+    * [x] Polls
+	* [x] Games
+	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
   * [ ] Message history
@@ -48,6 +52,8 @@
   * [x] Option to use bot to relay messages for unauthenticated Matrix users
   * [x] Option to use own Matrix account for messages sent from other Telegram clients
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
+  * [ ] ‡ Secret chats (not yet supported by Telethon)
+  * [ ] ‡ E2EE in Matrix rooms (not yet supported 
 
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

example-config.yaml

@@ -144,6 +144,8 @@ bridge:
     # Use inline images instead of a separate message for the caption.
     # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
     inline_images: false
+    # Maximum size of image in megabytes before sending to Telegram as a document.
+    image_as_file_size: 10
 
     # Whether to bridge Telegram bot messages as m.notices or m.texts.
     bot_messages_as_notices: true

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.5.0rc1"
+__version__ = "0.5.1"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -23,7 +23,6 @@ import sys
 import copy
 import signal
 
-from sqlalchemy import orm
 import sqlalchemy as sql
 
 from mautrix_appservice import AppService
@@ -73,15 +72,20 @@ log = logging.getLogger("mau.init")  # type: logging.Logger
 log.debug(f"Initializing mautrix-telegram {__version__}")
 
 db_engine = sql.create_engine(config["appservice.database"] or "sqlite:///mautrix-telegram.db")
-db_factory = orm.sessionmaker(bind=db_engine)
-db_session = orm.scoping.scoped_session(db_factory)
 Base.metadata.bind = db_engine
 
-session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
+session_container = AlchemySessionContainer(engine=db_engine, table_base=Base, session=False,
-                                            table_base=Base, table_prefix="telethon_",
+                                            table_prefix="telethon_", manage_tables=False)
-                                            manage_tables=False)
 session_container.core_mode = True
 
+try:
+    import uvloop
+
+    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+    log.debug("Using uvloop for asyncio")
+except ImportError:
+    pass
+
 loop = asyncio.get_event_loop()  # type: asyncio.AbstractEventLoop
 
 state_store = SQLStateStore()
@@ -94,8 +98,8 @@ appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                      aiohttp_params={
                          "client_max_size": config["appservice.max_body_size"] * mebibyte
                      })
-
+bot = init_bot(config)
-context = Context(appserv, db_session, config, loop, session_container)
+context = Context(appserv, config, loop, session_container, bot)
 
 if config["appservice.public.enabled"]:
     public_website = PublicBridgeWebsite(loop)
@@ -108,12 +112,12 @@ if config["appservice.provisioning.enabled"]:
                            provisioning_api.app)
     context.provisioning_api = provisioning_api
 
+context.mx = MatrixHandler(context)
+
 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
     start_ts = time()
     init_db(db_engine)
     init_abstract_user(context)
-    context.bot = init_bot(context)
-    context.mx = MatrixHandler(context)
     init_formatter(context)
     init_portal(context)
     startup_actions = (init_puppet(context) +
@@ -142,3 +146,6 @@ with appserv.run(config["appservice.hostname"], config["appservice.port"]) as st
             asyncio.gather(*[user.stop() for user in User.by_tgid.values()], loop=loop))
         log.debug("Clients stopped, shutting down")
         sys.exit(0)
+    except Exception as e:
+        log.exception("Unexpected error")
+        sys.exit(1)

mautrix_telegram/abstract_user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -20,7 +20,6 @@ import asyncio
 import logging
 import platform
 
-from sqlalchemy import orm
 from telethon.tl.patched import MessageService, Message
 from telethon.tl.types import (
     Channel, ChannelForbidden, Chat, ChatForbidden, MessageActionChannelMigrateFrom, PeerUser,
@@ -56,7 +55,6 @@ class AbstractUser(ABC):
     session_container = None  # type: AlchemySessionContainer
     loop = None  # type: asyncio.AbstractEventLoop
     log = None  # type: logging.Logger
-    db = None  # type: orm.Session
     az = None  # type: AppService
     bot = None  # type: Bot
     ignore_incoming_bot_events = True  # type: bool
@@ -175,11 +173,8 @@ class AbstractUser(ABC):
     async def ensure_started(self, even_if_no_session=False) -> 'AbstractUser':
         if not self.puppet_whitelisted or self.connected:
             return self
-        session_count = self.session_container.Session.query.filter(
+        self.log.debug("ensure_started(%s, even_if_no_session=%s)", self.mxid, even_if_no_session)
-            self.session_container.Session.session_id == self.mxid).count()
+        if even_if_no_session or self.session_container.has_session(self.mxid):
-        self.log.debug("ensure_started(%s, even_if_no_session=%s, session_count=%s)",
-                       self.mxid, even_if_no_session, session_count)
-        if even_if_no_session or session_count > 0:
             await self.start(delete_unless_authenticated=not even_if_no_session)
         return self
 
@@ -388,7 +383,7 @@ class AbstractUser(ABC):
 
 def init(context: "Context") -> None:
     global config, MAX_DELETIONS
-    AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, AbstractUser.relaybot = context.core
+    AbstractUser.az, config, AbstractUser.loop, AbstractUser.relaybot = context.core
     AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
     AbstractUser.session_container = context.session_container
     MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/bot.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -23,7 +23,7 @@ from telethon.tl.types import (
     ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
     ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser,
     MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
-    UpdateNewChannelMessage, UpdateNewMessage)
+    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo)
 from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
@@ -56,7 +56,7 @@ class Bot(AbstractUser):
         self.username = None  # type: str
         self.is_relaybot = True  # type: bool
         self.is_bot = True  # type: bool
-        self.chats = {chat.id: chat.type for chat in BotChat.all()}  # type: Dict[int, str]
+        self.chats = {}  # type: Dict[int, str]
         self.tg_whitelist = []  # type: List[int]
         self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
                                        or False)  # type: bool
@@ -74,6 +74,7 @@ class Bot(AbstractUser):
                 self.tg_whitelist.append(user_id)
 
     async def start(self, delete_unless_authenticated: bool = False) -> 'Bot':
+        self.chats = {chat.id: chat.type for chat in BotChat.all()}
         await super().start(delete_unless_authenticated)
         if not await self.is_logged_in():
             await self.client.sign_in(bot_token=self.token)
@@ -238,7 +239,7 @@ class Bot(AbstractUser):
             await self.handle_command_invite(portal, reply, mxid_input=mxid)
 
     def handle_service_message(self, message: MessageService) -> None:
-        to_id = message.to_id
+        to_id = message.to_id  # type: TelegramID
         if isinstance(to_id, PeerChannel):
             to_id = to_id.channel_id
             chat_type = "channel"
@@ -250,9 +251,12 @@ class Bot(AbstractUser):
 
         action = message.action
         if isinstance(action, MessageActionChatAddUser) and self.tgid in action.users:
-            self.add_chat(TelegramID(to_id), chat_type)
+            self.add_chat(to_id, chat_type)
         elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
-            self.remove_chat(TelegramID(to_id))
+            self.remove_chat(to_id)
+        elif isinstance(action, MessageActionChatMigrateTo):
+            self.remove_chat(to_id)
+            self.add_chat(TelegramID(action.channel_id), "channel")
 
     async def update(self, update) -> bool:
         if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
@@ -277,9 +281,9 @@ class Bot(AbstractUser):
         return "bot"
 
 
-def init(context: 'Context') -> Optional[Bot]:
+def init(cfg: 'Config') -> Optional[Bot]:
     global config
-    config = context.config
+    config = cfg
     token = config["telegram.bot_token"]
     if token and not token.lower().startswith("disable"):
         return Bot(token)

mautrix_telegram/commands/clean_rooms.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/commands/handler.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,15 +14,16 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""This module contains classes handling commands issued by Matrix users."""
 from typing import Awaitable, Callable, Dict, List, NamedTuple, Optional
-import traceback
 import logging
+import traceback
 
 import commonmark
 
 from telethon.errors import FloodWaitError
 
-from ..types import MatrixRoomID
+from ..types import MatrixRoomID, MatrixEventID
 from ..util import format_duration
 from .. import user as u, context as c
 
@@ -59,9 +60,32 @@ md_parser = commonmark.Parser()
 md_renderer = HtmlEscapingRenderer()
 
 
+def ensure_trailing_newline(s: str) -> str:
+    """Returns the passed string, but with a guaranteed trailing newline."""
+    return s + ("" if s[-1] == "\n" else "\n")
+
+
 class CommandEvent:
-    def __init__(self, processor: 'CommandProcessor', room: MatrixRoomID, sender: u.User,
+    """Holds information about a command issued in a Matrix room.
-                 command: str, args: List[str], is_management: bool, is_portal: bool) -> None:
+
+    When a Matrix command was issued to the bot, CommandEvent will hold
+    information regarding the event.
+
+    Attributes:
+        room_id: The id of the Matrix room in which the command was issued.
+        event_id: The id of the matrix event which contained the command.
+        sender: The user who issued the command.
+        command: The issued command.
+        args: Arguments given with the issued command.
+        is_management: Determines whether the room in which the command wa
+            issued is a management room.
+        is_portal: Determines whether the room in which the command was issued
+            is a portal.
+    """
+
+    def __init__(self, processor: 'CommandProcessor', room: MatrixRoomID, event: MatrixEventID,
+                 sender: u.User, command: str, args: List[str], is_management: bool,
+                 is_portal: bool) -> None:
         self.az = processor.az
         self.log = processor.log
         self.loop = processor.loop
@@ -70,6 +94,7 @@ class CommandEvent:
         self.public_website = processor.public_website
         self.command_prefix = processor.command_prefix
         self.room_id = room
+        self.event_id = event
         self.sender = sender
         self.command = command
         self.args = args
@@ -78,23 +103,102 @@ class CommandEvent:
 
     def reply(self, message: str, allow_html: bool = False, render_markdown: bool = True
               ) -> Awaitable[Dict]:
-        message = message.replace("$cmdprefix+sp ",
+        """Write a reply to the room in which the command was issued.
-                                  "" if self.is_management else f"{self.command_prefix} ")
+
-        message = message.replace("$cmdprefix", self.command_prefix)
+        Replaces occurences of "$cmdprefix" in the message with the command
-        html = None
+        prefix and replaces occurences of "$cmdprefix+sp " with the command
+        prefix if the command was not issued in a management room.
+        If allow_html and render_markdown are both False, the message will not
+        be rendered to html and sending of html is disabled.
+
+        Args:
+            message: The message to post in the room.
+            allow_html: Escape html in the message or don't render html at all
+                if markdown is disabled.
+            render_markdown: Use markdown formatting to render the passed
+                message to html.
+
+        Returns:
+            Handler for the message sending function.
+        """
+        message_cmd = self._replace_command_prefix(message)
+        html = self._render_message(message_cmd, allow_html=allow_html,
+                                    render_markdown=render_markdown)
+
+        return self.az.intent.send_notice(self.room_id, message_cmd, html=html)
+
+    def mark_read(self) -> Awaitable[Dict]:
+        """Marks the command as read by the bot."""
+        return self.az.intent.mark_read(self.room_id, self.event_id)
+
+    def _replace_command_prefix(self, message: str) -> str:
+        """Returns the string with the proper command prefix entered."""
+        message = message.replace(
+            "$cmdprefix+sp ", "" if self.is_management else f"{self.command_prefix} "
+        )
+        return message.replace("$cmdprefix", self.command_prefix)
+
+    @staticmethod
+    def _render_message(message: str, allow_html: bool, render_markdown: bool) -> Optional[str]:
+        """Renders the message as HTML.
+
+        Args:
+            allow_html: Flag to allow custom HTML in the message.
+            render_markdown: If true, markdown styling is applied to the message.
+
+        Returns:
+            The message rendered as HTML.
+            None is returned if no styled output is required.
+        """
+        html = ""
         if render_markdown:
             md_renderer.allow_html = allow_html
             html = md_renderer.render(md_parser.parse(message))
         elif allow_html:
             html = message
-        return self.az.intent.send_notice(self.room_id, message, html=html)
+        return ensure_trailing_newline(html) if html else None
 
 
 class CommandHandler:
+    """A command which can be executed from a Matrix room.
+
+    The command manages its permission and help texts.
+    When called, it will check the permission of the command event and execute
+    the command or, in case of error, report back to the user.
+
+    Attributes:
+        needs_auth: Flag indicating if the sender is required to be logged in.
+        needs_puppeting: Flag indicating if the sender is required to use
+            Telegram puppeteering for this command.
+        needs_matrix_puppeting: Flag indicating if the sender is required to use
+            Matrix pupeteering.
+        needs_admin: Flag for whether only admin users can issue this command.
+        management_only: Whether the command can exclusively be issued in a
+            management room.
+        name: The name of this command.
+        help_section: Section of the help in which this command will appear.
+    """
+
     def __init__(self, handler: Callable[[CommandEvent], Awaitable[Dict]], needs_auth: bool,
                  needs_puppeting: bool, needs_matrix_puppeting: bool, needs_admin: bool,
                  management_only: bool, name: str, help_text: str, help_args: str,
                  help_section: HelpSection) -> None:
+        """
+        Args:
+            handler: The function handling the execution of this command.
+            needs_auth: Flag indicating if the sender is required to be logged in.
+            needs_puppeting: Flag indicating if the sender is required to use
+                Telegram puppeteering for this command.
+            needs_matrix_puppeting: Flag indicating if the sender is required to
+                use Matrix pupeteering.
+            needs_admin: Flag for whether only admin users can issue this command.
+            management_only: Whether the command can exclusively be issued
+                in a management room.
+            name: The name of this command.
+            help_text: The text displayed in the help for this command.
+            help_args: Help text for the arguments of this command.
+            help_section: Section of the help in which this command will appear.
+        """
         self._handler = handler
         self.needs_auth = needs_auth
         self.needs_puppeting = needs_puppeting
@@ -107,6 +211,14 @@ class CommandHandler:
         self.help_section = help_section
 
     async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
+        """Returns the reason why the command could not be issued.
+
+        Args:
+            evt: The event for which to get the error information.
+
+        Returns:
+            A string describing the error or None if there was no error.
+        """
         if self.management_only and not evt.is_management:
             return (f"`{evt.command}` is a restricted command: "
                     "you may only run it in management rooms.")
@@ -122,6 +234,22 @@ class CommandHandler:
 
     def has_permission(self, is_management: bool, puppet_whitelisted: bool,
                        matrix_puppet_whitelisted: bool, is_admin: bool, is_logged_in: bool) -> bool:
+        """Checks the permission for this command with the given status.
+
+        Args:
+            is_management: If the room in which the command will be issued is a
+                management room.
+            puppet_whitelisted: If the connected Telegram account puppet is
+                allowed to issue the command.
+            matrix_puppet_whitelisted: If the connected Matrix account puppet is
+                allowed to issue the command.
+            is_admin: If the issuing user is an admin.
+            is_logged_in: If the issuing user is logged in.
+
+        Returns:
+            True if a user with the given state is allowed to issue the
+            command.
+        """
         return ((not self.management_only or is_management) and
                 (not self.needs_puppeting or puppet_whitelisted) and
                 (not self.needs_matrix_puppeting or matrix_puppet_whitelisted) and
@@ -129,6 +257,17 @@ class CommandHandler:
                 (not self.needs_auth or is_logged_in))
 
     async def __call__(self, evt: CommandEvent) -> Dict:
+        """Executes the command if evt was issued with proper rights.
+
+        Args:
+            evt: The CommandEvent for which to check permissions.
+
+        Returns:
+            The result of the command or the error message function.
+
+        Raises:
+            FloodWaitError
+        """
         error = await self.get_permission_error(evt)
         if error is not None:
             return await evt.reply(error)
@@ -136,26 +275,22 @@ class CommandHandler:
 
     @property
     def has_help(self) -> bool:
+        """Returns true if this command has a help text."""
         return bool(self.help_section) and bool(self._help_text)
 
     @property
     def help(self) -> str:
+        """Returns the help text to this command."""
         return f"**{self.name}** {self._help_args} - {self._help_text}"
 
 
 def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] = None, *,
-                    needs_auth: bool = True,
+                    needs_auth: bool = True, needs_puppeting: bool = True,
-                    needs_puppeting: bool = True,
+                    needs_matrix_puppeting: bool = False, needs_admin: bool = False,
-                    needs_matrix_puppeting: bool = False,
+                    management_only: bool = False, name: Optional[str] = None,
-                    needs_admin: bool = False,
+                    help_text: str = "", help_args: str = "", help_section: HelpSection = None
-                    management_only: bool = False,
-                    name: Optional[str] = None,
-                    help_text: str = "",
-                    help_args: str = "",
-                    help_section: HelpSection = None
                     ) -> Callable[[Callable[[CommandEvent], Awaitable[Optional[Dict]]]],
                                   CommandHandler]:
-
     def decorator(func: Callable[[CommandEvent], Awaitable[Optional[Dict]]]) -> CommandHandler:
         actual_name = name or func.__name__.replace("_", "-")
         handler = CommandHandler(func, needs_auth, needs_puppeting, needs_matrix_puppeting,
@@ -168,16 +303,40 @@ def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] =
 
 
 class CommandProcessor:
+    """Handles the raw commands issued by a user to the Matrix bot."""
     log = logging.getLogger("mau.commands")
 
     def __init__(self, context: c.Context) -> None:
-        self.az, self.db, self.config, self.loop, self.tgbot = context.core
+        self.az, self.config, self.loop, self.tgbot = context.core
         self.public_website = context.public_website
         self.command_prefix = self.config["bridge.command_prefix"]
 
-    async def handle(self, room: MatrixRoomID, sender: u.User, command: str, args: List[str],
+    async def handle(self, room: MatrixRoomID, event_id: MatrixEventID, sender: u.User,
-                     is_management: bool, is_portal: bool) -> Optional[Dict]:
+                     command: str, args: List[str], is_management: bool, is_portal: bool
-        evt = CommandEvent(self, room, sender, command, args, is_management, is_portal)
+                     ) -> Optional[Dict]:
+        """Handles the raw commands issued by a user to the Matrix bot.
+
+        If the command is not known, it might be a followup command and is
+        delegated to a command handler registered for that purpose in the
+        senders command_status as "next".
+
+        Args:
+            room: ID of the Matrix room in which the command was issued.
+            event_id: ID of the event by which the command was issued.
+            sender: The sender who issued the command.
+            command: The issued command, case insensitive.
+            args: Arguments given with the command.
+            is_management: Whether the room is a management room.
+            is_portal: Whether the room is a portal.
+
+        Returns:
+            The result of the error message function or None if no error
+            occured. Unknown and delegated commands do not count as errors.
+        """
+        if not command_handlers or "unknown-command" not in command_handlers:
+            raise ValueError("command_handlers are not properly initialized.")
+
+        evt = CommandEvent(self, room, event_id, sender, command, args, is_management, is_portal)
         orig_command = command
         command = command.lower()
         try:

mautrix_telegram/commands/matrix_auth.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/commands/meta.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -65,8 +65,8 @@ def _get_management_status(evt: CommandEvent) -> str:
     return "**This is not a management room**: you must prefix commands with `$cmdprefix`."
 
 
-@command_handler(needs_auth=False, needs_puppeting=False,
+@command_handler(name="help", needs_auth=False, needs_puppeting=False,
                  help_section=SECTION_GENERAL,
                  help_text="Show this help message.")
-async def help(evt: CommandEvent) -> Optional[Dict]:
+async def help_cmd(evt: CommandEvent) -> Optional[Dict]:
     return await evt.reply(_get_management_status(evt) + "\n" + await _get_help_text(evt))

mautrix_telegram/commands/portal/admin.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -52,7 +52,7 @@ async def set_power_level(evt: CommandEvent) -> Dict:
 async def clear_db_cache(evt: CommandEvent) -> Dict:
     try:
         section = evt.args[0].lower()
-    except KeyError:
+    except IndexError:
         return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
     if section == "portal":
         po.Portal.by_tgid = {}

mautrix_telegram/commands/portal/bridge.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/commands/portal/config.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,9 +18,10 @@ from typing import Dict, Awaitable
 from io import StringIO
 
 from ...config import yaml
-from ... import portal as po, user as u, util
+from ... import portal as po, util
 from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
 
+
 @command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
                  help_text="View or change per-portal settings.",
                  help_args="<`help`|_subcommand_> [...]")

mautrix_telegram/commands/portal/create_chat.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,6 +17,7 @@
 from typing import Dict
 
 from ... import portal as po
+from ...types import TelegramID
 from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
 from .util import user_has_power_level, get_initial_state
 
@@ -50,7 +51,8 @@ async def create(evt: CommandEvent) -> Dict:
         "group": "chat",
     }[type]
 
-    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
+    portal = po.Portal(tgid=TelegramID(0), peer_type=type,
+                       mxid=evt.room_id, title=title, about=about)
     try:
         await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
     except ValueError as e:

mautrix_telegram/commands/portal/filter.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -46,11 +46,11 @@ async def filter_mode(evt: CommandEvent) -> Dict:
                                "`!filter blacklist <chat ID>`.")
 
 
-@command_handler(needs_admin=True,
+@command_handler(name="filter", needs_admin=True,
                  help_section=SECTION_ADMIN,
                  help_args="<`whitelist`|`blacklist`> <_chat ID_>",
                  help_text="Allow or disallow bridging a specific chat.")
-async def filter(evt: CommandEvent) -> Optional[Dict]:
+async def edit_filter(evt: CommandEvent) -> Optional[Dict]:
     try:
         action = evt.args[0]
         if action not in ("whitelist", "blacklist", "add", "remove"):
@@ -58,11 +58,11 @@ async def filter(evt: CommandEvent) -> Optional[Dict]:
 
         id_str = evt.args[1]
         if id_str.startswith("-100"):
-            id = int(id_str[4:])
+            filter_id = int(id_str[4:])
         elif id_str.startswith("-"):
-            id = int(id_str[1:])
+            filter_id = int(id_str[1:])
         else:
-            id = int(id_str)
+            filter_id = int(id_str)
     except (IndexError, ValueError):
         return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
 
@@ -70,26 +70,26 @@ async def filter(evt: CommandEvent) -> Optional[Dict]:
     if mode not in ("blacklist", "whitelist"):
         return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
 
-    list = evt.config["bridge.filter.list"]
+    filter_id_list = evt.config["bridge.filter.list"]
 
     if action in ("blacklist", "whitelist"):
         action = "add" if mode == action else "remove"
 
     def save() -> None:
-        evt.config["bridge.filter.list"] = list
+        evt.config["bridge.filter.list"] = filter_id_list
         evt.config.save()
-        po.Portal.filter_list = list
+        po.Portal.filter_list = filter_id_list
 
     if action == "add":
-        if id in list:
+        if filter_id in filter_id_list:
             return await evt.reply(f"That chat is already {mode}ed.")
-        list.append(id)
+        filter_id_list.append(filter_id)
         save()
         return await evt.reply(f"Chat ID added to {mode}.")
     elif action == "remove":
-        if id not in list:
+        if filter_id not in filter_id_list:
             return await evt.reply(f"That chat is not {mode}ed.")
-        list.remove(id)
+        filter_id_list.remove(filter_id)
         save()
         return await evt.reply(f"Chat ID removed from {mode}.")
     return None

mautrix_telegram/commands/portal/misc.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -38,10 +38,10 @@ async def sync_state(evt: CommandEvent) -> Dict:
     await evt.reply("Synchronization complete")
 
 
-@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
+@command_handler(name="id", needs_admin=False, needs_puppeting=False, needs_auth=False,
                  help_section=SECTION_MISC,
                  help_text="Get the ID of the Telegram chat where this room is bridged.")
-async def id(evt: CommandEvent) -> Dict:
+async def get_id(evt: CommandEvent) -> Dict:
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")

mautrix_telegram/commands/portal/unbridge.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,7 +19,7 @@ from typing import Dict, Callable, Optional
 from ...types import MatrixRoomID
 from ... import portal as po
 from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
-from .util import user_has_power_level, get_initial_state
+from .util import user_has_power_level
 
 
 async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,

mautrix_telegram/commands/portal/util.py

@@ -1,3 +1,19 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Dict, Tuple
 
 from mautrix_appservice import MatrixRequestError, IntentAPI

mautrix_telegram/commands/telegram/account.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,12 +17,12 @@
 from typing import Dict, Optional
 
 from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
-                             HashInvalidError)
+                             HashInvalidError, AuthKeyError)
 from telethon.tl.types import Authorization
 from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
                                            ResetAuthorizationRequest)
 
-from mautrix_telegram.commands import command_handler, CommandEvent, SECTION_AUTH
+from .. import command_handler, CommandEvent, SECTION_AUTH
 
 
 @command_handler(needs_auth=True,
@@ -94,6 +94,11 @@ async def session(evt: CommandEvent) -> Optional[Dict]:
             ok = await evt.sender.client(ResetAuthorizationRequest(hash=session_hash))
         except HashInvalidError:
             return await evt.reply("Invalid session hash.")
+        except AuthKeyError as e:
+            if e.message == "FRESH_RESET_AUTHORISATION_FORBIDDEN":
+                return await evt.reply("New sessions can't terminate other sessions. "
+                                       "Please wait a while.")
+            raise
         if ok:
             return await evt.reply("Session terminated successfully.")
         else:

mautrix_telegram/commands/telegram/auth.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -23,9 +23,9 @@ from telethon.errors import (
     PhoneNumberAppSignupForbiddenError, PhoneNumberBannedError, PhoneNumberFloodError,
     PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError)
 
-from mautrix_telegram.commands import command_handler, CommandEvent, SECTION_AUTH
+from ... import puppet as pu, user as u
-from mautrix_telegram import puppet as pu, user as u
+from ...commands import command_handler, CommandEvent, SECTION_AUTH
-from mautrix_telegram.util import format_duration, ignore_coro
+from ...util import format_duration, ignore_coro
 
 
 @command_handler(needs_auth=False,

mautrix_telegram/commands/telegram/misc.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,18 +19,21 @@ import codecs
 import base64
 import re
 
-from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError,
+from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
                              UserAlreadyParticipantError)
-from telethon.tl.types import User as TLUser, TypeUpdates, MessageMediaGame
+from telethon.tl.patched import Message
+from telethon.tl.types import (User as TLUser, TypeUpdates, MessageMediaGame, MessageMediaPoll,
+                               TypePeer)
 from telethon.tl.types.messages import BotCallbackAnswer
 from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
-                                            GetBotCallbackAnswerRequest)
+                                            GetBotCallbackAnswerRequest, SendVoteRequest)
 from telethon.tl.functions.channels import JoinChannelRequest
 
-from mautrix_telegram import puppet as pu, portal as po
+from ... import puppet as pu, portal as po
-from mautrix_telegram.db import Message as DBMessage
+from ...abstract_user import AbstractUser
-from mautrix_telegram.types import TelegramID
+from ...db import Message as DBMessage
-from mautrix_telegram.commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
+from ...types import TelegramID
+from ...commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
 
 
 @command_handler(help_section=SECTION_MISC,
@@ -167,6 +170,45 @@ async def sync(evt: CommandEvent) -> Optional[Dict]:
 PEER_TYPE_CHAT = b"g"
 
 
+class MessageIDError(ValueError):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+
+async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
+                               ) -> Tuple[TypePeer, Message]:
+    try:
+        enc_id += (4 - len(enc_id) % 4) * "="
+        enc_id = base64.b64decode(enc_id)
+        peer_type, enc_id = bytes([enc_id[0]]), enc_id[1:]
+        tgid = TelegramID(int(codecs.encode(enc_id[0:5], "hex_codec"), 16))
+        msg_id = TelegramID(int(codecs.encode(enc_id[5:10], "hex_codec"), 16))
+        space = None
+        if peer_type == PEER_TYPE_CHAT:
+            space = TelegramID(int(codecs.encode(enc_id[10:15], "hex_codec"), 16))
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (format)") from e
+
+    if peer_type == PEER_TYPE_CHAT:
+        orig_msg = DBMessage.get_by_tgid(msg_id, space)
+        if not orig_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
+        new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
+        if not new_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (your copy of message not found in db)")
+        msg_id = new_msg.tgid
+    try:
+        peer = await user.client.get_input_entity(tgid)
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (chat not found)") from e
+
+    msg = await user.client.get_messages(entity=peer, ids=msg_id)
+    if not msg:
+        raise MessageIDError(f"Invalid {type_name} ID (message not found)")
+    return peer, msg
+
+
 @command_handler(help_section=SECTION_MISC,
                  help_args="<_play ID_>",
                  help_text="Play a Telegram game.")
@@ -179,38 +221,45 @@ async def play(evt: CommandEvent) -> Optional[Dict]:
         return await evt.reply("Bots can't play games :(")
 
     try:
-        play_id = evt.args[0]
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="play")
-        play_id += (4 - len(play_id) % 4) * "="
+    except MessageIDError as e:
-        play_id = base64.b64decode(play_id)
+        return await evt.reply(e.message)
-        peer_type, play_id = bytes([play_id[0]]), play_id[1:]
-        tgid = TelegramID(int(codecs.encode(play_id[0:5], "hex_codec"), 16))
-        msg_id = TelegramID(int(codecs.encode(play_id[5:10], "hex_codec"), 16))
-        space = None
-        if peer_type == PEER_TYPE_CHAT:
-            space = TelegramID(int(codecs.encode(play_id[10:15], "hex_codec"), 16))
-    except ValueError:
-        return await evt.reply("Invalid play ID (format)")
 
-    if peer_type == PEER_TYPE_CHAT:
+    if not isinstance(msg.media, MessageMediaGame):
-        orig_msg = DBMessage.get_by_tgid(msg_id, space)
-        if not orig_msg:
-            return await evt.reply("Invalid play ID (original message not found in db)")
-        new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, evt.sender.tgid)
-        if not new_msg:
-            return await evt.reply("Invalid play ID (your copy of message not found in db)")
-        msg_id = new_msg.tgid
-    try:
-        peer = await evt.sender.client.get_input_entity(tgid)
-    except ValueError:
-        return await evt.reply("Invalid play ID (chat not found)")
-
-    msg = await evt.sender.client.get_messages(entity=peer, ids=msg_id)
-    if not msg or not isinstance(msg.media, MessageMediaGame):
         return await evt.reply("Invalid play ID (message doesn't look like a game)")
 
-    game = await evt.sender.client(GetBotCallbackAnswerRequest(peer=peer, msg_id=msg_id, game=True))
+    game = await evt.sender.client(GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True))
     if not isinstance(game, BotCallbackAnswer):
         return await evt.reply("Game request response invalid")
 
     await evt.reply(f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
                     f"{msg.media.game.description}")
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_poll ID_> <_choice ID_>",
+                 help_text="Vote in a Telegram poll.")
+async def vote(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) < 2:
+        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice ID>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to vote in polls.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't vote in polls :(")
+
+    try:
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="poll")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
+
+    if not isinstance(msg.media, MessageMediaPoll):
+        return await evt.reply("Invalid poll ID (message doesn't look like a poll)")
+
+    options = [base64.b64decode(option + (3 - (len(option) + 3) % 4) * "=")
+               for option in evt.args[1:]]
+    try:
+        resp = await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))
+    except OptionsTooMuchError:
+        return await evt.reply("You passed too many options.")
+    # TODO use response
+    return await evt.mark_read()

mautrix_telegram/config.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -206,6 +206,7 @@ class Config(DictWithRecursion):
         copy("bridge.sync_with_custom_puppets")
         copy("bridge.telegram_link_preview")
         copy("bridge.inline_images")
+        copy("bridge.image_as_file_size")
 
         copy("bridge.bot_messages_as_notices")
         if isinstance(self["bridge.bridge_notices"], bool):

mautrix_telegram/context.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,8 +19,6 @@ from typing import Optional, Tuple, TYPE_CHECKING
 if TYPE_CHECKING:
     import asyncio
 
-    from sqlalchemy.orm import scoped_session
-
     from alchemysession import AlchemySessionContainer
     from mautrix_appservice import AppService
 
@@ -31,20 +29,17 @@ if TYPE_CHECKING:
 
 
 class Context:
-    def __init__(self, az: 'AppService', db: 'scoped_session', config: 'Config',
+    def __init__(self, az: 'AppService', config: 'Config', loop: 'asyncio.AbstractEventLoop',
-                 loop: 'asyncio.AbstractEventLoop', session_container: 'AlchemySessionContainer'
+                 session_container: 'AlchemySessionContainer', bot: Optional['Bot']) -> None:
-                 ) -> None:
         self.az = az  # type: AppService
-        self.db = db  # type: scoped_session
         self.config = config  # type: Config
         self.loop = loop  # type: asyncio.AbstractEventLoop
-        self.bot = None  # type: Optional[Bot]
+        self.bot = bot  # type: Optional[Bot]
-        self.mx = None  # type: MatrixHandler
+        self.mx = None  # type: Optional[MatrixHandler]
         self.session_container = session_container  # type: AlchemySessionContainer
-        self.public_website = None  # type: PublicBridgeWebsite
+        self.public_website = None  # type: Optional[PublicBridgeWebsite]
-        self.provisioning_api = None  # type: ProvisioningAPI
+        self.provisioning_api = None  # type: Optional[ProvisioningAPI]
 
     @property
-    def core(self) -> Tuple['AppService', 'scoped_session', 'Config',
+    def core(self) -> Tuple['AppService', 'Config', 'asyncio.AbstractEventLoop', Optional['Bot']]:
-                            'asyncio.AbstractEventLoop', Optional['Bot']]:
+        return self.az, self.config, self.loop, self.bot
-        return (self.az, self.db, self.config, self.loop, self.bot)

mautrix_telegram/db/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/db/base.py

@@ -1,3 +1,19 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from abc import abstractmethod
 
 from sqlalchemy import Table
@@ -28,14 +44,15 @@ class BaseBase:
         pass
 
     def update(self, **values) -> None:
-        self.db.execute(self.t.update()
+        with self.db.begin() as conn:
-                        .where(self._edit_identity)
+            conn.execute(self.t.update()
-                        .values(**values))
+                         .where(self._edit_identity)
+                         .values(**values))
         for key, value in values.items():
             setattr(self, key, value)
 
     def delete(self) -> None:
-        self.db.execute(self.t.delete().where(self._edit_identity))
+        with self.db.begin() as conn:
-
+            conn.execute(self.t.delete().where(self._edit_identity))
 
 Base = declarative_base(cls=BaseBase)

mautrix_telegram/db/bot_chat.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -29,15 +29,17 @@ class BotChat(Base):
     type = Column(String, nullable=False)
 
     @classmethod
-    def delete(cls, id: TelegramID) -> None:
+    def delete(cls, chat_id: TelegramID) -> None:
-        cls.db.execute(cls.t.delete().where(cls.c.id == id))
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.id == chat_id))
 
     @classmethod
     def all(cls) -> Iterable['BotChat']:
         rows = cls.db.execute(cls.t.select())
         for row in rows:
-            id, type = row
+            chat_id, chat_type = row
-            yield cls(id=id, type=type)
+            yield cls(id=chat_id, type=chat_type)
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(id=self.id, type=self.type))
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(id=self.id, type=self.type))

mautrix_telegram/db/message.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -68,20 +68,23 @@ class Message(Base):
 
     @classmethod
     def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, **values) -> None:
-        cls.db.execute(cls.t.update()
+        with cls.db.begin() as conn:
-                       .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space))
+            conn.execute(cls.t.update()
-                       .values(**values))
+                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space))
+                         .values(**values))
 
     @classmethod
     def update_by_mxid(cls, s_mxid: MatrixEventID, s_mx_room: MatrixRoomID, **values) -> None:
-        cls.db.execute(cls.t.update()
+        with cls.db.begin() as conn:
-                       .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
+            conn.execute(cls.t.update()
-                       .values(**values))
+                         .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
+                         .values(**values))
 
     @property
     def _edit_identity(self):
         return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space)
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room, tgid=self.tgid,
+        with self.db.begin() as conn:
-                                               tg_space=self.tg_space))
+            conn.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room,
+                                                tgid=self.tgid, tg_space=self.tg_space))

mautrix_telegram/db/portal.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -74,7 +74,8 @@ class Portal(Base):
         return and_(self.c.tgid == self.tgid, self.c.tg_receiver == self.tg_receiver)
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(
+        with self.db.begin() as conn:
-            tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
+            conn.execute(self.t.insert().values(
-            megagroup=self.megagroup, mxid=self.mxid, config=self.config, username=self.username,
+                tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
-            title=self.title, about=self.about, photo_id=self.photo_id))
+                megagroup=self.megagroup, mxid=self.mxid, config=self.config,
+                username=self.username, title=self.title, about=self.about, photo_id=self.photo_id))

mautrix_telegram/db/puppet.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -79,8 +79,9 @@ class Puppet(Base):
         return self.c.id == self.id
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(
+        with self.db.begin() as conn:
-            id=self.id, custom_mxid=self.custom_mxid, access_token=self.access_token,
+            conn.execute(self.t.insert().values(
-            displayname=self.displayname, displayname_source=self.displayname_source,
+                id=self.id, custom_mxid=self.custom_mxid, access_token=self.access_token,
-            username=self.username, photo_id=self.photo_id, is_bot=self.is_bot,
+                displayname=self.displayname, displayname_source=self.displayname_source,
-            matrix_registered=self.matrix_registered))
+                username=self.username, photo_id=self.photo_id, is_bot=self.is_bot,
+                matrix_registered=self.matrix_registered))

mautrix_telegram/db/room_state.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -47,14 +47,16 @@ class RoomState(Base):
             return None
 
     def update(self) -> None:
-        self.db.execute(self.t.update()
+        with self.db.begin() as conn:
-                        .where(self.c.room_id == self.room_id)
+            conn.execute(self.t.update()
-                        .values(power_levels=self._power_levels_text))
+                         .where(self.c.room_id == self.room_id)
+                         .values(power_levels=self._power_levels_text))
 
     @property
     def _edit_identity(self):
         return self.c.room_id == self.room_id
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(room_id=self.room_id,
+        with self.db.begin() as conn:
-                                               power_levels=self._power_levels_text))
+            conn.execute(self.t.insert().values(room_id=self.room_id,
+                                                power_levels=self._power_levels_text))

mautrix_telegram/db/telegram_file.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,7 +15,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean
-from sqlalchemy.orm import relationship
 from typing import Optional
 
 from .base import Base
@@ -33,23 +32,25 @@ class TelegramFile(Base):
     width = Column(Integer, nullable=True)
     height = Column(Integer, nullable=True)
     thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
-    thumbnail = relationship("TelegramFile", uselist=False)
+    thumbnail = None  # type: Optional[TelegramFile]
 
     @classmethod
-    def get(cls, id: str) -> Optional['TelegramFile']:
+    def get(cls, loc_id: str) -> Optional['TelegramFile']:
-        rows = cls.db.execute(cls.t.select().where(cls.c.id == id))
+        rows = cls.db.execute(cls.t.select().where(cls.c.id == loc_id))
         try:
-            id, mxc, mime, conv, ts, s, w, h, thumb_id = next(rows)
+            loc_id, mxc, mime, conv, ts, s, w, h, thumb_id = next(rows)
             thumb = None
             if thumb_id:
                 thumb = cls.get(thumb_id)
-            return cls(id=id, mxc=mxc, mime_type=mime, was_converted=conv, timestamp=ts,
+            return cls(id=loc_id, mxc=mxc, mime_type=mime, was_converted=conv, timestamp=ts,
                        size=s, width=w, height=h, thumbnail_id=thumb_id, thumbnail=thumb)
         except StopIteration:
             return None
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(
+        with self.db.begin() as conn:
-            id=self.id, mxc=self.mxc, mime_type=self.mime_type, was_converted=self.was_converted,
+            conn.execute(self.t.insert().values(
-            timestamp=self.timestamp, size=self.size, width=self.width, height=self.height,
+                id=self.id, mxc=self.mxc, mime_type=self.mime_type,
-            thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))
+                was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
+                width=self.width, height=self.height,
+                thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))

mautrix_telegram/db/user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,7 +18,7 @@ from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String
 from sqlalchemy.engine.result import RowProxy
 from typing import Optional, Iterable, Tuple
 
-from ..types import MatrixUserID, MatrixRoomID, TelegramID
+from ..types import MatrixUserID, TelegramID
 from .base import Base
 
 
@@ -65,9 +65,10 @@ class User(Base):
         return self.c.mxid == self.mxid
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(
+        with self.db.begin() as conn:
-            mxid=self.mxid, tgid=self.tgid, tg_username=self.tg_username, tg_phone=self.tg_phone,
+            conn.execute(self.t.insert().values(
-            saved_contacts=self.saved_contacts))
+                mxid=self.mxid, tgid=self.tgid, tg_username=self.tg_username,
+                tg_phone=self.tg_phone, saved_contacts=self.saved_contacts))
 
     @property
     def contacts(self) -> Iterable[TelegramID]:
@@ -78,10 +79,11 @@ class User(Base):
 
     @contacts.setter
     def contacts(self, puppets: Iterable[TelegramID]) -> None:
-        self.db.execute(Contact.t.delete().where(Contact.c.user == self.tgid))
+        with self.db.begin() as conn:
-        if puppets:
+            conn.execute(Contact.t.delete().where(Contact.c.user == self.tgid))
-            self.db.execute(Contact.t.insert(), [{"user": self.tgid, "contact": tgid}
+            insert_puppets = [{"user": self.tgid, "contact": tgid} for tgid in puppets]
-                                                 for tgid in puppets])
+            if insert_puppets:
+                conn.execute(Contact.t.insert(), insert_puppets)
 
     @property
     def portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
@@ -92,14 +94,15 @@ class User(Base):
 
     @portals.setter
     def portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
-        self.db.execute(UserPortal.t.delete().where(UserPortal.c.user == self.tgid))
+        with self.db.begin() as conn:
-        if portals:
+            conn.execute(UserPortal.t.delete().where(UserPortal.c.user == self.tgid))
-            self.db.execute(UserPortal.t.insert(),
+            insert_portals = [{
-                            [{
+                "user": self.tgid,
-                                "user": self.tgid,
+                "portal": tgid,
-                                "portal": tgid,
+                "portal_receiver": tg_receiver
-                                "portal_receiver": tg_receiver
+            } for tgid, tg_receiver in portals]
-                            } for tgid, tg_receiver in portals])
+            if insert_portals:
+                conn.execute(UserPortal.t.insert(), insert_portals)
 
     def delete(self) -> None:
         super().delete()
@@ -125,4 +128,3 @@ class Contact(Base):
 
     user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)  # type: TelegramID
     contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)  # type: TelegramID
-

mautrix_telegram/db/user_profile.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -50,7 +50,8 @@ class UserProfile(Base):
 
     @classmethod
     def delete_all(cls, room_id: MatrixRoomID) -> None:
-        cls.db.execute(cls.t.delete().where(cls.c.room_id == room_id))
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.room_id == room_id))
 
     def update(self) -> None:
         super().update(membership=self.membership, displayname=self.displayname,
@@ -61,8 +62,8 @@ class UserProfile(Base):
         return and_(self.c.room_id == self.room_id, self.c.user_id == self.user_id)
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(room_id=self.room_id, user_id=self.user_id,
+        with self.db.begin() as conn:
-                                               membership=self.membership,
+            conn.execute(self.t.insert().values(room_id=self.room_id, user_id=self.user_id,
-                                               displayname=self.displayname,
+                                                membership=self.membership,
-                                               avatar_url=self.avatar_url))
+                                                displayname=self.displayname,
-
+                                                avatar_url=self.avatar_url))

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -36,7 +36,7 @@ should_bridge_plaintext_highlights = False  # type: bool
 
 command_regex = re.compile(r"^!([A-Za-z0-9@]+)")  # type: Pattern
 not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")  # type: Pattern
-plain_mention_regex = None  # type: Pattern
+plain_mention_regex = None  # type: Optional[Pattern]
 
 
 def plain_mention_to_html(match: Match) -> str:
@@ -76,7 +76,6 @@ def matrix_to_telegram(html: str) -> ParsedMessage:
         if should_bridge_plaintext_highlights:
             html = plain_mention_regex.sub(plain_mention_to_html, html)
 
-        html = add_surrogates(html)
         text, entities = parse_html(add_surrogates(html))
         text = remove_surrogates(text.strip())
         text, entities = cut_long_message(text, entities)
@@ -148,5 +147,5 @@ def init_mx(context: "Context") -> None:
     config = context.config
     dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
     dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
-    plain_mention_regex = re.compile(f"(\s|^)({dn_template})")
+    plain_mention_regex = re.compile(f"^({dn_template})")
     should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False

mautrix_telegram/formatter/from_matrix/html_reader.py

@@ -1,4 +1,58 @@
-try:
+# -*- coding: future_fstrings -*-
-    from .html_reader_lxml import HTMLNode, read_html
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
-except ImportError:
+# Copyright (C) 2019 Tulir Asokan
-    from .html_reader_htmlparser import HTMLNode, read_html
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Tuple
+
+from html.parser import HTMLParser
+
+
+class HTMLNode(list):
+    def __init__(self, tag: str, attrs: List[Tuple[str, str]]):
+        super().__init__()
+        self.tag = tag  # type: str
+        self.text = ""  # type: str
+        self.tail = ""  # type: str
+        self.attrib = dict(attrs)  # type: Dict[str, str]
+
+
+class NodeifyingParser(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.stack = [HTMLNode("html", [])]  # type: List[HTMLNode]
+
+    def handle_starttag(self, tag, attrs):
+        node = HTMLNode(tag, attrs)
+        self.stack[-1].append(node)
+        self.stack.append(node)
+
+    def handle_endtag(self, tag):
+        if tag == self.stack[-1].tag:
+            self.stack.pop()
+
+    def handle_data(self, data):
+        if len(self.stack[-1]) > 0:
+            self.stack[-1][-1].tail += data
+        else:
+            self.stack[-1].text += data
+
+    def error(self, message):
+        pass
+
+
+def read_html(data: str) -> HTMLNode:
+    parser = NodeifyingParser()
+    parser.feed(data)
+    return parser.stack[0]

mautrix_telegram/formatter/from_matrix/html_reader_htmlparser.py

@@ -1,58 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, List, Tuple
-
-from html.parser import HTMLParser
-
-
-class HTMLNode(list):
-    def __init__(self, tag: str, attrs: List[Tuple[str, str]]):
-        super().__init__()
-        self.tag = tag  # type: str
-        self.text = ""  # type: str
-        self.tail = ""  # type: str
-        self.attrib = dict(attrs)  # type: Dict[str, str]
-
-
-class NodeifyingParser(HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self.stack = [HTMLNode("html", [])]  # type: List[HTMLNode]
-
-    def handle_starttag(self, tag, attrs):
-        node = HTMLNode(tag, attrs)
-        self.stack[-1].append(node)
-        self.stack.append(node)
-
-    def handle_endtag(self, tag):
-        if tag == self.stack[-1].tag:
-            self.stack.pop()
-
-    def handle_data(self, data):
-        if len(self.stack[-1]) > 0:
-            self.stack[-1][-1].tail += data
-        else:
-            self.stack[-1].text += data
-
-    def error(self, message):
-        pass
-
-
-def read_html(data: str) -> HTMLNode:
-    parser = NodeifyingParser()
-    parser.feed(data)
-    return parser.stack[0]

mautrix_telegram/formatter/from_matrix/html_reader_lxml.py

@@ -1,23 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from lxml import html
-
-HTMLNode = html.HtmlElement
-
-
-def read_html(data: str) -> HTMLNode:
-    return html.fromstring(data)

mautrix_telegram/formatter/from_matrix/parser.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/formatter/from_matrix/telegram_message.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/formatter/from_telegram.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -44,7 +44,6 @@ try:
 except ImportError:
     htmldiff = None  # type: ignore
 
-
 log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
 should_highlight_edits = False  # type: bool
 
@@ -71,7 +70,7 @@ async def _add_forward_header(source, text: str, html: Optional[str],
         html = escape(text)
     fwd_from_html, fwd_from_text = None, None
     if fwd_from.from_id:
-        user = u.User.get_by_tgid(fwd_from.from_id)
+        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
             fwd_from_html = f"<a href='https://matrix.to/#/{user.mxid}'>{fwd_from_text}</a>"
@@ -193,8 +192,8 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
                              main_intent: Optional[IntentAPI] = None,
                              is_edit: bool = False, prefix_text: Optional[str] = None,
                              prefix_html: Optional[str] = None, override_text: str = None,
-                             override_entities: List[TypeMessageEntity] = None
+                             override_entities: List[TypeMessageEntity] = None,
-                             ) -> Tuple[str, str, Dict]:
+                             no_reply_fallback: bool = False) -> Tuple[str, str, Dict]:
     text = add_surrogates(override_text or evt.message)
     entities = override_entities or evt.entities
     html = _telegram_entities_to_matrix_catch(text, entities) if entities else None
@@ -208,7 +207,7 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
     if evt.fwd_from:
         text, html = await _add_forward_header(source, text, html, evt.fwd_from)
 
-    if evt.reply_to_msg_id:
+    if evt.reply_to_msg_id and not no_reply_fallback:
         text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
                                              is_edit)
 
@@ -258,9 +257,9 @@ def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -
         elif entity_type == MessageEntityItalic:
             html.append(f"<em>{entity_text}</em>")
         elif entity_type == MessageEntityCode:
-            html.append(f"<pre><code>{entity_text}</code></pre>"
+            html.append(("<pre><code>{entity_text}</code></pre>"
-                        if "\n" in entity_text
+                         if "\n" in entity_text
-                        else f"<code>{entity_text}</code>")
+                         else "<code>{entity_text}</code>").format(entity_text=entity_text))
         elif entity_type == MessageEntityPre:
             skip_entity = _parse_pre(html, entity_text, entity.language)
         elif entity_type == MessageEntityMention:

mautrix_telegram/formatter/util.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/matrix.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -32,7 +32,7 @@ class MatrixHandler:
     log = logging.getLogger("mau.mx")  # type: logging.Logger
 
     def __init__(self, context: 'Context') -> None:
-        self.az, self.db, self.config, _, self.tgbot = context.core
+        self.az, self.config, _, self.tgbot = context.core
         self.commands = com.CommandProcessor(context)  # type: com.CommandProcessor
         self.previously_typing = []  # type: List[MatrixUserID]
 
@@ -248,7 +248,7 @@ class MatrixHandler:
                 # Not enough values to unpack, i.e. no arguments
                 command = text
                 args = []
-            await self.commands.handle(room, sender, command, args, is_management,
+            await self.commands.handle(room, event_id, sender, command, args, is_management,
                                        is_portal=portal is not None)
 
     @staticmethod

mautrix_telegram/portal.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -38,17 +38,17 @@ from telethon.tl.functions.messages import (
     EditChatPhotoRequest, EditChatTitleRequest, ExportChatInviteRequest, GetFullChatRequest,
     UpdatePinnedMessageRequest, MigrateChatRequest, SetTypingRequest, EditChatAboutRequest)
 from telethon.tl.functions.channels import (
-    CreateChannelRequest, EditAdminRequest, EditBannedRequest, EditPhotoRequest,
+    CreateChannelRequest, EditAdminRequest, EditBannedRequest, EditPhotoRequest, EditTitleRequest,
-    EditTitleRequest, GetParticipantsRequest, InviteToChannelRequest,
+    GetParticipantsRequest, InviteToChannelRequest, JoinChannelRequest, LeaveChannelRequest,
-    JoinChannelRequest, LeaveChannelRequest, UpdateUsernameRequest)
+    UpdateUsernameRequest)
 from telethon.tl.functions.messages import ReadHistoryRequest as ReadMessageHistoryRequest
 from telethon.tl.functions.channels import ReadHistoryRequest as ReadChannelHistoryRequest
 from telethon.errors import ChatAdminRequiredError, ChatNotModifiedError
 from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
     Channel, ChatAdminRights, ChatBannedRights, ChannelFull, ChannelParticipantAdmin,
-    ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat,
+    ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat, ChatFull,
-    ChatFull, ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto,
+    ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto, Poll, PollAnswer,
     DocumentAttributeFilename, DocumentAttributeImageSize, DocumentAttributeSticker,
     DocumentAttributeVideo, FileLocation, GeoPoint, InputChannel, InputChatUploadedPhoto,
     InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf,
@@ -56,12 +56,12 @@ from telethon.tl.types import (
     MessageActionChatDeletePhoto, MessageActionChatDeleteUser, MessageActionChatEditPhoto,
     MessageActionChatEditTitle, MessageActionChatJoinedByLink, MessageActionChatMigrateTo,
     MessageActionPinMessage, MessageActionGameScore, MessageMediaContact, MessageMediaDocument,
-    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame, PeerChannel,
+    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame, MessageMediaPoll,
-    PeerChat, PeerUser, Photo, PhotoCachedSize, SendMessageCancelAction, SendMessageTypingAction,
+    PeerChannel, PeerChat, PeerUser, Photo, PhotoCachedSize, SendMessageCancelAction,
-    TypeChannelParticipant, TypeChat, TypeChatParticipant, TypeDocumentAttribute, TypeInputPeer,
+    SendMessageTypingAction, TypeChannelParticipant, TypeChat, TypeChatParticipant,
-    TypeMessageAction, TypeMessageEntity, TypePeer, TypePhotoSize, TypeUpdates, TypeUser, PhotoSize,
+    TypeDocumentAttribute, TypeInputPeer, TypeMessageAction, TypeMessageEntity, TypePeer,
-    TypeUserFull, UpdateChatUserTyping, UpdateNewChannelMessage, UpdateNewMessage, UpdateUserTyping,
+    TypePhotoSize, TypeUpdates, TypeUser, PhotoSize, TypeUserFull, UpdateChatUserTyping,
-    User, UserFull, MessageEntityPre)
+    UpdateNewChannelMessage, UpdateNewMessage, UpdateUserTyping, User, UserFull, MessageEntityPre)
 from mautrix_appservice import MatrixRequestError, IntentError, AppService, IntentAPI
 
 from .types import MatrixEventID, MatrixRoomID, MatrixUserID, TelegramID
@@ -418,24 +418,40 @@ class Portal:
 
     def _get_base_power_levels(self, levels: dict = None, entity: TypeChat = None) -> dict:
         levels = levels or {}
-        levels["ban"] = 99
+        if self.peer_type == "user":
-        levels["kick"] = 50
+            levels["ban"] = 100
-        levels["invite"] = 50 if entity.default_banned_rights.invite_users else 0
+            levels["kick"] = 100
-        if "events" not in levels:
+            levels["invite"] = 100
-            levels["events"] = {}
+            levels.setdefault("events", {})
-        levels["events"]["m.room.name"] = 50 if entity.default_banned_rights.change_info else 0
+            levels["events"]["m.room.name"] = 0
-        levels["events"]["m.room.avatar"] = 50 if entity.default_banned_rights.change_info else 0
+            levels["events"]["m.room.avatar"] = 0
-        levels["events"]["m.room.topic"] = 50 if entity.default_banned_rights.change_info else 0
+            levels["events"]["m.room.topic"] = 0
-        levels["events"][
+            levels["state_default"] = 0
-            "m.room.pinned_events"] = 50 if entity.default_banned_rights.pin_messages else 0
+            levels["users_default"] = 0
-        levels["events"]["m.sticker"] = 50 if entity.default_banned_rights.send_stickers else 0
+            levels["events_default"] = 0
-        levels["events"]["m.room.power_levels"] = 75
+        else:
-        levels["events"]["m.room.history_visibility"] = 75
+            dbr = entity.default_banned_rights
-        levels["state_default"] = 50
+            if not dbr:
-        levels["users_default"] = 0
+                self.log.debug(f"default_banned_rights is None in {entity}")
-        levels["events_default"] = (50 if (self.peer_type == "channel" and not entity.megagroup
+                dbr = ChatBannedRights(invite_users=True, change_info=True, pin_messages=True,
-                                           or entity.default_banned_rights.send_messages)
+                                       send_stickers=False, send_messages=False, until_date=0)
-                                    else 0)
+            levels["ban"] = 99
+            levels["kick"] = 50
+            levels["invite"] = 50 if dbr.invite_users else 0
+            levels.setdefault("events", {})
+            levels["events"]["m.room.name"] = 50 if dbr.change_info else 0
+            levels["events"]["m.room.avatar"] = 50 if dbr.change_info else 0
+            levels["events"]["m.room.topic"] = 50 if dbr.change_info else 0
+            levels["events"][
+                "m.room.pinned_events"] = 50 if dbr.pin_messages else 0
+            levels["events"]["m.room.power_levels"] = 75
+            levels["events"]["m.room.history_visibility"] = 75
+            levels["state_default"] = 50
+            levels["users_default"] = 0
+            levels["events_default"] = (50 if (self.peer_type == "channel" and not entity.megagroup
+                                               or entity.default_banned_rights.send_messages)
+                                        else 0)
+            levels["events"]["m.sticker"] = 50 if dbr.send_stickers else levels["events_default"]
         if "users" not in levels:
             levels["users"] = {
                 self.main_intent.mxid: 100
@@ -598,8 +614,11 @@ class Portal:
         return False
 
     @staticmethod
-    def _get_largest_photo_size(photo: Photo) -> TypePhotoSize:
+    def _get_largest_photo_size(photo: Union[Photo, List[TypePhotoSize]]
-        return max(photo.sizes, key=(lambda photo2: (
+                                ) -> Optional[TypePhotoSize]:
+        if not photo:
+            return None
+        return max(photo.sizes if isinstance(photo, Photo) else photo, key=(lambda photo2: (
             len(photo2.bytes) if not isinstance(photo2, PhotoSize) else photo2.size)))
 
     async def remove_avatar(self, _: 'AbstractUser', save: bool = False) -> None:
@@ -756,7 +775,7 @@ class Portal:
 
         tpl_args = dict(mxid=user.mxid,
                         username=user.mxid_localpart,
-                        displayname=displayname)
+                        displayname=escape_html(displayname))
         tpl_args = {**tpl_args, **(arguments or {})}
         message = Template(tpl).safe_substitute(tpl_args)
         return {
@@ -888,7 +907,7 @@ class Portal:
         displayname = await self.get_displayname(sender)
         tpl_args = dict(sender_mxid=sender.mxid,
                         sender_username=sender.mxid_localpart,
-                        sender_displayname=displayname,
+                        sender_displayname=escape_html(displayname),
                         message=body)
         message["formatted_body"] = Template(tpl).safe_substitute(tpl_args)
 
@@ -969,7 +988,9 @@ class Portal:
 
         caption = message["body"] if message["body"].lower() != file_name.lower() else None
 
-        media = await client.upload_file_direct(file, mime, attributes, file_name)
+        media = await client.upload_file_direct(
+            file, mime, attributes, file_name,
+            max_image_size=config["bridge.image_as_file_size"] * 1000 ** 2)
         lock = self.require_send_lock(sender_id)
         async with lock:
             response = await client.send_media(self.peer, media, reply_to=reply_to,
@@ -1237,8 +1258,8 @@ class Portal:
 
         invites = await self._get_telegram_users_in_matrix_room()
         if len(invites) < 2:
-            raise ValueError("Not enough Telegram users to create a chat")
+            raise ValueError("Not enough Telegram users to create a chat. "
-
+                             "Invite more Telegram ghost users to the room, such as the relaybot.")
         if self.peer_type == "chat":
             response = await source.client(CreateChatRequest(title=self.title, users=invites))
             entity = response.chats[0]
@@ -1325,9 +1346,10 @@ class Portal:
                                          relates_to=relates_to, timestamp=evt.date,
                                          external_url=self.get_external_url(evt))
         if evt.message:
-            text, html, _ = await formatter.telegram_to_matrix(evt, source, self.main_intent)
+            text, html, _ = await formatter.telegram_to_matrix(evt, source, self.main_intent,
-            await intent.send_text(self.mxid, text, html=html, timestamp=evt.date,
+                                                               no_reply_fallback=True)
-                                   external_url=self.get_external_url(evt))
+            result = await intent.send_text(self.mxid, text, html=html, timestamp=evt.date,
+                                            external_url=self.get_external_url(evt))
         return result
 
     @staticmethod
@@ -1352,8 +1374,8 @@ class Portal:
         return attrs
 
     @staticmethod
-    def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: Dict
+    def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: Dict,
-                                      ) -> Tuple[Dict, str]:
+                                      thumb: TypePhotoSize) -> Tuple[Dict, str]:
         document = evt.media.document
         name = evt.message or attrs["name"]
         if attrs["is_sticker"]:
@@ -1381,8 +1403,8 @@ class Portal:
             info["thumbnail_url"] = file.thumbnail.mxc
             info["thumbnail_info"] = {
                 "mimetype": file.thumbnail.mime_type,
-                "h": file.thumbnail.height or document.thumb.h,
+                "h": file.thumbnail.height or thumb.h,
-                "w": file.thumbnail.width or document.thumb.w,
+                "w": file.thumbnail.width or thumb.w,
                 "size": file.thumbnail.size,
             }
 
@@ -1393,12 +1415,16 @@ class Portal:
         document = evt.media.document
         attrs = self._parse_telegram_document_attributes(document.attributes)
 
-        file = await util.transfer_file_to_matrix(source.client, intent, document,
+        thumb = self._get_largest_photo_size(document.thumbs)
-                                                  document.thumb, is_sticker=attrs["is_sticker"])
+        if thumb and not isinstance(thumb, (PhotoSize, PhotoCachedSize)):
+            self.log.debug(f"Unsupported thumbnail type {type(thumb)}")
+            thumb = None
+        file = await util.transfer_file_to_matrix(source.client, intent, document, thumb,
+                                                  is_sticker=attrs["is_sticker"])
         if not file:
             return None
 
-        info, name = self._parse_telegram_document_meta(evt, file, attrs)
+        info, name = self._parse_telegram_document_meta(evt, file, attrs, thumb)
 
         await intent.set_typing(self.mxid, is_typing=False)
 
@@ -1481,30 +1507,57 @@ class Portal:
             "net.maunium.telegram.unsupported": True,
         }, timestamp=evt.date, external_url=self.get_external_url(evt))
 
+    async def handle_telegram_poll(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
+                                   relates_to: dict) -> dict:
+        poll = evt.media.poll  # type: Poll
+        poll_id = self._encode_msgid(source, evt)
+
+        def enc(answer: PollAnswer) -> str:
+            return base64.b64encode(answer.option).decode("utf-8").rstrip("=")
+
+        text = (f"Poll ID {poll_id}: {poll.question}\n"
+                + "\n".join(f"* {enc(answer)}: {answer.text}" for answer in poll.answers) +
+                "\n"
+                f"Vote with !tg vote <poll ID> <choice ID>")
+
+        html = (f"<strong>Poll</strong> ID <code>{poll_id}</code>: {poll.question}<br/>\n"
+                f"<ul>"
+                + "\n".join(f"<li><code>{enc(answer)}</code>: {answer.text}</li>"
+                            for answer in poll.answers) +
+                "</ul>\n"
+                f"Vote with <code>!tg vote &lt;poll ID&gt; &lt;choice ID&gt;</code>")
+        await intent.set_typing(self.mxid, is_typing=False)
+        return await intent.send_text(self.mxid, text, html=html, relates_to=relates_to,
+                                      msgtype="m.text", timestamp=evt.date,
+                                      external_url=self.get_external_url(evt))
+
     @staticmethod
     def _int_to_bytes(i: int) -> bytes:
         hex_value = "{0:010x}".format(i)
         return codecs.decode(hex_value, "hex_codec")
 
-    async def handle_telegram_game(self, source: 'AbstractUser', intent: IntentAPI,
+    def _encode_msgid(self, source: 'AbstractUser', evt: Message) -> str:
-                                   evt: Message, _: dict = None):
-        game = evt.media.game
         if self.peer_type == "channel":
-            play_id = base64.b64encode(b"c"
+            play_id = (b"c"
-                                       + self._int_to_bytes(self.tgid)
+                       + self._int_to_bytes(self.tgid)
-                                       + self._int_to_bytes(evt.id))
+                       + self._int_to_bytes(evt.id))
         elif self.peer_type == "chat":
-            play_id = base64.b64encode(b"g"
+            play_id = (b"g"
-                                       + self._int_to_bytes(self.tgid)
+                       + self._int_to_bytes(self.tgid)
-                                       + self._int_to_bytes(evt.id)
+                       + self._int_to_bytes(evt.id)
-                                       + self._int_to_bytes(source.tgid))
+                       + self._int_to_bytes(source.tgid))
         elif self.peer_type == "user":
-            play_id = base64.b64encode(b"u"
+            play_id = (b"u"
-                                       + self._int_to_bytes(self.tgid)
+                       + self._int_to_bytes(self.tgid)
-                                       + self._int_to_bytes(evt.id))
+                       + self._int_to_bytes(evt.id))
         else:
             raise ValueError("Portal has invalid peer type")
-        play_id = play_id.decode("utf-8").rstrip("=")
+        return base64.b64encode(play_id).decode("utf-8").rstrip("=")
+
+    async def handle_telegram_game(self, source: 'AbstractUser', intent: IntentAPI,
+                                   evt: Message, relates_to: dict = None):
+        game = evt.media.game
+        play_id = self._encode_msgid(source, evt)
         command = f"!tg play {play_id}"
         override_text = f"Run {command} in your bridge management room to play {game.title}"
         override_entities = [MessageEntityPre(offset=len("Run "), length=len(command), language="")]
@@ -1609,7 +1662,7 @@ class Portal:
             await sender.update_info(source, entity)
 
         allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo, MessageMediaGame,
-                         MessageMediaUnsupported)
+                         MessageMediaPoll, MessageMediaUnsupported)
         media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
                                                                   allowed_media) else None
         intent = sender.intent if sender else self.main_intent
@@ -1621,6 +1674,7 @@ class Portal:
                 MessageMediaPhoto: self.handle_telegram_photo,
                 MessageMediaDocument: self.handle_telegram_document,
                 MessageMediaGeo: self.handle_telegram_location,
+                MessageMediaPoll: self.handle_telegram_poll,
                 MessageMediaUnsupported: self.handle_telegram_unsupported,
                 MessageMediaGame: self.handle_telegram_game,
             }[type(media)](source, intent, evt,
@@ -1986,7 +2040,7 @@ class Portal:
 
 def init(context: Context) -> None:
     global config
-    Portal.az, _, config, Portal.loop, Portal.bot = context.core
+    Portal.az, config, Portal.loop, Portal.bot = context.core
     Portal.max_initial_member_sync = config["bridge.max_initial_member_sync"]
     Portal.sync_channel_members = config["bridge.sync_channel_members"]
     Portal.sync_matrix_state = config["bridge.sync_matrix_state"]

mautrix_telegram/puppet.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,8 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import (Awaitable, Coroutine, Dict, List, Iterable, Optional, Pattern, Union,
+from typing import Awaitable, Any, Dict, List, Iterable, Optional, Pattern, Union, TYPE_CHECKING
-                    TYPE_CHECKING)
 from difflib import SequenceMatcher
 from enum import Enum
 from aiohttp import ServerDisconnectedError
@@ -23,8 +22,6 @@ import asyncio
 import logging
 import re
 
-from sqlalchemy import orm
-
 from telethon.tl.types import UserProfilePhoto, User, FileLocation, UpdateUserName, PeerUser
 from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
 
@@ -45,7 +42,6 @@ config = None  # type: Config
 
 class Puppet:
     log = logging.getLogger("mau.puppet")  # type: logging.Logger
-    db = None  # type: orm.Session
     az = None  # type: AppService
     mx = None  # type: MatrixHandler
     loop = None  # type: asyncio.AbstractEventLoop
@@ -400,8 +396,7 @@ class Puppet:
 
         if create:
             puppet = cls(tgid)
-            cls.db.add(puppet.db_instance)
+            puppet.db_instance.insert()
-            cls.db.commit()
             return puppet
 
         return None
@@ -481,9 +476,9 @@ class Puppet:
     # endregion
 
 
-def init(context: 'Context') -> List[Coroutine]:  # [None, None, PuppetError]
+def init(context: 'Context') -> List[Awaitable[Any]]:  # [None, None, PuppetError]
     global config
-    Puppet.az, Puppet.db, config, Puppet.loop, _ = context.core
+    Puppet.az, config, Puppet.loop, _ = context.core
     Puppet.mx = context.mx
     Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}")
     Puppet.hs_domain = config["homeserver"]["domain"]

mautrix_telegram/scripts/dbms_migrate/__main__.py

@@ -11,12 +11,19 @@ parser.add_argument("-f", "--from-url", type=str, required=True, metavar="<url>"
                     help="the old database path")
 parser.add_argument("-t", "--to-url", type=str, required=True, metavar="<url>",
                     help="the new database path")
+parser.add_argument("-v", "--verbose", action="store_true", help="Verbose logs while migrating")
 args = parser.parse_args()
+verbose = args.verbose or False
+
+
+def log(message, end="\n"):
+    if verbose:
+        print(message, end=end, flush=True)
 
 
 def connect(to):
-    import mautrix_telegram.base as base
+    import mautrix_telegram.db.base as base
-    base.Base = declarative_base()
+    base.Base = declarative_base(cls=base.BaseBase)
     from mautrix_telegram.db import (Portal, Message, UserPortal, User, RoomState, UserProfile,
                                      Contact, Puppet, BotChat, TelegramFile)
     db_engine = sql.create_engine(to)
@@ -45,15 +52,30 @@ def connect(to):
         "TelegramFile": TelegramFile,
     }
 
-
+log("Connecting to old database")
 session, tables = connect(args.from_url)
 
 data = {}
 for name, table in tables.items():
+    log("Reading table {name}...".format(name=name), end=" ")
     data[name] = session.query(table).all()
+    log("Done!")
 
+log("Connecting to new database")
 session, tables = connect(args.to_url)
+
 for name, table in tables.items():
+    log("Writing table {name}".format(name=name), end="")
+    length = len(data[name])
+    n = 0
     for row in data[name]:
         session.merge(row)
+        n += 5
+        if n >= length:
+            log(".", end="")
+            n = 0
+    log(" Done!")
+
+log("Committing changes to database...", end=" ")
 session.commit()
+log("Done!")

mautrix_telegram/scripts/telematrix_import/__main__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/sqlstatestore.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/tgclient.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -27,11 +27,11 @@ from telethon.tl.patched import Message
 class MautrixTelegramClient(TelegramClient):
     async def upload_file_direct(self, file: bytes, mime_type: str = None,
                                  attributes: List[TypeDocumentAttribute] = None,
-                                 file_name: str = None
+                                 file_name: str = None, max_image_size: float = 10 * 1000 ** 2,
                                  ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]:
         file_handle = await super().upload_file(file, file_name=file_name, use_cache=False)
 
-        if mime_type == "image/png" or mime_type == "image/jpeg":
+        if (mime_type == "image/png" or mime_type == "image/jpeg") and len(file) < max_image_size:
             return InputMediaUploadedPhoto(file_handle)
         else:
             attributes = attributes or []

mautrix_telegram/user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -65,7 +65,7 @@ class User(AbstractUser):
         self.db_portals = db_portals or []
         self._db_instance = db_instance  # type: Optional[DBUser]
 
-        self.command_status = None  # type: Dict
+        self.command_status = None  # type: Optional[Dict]
 
         (self.relaybot_whitelisted,
          self.whitelisted,
@@ -102,7 +102,9 @@ class User(AbstractUser):
 
     @property
     def db_contacts(self) -> Iterable[TelegramID]:
-        return (puppet.id for puppet in self.contacts)
+        return (puppet.id
+                for puppet in self.contacts
+                if puppet)
 
     @db_contacts.setter
     def db_contacts(self, contacts: Iterable[TelegramID]) -> None:
@@ -110,7 +112,9 @@ class User(AbstractUser):
 
     @property
     def db_portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
-        return (portal.tgid_full for portal in self.portals.values() if not portal.deleted)
+        return (portal.tgid_full
+                for portal in self.portals.values()
+                if portal and not portal.deleted)
 
     @db_portals.setter
     def db_portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
@@ -129,12 +133,15 @@ class User(AbstractUser):
 
     def new_db_instance(self) -> DBUser:
         return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
-                      contacts=self.db_contacts, saved_contacts=self.saved_contacts,
+                      saved_contacts=self.saved_contacts, portals=self.db_portals)
-                      portals=self.db_portals)
 
-    def save(self) -> None:
+    def save(self, contacts: bool = False, portals: bool = False) -> None:
         self.db_instance.update(tgid=self.tgid, tg_username=self.username, tg_phone=self.phone,
                                 saved_contacts=self.saved_contacts)
+        if contacts:
+            self.db_instance.contacts = self.db_contacts
+        if portals:
+            self.db_instance.portals = self.db_portals
 
     def delete(self, delete_db: bool = True) -> None:
         try:
@@ -232,7 +239,7 @@ class User(AbstractUser):
         if puppet.is_real_user:
             await puppet.switch_mxid(None, None)
         for _, portal in self.portals.items():
-            if not portal.mxid or portal.has_bot:
+            if not portal or portal.deleted or not portal.mxid or portal.has_bot:
                 continue
             try:
                 await portal.main_intent.kick(portal.mxid, self.mxid, "Logged out of Telegram.")
@@ -240,7 +247,7 @@ class User(AbstractUser):
                 pass
         self.portals = {}
         self.contacts = []
-        self.save()
+        self.save(portals=True, contacts=True)
         if self.tgid:
             try:
                 del self.by_tgid[self.tgid]
@@ -295,7 +302,7 @@ class User(AbstractUser):
             creators.append(
                 portal.create_matrix_room(self, entity, invites=[self.mxid],
                                           synchronous=synchronous_create))
-        self.save()
+        self.save(portals=True)
         await asyncio.gather(*creators, loop=self.loop)
 
     def register_portal(self, portal: po.Portal) -> None:
@@ -305,12 +312,12 @@ class User(AbstractUser):
         except KeyError:
             pass
         self.portals[portal.tgid_full] = portal
-        self.save()
+        self.save(portals=True)
 
     def unregister_portal(self, portal: po.Portal) -> None:
         try:
             del self.portals[portal.tgid_full]
-            self.save()
+            self.save(portals=True)
         except KeyError:
             pass
 
@@ -335,7 +342,7 @@ class User(AbstractUser):
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
             self.contacts.append(puppet)
-        self.save()
+        self.save(contacts=True)
 
     # endregion
     # region Class instance lookup

mautrix_telegram/util/file_transfer.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -23,8 +23,8 @@ import asyncio
 import magic
 from sqlalchemy.exc import IntegrityError, InvalidRequestError
 
-from telethon.tl.types import (Document, FileLocation, InputFileLocation,
+from telethon.tl.types import (Document, FileLocation, InputFileLocation, InputDocumentFileLocation,
-                               InputDocumentFileLocation, PhotoSize, PhotoCachedSize)
+                               TypePhotoSize, PhotoSize, PhotoCachedSize)
 from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, LocationInvalidError,
                              SecurityError)
 from mautrix_appservice import IntentAPI
@@ -147,9 +147,11 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
 
 transfer_locks = {}  # type: Dict[str, asyncio.Lock]
 
+TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
+
 
 async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                  location: TypeLocation, thumbnail: Optional[TypeLocation] = None,
+                                  location: TypeLocation, thumbnail: TypeThumbnail = None,
                                   is_sticker: bool = False) -> Optional[DBTelegramFile]:
     location_id = _location_to_id(location)
     if not location_id:
@@ -171,8 +173,8 @@ async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentA
 
 async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
                                             loc_id: str, location: TypeLocation,
-                                            thumbnail: Optional[TypeLocation],
+                                            thumbnail: TypeThumbnail, is_sticker: bool
-                                            is_sticker: bool) -> Optional[DBTelegramFile]:
+                                            ) -> Optional[DBTelegramFile]:
     db_file = DBTelegramFile.get(loc_id)
     if db_file:
         return db_file

mautrix_telegram/util/format_duration.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/util/recursive_dict.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/util/signed_token.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/web/common/auth_api.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -87,7 +87,8 @@ class AuthAPI(abc.ABC):
         except PhoneNumberAppSignupForbiddenError:
             return self.get_login_response(mxid=user.mxid, state="request", status=403,
                                            errcode="phone_number_app_signup_forbidden",
-                                           error="You have disabled 3rd party apps on your account.")
+                                           error="You have disabled 3rd party apps on your "
+                                                 "account.")
         except PhoneNumberUnoccupiedError:
             return self.get_login_response(mxid=user.mxid, state="request", status=404,
                                            errcode="phone_number_unoccupied",

mautrix_telegram/web/provisioning/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -247,7 +247,7 @@ class ProvisioningAPI(AuthAPI):
             "group": "chat",
         }[type]
 
-        portal = Portal(tgid=None, mxid=room_id, title=title, about=about, peer_type=type)
+        portal = Portal(tgid=TelegramID(0), mxid=room_id, title=title, about=about, peer_type=type)
         try:
             await portal.create_telegram_chat(user, supergroup=supergroup)
         except ValueError as e:

mautrix_telegram/web/public/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/web/public/login.css

@@ -1,6 +1,6 @@
 /*
  * mautrix-telegram - A Matrix-Telegram puppeting bridge
- * Copyright (C) 2018 Tulir Asokan
+ * Copyright (C) 2019 Tulir Asokan
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/web/public/login.html.mako

@@ -1,6 +1,6 @@
 <!--
 mautrix-telegram - A Matrix-Telegram puppeting bridge
-Copyright (C) 2018 Tulir Asokan
+Copyright (C) 2019 Tulir Asokan
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
 -->
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 	<title>Login - Mautrix-Telegram bridge</title>
 	<link rel="icon" type="image/png" href="favicon.png"/>
@@ -25,9 +25,10 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 	<meta property="og:image" content="favicon.png">
 	<meta charset="utf-8">
 	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
-	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
 	<link rel="stylesheet"
-		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+		  href="https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.1/normalize.min.css">
+	<link rel="stylesheet"
+		  href="https://cdnjs.cloudflare.com/ajax/libs/milligram/1.3.0/milligram.min.css">
 	<link rel="stylesheet" href="login.css"/>
 
 	<script>
@@ -100,7 +101,8 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 					</button>
 				% elif state == "bot_token":
 					<label for="value">Bot token</label>
-					<input type="text" id="value" name="bot_token" placeholder="Enter bot API token"/>
+					<input type="text" id="value" name="bot_token"
+						   placeholder="Enter bot API token"/>
 					<button type="submit">Sign in</button>
 				% elif state == "code":
 					<label for="value">Phone code</label>

mautrix_telegram/web/public/matrix-login.html.mako

@@ -1,6 +1,6 @@
 <!--
 mautrix-telegram - A Matrix-Telegram puppeting bridge
-Copyright (C) 2018 Tulir Asokan
+Copyright (C) 2019 Tulir Asokan
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
 -->
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 	<title>Matrix login - Mautrix-Telegram bridge</title>
 	<link rel="icon" type="image/png" href="favicon.png"/>
@@ -25,9 +25,10 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 	<meta property="og:image" content="favicon.png">
 	<meta charset="utf-8">
 	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
-	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
 	<link rel="stylesheet"
-		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+		  href="https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.1/normalize.min.css">
+	<link rel="stylesheet"
+		  href="https://cdnjs.cloudflare.com/ajax/libs/milligram/1.3.0/milligram.min.css">
 	<link rel="stylesheet" href="login.css"/>
 </head>
 <body>

setup.cfg

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

setup.py

@@ -4,7 +4,6 @@ import mautrix_telegram
 
 extras = {
     "highlight_edits": ["lxml>=4.1.1,<5"],
-    "better_formatter": ["lxml>=4.1.1,<5"],
     "fast_crypto": ["cryptg>=0.1,<0.2"],
     "webp_convert": ["Pillow>=4.3.0,<6"],
     "hq_thumbnails": ["moviepy>=1.0,<2.0"],
@@ -39,11 +38,14 @@ setuptools.setup(
         "ruamel.yaml>=0.15.35,<0.16",
         "future-fstrings>=0.4.2",
         "python-magic>=0.4.15,<0.5",
-        "telethon>=1.5.5,<1.6",
+        "telethon>=1.5.5,<1.7",
-        "telethon-session-sqlalchemy>=0.2.7,<0.3",
+        "telethon-session-sqlalchemy>=0.2.12,<0.3",
     ],
     extras_require=extras,
 
+    setup_requires=["pytest-runner"],
+    tests_require=["pytest", "pytest-asyncio", "pytest-mock"],
+
     classifiers=[
         "Development Status :: 4 - Beta",
         "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",

tests/commands/test_handler.py

@@ -0,0 +1,370 @@
+from typing import Tuple
+from unittest.mock import Mock
+
+import pytest
+from _pytest.fixtures import FixtureRequest
+from pytest_mock import MockFixture
+
+import mautrix_telegram.commands.handler
+from mautrix_telegram.commands.handler import (CommandEvent, CommandHandler, CommandProcessor,
+                                               HelpSection)
+from mautrix_telegram.config import Config
+from mautrix_telegram.context import Context
+from mautrix_telegram.types import MatrixEventID, MatrixRoomID, MatrixUserID
+import mautrix_telegram.user as u
+
+from tests.utils.helpers import AsyncMock, list_true_once_each
+
+
+@pytest.fixture
+def context(request: FixtureRequest) -> Context:
+    """Returns a Context with mocked Attributes.
+
+    Uses the attribute cls.config as Config.
+    """
+    # Config(path, registration_path, base_path)
+    config = getattr(request.cls, 'config', Config("", "", ""))
+    return Context(az=Mock(), config=config, loop=Mock(), session_container=Mock(), bot=Mock())
+
+
+@pytest.fixture
+def command_processor(context: Context) -> CommandProcessor:
+    """Returns a mocked CommandProcessor."""
+    return CommandProcessor(context)
+
+
+class TestCommandEvent:
+    config = Config("", "", "")
+    config["bridge.command_prefix"] = "tg"
+    config["bridge.permissions"] = {"*": "noperm"}
+
+    def test_reply(
+        self, command_processor: CommandProcessor, mocker: MockFixture
+    ) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        evt = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=u.User(MatrixUserID("@sender:example.org")),
+            command="help",
+            args=[],
+            is_management=True,
+            is_portal=False,
+        )
+
+        mock_az = command_processor.az
+
+        message = "**This** <i>was</i><br/><strong>all</strong>fun*!"
+
+        # html, no markdown
+        evt.reply(message, allow_html=True, render_markdown=False)
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "**This** <i>was</i><br/><strong>all</strong>fun*!",
+            html="**This** <i>was</i><br/><strong>all</strong>fun*!\n",
+        )
+
+        # html, markdown (default)
+        evt.reply(message, allow_html=True, render_markdown=True)
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "**This** <i>was</i><br/><strong>all</strong>fun*!",
+            html=(
+                "<p><strong>This</strong> <i>was</i><br/>"
+                "<strong>all</strong>fun*!</p>\n"
+            ),
+        )
+
+        # no html, no markdown
+        evt.reply(message, allow_html=False, render_markdown=False)
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "**This** <i>was</i><br/><strong>all</strong>fun*!",
+            html=None,
+        )
+
+        # no html, markdown
+        evt.reply(message, allow_html=False, render_markdown=True)
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "**This** <i>was</i><br/><strong>all</strong>fun*!",
+            html="<p><strong>This</strong> &lt;i&gt;was&lt;/i&gt;&lt;br/&gt;"
+                 "&lt;strong&gt;all&lt;/strong&gt;fun*!</p>\n"
+        )
+
+    def test_reply_with_cmdprefix(self, command_processor: CommandProcessor, mocker: MockFixture
+                                  ) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        evt = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=u.User(MatrixUserID("@sender:example.org")),
+            command="help",
+            args=[],
+            is_management=False,
+            is_portal=False,
+        )
+
+        mock_az = command_processor.az
+
+        evt.reply("$cmdprefix+sp ....$cmdprefix+sp...$cmdprefix $cmdprefix", allow_html=False,
+                  render_markdown=False)
+
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "tg ....tg+sp...tg tg",
+            html=None,
+        )
+
+    def test_reply_with_cmdprefix_in_management_room(self, command_processor: CommandProcessor,
+                                                     mocker: MockFixture) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        evt = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=u.User(MatrixUserID("@sender:example.org")),
+            command="help",
+            args=[],
+            is_management=True,
+            is_portal=False,
+        )
+
+        mock_az = command_processor.az
+
+        evt.reply(
+            "$cmdprefix+sp ....$cmdprefix+sp...$cmdprefix $cmdprefix",
+            allow_html=True,
+            render_markdown=True,
+        )
+
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "....tg+sp...tg tg",
+            html="<p>....tg+sp...tg tg</p>\n",
+        )
+
+
+class TestCommandHandler:
+    config = Config("", "", "")
+    config["bridge.permissions"] = {"*": "noperm"}
+
+    @pytest.mark.parametrize(
+        (
+            "needs_auth,"
+            "needs_puppeting,"
+            "needs_matrix_puppeting,"
+            "needs_admin,"
+            "management_only,"
+        ),
+        [l for l in list_true_once_each(length=5)]
+    )
+    @pytest.mark.asyncio
+    async def test_permissions_denied(
+        self,
+        needs_auth: bool,
+        needs_puppeting: bool,
+        needs_matrix_puppeting: bool,
+        needs_admin: bool,
+        management_only: bool,
+        command_processor: CommandProcessor,
+        boolean: bool,
+        mocker: MockFixture,
+    ) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        command = "testcmd"
+
+        mock_handler = Mock()
+
+        command_handler = CommandHandler(
+            handler=mock_handler,
+            needs_auth=needs_auth,
+            needs_puppeting=needs_puppeting,
+            needs_matrix_puppeting=needs_matrix_puppeting,
+            needs_admin=needs_admin,
+            management_only=management_only,
+            name=command,
+            help_text="No real command",
+            help_args="mock mockmock",
+            help_section=HelpSection("Mock Section", 42, ""),
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+        sender.puppet_whitelisted = False
+        sender.matrix_puppet_whitelisted = False
+        sender.is_admin = False
+
+        event = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=sender,
+            command=command,
+            args=[],
+            is_management=False,
+            is_portal=boolean,
+        )
+
+        assert await command_handler.get_permission_error(event)
+        assert not command_handler.has_permission(False, False, False, False, False)
+
+    @pytest.mark.parametrize(
+        (
+            "is_management,"
+            "puppet_whitelisted,"
+            "matrix_puppet_whitelisted,"
+            "is_admin,"
+            "is_logged_in,"
+        ),
+        [l for l in list_true_once_each(length=5)]
+    )
+    @pytest.mark.asyncio
+    async def test_permission_granted(
+        self,
+        is_management: bool,
+        puppet_whitelisted: bool,
+        matrix_puppet_whitelisted: bool,
+        is_admin: bool,
+        is_logged_in: bool,
+        command_processor: CommandProcessor,
+        boolean: bool,
+        mocker: MockFixture,
+    ) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        command = "testcmd"
+
+        mock_handler = Mock()
+
+        command_handler = CommandHandler(
+            handler=mock_handler,
+            needs_auth=False,
+            needs_puppeting=False,
+            needs_matrix_puppeting=False,
+            needs_admin=False,
+            management_only=False,
+            name=command,
+            help_text="No real command",
+            help_args="mock mockmock",
+            help_section=HelpSection("Mock Section", 42, ""),
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+        sender.puppet_whitelisted = puppet_whitelisted
+        sender.matrix_puppet_whitelisted = matrix_puppet_whitelisted
+        sender.is_admin = is_admin
+        mocker.patch.object(u.User, 'is_logged_in', return_value=is_logged_in)
+
+        event = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=sender,
+            command=command,
+            args=[],
+            is_management=is_management,
+            is_portal=boolean,
+        )
+
+        assert not await command_handler.get_permission_error(event)
+        assert command_handler.has_permission(
+            is_management=is_management,
+            puppet_whitelisted=puppet_whitelisted,
+            matrix_puppet_whitelisted=matrix_puppet_whitelisted,
+            is_admin=is_admin,
+            is_logged_in=is_logged_in,
+        )
+
+
+class TestCommandProcessor:
+    config = Config("", "", "")
+    config["bridge.command_prefix"] = "tg"
+    config["bridge.permissions"] = {"*": "relaybot"}
+
+    @pytest.mark.asyncio
+    async def test_handle(self, command_processor: CommandProcessor, boolean2: Tuple[bool, bool],
+                          mocker: MockFixture) -> None:
+        mocker.patch('mautrix_telegram.user.config', self.config)
+        mocker.patch(
+            'mautrix_telegram.commands.handler.command_handlers',
+            {"help": AsyncMock(), "unknown-command": AsyncMock()}
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+
+        result = await command_processor.handle(
+            room=MatrixRoomID("#mock_room:example.org"),
+            event_id=MatrixEventID("$H45H:example.org"),
+            sender=sender,
+            command="hElp",
+            args=[],
+            is_management=boolean2[0],
+            is_portal=boolean2[1],
+        )
+
+        assert result is None
+        command_handlers = mautrix_telegram.commands.handler.command_handlers
+        command_handlers["help"].mock.assert_called_once()  # type: ignore
+
+    @pytest.mark.asyncio
+    async def test_handle_unknown_command(self, command_processor: CommandProcessor,
+                                          boolean2: Tuple[bool, bool], mocker: MockFixture) -> None:
+        mocker.patch('mautrix_telegram.user.config', self.config)
+        mocker.patch(
+            'mautrix_telegram.commands.handler.command_handlers',
+            {"help": AsyncMock(), "unknown-command": AsyncMock()}
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+        sender.command_status = {}
+
+        result = await command_processor.handle(
+            room=MatrixRoomID("#mock_room:example.org"),
+            event_id=MatrixEventID("$H45H:example.org"),
+            sender=sender,
+            command="foo",
+            args=[],
+            is_management=boolean2[0],
+            is_portal=boolean2[1],
+        )
+
+        assert result is None
+        command_handlers = mautrix_telegram.commands.handler.command_handlers
+        command_handlers["help"].mock.assert_not_called()  # type: ignore
+        command_handlers["unknown-command"].mock.assert_called_once()  # type: ignore
+
+    @pytest.mark.asyncio
+    async def test_handle_delegated_handler(self, command_processor: CommandProcessor,
+                                            boolean2: Tuple[bool, bool],
+                                            mocker: MockFixture) -> None:
+        mocker.patch('mautrix_telegram.user.config', self.config)
+        mocker.patch(
+            'mautrix_telegram.commands.handler.command_handlers',
+            {"help": AsyncMock(), "unknown-command": AsyncMock()}
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+        sender.command_status = {"foo": AsyncMock(), "next": AsyncMock()}
+
+        result = await command_processor.handle(
+            room=MatrixRoomID("#mock_room:example.org"),
+            event_id=MatrixEventID("$H45H:example.org"),
+            sender=sender,  # u.User
+            command="foo",
+            args=[],
+            is_management=boolean2[0],
+            is_portal=boolean2[1]
+        )
+
+        assert result is None
+        command_handlers = mautrix_telegram.commands.handler.command_handlers
+        command_handlers["help"].mock.assert_not_called()  # type: ignore
+        command_handlers["unknown-command"].mock.assert_not_called()  # type: ignore
+        sender.command_status["foo"].mock.assert_not_called()  # type: ignore
+        sender.command_status["next"].mock.assert_called_once()  # type: ignore

tests/conftest.py

@@ -0,0 +1,3 @@
+pytest_plugins = [
+    "tests.utils.fixtures",
+]

tests/utils/fixtures.py

@@ -0,0 +1,27 @@
+"""This module provides utility fixtures for testing."""
+from typing import Tuple
+
+from _pytest.fixtures import FixtureRequest
+import pytest
+
+
+@pytest.fixture(params=[True, False])
+def boolean(request: FixtureRequest) -> bool:
+    return request.param
+
+
+@pytest.fixture
+def boolean1(boolean: bool) -> Tuple[bool]:
+    return boolean,
+
+
+@pytest.fixture(params=[True, False])
+def boolean2(request: FixtureRequest, boolean: bool) -> Tuple[bool, bool]:
+    return boolean, request.param
+
+
+@pytest.fixture(params=[True, False])
+def boolean3(request: FixtureRequest, boolean2: Tuple[bool, bool]) -> Tuple[bool, bool, bool]:
+    return boolean2[0], boolean2[1], request.param
+
+# …

tests/utils/helpers.py

@@ -0,0 +1,24 @@
+"""This module provides utility functions for testing."""
+from typing import Generator, Tuple
+from unittest.mock import Mock
+
+
+def AsyncMock(*args, **kwargs):
+    """Mocks a asyncronous coroutine which can be called with 'await'."""
+    m = Mock(*args, **kwargs)
+
+    async def mock_coro(*args, **kwargs):
+        return m(*args, **kwargs)
+
+    mock_coro.mock = m
+    return mock_coro
+
+
+def list_true_once_each(length: int) -> Generator[Tuple[bool, ...], None, None]:
+    """Yields tuples of bools with exactly one entry being True, starting left.
+
+    Args:
+        length: Length of the resulting tuples
+    """
+    for i in range(length):
+        yield tuple(i == j for j in range(length))