Bump version to 0.10.0

Fixes #375

.gitlab-ci.yml

@@ -17,6 +17,13 @@ build amd64:
   - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
   - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
   - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  after_script:
+  - |
+    if [ "$CI_COMMIT_BRANCH" = "master" ]; then
+      apk add --update curl
+      rm -rf /var/cache/apk/*
+      curl "$NOVA_ADMIN_API_URL" -H "Content-Type: application/json" -d '{"password":"'"$NOVA_ADMIN_NIGHTLY_PASS"'","bridge":"'$NOVA_BRIDGE_TYPE'","image":"'$CI_REGISTRY_IMAGE':'$CI_COMMIT_SHA'-amd64"}'
+    fi
 
 build arm64:
   stage: build

Dockerfile

@@ -1,12 +1,7 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.12
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.13
 
 ARG TARGETARCH=amd64
 
-RUN echo $'\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\n\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/testing\n\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
-
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
       py3-virtualenv \
@@ -14,11 +9,11 @@ RUN apk add --no-cache \
       py3-aiohttp \
       py3-magic \
       py3-sqlalchemy \
-      py3-telethon-session-sqlalchemy@edge \
+      py3-telethon-session-sqlalchemy \
-      py3-alembic@edge \
+      py3-alembic \
       py3-psycopg2 \
       py3-ruamel.yaml \
-      py3-commonmark@edge \
+      py3-commonmark \
       # Indirect dependencies
       py3-idna \
       #moviepy
@@ -27,12 +22,12 @@ RUN apk add --no-cache \
         py3-requests \
         #imageio
           py3-numpy \
-      #py3-telethon@edge \ (outdated)
+      #py3-telethon \ (outdated)
         # Optional for socks proxies
         py3-pysocks \
         # cryptg
           py3-cffi \
-	  py3-qrcode@edge \
+	  py3-qrcode \
       py3-brotli \
       # Other dependencies
       ffmpeg \
@@ -46,9 +41,8 @@ RUN apk add --no-cache \
       py3-future \
       bash \
       curl \
-      jq && \
+      jq \
-  curl -sLo yq https://github.com/mikefarah/yq/releases/download/3.3.2/yq_linux_${TARGETARCH} && \
+      yq
-  chmod +x yq && mv yq /usr/bin/yq
 
 COPY requirements.txt /opt/mautrix-telegram/requirements.txt
 COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt

README.md

@@ -10,15 +10,16 @@ A Matrix-Telegram hybrid puppeting/relaybot bridge.
 ## Sponsors
 * [Joel Lehtonen / Zouppen](https://github.com/zouppen)
 
-### Wiki
+### Documentation
-All setup and usage instructions are located in the GitHub
+All setup and usage instructions are located on
-[wiki](https://github.com/tulir/mautrix-telegram/wiki). Some quick links:
+[docs.mau.fi](https://docs.mau.fi/bridges/python/telegram/index.html).
+Some quick links:
 
-* [Bridge setup](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup)
+* [Bridge setup](https://docs.mau.fi/bridges/python/setup/index.html?bridge=telegram)
-  (or [with Docker](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup-with-Docker))
+  (or [with Docker](https://docs.mau.fi/bridges/python/setup/docker.html?bridge=telegram))
-* Basic usage: [Authentication](https://github.com/tulir/mautrix-telegram/wiki/Authentication),
+* Basic usage: [Authentication](https://docs.mau.fi/bridges/python/telegram/authentication.html),
-  [Creating chats](https://github.com/tulir/mautrix-telegram/wiki/Creating-and-managing-chats),
+  [Creating chats](https://docs.mau.fi/bridges/python/telegram/creating-and-managing-chats.html),
-  [Relaybot setup](https://github.com/tulir/mautrix-telegram/wiki/Relay-bot)
+  [Relaybot setup](https://docs.mau.fi/bridges/python/telegram/relay-bot.html)
 
 ### Features & Roadmap
 [ROADMAP.md](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)

alembic/versions/7de69cf5809e_add_matrix_redaction_state_to_message_.py

@@ -0,0 +1,25 @@
+"""Add Matrix redaction state to message table
+
+Revision ID: 7de69cf5809e
+Revises: 888275d58e57
+Create Date: 2020-12-19 12:39:57.368568
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '7de69cf5809e'
+down_revision = '888275d58e57'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('message', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('redacted', sa.Boolean(), server_default=sa.false(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table('message', schema=None) as batch_op:
+        batch_op.drop_column('redacted')

alembic/versions/990f4395afc6_store_displayname_contact_status_in_.py

@@ -0,0 +1,32 @@
+"""Store displayname contact status in puppet table
+
+Revision ID: 990f4395afc6
+Revises: 7de69cf5809e
+Create Date: 2021-01-01 11:56:54.610681
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '990f4395afc6'
+down_revision = '7de69cf5809e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('displayname_contact', sa.Boolean(), server_default=sa.true(), nullable=False))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('displayname_contact')
+
+    # ### end Alembic commands ###

alembic/versions/bfc0a39bfe02_store_displayname_quality_in_puppet_.py

@@ -0,0 +1,32 @@
+"""Store displayname quality in puppet table
+
+Revision ID: bfc0a39bfe02
+Revises: ec1d3dcc77e9
+Create Date: 2021-03-23 20:03:08.825333
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'bfc0a39bfe02'
+down_revision = 'ec1d3dcc77e9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('displayname_quality', sa.Integer(), server_default='0', nullable=False))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('displayname_quality')
+
+    # ### end Alembic commands ###

alembic/versions/ec1d3dcc77e9_switch_telegram_ids_to_bigints.py

@@ -0,0 +1,44 @@
+"""Switch Telegram IDs to bigints
+
+Revision ID: ec1d3dcc77e9
+Revises: 990f4395afc6
+Create Date: 2021-03-09 21:36:58.443727
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'ec1d3dcc77e9'
+down_revision = '990f4395afc6'
+branch_labels = None
+depends_on = None
+
+
+columns_to_upgrade = (
+    ("bot_chat", "id"),
+    ("message", "tgid"),
+    ("message", "tg_space"),
+    ("portal", "tgid"),
+    ("portal", "tg_receiver"),
+    ("puppet", "id"),
+    ("puppet", "displayname_source"),
+    ("user", "tgid"),
+    ("user_portal", "user"),
+    ("user_portal", "portal"),
+    ("user_portal", "portal_receiver"),
+    ("contact", "user"),
+    ("contact", "contact"),
+)
+
+
+def upgrade():
+    if op.get_context().dialect.name == "postgresql":
+        for table, column in columns_to_upgrade:
+            op.alter_column(table, column, existing_type=sa.Integer, type_=sa.BigInteger)
+
+
+def downgrade():
+    if op.get_context().dialect.name == "postgresql":
+        for table, column in columns_to_upgrade:
+            op.alter_column(table, column, existing_type=sa.BigInteger, type_=sa.Integer)

docker-run.sh

@@ -26,9 +26,6 @@ fi
 
 if [ ! -f /data/registration.yaml ]; then
 	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
-	echo "Didn't find a registration file."
-	echo "Generated one for you."
-	echo "Copy that over to synapses app service directory."
 	fixperms
 	exit
 fi

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.9.0rc1"
+__version__ = "0.10.0"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/abstract_user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,9 +15,9 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Tuple, Optional, Union, Dict, Type, Any, TYPE_CHECKING
 from abc import ABC, abstractmethod
+import platform
 import asyncio
 import logging
-import platform
 import time
 
 from telethon.sessions import Session
@@ -25,13 +25,14 @@ from telethon.network import (ConnectionTcpMTProxyRandomizedIntermediate, Connec
                               Connection)
 from telethon.tl.patched import MessageService, Message
 from telethon.tl.types import (
-    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdateChatPinnedMessage,
+    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdatePinnedMessages,
-    UpdateChannelPinnedMessage, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
+    UpdatePinnedChannelMessages, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
     UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateNewMessage, UpdateDeleteMessages,
     UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateReadHistoryOutbox,
     UpdateShortChatMessage, UpdateShortMessage, UpdateUserName, UpdateUserPhoto, UpdateUserStatus,
     UpdateUserTyping, User, UserStatusOffline, UserStatusOnline, UpdateReadHistoryInbox,
-    UpdateReadChannelInbox, MessageEmpty)
+    UpdateReadChannelInbox, MessageEmpty, UpdateFolderPeers, UpdatePinnedDialogs,
+    UpdateNotifySettings, UpdateChannelUserTyping)
 
 from mautrix.types import UserID, PresenceState
 from mautrix.errors import MatrixError
@@ -57,6 +58,7 @@ MAX_DELETIONS: int = 10
 UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                       UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
 UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
+UpdateTyping = Union[UpdateUserTyping, UpdateChatUserTyping, UpdateChannelUserTyping]
 
 UPDATE_TIME = Histogram("bridge_telegram_update", "Time spent processing Telegram updates",
                         ("update_type",))
@@ -235,8 +237,7 @@ class AbstractUser(ABC):
     # region Telegram update handling
 
     async def _update(self, update: TypeUpdate) -> None:
-        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
+        asyncio.create_task(self._handle_entity_updates(getattr(update, "_entities", {})))
-                              loop=self.loop)
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -244,7 +245,7 @@ class AbstractUser(ABC):
             await self.delete_message(update)
         elif isinstance(update, UpdateDeleteChannelMessages):
             await self.delete_channel_message(update)
-        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
+        elif isinstance(update, (UpdateChatUserTyping, UpdateChannelUserTyping, UpdateUserTyping)):
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
             await self.update_status(update)
@@ -252,7 +253,7 @@ class AbstractUser(ABC):
             await self.update_admin(update)
         elif isinstance(update, UpdateChatParticipants):
             await self.update_participants(update)
-        elif isinstance(update, (UpdateChannelPinnedMessage, UpdateChatPinnedMessage)):
+        elif isinstance(update, (UpdatePinnedMessages, UpdatePinnedChannelMessages)):
             await self.update_pinned_messages(update)
         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
             await self.update_others_info(update)
@@ -260,17 +261,33 @@ class AbstractUser(ABC):
             await self.update_read_receipt(update)
         elif isinstance(update, (UpdateReadHistoryInbox, UpdateReadChannelInbox)):
             await self.update_own_read_receipt(update)
+        elif isinstance(update, UpdateFolderPeers):
+            await self.update_folder_peers(update)
+        elif isinstance(update, UpdatePinnedDialogs):
+            await self.update_pinned_dialogs(update)
+        elif isinstance(update, UpdateNotifySettings):
+            await self.update_notify_settings(update)
         else:
             self.log.trace("Unhandled update: %s", update)
 
-    async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
+    async def update_folder_peers(self, update: UpdateFolderPeers) -> None:
-                                                         UpdateChatPinnedMessage]) -> None:
+        pass
-        if isinstance(update, UpdateChatPinnedMessage):
+
-            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+    async def update_pinned_dialogs(self, update: UpdatePinnedDialogs) -> None:
+        pass
+
+    async def update_notify_settings(self, update: UpdateNotifySettings) -> None:
+        pass
+
+    async def update_pinned_messages(self, update: Union[UpdatePinnedMessages,
+                                                         UpdatePinnedChannelMessages]) -> None:
+        if isinstance(update, UpdatePinnedMessages):
+            portal = po.Portal.get_by_entity(update.peer, receiver_id=self.tgid)
         else:
             portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if portal and portal.mxid:
-            await portal.receive_telegram_pin_id(update.id, self.tgid)
+            await portal.receive_telegram_pin_ids(update.messages, self.tgid,
+                                                  remove=not update.pinned)
 
     @staticmethod
     async def update_participants(update: UpdateChatParticipants) -> None:
@@ -329,16 +346,27 @@ class AbstractUser(ABC):
 
         await portal.set_telegram_admin(TelegramID(update.user_id))
 
-    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
+    async def update_typing(self, update: UpdateTyping) -> None:
+        sender = None
         if isinstance(update, UpdateUserTyping):
             portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
-        else:
+            sender = pu.Puppet.get(TelegramID(update.user_id))
+        elif isinstance(update, UpdateChannelUserTyping):
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        elif isinstance(update, UpdateChatUserTyping):
             portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
-
+        else:
-        if not portal or not portal.mxid:
+            return
+
+        if isinstance(update, (UpdateChannelUserTyping, UpdateChatUserTyping)):
+            # Can typing notifications come from non-user peers?
+            if not update.from_id.user_id:
+                return
+            sender = pu.Puppet.get(TelegramID(update.from_id.user_id))
+
+        if not sender or not portal or not portal.mxid:
             return
 
-        sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
     async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
@@ -419,6 +447,8 @@ class AbstractUser(ABC):
 
         for message_id in update.messages:
             for message in DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid):
+                if message.redacted:
+                    continue
                 message.delete()
                 number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
                 if number_left == 0:
@@ -432,6 +462,8 @@ class AbstractUser(ABC):
 
         for message_id in update.messages:
             for message in DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id):
+                if message.redacted:
+                    continue
                 message.delete()
                 await self._try_redact(message)
 
@@ -468,7 +500,7 @@ class AbstractUser(ABC):
                 await self.register_portal(portal)
                 return
             self.log.trace("Handling action %s to %s by %d", update.action, portal.tgid_log,
-                           sender.id)
+                           (sender.id if sender else 0))
             return await portal.handle_telegram_action(self, sender, update)
 
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):

mautrix_telegram/commands/__init__.py

@@ -1,7 +1,7 @@
 from .handler import (command_handler, CommandHandler, CommandProcessor, CommandEvent,
                       SECTION_AUTH, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT,
                       SECTION_MISC, SECTION_ADMIN)
-from . import portal, telegram, clean_rooms, matrix_auth, manhole
+from . import portal, telegram, matrix_auth, manhole
 
 __all__ = ["command_handler", "CommandHandler", "CommandProcessor", "CommandEvent",
            "SECTION_AUTH", "SECTION_MISC", "SECTION_ADMIN", "SECTION_CREATING_PORTALS",

mautrix_telegram/commands/clean_rooms.py

@@ -1,195 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, NamedTuple, Tuple, Union
-
-from mautrix.appservice import IntentAPI
-from mautrix.errors import MatrixRequestError
-from mautrix.types import RoomID, UserID, EventID, EventType
-
-from . import command_handler, CommandEvent, SECTION_ADMIN
-from .. import puppet as pu, portal as po
-
-ManagementRoom = NamedTuple('ManagementRoom', room_id=RoomID, user_id=UserID)
-
-
-async def _find_rooms(intent: IntentAPI) -> Tuple[List[ManagementRoom], List[RoomID], List[RoomID],
-                                                  List['po.Portal'], List['po.Portal']]:
-    management_rooms: List[ManagementRoom] = []
-    unidentified_rooms: List[RoomID] = []
-    tombstoned_rooms: List[RoomID] = []
-    portals: List[po.Portal] = []
-    empty_portals: List[po.Portal] = []
-
-    rooms = await intent.get_joined_rooms()
-    for room_id in rooms:
-        portal = po.Portal.get_by_mxid(room_id)
-        if not portal:
-            try:
-                tombstone = await intent.get_state_event(room_id, EventType.ROOM_TOMBSTONE)
-                if tombstone and tombstone.replacement_room:
-                    tombstoned_rooms.append(room_id)
-                    continue
-            except MatrixRequestError:
-                pass
-            try:
-                members = await intent.get_room_members(room_id)
-            except MatrixRequestError:
-                members = []
-            if len(members) == 2:
-                other_member = members[0] if members[0] != intent.mxid else members[1]
-                if pu.Puppet.get_id_from_mxid(other_member):
-                    unidentified_rooms.append(room_id)
-                else:
-                    management_rooms.append(ManagementRoom(room_id, other_member))
-            else:
-                unidentified_rooms.append(room_id)
-        else:
-            members = await portal.get_authenticated_matrix_users()
-            if len(members) == 0:
-                empty_portals.append(portal)
-            else:
-                portals.append(portal)
-
-    return management_rooms, unidentified_rooms, tombstoned_rooms, portals, empty_portals
-
-
-@command_handler(needs_admin=True, needs_auth=False, management_only=True, name="clean-rooms",
-                 help_section=SECTION_ADMIN,
-                 help_text="Clean up unused portal/management rooms.")
-async def clean_rooms(evt: CommandEvent) -> EventID:
-    (management_rooms, unidentified_rooms, tombstoned_rooms,
-     portals, empty_portals) = await _find_rooms(evt.az.intent)
-
-    reply = ["#### Management rooms (M)"]
-    reply += ([f"{n+1}. [M{n+1}](https://matrix.to/#/{room}) (with {other_member}"
-               for n, (room, other_member) in enumerate(management_rooms)]
-              or ["No management rooms found."])
-    reply.append("#### Active portal rooms (A)")
-    reply += ([f"{n+1}. [A{n+1}](https://matrix.to/#/{portal.mxid}) "
-               f"(to Telegram chat \"{portal.title}\")"
-               for n, portal in enumerate(portals)]
-              or ["No active portal rooms found."])
-    reply.append("#### Unidentified rooms (U)")
-    reply += ([f"{n+1}. [U{n+1}](https://matrix.to/#/{room})"
-               for n, room in enumerate(unidentified_rooms)]
-              or ["No unidentified rooms found."])
-    reply.append("#### Tombstoned rooms (T)")
-    reply += ([f"{n+1}. [T{n+1}](https://matrix.to/#/{room})"
-               for n, room in enumerate(tombstoned_rooms)]
-              or ["No tombstoned rooms found."])
-    reply.append("#### Inactive portal rooms (I)")
-    reply += ([f"{n}. [I{n}](https://matrix.to/#/{portal.mxid}) "
-               f"(to Telegram chat \"{portal.title}\")"
-               for n, portal in enumerate(empty_portals)]
-              or ["No inactive portal rooms found."])
-
-    reply += ["#### Usage",
-              ("To clean the recommended set of rooms (unidentified & inactive portals), "
-               "type `$cmdprefix+sp clean-recommended`"),
-              "",
-              ("To clean other groups of rooms, type `$cmdprefix+sp clean-groups <letters>` "
-               "where `letters` are the first letters of the group names (M, A, U, I, T)"),
-              "",
-              ("To clean specific rooms, type `$cmdprefix+sp clean-range <range>` "
-               "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
-               "the group name. (e.g. `I2-6`)"),
-              "",
-              ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
-               "between each use of the commands above.")]
-
-    evt.sender.command_status = {
-        "next": lambda clean_evt: set_rooms_to_clean(clean_evt, management_rooms,
-                                                     unidentified_rooms, tombstoned_rooms, portals,
-                                                     empty_portals),
-        "action": "Room cleaning",
-    }
-
-    return await evt.reply("\n".join(reply))
-
-
-async def set_rooms_to_clean(evt, management_rooms: List[ManagementRoom],
-                             unidentified_rooms: List[RoomID], tombstoned_rooms: List[RoomID],
-                             portals: List["po.Portal"], empty_portals: List["po.Portal"]) -> None:
-    command = evt.args[0]
-    rooms_to_clean: List[Union[po.Portal, RoomID]] = []
-    if command == "clean-recommended":
-        rooms_to_clean += empty_portals
-        rooms_to_clean += unidentified_rooms
-    elif command == "clean-groups":
-        if len(evt.args) < 2:
-            return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
-        groups_to_clean = evt.args[1].upper()
-        if "M" in groups_to_clean:
-            rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
-        if "A" in groups_to_clean:
-            rooms_to_clean += portals
-        if "U" in groups_to_clean:
-            rooms_to_clean += unidentified_rooms
-        if "I" in groups_to_clean:
-            rooms_to_clean += empty_portals
-        if "T" in groups_to_clean:
-            rooms_to_clean += tombstoned_rooms
-    elif command == "clean-range":
-        try:
-            clean_range = evt.args[1]
-            group, clean_range = clean_range[0], clean_range[1:]
-            start, end = clean_range.split("-")
-            start, end = int(start), int(end)
-            if group == "M":
-                group = [room_id for (room_id, user_id) in management_rooms]
-            elif group == "A":
-                group = portals
-            elif group == "U":
-                group = unidentified_rooms
-            elif group == "I":
-                group = empty_portals
-            elif group == "T":
-                group = tombstoned_rooms
-            else:
-                raise ValueError("Unknown group")
-            rooms_to_clean = group[start - 1:end]
-        except (KeyError, ValueError):
-            return await evt.reply(
-                "**Usage:** `$cmdprefix+sp clean-groups <_M|A|U|I_><range>")
-    else:
-        return await evt.reply(f"Unknown room cleaning action `{command}`. "
-                               "Use `$cmdprefix+sp cancel` to cancel room "
-                               "cleaning.")
-
-    evt.sender.command_status = {
-        "next": lambda confirm: execute_room_cleanup(confirm, rooms_to_clean),
-        "action": "Room cleaning",
-    }
-    await evt.reply(f"To confirm cleaning up {len(rooms_to_clean)} rooms, type "
-                    "`$cmdprefix+sp confirm-clean`.")
-
-
-async def execute_room_cleanup(evt, rooms_to_clean: List[Union[po.Portal, RoomID]]) -> None:
-    if len(evt.args) > 0 and evt.args[0] == "confirm-clean":
-        await evt.reply(f"Cleaning {len(rooms_to_clean)} rooms. "
-                        "This might take a while.")
-        cleaned = 0
-        for room in rooms_to_clean:
-            if isinstance(room, po.Portal):
-                await room.cleanup_and_delete()
-                cleaned += 1
-            else:
-                await po.Portal.cleanup_room(evt.az.intent, room, "Room deleted")
-                cleaned += 1
-        evt.sender.command_status = None
-        await evt.reply(f"{cleaned} rooms cleaned up successfully.")
-    else:
-        await evt.reply("Room cleaning cancelled.")

mautrix_telegram/commands/portal/bridge.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Tuple, Coroutine
+from typing import Optional, Tuple, Awaitable
 import asyncio
 
 from telethon.tl.types import ChatForbidden, ChannelForbidden
@@ -105,18 +105,17 @@ async def bridge(evt: CommandEvent) -> EventID:
 
 
 async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
-                                            ) -> Tuple[
+                                            ) -> Tuple[bool, Optional[Awaitable[None]]]:
-    bool, Optional[Coroutine[None, None, None]]]:
     if not portal.mxid:
         await evt.reply("The portal seems to have lost its Matrix room between you"
                         "calling `$cmdprefix+sp bridge` and this command.\n\n"
                         "Continuing without touching previous Matrix room...")
         return True, None
     elif evt.args[0] == "delete-and-continue":
-        return True, portal.cleanup_portal("Portal deleted (moving to another room)")
+        return True, portal.cleanup_portal("Portal deleted (moving to another room)", delete=False)
     elif evt.args[0] == "unbridge-and-continue":
         return True, portal.cleanup_portal("Room unbridged (portal moving to another room)",
-                                           puppets_only=True)
+                                           puppets_only=True, delete=False)
     else:
         await evt.reply(
             "The chat you were trying to bridge already has a Matrix portal room.\n\n"
@@ -137,6 +136,9 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
         return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
                                "This shouldn't happen unless you're messing with the command "
                                "handler code.")
+
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
+
     if "mxid" in status:
         ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
         if not ok:
@@ -154,7 +156,13 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
                                "`$cmdprefix+sp cancel` to cancel.")
 
     evt.sender.command_status = None
-    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
+    async with portal._room_create_lock:
+        await _locked_confirm_bridge(evt, portal=portal, room_id=bridge_to_mxid,
+                                     is_logged_in=is_logged_in)
+
+
+async def _locked_confirm_bridge(evt: CommandEvent, portal: 'po.Portal', room_id: RoomID,
+                                 is_logged_in: bool) -> Optional[EventID]:
     user = evt.sender if is_logged_in else evt.tgbot
     try:
         entity = await user.client.get_entity(portal.peer)
@@ -172,14 +180,15 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
         else:
             return await evt.reply("The bot doesn't seem to be in that chat.")
 
-    direct = False
+    portal.mxid = room_id
-
+    portal.by_mxid[portal.mxid] = portal
-    portal.mxid = bridge_to_mxid
+    (portal.title, portal.about, levels,
-    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+     portal.encrypted) = await get_initial_state(evt.az.intent, evt.room_id)
     portal.photo_id = ""
     await portal.save()
+    await portal.update_bridge_info()
 
-    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
+    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct=False, levels=levels),
                           loop=evt.loop)
 
     return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/create_chat.py

@@ -38,7 +38,7 @@ async def create(evt: CommandEvent) -> EventID:
     if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
         return await evt.reply("You do not have the permissions to bridge this room.")
 
-    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    title, about, levels, encrypted = await get_initial_state(evt.az.intent, evt.room_id)
     if not title:
         return await evt.reply("Please set a title before creating a Telegram chat.")
 
@@ -50,11 +50,17 @@ async def create(evt: CommandEvent) -> EventID:
         "group": "chat",
     }[type]
 
-    portal = po.Portal(tgid=TelegramID(0), peer_type=type,
+    portal = po.Portal(tgid=TelegramID(0), peer_type=type, mxid=evt.room_id,
-                       mxid=evt.room_id, title=title, about=about)
+                       title=title, about=about, encrypted=encrypted)
+    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender)
+    if len(errors) > 0:
+        error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
+        await evt.reply(f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
+                        "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
+                        "those users.")
     try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
+        await portal.create_telegram_chat(evt.sender, invites=invites, supergroup=supergroup)
     except ValueError as e:
-        portal.delete()
+        await portal.delete()
         return await evt.reply(e.args[0])
     return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/misc.py

@@ -13,6 +13,10 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple
+from datetime import timedelta, datetime
+import re
+
 from telethon.tl.functions.channels import GetFullChannelRequest
 from telethon.tl.functions.messages import GetFullChatRequest
 from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
@@ -80,9 +84,81 @@ async def get_id(evt: CommandEvent) -> EventID:
     await evt.reply(f"This room is bridged to Telegram chat ID `{tgid}`.")
 
 
+invite_link_usage = ("**Usage:** `$cmdprefix+sp invite-link [--uses=<amount>] [--expire=<delta>]`"
+                     "\n\n"
+                     "* `--uses`: the number of times the invite link can be used."
+                     "            Defaults to unlimited.\n"
+                     "* `--expire`: the duration after which the link will expire."
+                     "              A number suffixed with d(ay), h(our), m(inute) or s(econd)")
+
+
+def _parse_flag(args: List[str]) -> Tuple[str, str]:
+    arg = args.pop(0).lower()
+    if arg.startswith("--"):
+        value_start = arg.index("=")
+        if value_start:
+            flag = arg[2:value_start]
+            value = arg[value_start+1:]
+        else:
+            flag = arg[2:]
+            value = args.pop(0).lower()
+    elif arg.startswith("-"):
+        flag = arg[1]
+        if len(arg) > 3 and arg[2] == "=":
+            value = arg[3:]
+        else:
+            value = args.pop(0).lower()
+    else:
+        raise ValueError("invalid flag")
+    return flag, value
+
+
+delta_regex = re.compile("([0-9]+)(w(?:eek)?|d(?:ay)?|h(?:our)?|m(?:in(?:ute)?)?|s(?:ec(?:ond)?)?)")
+
+
+def _parse_delta(value: str) -> Optional[timedelta]:
+    match = delta_regex.fullmatch(value)
+    if not match:
+        return None
+    number = int(match.group(1))
+    unit = match.group(2)[0]
+    if unit == "w":
+        return timedelta(weeks=number)
+    elif unit == "d":
+        return timedelta(days=number)
+    elif unit == "h":
+        return timedelta(hours=number)
+    elif unit == "m":
+        return timedelta(minutes=number)
+    elif unit == "s":
+        return timedelta(seconds=number)
+    else:
+        return None
+
+
 @command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Get a Telegram invite link to the current chat.")
+                 help_text="Get a Telegram invite link to the current chat.",
+                 help_args="[--uses=<amount>] [--expire=<time delta, e.g. 1d>]")
 async def invite_link(evt: CommandEvent) -> EventID:
+    # TODO once we switch to Python 3.9 minimum, use argparse with exit_on_error=False
+    uses = None
+    expire = None
+    while evt.args:
+        try:
+            flag, value = _parse_flag(evt.args)
+        except (ValueError, IndexError):
+            return await evt.reply(invite_link_usage)
+        if flag in ("uses", "u"):
+            try:
+                uses = int(value)
+            except ValueError:
+                await evt.reply("The number of uses must be an integer")
+        elif flag in ("expire", "e"):
+            expire_delta = _parse_delta(value)
+            if not expire_delta:
+                await evt.reply("Invalid format for expiry time delta")
+            expire = datetime.now() + expire_delta
+
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
@@ -91,7 +167,7 @@ async def invite_link(evt: CommandEvent) -> EventID:
         return await evt.reply("You can't invite users to private chats.")
 
     try:
-        link = await portal.get_invite_link(evt.sender)
+        link = await portal.get_invite_link(evt.sender, uses=uses, expire=expire)
         return await evt.reply(f"Invite link to {portal.title}: {link}")
     except ValueError as e:
         return await evt.reply(e.args[0])

mautrix_telegram/commands/portal/unbridge.py

@@ -31,6 +31,12 @@ async def _get_portal_and_check_permission(evt: CommandEvent) -> Optional[po.Por
         await evt.reply(f"{that_this} is not a portal room.")
         return None
 
+    if portal.peer_type == "user":
+        if portal.tg_receiver != evt.sender.tgid:
+            await evt.reply("You do not have the permissions to unbridge that portal.")
+            return None
+        return portal
+
     if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
         await evt.reply("You do not have the permissions to unbridge that portal.")
         return None

mautrix_telegram/commands/portal/util.py

@@ -25,11 +25,12 @@ OptStr = Optional[str]
 
 
 async def get_initial_state(intent: IntentAPI, room_id: RoomID
-                            ) -> Tuple[OptStr, OptStr, Optional[PowerLevelStateEventContent]]:
+                            ) -> Tuple[OptStr, OptStr, Optional[PowerLevelStateEventContent], bool]:
     state = await intent.get_state(room_id)
     title: OptStr = None
     about: OptStr = None
     levels: Optional[PowerLevelStateEventContent] = None
+    encrypted: bool = False
     for event in state:
         try:
             if event.type == EventType.ROOM_NAME:
@@ -40,10 +41,12 @@ async def get_initial_state(intent: IntentAPI, room_id: RoomID
                 levels = event.content
             elif event.type == EventType.ROOM_CANONICAL_ALIAS:
                 title = title or event.content.canonical_alias
+            elif event.type == EventType.ROOM_ENCRYPTION:
+                encrypted = True
         except KeyError:
             # Some state event probably has empty content
             pass
-    return title, about, levels
+    return title, about, levels, encrypted
 
 
 async def user_has_power_level(room_id: RoomID, intent: IntentAPI, sender: u.User,

mautrix_telegram/commands/telegram/account.py

@@ -16,7 +16,7 @@
 from typing import Optional
 
 from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
-                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
+                             HashInvalidError, AuthKeyError, FirstNameInvalidError, AboutTooLongError)
 from telethon.tl.types import Authorization
 from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
                                            ResetAuthorizationRequest, UpdateProfileRequest)
@@ -53,6 +53,23 @@ async def username(evt: CommandEvent) -> EventID:
     else:
         await evt.reply(f"Username changed to {evt.sender.username}")
 
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_new about_>",
+                 help_text="Change your Telegram about section.")
+async def about(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp about <new about>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own about section.")
+    new_about = " ".join(evt.args)
+    if new_about == "-":
+        new_about = ""
+    try:
+        await evt.sender.client(UpdateProfileRequest(about=new_about))
+    except AboutTooLongError:
+        return await evt.reply("The provided about section is too long")
+    return await evt.reply("About section updated")
 
 @command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
                  help_text="Change your Telegram displayname.")

mautrix_telegram/commands/telegram/auth.py

@@ -31,7 +31,7 @@ from mautrix.types import (EventID, UserID, MediaMessageEventContent, ImageInfo,
 from ... import user as u
 from ...types import TelegramID
 from ...commands import command_handler, CommandEvent, SECTION_AUTH
-from ...util import format_duration
+from ...util import format_duration as fmt_duration
 
 try:
     import qrcode
@@ -70,7 +70,7 @@ async def ping_bot(evt: CommandEvent) -> EventID:
                  help_section=SECTION_AUTH,
                  help_args="<_phone_> <_full name_>",
                  help_text="Register to Telegram")
-async def register(evt: CommandEvent) -> Optional[EventID]:
+async def register(evt: CommandEvent) -> EventID:
     if await evt.sender.is_logged_in():
         return await evt.reply("You are already logged in.")
     elif len(evt.args) < 1:
@@ -87,7 +87,8 @@ async def register(evt: CommandEvent) -> Optional[EventID]:
         "action": "Register",
         "full_name": full_name,
     })
-    return None
+    return await evt.reply("By signing up for Telegram, you agree to "
+                           "the terms of service: https://telegram.org/tos")
 
 
 async def enter_code_register(evt: CommandEvent) -> EventID:
@@ -222,21 +223,18 @@ async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[
         ok = True
         return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
     except PhoneNumberAppSignupForbiddenError:
-        return await evt.reply(
+        return await evt.reply("Your phone number does not allow 3rd party apps to sign in.")
-            "Your phone number does not allow 3rd party apps to sign in.")
     except PhoneNumberFloodError:
-        return await evt.reply(
+        return await evt.reply("Your phone number has been temporarily blocked for flooding. "
-            "Your phone number has been temporarily blocked for flooding. "
+                               "The ban is usually applied for around a day.")
-            "The ban is usually applied for around a day.")
     except FloodWaitError as e:
-        return await evt.reply(
+        return await evt.reply("Your phone number has been temporarily blocked for flooding. "
-            "Your phone number has been temporarily blocked for flooding. "
+                               f"Please wait for {fmt_duration(e.seconds)} before trying again.")
-            f"Please wait for {format_duration(e.seconds)} before trying again.")
     except PhoneNumberBannedError:
-        return await  evt.reply("Your phone number has been banned from Telegram.")
+        return await evt.reply("Your phone number has been banned from Telegram.")
     except PhoneNumberUnoccupiedError:
-        return await  evt.reply("That phone number has not been registered. "
+        return await evt.reply("That phone number has not been registered. "
-                                "Please register with `$cmdprefix+sp register <phone>`.")
+                               "Please register with `$cmdprefix+sp register <phone>`.")
     except PhoneNumberInvalidError:
         return await evt.reply("That phone number is not valid.")
     except Exception:

mautrix_telegram/commands/telegram/misc.py

@@ -14,11 +14,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import List, Optional, Tuple, cast
-import logging
 import codecs
 import base64
 import re
 
+from aiohttp import ClientSession, InvalidURL
+
 from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
                              UserAlreadyParticipantError, ChatIdInvalidError,
                              TakeoutInitDelayError, EmoticonInvalidError)
@@ -115,25 +116,25 @@ async def pm(evt: CommandEvent) -> EventID:
         return await evt.reply("That doesn't seem to be a user.")
     portal = po.Portal.get_by_entity(user, evt.sender.tgid)
     await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
-    return await evt.reply("Created private chat room with "
+    displayname, _ = pu.Puppet.get_displayname(user, False)
-                           f"{pu.Puppet.get_displayname(user, False)}")
+    return await evt.reply(f"Created private chat room with {displayname}")
 
 
-async def _join(evt: CommandEvent, arg: str) -> Tuple[Optional[TypeUpdates], Optional[EventID]]:
+async def _join(evt: CommandEvent, identifier: str, link_type: str
-    if arg.startswith("joinchat/"):
+                ) -> Tuple[Optional[TypeUpdates], Optional[EventID]]:
-        invite_hash = arg[len("joinchat/"):]
+    if link_type == "joinchat":
         try:
-            await evt.sender.client(CheckChatInviteRequest(invite_hash))
+            await evt.sender.client(CheckChatInviteRequest(identifier))
         except InviteHashInvalidError:
             return None, await evt.reply("Invalid invite link.")
         except InviteHashExpiredError:
             return None, await evt.reply("Invite link expired.")
         try:
-            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
+            return (await evt.sender.client(ImportChatInviteRequest(identifier))), None
         except UserAlreadyParticipantError:
             return None, await evt.reply("You are already in that chat.")
     else:
-        channel = await evt.sender.client.get_entity(arg)
+        channel = await evt.sender.client.get_entity(identifier)
         if not channel:
             return None, await evt.reply("Channel/supergroup not found.")
         return await evt.sender.client(JoinChannelRequest(channel)), None
@@ -146,12 +147,26 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
 
-    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:joinchat/)?/(.+)")
+    url = evt.args[0]
-    arg = regex.match(evt.args[0])
+    if evt.config["bridge.invite_link_resolve"]:
+        try:
+            async with ClientSession() as sess, sess.get(url) as resp:
+                url = str(resp.url)
+        except InvalidURL:
+            return await evt.reply("That doesn't look like a Telegram invite link.")
+
+    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)"
+                       r"(?:/(?P<type>joinchat|s))?/(?P<id>[^/]+)/?", flags=re.IGNORECASE)
+    arg = regex.match(url)
     if not arg:
         return await evt.reply("That doesn't look like a Telegram invite link.")
 
-    updates, _ = await _join(evt, arg.group(1))
+    data = arg.groupdict()
+    identifier = data["id"]
+    link_type = data["type"]
+    if link_type:
+        link_type = link_type.lower()
+    updates, _ = await _join(evt, identifier, link_type)
     if not updates:
         return None
 
@@ -165,9 +180,8 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
             try:
                 await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
             except ChatIdInvalidError as e:
-                logging.getLogger("mau.commands").trace("ChatIdInvalidError while creating portal "
+                evt.log.trace("ChatIdInvalidError while creating portal from !tg join command: %s",
-                                                        "from !tg join command: %s",
+                              updates.stringify())
-                                                        updates.stringify())
                 raise e
             return await evt.reply(f"Created room for {portal.title}")
     return None

mautrix_telegram/config.py

@@ -13,14 +13,14 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Any, Dict, List, NamedTuple
+from typing import Any, List, NamedTuple
 from ruamel.yaml.comments import CommentedMap
 import os
 
 from mautrix.types import UserID
 from mautrix.client import Client
-from mautrix.bridge.config import (BaseBridgeConfig, ConfigUpdateHelper, ForbiddenKey,
+from mautrix.bridge.config import BaseBridgeConfig
-                                   ForbiddenDefault)
+from mautrix.util.config import ForbiddenKey, ForbiddenDefault, ConfigUpdateHelper
 
 Permissions = NamedTuple("Permissions", relaybot=bool, user=bool, puppeting=bool,
                          matrix_puppeting=bool, admin=bool, level=str)
@@ -114,6 +114,7 @@ class Config(BaseBridgeConfig):
         else:
             copy("bridge.login_shared_secret_map")
         copy("bridge.telegram_link_preview")
+        copy("bridge.invite_link_resolve")
         copy("bridge.inline_images")
         copy("bridge.image_as_file_size")
         copy("bridge.max_document_size")
@@ -131,6 +132,10 @@ class Config(BaseBridgeConfig):
         copy("bridge.delivery_receipts")
         copy("bridge.delivery_error_reports")
         copy("bridge.resend_bridge_info")
+        copy("bridge.mute_bridging")
+        copy("bridge.pinned_tag")
+        copy("bridge.archive_tag")
+        copy("bridge.tag_only_on_create")
         copy("bridge.backfill.invite_own_puppet")
         copy("bridge.backfill.takeout_limit")
         copy("bridge.backfill.initial_limit")
@@ -240,24 +245,3 @@ class Config(BaseBridgeConfig):
             return self._get_permissions(homeserver)
 
         return self._get_permissions("*")
-
-    @property
-    def namespaces(self) -> Dict[str, List[Dict[str, Any]]]:
-        homeserver = self["homeserver.domain"]
-
-        username_format = self["bridge.username_template"].format(userid=".+")
-        alias_format = self["bridge.alias_template"].format(groupname=".+")
-        group_id = ({"group_id": self["appservice.community_id"]}
-                    if self["appservice.community_id"] else {})
-
-        return {
-            "users": [{
-                "exclusive": True,
-                "regex": f"@{username_format}:{homeserver}",
-                **group_id,
-            }],
-            "aliases": [{
-                "exclusive": True,
-                "regex": f"#{alias_format}:{homeserver}",
-            }]
-        }

mautrix_telegram/db/bot_chat.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Iterable
 
-from sqlalchemy import Column, Integer, String
+from sqlalchemy import Column, BigInteger, String
 
 from mautrix.util.db import Base
 
@@ -25,7 +25,7 @@ from ..types import TelegramID
 # Fucking Telegram not telling bots what chats they are in 3:<
 class BotChat(Base):
     __tablename__ = "bot_chat"
-    id: TelegramID = Column(Integer, primary_key=True)
+    id: TelegramID = Column(BigInteger, primary_key=True)
     type: str = Column(String, nullable=False)
 
     @classmethod

mautrix_telegram/db/message.py

@@ -13,9 +13,10 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Iterator
+from typing import Optional, Iterator, List
 
-from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
+from sqlalchemy import (Column, UniqueConstraint, BigInteger, Integer, String, Boolean, and_, func,
+                        desc, select, false)
 
 from mautrix.types import RoomID, EventID
 from mautrix.util.db import Base
@@ -28,9 +29,10 @@ class Message(Base):
 
     mxid: EventID = Column(String)
     mx_room: RoomID = Column(String)
-    tgid: TelegramID = Column(Integer, primary_key=True)
+    tgid: TelegramID = Column(BigInteger, primary_key=True)
-    tg_space: TelegramID = Column(Integer, primary_key=True)
+    tg_space: TelegramID = Column(BigInteger, primary_key=True)
     edit_index: int = Column(Integer, primary_key=True)
+    redacted: bool = Column(Boolean, server_default=false())
 
     __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"),)
 
@@ -51,6 +53,12 @@ class Message(Base):
             return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
                                            cls.c.edit_index == edit_index)
 
+    @classmethod
+    def get_first_by_tgids(cls, tgids: List[TelegramID], tg_space: TelegramID
+                           ) -> Iterator['Message']:
+        return cls._select_all(cls.c.tgid.in_(tgids), cls.c.tg_space == tg_space,
+                               cls.c.edit_index == 0)
+
     @classmethod
     def count_spaces_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> int:
         rows = cls.db.execute(select([func.count(cls.c.tg_space)])
@@ -77,6 +85,12 @@ class Message(Base):
         return cls._select_one_or_none(cls.c.mxid == mxid, cls.c.mx_room == mx_room,
                                        cls.c.tg_space == tg_space)
 
+    @classmethod
+    def get_by_mxids(cls, mxids: List[EventID], mx_room: RoomID, tg_space: TelegramID
+                     ) -> Iterator['Message']:
+        return cls._select_all(cls.c.mxid.in_(mxids), cls.c.mx_room == mx_room,
+                               cls.c.tg_space == tg_space)
+
     @classmethod
     def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
                        **values) -> None:

mautrix_telegram/db/portal.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Iterable
 
-from sqlalchemy import Column, Integer, String, Boolean, Text, func, sql
+from sqlalchemy import Column, BigInteger, String, Boolean, Text, func, sql
 
 from mautrix.types import RoomID, ContentURI
 from mautrix.util.db import Base
@@ -27,8 +27,8 @@ class Portal(Base):
     __tablename__ = "portal"
 
     # Telegram chat information
-    tgid: TelegramID = Column(Integer, primary_key=True)
+    tgid: TelegramID = Column(BigInteger, primary_key=True)
-    tg_receiver: TelegramID = Column(Integer, primary_key=True)
+    tg_receiver: TelegramID = Column(BigInteger, primary_key=True)
     peer_type: str = Column(String, nullable=False)
     megagroup: bool = Column(Boolean)
 

mautrix_telegram/db/puppet.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Iterable
 
-from sqlalchemy import Column, Integer, String, Text, Boolean
+from sqlalchemy import Column, Integer, BigInteger, String, Text, Boolean
 from sqlalchemy.sql import expression, func
 
 from mautrix.types import UserID, SyncToken
@@ -27,13 +27,15 @@ from ..types import TelegramID
 class Puppet(Base):
     __tablename__ = "puppet"
 
-    id: TelegramID = Column(Integer, primary_key=True)
+    id: TelegramID = Column(BigInteger, primary_key=True)
     custom_mxid: UserID = Column(String, nullable=True)
     access_token: str = Column(String, nullable=True)
     next_batch: SyncToken = Column(String, nullable=True)
     base_url: str = Column(Text, nullable=True)
     displayname: str = Column(String, nullable=True)
-    displayname_source: TelegramID = Column(Integer, nullable=True)
+    displayname_source: TelegramID = Column(BigInteger, nullable=True)
+    displayname_contact: bool = Column(Boolean, nullable=False, server_default=expression.true())
+    displayname_quality: int = Column(Integer, nullable=False, server_default="0")
     username: str = Column(String, nullable=True)
     photo_id: str = Column(String, nullable=True)
     is_bot: bool = Column(Boolean, nullable=True)

mautrix_telegram/db/telegram_file.py

@@ -13,15 +13,17 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, cast, Dict, Any
+from typing import Optional, cast, Dict, Any, TYPE_CHECKING
 
 from sqlalchemy import (Column, ForeignKey, Integer, BigInteger, String, Boolean, Text,
                         TypeDecorator)
-from sqlalchemy.engine.result import RowProxy
 
 from mautrix.types import ContentURI, EncryptedFile
 from mautrix.util.db import Base
 
+if TYPE_CHECKING:
+    from sqlalchemy.engine.result import RowProxy
+
 
 class DBEncryptedFile(TypeDecorator):
     impl = Text
@@ -60,7 +62,7 @@ class TelegramFile(Base):
     thumbnail: Optional['TelegramFile'] = None
 
     @classmethod
-    def scan(cls, row: RowProxy) -> 'TelegramFile':
+    def scan(cls, row: 'RowProxy') -> 'TelegramFile':
         telegram_file = cast(TelegramFile, super().scan(row))
         if isinstance(telegram_file.thumbnail, str):
             telegram_file.thumbnail = cls.get(telegram_file.thumbnail)

mautrix_telegram/db/user.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Iterable, Tuple
 
-from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String, func
+from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, BigInteger, Integer, String, func
 
 from mautrix.types import UserID
 from mautrix.util.db import Base
@@ -27,7 +27,7 @@ class User(Base):
     __tablename__ = "user"
 
     mxid: UserID = Column(String, primary_key=True)
-    tgid: Optional[TelegramID] = Column(Integer, nullable=True, unique=True)
+    tgid: Optional[TelegramID] = Column(BigInteger, nullable=True, unique=True)
     tg_username: str = Column(String, nullable=True)
     tg_phone: str = Column(String, nullable=True)
     saved_contacts: int = Column(Integer, default=0, nullable=False)
@@ -91,10 +91,10 @@ class User(Base):
 class UserPortal(Base):
     __tablename__ = "user_portal"
 
-    user: TelegramID = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE",
+    user: TelegramID = Column(BigInteger, ForeignKey("user.tgid", onupdate="CASCADE",
                                                   ondelete="CASCADE"), primary_key=True)
-    portal: TelegramID = Column(Integer, primary_key=True)
+    portal: TelegramID = Column(BigInteger, primary_key=True)
-    portal_receiver: TelegramID = Column(Integer, primary_key=True)
+    portal_receiver: TelegramID = Column(BigInteger, primary_key=True)
 
     __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
                                            ("portal.tgid", "portal.tg_receiver"),
@@ -104,5 +104,5 @@ class UserPortal(Base):
 class Contact(Base):
     __tablename__ = "contact"
 
-    user: TelegramID = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
+    user: TelegramID = Column(BigInteger, ForeignKey("user.tgid"), primary_key=True)
-    contact: TelegramID = Column(Integer, ForeignKey("puppet.id"), primary_key=True)
+    contact: TelegramID = Column(BigInteger, ForeignKey("puppet.id"), primary_key=True)

mautrix_telegram/example-config.yaml

@@ -8,6 +8,12 @@ homeserver:
     # Only applies if address starts with https://
     verify_ssl: true
     asmux: false
+    # Number of retries for all HTTP requests if the homeserver isn't reachable.
+    http_retry_count: 4
+    # The URL to push real-time bridge status to.
+    # If set, the bridge will make POST requests to this URL whenever a user's Telegram connection state changes.
+    # The bridge will use the appservice as_token to authorize requests.
+    status_endpoint: null
 
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
@@ -194,8 +200,11 @@ bridge:
         example.com: foobar
     # Set to false to disable link previews in messages sent to Telegram.
     telegram_link_preview: true
+    # Whether or not the !tg join command should do a HTTP request
+    # to resolve redirects in invite links.
+    invite_link_resolve: false
     # Use inline images instead of a separate message for the caption.
-    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    # N.B. Inline images are not supported on all clients (e.g. Element iOS/Android).
     inline_images: false
     # Maximum size of image in megabytes before sending to Telegram as a document.
     image_as_file_size: 10
@@ -204,6 +213,7 @@ bridge:
     # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
     # streaming from/to Matrix and using many connections for Telegram.
     # Note that generating HQ thumbnails for videos is not possible with streamed transfers.
+    # This option uses internal Telethon implementation details and may break with minor updates.
     parallel_file_transfer: false
     # Whether or not created rooms should have federation enabled.
     # If false, created portal rooms will never be federated.
@@ -267,6 +277,15 @@ bridge:
     # This field will automatically be changed back to false after it,
     # except if the config file is not writable.
     resend_bridge_info: false
+    # When using double puppeting, should muted chats be muted in Matrix?
+    mute_bridging: false
+    # When using double puppeting, should pinned chats be moved to a specific tag in Matrix?
+    # The favorites tag is `m.favourite`.
+    pinned_tag: null
+    # Same as above for archived chats, the low priority tag is `m.lowpriority`.
+    archive_tag: null
+    # Whether or not mute status and tags should only be bridged when the portal room is created.
+    tag_only_on_create: true
     # Settings for backfilling messages from Telegram.
     backfill:
         # Whether or not the Telegram ghosts of logged in Matrix users should be

mautrix_telegram/formatter/__init__.py

@@ -1,5 +1,4 @@
-from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
+from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram, init_mx
-                          init_mx)
 from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
 from .. import context as c
 

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -18,10 +18,12 @@ import re
 import logging
 
 from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityItalic,
-                               TypeMessageEntity)
+                               TypeMessageEntity, InputMessageEntityMentionName)
 from telethon.helpers import add_surrogate, del_surrogate
+from telethon import TelegramClient
 
 from mautrix.types import RoomID, MessageEventContent
+from mautrix.util.logging import TraceLogger
 
 from ... import puppet as pu
 from ...types import TelegramID
@@ -31,30 +33,19 @@ from .parser import ParsedMessage, parse_html
 if TYPE_CHECKING:
     from ...context import Context
 
-log: logging.Logger = logging.getLogger("mau.fmt.mx")
+log: TraceLogger = logging.getLogger("mau.fmt.mx")
 should_bridge_plaintext_highlights: bool = False
 
 command_regex: Pattern = re.compile(r"^!([A-Za-z0-9@]+)")
 not_command_regex: Pattern = re.compile(r"^\\(![A-Za-z0-9@]+)")
 plain_mention_regex: Optional[Pattern] = None
 
-
-def plain_mention_to_html(match: Match) -> str:
-    puppet = pu.Puppet.find_by_displayname(match.group(2))
-    if puppet:
-        return (f"{match.group(1)}"
-                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
-                f"{puppet.displayname}"
-                "</a>")
-    return "".join(match.groups())
-
-
 MAX_LENGTH = 4096
 CUTOFF_TEXT = " [message cut]"
 CUT_MAX_LENGTH = MAX_LENGTH - len(CUTOFF_TEXT)
 
 
-def cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
+def _cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
     if len(message) > MAX_LENGTH:
         message = message[0:CUT_MAX_LENGTH] + CUTOFF_TEXT
         new_entities = []
@@ -73,23 +64,6 @@ class FormatError(Exception):
     pass
 
 
-def matrix_to_telegram(html: str) -> ParsedMessage:
-    try:
-        html = command_regex.sub(r"<command>\1</command>", html)
-        html = html.replace("\t", " " * 4)
-        html = not_command_regex.sub(r"\1", html)
-        if should_bridge_plaintext_highlights:
-            html = plain_mention_regex.sub(plain_mention_to_html, html)
-
-        text, entities = parse_html(add_surrogate(html))
-        text = del_surrogate(text.strip())
-        text, entities = cut_long_message(text, entities)
-
-        return text, entities
-    except Exception as e:
-        raise FormatError(f"Failed to convert Matrix format: {html}") from e
-
-
 def matrix_reply_to_telegram(content: MessageEventContent, tg_space: TelegramID,
                              room_id: Optional[RoomID] = None) -> Optional[TelegramID]:
     event_id = content.get_reply_to()
@@ -103,19 +77,61 @@ def matrix_reply_to_telegram(content: MessageEventContent, tg_space: TelegramID,
     return None
 
 
-def matrix_text_to_telegram(text: str) -> ParsedMessage:
+async def matrix_to_telegram(client: TelegramClient, *, text: Optional[str] = None,
+                             html: Optional[str] = None) -> ParsedMessage:
+    if html is not None:
+        text, entities = _matrix_html_to_telegram(html)
+    elif text is not None:
+        text, entities = _matrix_text_to_telegram(text)
+    else:
+        raise ValueError("text or html must be provided to convert formatting")
+    await _fix_name_mentions(client, entities)
+    return text, entities
+
+
+def _matrix_html_to_telegram(html: str) -> ParsedMessage:
+    try:
+        html = command_regex.sub(r"<command>\1</command>", html)
+        html = html.replace("\t", " " * 4)
+        html = not_command_regex.sub(r"\1", html)
+        if should_bridge_plaintext_highlights:
+            html = plain_mention_regex.sub(_plain_mention_to_html, html)
+
+        text, entities = parse_html(add_surrogate(html))
+        text = del_surrogate(text.strip())
+        text, entities = _cut_long_message(text, entities)
+
+        return text, entities
+    except Exception as e:
+        raise FormatError(f"Failed to convert Matrix format: {html}") from e
+
+
+def _matrix_text_to_telegram(text: str) -> ParsedMessage:
     text = command_regex.sub(r"/\1", text)
     text = text.replace("\t", " " * 4)
     text = not_command_regex.sub(r"\1", text)
     if should_bridge_plaintext_highlights:
-        entities, pmr_replacer = plain_mention_to_text()
+        entities, pmr_replacer = _plain_mention_to_text()
         text = plain_mention_regex.sub(pmr_replacer, text)
     else:
         entities = []
     return text, entities
 
 
-def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match], str]]:
+async def _fix_name_mentions(client: TelegramClient, entities: List[TypeMessageEntity]) -> None:
+    for index in reversed(range(len(entities))):
+        entity = entities[index]
+        if isinstance(entity, (MessageEntityMentionName, InputMessageEntityMentionName)):
+            try:
+                user = await client.get_input_entity(entity.user_id)
+            except (ValueError, TypeError) as e:
+                log.trace(f"Dropping mention of {entity.user_id}: {e}")
+                del entities[index]
+            else:
+                entities[index] = InputMessageEntityMentionName(entity.offset, entity.length, user)
+
+
+def _plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match], str]]:
     entities = []
 
     def replacer(match: Match) -> str:
@@ -136,6 +152,16 @@ def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match],
     return entities, replacer
 
 
+def _plain_mention_to_html(match: Match) -> str:
+    puppet = pu.Puppet.find_by_displayname(match.group(2))
+    if puppet:
+        return (f"{match.group(1)}"
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                f"{puppet.displayname}"
+                "</a>")
+    return "".join(match.groups())
+
+
 def init_mx(context: "Context") -> None:
     global plain_mention_regex, should_bridge_plaintext_highlights
     config = context.config

mautrix_telegram/formatter/from_telegram.py

@@ -51,7 +51,7 @@ def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Optional[R
                  else source.tgid)
         msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
         if msg:
-            return RelatesTo(rel_type=RelationType.REFERENCE, event_id=msg.mxid)
+            return RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
     return None
 
 
@@ -79,7 +79,7 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
             try:
                 user = await source.client.get_entity(fwd_from.from_id)
                 if user:
-                    fwd_from_text = pu.Puppet.get_displayname(user, False)
+                    fwd_from_text, _ = pu.Puppet.get_displayname(user, False)
                     fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
             except (ValueError, RPCError):
                 fwd_from_text = fwd_from_html = "unknown user"
@@ -87,7 +87,7 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
         from_id = (fwd_from.from_id.chat_id if isinstance(fwd_from.from_id, PeerChat)
                    else fwd_from.from_id.channel_id)
         portal = po.Portal.get_by_tgid(TelegramID(from_id))
-        if portal:
+        if portal and portal.title:
             fwd_from_text = portal.title
             if portal.alias:
                 fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"
@@ -126,7 +126,7 @@ async def _add_reply_header(source: 'AbstractUser', content: TextMessageEventCon
     if not msg:
         return
 
-    content.relates_to = RelatesTo(rel_type=RelationType.REFERENCE, event_id=msg.mxid)
+    content.relates_to = RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
 
     try:
         event: MessageEvent = await main_intent.get_event(msg.mx_room, msg.mxid)

mautrix_telegram/matrix.py

@@ -94,9 +94,7 @@ class MatrixHandler(BaseMatrixHandler):
                 except MatrixError:
                     pass
             portal.mxid = room_id
-            e2be_ok = None
+            e2be_ok = await portal.check_dm_encryption()
-            if self.config["bridge.encryption.default"] and self.e2ee:
-                e2be_ok = await portal.enable_dm_encryption()
             await portal.save()
             await inviter.register_portal(portal)
             if e2be_ok is True:
@@ -111,6 +109,7 @@ class MatrixHandler(BaseMatrixHandler):
                 if e2be_ok is False:
                     message += "\n\nWarning: Failed to enable end-to-bridge encryption"
                 await intent.send_notice(room_id, message)
+            await portal.update_bridge_info()
         else:
             await intent.join_room(room_id)
             await intent.send_notice(room_id, "This puppet will remain inactive until a "
@@ -283,13 +282,12 @@ class MatrixHandler(BaseMatrixHandler):
         portal = po.Portal.get_by_mxid(room_id)
         sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
         if await sender.has_full_access(allow_bot=True) and portal:
-            events = new_events - old_events
+            if not new_events:
-            if len(events) > 0:
+                await portal.handle_matrix_unpin_all(sender, event_id)
-                # New event pinned, set that as pinned in Telegram.
+            else:
-                await portal.handle_matrix_pin(sender, EventID(events.pop()), event_id)
+                changes = {event_id: event_id in new_events
-            elif len(new_events) == 0:
+                           for event_id in new_events ^ old_events}
-                # All pinned events removed, remove pinned event in Telegram.
+                await portal.handle_matrix_pin(sender, changes, event_id)
-                await portal.handle_matrix_pin(sender, None, event_id)
 
     @staticmethod
     async def handle_room_upgrade(room_id: RoomID, sender: UserID, new_room_id: RoomID,
@@ -328,17 +326,15 @@ class MatrixHandler(BaseMatrixHandler):
             return
 
         for user_id, event_id in receipts:
-            user = await u.User.get_by_mxid(user_id).ensure_started()
+            user = u.User.get_by_mxid(user_id, check_db=False, create=False)
-            if not await user.is_logged_in():
+            if user and await user.is_logged_in():
-                continue
+                await portal.mark_read(user, event_id)
-            await portal.mark_read(user, event_id)
 
     @staticmethod
     async def handle_presence(user_id: UserID, presence: PresenceState) -> None:
-        user = await u.User.get_by_mxid(user_id).ensure_started()
+        user = u.User.get_by_mxid(user_id, check_db=False, create=False)
-        if not await user.is_logged_in():
+        if user and await user.is_logged_in():
-            return
+            await user.set_presence(presence == PresenceState.ONLINE)
-        await user.set_presence(presence == PresenceState.ONLINE)
 
     async def handle_typing(self, room_id: RoomID, now_typing: Set[UserID]) -> None:
         portal = po.Portal.get_by_mxid(room_id)
@@ -353,11 +349,9 @@ class MatrixHandler(BaseMatrixHandler):
             if is_typing and was_typing:
                 continue
 
-            user = await u.User.get_by_mxid(user_id).ensure_started()
+            user = u.User.get_by_mxid(user_id, check_db=False, create=False)
-            if not await user.is_logged_in():
+            if user and await user.is_logged_in():
-                continue
+                await portal.set_typing(user, is_typing)
-
-            await portal.set_typing(user, is_typing)
 
         self.previously_typing[room_id] = now_typing
 

mautrix_telegram/portal/base.py

@@ -15,22 +15,23 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, Set, Iterable, TYPE_CHECKING
 from abc import ABC, abstractmethod
+from datetime import datetime
 import asyncio
 import logging
 import json
 
 from telethon.tl.functions.messages import ExportChatInviteRequest
-from telethon.tl.types import (Channel, ChannelFull, Chat, ChatFull, ChatInviteEmpty, InputChannel,
+from telethon.tl.types import (Channel, ChannelFull, Chat, ChatFull, InputChannel,
                                InputPeerChannel, InputPeerChat, InputPeerUser, InputUser,
                                PeerChannel, PeerChat, PeerUser, TypeChat, TypeInputPeer, TypePeer,
                                TypeUser, TypeUserFull, User, UserFull, TypeInputChannel, Photo,
                                Document, TypePhotoSize, PhotoSize, InputPhotoFileLocation,
                                TypeChatParticipant, TypeChannelParticipant, PhotoEmpty, ChatPhoto,
-                               ChatPhotoEmpty)
+                               ChatPhotoEmpty, PhotoSizeProgressive, PhotoSizeEmpty)
 
 from mautrix.errors import MatrixRequestError, IntentError
 from mautrix.appservice import AppService, IntentAPI
-from mautrix.types import (RoomID, RoomAlias, UserID, EventID, EventType, MessageEventContent,
+from mautrix.types import (RoomID, RoomAlias, UserID, EventID, EventType,
                            PowerLevelStateEventContent, ContentURI)
 from mautrix.util.simple_template import SimpleTemplate
 from mautrix.util.simple_lock import SimpleLock
@@ -104,9 +105,11 @@ class BasePortal(MautrixBasePortal, ABC):
 
     dedup: PortalDedup
     send_lock: PortalSendLock
+    _pin_lock: asyncio.Lock
 
     _db_instance: DBPortal
     _main_intent: Optional[IntentAPI]
+    _room_create_lock: asyncio.Lock
 
     def __init__(self, tgid: TelegramID, peer_type: str, tg_receiver: Optional[TelegramID] = None,
                  mxid: Optional[RoomID] = None, username: Optional[str] = None,
@@ -137,6 +140,7 @@ class BasePortal(MautrixBasePortal, ABC):
 
         self.dedup = PortalDedup(self)
         self.send_lock = PortalSendLock()
+        self._pin_lock = asyncio.Lock()
 
         if tgid:
             self.by_tgid[self.tgid_full] = self
@@ -155,6 +159,10 @@ class BasePortal(MautrixBasePortal, ABC):
             return str(self.tgid)
         return f"{self.tg_receiver}<->{self.tgid}"
 
+    @property
+    def name(self) -> str:
+        return self.title
+
     @property
     def alias(self) -> Optional[RoomAlias]:
         if not self.username:
@@ -176,6 +184,10 @@ class BasePortal(MautrixBasePortal, ABC):
         elif self.peer_type == "channel":
             return PeerChannel(channel_id=self.tgid)
 
+    @property
+    def is_direct(self) -> bool:
+        return self.peer_type == "user"
+
     @property
     def has_bot(self) -> bool:
         return (bool(self.bot)
@@ -210,7 +222,18 @@ class BasePortal(MautrixBasePortal, ABC):
         return config[f"bridge.{key}"]
 
     @staticmethod
-    def _get_largest_photo_size(photo: Union[Photo, Document]
+    def _photo_size_key(photo: TypePhotoSize) -> int:
+        if isinstance(photo, PhotoSize):
+            return photo.size
+        elif isinstance(photo, PhotoSizeProgressive):
+            return max(photo.sizes)
+        elif isinstance(photo, PhotoSizeEmpty):
+            return 0
+        else:
+            return len(photo.bytes)
+
+    @classmethod
+    def _get_largest_photo_size(cls, photo: Union[Photo, Document]
                                 ) -> Tuple[Optional[InputPhotoFileLocation],
                                            Optional[TypePhotoSize]]:
         if not photo or isinstance(photo, PhotoEmpty) or (isinstance(photo, Document)
@@ -218,9 +241,7 @@ class BasePortal(MautrixBasePortal, ABC):
             return None, None
 
         largest = max(photo.thumbs if isinstance(photo, Document) else photo.sizes,
-                      key=(lambda photo2: (len(photo2.bytes)
+                      key=cls._photo_size_key)
-                                           if not isinstance(photo2, PhotoSize)
-                                           else photo2.size)))
         return InputPhotoFileLocation(
             id=photo.id,
             access_hash=photo.access_hash,
@@ -259,74 +280,45 @@ class BasePortal(MautrixBasePortal, ABC):
                     return dialog.entity
             raise
 
-    async def get_invite_link(self, user: 'u.User') -> str:
+    async def get_invite_link(self, user: 'u.User', uses: Optional[int] = None,
+                              expire: Optional[datetime] = None) -> str:
         if self.peer_type == "user":
             raise ValueError("You can't invite users to private chats.")
         if self.username:
             return f"https://t.me/{self.username}"
-        link = await user.client(ExportChatInviteRequest(peer=await self.get_input_entity(user)))
+        link = await user.client(ExportChatInviteRequest(peer=await self.get_input_entity(user),
-        if isinstance(link, ChatInviteEmpty):
+                                                         expire_date=expire, usage_limit=uses))
-            raise ValueError("Failed to get invite link.")
         return link.link
 
     # endregion
     # region Matrix room cleanup
 
-    async def get_authenticated_matrix_users(self) -> List['u.User']:
+    async def get_authenticated_matrix_users(self) -> List[UserID]:
         try:
             members = await self.main_intent.get_room_members(self.mxid)
         except MatrixRequestError:
             return []
-        authenticated: List[u.User] = []
+        authenticated: List[UserID] = []
         has_bot = self.has_bot
-        for member_str in members:
+        for member in members:
-            member = UserID(member_str)
+            if p.Puppet.get_id_from_mxid(member) or member == self.az.bot_mxid:
-            if p.Puppet.get_id_from_mxid(member) or member == self.main_intent.mxid:
                 continue
             user = await u.User.get_by_mxid(member).ensure_started()
             authenticated_through_bot = has_bot and user.relaybot_whitelisted
             if authenticated_through_bot or await user.has_full_access(allow_bot=True):
-                authenticated.append(user)
+                authenticated.append(user.mxid)
         return authenticated
 
-    @classmethod
+    async def cleanup_portal(self, message: str, puppets_only: bool = False, delete: bool = True
-    async def cleanup_room(cls, intent: IntentAPI, room_id: RoomID, message: str,
+                             ) -> None:
-                           puppets_only: bool = False) -> None:
-        # TODO use the cleanup_room from BasePortal instead of this
-        try:
-            members = await intent.get_room_members(room_id)
-        except MatrixRequestError:
-            members = []
-        for user in members:
-            puppet = await p.Puppet.get_by_mxid(UserID(user), create=False)
-            if user != intent.mxid and (not puppets_only or puppet):
-                try:
-                    if puppet:
-                        await puppet.default_mxid_intent.leave_room(room_id)
-                    else:
-                        await intent.kick_user(room_id, user, message)
-                except (MatrixRequestError, IntentError):
-                    pass
-        try:
-            await intent.leave_room(room_id)
-        except (MatrixRequestError, IntentError):
-            cls.log.warning(f"Failed to leave room {room_id} when cleaning up room", exc_info=True)
-
-    async def cleanup_portal(self, message: str, puppets_only: bool = False) -> None:
         if self.username:
             try:
                 await self.main_intent.remove_room_alias(self.alias_localpart)
             except (MatrixRequestError, IntentError):
                 self.log.warning("Failed to remove alias when cleaning up room", exc_info=True)
         await self.cleanup_room(self.main_intent, self.mxid, message, puppets_only)
-
+        if delete:
-    async def unbridge(self) -> None:
+            await self.delete()
-        await self.cleanup_portal("Room unbridged", puppets_only=True)
-        self.delete()
-
-    async def cleanup_and_delete(self) -> None:
-        await self.cleanup_portal("Portal deleted")
-        self.delete()
 
     # endregion
     # region Database conversion
@@ -350,7 +342,10 @@ class BasePortal(MautrixBasePortal, ABC):
                               config=json.dumps(self.local_config), avatar_url=self.avatar_url,
                               encrypted=self.encrypted)
 
-    def delete(self) -> None:
+    async def delete(self) -> None:
+        self.delete_sync()
+
+    def delete_sync(self) -> None:
         try:
             del self.by_tgid[self.tgid_full]
         except KeyError:
@@ -544,6 +539,7 @@ def init(context: Context) -> None:
     global config
     BasePortal.az, config, BasePortal.loop, BasePortal.bot = context.core
     BasePortal.matrix = context.mx
+    MautrixBasePortal.bridge = context.bridge
     BasePortal.max_initial_member_sync = config["bridge.max_initial_member_sync"]
     BasePortal.sync_channel_members = config["bridge.sync_channel_members"]
     BasePortal.sync_matrix_state = config["bridge.sync_matrix_state"]

mautrix_telegram/portal/deduplication.py

@@ -61,9 +61,9 @@ class PortalDedup:
         if isinstance(event, MessageService):
             hash_content = [event.date.timestamp(), event.from_id, event.action]
         else:
-            hash_content = [event.date.timestamp(), event.message]
+            hash_content = [event.date.timestamp(), event.message.strip()]
             if event.fwd_from:
-                hash_content += [event.fwd_from.from_id, event.fwd_from.channel_id]
+                hash_content += [event.fwd_from.from_id]
             elif isinstance(event, Message) and event.media:
                 try:
                     hash_content += {

mautrix_telegram/portal/matrix.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, TYPE_CHECKING
+from typing import Awaitable, Dict, Optional, Union, Any, TYPE_CHECKING
 from html import escape as escape_html
 from string import Template
 from abc import ABC
@@ -22,17 +22,16 @@ import magic
 
 from telethon.tl.functions.messages import (EditChatPhotoRequest, EditChatTitleRequest,
                                             UpdatePinnedMessageRequest, SetTypingRequest,
-                                            EditChatAboutRequest)
+                                            EditChatAboutRequest, UnpinAllMessagesRequest)
 from telethon.tl.functions.channels import EditPhotoRequest, EditTitleRequest, JoinChannelRequest
-from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError,
+from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError, MessageIdInvalidError,
-                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError,
+                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, RPCError)
-                             RPCError)
 from telethon.tl.patched import Message, MessageService
-from telethon.tl.types import (
+from telethon.tl.types import (DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
-    DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
+                               InputChatUploadedPhoto, MessageActionChatEditPhoto, MessageMediaGeo,
-    InputChatUploadedPhoto, MessageActionChatEditPhoto, MessageMediaGeo,
+                               SendMessageCancelAction, SendMessageTypingAction, TypeInputPeer,
-    SendMessageCancelAction, SendMessageTypingAction, TypeInputPeer, TypeMessageEntity,
+                               UpdateNewMessage, InputMediaUploadedDocument,
-    UpdateNewMessage, InputMediaUploadedDocument, InputMediaUploadedPhoto)
+                               InputMediaUploadedPhoto)
 
 from mautrix.types import (EventID, RoomID, UserID, ContentURI, MessageType, MessageEventContent,
                            TextMessageEventContent, MediaMessageEventContent, Format,
@@ -87,7 +86,7 @@ class PortalMatrix(BasePortal, ABC):
             message = await self._get_state_change_message(event, user, **kwargs)
             if not message:
                 return
-            message, entities = formatter.matrix_to_telegram(message)
+            message, entities = await formatter.matrix_to_telegram(self.bot.client, html=message)
             response = await self.bot.client.send_message(self.peer, message,
                                                           formatting_entities=entities)
             space = self.tgid if self.peer_type == "channel" else self.bot.tgid
@@ -113,7 +112,19 @@ class PortalMatrix(BasePortal, ABC):
         space = self.tgid if self.peer_type == "channel" else user.tgid
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
-            return
+            message = DBMessage.find_last(self.mxid, space)
+            if not message:
+                self.log.debug(f"Dropping Matrix read receipt from {user.mxid}: "
+                               f"target message {event_id} not known and last message"
+                               " in chat not found")
+                return
+            else:
+                self.log.debug(f"Matrix read receipt target {event_id} not known, marking "
+                               f"messages up to most recent ({message.mxid}/{message.tgid}) "
+                               f"as read by {user.mxid}/{user.tgid}")
+        else:
+            self.log.debug("Handling Matrix read receipt: marking messages up to "
+                           f"{message.mxid}/{message.tgid} as read by {user.mxid}/{user.tgid}")
         await user.client.send_read_acknowledge(self.peer, max_id=message.tgid,
                                                 clear_mentions=True)
 
@@ -122,7 +133,7 @@ class PortalMatrix(BasePortal, ABC):
         if user.tgid == source.tgid:
             return None
         if self.peer_type == "user" and user.tgid == self.tgid:
-            self.delete()
+            await self.delete()
             return None
         if isinstance(user, u.User) and await user.needs_relaybot(self):
             if not self.bot:
@@ -152,7 +163,7 @@ class PortalMatrix(BasePortal, ABC):
 
         if self.peer_type == "user":
             await self.main_intent.leave_room(self.mxid)
-            self.delete()
+            await self.delete()
             try:
                 del self.by_tgid[self.tgid_full]
                 del self.by_mxid[self.mxid]
@@ -214,27 +225,11 @@ class PortalMatrix(BasePortal, ABC):
         elif content.msgtype == MessageType.EMOTE:
             await self._apply_emote_format(sender, content)
 
-    @staticmethod
-    def _matrix_event_to_entities(event: Union[str, MessageEventContent]
-                                  ) -> Tuple[str, Optional[List[TypeMessageEntity]]]:
-        try:
-            if isinstance(event, str):
-                message, entities = formatter.matrix_to_telegram(event)
-            elif isinstance(event, TextMessageEventContent) and event.format == Format.HTML:
-                message, entities = formatter.matrix_to_telegram(event.formatted_body)
-            else:
-                message, entities = formatter.matrix_text_to_telegram(event.body)
-        except KeyError:
-            message, entities = None, None
-        return message, entities
-
     async def _handle_matrix_text(self, sender_id: TelegramID, event_id: EventID,
                                   space: TelegramID, client: 'MautrixTelegramClient',
                                   content: TextMessageEventContent, reply_to: TelegramID) -> None:
-        if content.formatted_body and content.format == Format.HTML:
+        message, entities = await formatter.matrix_to_telegram(client, text=content.body,
-            message, entities = formatter.matrix_to_telegram(content.formatted_body)
+                                                               html=content.formatted(Format.HTML))
-        else:
-            message, entities = formatter.matrix_text_to_telegram(content.body)
         async with self.send_lock(sender_id):
             lp = self.get_config("telegram_link_preview")
             if content.get_edit():
@@ -301,25 +296,21 @@ class PortalMatrix(BasePortal, ABC):
             media = InputMediaUploadedDocument(file=file_handle, attributes=attributes,
                                                mime_type=mime or "application/octet-stream")
 
-        if caption:
+        capt, entities = (await formatter.matrix_to_telegram(client, text=caption.body,
-            if caption.formatted_body and caption.format == Format.HTML:
+                                                             html=caption.formatted(Format.HTML))
-                caption, entities = formatter.matrix_to_telegram(caption.formatted_body)
+                          if caption else (None, None))
-            else:
-                caption, entities = formatter.matrix_text_to_telegram(content.body)
-        else:
-            caption, entities = None, None
 
         async with self.send_lock(sender_id):
-            if await self._matrix_document_edit(client, content, space, caption, media, event_id):
+            if await self._matrix_document_edit(client, content, space, capt, media, event_id):
                 return
             try:
                 response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                                   caption=caption, entities=entities)
+                                                   caption=capt, entities=entities)
             except (PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, PhotoExtInvalidError):
                 media = InputMediaUploadedDocument(file=media.file, mime_type=mime,
                                                    attributes=attributes)
                 response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                                   caption=caption, entities=entities)
+                                                   caption=capt, entities=entities)
             self._add_telegram_message_to_db(event_id, space, 0, response)
         await self._send_delivery_receipt(event_id)
 
@@ -346,8 +337,8 @@ class PortalMatrix(BasePortal, ABC):
         except (KeyError, ValueError):
             self.log.exception("Failed to parse location")
             return None
-        caption, entities = formatter.matrix_text_to_telegram(content.body)
+        caption, entities = await formatter.matrix_to_telegram(client, text=content.body)
-        media = MessageMediaGeo(geo=GeoPoint(lat, long, access_hash=0))
+        media = MessageMediaGeo(geo=GeoPoint(lat=lat, long=long, access_hash=0))
 
         async with self.send_lock(sender_id):
             if await self._matrix_document_edit(client, content, space, caption, media, event_id):
@@ -432,23 +423,23 @@ class PortalMatrix(BasePortal, ABC):
         else:
             self.log.trace("Unhandled Matrix event: %s", content)
 
-    async def handle_matrix_pin(self, sender: 'u.User', pinned_message: Optional[EventID],
+    async def handle_matrix_unpin_all(self, sender: 'u.User', pin_event_id: EventID) -> None:
+        await sender.client(UnpinAllMessagesRequest(peer=self.peer))
+        await self._send_delivery_receipt(pin_event_id)
+
+    async def handle_matrix_pin(self, sender: 'u.User', changes: Dict[EventID, bool],
                                 pin_event_id: EventID) -> None:
-        if self.peer_type != "chat" and self.peer_type != "channel":
+        tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
-            return
+        ids = {msg.mxid: msg.tgid
-        try:
+               for msg in DBMessage.get_by_mxids(list(changes.keys()),
-            if not pinned_message:
+                                                 mx_room=self.mxid, tg_space=tg_space)}
-                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=0))
+        for event_id, pinned in changes.items():
-            else:
+            try:
-                tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
+                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=ids[event_id],
-                message = DBMessage.get_by_mxid(pinned_message, self.mxid, tg_space)
+                                                               unpin=not pinned))
-                if message is None:
+            except (ChatNotModifiedError, MessageIdInvalidError, KeyError):
-                    self.log.warning(f"Could not find pinned {pinned_message} in {self.mxid}")
+                pass
-                    return
+        await self._send_delivery_receipt(pin_event_id)
-                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=message.tgid))
-            await self._send_delivery_receipt(pin_event_id)
-        except ChatNotModifiedError:
-            pass
 
     async def handle_matrix_deletion(self, deleter: 'u.User', event_id: EventID,
                                      redaction_event_id: EventID) -> None:
@@ -456,12 +447,18 @@ class PortalMatrix(BasePortal, ABC):
         space = self.tgid if self.peer_type == "channel" else real_deleter.tgid
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
-            return
+            self.log.trace(f"Ignoring Matrix redaction of unknown event {event_id}")
-        if message.edit_index == 0:
+        elif message.redacted:
+            self.log.debug("Ignoring Matrix redaction of already redacted event "
+                           f"{message.mxid} in {message.mx_room}")
+        elif message.edit_index != 0:
+            message.edit(redacted=True)
+            self.log.debug("Ignoring Matrix redaction of edit event "
+                           f"{message.mxid} in {message.mx_room}")
+        else:
+            message.edit(redacted=True)
             await real_deleter.client.delete_messages(self.peer, [message.tgid])
             await self._send_delivery_receipt(redaction_event_id)
-        else:
-            self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}")
 
     async def _update_telegram_power_level(self, sender: 'u.User', user_id: TelegramID,
                                            level: int) -> None:

mautrix_telegram/portal/metadata.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Optional, Iterable, Union, Dict, Any, TYPE_CHECKING
+from typing import List, Optional, Iterable, Union, Dict, Any, Tuple, TYPE_CHECKING
 from abc import ABC
 import asyncio
 
@@ -26,7 +26,8 @@ from telethon.tl.types import (
     Channel, ChatBannedRights, ChannelParticipantsRecent, ChannelParticipantsSearch, ChatPhoto,
     PhotoEmpty, InputChannel, InputUser, ChatPhotoEmpty, PeerUser, Photo, TypeChat, TypeInputPeer,
     TypeUser, User, InputPeerPhotoFileLocation, ChatParticipantAdmin, ChannelParticipantAdmin,
-    ChatParticipantCreator, ChannelParticipantCreator, UserProfilePhoto, UserProfilePhotoEmpty)
+    ChatParticipantCreator, ChannelParticipantCreator, UserProfilePhoto, UserProfilePhotoEmpty,
+    InputPeerUser)
 
 from mautrix.errors import MForbidden
 from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership,
@@ -58,21 +59,30 @@ class PortalMetadata(BasePortal, ABC):
 
     # region Matrix -> Telegram
 
-    async def _get_telegram_users_in_matrix_room(self) -> List[Union[InputUser, PeerUser]]:
+    async def get_telegram_users_in_matrix_room(self, source: 'u.User'
-        user_tgids = set()
+                                                ) -> Tuple[List[InputPeerUser], List[UserID]]:
+        user_tgids = {}
         user_mxids = await self.main_intent.get_room_members(self.mxid, (Membership.JOIN,
                                                                          Membership.INVITE))
-        for user_str in user_mxids:
+        for mxid in user_mxids:
-            user = UserID(user_str)
+            if mxid == self.az.bot_mxid:
-            if user == self.az.bot_mxid:
                 continue
-            mx_user = u.User.get_by_mxid(user, create=False)
+            mx_user = u.User.get_by_mxid(mxid, create=False)
             if mx_user and mx_user.tgid:
-                user_tgids.add(mx_user.tgid)
+                user_tgids[mx_user.tgid] = mxid
-            puppet_id = p.Puppet.get_id_from_mxid(user)
+            puppet_id = p.Puppet.get_id_from_mxid(mxid)
             if puppet_id:
-                user_tgids.add(puppet_id)
+                user_tgids[puppet_id] = mxid
-        return [PeerUser(user_id) for user_id in user_tgids]
+        input_users = []
+        errors = []
+        for tgid, mxid in user_tgids.items():
+            try:
+                input_users.append(await source.client.get_input_entity(tgid))
+            except ValueError as e:
+                source.log.debug(f"Failed to find the input entity for {tgid} ({mxid}) for "
+                                 f"creating a group: {e}")
+                errors.append(mxid)
+        return input_users, errors
 
     async def upgrade_telegram_chat(self, source: 'u.User') -> None:
         if self.peer_type != "chat":
@@ -97,7 +107,7 @@ class PortalMetadata(BasePortal, ABC):
             pass
         try:
             existing = self.by_tgid[(new_id, new_id)]
-            existing.delete()
+            existing.delete_sync()
         except KeyError:
             pass
         self.db_instance.edit(tgid=new_id, tg_receiver=new_id, peer_type=self.peer_type)
@@ -116,13 +126,13 @@ class PortalMetadata(BasePortal, ABC):
         if await self._update_username(username):
             await self.save()
 
-    async def create_telegram_chat(self, source: 'u.User', supergroup: bool = False) -> None:
+    async def create_telegram_chat(self, source: 'u.User', invites: List[InputUser],
+                                   supergroup: bool = False) -> None:
         if not self.mxid:
             raise ValueError("Can't create Telegram chat for portal without Matrix room.")
         elif self.tgid:
             raise ValueError("Can't create Telegram chat for portal with existing Telegram chat.")
 
-        invites = await self._get_telegram_users_in_matrix_room()
         if len(invites) < 2:
             if self.bot is not None:
                 info, mxid = await self.bot.get_me()
@@ -160,6 +170,7 @@ class PortalMetadata(BasePortal, ABC):
             levels = self._get_base_power_levels(levels, entity)
             await self.main_intent.set_power_levels(self.mxid, levels)
         await self.handle_matrix_power_levels(source, levels.users, {}, None)
+        await self.update_bridge_info()
 
     async def invite_telegram(self, source: 'u.User',
                               puppet: Union[p.Puppet, 'AbstractUser']) -> None:
@@ -295,17 +306,14 @@ class PortalMetadata(BasePortal, ABC):
 
     async def _create_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
                                   invites: InviteList) -> Optional[RoomID]:
-        direct = self.peer_type == "user"
-
-        if invites is None:
-            invites = []
-
         if self.mxid:
             return self.mxid
-
+        elif not self.allow_bridging:
-        if not self.allow_bridging:
             return None
 
+        direct = self.peer_type == "user"
+        invites = invites or []
+
         if not entity:
             entity = await self.get_entity(user)
             self.log.trace("Fetched data: %s", entity)
@@ -453,7 +461,7 @@ class PortalMetadata(BasePortal, ABC):
             levels.kick = overrides.get("kick", 50)
             levels.redact = overrides.get("redact", 50)
             levels.invite = overrides.get("invite", 50 if dbr.invite_users else 0)
-            levels.events[EventType.ROOM_ENCRYPTION] = 99
+            levels.events[EventType.ROOM_ENCRYPTION] = 50 if self.matrix.e2ee else 99
             levels.events[EventType.ROOM_TOMBSTONE] = 99
             levels.events[EventType.ROOM_NAME] = 50 if dbr.change_info else 0
             levels.events[EventType.ROOM_AVATAR] = 50 if dbr.change_info else 0
@@ -576,32 +584,38 @@ class PortalMetadata(BasePortal, ABC):
                               if self.max_initial_member_sync < 0
                               else len(allowed_tgids) < self.max_initial_member_sync - 10)
                              and (self.megagroup or self.peer_type != "channel"))
-        if trust_member_list:
+        if not trust_member_list:
-            joined_mxids = await self.main_intent.get_room_members(self.mxid)
+            return
-            for user_mxid in joined_mxids:
-                if user_mxid == self.az.bot_mxid:
-                    continue
-                puppet_id = p.Puppet.get_id_from_mxid(user_mxid)
-                if puppet_id and puppet_id not in allowed_tgids:
-                    if self.bot and puppet_id == self.bot.tgid:
-                        self.bot.remove_chat(self.tgid)
-                    try:
-                        await self.main_intent.kick_user(self.mxid, user_mxid,
-                                                         "User had left this Telegram chat.")
-                    except MForbidden:
-                        pass
-                    continue
-                mx_user = u.User.get_by_mxid(user_mxid, create=False)
-                if mx_user and mx_user.is_bot and mx_user.tgid not in allowed_tgids:
-                    await mx_user.unregister_portal(*self.tgid_full)
 
-                if mx_user and not self.has_bot and mx_user.tgid not in allowed_tgids:
+        for user_mxid in await self.main_intent.get_room_members(self.mxid):
+            if user_mxid == self.az.bot_mxid:
+                continue
+
+            puppet_id = p.Puppet.get_id_from_mxid(user_mxid)
+            if puppet_id:
+                if puppet_id in allowed_tgids:
+                    continue
+                if self.bot and puppet_id == self.bot.tgid:
+                    self.bot.remove_chat(self.tgid)
+                try:
+                    await self.main_intent.kick_user(self.mxid, user_mxid,
+                                                     "User had left this Telegram chat.")
+                except MForbidden:
+                    pass
+                continue
+
+            mx_user = u.User.get_by_mxid(user_mxid, create=False)
+            if mx_user:
+                if mx_user.tgid in allowed_tgids:
+                    continue
+                if mx_user.is_bot:
+                    await mx_user.unregister_portal(*self.tgid_full)
+                if not self.has_bot:
                     try:
                         await self.main_intent.kick_user(self.mxid, mx_user.mxid,
                                                          "You had left this Telegram chat.")
                     except MForbidden:
                         pass
-                    continue
 
     async def _add_telegram_user(self, user_id: TelegramID, source: Optional['AbstractUser'] = None
                                  ) -> None:

mautrix_telegram/portal/telegram.py

@@ -34,7 +34,8 @@ from telethon.tl.types import (
     MessageMediaPhoto, MessageMediaDice, MessageMediaGame, MessageMediaUnsupported, PeerUser,
     PhotoCachedSize, TypeChannelParticipant, TypeChatParticipant, TypeDocumentAttribute,
     TypeMessageAction, TypePhotoSize, PhotoSize, UpdateChatUserTyping, UpdateUserTyping,
-    MessageEntityPre, ChatPhotoEmpty)
+    MessageEntityPre, ChatPhotoEmpty, DocumentAttributeImageSize, DocumentAttributeAnimated,
+    UpdateChannelUserTyping, SendMessageTypingAction)
 
 from mautrix.appservice import IntentAPI
 from mautrix.types import (EventID, UserID, ImageInfo, ThumbnailInfo, RelatesTo, MessageType,
@@ -56,16 +57,18 @@ if TYPE_CHECKING:
 
 InviteList = Union[UserID, List[UserID]]
 TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant]
+UpdateTyping = Union[UpdateUserTyping, UpdateChatUserTyping, UpdateChannelUserTyping]
 DocAttrs = NamedTuple("DocAttrs", name=Optional[str], mime_type=Optional[str], is_sticker=bool,
-                      sticker_alt=Optional[str], width=int, height=int)
+                      sticker_alt=Optional[str], width=int, height=int, is_gif=bool)
 
 config: Optional['Config'] = None
 
 
 class PortalTelegram(BasePortal, ABC):
-    async def handle_telegram_typing(self, user: p.Puppet,
+    async def handle_telegram_typing(self, user: p.Puppet, update: UpdateTyping) -> None:
-                                     _: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
+        is_typing = isinstance(update.action, SendMessageTypingAction)
-        await user.intent_for(self).set_typing(self.mxid, is_typing=True)
+        # Always use the default puppet here to avoid any problems with echoing
+        await user.default_mxid_intent.set_typing(self.mxid, is_typing=is_typing)
 
     def _get_external_url(self, evt: Message) -> Optional[str]:
         if self.peer_type == "channel" and self.username is not None:
@@ -109,8 +112,7 @@ class PortalTelegram(BasePortal, ABC):
             return await self._send_message(intent, content, timestamp=evt.date)
         info = ImageInfo(
             height=largest_size.h, width=largest_size.w, orientation=0, mimetype=file.mime_type,
-            size=(len(largest_size.bytes) if (isinstance(largest_size, PhotoCachedSize))
+            size=self._photo_size_key(largest_size))
-                  else largest_size.size))
         ext = sane_mimetypes.guess_extension(file.mime_type)
         name = f"disappearing_image{ext}" if media.ttl_seconds else f"image{ext}"
         await intent.set_typing(self.mxid, is_typing=False)
@@ -135,6 +137,7 @@ class PortalTelegram(BasePortal, ABC):
     @staticmethod
     def _parse_telegram_document_attributes(attributes: List[TypeDocumentAttribute]) -> DocAttrs:
         name, mime_type, is_sticker, sticker_alt, width, height = None, None, False, None, 0, 0
+        is_gif = False
         for attr in attributes:
             if isinstance(attr, DocumentAttributeFilename):
                 name = name or attr.file_name
@@ -142,9 +145,13 @@ class PortalTelegram(BasePortal, ABC):
             elif isinstance(attr, DocumentAttributeSticker):
                 is_sticker = True
                 sticker_alt = attr.alt
+            elif isinstance(attr, DocumentAttributeAnimated):
+                is_gif = True
             elif isinstance(attr, DocumentAttributeVideo):
                 width, height = attr.w, attr.h
-        return DocAttrs(name, mime_type, is_sticker, sticker_alt, width, height)
+            elif isinstance(attr, DocumentAttributeImageSize):
+                width, height = attr.w, attr.h
+        return DocAttrs(name, mime_type, is_sticker, sticker_alt, width, height, is_gif)
 
     @staticmethod
     def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: DocAttrs,
@@ -185,7 +192,7 @@ class PortalTelegram(BasePortal, ABC):
                                                 width=file.thumbnail.width or thumb_size.w,
                                                 size=file.thumbnail.size)
         else:
-            # This is a hack for bad clients like Riot iOS that require a thumbnail
+            # This is a hack for bad clients like Element iOS that require a thumbnail
             if file.decryption_info:
                 info.thumbnail_file = file.decryption_info
             else:
@@ -226,9 +233,26 @@ class PortalTelegram(BasePortal, ABC):
         await intent.set_typing(self.mxid, is_typing=False)
 
         event_type = EventType.ROOM_MESSAGE
-        # Riot only supports images as stickers, so send animated webm stickers as m.video
+        # Elements only support images as stickers, so send animated webm stickers as m.video
         if attrs.is_sticker and file.mime_type.startswith("image/"):
             event_type = EventType.STICKER
+            # Tell clients to render the stickers as 256x256 if they're bigger
+            if info.width > 256 or info.height > 256:
+                if info.width > info.height:
+                    info.height = int(info.height / (info.width / 256))
+                    info.width = 256
+                else:
+                    info.width = int(info.width / (info.height / 256))
+                    info.height = 256
+            if info.thumbnail_info:
+                info.thumbnail_info.width = info.width
+                info.thumbnail_info.height = info.height
+        if attrs.is_gif:
+            info["fi.mau.telegram.gif"] = True
+            info["fi.mau.loop"] = True
+            info["fi.mau.autoplay"] = True
+            info["fi.mau.no_audio"] = True
+
         content = MediaMessageEventContent(
             body=name or "unnamed file", info=info, relates_to=relates_to,
             external_url=self._get_external_url(evt),
@@ -318,16 +342,63 @@ class PortalTelegram(BasePortal, ABC):
         await intent.set_typing(self.mxid, is_typing=False)
         return await self._send_message(intent, content, timestamp=evt.date)
 
+    @staticmethod
+    def _format_dice(roll: MessageMediaDice) -> str:
+        if roll.emoticon == "\U0001F3B0":
+            emojis = {
+                0: "\U0001F36B",  # "🍫",
+                1: "\U0001F352",  # "🍒",
+                2: "\U0001F34B",  # "🍋",
+                3: "7\ufe0f\u20e3"  # "7️⃣",
+            }
+            res = roll.value - 1
+            slot1, slot2, slot3 = emojis[res % 4], emojis[res // 4 % 4], emojis[res // 16]
+            return f"{slot1} {slot2} {slot3} ({roll.value})"
+        elif roll.emoticon == "\u26BD":
+            results = {
+                1: "miss",
+                2: "hit the woodwork",
+                3: "goal",  # seems to go in through the center
+                4: "goal",
+                5: "goal 🎉",  # seems to go in through the top right corner, includes confetti
+            }
+        elif roll.emoticon == "\U0001F3B3":
+            results = {
+                1: "miss",
+                2: "1 pin down",
+                3: "3 pins down, split",
+                4: "4 pins down, split",
+                5: "5 pins down",
+                6: "strike 🎉",
+            }
+        # elif roll.emoticon == "\U0001F3C0":
+        #     results = {
+        #         2: "rolled off",
+        #         3: "stuck",
+        #     }
+        # elif roll.emoticon == "\U0001F3AF":
+        #     results = {
+        #         1: "bounced off",
+        #         2: "outer rim",
+        #
+        #         6: "bullseye",
+        #     }
+        else:
+            return str(roll.value)
+        return f"{results[roll.value]} ({roll.value})"
+
     async def handle_telegram_dice(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
                                    relates_to: RelatesTo) -> EventID:
         emoji_text = {
             "\U0001F3AF": " Dart throw",
             "\U0001F3B2": " Dice roll",
             "\U0001F3C0": " Basketball throw",
+            "\U0001F3B0": " Slot machine",
+            "\U0001F3B3": " Bowling",
             "\u26BD": " Football kick"
         }
         roll: MessageMediaDice = evt.media
-        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {roll.value}"
+        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {self._format_dice(roll)}"
         content = TextMessageEventContent(msgtype=MessageType.TEXT, format=Format.HTML, body=text,
                                           formatted_body=f"<h4>{text}</h4>", relates_to=relates_to,
                                           external_url=self._get_external_url(evt))
@@ -575,6 +646,9 @@ class PortalTelegram(BasePortal, ABC):
                            "displayname, updating info...")
             entity = await source.client.get_entity(PeerUser(sender.tgid))
             await sender.update_info(source, entity)
+            if not sender.displayname:
+                self.log.debug(f"Telegram user {sender.tgid} doesn't have a displayname even after"
+                               f" updating with data {entity!s}")
 
         allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
                          MessageMediaGame, MessageMediaDice, MessageMediaPoll,
@@ -694,13 +768,20 @@ class PortalTelegram(BasePortal, ABC):
             levels.users[puppet.mxid] = 50
         await self.main_intent.set_power_levels(self.mxid, levels)
 
-    async def receive_telegram_pin_id(self, msg_id: TelegramID, receiver: TelegramID) -> None:
+    async def receive_telegram_pin_ids(self, msg_ids: List[TelegramID], receiver: TelegramID,
-        tg_space = receiver if self.peer_type != "channel" else self.tgid
+                                       remove: bool) -> None:
-        message = DBMessage.get_one_by_tgid(msg_id, tg_space) if msg_id != 0 else None
+        async with self._pin_lock:
-        if message:
+            tg_space = receiver if self.peer_type != "channel" else self.tgid
-            await self.main_intent.set_pinned_messages(self.mxid, [message.mxid])
+            previously_pinned = await self.main_intent.get_pinned_messages(self.mxid)
-        else:
+            currently_pinned_dict = {event_id: True for event_id in previously_pinned}
-            await self.main_intent.set_pinned_messages(self.mxid, [])
+            for message in DBMessage.get_first_by_tgids(msg_ids, tg_space):
+                if remove:
+                    currently_pinned_dict.pop(message.mxid, None)
+                else:
+                    currently_pinned_dict[message.mxid] = True
+            currently_pinned = list(currently_pinned_dict.keys())
+            if currently_pinned != previously_pinned:
+                await self.main_intent.set_pinned_messages(self.mxid, currently_pinned)
 
     async def set_telegram_admins_enabled(self, enabled: bool) -> None:
         level = 50 if enabled else 10

mautrix_telegram/puppet.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Any, Dict, Iterable, Optional, Union, TYPE_CHECKING
+from typing import Awaitable, Any, Dict, Iterable, Optional, Union, Tuple, TYPE_CHECKING
 from difflib import SequenceMatcher
 import unicodedata
 import asyncio
@@ -24,10 +24,11 @@ from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser,
 from yarl import URL
 
 from mautrix.appservice import AppService, IntentAPI
-from mautrix.errors import MatrixRequestError
+from mautrix.errors import MatrixRequestError, MatrixError
 from mautrix.bridge import BasePuppet
-from mautrix.types import UserID, SyncToken, RoomID
+from mautrix.types import UserID, SyncToken, RoomID, ContentURI
 from mautrix.util.simple_template import SimpleTemplate
+from mautrix.util.logging import TraceLogger
 
 from .types import TelegramID
 from .db import Puppet as DBPuppet
@@ -43,7 +44,7 @@ config: Optional['Config'] = None
 
 
 class Puppet(BasePuppet):
-    log: logging.Logger = logging.getLogger("mau.puppet")
+    log: TraceLogger = logging.getLogger("mau.puppet")
     az: AppService
     mx: 'MatrixHandler'
     loop: asyncio.AbstractEventLoop
@@ -64,6 +65,8 @@ class Puppet(BasePuppet):
     username: Optional[str]
     displayname: Optional[str]
     displayname_source: Optional[TelegramID]
+    displayname_contact: bool
+    displayname_quality: int
     photo_id: Optional[str]
     is_bot: bool
     is_registered: bool
@@ -85,6 +88,8 @@ class Puppet(BasePuppet):
                  username: Optional[str] = None,
                  displayname: Optional[str] = None,
                  displayname_source: Optional[TelegramID] = None,
+                 displayname_contact: bool = True,
+                 displayname_quality: int = 0,
                  photo_id: Optional[str] = None,
                  is_bot: bool = False,
                  is_registered: bool = False,
@@ -100,6 +105,8 @@ class Puppet(BasePuppet):
         self.username = username
         self.displayname = displayname
         self.displayname_source = displayname_source
+        self.displayname_contact = displayname_contact
+        self.displayname_quality = displayname_quality
         self.photo_id = photo_id
         self.is_bot = is_bot
         self.is_registered = is_registered
@@ -164,8 +171,10 @@ class Puppet(BasePuppet):
         return dict(access_token=self.access_token, next_batch=self._next_batch,
                     custom_mxid=self.custom_mxid, username=self.username, is_bot=self.is_bot,
                     displayname=self.displayname, displayname_source=self.displayname_source,
-                    photo_id=self.photo_id, matrix_registered=self.is_registered,
+                    displayname_contact=self.displayname_contact,
-                    disable_updates=self.disable_updates, base_url=self.base_url)
+                    displayname_quality=self.displayname_quality, photo_id=self.photo_id,
+                    matrix_registered=self.is_registered, disable_updates=self.disable_updates,
+                    base_url=str(self.base_url) if self.base_url else None)
 
     def new_db_instance(self) -> DBPuppet:
         return DBPuppet(id=self.id, **self._fields)
@@ -177,9 +186,10 @@ class Puppet(BasePuppet):
     def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
         return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
                       db_puppet.next_batch, db_puppet.base_url, db_puppet.username,
-                      db_puppet.displayname, db_puppet.displayname_source, db_puppet.photo_id,
+                      db_puppet.displayname, db_puppet.displayname_source,
-                      db_puppet.is_bot, db_puppet.matrix_registered, db_puppet.disable_updates,
+                      db_puppet.displayname_contact, db_puppet.displayname_quality,
-                      db_instance=db_puppet)
+                      db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
+                      db_puppet.disable_updates, db_instance=db_puppet)
 
     # endregion
     # region Info updating
@@ -199,11 +209,13 @@ class Puppet(BasePuppet):
         whitespace = ("\t\n\r\v\f \u00a0\u034f\u180e\u2063\u202f\u205f\u2800\u3000\u3164\ufeff"
                       "\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b"
                       "\u200c\u200d\u200e\u200f\ufe0f")
-        name = "".join(c for c in name.strip(whitespace) if unicodedata.category(c) != 'Cf')
+        allowed_other_format = ("\u200d", "\u200c")
+        name = "".join(c for c in name.strip(whitespace) if unicodedata.category(c) != 'Cf'
+                                                         or c in allowed_other_format)
         return name
 
     @classmethod
-    def get_displayname(cls, info: User, enable_format: bool = True) -> str:
+    def get_displayname(cls, info: User, enable_format: bool = True) -> Tuple[str, int]:
         fn = cls._filter_name(info.first_name)
         ln = cls._filter_name(info.last_name)
         data = {
@@ -216,19 +228,21 @@ class Puppet(BasePuppet):
         }
         preferences = config["bridge.displayname_preference"]
         name = None
+        quality = 99
         for preference in preferences:
             name = data[preference]
             if name:
                 break
+            quality -= 1
 
         if isinstance(info, User) and info.deleted:
             name = f"Deleted account {info.id}"
+            quality = 99
         elif not name:
             name = str(info.id)
+            quality = 0
 
-        if not enable_format:
+        return (cls.displayname_template.format_full(name) if enable_format else name), quality
-            return name
-        return cls.displayname_template.format_full(name)
 
     async def try_update_info(self, source: 'AbstractUser', info: User) -> None:
         try:
@@ -264,27 +278,40 @@ class Puppet(BasePuppet):
             allow_because = "user is the primary source"
         elif not isinstance(info, UpdateUserName) and not info.contact:
             allow_because = "user is not a contact"
-        elif self.displayname_source is None:
+        elif not self.displayname_source:
             allow_because = "no primary source set"
+        elif not self.displayname:
+            allow_because = "user has no name"
         else:
             return False
 
         if isinstance(info, UpdateUserName):
             info = await source.client.get_entity(PeerUser(self.tgid))
+        if not info.contact:
+            self.displayname_contact = False
+        elif not self.displayname_contact:
+            if not self.displayname:
+                self.displayname_contact = True
+            else:
+                return False
 
-        displayname = self.get_displayname(info)
+        displayname, quality = self.get_displayname(info)
-        if displayname != self.displayname:
+        if displayname != self.displayname and quality >= self.displayname_quality:
+            allow_because = f"{allow_because} and quality {quality} >= {self.displayname_quality}"
             self.log.debug(f"Updating displayname of {self.id} (src: {source.tgid}, allowed "
                            f"because {allow_because}) from {self.displayname} to {displayname}")
+            self.log.trace("Displayname source data: %s", info)
             self.displayname = displayname
             self.displayname_source = source.tgid
+            self.displayname_quality = quality
             try:
                 await self.default_mxid_intent.set_displayname(
                     displayname[:config["bridge.displayname_max_length"]])
-            except MatrixRequestError:
+            except MatrixError:
                 self.log.exception("Failed to set displayname")
                 self.displayname = ""
                 self.displayname_source = None
+                self.displayname_quality = 0
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
@@ -309,8 +336,8 @@ class Puppet(BasePuppet):
             if not photo_id:
                 self.photo_id = ""
                 try:
-                    await self.default_mxid_intent.set_avatar_url("")
+                    await self.default_mxid_intent.set_avatar_url(ContentURI(""))
-                except MatrixRequestError:
+                except MatrixError:
                     self.log.exception("Failed to set avatar")
                     self.photo_id = ""
                 return True
@@ -326,13 +353,13 @@ class Puppet(BasePuppet):
                 self.photo_id = photo_id
                 try:
                     await self.default_mxid_intent.set_avatar_url(file.mxc)
-                except MatrixRequestError:
+                except MatrixError:
                     self.log.exception("Failed to set avatar")
                     self.photo_id = ""
                 return True
         return False
 
-    def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
+    async def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
         portal: p.Portal = p.Portal.get_by_mxid(room_id)
         return portal and not portal.backfill_lock.locked and portal.peer_type != "user"
 

mautrix_telegram/user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,27 +15,28 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import (Awaitable, Dict, List, Iterable, NamedTuple, Optional, Tuple, Any, cast,
                     TYPE_CHECKING)
-from collections import defaultdict
+from datetime import datetime, timezone
 import logging
 import asyncio
 
-from telethon.tl.types import (TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage, PeerUser,
+from telethon.tl.types import (TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage,
                                UpdateShortChatMessage, UpdateShortMessage, User as TLUser, Chat,
-                               ChatForbidden)
+                               ChatForbidden, UpdateFolderPeers, UpdatePinnedDialogs,
+                               UpdateNotifySettings, NotifyPeer)
 from telethon.tl.custom import Dialog
 from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
 from telethon.tl.functions.account import UpdateStatusRequest
 
 from mautrix.client import Client
-from mautrix.errors import MatrixRequestError
+from mautrix.errors import MatrixRequestError, MNotFound
-from mautrix.types import UserID, RoomID
+from mautrix.types import UserID, RoomID, PushRuleScope, PushRuleKind, PushActionType, RoomTagInfo
-from mautrix.bridge import BaseUser
+from mautrix.bridge import BaseUser, BridgeState
 from mautrix.util.logging import TraceLogger
 from mautrix.util.opt_prometheus import Gauge
 
 from .types import TelegramID
-from .db import User as DBUser, Portal as DBPortal
+from .db import User as DBUser, Portal as DBPortal, Message as DBMessage
 from .abstract_user import AbstractUser
 from . import portal as po, puppet as pu
 
@@ -48,7 +49,12 @@ config: Optional['Config'] = None
 SearchResult = NamedTuple('SearchResult', puppet='pu.Puppet', similarity=int)
 
 METRIC_LOGGED_IN = Gauge('bridge_logged_in', 'Users logged into bridge')
-METRIC_CONNECTED = Gauge('bridge_connected', 'Users connected')
+METRIC_CONNECTED = Gauge('bridge_connected', 'Users connected to Telegram')
+
+BridgeState.human_readable_errors.update({
+    "tg-not-connected": "Your Telegram connection failed",
+    "logged-out": "You're not logged into Telegram",
+})
 
 
 class User(AbstractUser, BaseUser):
@@ -72,8 +78,9 @@ class User(AbstractUser, BaseUser):
                  saved_contacts: int = 0, is_bot: bool = False,
                  db_portals: Optional[Iterable[Tuple[TelegramID, TelegramID]]] = None,
                  db_instance: Optional[DBUser] = None) -> None:
-        super().__init__()
+        AbstractUser.__init__(self)
         self.mxid = mxid
+        BaseUser.__init__(self)
         self.tgid = tgid
         self.is_bot = is_bot
         self.username = username
@@ -85,12 +92,8 @@ class User(AbstractUser, BaseUser):
         self.db_portals = db_portals or []
         self._db_instance = db_instance
         self._ensure_started_lock = asyncio.Lock()
-        self.dm_update_lock = asyncio.Lock()
-        self._metric_value = defaultdict(lambda: False)
         self._track_connection_task = None
 
-        self.command_status = None
-
         (self.relaybot_whitelisted,
          self.whitelisted,
          self.puppet_whitelisted,
@@ -102,8 +105,6 @@ class User(AbstractUser, BaseUser):
         if tgid:
             self.by_tgid[tgid] = self
 
-        self.log = self.log.getChild(self.mxid)
-
     @property
     def name(self) -> str:
         return self.mxid
@@ -201,16 +202,12 @@ class User(AbstractUser, BaseUser):
 
     async def start(self, delete_unless_authenticated: bool = False) -> 'User':
         await super().start()
-        self._track_metric(METRIC_CONNECTED, True)
         if await self.is_logged_in():
             self.log.debug(f"Ensuring post_login() for {self.name}")
             self.loop.create_task(self.post_login())
-            if config["metrics.enabled"]:
-                self._track_connection_task = self.loop.create_task(self._track_connection())
         elif delete_unless_authenticated:
             self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting session...")
             await self.client.disconnect()
-            self._track_metric(METRIC_CONNECTED, False)
             self.client.session.delete()
         return self
 
@@ -221,6 +218,21 @@ class User(AbstractUser, BaseUser):
             connected = bool(self.client._sender._transport_connected
                              if self.client and self.client._sender else False)
             self._track_metric(METRIC_CONNECTED, connected)
+            await self.push_bridge_state(ok=connected, ttl=3600 if connected else 240,
+                                         error="tg-not-connected" if not connected else None)
+
+    async def fill_bridge_state(self, state: BridgeState) -> None:
+        await super().fill_bridge_state(state)
+        state.remote_id = str(self.tgid)
+        state.remote_name = self.human_tg_id
+
+    async def get_bridge_state(self) -> BridgeState:
+        if not self.client:
+            return BridgeState(ok=False, error="logged-out")
+        elif not self.client._sender or not self.client._sender._transport_connected:
+            return BridgeState(ok=False, error="tg-not-connected")
+        else:
+            return BridgeState(ok=True)
 
     async def stop(self) -> None:
         await super().stop()
@@ -228,8 +240,12 @@ class User(AbstractUser, BaseUser):
             self._track_connection_task.cancel()
             self._track_connection_task = None
         self._track_metric(METRIC_CONNECTED, False)
+        await self.push_bridge_state(ok=False, error="tg-not-connected")
 
     async def post_login(self, info: TLUser = None, first_login: bool = False) -> None:
+        if config["metrics.enabled"] and not self._track_connection_task:
+            self._track_connection_task = self.loop.create_task(self._track_connection())
+
         try:
             await self.update_info(info)
         except Exception:
@@ -305,11 +321,14 @@ class User(AbstractUser, BaseUser):
         for _, portal in self.portals.items():
             if not portal or portal.deleted or not portal.mxid or portal.has_bot:
                 continue
-            try:
+            if portal.peer_type == "user":
-                await portal.main_intent.kick_user(portal.mxid, self.mxid,
+                await portal.cleanup_portal("Logged out of Telegram")
-                                                   "Logged out of Telegram.")
+            else:
-            except MatrixRequestError:
+                try:
-                pass
+                    await portal.main_intent.kick_user(portal.mxid, self.mxid,
+                                                       "Logged out of Telegram.")
+                except MatrixRequestError:
+                    pass
         self.portals = {}
         self.contacts = []
         await self.save(portals=True, contacts=True)
@@ -324,7 +343,9 @@ class User(AbstractUser, BaseUser):
         if not ok:
             return False
         self.delete()
+        await self.stop()
         self._track_metric(METRIC_LOGGED_IN, False)
+        await self.push_bridge_state(ok=False, error="logged-out")
         return True
 
     def _search_local(self, query: str, max_results: int = 5, min_similarity: int = 45
@@ -373,6 +394,101 @@ class User(AbstractUser, BaseUser):
             if portal.mxid
         }
 
+    async def _tag_room(self, puppet: pu.Puppet, portal: po.Portal, tag: str, active: bool
+                        ) -> None:
+        if not tag or not portal or not portal.mxid:
+            return
+        tag_info = await puppet.intent.get_room_tag(portal.mxid, tag)
+        if active and tag_info is None:
+            tag_info = RoomTagInfo(order=0.5)
+            tag_info[self.bridge.real_user_content_key] = True
+            await puppet.intent.set_room_tag(portal.mxid, tag, tag_info)
+        elif not active and tag_info and tag_info.get(self.bridge.real_user_content_key, False):
+            await puppet.intent.remove_room_tag(portal.mxid, tag)
+
+    @staticmethod
+    async def _mute_room(puppet: pu.Puppet, portal: po.Portal, mute_until: datetime) -> None:
+        if not config["bridge.mute_bridging"] or not portal or not portal.mxid:
+            return
+        now = datetime.utcnow().replace(tzinfo=timezone.utc)
+        if mute_until is not None and mute_until > now:
+            await puppet.intent.set_push_rule(PushRuleScope.GLOBAL, PushRuleKind.ROOM, portal.mxid,
+                                              actions=[PushActionType.DONT_NOTIFY])
+        else:
+            try:
+                await puppet.intent.remove_push_rule(PushRuleScope.GLOBAL, PushRuleKind.ROOM,
+                                                     portal.mxid)
+            except MNotFound:
+                pass
+
+    async def update_folder_peers(self, update: UpdateFolderPeers) -> None:
+        if config["bridge.tag_only_on_create"]:
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        for peer in update.folder_peers:
+            portal = po.Portal.get_by_entity(peer.peer, receiver_id=self.tgid, create=False)
+            await self._tag_room(puppet, portal, config["bridge.archive_tag"],
+                                 peer.folder_id == 1)
+
+    async def update_pinned_dialogs(self, update: UpdatePinnedDialogs) -> None:
+        if config["bridge.tag_only_on_create"]:
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        # TODO bridge unpinning properly
+        for pinned in update.order:
+            portal = po.Portal.get_by_entity(pinned.peer, receiver_id=self.tgid, create=False)
+            await self._tag_room(puppet, portal, config["bridge.pinned_tag"], True)
+
+    async def update_notify_settings(self, update: UpdateNotifySettings) -> None:
+        if config["bridge.tag_only_on_create"]:
+            return
+        elif not isinstance(update.peer, NotifyPeer):
+            # TODO handle global notification setting changes?
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        portal = po.Portal.get_by_entity(update.peer.peer, receiver_id=self.tgid, create=False)
+        await self._mute_room(puppet, portal, update.notify_settings.mute_until)
+
+    async def _sync_dialog(self, portal: po.Portal, dialog: Dialog, should_create: bool,
+                           puppet: Optional[pu.Puppet]) -> None:
+        was_created = False
+        if portal.mxid:
+            try:
+                await portal.backfill(self, last_id=dialog.message.id)
+            except Exception:
+                self.log.exception(f"Error while backfilling {portal.tgid_log}")
+            try:
+                await portal.update_matrix_room(self, dialog.entity)
+            except Exception:
+                self.log.exception(f"Error while updating {portal.tgid_log}")
+        elif should_create:
+            try:
+                await portal.create_matrix_room(self, dialog.entity, invites=[self.mxid])
+                was_created = True
+            except Exception:
+                self.log.exception(f"Error while creating {portal.tgid_log}")
+        if portal.mxid and puppet and puppet.is_real_user:
+            tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+            if dialog.unread_count == 0:
+                # This is usually more reliable than finding a specific message
+                # e.g. if the last read message is a service message that isn't in the message db
+                last_read = DBMessage.find_last(portal.mxid, tg_space)
+            else:
+                last_read = DBMessage.get_one_by_tgid(portal.tgid, tg_space,
+                                                      dialog.dialog.read_inbox_max_id)
+            if last_read:
+                await puppet.intent.mark_read(last_read.mx_room, last_read.mxid)
+            if was_created or not config["bridge.tag_only_on_create"]:
+                await self._mute_room(puppet, portal, dialog.dialog.notify_settings.mute_until)
+                await self._tag_room(puppet, portal, config["bridge.pinned_tag"], dialog.pinned)
+                await self._tag_room(puppet, portal, config["bridge.archive_tag"], dialog.archived)
+
     async def sync_dialogs(self) -> None:
         if self.is_bot:
             return
@@ -382,6 +498,7 @@ class User(AbstractUser, BaseUser):
         index = 0
         self.log.debug(f"Syncing dialogs (update_limit={update_limit}, "
                        f"create_limit={create_limit})")
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
         dialog: Dialog
         async for dialog in self.client.iter_dialogs(limit=update_limit, ignore_migrated=True,
                                                      archived=False):
@@ -397,17 +514,9 @@ class User(AbstractUser, BaseUser):
                 continue
             portal = po.Portal.get_by_entity(entity, receiver_id=self.tgid)
             self.portals[portal.tgid_full] = portal
-            if portal.mxid:
+            coro = self._sync_dialog(portal=portal, dialog=dialog, puppet=puppet,
-                update_task = portal.update_matrix_room(self, entity)
+                                     should_create=not create_limit or index < create_limit)
-                backfill_task = portal.backfill(self, last_id=dialog.message.id)
+            creators.append(self.loop.create_task(coro))
-                creators.append(self._catch(f"updating {portal.tgid_log}",
-                                            self.loop.create_task(update_task)))
-                creators.append(self._catch(f"backfilling {portal.tgid_log}",
-                                            self.loop.create_task(backfill_task)))
-            elif not create_limit or index < create_limit:
-                create_task = portal.create_matrix_room(self, entity, invites=[self.mxid])
-                creators.append(self._catch(f"creating {portal.tgid_log}",
-                                            self.loop.create_task(create_task)))
             index += 1
         await self.save(portals=True)
         await asyncio.gather(*creators)
@@ -459,7 +568,8 @@ class User(AbstractUser, BaseUser):
     # region Class instance lookup
 
     @classmethod
-    def get_by_mxid(cls, mxid: UserID, create: bool = True) -> Optional['User']:
+    def get_by_mxid(cls, mxid: UserID, create: bool = True, check_db: bool = True
+                    ) -> Optional['User']:
         if not mxid:
             raise ValueError("Matrix ID can't be empty")
 
@@ -468,10 +578,11 @@ class User(AbstractUser, BaseUser):
         except KeyError:
             pass
 
-        user = DBUser.get_by_mxid(mxid)
+        if check_db:
-        if user:
+            user = DBUser.get_by_mxid(mxid)
-            user = cls.from_db(user)
+            if user:
-            return user
+                user = cls.from_db(user)
+                return user
 
         if create:
             user = cls(mxid)

mautrix_telegram/util/file_transfer.py

@@ -30,7 +30,6 @@ from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, Locatio
                              SecurityError, FileIdInvalidError)
 
 from mautrix.appservice import IntentAPI
-from mautrix.types import EncryptedFile
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
@@ -221,9 +220,9 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
 
         image_converted = False
         # A weird bug in alpine/magic makes it return application/octet-stream for gzips...
-        is_tgs = (mime_type == "application/gzip" or (mime_type == "application/octet-stream"
+        is_tgs = (mime_type == "application/gzip"
-                                                      and magic.from_buffer(file).startswith(
+                  or (mime_type == "application/octet-stream"
-                "gzip")))
+                      and magic.from_buffer(file).startswith("gzip")))
         if is_sticker and tgs_convert and is_tgs:
             converted_anim = await convert_tgs_to(file, tgs_convert["target"],
                                                   **tgs_convert["args"])
@@ -233,14 +232,6 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
             image_converted = mime_type != "application/gzip"
             thumbnail = None
 
-        if mime_type == "image/webp":
-            new_mime_type, file, width, height = convert_image(
-                file, source_mime="image/webp", target_type="png",
-                thumbnail_to=(256, 256) if is_sticker else None)
-            image_converted = new_mime_type != mime_type
-            mime_type = new_mime_type
-            thumbnail = None
-
         decryption_info = None
         upload_mime_type = mime_type
         if encrypt and encrypt_attachment:

mautrix_telegram/util/parallel_file_transfer.py

@@ -27,8 +27,10 @@ from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLoc
                                InputPhotoFileLocation, InputPeerPhotoFileLocation, TypeInputFile,
                                InputFileBig, InputFile)
 from telethon.tl.functions.auth import ExportAuthorizationRequest, ImportAuthorizationRequest
+from telethon.tl.functions import InvokeWithLayerRequest
 from telethon.tl.functions.upload import (GetFileRequest, SaveFilePartRequest,
                                           SaveBigFilePartRequest)
+from telethon.tl.alltlobjects import LAYER
 from telethon.network import MTProtoSender
 from telethon.crypto import AuthKey
 from telethon import utils, helpers
@@ -193,9 +195,9 @@ class ParallelTransferrer:
         if not self.auth_key:
             log.debug(f"Exporting auth to DC {self.dc_id}")
             auth = await self.client(ExportAuthorizationRequest(self.dc_id))
-            req = self.client._init_with(ImportAuthorizationRequest(
+            self.client._init_request.query = ImportAuthorizationRequest(id=auth.id,
-                id=auth.id, bytes=auth.bytes
+                                                                         bytes=auth.bytes)
-            ))
+            req = InvokeWithLayerRequest(LAYER, self.client._init_request)
             await sender.send(req)
             self.auth_key = sender.auth_key
         return sender

mautrix_telegram/web/provisioning/__init__.py

@@ -141,6 +141,12 @@ class ProvisioningAPI(AuthAPI):
             return self.get_error_response(403, "not_enough_permissions",
                                            "You do not have the permissions to bridge that room.")
 
+        is_logged_in = user is not None and await user.is_logged_in()
+        acting_user = user if is_logged_in else self.context.bot
+        if not acting_user:
+            return self.get_login_response(status=403, errcode="not_logged_in",
+                                           error="You are not logged in and there is no relay bot.")
+
         portal = Portal.get_by_tgid(tgid, peer_type=peer_type)
         if portal.mxid == room_id:
             return self.get_error_response(200, "bridge_exists",
@@ -157,35 +163,30 @@ class ProvisioningAPI(AuthAPI):
                                                "Telegram chat is already bridged to another "
                                                "Matrix room.")
 
-        is_logged_in = user is not None and await user.is_logged_in()
+        async with portal._room_create_lock:
-        acting_user = user if is_logged_in else self.context.bot
+            entity: Optional[TypeChat] = None
-        if not acting_user:
+            try:
-            return self.get_login_response(status=403, errcode="not_logged_in",
+                entity = await acting_user.client.get_entity(portal.peer)
-                                           error="You are not logged in and there is no relay bot.")
+            except Exception:
+                self.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
 
-        entity: Optional[TypeChat] = None
+            if not entity or isinstance(entity, (ChatForbidden, ChannelForbidden)):
-        try:
+                if is_logged_in:
-            entity = await acting_user.client.get_entity(portal.peer)
+                    return self.get_error_response(403, "user_not_in_chat",
-        except Exception:
+                                                   "Failed to get info of Telegram chat. "
-            self.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
+                                                   "Are you in the chat?")
-
+                return self.get_error_response(403, "bot_not_in_chat",
-        if not entity or isinstance(entity, (ChatForbidden, ChannelForbidden)):
-            if is_logged_in:
-                return self.get_error_response(403, "user_not_in_chat",
                                                "Failed to get info of Telegram chat. "
-                                               "Are you in the chat?")
+                                               "Is the relay bot in the chat?")
-            return self.get_error_response(403, "bot_not_in_chat",
-                                           "Failed to get info of Telegram chat. "
-                                           "Is the relay bot in the chat?")
 
-        direct = False
+            portal.mxid = room_id
+            portal.by_mxid[portal.mxid] = portal
+            (portal.title, portal.about, levels,
+             portal.encrypted) = await get_initial_state(self.az.intent, room_id)
+            portal.photo_id = ""
+            await portal.save()
 
-        portal.mxid = room_id
+        asyncio.ensure_future(portal.update_matrix_room(user, entity, direct=False, levels=levels),
-        portal.title, portal.about, levels = await get_initial_state(self.az.intent, room_id)
-        portal.photo_id = ""
-        await portal.save()
-
-        asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
                               loop=self.loop)
 
         return web.Response(status=202, body="{}")
@@ -216,7 +217,7 @@ class ProvisioningAPI(AuthAPI):
                                            "You do not have the permissions to bridge that room.")
 
         try:
-            title, about, _ = await get_initial_state(self.az.intent, room_id)
+            title, about, _, encrypted = await get_initial_state(self.az.intent, room_id)
         except (MatrixRequestError, IntentError):
             return self.get_error_response(403, "bot_not_in_room",
                                            "The bridge bot is not in the given room.")
@@ -240,11 +241,12 @@ class ProvisioningAPI(AuthAPI):
             "group": "chat",
         }[type]
 
-        portal = Portal(tgid=TelegramID(0), mxid=room_id, title=title, about=about, peer_type=type)
+        portal = Portal(tgid=TelegramID(0), mxid=room_id, title=title, about=about, peer_type=type,
+                        encrypted=encrypted)
         try:
             await portal.create_telegram_chat(user, supergroup=supergroup)
         except ValueError as e:
-            portal.delete()
+            await portal.delete()
             return self.get_error_response(500, "unknown_error", e.args[0])
 
         return web.json_response({

optional-requirements.txt

@@ -7,24 +7,21 @@ cchardet
 aiodns
 brotli
 
-#/webp_convert
-pillow>=4,<8
-
 #/qr_login
-pillow>=4,<8
+pillow>=4,<9
 qrcode>=6,<7
 
 #/hq_thumbnails
 moviepy>=1,<2
 
 #/metrics
-prometheus_client>=0.6,<0.9
+prometheus_client>=0.6,<0.12
 
 #/postgres
 psycopg2-binary>=2,<3
 
 #/e2be
-asyncpg>=0.20,<0.22
+asyncpg>=0.20,<0.24
 python-olm>=3,<4
 pycryptodome>=3,<4
 unpaddedbase64>=1,<2

requirements.txt

@@ -1,10 +1,10 @@
-SQLAlchemy>=1.2,<2
+SQLAlchemy>=1.2,<1.4
 alembic>=1,<2
-ruamel.yaml>=0.15.35,<0.17
+ruamel.yaml>=0.15.35,<0.18
 python-magic>=0.4,<0.5
 commonmark>=0.8,<0.10
-aiohttp>=3,<3.7
+aiohttp>=3,<4
-yarl<1.6
+yarl>=1,<2
-mautrix==0.8.0rc1
+mautrix>=0.9.3,<0.10
-telethon>=1.17,<1.18
+telethon>=1.20,<1.22
 telethon-session-sqlalchemy>=0.2.14,<0.3

setup.py

@@ -49,7 +49,7 @@ setuptools.setup(
 
     install_requires=install_requires,
     extras_require=extras_require,
-    python_requires="~=3.6",
+    python_requires="~=3.7",
 
     setup_requires=["pytest-runner"],
     tests_require=["pytest", "pytest-asyncio", "pytest-mock"],