Bump version to 0.6.0

Users who the bridge only saw via logged in users with the target user
in their contact lists wouldn't get their displayname updated due to an
invalid condition in the update_displayname function.

.gitignore

@@ -14,3 +14,4 @@ config.yaml
 registration.yaml
 *.log*
 *.db
+*.bak

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/alpine:3.9
+FROM docker.io/alpine:3.10
 
 ENV UID=1337 \
     GID=1337 \
@@ -14,8 +14,7 @@ RUN apk add --no-cache \
       py3-sqlalchemy \
       py3-markdown \
       py3-psycopg2 \
-      # Not yet in stable repos:
+      py3-ruamel.yaml \
-      #py3-ruamel \
       # Indirect dependencies
       #commonmark
         py3-future \
@@ -33,6 +32,7 @@ RUN apk add --no-cache \
         py3-rsa \
       # Other dependencies
       python3-dev \
+      libffi-dev \
       build-base \
       ffmpeg \
       ca-certificates \

example-config.yaml

@@ -60,11 +60,15 @@ appservice:
     bot_displayname: Telegram bridge bot
     bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
 
+    # Community ID for bridged users (changes registration file) and rooms.
+    # Must be created manually.
+    community_id: false
+
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
     hs_token: "This value is generated when generating the registration"
 
-# Prometheus telemetry config. Requires prometheus-aio to be installed.
+# Prometheus telemetry config. Requires prometheus-client to be installed.
 metrics:
     enabled: false
     listen_port: 8000

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.6.0rc1"
+__version__ = "0.6.0"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/config.py

@@ -189,6 +189,8 @@ class Config(DictWithRecursion):
         copy("appservice.bot_displayname")
         copy("appservice.bot_avatar")
 
+        copy("appservice.community_id")
+
         copy("appservice.as_token")
         copy("appservice.hs_token")
 
@@ -352,3 +354,6 @@ class Config(DictWithRecursion):
             "sender_localpart": self["appservice.bot_username"],
             "rate_limited": False
         }
+        if self["appservice.community_id"]:
+            self._registration["namespaces"]["users"][0]["group_id"] \
+                = self["appservice.community_id"]

mautrix_telegram/formatter/from_matrix/parser.py

@@ -18,15 +18,15 @@ from typing import List, Tuple, Pattern
 import re
 
 from telethon.tl.types import (MessageEntityMention as Mention, MessageEntityBotCommand as Command,
-                               MessageEntityMentionName as MentionName, MessageEntityEmail as Email,
+                               MessageEntityMentionName as MentionName, MessageEntityUrl as URL,
-                               MessageEntityUrl as URL, MessageEntityTextUrl as TextURL,
+                               MessageEntityEmail as Email, MessageEntityTextUrl as TextURL,
                                MessageEntityBold as Bold, MessageEntityItalic as Italic,
                                MessageEntityCode as Code, MessageEntityPre as Pre,
-                               TypeMessageEntity)
+                               MessageEntityStrike as Strike, MessageEntityUnderline as Underline,
+                               MessageEntityBlockquote as Blockquote, TypeMessageEntity)
 
 from ... import user as u, puppet as pu, portal as po
 from ...types import MatrixUserID
-from ..util import html_to_unicode
 from .telegram_message import TelegramMessage, Entity, offset_length_multiply
 
 from .html_reader import HTMLNode, read_html
@@ -101,13 +101,6 @@ class MatrixParser:
             children.append(child)
         return TelegramMessage.join(children, "\n")
 
-    @classmethod
-    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
-        msg = cls.tag_aware_parse_node(node, ctx)
-        children = msg.trim().split("\n")
-        children = [child.prepend("> ") for child in children]
-        return TelegramMessage.join(children, "\n")
-
     @classmethod
     def header_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         children = cls.node_to_tmessages(node, ctx)
@@ -122,15 +115,14 @@ class MatrixParser:
             msg.format(Bold)
         elif node.tag in ("i", "em"):
             msg.format(Italic)
+        elif node.tag in ("s", "strike", "del"):
+            msg.format(Strike)
+        elif node.tag in ("u", "ins"):
+            msg.format(Underline)
+        elif node == "blockquote":
+            msg.format(Blockquote)
         elif node.tag == "command":
             msg.format(Command)
-        elif node.tag in ("s", "strike", "del"):
-            msg.text = html_to_unicode(msg.text, "\u0336")
-        elif node.tag in ("u", "ins"):
-            msg.text = html_to_unicode(msg.text, "\u0332")
-
-        if node.tag in ("s", "strike", "del", "u", "ins"):
-            msg.entities = Entity.adjust(msg.entities, offset_length_multiply(2))
 
         return msg
 
@@ -169,11 +161,16 @@ class MatrixParser:
                 if msg.text == href
                 else msg.format(TextURL, url=href))
 
+    @classmethod
+    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        children = msg.trim().split("\n")
+        children = [child.prepend("> ") for child in children]
+        return TelegramMessage.join(children, "\n")
+
     @classmethod
     def node_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
-        if node.tag == "blockquote":
+        if node.tag == "ol":
-            return cls.blockquote_to_tmessage(node, ctx)
-        elif node.tag == "ol":
             return cls.list_to_tmessage(node, ctx)
         elif node.tag == "ul":
             return cls.list_to_tmessage(node, ctx.enter_list())
@@ -183,6 +180,11 @@ class MatrixParser:
             return TelegramMessage("\n")
         elif node.tag in ("b", "strong", "i", "em", "s", "del", "u", "ins", "command"):
             return cls.basic_format_to_tmessage(node, ctx)
+        elif node.tag == "blockquote":
+            # Telegram already has blockquote entities in the protocol schema, but it strips them
+            # server-side and none of the official clients support them.
+            # TODO once Telegram changes that, use the above if block for blockquotes too.
+            return cls.blockquote_to_tmessage(node, ctx)
         elif node.tag == "a":
             return cls.link_to_tstring(node, ctx)
         elif node.tag == "p":

mautrix_telegram/formatter/from_telegram.py

@@ -24,7 +24,8 @@ from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, M
                                MessageEntityItalic, MessageEntityCode, MessageEntityPre,
                                MessageEntityBotCommand, MessageEntityHashtag, MessageEntityCashtag,
                                MessageEntityPhone, TypeMessageEntity, Message, PeerChannel,
-                               MessageFwdHeader, PeerUser)
+                               MessageEntityBlockquote, MessageEntityStrike, MessageFwdHeader,
+                               MessageEntityUnderline, PeerUser)
 
 from mautrix_appservice import MatrixRequestError
 from mautrix_appservice.intent_api import IntentAPI
@@ -33,7 +34,7 @@ from .. import user as u, puppet as pu, portal as po
 from ..types import TelegramID
 from ..db import Message as DBMessage
 from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
-                   trim_reply_fallback_text, unicode_to_html)
+                   trim_reply_fallback_text)
 
 if TYPE_CHECKING:
     from ..abstract_user import AbstractUser
@@ -194,9 +195,6 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
         text += f"\n- {evt.post_author}"
         html += f"<br/><i>- <u>{evt.post_author}</u></i>"
 
-    html = unicode_to_html(text, html, "\u0336", "del")
-    html = unicode_to_html(text, html, "\u0332", "u")
-
     if html:
         html = html.replace("\n", "<br/>")
 
@@ -214,29 +212,43 @@ def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEnti
     return "[failed conversion in _telegram_entities_to_matrix]"
 
 
-def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -> str:
+def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity],
+                                 offset: int = 0, length: int = None) -> str:
     if not entities:
-        return text
+        return escape(text)
+    if length is None:
+        length = len(text)
     html = []
     last_offset = 0
-    for entity in entities:
+    for i, entity in enumerate(entities):
-        if entity.offset > last_offset:
+        if entity.offset > offset + length:
-            html.append(escape(text[last_offset:entity.offset]))
+            break
-        elif entity.offset < last_offset:
+        relative_offset = entity.offset - offset
+        if relative_offset > last_offset:
+            html.append(escape(text[last_offset:relative_offset]))
+        elif relative_offset < last_offset:
             continue
 
         skip_entity = False
-        entity_text = escape(text[entity.offset:entity.offset + entity.length])
+        entity_text = _telegram_entities_to_matrix(
+            text=text[relative_offset:relative_offset + entity.length],
+            entities=entities[i + 1:], offset=entity.offset, length=entity.length)
         entity_type = type(entity)
 
         if entity_type == MessageEntityBold:
             html.append(f"<strong>{entity_text}</strong>")
         elif entity_type == MessageEntityItalic:
             html.append(f"<em>{entity_text}</em>")
+        elif entity_type == MessageEntityUnderline:
+            html.append(f"<u>{entity_text}</u>")
+        elif entity_type == MessageEntityStrike:
+            html.append(f"<del>{entity_text}</del>")
+        elif entity_type == MessageEntityBlockquote:
+            html.append(f"<blockquote>{entity_text}</blockquote>")
         elif entity_type == MessageEntityCode:
-            html.append(("<pre><code>{entity_text}</code></pre>"
+            html.append(f"<pre><code>{entity_text}</code></pre>"
-                         if "\n" in entity_text
+                        if "\n" in entity_text
-                         else "<code>{entity_text}</code>").format(entity_text=entity_text))
+                        else f"<code>{entity_text}</code>")
         elif entity_type == MessageEntityPre:
             skip_entity = _parse_pre(html, entity_text, entity.language)
         elif entity_type == MessageEntityMention:
@@ -254,8 +266,8 @@ def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -
             html.append(f"<font color='blue'>{entity_text}</font>")
         else:
             skip_entity = True
-        last_offset = entity.offset + (0 if skip_entity else entity.length)
+        last_offset = relative_offset + (0 if skip_entity else entity.length)
-    html.append(text[last_offset:])
+    html.append(escape(text[last_offset:]))
 
     return "".join(html)
 

mautrix_telegram/formatter/util.py

@@ -20,38 +20,6 @@ import struct
 import re
 
 
-def unicode_to_html(text: str, html: str, ctrl: str, tag: str) -> str:
-    if ctrl not in text:
-        return html
-    if not html:
-        html = escape(text)
-    tag_start = f"<{tag}>"
-    tag_end = f"</{tag}>"
-    characters = html.split(ctrl)
-    html = ""
-    in_tag = False
-    for char in characters:
-        if not in_tag:
-            if len(char) > 1:
-                html += char[0:-1]
-                char = char[-1]
-            html += tag_start
-            in_tag = True
-            html += char
-        else:
-            if len(char) > 1:
-                html += tag_end
-                in_tag = False
-            html += char
-    if in_tag:
-        html += tag_end
-    return html
-
-
-def html_to_unicode(text: str, ctrl: str) -> str:
-    return ctrl.join(text) + ctrl
-
-
 # add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
 # Licensed under the MIT license.
 # https://github.com/LonamiWebs/Telethon/blob/7cce7aa3e4c6c7019a55530391b1761d33e5a04e/telethon/helpers.py

mautrix_telegram/matrix.py

@@ -399,12 +399,24 @@ class MatrixHandler:
         return (sender == self.az.bot_mxid
                 or pu.Puppet.get_id_from_mxid(sender) is not None)
 
-    async def try_handle_event(self, evt: MatrixEvent) -> None:
+    async def try_handle_ephemeral_event(self, evt: MatrixEvent) -> None:
         try:
-            await self.handle_event(evt)
+            await self.handle_ephemeral_event(evt)
         except Exception:
             self.log.exception("Error handling manually received Matrix event")
 
+    async def handle_ephemeral_event(self, evt: MatrixEvent) -> None:
+        evt_type = evt.get("type", "m.unknown")  # type: str
+        room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
+        sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
+        content = evt.get("content", {})  # type: Dict
+        if evt_type == "m.receipt":
+            await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
+        elif evt_type == "m.presence":
+            await self.handle_presence(sender, content.get("presence", "offline"))
+        elif evt_type == "m.typing":
+            await self.handle_typing(room_id, content.get("user_ids", []))
+
     async def handle_event(self, evt: MatrixEvent) -> None:
         if self.filter_matrix_event(evt):
             return
@@ -414,53 +426,52 @@ class MatrixHandler:
         room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
         event_id = evt.get("event_id", None)  # type: Optional[MatrixEventID]
         sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
+        state_key = evt.get("state_key", None)
         content = evt.get("content", {})  # type: Dict
-        if evt_type == "m.room.member":
+        if state_key is not None:
-            state_key = evt["state_key"]  # type: MatrixUserID
+            if evt_type == "m.room.member":
-            prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: Dict
+                prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: Dict
-            membership = content.get("membership", "")  # type: str
+                membership = content.get("membership", "")  # type: str
-            prev_membership = prev_content.get("membership", "leave")  # type: str
+                prev_membership = prev_content.get("membership", "leave")  # type: str
-            if membership == prev_membership:
+                if membership == prev_membership:
-                match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
+                    match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
-                mxid = match.group(0)  # type: str
+                    mxid = match.group(0)  # type: str
-                displayname = content.get("displayname", None) or mxid  # type: str
+                    displayname = content.get("displayname", None) or mxid  # type: str
-                prev_displayname = prev_content.get("displayname", None) or mxid  # type: str
+                    prev_displayname = prev_content.get("displayname", None) or mxid  # type: str
-                if displayname != prev_displayname:
+                    if displayname != prev_displayname:
-                    await self.handle_name_change(room_id, state_key, displayname,
+                        await self.handle_name_change(room_id, state_key, displayname,
-                                                  prev_displayname, event_id)
+                                                      prev_displayname, event_id)
-            elif membership == "invite":
+                elif membership == "invite":
-                await self.handle_invite(room_id, state_key, sender)
+                    await self.handle_invite(room_id, state_key, sender)
-            elif prev_membership == "join" and membership == "leave":
+                elif prev_membership == "join" and membership == "leave":
-                await self.handle_part(room_id, state_key, sender, event_id)
+                    await self.handle_part(room_id, state_key, sender, event_id)
-            elif membership == "join":
+                elif membership == "join":
-                await self.handle_join(room_id, state_key, event_id)
+                    await self.handle_join(room_id, state_key, event_id)
-        elif evt_type in ("m.room.message", "m.sticker"):
+            elif evt_type == "m.room.power_levels":
-            if evt_type != "m.room.message":
+                prev_content = evt.get("unsigned", {}).get("prev_content", {})
-                content["msgtype"] = evt_type
+                await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
-            await self.handle_message(room_id, sender, content, event_id)
+            elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
-        elif evt_type == "m.room.redaction":
+                await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
-            await self.handle_redaction(room_id, sender, evt["redacts"])
+            elif evt_type == "m.room.pinned_events":
-        elif evt_type == "m.room.power_levels":
+                new_events = set(evt["content"]["pinned"])
-            prev_content = evt.get("unsigned", {}).get("prev_content", {})
+                try:
-            await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
+                    old_events = set(evt["unsigned"]["prev_content"]["pinned"])
-        elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
+                except KeyError:
-            await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
+                    old_events = set()
-        elif evt_type == "m.room.pinned_events":
+                await self.handle_room_pin(room_id, sender, new_events, old_events)
-            new_events = set(evt["content"]["pinned"])
+            elif evt_type == "m.room.tombstone":
-            try:
+                await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
-                old_events = set(evt["unsigned"]["prev_content"]["pinned"])
+            else:
-            except KeyError:
+                return
-                old_events = set()
-            await self.handle_room_pin(room_id, sender, new_events, old_events)
-        elif evt_type == "m.room.tombstone":
-            await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
-        elif evt_type == "m.receipt":
-            await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
-        elif evt_type == "m.presence":
-            await self.handle_presence(sender, content.get("presence", "offline"))
-        elif evt_type == "m.typing":
-            await self.handle_typing(room_id, content.get("user_ids", []))
         else:
-            return
+            if evt_type in ("m.room.message", "m.sticker"):
+                if evt_type != "m.room.message":
+                    content["msgtype"] = evt_type
+                await self.handle_message(room_id, sender, content, event_id)
+            elif evt_type == "m.room.redaction":
+                await self.handle_redaction(room_id, sender, evt["redacts"])
+            else:
+                return
+
         if EVENT_TIME:
             EVENT_TIME.labels(event_type=evt_type).observe(time.time() - start_time)

mautrix_telegram/portal.py

@@ -397,6 +397,11 @@ class Portal:
             "type": "m.room.power_levels",
             "content": power_levels,
         }]
+        if config["appservice.community_id"]:
+            initial_state.append({
+                "type": "m.room.related_groups",
+                "content": {"groups": [config["appservice.community_id"]]},
+            })
 
         room_id = await self.main_intent.create_room(alias=alias, is_public=public,
                                                      is_direct=direct, invitees=invites or [],
@@ -1578,7 +1583,7 @@ class Portal:
                                       external_url=self.get_external_url(evt))
 
     async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI,
-                                          evt: Message, _: dict = None) -> dict:
+                                          evt: Message, relates_to: dict = None) -> dict:
         override_text = ("This message is not supported on your version of Mautrix-Telegram. "
                          "Please check https://github.com/tulir/mautrix-telegram or ask your "
                          "bridge administrator about possible updates.")
@@ -1769,7 +1774,7 @@ class Portal:
                 return
 
         if sender and not sender.displayname:
-            self.log.debug(f"Telegram user {sender.tgid} sent a message, but doesn't have a"
+            self.log.debug(f"Telegram user {sender.tgid} sent a message, but doesn't have a "
                            "displayname, updating info...")
             entity = await source.client.get_entity(PeerUser(sender.tgid))
             await sender.update_info(source, entity)

mautrix_telegram/puppet.py

@@ -221,13 +221,13 @@ class Puppet:
         return new_events
 
     def handle_sync(self, presence: List, ephemeral: Dict) -> None:
-        presence_events = [self.mx.try_handle_event(event) for event in presence]
+        presence_events = [self.mx.try_handle_ephemeral_event(event) for event in presence]
 
         for room_id, events in ephemeral.items():
             for event in events:
                 event["room_id"] = room_id
 
-        ephemeral_events = [self.mx.try_handle_event(event)
+        ephemeral_events = [self.mx.try_handle_ephemeral_event(event)
                             for events in ephemeral.values()
                             for event in self.filter_events(events)]
 
@@ -363,11 +363,13 @@ class Puppet:
                                  ) -> bool:
         if self.disable_updates:
             return False
-        is_main_source = (source.is_relaybot or (self.displayname_source is not None
+        allow_source = (source.is_relaybot
-                                                 and self.displayname_source == source.tgid))
+                        or self.displayname_source == source.tgid
-        # No phone -> not in contact list -> can't set custom name -> name is trustworthy
+                        # No displayname source, so just trust anything
-        is_trustworthy_source = isinstance(info, User) and info.phone is None
+                        or self.displayname_source is None
-        if not is_main_source and not is_trustworthy_source:
+                        # No phone -> not in contact list -> can't set custom name
+                        or (isinstance(info, User) and info.phone is None))
+        if not allow_source:
             return False
         elif isinstance(info, UpdateUserName):
             info = await source.client.get_entity(PeerUser(self.tgid))

mautrix_telegram/util/file_transfer.py

@@ -27,7 +27,7 @@ from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLoc
                                TypePhotoSize, PhotoSize, PhotoCachedSize, InputPhotoFileLocation,
                                InputPeerPhotoFileLocation)
 from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, LocationInvalidError,
-                             SecurityError)
+                             SecurityError, FileIdInvalidError)
 from mautrix_appservice import IntentAPI
 
 from ..tgclient import MautrixTelegramClient
@@ -184,7 +184,7 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
 
     try:
         file = await client.download_file(location)
-    except LocationInvalidError:
+    except (LocationInvalidError, FileIdInvalidError):
         return None
     except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
         log.exception(f"{e.__class__.__name__} while downloading a file.")

setup.py

@@ -6,7 +6,7 @@ extras = {
     "fast_crypto": ["cryptg>=0.1,<0.3"],
     "webp_convert": ["Pillow>=4.3.0,<7"],
     "hq_thumbnails": ["moviepy>=1.0,<2.0"],
-    "metrics": ["prometheus-client>=0.6.0,<0.7.0"],
+    "metrics": ["prometheus-client>=0.6.0,<0.8.0"],
 }
 extras["all"] = list({dep for deps in extras.values() for dep in deps})
 
@@ -38,7 +38,7 @@ setuptools.setup(
         "ruamel.yaml>=0.15.35,<0.16",
         "future-fstrings>=0.4.2",
         "python-magic>=0.4.15,<0.5",
-        "telethon>=1.7,<1.9",
+        "telethon>=1.9,<1.10",
         "telethon-session-sqlalchemy>=0.2.14,<0.3",
     ],
     extras_require=extras,