Bump version to 0.9.0

.gitlab-ci.yml

@@ -17,6 +17,13 @@ build amd64:
   - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
   - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
   - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  after_script:
+  - |
+    if [ "$CI_COMMIT_BRANCH" = "master" ]; then
+      apk add --update curl
+      rm -rf /var/cache/apk/*
+      curl "$NOVA_ADMIN_API_URL" -H "Content-Type: application/json" -d '{"password":"'"$NOVA_ADMIN_NIGHTLY_PASS"'","bridge":"'$NOVA_BRIDGE_TYPE'","image":"'$CI_REGISTRY_IMAGE':'$CI_COMMIT_SHA'-amd64"}'
+    fi
 
 build arm64:
   stage: build

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.9.0rc2"
+__version__ = "0.9.0"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/formatter/__init__.py

@@ -1,5 +1,4 @@
-from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
+from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram, init_mx
-                          init_mx)
 from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
 from .. import context as c
 

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -18,10 +18,12 @@ import re
 import logging
 
 from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityItalic,
-                               TypeMessageEntity)
+                               TypeMessageEntity, InputMessageEntityMentionName)
 from telethon.helpers import add_surrogate, del_surrogate
+from telethon import TelegramClient
 
 from mautrix.types import RoomID, MessageEventContent
+from mautrix.util.logging import TraceLogger
 
 from ... import puppet as pu
 from ...types import TelegramID
@@ -31,30 +33,19 @@ from .parser import ParsedMessage, parse_html
 if TYPE_CHECKING:
     from ...context import Context
 
-log: logging.Logger = logging.getLogger("mau.fmt.mx")
+log: TraceLogger = logging.getLogger("mau.fmt.mx")
 should_bridge_plaintext_highlights: bool = False
 
 command_regex: Pattern = re.compile(r"^!([A-Za-z0-9@]+)")
 not_command_regex: Pattern = re.compile(r"^\\(![A-Za-z0-9@]+)")
 plain_mention_regex: Optional[Pattern] = None
 
-
-def plain_mention_to_html(match: Match) -> str:
-    puppet = pu.Puppet.find_by_displayname(match.group(2))
-    if puppet:
-        return (f"{match.group(1)}"
-                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
-                f"{puppet.displayname}"
-                "</a>")
-    return "".join(match.groups())
-
-
 MAX_LENGTH = 4096
 CUTOFF_TEXT = " [message cut]"
 CUT_MAX_LENGTH = MAX_LENGTH - len(CUTOFF_TEXT)
 
 
-def cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
+def _cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
     if len(message) > MAX_LENGTH:
         message = message[0:CUT_MAX_LENGTH] + CUTOFF_TEXT
         new_entities = []
@@ -73,23 +64,6 @@ class FormatError(Exception):
     pass
 
 
-def matrix_to_telegram(html: str) -> ParsedMessage:
-    try:
-        html = command_regex.sub(r"<command>\1</command>", html)
-        html = html.replace("\t", " " * 4)
-        html = not_command_regex.sub(r"\1", html)
-        if should_bridge_plaintext_highlights:
-            html = plain_mention_regex.sub(plain_mention_to_html, html)
-
-        text, entities = parse_html(add_surrogate(html))
-        text = del_surrogate(text.strip())
-        text, entities = cut_long_message(text, entities)
-
-        return text, entities
-    except Exception as e:
-        raise FormatError(f"Failed to convert Matrix format: {html}") from e
-
-
 def matrix_reply_to_telegram(content: MessageEventContent, tg_space: TelegramID,
                              room_id: Optional[RoomID] = None) -> Optional[TelegramID]:
     event_id = content.get_reply_to()
@@ -103,19 +77,61 @@ def matrix_reply_to_telegram(content: MessageEventContent, tg_space: TelegramID,
     return None
 
 
-def matrix_text_to_telegram(text: str) -> ParsedMessage:
+async def matrix_to_telegram(client: TelegramClient, *, text: Optional[str] = None,
+                             html: Optional[str] = None) -> ParsedMessage:
+    if html is not None:
+        text, entities = _matrix_html_to_telegram(html)
+    elif text is not None:
+        text, entities = _matrix_text_to_telegram(text)
+    else:
+        raise ValueError("text or html must be provided to convert formatting")
+    await _fix_name_mentions(client, entities)
+    return text, entities
+
+
+def _matrix_html_to_telegram(html: str) -> ParsedMessage:
+    try:
+        html = command_regex.sub(r"<command>\1</command>", html)
+        html = html.replace("\t", " " * 4)
+        html = not_command_regex.sub(r"\1", html)
+        if should_bridge_plaintext_highlights:
+            html = plain_mention_regex.sub(_plain_mention_to_html, html)
+
+        text, entities = parse_html(add_surrogate(html))
+        text = del_surrogate(text.strip())
+        text, entities = _cut_long_message(text, entities)
+
+        return text, entities
+    except Exception as e:
+        raise FormatError(f"Failed to convert Matrix format: {html}") from e
+
+
+def _matrix_text_to_telegram(text: str) -> ParsedMessage:
     text = command_regex.sub(r"/\1", text)
     text = text.replace("\t", " " * 4)
     text = not_command_regex.sub(r"\1", text)
     if should_bridge_plaintext_highlights:
-        entities, pmr_replacer = plain_mention_to_text()
+        entities, pmr_replacer = _plain_mention_to_text()
         text = plain_mention_regex.sub(pmr_replacer, text)
     else:
         entities = []
     return text, entities
 
 
-def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match], str]]:
+async def _fix_name_mentions(client: TelegramClient, entities: List[TypeMessageEntity]) -> None:
+    for index in reversed(range(len(entities))):
+        entity = entities[index]
+        if isinstance(entity, (MessageEntityMentionName, InputMessageEntityMentionName)):
+            try:
+                user = await client.get_input_entity(entity.user_id)
+            except (ValueError, TypeError) as e:
+                log.trace(f"Dropping mention of {entity.user_id}: {e}")
+                del entities[index]
+            else:
+                entities[index] = InputMessageEntityMentionName(entity.offset, entity.length, user)
+
+
+def _plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match], str]]:
     entities = []
 
     def replacer(match: Match) -> str:
@@ -136,6 +152,16 @@ def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match],
     return entities, replacer
 
 
+def _plain_mention_to_html(match: Match) -> str:
+    puppet = pu.Puppet.find_by_displayname(match.group(2))
+    if puppet:
+        return (f"{match.group(1)}"
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                f"{puppet.displayname}"
+                "</a>")
+    return "".join(match.groups())
+
+
 def init_mx(context: "Context") -> None:
     global plain_mention_regex, should_bridge_plaintext_highlights
     config = context.config

mautrix_telegram/formatter/from_telegram.py

@@ -51,7 +51,7 @@ def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Optional[R
                  else source.tgid)
         msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
         if msg:
-            return RelatesTo(rel_type=RelationType.REFERENCE, event_id=msg.mxid)
+            return RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
     return None
 
 
@@ -126,7 +126,7 @@ async def _add_reply_header(source: 'AbstractUser', content: TextMessageEventCon
     if not msg:
         return
 
-    content.relates_to = RelatesTo(rel_type=RelationType.REFERENCE, event_id=msg.mxid)
+    content.relates_to = RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
 
     try:
         event: MessageEvent = await main_intent.get_event(msg.mx_room, msg.mxid)

mautrix_telegram/portal/base.py

@@ -523,7 +523,7 @@ def init(context: Context) -> None:
     global config
     BasePortal.az, config, BasePortal.loop, BasePortal.bot = context.core
     BasePortal.matrix = context.mx
-    BasePortal.bridge = context.bridge
+    MautrixBasePortal.bridge = context.bridge
     BasePortal.max_initial_member_sync = config["bridge.max_initial_member_sync"]
     BasePortal.sync_channel_members = config["bridge.sync_channel_members"]
     BasePortal.sync_matrix_state = config["bridge.sync_matrix_state"]

mautrix_telegram/portal/matrix.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, TYPE_CHECKING
+from typing import Awaitable, Dict, Optional, Union, Any, TYPE_CHECKING
 from html import escape as escape_html
 from string import Template
 from abc import ABC
@@ -28,11 +28,11 @@ from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError,
                              PhotoInvalidDimensionsError, PhotoSaveFileInvalidError,
                              RPCError)
 from telethon.tl.patched import Message, MessageService
-from telethon.tl.types import (
+from telethon.tl.types import (DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
-    DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
+                               InputChatUploadedPhoto, MessageActionChatEditPhoto, MessageMediaGeo,
-    InputChatUploadedPhoto, MessageActionChatEditPhoto, MessageMediaGeo,
+                               SendMessageCancelAction, SendMessageTypingAction, TypeInputPeer,
-    SendMessageCancelAction, SendMessageTypingAction, TypeInputPeer, TypeMessageEntity,
+                               UpdateNewMessage, InputMediaUploadedDocument,
-    UpdateNewMessage, InputMediaUploadedDocument, InputMediaUploadedPhoto)
+                               InputMediaUploadedPhoto)
 
 from mautrix.types import (EventID, RoomID, UserID, ContentURI, MessageType, MessageEventContent,
                            TextMessageEventContent, MediaMessageEventContent, Format,
@@ -87,7 +87,7 @@ class PortalMatrix(BasePortal, ABC):
             message = await self._get_state_change_message(event, user, **kwargs)
             if not message:
                 return
-            message, entities = formatter.matrix_to_telegram(message)
+            message, entities = await formatter.matrix_to_telegram(self.bot.client, html=message)
             response = await self.bot.client.send_message(self.peer, message,
                                                           formatting_entities=entities)
             space = self.tgid if self.peer_type == "channel" else self.bot.tgid
@@ -214,27 +214,11 @@ class PortalMatrix(BasePortal, ABC):
         elif content.msgtype == MessageType.EMOTE:
             await self._apply_emote_format(sender, content)
 
-    @staticmethod
-    def _matrix_event_to_entities(event: Union[str, MessageEventContent]
-                                  ) -> Tuple[str, Optional[List[TypeMessageEntity]]]:
-        try:
-            if isinstance(event, str):
-                message, entities = formatter.matrix_to_telegram(event)
-            elif isinstance(event, TextMessageEventContent) and event.format == Format.HTML:
-                message, entities = formatter.matrix_to_telegram(event.formatted_body)
-            else:
-                message, entities = formatter.matrix_text_to_telegram(event.body)
-        except KeyError:
-            message, entities = None, None
-        return message, entities
-
     async def _handle_matrix_text(self, sender_id: TelegramID, event_id: EventID,
                                   space: TelegramID, client: 'MautrixTelegramClient',
                                   content: TextMessageEventContent, reply_to: TelegramID) -> None:
-        if content.formatted_body and content.format == Format.HTML:
+        message, entities = await formatter.matrix_to_telegram(client, text=content.body,
-            message, entities = formatter.matrix_to_telegram(content.formatted_body)
+                                                               html=content.formatted(Format.HTML))
-        else:
-            message, entities = formatter.matrix_text_to_telegram(content.body)
         async with self.send_lock(sender_id):
             lp = self.get_config("telegram_link_preview")
             if content.get_edit():
@@ -301,25 +285,21 @@ class PortalMatrix(BasePortal, ABC):
             media = InputMediaUploadedDocument(file=file_handle, attributes=attributes,
                                                mime_type=mime or "application/octet-stream")
 
-        if caption:
+        capt, entities = (await formatter.matrix_to_telegram(client, text=caption.body,
-            if caption.formatted_body and caption.format == Format.HTML:
+                                                             html=caption.formatted(Format.HTML))
-                caption, entities = formatter.matrix_to_telegram(caption.formatted_body)
+                          if caption else (None, None))
-            else:
-                caption, entities = formatter.matrix_text_to_telegram(caption.body)
-        else:
-            caption, entities = None, None
 
         async with self.send_lock(sender_id):
-            if await self._matrix_document_edit(client, content, space, caption, media, event_id):
+            if await self._matrix_document_edit(client, content, space, capt, media, event_id):
                 return
             try:
                 response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                                   caption=caption, entities=entities)
+                                                   caption=capt, entities=entities)
             except (PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, PhotoExtInvalidError):
                 media = InputMediaUploadedDocument(file=media.file, mime_type=mime,
                                                    attributes=attributes)
                 response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                                   caption=caption, entities=entities)
+                                                   caption=capt, entities=entities)
             self._add_telegram_message_to_db(event_id, space, 0, response)
         await self._send_delivery_receipt(event_id)
 
@@ -346,7 +326,7 @@ class PortalMatrix(BasePortal, ABC):
         except (KeyError, ValueError):
             self.log.exception("Failed to parse location")
             return None
-        caption, entities = formatter.matrix_text_to_telegram(content.body)
+        caption, entities = await formatter.matrix_to_telegram(client, text=content.body)
         media = MessageMediaGeo(geo=GeoPoint(lat, long, access_hash=0))
 
         async with self.send_lock(sender_id):

mautrix_telegram/util/file_transfer.py

@@ -30,7 +30,7 @@ from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, Locatio
                              SecurityError, FileIdInvalidError)
 
 from mautrix.appservice import IntentAPI
-from mautrix.types import EncryptedFile
+from mautrix.util.network_retry import call_with_net_retry
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
@@ -145,7 +145,8 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if encrypt:
         file, decryption_info = encrypt_attachment(file)
         upload_mime_type = "application/octet-stream"
-    content_uri = await intent.upload_media(file, upload_mime_type)
+    content_uri = await call_with_net_retry(intent.upload_media, file, upload_mime_type,
+                                            _action="upload media")
     if decryption_info:
         decryption_info.url = content_uri
 
@@ -246,7 +247,8 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
         if encrypt and encrypt_attachment:
             file, decryption_info = encrypt_attachment(file)
             upload_mime_type = "application/octet-stream"
-        content_uri = await intent.upload_media(file, upload_mime_type)
+        content_uri = await call_with_net_retry(intent.upload_media, file, upload_mime_type,
+                                                _action="upload media")
         if decryption_info:
             decryption_info.url = content_uri
 

requirements.txt

@@ -5,6 +5,6 @@ python-magic>=0.4,<0.5
 commonmark>=0.8,<0.10
 aiohttp>=3,<4
 yarl>=1,<2
-mautrix==0.8.0rc5
+mautrix>=0.8.3,<0.9
 telethon>=1.17,<1.18
 telethon-session-sqlalchemy>=0.2.14,<0.3