Merge pull request #366 from Eramde/rlottie

TGS animation support

Dockerfile

@@ -1,9 +1,31 @@
+FROM docker.io/alpine:3.10 AS lottieconverter
+
+WORKDIR /build
+
+RUN apk add --no-cache git build-base cmake \
+  && git clone https://github.com/Samsung/rlottie.git \
+  && cd rlottie \
+  && mkdir build \
+  && cd build \
+  && cmake .. \
+  && make -j2 \
+  && make install \
+  && cd ../..
+
+RUN apk add --no-cache libpng libpng-dev zlib zlib-dev \
+  && git clone https://github.com/Eramde/LottieConverter.git \
+  && cd LottieConverter \
+  && make
+
 FROM docker.io/alpine:3.10
 
 ENV UID=1337 \
     GID=1337 \
     FFMPEG_BINARY=/usr/bin/ffmpeg
 
+COPY --from=lottieconverter /usr/lib/librlottie* /usr/lib/
+COPY --from=lottieconverter /build/LottieConverter/dist/Debug/GNU-Linux/lottieconverter /usr/local/bin/lottieconverter
+
 COPY . /opt/mautrix-telegram
 WORKDIR /opt/mautrix-telegram
 RUN apk add --no-cache --virtual .build-deps \
@@ -41,6 +63,8 @@ RUN apk add --no-cache --virtual .build-deps \
       ca-certificates \
       su-exec \
       netcat-openbsd \
+      # lottieconverter
+      zlib libpng \
  && pip3 install .[speedups,hq_thumbnails,metrics] \
  && apk del .build-deps
 

example-config.yaml

@@ -170,6 +170,21 @@ bridge:
     # Whether or not created rooms should have federation enabled.
     # If false, created portal rooms will never be federated.
     federate_rooms: true
+    # Settings for converting animated stickers.
+    animated_sticker:
+        # Format to which animated stickers should be converted.
+        # disable - No conversion, send as-is (gzipped lottie)
+        # png - converts to non-animated png (fastest),
+        # gif - converts to animated gif, but loses transparency
+        # webm - converts to webm video, requires ffmpeg executable with vp9 codec and webm container support
+        target: gif
+        # Arguments for converter. All converters take width and height.
+        # GIF converter takes background as a hex color.
+        args:
+            width: 256
+            height: 256
+            background: "020202"  # only for gif
+            fps: 30               # only for webm
 
     # Whether to bridge Telegram bot messages as m.notices or m.texts.
     bot_messages_as_notices: true

mautrix_telegram/config.py

@@ -103,6 +103,8 @@ class Config(BaseBridgeConfig):
         copy("bridge.max_document_size")
         copy("bridge.parallel_file_transfer")
         copy("bridge.federate_rooms")
+        copy("bridge.animated_sticker.target")
+        copy("bridge.animated_sticker.args")
 
         copy("bridge.bot_messages_as_notices")
         if isinstance(self["bridge.bridge_notices"], bool):

mautrix_telegram/portal/telegram.py

@@ -183,8 +183,9 @@ class PortalTelegram(BasePortal, ABC):
             thumb_size = None
         parallel_id = source.tgid if config["bridge.parallel_file_transfer"] else None
         file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc,
-                                                  is_sticker=attrs.is_sticker, filename=attrs.name,
+                                                  is_sticker=attrs.is_sticker,
-                                                  parallel_id=parallel_id)
+                                                  tgs_convert=config["bridge.animated_sticker"],
+                                                  filename=attrs.name, parallel_id=parallel_id)
         if not file:
             return None
 
@@ -192,7 +193,10 @@ class PortalTelegram(BasePortal, ABC):
 
         await intent.set_typing(self.mxid, is_typing=False)
 
-        event_type = EventType.STICKER if attrs.is_sticker else EventType.ROOM_MESSAGE
+        event_type = EventType.ROOM_MESSAGE
+        # Riot only supports images as stickers, so send animated webm stickers as m.video
+        if attrs.is_sticker and file.mime_type.startswith("image/"):
+            event_type = EventType.STICKER
         content = MediaMessageEventContent(
             body=name or "unnamed file", info=info, url=file.mxc, relates_to=relates_to,
             external_url=self._get_external_url(evt),

mautrix_telegram/util/file_transfer.py

@@ -30,6 +30,7 @@ from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, Locatio
 
 from mautrix.appservice import IntentAPI
 
+
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
 from ..util import sane_mimetypes
@@ -49,6 +50,8 @@ try:
 except ImportError:
     VideoFileClip = random = string = os = mimetypes = None
 
+from .tgs_converter import convert_tgs_to
+
 log: logging.Logger = logging.getLogger("mau.util")
 
 TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
@@ -159,8 +162,9 @@ TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
 
 async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
                                   location: TypeLocation, thumbnail: TypeThumbnail = None,
-                                  is_sticker: bool = False, filename: Optional[str] = None,
+                                  is_sticker: bool = False, tgs_convert: Optional[dict] = None,
-                                  parallel_id: Optional[int] = None) -> Optional[DBTelegramFile]:
+                                  filename: Optional[str] = None, parallel_id: Optional[int] = None
+                                  ) -> Optional[DBTelegramFile]:
     location_id = _location_to_id(location)
     if not location_id:
         return None
@@ -176,21 +180,21 @@ async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentA
         transfer_locks[location_id] = lock
     async with lock:
         return await _unlocked_transfer_file_to_matrix(client, intent, location_id, location,
-                                                       thumbnail, is_sticker, filename,
+                                                       thumbnail, is_sticker, tgs_convert,
-                                                       parallel_id)
+                                                       filename, parallel_id)
 
 
 async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
                                             loc_id: str, location: TypeLocation,
                                             thumbnail: TypeThumbnail, is_sticker: bool,
-                                            filename: Optional[str],
+                                            tgs_convert: Optional[dict], filename: Optional[str],
-                                            parallel_id: Optional[int] = None
+                                            parallel_id: Optional[int]
                                             ) -> Optional[DBTelegramFile]:
     db_file = DBTelegramFile.get(loc_id)
     if db_file:
         return db_file
 
-    if parallel_id and isinstance(location, Document):
+    if parallel_id and isinstance(location, Document) and (not is_sticker or not tgs_convert):
         db_file = await parallel_transfer_to_matrix(client, intent, loc_id, location, filename,
                                                     parallel_id)
         mime_type = location.mime_type
@@ -208,6 +212,15 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
         mime_type = magic.from_buffer(file, mime=True)
 
         image_converted = False
+        # A weird bug in alpine/magic makes it return application/octet-stream for gzips...
+        if is_sticker and tgs_convert and (mime_type == "application/gzip" or (
+               mime_type == "application/octet-stream"
+               and magic.from_buffer(file).startswith("gzip"))):
+            mime_type, file, width, height = await convert_tgs_to(
+                file, tgs_convert["target"], **tgs_convert["args"])
+            thumbnail = None
+            image_converted = mime_type != "application/gzip"
+
         if mime_type == "image/webp":
             new_mime_type, file, width, height = convert_image(
                 file, source_mime="image/webp", target_type="png",

mautrix_telegram/util/tgs_converter.py

@@ -0,0 +1,112 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Telegram lottie sticker converter
+# Copyright (C) 2019 Randall Eramde Lawrence
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Callable, Awaitable, Optional, Tuple, Any
+import asyncio.subprocess
+import logging
+import shutil
+import os.path
+import tempfile
+
+log: logging.Logger = logging.getLogger("mau.util.tgs")
+converters: Dict[str, Callable[[bytes, int, int, Any], Awaitable[Tuple[str, bytes]]]] = {}
+
+
+def abswhich(program: Optional[str]) -> Optional[str]:
+    path = shutil.which(program)
+    return os.path.abspath(path) if path else None
+
+
+lottieconverter = abswhich("lottieconverter")
+ffmpeg = abswhich("ffmpeg")
+
+if lottieconverter:
+    async def tgs_to_png(file: bytes, width: int, height: int, **_: Any) -> Tuple[str, bytes]:
+        frame = 1
+        proc = await asyncio.create_subprocess_exec(lottieconverter, "-", "-", "png",
+                                                    f"{width}x{height}", str(frame),
+                                                    stdout=asyncio.subprocess.PIPE,
+                                                    stdin=asyncio.subprocess.PIPE)
+        stdout, stderr = await proc.communicate(file)
+        if proc.returncode == 0:
+            return "image/png", stdout
+        else:
+            log.error("lottieconverter error: " + stderr.decode("utf-8") if stderr is not None
+                      else "unknown")
+            return "application/gzip", file
+
+
+    async def tgs_to_gif(file: bytes, width: int, height: int, background: str = "202020",
+                         **_: Any) -> Tuple[str, bytes]:
+        proc = await asyncio.create_subprocess_exec(lottieconverter, "-", "-", "gif",
+                                                    f"{width}x{height}", f"0x{background}",
+                                                    stdout=asyncio.subprocess.PIPE,
+                                                    stdin=asyncio.subprocess.PIPE)
+        stdout, stderr = await proc.communicate(file)
+        if proc.returncode == 0:
+            return "image/gif", stdout
+        else:
+            log.error("lottieconverter error: " + stderr.decode("utf-8") if stderr is not None
+                      else "unknown")
+            return "application/gzip", file
+
+
+    converters["png"] = tgs_to_png
+    converters["gif"] = tgs_to_gif
+
+if lottieconverter and ffmpeg:
+    async def tgs_to_webm(file: bytes, width: int, height: int, fps: int = 30,
+                          **_: Any) -> Tuple[str, bytes]:
+        with tempfile.TemporaryDirectory(prefix="tgs_") as tmpdir:
+            file_template = tmpdir + "/out_"
+            proc = await asyncio.create_subprocess_exec(lottieconverter, "-", file_template,
+                                                        "pngs", f"{width}x{height}", str(fps),
+                                                        stdout=asyncio.subprocess.PIPE,
+                                                        stdin=asyncio.subprocess.PIPE)
+            _, stderr = await proc.communicate(file)
+            if proc.returncode == 0:
+                proc = await asyncio.create_subprocess_exec(ffmpeg, "-hide_banner", "-loglevel",
+                                                            "error", "-framerate", str(fps),
+                                                            "-pattern_type", "glob", "-i",
+                                                            file_template + "*.png",
+                                                            "-c:v", "libvpx-vp9", "-pix_fmt",
+                                                            "yuva420p", "-f", "webm", "-",
+                                                            stdout=asyncio.subprocess.PIPE,
+                                                            stdin=asyncio.subprocess.PIPE)
+                stdout, stderr = await proc.communicate()
+                if proc.returncode == 0:
+                    return "video/webm", stdout
+                else:
+                    log.error("ffmpeg error: " + stderr.decode("utf-8") if stderr is not None
+                              else "unknown")
+            else:
+                log.error("lottieconverter error: " + stderr.decode("utf-8") if stderr is not None
+                          else "unknown")
+        return "application/gzip", file
+
+
+    converters["webm"] = tgs_to_webm
+
+
+async def convert_tgs_to(file: bytes, convert_to: str, width: int, height: int, **kwargs: Any
+                         ) -> Tuple[str, bytes, Optional[int], Optional[int]]:
+    if convert_to in converters:
+        converter = converters[convert_to]
+        mime, out = await converter(file, width, height, **kwargs)
+        return mime, out, width, height
+    elif convert_to != "disable":
+        log.warning(f"Unable to convert animated sticker, type {convert_to} not supported")
+    return "application/gzip", file, None, None