stickers: support receiving and converting

Signed-off-by: Sumner Evans <sumner.evans@automattic.com>

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/gotd/td v0.102.0
 	github.com/rs/zerolog v1.33.0
 	github.com/stretchr/testify v1.9.0
-	go.mau.fi/util v0.5.1-0.20240708233020-c2f9af6fecf8
+	go.mau.fi/util v0.5.1-0.20240710154926-931b33d6d530
 	go.mau.fi/zerozap v0.1.1
 	go.uber.org/zap v1.27.0
 	maunium.net/go/mautrix v0.19.0-beta.1.0.20240706124659-b4057a26c3ed

go.sum

@@ -67,8 +67,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
 github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
-go.mau.fi/util v0.5.1-0.20240708233020-c2f9af6fecf8 h1:7ntkhSR0G/dIwAPjcoOoJz+bPne0gA4PypEn8euMMx0=
+go.mau.fi/util v0.5.1-0.20240710154926-931b33d6d530 h1:ZWMrLC+Fn2AmKL8HM04YY0zyMDMOagQZVukpxp0rmic=
-go.mau.fi/util v0.5.1-0.20240708233020-c2f9af6fecf8/go.mod h1:DsJzUrJAG53lCZnnYvq9/mOyLuPScWwYhvETiTrpdP4=
+go.mau.fi/util v0.5.1-0.20240710154926-931b33d6d530/go.mod h1:DsJzUrJAG53lCZnnYvq9/mOyLuPScWwYhvETiTrpdP4=
 go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto=
 go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70=
 go.mau.fi/zerozap v0.1.1 h1:mxE/dW4wtkqBYOXOEEzXldk5qKB+ahsZXjoTGnvEhZQ=

pkg/connector/config.go

@@ -3,6 +3,7 @@ package connector
 import (
 	_ "embed"
 	"fmt"
+	"slices"
 
 	up "go.mau.fi/util/configupgrade"
 	"maunium.net/go/mautrix/bridgev2"
@@ -25,11 +26,11 @@ var ExampleConfig string
 func upgradeConfig(helper up.Helper) {
 	helper.Copy(up.Int, "app_id")
 	helper.Copy(up.Str, "app_hash")
-	helper.Copy(up.Str, "animated_sticker.target")
+	helper.Copy(up.Str, "animated_sticker", "target")
-	helper.Copy(up.Bool, "animated_sticker.convert_from_webm")
+	helper.Copy(up.Bool, "animated_sticker", "convert_from_webm")
-	helper.Copy(up.Int, "animated_sticker.args.width")
+	helper.Copy(up.Int, "animated_sticker", "args", "width")
-	helper.Copy(up.Int, "animated_sticker.args.height")
+	helper.Copy(up.Int, "animated_sticker", "args", "height")
-	helper.Copy(up.Int, "animated_sticker.args.fps")
+	helper.Copy(up.Int, "animated_sticker", "args", "fps")
 }
 
 func (tg *TelegramConnector) GetConfig() (example string, data any, upgrader up.Upgrader) {
@@ -43,5 +44,8 @@ func (tg *TelegramConnector) ValidateConfig() error {
 	if tg.Config.AppHash == "" {
 		return fmt.Errorf("app_hash is required")
 	}
+	if !slices.Contains([]string{"disable", "gif", "png", "webp", "webm"}, tg.Config.AnimatedSticker.Target) {
+		return fmt.Errorf("unsupported animated sticker target: %s", tg.Config.AnimatedSticker.Target)
+	}
 	return nil
 }

pkg/connector/directdownload.go

@@ -86,6 +86,16 @@ func (tc *TelegramConnector) Download(ctx context.Context, mediaID networkid.Med
 	case *tg.MessageMediaPhoto:
 		readyTransferer = transferer.WithPhoto(msgMedia.Photo)
 	case *tg.MessageMediaDocument:
+		document, ok := msgMedia.Document.(*tg.Document)
+		if !ok {
+			return nil, fmt.Errorf("unknown document type %T", msgMedia.Document)
+		}
+		for _, attr := range document.GetAttributes() {
+			if attr.TypeID() == tg.DocumentAttributeStickerTypeID {
+				transferer = transferer.WithStickerConfig(tc.Config.AnimatedSticker)
+			}
+		}
+
 		readyTransferer = transferer.WithDocument(msgMedia.Document, info.Thumbnail)
 	default:
 		return nil, fmt.Errorf("unhandled media type %T", msgMedia)
@@ -93,6 +103,7 @@ func (tc *TelegramConnector) Download(ctx context.Context, mediaID networkid.Med
 
 	data, fileInfo, err := readyTransferer.Download(ctx)
 	if err != nil {
+		log.Err(err).Msg("failed to download media")
 		return nil, err
 	}
 

pkg/connector/media/sticker.go

@@ -0,0 +1,85 @@
+package media
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+
+	"github.com/rs/zerolog"
+	"go.mau.fi/util/ffmpeg"
+	"go.mau.fi/util/lottie"
+)
+
+type AnimatedStickerConfig struct {
+	Target          string `yaml:"target"`
+	ConvertFromWebm bool   `yaml:"convert_from_webm"`
+	Args            struct {
+		Width  int `yaml:"width"`
+		Height int `yaml:"height"`
+		FPS    int `yaml:"fps"`
+	} `yaml:"args"`
+}
+
+type ConvertedSticker struct {
+	Data              []byte
+	MIMEType          string
+	ThumbnailData     []byte
+	ThumbnailMIMEType string
+	Width             int
+	Height            int
+}
+
+func (c AnimatedStickerConfig) convert(ctx context.Context, data []byte) ConvertedSticker {
+	if c.Target == "disable" {
+		return ConvertedSticker{Data: data, MIMEType: "application/x-tgsticker"}
+	}
+
+	log := zerolog.Ctx(ctx).With().Str("animated_sticker_target", c.Target).Logger()
+
+	if !lottie.Supported() {
+		log.Warn().Msg("lottie not supported, cannot convert animated stickers")
+		return ConvertedSticker{Data: data, MIMEType: "application/x-tgsticker"}
+	} else if (c.Target == "webp" || c.Target == "webm") && !ffmpeg.Supported() {
+		log.Warn().Msg("ffmpeg not supported, cannot convert animated stickers")
+		return ConvertedSticker{Data: data, MIMEType: "application/x-tgsticker"}
+	}
+
+	input := bytes.NewBuffer(data)
+	outputWriter := new(bytes.Buffer)
+	var thumbnailData []byte
+	var mimeType, thumbnailMIMEType string
+
+	var err error
+	switch c.Target {
+	case "png":
+		mimeType = "image/png"
+		err = lottie.Convert(ctx, input, "", outputWriter, c.Target, c.Args.Width, c.Args.Height, "1")
+	case "gif":
+		mimeType = "image/gif"
+		err = lottie.Convert(ctx, input, "", outputWriter, c.Target, c.Args.Width, c.Args.Height, strconv.Itoa(c.Args.FPS))
+	case "webm", "webp":
+		thumbnailMIMEType = "image/png"
+		outputWriter, mimeType, thumbnailData, err = lottie.FfmpegConvert(ctx, input, c.Target, c.Args.Width, c.Args.Height, c.Args.FPS)
+	default:
+		err = fmt.Errorf("unsupported target format %s", c.Target)
+	}
+	if err != nil {
+		log.Err(err).
+			Str("target", c.Target).
+			Msg("failed to convert animated sticker to target format")
+
+		// Fallback to original data
+		return ConvertedSticker{Data: data, MIMEType: "application/x-tgsticker"}
+	}
+
+	return ConvertedSticker{
+		Data:              outputWriter.Bytes(),
+		MIMEType:          mimeType,
+		ThumbnailData:     thumbnailData,
+		ThumbnailMIMEType: thumbnailMIMEType,
+		Width:             c.Args.Width,
+		Height:            c.Args.Height,
+	}
+
+}

pkg/connector/media/transfer.go

@@ -13,7 +13,7 @@ import (
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 
-	"go.mau.fi/util/lottie"
+	"go.mau.fi/util/gnuzip"
 
 	"go.mau.fi/mautrix-telegram/pkg/connector/ids"
 	"go.mau.fi/mautrix-telegram/pkg/connector/store"
@@ -80,24 +80,6 @@ func getLocationID(loc any) (locID store.TelegramFileLocationID) {
 	return store.TelegramFileLocationID(id)
 }
 
-type AnimatedStickerConfig struct {
-	Target          string `yaml:"target"`
-	ConvertFromWebm bool   `yaml:"convert_from_webm"`
-	Args            struct {
-		Width  int `yaml:"width"`
-		Height int `yaml:"height"`
-		FPS    int `yaml:"fps"`
-	} `yaml:"args"`
-}
-
-func (c AnimatedStickerConfig) TGSConvert() bool {
-	return c.Target == "gif" || c.Target == "png"
-}
-
-func (c AnimatedStickerConfig) WebmConvert() bool {
-	return c.ConvertFromWebm && c.Target != "webm"
-}
-
 // Transferer is a utility for downloading media from Telegram and uploading it
 // to Matrix.
 // TODO better name?
@@ -134,14 +116,19 @@ func (t *Transferer) WithFilename(filename string) *Transferer {
 	return t
 }
 
-func (t *Transferer) WithMIMEType(mimeType string) *Transferer {
-	t.fileInfo.MimeType = mimeType
-	return t
-}
-
 // WithStickerConfig sets the animated sticker config for the [Transferer].
 func (t *Transferer) WithStickerConfig(cfg AnimatedStickerConfig) *Transferer {
 	t.animatedStickerConfig = &cfg
+	switch cfg.Target {
+	case "png":
+		t.fileInfo.MimeType = "image/png"
+	case "gif":
+		t.fileInfo.MimeType = "image/gif"
+	case "webp":
+		t.fileInfo.MimeType = "image/webp"
+	case "webm":
+		t.fileInfo.MimeType = "video/webm"
+	}
 	return t
 }
 
@@ -158,6 +145,11 @@ func (t *Transferer) WithVideo(attr *tg.DocumentAttributeVideo) *Transferer {
 	return t
 }
 
+func (t *Transferer) WithImageSize(attr *tg.DocumentAttributeImageSize) *Transferer {
+	t.fileInfo.Width, t.fileInfo.Height = attr.W, attr.H
+	return t
+}
+
 // WithDocument transforms a [Transferer] to a [ReadyTransferer] by setting the
 // given document as the location that will be downloaded by the
 // [ReadyTransferer].
@@ -173,7 +165,9 @@ func (t *Transferer) WithDocument(doc tg.DocumentClass, thumbnail bool) *ReadyTr
 		documentFileLocation.ThumbSize = largestThumbnail.GetType()
 	} else {
 		t.fileInfo.Size = int(document.Size)
-		t.fileInfo.MimeType = document.GetMimeType()
+		if t.fileInfo.MimeType == "" {
+			t.fileInfo.MimeType = document.GetMimeType()
+		}
 	}
 	return &ReadyTransferer{t, &documentFileLocation}
 }
@@ -225,6 +219,7 @@ func (t *ReadyTransferer) Transfer(ctx context.Context, store *store.Container,
 		Str("component", "media_transfer").
 		Str("location_id", string(locationID)).
 		Logger()
+	ctx = log.WithContext(ctx)
 
 	if file, err := store.TelegramFile.GetByLocationID(ctx, locationID); err != nil {
 		return "", nil, nil, fmt.Errorf("failed to search for Telegram file by location ID: %w", err)
@@ -238,22 +233,26 @@ func (t *ReadyTransferer) Transfer(ctx context.Context, store *store.Container,
 		return "", nil, nil, fmt.Errorf("downloading file failed: %w", err)
 	}
 
-	if t.inner.animatedStickerConfig != nil {
+	if t.inner.animatedStickerConfig != nil && t.inner.fileInfo.MimeType == "application/x-tgsticker" {
-		if lottie.Supported() && t.inner.animatedStickerConfig.TGSConvert() && t.inner.fileInfo.MimeType == "application/x-tgsticker" {
+		converted := t.inner.animatedStickerConfig.convert(ctx, data)
-			newData, err := lottie.ConvertBytes(ctx, data,
+		data = converted.Data
-				t.inner.animatedStickerConfig.Target,
+		t.inner.fileInfo.MimeType = converted.MIMEType
-				t.inner.animatedStickerConfig.Args.Width,
+		t.inner.fileInfo.Width = converted.Width
-				t.inner.animatedStickerConfig.Args.Height,
+		t.inner.fileInfo.Height = converted.Height
-				fmt.Sprintf("%d", t.inner.animatedStickerConfig.Args.FPS))
+		t.inner.fileInfo.Size = len(data)
+
+		if len(converted.ThumbnailData) > 0 {
+			thumbnailMXC, thumbnailFileInfo, err := intent.UploadMedia(ctx, t.inner.roomID, converted.ThumbnailData, t.inner.filename, converted.ThumbnailMIMEType)
 			if err != nil {
-				log.Err(err).Msg("failed to convert animated sticker")
+				log.Err(err).Msg("failed to upload animated sticker thumbnail to Matrix")
 			} else {
-				data = newData
+				t.inner = t.inner.WithThumbnail(thumbnailMXC, thumbnailFileInfo, &event.FileInfo{
-				t.inner.fileInfo.Size = len(data)
+					MimeType: converted.ThumbnailMIMEType,
-				t.inner.fileInfo.MimeType = fmt.Sprintf("image/%s", t.inner.animatedStickerConfig.Target)
+					Width:    converted.Width,
+					Height:   converted.Height,
+					Size:     len(converted.ThumbnailData),
+				})
 			}
-			// TODO support ffmpeg conversion
-			// } else if ffmpeg.Supported() && t.Config.WebmConvert() && mimeType == "video/webm" {
 		}
 	}
 
@@ -278,7 +277,7 @@ func (t *ReadyTransferer) Transfer(ctx context.Context, store *store.Container,
 }
 
 // Download downloads the media from Telegram.
-func (t *ReadyTransferer) Download(ctx context.Context) (data []byte, fileInfo *event.FileInfo, err error) {
+func (t *ReadyTransferer) Download(ctx context.Context) ([]byte, *event.FileInfo, error) {
 	// TODO convert entire function to streaming? Maybe at least stream to file?
 	var buf bytes.Buffer
 	storageFileTypeClass, err := downloader.NewDownloader().Download(t.inner.client, t.loc).Stream(ctx, &buf)
@@ -307,7 +306,22 @@ func (t *ReadyTransferer) Download(ctx context.Context) (data []byte, fileInfo *
 			t.inner.fileInfo.MimeType = http.DetectContentType(buf.Bytes())
 		}
 	}
-	t.inner.fileInfo.Size = len(data)
+	t.inner.fileInfo.Size = buf.Len()
+
+	if t.inner.animatedStickerConfig != nil {
+		detected := http.DetectContentType(buf.Bytes())
+		if detected == "application/x-tgsticker" || detected == "application/x-gzip" {
+			if unzipped, err := gnuzip.MaybeGUnzip(buf.Bytes()); err != nil {
+				zerolog.Ctx(ctx).Err(err).Msg("failed to unzip animated sticker")
+			} else {
+				converted := t.inner.animatedStickerConfig.convert(ctx, unzipped)
+				t.inner.fileInfo.MimeType = converted.MIMEType
+				t.inner.fileInfo.Size = len(converted.Data)
+				return converted.Data, &t.inner.fileInfo, nil
+			}
+		}
+	}
+
 	return buf.Bytes(), &t.inner.fileInfo, nil
 }
 

pkg/connector/msgconv/tomatrix.go

@@ -154,8 +154,11 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por
 		Str("portal_id", string(portal.ID)).
 		Int("msg_id", msgID).
 		Logger()
+	eventType := event.EventMessage
 	var partID networkid.PartID
 	var content event.MessageEventContent
+	var isSticker, isAnimatedSticker bool
+	extra := map[string]any{}
 
 	transferer := media.NewTransferer(mc.client.API()).WithRoomID(portal.MXID)
 	var mediaTransferer *media.ReadyTransferer
@@ -176,7 +179,81 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por
 		partID = networkid.PartID("document")
 		content.MsgType = event.MsgFile
 
-		if _, ok := document.GetThumbs(); ok {
+		extraInfo := map[string]any{}
+		for _, attr := range document.GetAttributes() {
+			switch a := attr.(type) {
+			case *tg.DocumentAttributeFilename:
+				if content.Body == "" {
+					content.Body = a.GetFileName()
+				} else {
+					content.FileName = a.GetFileName()
+				}
+			case *tg.DocumentAttributeVideo:
+				content.MsgType = event.MsgVideo
+				transferer = transferer.WithVideo(a)
+			case *tg.DocumentAttributeAudio:
+				content.MsgType = event.MsgAudio
+				content.MSC1767Audio = &event.MSC1767Audio{
+					Duration: a.Duration * 1000,
+				}
+				if wf, ok := a.GetWaveform(); ok {
+					for _, v := range waveform.Decode(wf) {
+						content.MSC1767Audio.Waveform = append(content.MSC1767Audio.Waveform, int(v)<<5)
+					}
+				}
+				if a.Voice {
+					content.MSC3245Voice = &event.MSC3245Voice{}
+				}
+			case *tg.DocumentAttributeImageSize:
+				transferer = transferer.WithImageSize(a)
+			case *tg.DocumentAttributeSticker:
+				isSticker = true
+				if mc.animatedStickerConfig.Target == "webm" {
+					content.MsgType = event.MsgVideo
+				} else {
+					eventType = event.EventSticker
+					content.MsgType = ""
+				}
+				if content.Body == "" {
+					content.Body = a.Alt
+				} else {
+					content.FileName = content.Body
+					content.Body = a.Alt
+				}
+				stickerInfo := map[string]any{"alt": a.Alt, "id": document.ID}
+
+				if setID, ok := a.Stickerset.(*tg.InputStickerSetID); ok {
+					stickerInfo["pack"] = map[string]any{
+						"id":          setID.ID,
+						"access_hash": setID.AccessHash,
+					}
+				} else if shortName, ok := a.Stickerset.(*tg.InputStickerSetShortName); ok {
+					stickerInfo["pack"] = map[string]any{
+						"short_name": shortName.ShortName,
+					}
+				}
+				extraInfo["fi.mau.telegram.sticker"] = stickerInfo
+				extraInfo["fi.mau.gif"] = true
+				extraInfo["fi.mau.loop"] = true
+				extraInfo["fi.mau.autoplay"] = true
+				extraInfo["fi.mau.hide_controls"] = true
+				extraInfo["fi.mau.no_audio"] = true
+				transferer = transferer.WithStickerConfig(mc.animatedStickerConfig)
+			case *tg.DocumentAttributeAnimated:
+				isAnimatedSticker = true
+			}
+		}
+
+		if isAnimatedSticker || (isSticker && mc.animatedStickerConfig.Target == "webm") {
+			if isAnimatedSticker {
+				extraInfo["fi.mau.telegram.gif"] = true
+			} else {
+				extraInfo["fi.mau.telegram.animated_sticker"] = true
+			}
+		}
+		extra["info"] = extraInfo
+
+		if _, ok := document.GetThumbs(); ok && eventType != event.EventSticker {
 			var thumbnailURL id.ContentURIString
 			var thumbnailFile *event.EncryptedFileInfo
 			var thumbnailInfo *event.FileInfo
@@ -201,29 +278,6 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por
 			transferer = transferer.WithThumbnail(thumbnailURL, thumbnailFile, thumbnailInfo)
 		}
 
-		for _, attr := range document.GetAttributes() {
-			switch a := attr.(type) {
-			case *tg.DocumentAttributeFilename:
-				content.Body = a.GetFileName()
-			case *tg.DocumentAttributeVideo:
-				content.MsgType = event.MsgVideo
-				transferer = transferer.WithVideo(a)
-			case *tg.DocumentAttributeAudio:
-				content.MsgType = event.MsgAudio
-				content.MSC1767Audio = &event.MSC1767Audio{
-					Duration: a.Duration * 1000,
-				}
-				if wf, ok := a.GetWaveform(); ok {
-					for _, v := range waveform.Decode(wf) {
-						content.MSC1767Audio.Waveform = append(content.MSC1767Audio.Waveform, int(v)<<5)
-					}
-				}
-				if a.Voice {
-					content.MSC3245Voice = &event.MSC3245Voice{}
-				}
-			}
-		}
-
 		mediaTransferer = transferer.
 			WithFilename(content.Body).
 			WithDocument(msgMedia.Document, false)
@@ -232,7 +286,7 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por
 	}
 
 	var err error
-	if mc.useDirectMedia {
+	if mc.useDirectMedia && (!isSticker || mc.animatedStickerConfig.Target == "disable") {
 		content.URL, content.Info, err = mediaTransferer.DirectDownloadURL(ctx, portal, msgID, false)
 		if err != nil {
 			log.Err(err).Msg("error getting direct download URL for media")
@@ -248,15 +302,17 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por
 		}
 	}
 
-	extra := map[string]any{}
-
 	// Handle spoilers
 	// See: https://github.com/matrix-org/matrix-spec-proposals/pull/3725
 	if s, ok := msgMedia.(spoilable); ok && s.GetSpoiler() {
 		extra["town.robin.msc3725.content_warning"] = map[string]any{
 			"type": "town.robin.msc3725.spoiler",
 		}
-		extra["fi.mau.telegram.spoiler"] = true
+		if extra["info"] == nil {
+			extra["info"] = map[string]any{}
+		}
+		info := extra["info"].(map[string]any)
+		info["fi.mau.telegram.spoiler"] = true
 	}
 
 	// Handle disappearing messages
@@ -275,7 +331,7 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por
 
 	return &bridgev2.ConvertedMessagePart{
 		ID:      partID,
-		Type:    event.EventMessage,
+		Type:    eventType,
 		Content: &content,
 		Extra:   extra,
 	}, disappearingSetting, nil