Merge pull request #190 from tulir/replace_matrix_puppet

Add option to replace the Matrix puppet of own Telegram account with real Matrix account

alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py

@@ -0,0 +1,26 @@
+"""Add access_token and custom_mxid fields for puppets
+
+Revision ID: d5f7b8b4b456
+Revises: 6ca3d74d51e4
+Create Date: 2018-07-20 12:09:30.277960
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "d5f7b8b4b456"
+down_revision = "6ca3d74d51e4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("puppet", sa.Column("access_token", sa.String(), nullable=True))
+    op.add_column("puppet", sa.Column("custom_mxid", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("custom_mxid")
+        batch_op.drop_column("access_token")

example-config.yaml

@@ -125,6 +125,9 @@ bridge:
     # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
     # WARNING: Probably buggy, might get stuck in infinite loop.
     catch_up: false
+    # Whether or not to use /sync to get presence, read receipts and typing notifications when using
+    # your own Matrix account as the Matrix puppet for your Telegram account.
+    sync_with_custom_puppets: true
 
     # The formats to use when sending messages to Telegram via the relay bot.
     #

mautrix_telegram/__main__.py

@@ -86,7 +86,8 @@ state_store = SQLStateStore(db_session)
 appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                      config["appservice.as_token"], config["appservice.hs_token"],
                      config["appservice.bot_username"], log="mau.as", loop=loop,
-                     verify_ssl=config["homeserver.verify_ssl"], state_store=state_store)
+                     verify_ssl=config["homeserver.verify_ssl"], state_store=state_store,
+                     real_user_content_key="net.maunium.telegram.puppet")
 
 public_website = None
 provisioning_api = None
@@ -110,8 +111,10 @@ with appserv.run(config["appservice.hostname"], config["appservice.port"]) as st
     context.mx = MatrixHandler(context)
     init_formatter(context)
     init_portal(context)
-    init_puppet(context)
+    startup_actions = (init_puppet(context) +
-    startup_actions = init_user(context) + [start, context.mx.init_as_bot()]
+                       init_user(context) +
+                       [start,
+                        context.mx.init_as_bot()])
 
     if context.bot:
         startup_actions.append(context.bot.start())

mautrix_telegram/abstract_user.py

@@ -36,15 +36,15 @@ class AbstractUser:
     az = None
 
     def __init__(self):
-        self.puppet_whitelisted = False
+        self.puppet_whitelisted = False  # type: bool
-        self.whitelisted = False
+        self.whitelisted = False  # type: bool
-        self.relaybot_whitelisted = False
+        self.relaybot_whitelisted = False  # type: bool
-        self.is_admin = False
+        self.is_admin = False  # type: bool
-        self.client = None
+        self.client = None  # type: MautrixTelegramClient
-        self.tgid = None
+        self.tgid = None  # type: int
-        self.mxid = None
+        self.mxid = None  # type: str
-        self.is_relaybot = False
+        self.is_relaybot = False  # type: bool
-        self.is_bot = False
+        self.is_bot = False  # type: bool
 
     @property
     def connected(self):
@@ -124,7 +124,7 @@ class AbstractUser:
         self.log.debug("%s connected: %s", self.mxid, self.connected)
         return self
 
-    async def ensure_started(self, even_if_no_session=False):
+    async def ensure_started(self, even_if_no_session=False) -> "AbstractUser":
         if not self.puppet_whitelisted:
             return self
         self.log.debug("ensure_started(%s, connected=%s, even_if_no_session=%s, session_count=%s)",
@@ -229,9 +229,9 @@ class AbstractUser:
     async def update_status(self, update):
         puppet = pu.Puppet.get(update.user_id)
         if isinstance(update.status, UserStatusOnline):
-            await puppet.intent.set_presence("online")
+            await puppet.default_mxid_intent.set_presence("online")
         elif isinstance(update.status, UserStatusOffline):
-            await puppet.intent.set_presence("offline")
+            await puppet.default_mxid_intent.set_presence("offline")
         else:
             self.log.warning("Unexpected user status update: %s", update)
         return

mautrix_telegram/commands/auth.py

@@ -49,6 +49,70 @@ async def ping_bot(evt: CommandEvent):
                            "To use the bot, simply invite it to a portal room.")
 
 
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Revert your Telegram account's Matrix puppet to use the default Matrix "
+                           "account.")
+async def logout_matrix(evt: CommandEvent):
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    await puppet.switch_mxid(None, None)
+    await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
+
+
+@command_handler(needs_auth=True, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
+                           "account")
+async def login_matrix(evt: CommandEvent):
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login:
+        evt.sender.command_status = {
+            "next": enter_matrix_token,
+            "action": "Matrix login",
+        }
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        url = f"{prefix}/matrix-login?token={evt.public_website.make_token(evt.sender.mxid, '/matrix-login')}"
+        if allow_matrix_login:
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your Matrix access token "
+                "here.\n"
+                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
+                "your access token in the message history.")
+        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
+                               f"Please visit [the login page]({url}) to log in.")
+    elif allow_matrix_login:
+        return await evt.reply(
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your Matrix access token here to log in.")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+async def enter_matrix_token(evt: CommandEvent):
+    evt.sender.command_status = None
+
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+
+    resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
+    if resp == 2:
+        return await evt.reply("You can only log in as your own Matrix user.")
+    elif resp == 1:
+        return await evt.reply("Failed to verify access token.")
+    return await evt.reply(
+        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.")
+
+
 @command_handler(needs_auth=False, management_only=True,
                  help_section=SECTION_AUTH,
                  help_args="<_phone_> <_full name_>",
@@ -114,8 +178,8 @@ async def login(evt: CommandEvent):
 
     if evt.config["appservice.public.enabled"]:
         prefix = evt.config["appservice.public.external"]
-        url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid)}"
+        url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
-        if evt.config.get("bridge.allow_matrix_login", True):
+        if allow_matrix_login:
             return await evt.reply(
                 "This bridge instance allows you to log in inside or outside Matrix.\n\n"
                 "If you would like to log in within Matrix, please send your phone number or bot "
@@ -128,7 +192,7 @@ async def login(evt: CommandEvent):
     elif allow_matrix_login:
         return await evt.reply(
             "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your phone number or bot aut token here to start the login process.")
+            "Please send your phone number or bot auth token here to start the login process.")
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 

mautrix_telegram/config.py

@@ -191,6 +191,7 @@ class Config(DictWithRecursion):
         copy("bridge.public_portals")
         copy("bridge.native_stickers")
         copy("bridge.catch_up")
+        copy("bridge.sync_with_custom_puppets")
 
         if "bridge.message_formats.m_text" in self:
             del self["bridge.message_formats"]

mautrix_telegram/db.py

@@ -17,14 +17,14 @@
 from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer,
                         BigInteger, String, Boolean, Text)
 from sqlalchemy.sql import expression
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import relationship, Query
 import json
 
 from .base import Base
 
 
 class Portal(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "portal"
 
     # Telegram chat information
@@ -42,9 +42,8 @@ class Portal(Base):
     about = Column(String, nullable=True)
     photo_id = Column(String, nullable=True)
 
-
 class Message(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "message"
 
     mxid = Column(String)
@@ -56,7 +55,7 @@ class Message(Base):
 
 
 class UserPortal(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "user_portal"
 
     user = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE", ondelete="CASCADE"),
@@ -70,7 +69,7 @@ class UserPortal(Base):
 
 
 class User(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "user"
 
     mxid = Column(String, primary_key=True)
@@ -83,7 +82,7 @@ class User(Base):
 
 
 class RoomState(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "mx_room_state"
 
     room_id = Column(String, primary_key=True)
@@ -107,7 +106,7 @@ class RoomState(Base):
 
 
 class UserProfile(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "mx_user_profile"
 
     room_id = Column(String, primary_key=True)
@@ -125,7 +124,7 @@ class UserProfile(Base):
 
 
 class Contact(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "contact"
 
     user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
@@ -133,10 +132,12 @@ class Contact(Base):
 
 
 class Puppet(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "puppet"
 
     id = Column(Integer, primary_key=True)
+    custom_mxid = Column(String, nullable=True)
+    access_token = Column(String, nullable=True)
     displayname = Column(String, nullable=True)
     displayname_source = Column(Integer, nullable=True)
     username = Column(String, nullable=True)
@@ -147,14 +148,14 @@ class Puppet(Base):
 
 # Fucking Telegram not telling bots what chats they are in 3:<
 class BotChat(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "bot_chat"
     id = Column(Integer, primary_key=True)
     type = Column(String, nullable=False)
 
 
 class TelegramFile(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "telegram_file"
 
     id = Column(String, primary_key=True)

mautrix_telegram/matrix.py

@@ -14,6 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Dict
 import logging
 import asyncio
 import re
@@ -32,6 +33,7 @@ class MatrixHandler:
     def __init__(self, context):
         self.az, self.db, self.config, _, self.tgbot = context
         self.commands = CommandProcessor(context)
+        self.previously_typing = []
 
         self.az.matrix_event_handler(self.handle_event)
 
@@ -39,7 +41,8 @@ class MatrixHandler:
         displayname = self.config["appservice.bot_displayname"]
         if displayname:
             try:
-                await self.az.intent.set_display_name(displayname if displayname != "remove" else "")
+                await self.az.intent.set_display_name(
+                    displayname if displayname != "remove" else "")
             except asyncio.TimeoutError:
                 self.log.exception("TimeoutError when trying to set displayname")
 
@@ -51,19 +54,20 @@ class MatrixHandler:
                 self.log.exception("TimeoutError when trying to set avatar")
 
     async def handle_puppet_invite(self, room, puppet, inviter):
+        intent = puppet.default_mxid_intent
         self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room}")
         if not await inviter.is_logged_in():
-            await puppet.intent.error_and_leave(
+            await intent.error_and_leave(
                 room, text="Please log in before inviting Telegram puppets.")
             return
         portal = Portal.get_by_mxid(room)
         if portal:
             if portal.peer_type == "user":
-                await puppet.intent.error_and_leave(
+                await intent.error_and_leave(
                     room, text="You can not invite additional users to private chats.")
                 return
             await portal.invite_telegram(inviter, puppet)
-            await puppet.intent.join_room(room)
+            await intent.join_room(room)
             return
         try:
             members = await self.az.intent.get_room_members(room)
@@ -71,34 +75,34 @@ class MatrixHandler:
             members = []
         if self.az.bot_mxid not in members:
             if len(members) > 1:
-                await puppet.intent.error_and_leave(room, text=None, html=(
+                await intent.error_and_leave(room, text=None, html=(
                     f"Please invite "
                     f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
                     f"first if you want to create a Telegram chat."))
                 return
 
-            await puppet.intent.join_room(room)
+            await intent.join_room(room)
             portal = Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
             if portal.mxid:
                 try:
-                    await puppet.intent.invite(portal.mxid, inviter.mxid)
+                    await intent.invite(portal.mxid, inviter.mxid)
-                    await puppet.intent.send_notice(room, text=None, html=(
+                    await intent.send_notice(room, text=None, html=(
                         "You already have a private chat with me: "
                         f"<a href='https://matrix.to/#/{portal.mxid}'>"
                         "Link to room"
                         "</a>"))
-                    await puppet.intent.leave_room(room)
+                    await intent.leave_room(room)
                     return
                 except MatrixRequestError:
                     pass
             portal.mxid = room
             portal.save()
             inviter.register_portal(portal)
-            await puppet.intent.send_notice(room, "Portal to private chat created.")
+            await intent.send_notice(room, "Portal to private chat created.")
         else:
-            await puppet.intent.join_room(room)
+            await intent.join_room(room)
-            await puppet.intent.send_notice(room, "This puppet will remain inactive until a "
+            await intent.send_notice(room, "This puppet will remain inactive until a "
-                                                  "Telegram chat is created for this room.")
+                                           "Telegram chat is created for this room.")
 
     async def accept_bot_invite(self, room, inviter):
         tries = 0
@@ -215,7 +219,7 @@ class MatrixHandler:
             await portal.handle_matrix_message(sender, message, event_id)
             return
 
-        if not sender.whitelisted or message["msgtype"] != "m.text":
+        if not sender.whitelisted or message.get("msgtype", "m.unknown") != "m.text":
             return
 
         try:
@@ -286,18 +290,69 @@ class MatrixHandler:
         if await user.needs_relaybot(portal):
             await portal.name_change_matrix(user, displayname, prev_displayname, event_id)
 
+    @staticmethod
+    def parse_read_receipts(content: dict) -> Dict[str, str]:
+        return {user_id: event_id
+                for event_id, receipts in content.items()
+                for user_id in receipts.get("m.read", {})}
+
+    async def handle_read_receipts(self, room_id: str, receipts: Dict[str, str]):
+        portal = Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        for user_id, event_id in receipts.items():
+            user = await User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
+            await portal.mark_read(user, event_id)
+
+    async def handle_presence(self, user: str, presence: str):
+        user = await User.get_by_mxid(user).ensure_started()
+        if not await user.is_logged_in():
+            return
+        await user.set_presence(presence == "online")
+
+    async def handle_typing(self, room_id: str, now_typing: List[str]):
+        portal = Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        for user_id in set(self.previously_typing + now_typing):
+            is_typing = user_id in now_typing
+            was_typing = user_id in self.previously_typing
+            if is_typing and was_typing:
+                continue
+
+            user = await User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
+
+            await portal.set_typing(user, is_typing)
+
+        self.previously_typing = now_typing
+
     def filter_matrix_event(self, event):
-        return (event["sender"] == self.az.bot_mxid
+        sender = event.get("sender", None)
-                or Puppet.get_id_from_mxid(event["sender"]) is not None)
+        if not sender:
+            return False
+        return (sender == self.az.bot_mxid
+                or Puppet.get_id_from_mxid(sender) is not None)
+
+    async def try_handle_event(self, evt):
+        try:
+            await self.handle_event(evt)
+        except Exception:
+            self.log.exception("Error handling manually received Matrix event")
 
     async def handle_event(self, evt):
         if self.filter_matrix_event(evt):
             return
         self.log.debug("Received event: %s", evt)
-        type = evt["type"]
+        type = evt.get("type", "m.unknown")
-        room_id = evt["room_id"]
+        room_id = evt.get("room_id", None)
-        event_id = evt["event_id"]
+        event_id = evt.get("event_id", None)
-        sender = evt["sender"]
+        sender = evt.get("sender", None)
         content = evt.get("content", {})
         if type == "m.room.member":
             state_key = evt["state_key"]
@@ -335,3 +390,9 @@ class MatrixHandler:
             except KeyError:
                 old_events = set()
             await self.handle_room_pin(room_id, sender, new_events, old_events)
+        elif type == "m.receipt":
+            await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
+        elif type == "m.presence":
+            await self.handle_presence(sender, content.get("presence", "offline"))
+        elif type == "m.typing":
+            await self.handle_typing(room_id, content.get("user_ids", []))

mautrix_telegram/portal.py

@@ -31,6 +31,8 @@ from sqlalchemy.orm.exc import FlushError
 
 from telethon.tl.functions.messages import *
 from telethon.tl.functions.channels import *
+from telethon.tl.functions.messages import ReadHistoryRequest
+from telethon.tl.functions.channels import ReadHistoryRequest as ReadChannelHistoryRequest
 from telethon.errors import *
 from telethon.tl.types import *
 from mautrix_appservice import MatrixRequestError, IntentError
@@ -652,6 +654,23 @@ class Portal:
         return (await self.main_intent.get_displayname(self.mxid, user.mxid)
                 or user.mxid_localpart)
 
+    def set_typing(self, user, typing=True, action=SendMessageTypingAction):
+        return user.client(
+            SetTypingRequest(self.peer, action() if typing else SendMessageCancelAction()))
+
+    async def mark_read(self, user, event_id):
+        space = self.tgid if self.peer_type == "channel" else user.tgid
+        message = DBMessage.query.filter(DBMessage.mxid == event_id,
+                                         DBMessage.mx_room == self.mxid,
+                                         DBMessage.tg_space == space).one_or_none()
+        if not message:
+            return
+        if self.peer_type == "channel":
+            await user.client(ReadChannelHistoryRequest(
+                channel=await self.get_input_entity(user), max_id=message.tgid))
+        else:
+            await user.client(ReadHistoryRequest(peer=self.peer, max_id=message.tgid))
+
     async def leave_matrix(self, user, source, event_id):
         if await user.needs_relaybot(self):
             async with self.require_send_lock(self.bot.tgid):
@@ -824,7 +843,12 @@ class Portal:
             mxid=event_id))
         self.db.commit()
 
-    async def handle_matrix_message(self, sender, message, event_id):
+    async def handle_matrix_message(self, sender: u.User, message: dict, event_id: str):
+        puppet = p.Puppet.get_by_custom_mxid(sender.mxid)
+        if puppet and message.get("net.maunium.telegram.puppet", False):
+            self.log.debug("Ignoring puppet-sent message by confirmed puppet user %s", sender.mxid)
+            return
+
         logged_in = not await sender.needs_relaybot(self)
         client = sender.client if logged_in else self.bot.client
         sender_id = sender.tgid if logged_in else self.bot.tgid

mautrix_telegram/puppet.py

@@ -15,13 +15,16 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from difflib import SequenceMatcher
+from typing import Optional, Awaitable
 import re
 import logging
+import asyncio
 
 from telethon.tl.types import UserProfilePhoto
+from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
 
 from .db import Puppet as DBPuppet
-from . import util
+from . import util, matrix
 
 config = None
 
@@ -29,16 +32,24 @@ config = None
 class Puppet:
     log = logging.getLogger("mau.puppet")
     db = None
-    az = None
+    az = None  # type: AppService
+    mx = None  # type: matrix.MatrixHandler
+    loop = None  # type: asyncio.AbstractEventLoop
     mxid_regex = None
     username_template = None
     hs_domain = None
     cache = {}
+    by_custom_mxid = {}
 
-    def __init__(self, id=None, username=None, displayname=None, displayname_source=None,
+    def __init__(self, id=None, access_token=None, custom_mxid=None, username=None,
-                 photo_id=None, is_bot=None, is_registered=False, db_instance=None):
+                 displayname=None, displayname_source=None, photo_id=None, is_bot=None,
+                 is_registered=False, db_instance=None):
         self.id = id
-        self.mxid = self.get_mxid_from_id(self.id)
+        self.access_token = access_token
+        self.custom_mxid = custom_mxid
+        self.is_real_user = self.custom_mxid and self.access_token
+        self.default_mxid = self.get_mxid_from_id(self.id)
+        self.mxid = self.custom_mxid or self.default_mxid
 
         self.username = username
         self.displayname = displayname
@@ -48,9 +59,13 @@ class Puppet:
         self.is_registered = is_registered
         self._db_instance = db_instance
 
-        self.intent = self.az.intent.user(self.mxid)
+        self.default_mxid_intent = self.az.intent.user(self.default_mxid)
+        self.intent = None  # type: IntentAPI
+        self.refresh_intents()
 
         self.cache[id] = self
+        if self.custom_mxid:
+            self.by_custom_mxid[self.custom_mxid] = self
 
     @property
     def tgid(self):
@@ -59,6 +74,136 @@ class Puppet:
     async def is_logged_in(self):
         return True
 
+    # region Custom puppet management
+    def refresh_intents(self):
+        self.is_real_user = self.custom_mxid and self.access_token
+        self.intent = (self.az.intent.user(self.custom_mxid, self.access_token)
+                       if self.is_real_user else self.default_mxid_intent)
+
+    async def switch_mxid(self, access_token, mxid):
+        prev_mxid = self.custom_mxid
+        self.custom_mxid = mxid
+        self.access_token = access_token
+        self.refresh_intents()
+
+        err = await self.init_custom_mxid()
+        if err != 0:
+            return err
+
+        try:
+            del self.by_custom_mxid[prev_mxid]
+        except KeyError:
+            pass
+        self.mxid = self.custom_mxid or self.default_mxid
+        if self.mxid != self.default_mxid:
+            self.by_custom_mxid[self.mxid] = self
+            await self.leave_rooms_with_default_user()
+        self.save()
+        return 0
+
+    async def init_custom_mxid(self):
+        if not self.is_real_user:
+            return 0
+
+        mxid = await self.intent.whoami()
+        if not mxid or mxid != self.custom_mxid:
+            self.custom_mxid = None
+            self.access_token = None
+            self.refresh_intents()
+            if mxid != self.custom_mxid:
+                return 2
+            return 1
+        if config["bridge.sync_with_custom_puppets"]:
+            asyncio.ensure_future(self.sync(), loop=self.loop)
+        return 0
+
+    async def leave_rooms_with_default_user(self):
+        for room_id in await self.default_mxid_intent.get_joined_rooms():
+            try:
+                await self.default_mxid_intent.leave_room(room_id)
+                await self.intent.ensure_joined(room_id)
+            except (IntentError, MatrixRequestError):
+                pass
+
+    def create_sync_filter(self) -> Awaitable[str]:
+        return self.intent.client.create_filter(self.custom_mxid, {
+            "room": {
+                "include_leave": False,
+                "state": {
+                    "types": []
+                },
+                "timeline": {
+                    "types": [],
+                },
+                "ephemeral": {
+                    "types": ["m.typing", "m.receipt"]
+                },
+                "account_data": {
+                    "types": []
+                }
+            },
+            "account_data": {
+                "types": [],
+            },
+            "presence": {
+                "types": ["m.presence"]
+            },
+        })
+
+    def handle_sync(self, presence, ephemeral):
+        presence = [self.mx.try_handle_event(event) for event in presence]
+
+        for room_id, events in ephemeral.items():
+            for event in events:
+                event["room_id"] = room_id
+
+        ephemeral = [self.mx.try_handle_event(event)
+                     for events in ephemeral.values()
+                     for event in events]
+
+        events = ephemeral + presence
+        coro = asyncio.gather(*events, loop=self.loop)
+        asyncio.ensure_future(coro, loop=self.loop)
+
+    async def sync(self):
+        try:
+            await self._sync()
+        except Exception:
+            self.log.exception("Fatal error syncing")
+
+    async def _sync(self):
+        if not self.is_real_user:
+            self.log.warning("Called sync() for non-custom puppet.")
+            return
+        custom_mxid = self.custom_mxid
+        access_token_at_start = self.access_token
+        errors = 0
+        next_batch = None
+        filter_id = await self.create_sync_filter()
+        self.log.debug(f"Starting syncer for {custom_mxid} with sync filter {filter_id}.")
+        while access_token_at_start == self.access_token:
+            try:
+                sync_resp = await self.intent.client.sync(filter=filter_id, since=next_batch,
+                                                          set_presence="offline")
+                errors = 0
+                if next_batch is not None:
+                    presence = sync_resp.get("presence", {}).get("events", [])
+                    ephemeral = {room: data.get("ephemeral", {}).get("events", [])
+                                 for room, data
+                                 in sync_resp.get("rooms", {}).get("join", {}).items()}
+                    self.handle_sync(presence, ephemeral)
+                next_batch = sync_resp.get("next_batch", None)
+            except MatrixRequestError as e:
+                wait = min(errors, 11) ** 2
+                self.log.warning(f"Syncer for {custom_mxid} errored: {e}. "
+                                 f"Waiting for {wait} seconds...")
+                errors += 1
+                await asyncio.sleep(wait)
+        self.log.debug(f"Syncer for custom puppet {custom_mxid} stopped.")
+
+    # endregion
+    # region DB conversion
+
     @property
     def db_instance(self):
         if not self._db_instance:
@@ -66,17 +211,21 @@ class Puppet:
         return self._db_instance
 
     def new_db_instance(self):
-        return DBPuppet(id=self.id, username=self.username, displayname=self.displayname,
+        return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid,
+                        username=self.username, displayname=self.displayname,
                         displayname_source=self.displayname_source, photo_id=self.photo_id,
                         is_bot=self.is_bot, matrix_registered=self.is_registered)
 
     @classmethod
     def from_db(cls, db_puppet):
-        return Puppet(db_puppet.id, db_puppet.username, db_puppet.displayname,
+        return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
-                      db_puppet.displayname_source, db_puppet.photo_id, db_puppet.is_bot,
+                      db_puppet.username, db_puppet.displayname, db_puppet.displayname_source,
-                      db_puppet.matrix_registered, db_instance=db_puppet)
+                      db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
+                      db_instance=db_puppet)
 
     def save(self):
+        self.db_instance.access_token = self.access_token
+        self.db_instance.custom_mxid = self.custom_mxid
         self.db_instance.username = self.username
         self.db_instance.displayname = self.displayname
         self.db_instance.displayname_source = self.displayname_source
@@ -85,6 +234,8 @@ class Puppet:
         self.db_instance.matrix_registered = self.is_registered
         self.db.commit()
 
+    # endregion
+    # region Info updating
     def similarity(self, query):
         username_similarity = (SequenceMatcher(None, self.username, query).ratio()
                                if self.username else 0)
@@ -145,7 +296,7 @@ class Puppet:
 
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
-            await self.intent.set_display_name(displayname)
+            await self.default_mxid_intent.set_display_name(displayname)
             self.displayname = displayname
             self.displayname_source = source.tgid
             return True
@@ -156,15 +307,19 @@ class Puppet:
     async def update_avatar(self, source, photo):
         photo_id = f"{photo.volume_id}-{photo.local_id}"
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(self.db, source.client, self.intent, photo)
+            file = await util.transfer_file_to_matrix(self.db, source.client,
+                                                      self.default_mxid_intent, photo)
             if file:
-                await self.intent.set_avatar(file.mxc)
+                await self.default_mxid_intent.set_avatar(file.mxc)
                 self.photo_id = photo_id
                 return True
         return False
 
+    # endregion
+    # region Getters
+
     @classmethod
-    def get(cls, id, create=True):
+    def get(cls, id, create=True) -> "Optional[Puppet]":
         try:
             return cls.cache[id]
         except KeyError:
@@ -183,10 +338,34 @@ class Puppet:
         return None
 
     @classmethod
-    def get_by_mxid(cls, mxid, create=True):
+    def get_by_mxid(cls, mxid, create=True) -> "Optional[Puppet]":
         tgid = cls.get_id_from_mxid(mxid)
         return cls.get(tgid, create) if tgid else None
 
+    @classmethod
+    def get_by_custom_mxid(cls, mxid):
+        if not mxid:
+            raise ValueError("Matrix ID can't be empty")
+
+        try:
+            return cls.by_custom_mxid[mxid]
+        except KeyError:
+            pass
+
+        puppet = DBPuppet.query.filter(DBPuppet.custom_mxid == mxid).one_or_none()
+        if puppet:
+            puppet = cls.from_db(puppet)
+            return puppet
+
+        return None
+
+    @classmethod
+    def get_all_with_custom_mxid(cls):
+        return [cls.by_custom_mxid[puppet.mxid]
+                if puppet.custom_mxid in cls.by_custom_mxid
+                else cls.from_db(puppet)
+                for puppet in DBPuppet.query.filter(DBPuppet.custom_mxid is not None).all()]
+
     @classmethod
     def get_id_from_mxid(cls, mxid):
         match = cls.mxid_regex.match(mxid)
@@ -199,7 +378,7 @@ class Puppet:
         return f"@{cls.username_template.format(userid=id)}:{cls.hs_domain}"
 
     @classmethod
-    def find_by_username(cls, username):
+    def find_by_username(cls, username) -> "Optional[Puppet]":
         if not username:
             return None
 
@@ -214,7 +393,7 @@ class Puppet:
         return None
 
     @classmethod
-    def find_by_displayname(cls, displayname):
+    def find_by_displayname(cls, displayname) -> "Optional[Puppet]":
         if not displayname:
             return None
 
@@ -227,12 +406,15 @@ class Puppet:
             return cls.from_db(puppet)
 
         return None
+    # endregion
 
 
 def init(context):
     global config
-    Puppet.az, Puppet.db, config, _, _ = context
+    Puppet.az, Puppet.db, config, Puppet.loop, _ = context
+    Puppet.mx = context.mx
     Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}")
     Puppet.hs_domain = config["homeserver"]["domain"]
     localpart = Puppet.username_template.format(userid="(.+)")
     Puppet.mxid_regex = re.compile(f"@{localpart}:{Puppet.hs_domain}")
+    return [puppet.init_custom_mxid() for puppet in Puppet.get_all_with_custom_mxid()]

mautrix_telegram/user.py

@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict
+from typing import Dict, Awaitable, Optional
 import logging
 import asyncio
 import re
@@ -22,6 +22,7 @@ import re
 from telethon.tl.types import *
 from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
+from telethon.tl.functions.account import UpdateStatusRequest
 from mautrix_appservice import MatrixRequestError
 
 from .db import User as DBUser, Contact as DBContact
@@ -111,14 +112,14 @@ class User(AbstractUser):
 
     def new_db_instance(self):
         return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
-                      contacts=self.db_contacts, saved_contacts=self.saved_contacts,
+                      contacts=self.db_contacts, saved_contacts=self.saved_contacts or 0,
                       portals=self.db_portals)
 
     def save(self):
         self.db_instance.tgid = self.tgid
         self.db_instance.username = self.username
         self.db_instance.contacts = self.db_contacts
-        self.db_instance.saved_contacts = self.saved_contacts
+        self.db_instance.saved_contacts = self.saved_contacts or 0
         self.db_instance.portals = self.db_portals
         self.db.commit()
 
@@ -185,6 +186,12 @@ class User(AbstractUser):
     # endregion
     # region Telegram actions that need custom methods
 
+    def ensure_started(self, even_if_no_session=False) -> "Awaitable[User]":
+        return super().ensure_started(even_if_no_session)
+
+    def set_presence(self, online: bool = True):
+        return self.client(UpdateStatusRequest(offline=not online))
+
     async def update_info(self, info: User = None):
         info = info or await self.client.get_me()
         changed = False
@@ -309,7 +316,7 @@ class User(AbstractUser):
     # region Class instance lookup
 
     @classmethod
-    def get_by_mxid(cls, mxid, create=True):
+    def get_by_mxid(cls, mxid, create=True) -> "Optional[User]":
         if not mxid:
             raise ValueError("Matrix ID can't be empty")
 
@@ -332,7 +339,7 @@ class User(AbstractUser):
         return None
 
     @classmethod
-    def get_by_tgid(cls, tgid):
+    def get_by_tgid(cls, tgid) -> "Optional[User]":
         try:
             return cls.by_tgid[tgid]
         except KeyError:
@@ -346,7 +353,7 @@ class User(AbstractUser):
         return None
 
     @classmethod
-    def find_by_username(cls, username):
+    def find_by_username(cls, username) -> "Optional[User]":
         if not username:
             return None
 

mautrix_telegram/web/common/auth_api.py

@@ -23,6 +23,8 @@ from telethon.errors import *
 
 from ...commands.auth import enter_password
 from ...util import format_duration
+from ...puppet import Puppet
+from ...user import User
 
 
 class AuthAPI(abc.ABC):
@@ -36,6 +38,32 @@ class AuthAPI(abc.ABC):
                            errcode=""):
         raise NotImplementedError()
 
+    @abstractmethod
+    def get_mx_login_response(self, status=200, state="", username="", mxid="", message="",
+                              error="", errcode=""):
+        raise NotImplementedError()
+
+    async def post_matrix_token(self, user: User, token):
+        puppet = Puppet.get(user.tgid)
+        if puppet.is_real_user:
+            return self.get_mx_login_response(state="already-logged-in", status=409,
+                                              error="You have already logged in with your Matrix "
+                                                    "account.", errcode="already-logged-in")
+
+        resp = await puppet.switch_mxid(token, user.mxid)
+        if resp == 2:
+            return self.get_mx_login_response(status=403, errcode="only-login-self",
+                                              error="You can only log in as your own Matrix user.")
+        elif resp == 1:
+            return self.get_mx_login_response(status=401, errcode="invalid-access-token",
+                                              error="Failed to verify access token.")
+
+        return self.get_mx_login_response(mxid=user.mxid, status=200, state="logged-in")
+
+    async def post_matrix_password(self, user, password):
+        return self.get_mx_login_response(mxid=user.mxid, status=501, error="Not yet implemented",
+                                          errcode="not-yet-implemented")
+
     async def post_login_phone(self, user, phone):
         try:
             await user.client.sign_in(phone or "+123")

mautrix_telegram/web/provisioning/__init__.py

@@ -297,6 +297,10 @@ class ProvisioningAPI(AuthAPI):
             "errcode": errcode,
         }, status=status)
 
+    def get_mx_login_response(self, status=200, state="", username="", mxid="", message="",
+                              error="", errcode=""):
+        raise NotImplementedError()
+
     def get_login_response(self, status=200, state="", username="", mxid="", message="", error="",
                            errcode="") -> web.Response:
         if username:

mautrix_telegram/web/public/__init__.py

@@ -24,6 +24,7 @@ import time
 
 from ...util import sign_token, verify_token
 from ...user import User
+from ...puppet import Puppet
 from ..common import AuthAPI
 
 
@@ -38,28 +39,35 @@ class PublicBridgeWebsite(AuthAPI):
         self.login = Template(
             pkg_resources.resource_string("mautrix_telegram", "web/public/login.html.mako"))
 
+        self.mx_login = Template(
+            pkg_resources.resource_string("mautrix_telegram", "web/public/matrix-login.html.mako"))
+
         self.app = web.Application(loop=loop)
         self.app.router.add_route("GET", "/login", self.get_login)
         self.app.router.add_route("POST", "/login", self.post_login)
+        self.app.router.add_route("GET", "/matrix-login", self.get_matrix_login)
+        self.app.router.add_route("POST", "/matrix-login", self.post_matrix_login)
         self.app.router.add_static("/", pkg_resources.resource_filename("mautrix_telegram",
                                                                         "web/public/"))
 
-    def make_token(self, mxid, expires_in=900):
+    def make_token(self, mxid, endpoint="/login", expires_in=900):
         return sign_token(self.secret_key, {
             "mxid": mxid,
+            "endpoint": endpoint,
             "expiry": int(time.time()) + expires_in,
         })
 
-    def verify_token(self, token):
+    def verify_token(self, token, endpoint="/login"):
         token = verify_token(self.secret_key, token)
-        if token and token.get("expiry", 0) > int(time.time()):
+        if token and (token.get("expiry", 0) > int(time.time()) and
+                      token.get("endpoint", None) == endpoint):
             return token.get("mxid", None)
         return None
 
     async def get_login(self, request):
         state = "bot_token" if request.rel_url.query.get("mode", "") == "bot" else "request"
 
-        mxid = self.verify_token(request.rel_url.query.get("token", None))
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/login")
         if not mxid:
             return self.get_login_response(status=401, state="invalid-token")
         user = User.get_by_mxid(mxid, create=False) if mxid else None
@@ -75,14 +83,65 @@ class PublicBridgeWebsite(AuthAPI):
 
         return self.get_login_response(mxid=user.mxid, username=user.username)
 
+    async def get_matrix_login(self, request):
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        if not mxid:
+            return self.get_mx_login_response(status=401, state="invalid-token")
+        user = User.get_by_mxid(mxid, create=False) if mxid else None
+
+        if not user:
+            return self.get_mx_login_response(mxid=mxid)
+        elif not user.puppet_whitelisted:
+            return self.get_mx_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                              status=403)
+        await user.ensure_started()
+        if not await user.is_logged_in():
+            return self.get_mx_login_response(mxid=user.mxid, status=403,
+                                              error="You are not logged in to Telegram.")
+
+        puppet = Puppet.get(user.tgid)
+        if puppet.is_real_user:
+            return self.get_mx_login_response(state="already-logged-in", status=409)
+
+        return self.get_mx_login_response(mxid=user.mxid)
+
     def get_login_response(self, status=200, state="", username="", mxid="", message="", error="",
                            errcode=""):
         return web.Response(status=status, content_type="text/html",
                             text=self.login.render(username=username, state=state, error=error,
                                                    message=message, mxid=mxid))
 
+    def get_mx_login_response(self, status=200, state="", username="", mxid="", message="",
+                              error="", errcode=""):
+        return web.Response(status=status, content_type="text/html",
+                            text=self.mx_login.render(username=username, state=state, error=error,
+                                                      message=message, mxid=mxid))
+
+    async def post_matrix_login(self, request):
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        if not mxid:
+            return self.get_mx_login_response(status=401, state="invalid-token")
+
+        data = await request.post()
+
+        user = await User.get_by_mxid(mxid).ensure_started()
+        if not user.puppet_whitelisted:
+            return self.get_mx_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                              status=403)
+        elif not await user.is_logged_in():
+            return self.get_mx_login_response(mxid=user.mxid, status=403,
+                                              error="You are not logged in to Telegram.")
+        mode = data.get("mode", "access_token")
+        if mode == "password":
+            return await self.post_matrix_password(user, data["value"])
+        elif mode == "access_token":
+            return await self.post_matrix_token(user, data["value"])
+        return self.get_mx_login_response(mxid=user.mxid, status=400,
+                                          error="You must provide an access token or "
+                                                "password.")
+
     async def post_login(self, request):
-        mxid = self.verify_token(request.rel_url.query.get("token", None))
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/login")
         if not mxid:
             return self.get_login_response(status=401, state="invalid-token")
 

mautrix_telegram/web/public/login.css

@@ -19,8 +19,8 @@ form > div {
 	display: none;
 }
 
-form[data-status="request"]  > div.status-request,
+form[data-status="request"] > div.status-request,
-form[data-status="code"]     > div.status-code,
+form[data-status="code"] > div.status-code,
 form[data-status="password"] > div.status-password {
 	display: initial;
 }
@@ -48,3 +48,52 @@ form[data-status="password"] > div.status-password {
 	background-color: #d4edda;
 	color: #155724;
 }
+
+[type="checkbox"], [type="radio"] {
+	position: absolute;
+	opacity: 0;
+}
+
+[type="checkbox"] + label, [type="radio"] + label {
+	position: relative;
+	padding-left: 2.5rem;
+	cursor: pointer;
+	display: inline-block;
+}
+
+[type="checkbox"] + label:before, [type="radio"] + label:before {
+	content: '';
+	position: absolute;
+	left: 0;
+	top: 0.4rem;
+	width: 1.8rem;
+	height: 1.8rem;
+	border: 0.1rem solid #d1d1d1;
+}
+
+[type="radio"] + label:before, [type="radio"] + label:after {
+	border-radius: 50%;
+}
+
+[type="checkbox"]:checked + label:after,
+[type="radio"]:checked + label:after {
+	content: '';
+	width: 0.8rem;
+	height: 0.8rem;
+	background: #9b4dca;
+	position: absolute;
+	top: 0.9rem;
+	left: 0.5rem;
+}
+
+[type="radio"]:disabled + label:before, [type="checkbox"]:disabled + label:before {
+	background-color: #d1d1d1;
+}
+
+[type="radio"]:disabled + label, [type="checkbox"]:disabled + label {
+	color: #d1d1d1;
+}
+
+[type="radio"]:disabled:checked + label:after, [type="checkbox"]:disabled:checked + label:after {
+	background: #606c76;
+}

mautrix_telegram/web/public/login.html.mako

@@ -18,9 +18,9 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 <!DOCTYPE html>
 <html>
 <head>
-	<title>Mautrix-Telegram bridge</title>
+	<title>Login - Mautrix-Telegram bridge</title>
 	<link rel="icon" type="image/png" href="favicon.png"/>
-	<meta property="og:title" content="Mautrix-Telegram bridge">
+	<meta property="og:title" content="Login - Mautrix-Telegram bridge">
 	<meta property="og:description" content="A hybrid puppeting/relaybot Matrix-Telegram bridge">
 	<meta property="og:image" content="favicon.png">
 	<meta charset="utf-8">
@@ -40,10 +40,10 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 		function goBack() {
 			let params = new URLSearchParams(location.search.slice(1))
-			const mxid = params.get("mxid")
+			const token = params.get("token")
 			params = new URLSearchParams()
-			if (mxid) {
+			if (token) {
-				params.set("mxid", mxid)
+				params.set("token", token)
 			}
 			location.replace(location.href.split("?")[0] + "?" + params.toString())
 		}

mautrix_telegram/web/public/matrix-login.html.mako

@@ -0,0 +1,78 @@
+<!--
+mautrix-telegram - A Matrix-Telegram puppeting bridge
+Copyright (C) 2018 Tulir Asokan
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+-->
+<!DOCTYPE html>
+<html>
+<head>
+	<title>Matrix login - Mautrix-Telegram bridge</title>
+	<link rel="icon" type="image/png" href="favicon.png"/>
+	<meta property="og:title" content="Matrix login - Mautrix-Telegram bridge">
+	<meta property="og:description" content="A hybrid puppeting/relaybot Matrix-Telegram bridge">
+	<meta property="og:image" content="favicon.png">
+	<meta charset="utf-8">
+	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
+	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
+	<link rel="stylesheet"
+		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+	<link rel="stylesheet" href="login.css"/>
+</head>
+<body>
+<main class="container">
+	% if state == "logged-in":
+		<h1>Logged in successfully!</h1>
+		<p>
+			Logged in as ${mxid}.
+			You can now close this page.
+		</p>
+	% elif state == "already-logged-in":
+		<h1>You're already logged in!</h1>
+		<p>
+			If you want to log in with another account, log out using the
+			<code>logout-matrix</code> management command first.
+		</p>
+	% elif state == "invalid-token":
+		<h1>Invalid or expired token</h1>
+		<div class="error">Please ask the bridge bot for a new login link.</div>
+	% else:
+		<h1>Log in to Matrix</h1>
+	% if error:
+		<div class="error">${error}</div>
+	% endif
+	% if message:
+		<div class="message">${message}</div>
+	% endif
+		<form method="post">
+			<fieldset>
+				<label for="mxid">Matrix ID</label>
+				<input type="text" id="mxid" name="mxid" disabled value="${mxid}"/>
+
+				<input id="access_token" type="radio" name="mode" value="access_token" checked>
+				<label for="access_token">Access token</label><br>
+				<input id="password" type="radio" name="mode" value="password">
+				<label for="password">Password</label><br>
+
+				<label for="value">Value</label>
+				<input type="text" id="value" name="value"
+					   placeholder="Enter Matrix access token or password"/>
+
+				<button type="submit">Sign in</button>
+			</fieldset>
+		</form>
+	% endif
+</main>
+</body>
+</html>

setup.py

@@ -25,7 +25,7 @@ setuptools.setup(
 
     install_requires=[
         "aiohttp>=3.0.1,<4",
-        "mautrix-appservice>=0.3.0,<0.4.0",
+        "mautrix-appservice>=0.3.1,<0.4.0",
         "SQLAlchemy>=1.2.3,<2",
         "alembic>=1.0.0,<2",
         "Markdown>=2.6.11,<3",