Merge branch 'master' into allow-portals-without-power

ROADMAP.md

@@ -1,20 +1,20 @@
 # Features & roadmap
 
 * Matrix → Telegram
-  * [ ] Message content
+  * [x] Message content
     * [x] Plaintext messages
     * [x] Formatted messages
     * [x] Bot commands (!command -> /command)
     * [x] Mentions
     * [x] Rich quotes
-    * [ ] Locations (not implemented in Riot)
+    * [x] Locations (not implemented in Riot)
     * [x] Images
     * [x] Files
   * [x] Message redactions
   * [ ] † Presence
   * [ ] † Typing notifications
   * [ ] † Read receipts
-  * [ ] Pinning messages
+  * [x] Pinning messages
   * [x] Power level
     * [x] Normal chats
       * [ ] Non-hardcoded PL requirements
@@ -52,7 +52,7 @@
   * [x] Read receipts (private chat only)
   * [x] Pinning messages
   * [x] Admin/chat creator status
-  * [ ] Supergroup/channel permissions (precise per-user not supported in Matrix)
+  * [ ] Supergroup/channel permissions (precise per-user permissions not supported in Matrix)
   * [x] Membership actions
     * [x] Inviting
     * [x] Kicking

example-config.yaml

@@ -1,7 +1,11 @@
 # Homeserver details
 homeserver:
+    # The address that this appservice can use to connect to the homeserver.
     address: https://matrix.org
+    # The domain of the homeserver (for MXIDs, etc).
     domain: matrix.org
+    # Whether or not to verify the SSL certificate of the homeserver.
+    # Only applies if address starts with https://
     verify_ssl: true
 
 # Application service host/registration related details
@@ -15,6 +19,9 @@ appservice:
     hostname: localhost
     port: 8080
 
+    # The full URI to the database.
+    database: sqlite:///mautrix-telegram.db
+
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in
     # the HS database.
@@ -127,8 +134,4 @@ telegram:
     api_id: 12345
     api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
     # (Optional) Create your own bot at https://t.me/BotFather
-    #bot_token: 123456789:ABCD-QBPd3VrWRhg623xYh07WUWErYA9eMI
+    bot_token: disabled
-
-# The version of the config. The bridge will read this and automatically update the config if
-# the schema has changed. For the latest version, check the example config.
-version: 2

mautrix_telegram/__main__.py

@@ -26,7 +26,7 @@ from telethon_aio.sessions import AlchemySessionContainer
 from mautrix_appservice import AppService
 
 from .base import Base
-from .config import Config
+from .config import Config, DictWithRecursion
 from .matrix import MatrixHandler
 
 from .db import init as init_db
@@ -50,15 +50,18 @@ parser = argparse.ArgumentParser(
     prog="python -m mautrix-telegram")
 parser.add_argument("-c", "--config", type=str, default="config.yaml",
                     metavar="<path>", help="the path to your config file")
+parser.add_argument("-b", "--base-config", type=str, default="example-config.yaml",
+                    metavar="<path>", help="the path to the example config "
+                                           "(for automatic config updates)")
 parser.add_argument("-g", "--generate-registration", action="store_true",
                     help="generate registration and quit")
 parser.add_argument("-r", "--registration", type=str, default="registration.yaml",
                     metavar="<path>", help="the path to save the generated registration to")
 args = parser.parse_args()
 
-config = Config(args.config, args.registration)
+config = Config(args.config, args.registration, args.base_config)
 config.load()
-config.check_updates()
+config.update()
 
 if args.generate_registration:
     config.generate_registration()

mautrix_telegram/bot.py

@@ -253,6 +253,6 @@ def init(context):
     global config
     config = context.config
     token = config["telegram.bot_token"]
-    if token:
+    if token and not token.lower().startswith("disable"):
         return Bot(token)
     return None

mautrix_telegram/config.py

@@ -16,8 +16,6 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
-from ruamel.yaml.tokens import CommentToken
-from ruamel.yaml.error import CommentMark
 import random
 import string
 
@@ -93,38 +91,27 @@ class DictWithRecursion:
     def __delitem__(self, key):
         self.delete(key)
 
-    def comment(self, key, message):
-        indent = key.count(".") * 4
-        try:
-            path, key = key.rsplit(".", 1)
-        except ValueError:
-            path = None
-        entry = self[path] if path else self._data
-        c = entry.ca.items.setdefault(key, [None, [], None, None])
-        c[1] = []
-        entry.yaml_set_comment_before_after_key(key=key, before=message, indent=indent)
-
-    def comment_newline(self, key):
-        try:
-            path, key = key.rsplit(".", 1)
-        except ValueError:
-            path = None
-        entry = self[path] if path else self._data
-        c = entry.ca.items.setdefault(key, [None, [], None, None])
-        c[2] = CommentToken("\n\n", CommentMark(0), None)
-
 
 class Config(DictWithRecursion):
-    def __init__(self, path, registration_path):
+    def __init__(self, path, registration_path, base_path):
         super().__init__()
         self.path = path
         self.registration_path = registration_path
+        self.base_path = base_path
         self._registration = None
 
     def load(self):
         with open(self.path, 'r') as stream:
             self._data = yaml.load(stream)
 
+    def load_base(self):
+        try:
+            with open(self.base_path, 'r') as stream:
+                return DictWithRecursion(yaml.load(stream))
+        except OSError:
+            pass
+        return None
+
     def save(self):
         with open(self.path, 'w') as stream:
             yaml.dump(self._data, stream)
@@ -136,126 +123,85 @@ class Config(DictWithRecursion):
     def _new_token():
         return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(64))
 
-    def update_0_1(self):
+    def update(self):
-        permissions = self["bridge.permissions"] or CommentedMap()
+        base = self.load_base()
-        for entry in self["bridge.whitelist"] or []:
+        if not base:
-            permissions[entry] = "full"
+            return
-        for entry in self["bridge.admins"] or []:
-            permissions[entry] = "admin"
 
-        self["bridge.permissions"] = permissions
+        def copy(from_path, to_path=None):
-        del self["bridge.whitelist"]
+            if from_path in self:
-        del self["bridge.admins"]
+                base[to_path or from_path] = self[from_path]
 
-        if "bridge.authless_relaybot_portals" not in self:
+        def copy_dict(from_path, to_path=None):
-            self["bridge.authless_relaybot_portals"] = True
+            if from_path in self:
-            self.comment("bridge.authless_relaybot_portals",
+                to_path = to_path or from_path
-                         "Whether or not to allow creating portals from Telegram.")
+                base[to_path] = CommentedMap()
-        if "bridge.max_telegram_delete" not in self:
+                for key, value in self[from_path].items():
-            self["bridge.max_telegram_delete"] = 10
+                    base[to_path][key] = value
-            self.comment("bridge.max_telegram_delete",
-                         "The maximum number of simultaneous Telegram deletions to handle.\n"
-                         "A large number of simultaneous redactions could put strain on your "
-                         "homeserver.")
 
-        self.comment("bridge.permissions", "\n".join((
+        copy("homeserver.address")
-            "",
+        copy("homeserver.verify_ssl")
-            "Permissions for using the bridge.",
+        copy("homeserver.domain")
-            "Permitted values:",
-            "  relaybot - Only use the bridge via the relaybot, no access to commands.",
-            "      full - Full access to use the bridge via relaybot or logging in with Telegram account.",
-            "     admin - Full access to use the bridge and some extra administration commands.",
-            "Permitted keys:",
-            "       * - All Matrix users",
-            "  domain - All users on that homeserver",
-            "    mxid - Specific user")))
-        # The telegram section comment disappears for some reason 3:
-        self.comment("telegram", "\nTelegram config")
 
-        self["version"] = 1
+        copy("appservice.protocol")
-        # Add newline before version
+        copy("appservice.hostname")
-        self.comment("version",
+        copy("appservice.port")
-                     "\nThe version of the config. The bridge will read this and automatically "
-                     "update the config if\nthe schema has changed. For the latest version, "
-                     "check the example config.")
-        return self["version"]
 
-    def update_1_2(self):
+        copy("appservice.public.enabled")
-        del self["bridge.link_in_reply"]
+        copy("appservice.public.prefix")
-        del self["bridge.native_replies"]
+        copy("appservice.public.external")
-        if "bridge.bridge_notices" not in self:
+
-            self["bridge.bridge_notices"] = False
+        copy("appservice.debug")
-            self.comment("bridge.bridge_notices",
+
-                         "Whether or not Matrix bot messages (type m.notice) should be bridged.")
+        copy("appservice.id")
-        if "bridge.allow_matrix_login" not in self:
+        copy("appservice.bot_username")
-            self["bridge.allow_matrix_login"] = True
+        copy("appservice.bot_displayname")
-            self.comment("bridge.allow_matrix_login",
+
-                         "Allow logging in within Matrix. If false, the only way to log in is "
+        copy("appservice.as_token")
-                         "using the out-of-Matrix login website (see appservice.public config "
+        copy("appservice.hs_token")
-                         "section)")
+
-        if "bridge.inline_images" not in self:
+        copy("bridge.username_template")
-            self["bridge.inline_images"] = False
+        copy("bridge.alias_template")
-            self.comment("bridge.inline_images",
+        copy("bridge.displayname_template")
-                         "Use inline images instead of m.image to make rich captions possible.\n"
+
-                         "N.B. Inline images are not supported on all clients (e.g. Riot iOS).")
+        copy("bridge.displayname_preference")
-        if "appservice.public" not in self:
+
-            self["appservice.public.enabled"] = False
+        copy("bridge.edits_as_replies")
-            self["appservice.public.prefix"] = "/public"
+        copy("bridge.highlight_edits")
-            self["appservice.public.external"] = "https://example.com/public"
+        copy("bridge.bridge_notices")
-            self.comment("appservice.public",
+        copy("bridge.max_telegram_delete")
-                         "Public part of web server for out-of-Matrix interaction with the "
+        copy("bridge.allow_matrix_login")
-                         "bridge.\nUsed for things like login if the user wants to make sure the "
+        copy("bridge.inline_images")
-                         "2FA password isn't stored in the HS database.")
+        copy("bridge.plaintext_highlights")
-            self.comment("appservice.public.enabled",
+
-                         "Whether or not the public-facing endpoints should be enabled.")
+        copy("bridge.command_prefix")
-            self.comment("appservice.public.prefix",
+
-                         "The prefix to use in the public-facing endpoints.")
+        migrate_permissions = ("bridge.permissions" not in self
-            self.comment("appservice.public.external",
+                               or "bridge.whitelist" in self
-                         "The base URL where the public-facing endpoints are available. The "
+                               or "bridge.admins" in self)
-                         "prefix is not added\nimplicitly.")
+        if migrate_permissions:
-        if "homeserver.verify_ssl" not in self:
+            permissions = self["bridge.permissions"] or CommentedMap()
-            self["homeserver.verify_ssl"] = True
+            for entry in self["bridge.whitelist"] or []:
-        self["version"] = 2
+                permissions[entry] = "full"
-        return self["version"]
+            for entry in self["bridge.admins"] or []:
+                permissions[entry] = "admin"
+            base["bridge.permissions"] = permissions
+        else:
+            copy_dict("bridge.permissions")
 
-    def update_2_3(self):
-        if "bridge.plaintext_highlights" not in self:
-            self["bridge.plaintext_highlights"] = False
-            self.comment("bridge.plaintext_highlights",
-                         "Whether or not to bridge plaintext highlights.\n"
-                         "Only enable this if your displayname_template has some static part that "
-                         "the bridge can use to\nreliably identify what is a plaintext highlight.")
-        if "bridge.highlight_edits" not in self:
-            self["bridge.highlight_edits"] = False
-            self.comment("bridge.highlight_edits",
-                         "Highlight changed/added parts in edits. Requires lxml.")
         if "bridge.relaybot" not in self:
-            self["bridge.relaybot.authless_portals"] = bool(
+            copy("bridge.authless_relaybot_portals", "bridge.relaybot.authless_portals")
-                self["bridge.authless_relaybot_portals"]) or True
+        else:
-            del self["bridge.authless_relaybot_portals"]
+            copy("bridge.relaybot.authless_portals")
-            self["bridge.relaybot.whitelist_group_admins"] = True
+            copy("bridge.relaybot.whitelist_group_admins")
-            self["bridge.relaybot.whitelist"] = []
+            copy("bridge.relaybot.whitelist")
-            self.comment("bridge.relaybot", "Options related to the message relay Telegram bot.")
-            self.comment("bridge.relaybot.authless_portals",
-                         "Whether or not to allow creating portals from Telegram.")
-            self.comment("bridge.relaybot.whitelist_group_admins",
-                         "Whether or not to allow Telegram group admins to use the bot commands.")
-            self.comment("bridge.relaybot.whitelist",
-                         "List of usernames/user IDs who are also allowed to use the bot commands.")
-        self["version"] = 3
-        return self["version"]
 
-    def check_updates(self):
+        copy("telegram.api_id")
-        version = self.get("version", 0)
+        copy("telegram.api_hash")
-        new_version = version
+        copy("telegram.bot_token")
-        if version < 1:
+
-            new_version = self.update_0_1()
+        self._data = base._data
-        if version < 2:
+        self.save()
-            new_version = self.update_1_2()
-        if version < 3:
-            new_version = self.update_2_3()
-        if new_version != version:
-            self.save()
 
     def _get_permissions(self, key):
         level = self["bridge.permissions"].get(key, "")