Add another backfill column

mautrix_telegram/db/portal.py

@@ -46,6 +46,7 @@ class Portal:
     encrypted: bool
     first_event_id: EventID | None
     next_batch_id: BatchID | None
+    base_insertion_id: EventID | None
 
     sponsored_event_id: EventID | None
     sponsored_event_ts: int | None
@@ -80,6 +81,7 @@ class Portal:
             "encrypted",
             "first_event_id",
             "next_batch_id",
+            "base_insertion_id",
             "sponsored_event_id",
             "sponsored_event_ts",
             "sponsored_msg_random_id",
@@ -134,6 +136,7 @@ class Portal:
             self.encrypted,
             self.first_event_id,
             self.next_batch_id,
+            self.base_insertion_id,
             self.sponsored_event_id,
             self.sponsored_event_ts,
             self.sponsored_msg_random_id,
@@ -150,10 +153,11 @@ class Portal:
     async def save(self) -> None:
         q = """
         UPDATE portal
-        SET mxid=$4, avatar_url=$5, encrypted=$6, first_event_id=$7, next_batch_id=$8,
+        SET mxid=$4, avatar_url=$5, encrypted=$6,
-            sponsored_event_id=$9, sponsored_event_ts=$10, sponsored_msg_random_id=$11,
+            first_event_id=$7, next_batch_id=$8, base_insertion_id=$9,
-            username=$12, title=$13, about=$14, photo_id=$15, name_set=$16, avatar_set=$17,
+            sponsored_event_id=$10, sponsored_event_ts=$11, sponsored_msg_random_id=$12,
-            megagroup=$18, config=$19
+            username=$13, title=$14, about=$15, photo_id=$16, name_set=$17, avatar_set=$18,
+            megagroup=$19, config=$20
         WHERE tgid=$1 AND tg_receiver=$2 AND (peer_type=$3 OR true)
         """
         await self.db.execute(q, *self._values)
@@ -171,11 +175,12 @@ class Portal:
     async def insert(self) -> None:
         q = """
         INSERT INTO portal (
-            tgid, tg_receiver, peer_type, mxid, avatar_url, encrypted, first_event_id,
+            tgid, tg_receiver, peer_type, mxid, avatar_url, encrypted,
-            next_batch_id, sponsored_event_id, sponsored_event_ts, sponsored_msg_random_id,
+            first_event_id, base_insertion_id, next_batch_id,
+            sponsored_event_id, sponsored_event_ts, sponsored_msg_random_id,
             username, title, about, photo_id, name_set, avatar_set, megagroup, config
         ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,
-                  $19)
+                  $19, $20)
         """
         await self.db.execute(q, *self._values)
 

mautrix_telegram/db/upgrade/__init__.py

@@ -12,4 +12,5 @@ from . import (
     v07_puppet_phone_number,
     v08_portal_first_event,
     v09_puppet_username_index,
+    v10_more_backfill_fields,
 )

mautrix_telegram/db/upgrade/v00_latest_revision.py

@@ -15,8 +15,10 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from mautrix.util.async_db import Connection
 
+latest_version = 10
 
-async def create_v7_tables(conn: Connection) -> int:
+
+async def create_latest_tables(conn: Connection) -> int:
     await conn.execute(
         """CREATE TABLE "user" (
             mxid TEXT   PRIMARY KEY,
@@ -44,6 +46,10 @@ async def create_v7_tables(conn: Connection) -> int:
             megagroup   BOOLEAN,
             config      jsonb,
 
+            first_event_id    TEXT,
+            next_batch_id     TEXT,
+            base_insertion_id TEXT,
+
             sponsored_event_id     TEXT,
             sponsored_event_ts     BIGINT,
             sponsored_msg_random_id bytea,
@@ -112,6 +118,7 @@ async def create_v7_tables(conn: Connection) -> int:
             base_url     TEXT
         )"""
     )
+    await conn.execute("CREATE INDEX puppet_username_idx ON puppet(LOWER(username))")
     await conn.execute(
         """CREATE TABLE telegram_file (
             id              TEXT PRIMARY KEY,
@@ -197,4 +204,4 @@ async def create_v7_tables(conn: Connection) -> int:
             PRIMARY KEY (session_id, entity_id)
         )"""
     )
-    return 7
+    return latest_version

mautrix_telegram/db/upgrade/v01_initial_revision.py

@@ -18,7 +18,7 @@ from __future__ import annotations
 from mautrix.util.async_db import Connection, Scheme
 
 from . import upgrade_table
-from .v00_latest_revision import create_v7_tables
+from .v00_latest_revision import create_latest_tables, latest_version
 
 legacy_version_query = "SELECT version_num FROM alembic_version"
 last_legacy_version = "bfc0a39bfe02"
@@ -34,9 +34,9 @@ def table_exists(scheme: str, name: str) -> str:
 
 async def first_upgrade_target(conn: Connection, scheme: str) -> int:
     is_legacy = await conn.fetchval(table_exists(scheme, "alembic_version"))
-    # If it's a legacy db, the upgrade process will go to v1 and run each migration up to v7.
+    # If it's a legacy db, the upgrade process will go to v1 and run each migration up to latest.
-    # If it's a new db, we'll create the v7 tables directly (see the create_v7_tables call).
+    # If it's a new db, we'll create the latest tables directly (see create_latest_tables call).
-    return 1 if is_legacy else 7
+    return 1 if is_legacy else latest_version
 
 
 @upgrade_table.register(description="Initial asyncpg revision", upgrades_to=first_upgrade_target)
@@ -46,7 +46,7 @@ async def upgrade_v1(conn: Connection, scheme: str) -> int:
         await migrate_legacy_to_v1(conn, scheme)
         return 1
     else:
-        return await create_v7_tables(conn)
+        return await create_latest_tables(conn)
 
 
 async def drop_constraints(conn: Connection, table: str, contype: str) -> None:

mautrix_telegram/db/upgrade/v10_more_backfill_fields.py

@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add more portal columns related to infinite backfill")
+async def upgrade_v10(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE portal ADD COLUMN base_insertion_id TEXT")

mautrix_telegram/portal.py

@@ -274,6 +274,7 @@ class Portal(DBPortal, BasePortal):
         encrypted: bool = False,
         first_event_id: EventID | None = None,
         next_batch_id: BatchID | None = None,
+        base_insertion_id: EventID | None = None,
         sponsored_event_id: EventID | None = None,
         sponsored_event_ts: int | None = None,
         sponsored_msg_random_id: bytes | None = None,
@@ -295,6 +296,7 @@ class Portal(DBPortal, BasePortal):
             encrypted=encrypted,
             first_event_id=first_event_id,
             next_batch_id=next_batch_id,
+            base_insertion_id=base_insertion_id,
             sponsored_event_id=sponsored_event_id,
             sponsored_event_ts=sponsored_event_ts,
             sponsored_msg_random_id=sponsored_msg_random_id,
@@ -1843,9 +1845,9 @@ class Portal(DBPortal, BasePortal):
             ),
         )
         if isinstance(err, IgnoredMessageError):
-            status.status = MessageStatus.FAIL
-            status.error = str(err)
             status.reason = MessageStatusReason.UNSUPPORTED
+            status.error = str(err)
+            status.status = MessageStatus.FAIL
         elif err:
             status.reason = MessageStatusReason.GENERIC_ERROR
             status.error = str(err)