Add another backfill column

mautrix_telegram/db/upgrade/__init__.py

@@ -12,4 +12,5 @@ from . import (
     v07_puppet_phone_number,
     v08_portal_first_event,
     v09_puppet_username_index,
+    v10_more_backfill_fields,
 )

mautrix_telegram/db/upgrade/v00_latest_revision.py

@@ -15,8 +15,10 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from mautrix.util.async_db import Connection
 
+latest_version = 10
 
-async def create_v7_tables(conn: Connection) -> int:
+
+async def create_latest_tables(conn: Connection) -> int:
     await conn.execute(
         """CREATE TABLE "user" (
             mxid TEXT   PRIMARY KEY,
@@ -44,6 +46,10 @@ async def create_v7_tables(conn: Connection) -> int:
             megagroup   BOOLEAN,
             config      jsonb,
 
+            first_event_id    TEXT,
+            next_batch_id     TEXT,
+            base_insertion_id TEXT,
+
             sponsored_event_id     TEXT,
             sponsored_event_ts     BIGINT,
             sponsored_msg_random_id bytea,
@@ -112,6 +118,7 @@ async def create_v7_tables(conn: Connection) -> int:
             base_url     TEXT
         )"""
     )
+    await conn.execute("CREATE INDEX puppet_username_idx ON puppet(LOWER(username))")
     await conn.execute(
         """CREATE TABLE telegram_file (
             id              TEXT PRIMARY KEY,
@@ -197,4 +204,4 @@ async def create_v7_tables(conn: Connection) -> int:
             PRIMARY KEY (session_id, entity_id)
         )"""
     )
-    return 7
+    return latest_version

mautrix_telegram/db/upgrade/v01_initial_revision.py

@@ -18,7 +18,7 @@ from __future__ import annotations
 from mautrix.util.async_db import Connection, Scheme
 
 from . import upgrade_table
-from .v00_latest_revision import create_v7_tables
+from .v00_latest_revision import create_latest_tables, latest_version
 
 legacy_version_query = "SELECT version_num FROM alembic_version"
 last_legacy_version = "bfc0a39bfe02"
@@ -34,9 +34,9 @@ def table_exists(scheme: str, name: str) -> str:
 
 async def first_upgrade_target(conn: Connection, scheme: str) -> int:
     is_legacy = await conn.fetchval(table_exists(scheme, "alembic_version"))
-    # If it's a legacy db, the upgrade process will go to v1 and run each migration up to v7.
-    # If it's a new db, we'll create the v7 tables directly (see the create_v7_tables call).
-    return 1 if is_legacy else 7
+    # If it's a legacy db, the upgrade process will go to v1 and run each migration up to latest.
+    # If it's a new db, we'll create the latest tables directly (see create_latest_tables call).
+    return 1 if is_legacy else latest_version
 
 
 @upgrade_table.register(description="Initial asyncpg revision", upgrades_to=first_upgrade_target)
@@ -46,7 +46,7 @@ async def upgrade_v1(conn: Connection, scheme: str) -> int:
         await migrate_legacy_to_v1(conn, scheme)
         return 1
     else:
-        return await create_v7_tables(conn)
+        return await create_latest_tables(conn)
 
 
 async def drop_constraints(conn: Connection, table: str, contype: str) -> None:

mautrix_telegram/db/upgrade/v10_more_backfill_fields.py

@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add more portal columns related to infinite backfill")
+async def upgrade_v10(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE portal ADD COLUMN base_insertion_id TEXT")