diff --git a/mautrix_telegram/db/backfill_queue.py b/mautrix_telegram/db/backfill_queue.py
index e86c5def..a1ba408e 100644
--- a/mautrix_telegram/db/backfill_queue.py
+++ b/mautrix_telegram/db/backfill_queue.py
@@ -167,7 +167,7 @@ class Backfill:
         q = "UPDATE backfill_queue SET completed_at=$1 WHERE queue_id=$2"
         await self.db.execute(q, datetime.now(), self.queue_id)
 
-    async def set_cooldown_timeout(self, timeout) -> None:
+    async def set_cooldown_timeout(self, timeout: int) -> None:
         """
         Set the backfill request to cooldown for ``timeout`` seconds.
         """
diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py
index 34537383..3638602d 100644
--- a/mautrix_telegram/user.py
+++ b/mautrix_telegram/user.py
@@ -382,7 +382,10 @@ class User(DBUser, AbstractUser, BaseUser):
         while True:
             req = await Backfill.get_next(self.mxid)
             if not req:
-                await self.wakeup_backfill_task.wait()
+                try:
+                    await asyncio.wait_for(self.wakeup_backfill_task.wait(), timeout=300)
+                except asyncio.TimeoutError:
+                    pass
                 self.wakeup_backfill_task.clear()
             else:
                 await self._takeout_and_backfill(req)
@@ -439,7 +442,7 @@ class User(DBUser, AbstractUser, BaseUser):
                 await asyncio.sleep(req.post_batch_delay)
             except Exception:
                 self.log.exception("Error handling backfill request for %s", req.portal_tgid)
-                await req.set_cooldown_timeout(10)
+                await req.set_cooldown_timeout(1800)
 
     async def update(self, update: TypeUpdate) -> bool:
         if not self.is_bot: