Schedule stale gallery enrichment

mobilede_scraper/storage/db.py

@@ -5,7 +5,7 @@ from contextlib import contextmanager
 from datetime import datetime, timezone
 from typing import Any, Iterator
 
-from sqlalchemy import create_engine, delete, func, or_, select, text, update
+from sqlalchemy import case, create_engine, delete, func, or_, select, text, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session, sessionmaker
 
@@ -185,6 +185,8 @@ class PersistenceService:
 	@staticmethod
 	def _car_payload(record: CarRecord) -> dict[str, object]:
 		payload = record.model_dump(mode="python")
+		if record.details_confirmed:
+			payload["details_fetched_at"] = datetime.now(timezone.utc)
 		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
 
 	@staticmethod
@@ -194,6 +196,8 @@ class PersistenceService:
 		record: CarRecord,
 	) -> dict[str, object]:
 		prepared = dict(payload)
+		if not record.details_confirmed:
+			prepared["details_fetched_at"] = car.details_fetched_at
 		if record.preserve_existing_details and not record.details_confirmed:
 			for key, fallback_values in DETAIL_VALUE_FALLBACKS.items():
 				if prepared.get(key) in fallback_values:
@@ -907,24 +911,36 @@ class PersistenceService:
 		self,
 		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
 		*,
-		limit: int = 50,
+		limit: int | None = None,
 		max_existing_images: int = 1,
+		stale_before: datetime | None = None,
 	) -> list[tuple[int, str, str, int]]:
+		if limit is not None and int(limit) <= 0:
+			return []
 		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
 		with self.session_scope() as session:
 			image_count = func.count(Image.id)
-			result = session.execute(
+			conditions = [
+				_origin_prefix_filter(Car.origin_id, prefixes),
+				Car.is_sold == False,  # noqa: E712
+			]
+			if stale_before is not None:
+				conditions.append(or_(Car.details_fetched_at.is_(None), Car.details_fetched_at <= stale_before))
+			stmt = (
 				select(Car.id, Car.origin_id, Car.origin_url, image_count.label("image_count"))
 				.outerjoin(Image, Image.car_id == Car.id)
-				.where(
-					_origin_prefix_filter(Car.origin_id, prefixes),
-					Car.is_sold == False,  # noqa: E712
-				)
-				.group_by(Car.id, Car.origin_id, Car.origin_url)
+				.where(*conditions)
+				.group_by(Car.id, Car.origin_id, Car.origin_url, Car.details_fetched_at)
 				.having(image_count <= max(0, int(max_existing_images)))
-				.order_by(Car.last_seen_at.desc())
-				.limit(max(1, int(limit)))
+				.order_by(
+					case((Car.details_fetched_at.is_(None), 0), else_=1).asc(),
+					Car.details_fetched_at.asc(),
+					Car.id.asc(),
+				)
 			)
+			if limit is not None:
+				stmt = stmt.limit(int(limit))
+			result = session.execute(stmt)
 			return [
 				(int(row[0]), str(row[1]), str(row[2]), int(row[3] or 0))
 				for row in result

mobilede_scraper/storage/models.py

@@ -56,6 +56,7 @@ class Car(Base):
 	first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
 	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
 	sold_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
+	details_fetched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
 	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
 
 

mobilede_scraper/storage/schemas.py

@@ -40,6 +40,7 @@ class CarRecord(BaseModel):
 	first_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
 	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
 	sold_at: datetime | None = None
+	details_fetched_at: datetime | None = None
 	skip_image_sync: bool = False
 	details_confirmed: bool = False
 	images_confirmed: bool = False
@@ -91,5 +92,6 @@ class CarRead(BaseModel):
 	first_seen_at: datetime | None = None
 	last_seen_at: datetime | None = None
 	sold_at: datetime | None = None
+	details_fetched_at: datetime | None = None
 	images: list[ImageRead] = Field(default_factory=list)