Add gallery fetch status

This commit is contained in:
qananasikq
2026-08-20 10:43:00 +03:00
parent 7068b7e8fa
commit 1891f84291
9 changed files with 54 additions and 492 deletions
@@ -3,8 +3,8 @@ from typing import Sequence, Union
import sqlalchemy as sa import sqlalchemy as sa
from alembic import op from alembic import op
revision: str = "005_add_car_seen_sold_timestamps" revision: str = "004_add_car_seen_sold_timestamps"
down_revision: Union[str, None] = "004_add_ingestion_tables" down_revision: Union[str, None] = "003_add_composite_indexes"
branch_labels: Union[str, Sequence[str], None] = None branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None
@@ -1,17 +0,0 @@
# Заглушка миграции (таблицы ingestion пока не требуются)
from typing import Sequence, Union
from alembic import op
revision: str = "004_add_ingestion_tables"
down_revision: Union[str, None] = "003_add_composite_indexes"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -3,8 +3,8 @@ from typing import Sequence, Union
import sqlalchemy as sa import sqlalchemy as sa
from alembic import op from alembic import op
revision: str = "006_add_details_fetched_at" revision: str = "005_add_details_fetched_at"
down_revision: Union[str, None] = "005_add_car_seen_sold_timestamps" down_revision: Union[str, None] = "004_add_car_seen_sold_timestamps"
branch_labels: Union[str, Sequence[str], None] = None branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None
+35
View File
@@ -0,0 +1,35 @@
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "006_runtime_storage"
down_revision: Union[str, None] = "005_add_details_fetched_at"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute(
'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_last_seen_mobile_de" '
'ON "MOBILEDE_cars" (last_seen_at) '
"WHERE is_sold = FALSE AND origin_id LIKE 'mobile.de:%'"
)
op.add_column("MOBILEDE_cars", sa.Column("gallery_fetched_at", sa.DateTime(timezone=True), nullable=True))
op.create_index("ix_MOBILEDE_cars_gallery_fetched_at", "MOBILEDE_cars", ["gallery_fetched_at"])
op.execute(
'''
UPDATE "MOBILEDE_cars" AS car
SET gallery_fetched_at = car.details_fetched_at
WHERE car.details_fetched_at IS NOT NULL
AND EXISTS (
SELECT 1 FROM "MOBILEDE_images" AS image WHERE image.car_id = car.id
)
'''
)
def downgrade() -> None:
op.drop_index("ix_MOBILEDE_cars_gallery_fetched_at", table_name="MOBILEDE_cars")
op.drop_column("MOBILEDE_cars", "gallery_fetched_at")
op.execute('DROP INDEX IF EXISTS "ix_MOBILEDE_cars_active_last_seen_mobile_de"')
-73
View File
@@ -1,73 +0,0 @@
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "007_runtime_storage"
down_revision: Union[str, None] = "006_add_details_fetched_at"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute(
'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_last_seen_mobile_de" '
'ON "MOBILEDE_cars" (last_seen_at) '
"WHERE is_sold = FALSE AND origin_id LIKE 'mobile.de:%'"
)
op.create_table(
"MOBILEDE_detail_jobs",
sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False),
sa.Column("car_id", sa.BigInteger(), nullable=False),
sa.Column("listing_id", sa.String(length=128), nullable=False),
sa.Column("status", sa.String(length=32), server_default="pending", nullable=False),
sa.Column("priority", sa.Integer(), server_default="3", nullable=False),
sa.Column("attempts", sa.Integer(), server_default="0", nullable=False),
sa.Column("available_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("lease_owner", sa.String(length=64), nullable=True),
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_http_status", sa.Integer(), nullable=True),
sa.Column("error_kind", sa.String(length=64), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.ForeignKeyConstraint(["car_id"], ["MOBILEDE_cars.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("car_id"),
sa.UniqueConstraint("listing_id"),
)
op.create_index("ix_MOBILEDE_detail_jobs_listing_id", "MOBILEDE_detail_jobs", ["listing_id"])
op.create_index("ix_MOBILEDE_detail_jobs_status", "MOBILEDE_detail_jobs", ["status"])
op.create_index("ix_MOBILEDE_detail_jobs_available_at", "MOBILEDE_detail_jobs", ["available_at"])
op.create_index("ix_MOBILEDE_detail_jobs_lease_expires_at", "MOBILEDE_detail_jobs", ["lease_expires_at"])
op.create_index(
"ix_MOBILEDE_detail_jobs_ready",
"MOBILEDE_detail_jobs",
["status", "available_at", "priority", "created_at"],
)
op.create_index("ix_MOBILEDE_detail_jobs_lease", "MOBILEDE_detail_jobs", ["status", "lease_expires_at"])
op.add_column("MOBILEDE_cars", sa.Column("gallery_fetched_at", sa.DateTime(timezone=True), nullable=True))
op.create_index("ix_MOBILEDE_cars_gallery_fetched_at", "MOBILEDE_cars", ["gallery_fetched_at"])
op.execute(
'''
UPDATE "MOBILEDE_cars" AS car
SET gallery_fetched_at = car.details_fetched_at
WHERE car.details_fetched_at IS NOT NULL
AND EXISTS (
SELECT 1 FROM "MOBILEDE_images" AS image WHERE image.car_id = car.id
)
'''
)
def downgrade() -> None:
op.drop_index("ix_MOBILEDE_cars_gallery_fetched_at", table_name="MOBILEDE_cars")
op.drop_column("MOBILEDE_cars", "gallery_fetched_at")
op.drop_index("ix_MOBILEDE_detail_jobs_lease", table_name="MOBILEDE_detail_jobs")
op.drop_index("ix_MOBILEDE_detail_jobs_ready", table_name="MOBILEDE_detail_jobs")
op.drop_index("ix_MOBILEDE_detail_jobs_lease_expires_at", table_name="MOBILEDE_detail_jobs")
op.drop_index("ix_MOBILEDE_detail_jobs_available_at", table_name="MOBILEDE_detail_jobs")
op.drop_index("ix_MOBILEDE_detail_jobs_status", table_name="MOBILEDE_detail_jobs")
op.drop_index("ix_MOBILEDE_detail_jobs_listing_id", table_name="MOBILEDE_detail_jobs")
op.drop_table("MOBILEDE_detail_jobs")
op.execute('DROP INDEX IF EXISTS "ix_MOBILEDE_cars_active_last_seen_mobile_de"')
+11 -242
View File
@@ -1,17 +1,16 @@
import logging import logging
import os import os
import re import re
import uuid
from contextlib import contextmanager from contextlib import contextmanager
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Any, Iterator from typing import Iterator
from sqlalchemy import case, create_engine, delete, func, or_, select, text, update from sqlalchemy import case, create_engine, delete, func, or_, select, text, update
from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.orm import Session, sessionmaker
from ..core.config import Settings from ..core.config import Settings
from .models import Base, Car, DetailJob, Image, SyncRun from .models import Base, Car, Image, SyncRun
from .schemas import CarRecord from .schemas import CarRecord
logger = logging.getLogger("MOBILEDE_scraper.db") logger = logging.getLogger("MOBILEDE_scraper.db")
@@ -53,14 +52,6 @@ MOBILEDE_MATERIAL_CHANGE_FIELDS = (
_IN_CHUNK_SIZE = 5000 _IN_CHUNK_SIZE = 5000
MOBILEDE_ORIGIN_PREFIXES = ("mobile.de:", "mobilede:") MOBILEDE_ORIGIN_PREFIXES = ("mobile.de:", "mobilede:")
PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$") PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
def _utc_datetime(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
DETAIL_VALUE_FALLBACKS: dict[str, set[object]] = { DETAIL_VALUE_FALLBACKS: dict[str, set[object]] = {
"brand": {None, "", "UNKNOWN"}, "brand": {None, "", "UNKNOWN"},
"model": {None, "", "UNKNOWN"}, "model": {None, "", "UNKNOWN"},
@@ -866,7 +857,6 @@ class PersistenceService:
prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
*, *,
limit: int | None = None, limit: int | None = None,
max_existing_images: int = 1,
stale_before: datetime | None = None, stale_before: datetime | None = None,
) -> list[tuple[int, str, str, int]]: ) -> list[tuple[int, str, str, int]]:
if limit is not None and int(limit) <= 0: if limit is not None and int(limit) <= 0:
@@ -933,242 +923,21 @@ class PersistenceService:
"gallery_changed": changed, "gallery_changed": changed,
} }
def get_active_cars_batch_for_detail(
self,
prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
*,
limit: int | None = None,
stale_before: datetime | None = None,
) -> list[tuple[int, str, str]]:
"""Return fair Detail candidates without using gallery size as a proxy."""
if limit is not None and int(limit) <= 0:
return []
prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
conditions = [
_origin_prefix_filter(Car.origin_id, prefixes),
Car.is_sold == False, # noqa: E712
]
if stale_before is None:
conditions.append(Car.details_fetched_at.is_(None))
else:
conditions.append(or_(Car.details_fetched_at.is_(None), Car.details_fetched_at <= stale_before))
stmt = (
select(Car.id, Car.origin_id, Car.origin_url)
.where(*conditions)
.order_by(
case((Car.details_fetched_at.is_(None), 0), else_=1).asc(),
case((Car.details_fetched_at.is_(None), Car.first_seen_at), else_=None).asc(),
Car.details_fetched_at.asc(),
Car.id.asc(),
)
)
if limit is not None:
stmt = stmt.limit(int(limit))
with self.session_scope() as session:
rows = session.execute(stmt).all()
return [
(int(row[0]), str(row[1]), str(row[2]))
for row in rows
if row and row[0] and row[1] and row[2]
]
@staticmethod @staticmethod
def _detail_listing_id(origin_id: str) -> str: def _detail_listing_id(origin_id: str) -> str:
return str(origin_id).rsplit(":", 1)[-1].strip() return str(origin_id).rsplit(":", 1)[-1].strip()
def enqueue_detail_jobs( def is_gallery_fetched(self, origin_id: str) -> bool:
self, listing_id = self._detail_listing_id(origin_id)
origin_ids: list[str], candidate_origin_ids = {str(origin_id)}
*,
priority: int = 3,
reset: bool = False,
) -> dict[str, int]:
"""Create or reactivate durable jobs for known cars."""
unique_inputs = list(dict.fromkeys(str(value).strip() for value in origin_ids if str(value).strip()))
listing_ids = [self._detail_listing_id(value) for value in unique_inputs]
listing_ids = [value for value in listing_ids if value]
if not listing_ids:
return {"ready": 0, "deferred": 0, "missing": 0}
candidate_origin_ids = set(unique_inputs)
for listing_id in listing_ids:
candidate_origin_ids.update(f"{prefix}{listing_id}" for prefix in MOBILEDE_ORIGIN_PREFIXES) candidate_origin_ids.update(f"{prefix}{listing_id}" for prefix in MOBILEDE_ORIGIN_PREFIXES)
now = datetime.now(timezone.utc)
ready = 0
deferred = 0
with self.session_scope() as session: with self.session_scope() as session:
cars = session.execute( return bool(session.execute(
select(Car).where(Car.origin_id.in_(candidate_origin_ids)) select(Car.id)
).scalars().all() .where(Car.origin_id.in_(candidate_origin_ids))
cars_by_listing = {self._detail_listing_id(car.origin_id): car for car in cars} .where(Car.gallery_fetched_at.is_not(None))
existing_jobs = session.execute( .limit(1)
select(DetailJob).where(DetailJob.listing_id.in_(listing_ids)) ).scalar_one_or_none())
).scalars().all()
jobs_by_listing = {job.listing_id: job for job in existing_jobs}
for listing_id in listing_ids:
car = cars_by_listing.get(listing_id)
if car is None:
continue
job = jobs_by_listing.get(listing_id)
if job is None:
job = DetailJob(
car_id=int(car.id),
listing_id=listing_id,
status="pending",
priority=max(0, min(9, int(priority))),
available_at=now,
)
session.add(job)
jobs_by_listing[listing_id] = job
ready += 1
continue
job.car_id = int(car.id)
job.priority = max(int(job.priority or 0), max(0, min(9, int(priority))))
lease_expires_at = _utc_datetime(job.lease_expires_at)
available_at = _utc_datetime(job.available_at)
lease_active = job.status == "leased" and lease_expires_at is not None and lease_expires_at > now
cooldown_active = (
job.status in {"unavailable", "retry"}
and available_at is not None
and available_at > now
)
if lease_active or (cooldown_active and not reset):
deferred += 1
continue
if job.status == "completed" and car.gallery_fetched_at is not None and not reset:
deferred += 1
continue
job.status = "pending"
job.available_at = now
job.lease_owner = None
job.lease_expires_at = None
if reset:
job.error_kind = None
job.last_error = None
ready += 1
return {"ready": ready, "deferred": deferred, "missing": len(listing_ids) - len(cars_by_listing)}
def reserve_detail_jobs(
self,
*,
limit: int,
lease_seconds: int = 900,
listing_ids: list[str] | None = None,
) -> list[dict[str, object]]:
"""Atomically lease ready jobs; PostgreSQL workers use SKIP LOCKED."""
if int(limit) <= 0:
return []
now = datetime.now(timezone.utc)
lease_until = now + timedelta(seconds=max(60, int(lease_seconds)))
with self.session_scope() as session:
session.execute(
update(DetailJob)
.where(DetailJob.status == "leased")
.where(DetailJob.lease_expires_at <= now)
.values(status="pending", lease_owner=None, lease_expires_at=None, updated_at=now)
)
conditions = [
DetailJob.status.in_(("pending", "retry")),
DetailJob.available_at <= now,
]
if listing_ids is not None:
conditions.append(DetailJob.listing_id.in_(listing_ids))
stmt = (
select(DetailJob)
.where(*conditions)
.order_by(
DetailJob.priority.desc(),
DetailJob.available_at.asc(),
DetailJob.created_at.asc(),
DetailJob.id.asc(),
)
.limit(int(limit))
)
if self._is_postgres():
stmt = stmt.with_for_update(skip_locked=True)
jobs = session.execute(stmt).scalars().all()
leases: list[dict[str, object]] = []
for job in jobs:
owner = uuid.uuid4().hex
job.status = "leased"
job.lease_owner = owner
job.lease_expires_at = lease_until
job.attempts = int(job.attempts or 0) + 1
job.updated_at = now
leases.append({
"job_id": int(job.id),
"listing_id": str(job.listing_id),
"lease_owner": owner,
"attempts": int(job.attempts),
"priority": int(job.priority),
})
return leases
def complete_detail_job(self, listing_id: str, *, lease_owner: str | None = None) -> bool:
now = datetime.now(timezone.utc)
conditions = [DetailJob.listing_id == self._detail_listing_id(listing_id)]
if lease_owner:
conditions.append(DetailJob.lease_owner == lease_owner)
with self.session_scope() as session:
result = session.execute(
update(DetailJob)
.where(*conditions)
.values(
status="completed",
available_at=now,
lease_owner=None,
lease_expires_at=None,
last_http_status=200,
error_kind=None,
last_error=None,
updated_at=now,
)
)
return bool(result.rowcount)
def fail_detail_job(
self,
listing_id: str,
*,
lease_owner: str | None = None,
status: str = "retry",
cooldown_seconds: int = 60,
http_status: int | None = None,
error_kind: str | None = None,
last_error: str | None = None,
) -> bool:
now = datetime.now(timezone.utc)
conditions = [DetailJob.listing_id == self._detail_listing_id(listing_id)]
if lease_owner:
conditions.append(DetailJob.lease_owner == lease_owner)
with self.session_scope() as session:
result = session.execute(
update(DetailJob)
.where(*conditions)
.values(
status=status,
available_at=now + timedelta(seconds=max(0, int(cooldown_seconds))),
lease_owner=None,
lease_expires_at=None,
last_http_status=http_status,
error_kind=error_kind,
last_error=(last_error or "")[:4000] or None,
updated_at=now,
)
)
return bool(result.rowcount)
def release_detail_job(self, listing_id: str, *, lease_owner: str | None = None) -> bool:
return self.fail_detail_job(
listing_id,
lease_owner=lease_owner,
status="pending",
cooldown_seconds=0,
)
def mark_cars_sold_by_ids(self, car_ids: list[int], *, sold_at: datetime | None = None) -> int: def mark_cars_sold_by_ids(self, car_ids: list[int], *, sold_at: datetime | None = None) -> int:
if not car_ids: if not car_ids:
-38
View File
@@ -16,44 +16,6 @@ BODY_TYPE_ENUM_VALUES = (
"OTHER", "OTHER",
"NA", "NA",
) )
COUNTRY_ENUM_VALUES = (
"AT",
"BE",
"BG",
"CA",
"CH",
"CY",
"CZ",
"DE",
"DK",
"EE",
"ES",
"FI",
"FR",
"GB",
"GR",
"HR",
"HU",
"IE",
"IT",
"JP",
"KR",
"LI",
"LT",
"LU",
"LV",
"MT",
"NA",
"NL",
"NO",
"PL",
"PT",
"RO",
"SE",
"SI",
"SK",
"US",
)
ORIGIN_ENUM_VALUES = ( ORIGIN_ENUM_VALUES = (
"MOBILEDE", "MOBILEDE",
"MOBILE_DE", "MOBILE_DE",
-27
View File
@@ -71,33 +71,6 @@ class Image(Base):
car: Mapped[Car] = relationship("Car", back_populates="images") car: Mapped[Car] = relationship("Car", back_populates="images")
class DetailJob(Base):
__tablename__ = "MOBILEDE_detail_jobs"
__table_args__ = (
Index("ix_MOBILEDE_detail_jobs_ready", "status", "available_at", "priority", "created_at"),
Index("ix_MOBILEDE_detail_jobs_lease", "status", "lease_expires_at"),
)
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
car_id: Mapped[int] = mapped_column(
BigInteger().with_variant(Integer, "sqlite"),
ForeignKey("MOBILEDE_cars.id", ondelete="CASCADE"),
nullable=False,
unique=True,
)
listing_id: Mapped[str] = mapped_column(String(128), nullable=False, unique=True, index=True)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="pending", index=True)
priority: Mapped[int] = mapped_column(Integer, nullable=False, default=3)
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
last_http_status: Mapped[int | None] = mapped_column(Integer, nullable=True)
error_kind: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), onupdate=func.now())
class SyncRun(Base): class SyncRun(Base):
__tablename__ = "MOBILEDE_sync_runs" __tablename__ = "MOBILEDE_sync_runs"
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+2 -89
View File
@@ -10,7 +10,7 @@ from sqlalchemy import select
from mobilede_scraper.core.config import Settings from mobilede_scraper.core.config import Settings
from mobilede_scraper.storage.db import PersistenceService from mobilede_scraper.storage.db import PersistenceService
from mobilede_scraper.storage.models import Car, DetailJob, Image, SyncRun from mobilede_scraper.storage.models import Car, Image, SyncRun
from mobilede_scraper.storage.schemas import CarRecord, ImageRecord from mobilede_scraper.storage.schemas import CarRecord, ImageRecord
@@ -353,91 +353,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertIsNotNone(car.gallery_fetched_at) self.assertIsNotNone(car.gallery_fetched_at)
self.assertEqual(len(images), 3) self.assertEqual(len(images), 3)
def test_detail_candidates_ignore_gallery_size_and_prefer_oldest_unfetched(self) -> None:
old = self._search_record("mobile.de:old-detail")
old.first_seen_at = datetime(2026, 1, 1, tzinfo=timezone.utc)
old.images = [
ImageRecord(
fullres_image=f"https://example.test/{index}.jpg",
preview_image=f"https://example.test/{index}-preview.jpg",
order_index=index,
)
for index in range(4)
]
old.images_confirmed = True
newer = self._search_record("mobile.de:new-detail")
newer.first_seen_at = datetime(2026, 2, 1, tzinfo=timezone.utc)
self.persistence.upsert_car(newer)
self.persistence.upsert_car(old)
candidates = self.persistence.get_active_cars_batch_for_detail(limit=2)
self.assertEqual([origin_id for _car_id, origin_id, _url in candidates], [
"mobile.de:old-detail",
"mobile.de:new-detail",
])
def test_detail_jobs_enforce_cooldown_reset_and_exclusive_leases(self) -> None:
self.persistence.upsert_car(self._search_record("mobile.de:job-1"))
self.persistence.upsert_car(self._search_record("mobile.de:job-2"))
result = self.persistence.enqueue_detail_jobs(["mobile.de:job-1", "mobile.de:job-2"], priority=7)
self.assertEqual(result, {"ready": 2, "deferred": 0, "missing": 0})
first = self.persistence.reserve_detail_jobs(limit=1)
second = self.persistence.reserve_detail_jobs(limit=2)
self.assertEqual(len(first), 1)
self.assertEqual(len(second), 1)
self.assertNotEqual(first[0]["listing_id"], second[0]["listing_id"])
listing_id = str(first[0]["listing_id"])
lease_owner = str(first[0]["lease_owner"])
self.assertTrue(self.persistence.fail_detail_job(
listing_id,
lease_owner=lease_owner,
status="unavailable",
cooldown_seconds=3600,
http_status=404,
error_kind="unavailable",
last_error="not found",
))
self.assertEqual(
self.persistence.enqueue_detail_jobs([f"mobile.de:{listing_id}"]),
{"ready": 0, "deferred": 1, "missing": 0},
)
self.assertEqual(self.persistence.reserve_detail_jobs(limit=1, listing_ids=[listing_id]), [])
self.assertEqual(
self.persistence.enqueue_detail_jobs([f"mobile.de:{listing_id}"], priority=9, reset=True),
{"ready": 1, "deferred": 0, "missing": 0},
)
retry = self.persistence.reserve_detail_jobs(limit=1, listing_ids=[listing_id])
self.assertEqual(len(retry), 1)
self.assertTrue(self.persistence.complete_detail_job(listing_id, lease_owner=str(retry[0]["lease_owner"])))
with self.persistence.session_scope() as session:
job = session.execute(select(DetailJob).where(DetailJob.listing_id == listing_id)).scalar_one()
self.assertEqual(job.status, "completed")
self.assertEqual(job.priority, 9)
self.assertEqual(job.last_http_status, 200)
self.assertIsNone(job.lease_owner)
self.assertIsNone(job.error_kind)
def test_expired_detail_lease_is_recovered(self) -> None:
self.persistence.upsert_car(self._search_record("mobile.de:expired"))
self.persistence.enqueue_detail_jobs(["mobile.de:expired"])
lease = self.persistence.reserve_detail_jobs(limit=1)[0]
with self.persistence.session_scope() as session:
job = session.execute(select(DetailJob).where(DetailJob.listing_id == "expired")).scalar_one()
job.lease_expires_at = datetime.now(timezone.utc) - timedelta(minutes=1)
recovered = self.persistence.reserve_detail_jobs(limit=1, listing_ids=["expired"])
self.assertEqual(len(recovered), 1)
self.assertNotEqual(recovered[0]["lease_owner"], lease["lease_owner"])
self.assertEqual(recovered[0]["attempts"], 2)
def test_start_sync_run_keeps_parallel_active_runs_running(self) -> None: def test_start_sync_run_keeps_parallel_active_runs_running(self) -> None:
first_run_id = self.persistence.start_sync_run("lane-a") first_run_id = self.persistence.start_sync_run("lane-a")
second_run_id = self.persistence.start_sync_run("lane-b") second_run_id = self.persistence.start_sync_run("lane-b")
@@ -594,7 +509,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.persistence.replace_car_gallery(rich.origin_id, rich.images) self.persistence.replace_car_gallery(rich.origin_id, rich.images)
candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10, max_existing_images=1) candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10)
self.assertEqual([(origin_id, image_count) for _id, origin_id, _url, image_count in candidates], [("mobile.de:low", 1)]) self.assertEqual([(origin_id, image_count) for _id, origin_id, _url, image_count in candidates], [("mobile.de:low", 1)])
@@ -624,7 +539,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
candidates = self.persistence.get_active_cars_batch_for_image_enrich( candidates = self.persistence.get_active_cars_batch_for_image_enrich(
limit=10, limit=10,
max_existing_images=1,
stale_before=now - timedelta(days=7), stale_before=now - timedelta(days=7),
) )
@@ -635,7 +549,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertEqual( self.assertEqual(
self.persistence.get_active_cars_batch_for_image_enrich( self.persistence.get_active_cars_batch_for_image_enrich(
limit=0, limit=0,
max_existing_images=1,
stale_before=now, stale_before=now,
), ),
[], [],