qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add gallery fetch status

Browse Source
This commit is contained in:
qananasikq
2026-08-20 10:43:00 +03:00
parent 7068b7e8fa
commit 1891f84291
9 changed files with 54 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_db.py
+2 -89
View File
@@ -10,7 +10,7 @@ from sqlalchemy import select
from mobilede_scraper.core.config import Settings
from mobilede_scraper.storage.db import PersistenceService
from mobilede_scraper.storage.models import Car, DetailJob, Image, SyncRun
from mobilede_scraper.storage.models import Car, Image, SyncRun
from mobilede_scraper.storage.schemas import CarRecord, ImageRecord
@@ -353,91 +353,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertIsNotNone(car.gallery_fetched_at)
self.assertEqual(len(images), 3)
def test_detail_candidates_ignore_gallery_size_and_prefer_oldest_unfetched(self) -> None:
old = self._search_record("mobile.de:old-detail")
old.first_seen_at = datetime(2026, 1, 1, tzinfo=timezone.utc)
old.images = [
ImageRecord(
fullres_image=f"https://example.test/{index}.jpg",
preview_image=f"https://example.test/{index}-preview.jpg",
order_index=index,
)
for index in range(4)
]
old.images_confirmed = True
newer = self._search_record("mobile.de:new-detail")
newer.first_seen_at = datetime(2026, 2, 1, tzinfo=timezone.utc)
self.persistence.upsert_car(newer)
self.persistence.upsert_car(old)
candidates = self.persistence.get_active_cars_batch_for_detail(limit=2)
self.assertEqual([origin_id for _car_id, origin_id, _url in candidates], [
"mobile.de:old-detail",
"mobile.de:new-detail",
])
def test_detail_jobs_enforce_cooldown_reset_and_exclusive_leases(self) -> None:
self.persistence.upsert_car(self._search_record("mobile.de:job-1"))
self.persistence.upsert_car(self._search_record("mobile.de:job-2"))
result = self.persistence.enqueue_detail_jobs(["mobile.de:job-1", "mobile.de:job-2"], priority=7)
self.assertEqual(result, {"ready": 2, "deferred": 0, "missing": 0})
first = self.persistence.reserve_detail_jobs(limit=1)
second = self.persistence.reserve_detail_jobs(limit=2)
self.assertEqual(len(first), 1)
self.assertEqual(len(second), 1)
self.assertNotEqual(first[0]["listing_id"], second[0]["listing_id"])
listing_id = str(first[0]["listing_id"])
lease_owner = str(first[0]["lease_owner"])
self.assertTrue(self.persistence.fail_detail_job(
listing_id,
lease_owner=lease_owner,
status="unavailable",
cooldown_seconds=3600,
http_status=404,
error_kind="unavailable",
last_error="not found",
))
self.assertEqual(
self.persistence.enqueue_detail_jobs([f"mobile.de:{listing_id}"]),
{"ready": 0, "deferred": 1, "missing": 0},
)
self.assertEqual(self.persistence.reserve_detail_jobs(limit=1, listing_ids=[listing_id]), [])
self.assertEqual(
self.persistence.enqueue_detail_jobs([f"mobile.de:{listing_id}"], priority=9, reset=True),
{"ready": 1, "deferred": 0, "missing": 0},
)
retry = self.persistence.reserve_detail_jobs(limit=1, listing_ids=[listing_id])
self.assertEqual(len(retry), 1)
self.assertTrue(self.persistence.complete_detail_job(listing_id, lease_owner=str(retry[0]["lease_owner"])))
with self.persistence.session_scope() as session:
job = session.execute(select(DetailJob).where(DetailJob.listing_id == listing_id)).scalar_one()
self.assertEqual(job.status, "completed")
self.assertEqual(job.priority, 9)
self.assertEqual(job.last_http_status, 200)
self.assertIsNone(job.lease_owner)
self.assertIsNone(job.error_kind)
def test_expired_detail_lease_is_recovered(self) -> None:
self.persistence.upsert_car(self._search_record("mobile.de:expired"))
self.persistence.enqueue_detail_jobs(["mobile.de:expired"])
lease = self.persistence.reserve_detail_jobs(limit=1)[0]
with self.persistence.session_scope() as session:
job = session.execute(select(DetailJob).where(DetailJob.listing_id == "expired")).scalar_one()
job.lease_expires_at = datetime.now(timezone.utc) - timedelta(minutes=1)
recovered = self.persistence.reserve_detail_jobs(limit=1, listing_ids=["expired"])
self.assertEqual(len(recovered), 1)
self.assertNotEqual(recovered[0]["lease_owner"], lease["lease_owner"])
self.assertEqual(recovered[0]["attempts"], 2)
def test_start_sync_run_keeps_parallel_active_runs_running(self) -> None:
first_run_id = self.persistence.start_sync_run("lane-a")
second_run_id = self.persistence.start_sync_run("lane-b")
@@ -594,7 +509,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.persistence.replace_car_gallery(rich.origin_id, rich.images)
candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10, max_existing_images=1)
candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10)
self.assertEqual([(origin_id, image_count) for _id, origin_id, _url, image_count in candidates], [("mobile.de:low", 1)])
@@ -624,7 +539,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
candidates = self.persistence.get_active_cars_batch_for_image_enrich(
limit=10,
max_existing_images=1,
stale_before=now - timedelta(days=7),
)
@@ -635,7 +549,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertEqual(
self.persistence.get_active_cars_batch_for_image_enrich(
limit=0,
max_existing_images=1,
stale_before=now,
),
[],