qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add detail job storage

Browse Source
This commit is contained in:
qananasikq
2026-08-19 11:18:00 +03:00
parent f3329f76a5
commit 33b85ccecd
5 changed files with 832 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_db.py
+251 -10
View File
@@ -4,12 +4,13 @@ import tempfile
import unittest
from datetime import datetime, timedelta, timezone
from pathlib import Path
from unittest.mock import patch
from sqlalchemy import select
from mobilede_scraper.core.config import Settings
from mobilede_scraper.storage.db import PersistenceService
from mobilede_scraper.storage.models import Car, Image, SyncRun
from mobilede_scraper.storage.models import Car, DetailJob, Image, SyncRun
from mobilede_scraper.storage.schemas import CarRecord, ImageRecord
@@ -52,6 +53,15 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
],
)
@classmethod
def _search_record(cls, origin_id: str, *, price: int = 1000) -> CarRecord:
record = cls._record(origin_id, price=price)
record.details_confirmed = False
record.images_confirmed = False
record.preserve_existing_details = True
record.images = []
return record
@staticmethod
def _as_utc(value: datetime | None) -> datetime | None:
if value is None:
@@ -134,6 +144,59 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertEqual(result["images_upserted"], 1)
self.assertEqual(len(images), 1)
def test_batch_upsert_returns_only_inserted_origin_ids(self) -> None:
existing = self._record("mobile.de:existing")
self.persistence.upsert_car(existing)
updated = self._record("mobile.de:existing", price=2000)
inserted = self._record("mobile.de:new", price=3000)
result = self.persistence.upsert_cars_batch([updated, inserted])
self.assertEqual(result["inserted"], 1)
self.assertEqual(result["updated"], 1)
self.assertEqual(result["inserted_origin_ids"], ["mobile.de:new"])
def test_batch_search_upsert_classifies_changed_unchanged_and_reappeared(self) -> None:
unchanged = self._search_record("mobile.de:unchanged", price=1000)
changed = self._search_record("mobile.de:changed", price=1000)
reappeared = self._search_record("mobile.de:reappeared", price=1000)
self.persistence.upsert_cars_batch([unchanged, changed, reappeared])
with self.persistence.session_scope() as session:
car = session.execute(select(Car).where(Car.origin_id == reappeared.origin_id)).scalar_one()
car.is_sold = True
car.sold_at = datetime.now(timezone.utc)
unchanged_again = self._search_record("mobile.de:unchanged", price=1000)
unchanged_again.last_seen_at = datetime.now(timezone.utc) + timedelta(minutes=1)
changed_again = self._search_record("mobile.de:changed", price=1500)
reappeared_again = self._search_record("mobile.de:reappeared", price=1000)
result = self.persistence.upsert_cars_batch([unchanged_again, changed_again, reappeared_again])
self.assertEqual(result["updated"], 3)
self.assertEqual(result["changed"], 1)
self.assertEqual(result["unchanged"], 1)
self.assertEqual(result["reappeared"], 1)
self.assertEqual(result["changed_origin_ids"], ["mobile.de:changed"])
self.assertEqual(result["reappeared_origin_ids"], ["mobile.de:reappeared"])
def test_weak_search_fallback_does_not_create_false_material_change(self) -> None:
detail = self._record("mobile.de:fallback", price=1000)
detail.drive = "4WD"
detail.body_type = "SUV"
detail.color = "black"
self.persistence.upsert_car(detail)
search = self._search_record("mobile.de:fallback", price=1000)
search.drive = None
search.body_type = "OTHER"
search.color = "other"
result = self.persistence.upsert_cars_batch([search])
self.assertEqual(result["changed"], 0)
self.assertEqual(result["unchanged"], 1)
self.assertEqual(result["changed_origin_ids"], [])
def test_search_update_preserves_confirmed_details_and_gallery(self) -> None:
detail = self._record("mobile.de:confirmed", price=1000)
detail.drive = "4WD"
@@ -217,7 +280,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertEqual(car.body_type, "COUPE")
self.assertEqual([image.fullres_image for image in images], ["https://example.test/refreshed.jpg"])
def test_detail_timestamp_is_set_and_preserved_by_search_update(self) -> None:
def test_search_update_preserves_detail_and_gallery_completion(self) -> None:
search = self._record("mobile.de:detail-ts")
search.details_confirmed = False
search.images_confirmed = False
@@ -236,6 +299,15 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
fetched_at = self._as_utc(car.details_fetched_at)
gallery = [
ImageRecord(
fullres_image="https://example.test/gallery.jpg",
preview_image="https://example.test/gallery-preview.jpg",
order_index=0,
)
]
self.persistence.replace_car_gallery(search.origin_id, gallery)
self.assertIsNotNone(fetched_at)
self.assertGreaterEqual(fetched_at, before)
@@ -249,8 +321,124 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
self.assertEqual(self._as_utc(car.details_fetched_at), fetched_at)
self.assertIsNotNone(car.gallery_fetched_at)
def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
def test_replace_car_gallery_changes_only_images_and_completion_timestamp(self) -> None:
search = self._search_record("mobile.de:gallery-only", price=1234)
search.brand = "BMW"
search.model = "M3"
self.persistence.upsert_car(search)
gallery = [
ImageRecord(
fullres_image=f"https://example.test/gallery-{index}.jpg",
preview_image=f"https://example.test/gallery-{index}-preview.jpg",
order_index=index,
)
for index in range(3)
]
result = self.persistence.replace_car_gallery(search.origin_id, gallery)
with self.persistence.session_scope() as session:
car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
images = session.execute(
select(Image).where(Image.car_id == car.id).order_by(Image.order_index.asc())
).scalars().all()
self.assertEqual(result["images_upserted"], 3)
self.assertEqual(car.brand, "BMW")
self.assertEqual(car.model, "M3")
self.assertEqual(car.price, 1234)
self.assertIsNone(car.details_fetched_at)
self.assertIsNotNone(car.gallery_fetched_at)
self.assertEqual(len(images), 3)
def test_detail_candidates_ignore_gallery_size_and_prefer_oldest_unfetched(self) -> None:
old = self._search_record("mobile.de:old-detail")
old.first_seen_at = datetime(2026, 1, 1, tzinfo=timezone.utc)
old.images = [
ImageRecord(
fullres_image=f"https://example.test/{index}.jpg",
preview_image=f"https://example.test/{index}-preview.jpg",
order_index=index,
)
for index in range(4)
]
old.images_confirmed = True
newer = self._search_record("mobile.de:new-detail")
newer.first_seen_at = datetime(2026, 2, 1, tzinfo=timezone.utc)
self.persistence.upsert_car(newer)
self.persistence.upsert_car(old)
candidates = self.persistence.get_active_cars_batch_for_detail(limit=2)
self.assertEqual([origin_id for _car_id, origin_id, _url in candidates], [
"mobile.de:old-detail",
"mobile.de:new-detail",
])
def test_detail_jobs_enforce_cooldown_reset_and_exclusive_leases(self) -> None:
self.persistence.upsert_car(self._search_record("mobile.de:job-1"))
self.persistence.upsert_car(self._search_record("mobile.de:job-2"))
result = self.persistence.enqueue_detail_jobs(["mobile.de:job-1", "mobile.de:job-2"], priority=7)
self.assertEqual(result, {"ready": 2, "deferred": 0, "missing": 0})
first = self.persistence.reserve_detail_jobs(limit=1)
second = self.persistence.reserve_detail_jobs(limit=2)
self.assertEqual(len(first), 1)
self.assertEqual(len(second), 1)
self.assertNotEqual(first[0]["listing_id"], second[0]["listing_id"])
listing_id = str(first[0]["listing_id"])
lease_owner = str(first[0]["lease_owner"])
self.assertTrue(self.persistence.fail_detail_job(
listing_id,
lease_owner=lease_owner,
status="unavailable",
cooldown_seconds=3600,
http_status=404,
error_kind="unavailable",
last_error="not found",
))
self.assertEqual(
self.persistence.enqueue_detail_jobs([f"mobile.de:{listing_id}"]),
{"ready": 0, "deferred": 1, "missing": 0},
)
self.assertEqual(self.persistence.reserve_detail_jobs(limit=1, listing_ids=[listing_id]), [])
self.assertEqual(
self.persistence.enqueue_detail_jobs([f"mobile.de:{listing_id}"], priority=9, reset=True),
{"ready": 1, "deferred": 0, "missing": 0},
)
retry = self.persistence.reserve_detail_jobs(limit=1, listing_ids=[listing_id])
self.assertEqual(len(retry), 1)
self.assertTrue(self.persistence.complete_detail_job(listing_id, lease_owner=str(retry[0]["lease_owner"])))
with self.persistence.session_scope() as session:
job = session.execute(select(DetailJob).where(DetailJob.listing_id == listing_id)).scalar_one()
self.assertEqual(job.status, "completed")
self.assertEqual(job.priority, 9)
self.assertEqual(job.last_http_status, 200)
self.assertIsNone(job.lease_owner)
self.assertIsNone(job.error_kind)
def test_expired_detail_lease_is_recovered(self) -> None:
self.persistence.upsert_car(self._search_record("mobile.de:expired"))
self.persistence.enqueue_detail_jobs(["mobile.de:expired"])
lease = self.persistence.reserve_detail_jobs(limit=1)[0]
with self.persistence.session_scope() as session:
job = session.execute(select(DetailJob).where(DetailJob.listing_id == "expired")).scalar_one()
job.lease_expires_at = datetime.now(timezone.utc) - timedelta(minutes=1)
recovered = self.persistence.reserve_detail_jobs(limit=1, listing_ids=["expired"])
self.assertEqual(len(recovered), 1)
self.assertNotEqual(recovered[0]["lease_owner"], lease["lease_owner"])
self.assertEqual(recovered[0]["attempts"], 2)
def test_start_sync_run_keeps_parallel_active_runs_running(self) -> None:
first_run_id = self.persistence.start_sync_run("lane-a")
second_run_id = self.persistence.start_sync_run("lane-b")
@@ -260,9 +448,36 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
first = session.get(SyncRun, first_run_id)
second = session.get(SyncRun, second_run_id)
self.assertEqual(first.status, "running")
self.assertIsNone(first.finished_at)
self.assertEqual(second.status, "running")
def test_start_sync_run_marks_only_old_running_runs_as_failed(self) -> None:
first_run_id = self.persistence.start_sync_run("lane-a")
with self.persistence.session_scope() as session:
first = session.get(SyncRun, first_run_id)
first.started_at = datetime(2020, 1, 1, tzinfo=timezone.utc)
self.persistence.start_sync_run("lane-b")
with self.persistence.session_scope() as session:
first = session.get(SyncRun, first_run_id)
self.assertEqual(first.status, "failed")
self.assertIsNotNone(first.finished_at)
self.assertEqual(second.status, "running")
def test_individual_upsert_fallback_reports_failed_records(self) -> None:
first = self._record("mobile.de:ok")
second = self._record("mobile.de:failed")
with patch.object(
self.persistence,
"upsert_car",
side_effect=[{"action": "inserted", "images_upserted": 0}, RuntimeError("db failure")],
):
result = self.persistence._upsert_cars_individually([first, second])
self.assertEqual(result["inserted"], 1)
self.assertEqual(result["failed"], 1)
self.assertEqual(result["failed_origin_ids"], ["mobile.de:failed"])
def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
first = self._record("OLD-ID")
@@ -331,8 +546,28 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertEqual(sold_map["mobile.de:old"], (True, seen_at))
self.assertEqual(sold_map["mobile.de:current"], (False, None))
def test_get_active_cars_batch_for_image_enrich_selects_low_image_active_cars(self) -> None:
def test_mark_sold_not_seen_since_limits_candidates_to_scope_brands(self) -> None:
seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
bmw = self._record("mobile.de:bmw")
bmw.brand = "BMW"
bmw.last_seen_at = seen_at - timedelta(days=1)
toyota = self._record("mobile.de:toyota")
toyota.brand = "Toyota"
toyota.last_seen_at = seen_at - timedelta(days=1)
self.persistence.upsert_car(bmw)
self.persistence.upsert_car(toyota)
marked = self.persistence.mark_sold_not_seen_since(seen_at, brands=("BMW",))
self.assertEqual(marked, 1)
with self.persistence.session_scope() as session:
cars = {car.origin_id: car for car in session.execute(select(Car)).scalars().all()}
self.assertTrue(cars[bmw.origin_id].is_sold)
self.assertFalse(cars[toyota.origin_id].is_sold)
def test_get_active_cars_batch_for_image_enrich_selects_unfetched_galleries(self) -> None:
low = self._record("mobile.de:low")
low.details_confirmed = False
low.images = [
ImageRecord(
fullres_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-640.jpg",
@@ -357,6 +592,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.persistence.upsert_car(rich)
self.persistence.upsert_car(sold)
self.persistence.replace_car_gallery(rich.origin_id, rich.images)
candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10, max_existing_images=1)
self.assertEqual([(origin_id, image_count) for _id, origin_id, _url, image_count in candidates], [("mobile.de:low", 1)])
@@ -364,10 +601,12 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
def test_image_enrich_selector_prefers_unfetched_then_oldest_stale(self) -> None:
unfetched = self._record("mobile.de:unfetched")
unfetched.details_confirmed = False
unfetched_new = self._record("mobile.de:unfetched-new")
unfetched_new.details_confirmed = False
stale_old = self._record("mobile.de:stale-old")
stale_new = self._record("mobile.de:stale-new")
fresh = self._record("mobile.de:fresh")
for record in (unfetched, stale_old, stale_new, fresh):
for record in (unfetched, unfetched_new, stale_old, stale_new, fresh):
record.images = []
self.persistence.upsert_car(record)
@@ -377,9 +616,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
car.origin_id: car
for car in session.execute(select(Car)).scalars().all()
}
cars[stale_old.origin_id].details_fetched_at = now - timedelta(days=10)
cars[stale_new.origin_id].details_fetched_at = now - timedelta(days=8)
cars[fresh.origin_id].details_fetched_at = now - timedelta(days=1)
cars[unfetched.origin_id].first_seen_at = now - timedelta(hours=2)
cars[unfetched_new.origin_id].first_seen_at = now - timedelta(hours=1)
cars[stale_old.origin_id].gallery_fetched_at = now - timedelta(days=10)
cars[stale_new.origin_id].gallery_fetched_at = now - timedelta(days=8)
cars[fresh.origin_id].gallery_fetched_at = now - timedelta(days=1)
candidates = self.persistence.get_active_cars_batch_for_image_enrich(
limit=10,
@@ -389,7 +630,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertEqual(
[origin_id for _id, origin_id, _url, _count in candidates],
[unfetched.origin_id, stale_old.origin_id, stale_new.origin_id],
[unfetched_new.origin_id, unfetched.origin_id, stale_old.origin_id, stale_new.origin_id],
)
self.assertEqual(
self.persistence.get_active_cars_batch_for_image_enrich(