648 lines
27 KiB
Python
648 lines
27 KiB
Python
from __future__ import annotations
|
|
|
|
import tempfile
|
|
import unittest
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from sqlalchemy import select
|
|
|
|
from mobilede_scraper.core.config import Settings
|
|
from mobilede_scraper.storage.db import PersistenceService
|
|
from mobilede_scraper.storage.models import Car, DetailJob, Image, SyncRun
|
|
from mobilede_scraper.storage.schemas import CarRecord, ImageRecord
|
|
|
|
|
|
class TestPersistenceServiceIntegration(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tmp_dir = tempfile.TemporaryDirectory()
|
|
db_path = Path(self.tmp_dir.name) / "test.sqlite"
|
|
|
|
self.settings = Settings()
|
|
self.settings.database.url = f"sqlite:///{db_path.as_posix()}"
|
|
self.settings.database.echo = False
|
|
|
|
self.persistence = PersistenceService(self.settings)
|
|
self.persistence.create_tables()
|
|
|
|
def tearDown(self) -> None:
|
|
self.persistence.engine.dispose()
|
|
self.tmp_dir.cleanup()
|
|
|
|
@staticmethod
|
|
def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
|
|
return CarRecord(
|
|
parser_id=f"mobilede:{origin_id}",
|
|
brand="Toyota",
|
|
model="Camry",
|
|
year=2014,
|
|
price=price,
|
|
origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US",
|
|
origin_id=origin_id,
|
|
slug=f"toyota-camry-{origin_id}",
|
|
details_confirmed=True,
|
|
images_confirmed=True,
|
|
preserve_existing_details=False,
|
|
images=[
|
|
ImageRecord(
|
|
fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633",
|
|
preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=400&height=300",
|
|
order_index=0,
|
|
)
|
|
],
|
|
)
|
|
|
|
@classmethod
|
|
def _search_record(cls, origin_id: str, *, price: int = 1000) -> CarRecord:
|
|
record = cls._record(origin_id, price=price)
|
|
record.details_confirmed = False
|
|
record.images_confirmed = False
|
|
record.preserve_existing_details = True
|
|
record.images = []
|
|
return record
|
|
|
|
@staticmethod
|
|
def _as_utc(value: datetime | None) -> datetime | None:
|
|
if value is None:
|
|
return None
|
|
if value.tzinfo is None:
|
|
return value.replace(tzinfo=timezone.utc)
|
|
return value.astimezone(timezone.utc)
|
|
|
|
def test_insert_update_and_skip_flow(self) -> None:
|
|
first = self._record("777", price=1000)
|
|
inserted = self.persistence.upsert_car(first)
|
|
self.assertEqual(inserted["action"], "inserted")
|
|
self.assertEqual(inserted["images_upserted"], 1)
|
|
|
|
same = self._record("777", price=1000)
|
|
updated_same = self.persistence.upsert_car(same)
|
|
self.assertEqual(updated_same["action"], "updated")
|
|
self.assertEqual(updated_same["images_upserted"], 1)
|
|
|
|
changed = self._record("777", price=1500)
|
|
updated = self.persistence.upsert_car(changed)
|
|
self.assertEqual(updated["action"], "updated")
|
|
self.assertEqual(updated["images_upserted"], 1)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
cars = session.execute(select(Car)).scalars().all()
|
|
images = session.execute(select(Image)).scalars().all()
|
|
|
|
self.assertEqual(len(cars), 1)
|
|
self.assertEqual(cars[0].price, 1500)
|
|
self.assertEqual(len(images), 1)
|
|
|
|
def test_update_replaces_old_images(self) -> None:
|
|
first = self._record("888")
|
|
self.persistence.upsert_car(first)
|
|
|
|
second = self._record("888")
|
|
second.images = [
|
|
ImageRecord(
|
|
fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=845&height=633",
|
|
preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300",
|
|
order_index=0,
|
|
)
|
|
]
|
|
self.persistence.upsert_car(second)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
images = session.execute(select(Image)).scalars().all()
|
|
|
|
self.assertEqual(len(images), 1)
|
|
self.assertIn("imageKeys=2", images[0].fullres_image)
|
|
|
|
def test_empty_update_preserves_existing_images(self) -> None:
|
|
first = self._record("mobile.de:keep-images")
|
|
self.persistence.upsert_car(first)
|
|
|
|
update = self._record("mobile.de:keep-images", price=1500)
|
|
update.images = []
|
|
result = self.persistence.upsert_car(update)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
images = session.execute(select(Image)).scalars().all()
|
|
|
|
self.assertEqual(result["action"], "updated")
|
|
self.assertEqual(result["images_upserted"], 0)
|
|
self.assertEqual(len(images), 1)
|
|
|
|
def test_batch_update_adds_images_when_existing_gallery_is_empty(self) -> None:
|
|
first = self._record("mobile.de:add-images")
|
|
first.images = []
|
|
self.persistence.upsert_car(first)
|
|
|
|
update = self._record("mobile.de:add-images", price=1500)
|
|
result = self.persistence.upsert_cars_batch([update])
|
|
|
|
with self.persistence.session_scope() as session:
|
|
images = session.execute(select(Image)).scalars().all()
|
|
|
|
self.assertEqual(result["updated"], 1)
|
|
self.assertEqual(result["images_upserted"], 1)
|
|
self.assertEqual(len(images), 1)
|
|
|
|
def test_batch_upsert_returns_only_inserted_origin_ids(self) -> None:
|
|
existing = self._record("mobile.de:existing")
|
|
self.persistence.upsert_car(existing)
|
|
|
|
updated = self._record("mobile.de:existing", price=2000)
|
|
inserted = self._record("mobile.de:new", price=3000)
|
|
result = self.persistence.upsert_cars_batch([updated, inserted])
|
|
|
|
self.assertEqual(result["inserted"], 1)
|
|
self.assertEqual(result["updated"], 1)
|
|
self.assertEqual(result["inserted_origin_ids"], ["mobile.de:new"])
|
|
|
|
def test_batch_search_upsert_classifies_changed_unchanged_and_reappeared(self) -> None:
|
|
unchanged = self._search_record("mobile.de:unchanged", price=1000)
|
|
changed = self._search_record("mobile.de:changed", price=1000)
|
|
reappeared = self._search_record("mobile.de:reappeared", price=1000)
|
|
self.persistence.upsert_cars_batch([unchanged, changed, reappeared])
|
|
|
|
with self.persistence.session_scope() as session:
|
|
car = session.execute(select(Car).where(Car.origin_id == reappeared.origin_id)).scalar_one()
|
|
car.is_sold = True
|
|
car.sold_at = datetime.now(timezone.utc)
|
|
|
|
unchanged_again = self._search_record("mobile.de:unchanged", price=1000)
|
|
unchanged_again.last_seen_at = datetime.now(timezone.utc) + timedelta(minutes=1)
|
|
changed_again = self._search_record("mobile.de:changed", price=1500)
|
|
reappeared_again = self._search_record("mobile.de:reappeared", price=1000)
|
|
result = self.persistence.upsert_cars_batch([unchanged_again, changed_again, reappeared_again])
|
|
|
|
self.assertEqual(result["updated"], 3)
|
|
self.assertEqual(result["changed"], 1)
|
|
self.assertEqual(result["unchanged"], 1)
|
|
self.assertEqual(result["reappeared"], 1)
|
|
self.assertEqual(result["changed_origin_ids"], ["mobile.de:changed"])
|
|
self.assertEqual(result["reappeared_origin_ids"], ["mobile.de:reappeared"])
|
|
|
|
def test_weak_search_fallback_does_not_create_false_material_change(self) -> None:
|
|
detail = self._record("mobile.de:fallback", price=1000)
|
|
detail.drive = "4WD"
|
|
detail.body_type = "SUV"
|
|
detail.color = "black"
|
|
self.persistence.upsert_car(detail)
|
|
|
|
search = self._search_record("mobile.de:fallback", price=1000)
|
|
search.drive = None
|
|
search.body_type = "OTHER"
|
|
search.color = "other"
|
|
result = self.persistence.upsert_cars_batch([search])
|
|
|
|
self.assertEqual(result["changed"], 0)
|
|
self.assertEqual(result["unchanged"], 1)
|
|
self.assertEqual(result["changed_origin_ids"], [])
|
|
|
|
def test_search_update_preserves_confirmed_details_and_gallery(self) -> None:
|
|
detail = self._record("mobile.de:confirmed", price=1000)
|
|
detail.drive = "4WD"
|
|
detail.gearbox = "AT"
|
|
detail.body_type = "SUV"
|
|
detail.engine_volume = 1998
|
|
detail.color = "black"
|
|
detail.mileage = 45000
|
|
detail.images = [
|
|
ImageRecord(
|
|
fullres_image=f"https://example.test/detail-{index}.jpg",
|
|
preview_image=f"https://example.test/detail-{index}-preview.jpg",
|
|
order_index=index,
|
|
)
|
|
for index in range(3)
|
|
]
|
|
self.persistence.upsert_car(detail)
|
|
|
|
search = self._record("mobile.de:confirmed", price=1200)
|
|
search.details_confirmed = False
|
|
search.images_confirmed = False
|
|
search.preserve_existing_details = True
|
|
search.drive = None
|
|
search.gearbox = None
|
|
search.body_type = "OTHER"
|
|
search.engine_volume = None
|
|
search.color = "other"
|
|
search.mileage = 0
|
|
search.images = [
|
|
ImageRecord(
|
|
fullres_image="https://example.test/search.jpg",
|
|
preview_image="https://example.test/search-preview.jpg",
|
|
order_index=0,
|
|
)
|
|
]
|
|
self.persistence.upsert_car(search)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
car = session.execute(select(Car).where(Car.origin_id == detail.origin_id)).scalar_one()
|
|
images = session.execute(
|
|
select(Image).where(Image.car_id == car.id).order_by(Image.order_index)
|
|
).scalars().all()
|
|
|
|
self.assertEqual(car.price, 1200)
|
|
self.assertEqual(car.drive, "4WD")
|
|
self.assertEqual(car.gearbox, "AT")
|
|
self.assertEqual(car.body_type, "SUV")
|
|
self.assertEqual(car.engine_volume, 1998)
|
|
self.assertEqual(car.color, "black")
|
|
self.assertEqual(car.mileage, 45000)
|
|
self.assertEqual([image.fullres_image for image in images], [
|
|
"https://example.test/detail-0.jpg",
|
|
"https://example.test/detail-1.jpg",
|
|
"https://example.test/detail-2.jpg",
|
|
])
|
|
|
|
def test_confirmed_detail_update_replaces_values_and_gallery(self) -> None:
|
|
first = self._record("mobile.de:detail-refresh")
|
|
first.drive = "FWD"
|
|
first.body_type = "SEDAN"
|
|
self.persistence.upsert_car(first)
|
|
|
|
refreshed = self._record("mobile.de:detail-refresh", price=2000)
|
|
refreshed.drive = "RWD"
|
|
refreshed.body_type = "COUPE"
|
|
refreshed.images = [
|
|
ImageRecord(
|
|
fullres_image="https://example.test/refreshed.jpg",
|
|
preview_image="https://example.test/refreshed-preview.jpg",
|
|
order_index=0,
|
|
)
|
|
]
|
|
self.persistence.upsert_car(refreshed)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
car = session.execute(select(Car).where(Car.origin_id == first.origin_id)).scalar_one()
|
|
images = session.execute(select(Image).where(Image.car_id == car.id)).scalars().all()
|
|
|
|
self.assertEqual(car.price, 2000)
|
|
self.assertEqual(car.drive, "RWD")
|
|
self.assertEqual(car.body_type, "COUPE")
|
|
self.assertEqual([image.fullres_image for image in images], ["https://example.test/refreshed.jpg"])
|
|
|
|
def test_search_update_preserves_detail_and_gallery_completion(self) -> None:
|
|
search = self._record("mobile.de:detail-ts")
|
|
search.details_confirmed = False
|
|
search.images_confirmed = False
|
|
search.preserve_existing_details = True
|
|
self.persistence.upsert_car(search)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
|
|
self.assertIsNone(car.details_fetched_at)
|
|
|
|
detail = self._record("mobile.de:detail-ts", price=1500)
|
|
before = datetime.now(timezone.utc)
|
|
self.persistence.upsert_car(detail)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
|
|
fetched_at = self._as_utc(car.details_fetched_at)
|
|
|
|
gallery = [
|
|
ImageRecord(
|
|
fullres_image="https://example.test/gallery.jpg",
|
|
preview_image="https://example.test/gallery-preview.jpg",
|
|
order_index=0,
|
|
)
|
|
]
|
|
self.persistence.replace_car_gallery(search.origin_id, gallery)
|
|
|
|
self.assertIsNotNone(fetched_at)
|
|
self.assertGreaterEqual(fetched_at, before)
|
|
|
|
search_again = self._record("mobile.de:detail-ts", price=2000)
|
|
search_again.details_confirmed = False
|
|
search_again.images_confirmed = False
|
|
search_again.preserve_existing_details = True
|
|
self.persistence.upsert_car(search_again)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
|
|
|
|
self.assertEqual(self._as_utc(car.details_fetched_at), fetched_at)
|
|
self.assertIsNotNone(car.gallery_fetched_at)
|
|
|
|
def test_replace_car_gallery_changes_only_images_and_completion_timestamp(self) -> None:
|
|
search = self._search_record("mobile.de:gallery-only", price=1234)
|
|
search.brand = "BMW"
|
|
search.model = "M3"
|
|
self.persistence.upsert_car(search)
|
|
|
|
gallery = [
|
|
ImageRecord(
|
|
fullres_image=f"https://example.test/gallery-{index}.jpg",
|
|
preview_image=f"https://example.test/gallery-{index}-preview.jpg",
|
|
order_index=index,
|
|
)
|
|
for index in range(3)
|
|
]
|
|
result = self.persistence.replace_car_gallery(search.origin_id, gallery)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
|
|
images = session.execute(
|
|
select(Image).where(Image.car_id == car.id).order_by(Image.order_index.asc())
|
|
).scalars().all()
|
|
|
|
self.assertEqual(result["images_upserted"], 3)
|
|
self.assertEqual(car.brand, "BMW")
|
|
self.assertEqual(car.model, "M3")
|
|
self.assertEqual(car.price, 1234)
|
|
self.assertIsNone(car.details_fetched_at)
|
|
self.assertIsNotNone(car.gallery_fetched_at)
|
|
self.assertEqual(len(images), 3)
|
|
|
|
def test_detail_candidates_ignore_gallery_size_and_prefer_oldest_unfetched(self) -> None:
|
|
old = self._search_record("mobile.de:old-detail")
|
|
old.first_seen_at = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
|
old.images = [
|
|
ImageRecord(
|
|
fullres_image=f"https://example.test/{index}.jpg",
|
|
preview_image=f"https://example.test/{index}-preview.jpg",
|
|
order_index=index,
|
|
)
|
|
for index in range(4)
|
|
]
|
|
old.images_confirmed = True
|
|
newer = self._search_record("mobile.de:new-detail")
|
|
newer.first_seen_at = datetime(2026, 2, 1, tzinfo=timezone.utc)
|
|
self.persistence.upsert_car(newer)
|
|
self.persistence.upsert_car(old)
|
|
|
|
candidates = self.persistence.get_active_cars_batch_for_detail(limit=2)
|
|
|
|
self.assertEqual([origin_id for _car_id, origin_id, _url in candidates], [
|
|
"mobile.de:old-detail",
|
|
"mobile.de:new-detail",
|
|
])
|
|
|
|
def test_detail_jobs_enforce_cooldown_reset_and_exclusive_leases(self) -> None:
|
|
self.persistence.upsert_car(self._search_record("mobile.de:job-1"))
|
|
self.persistence.upsert_car(self._search_record("mobile.de:job-2"))
|
|
|
|
result = self.persistence.enqueue_detail_jobs(["mobile.de:job-1", "mobile.de:job-2"], priority=7)
|
|
self.assertEqual(result, {"ready": 2, "deferred": 0, "missing": 0})
|
|
|
|
first = self.persistence.reserve_detail_jobs(limit=1)
|
|
second = self.persistence.reserve_detail_jobs(limit=2)
|
|
self.assertEqual(len(first), 1)
|
|
self.assertEqual(len(second), 1)
|
|
self.assertNotEqual(first[0]["listing_id"], second[0]["listing_id"])
|
|
|
|
listing_id = str(first[0]["listing_id"])
|
|
lease_owner = str(first[0]["lease_owner"])
|
|
self.assertTrue(self.persistence.fail_detail_job(
|
|
listing_id,
|
|
lease_owner=lease_owner,
|
|
status="unavailable",
|
|
cooldown_seconds=3600,
|
|
http_status=404,
|
|
error_kind="unavailable",
|
|
last_error="not found",
|
|
))
|
|
self.assertEqual(
|
|
self.persistence.enqueue_detail_jobs([f"mobile.de:{listing_id}"]),
|
|
{"ready": 0, "deferred": 1, "missing": 0},
|
|
)
|
|
self.assertEqual(self.persistence.reserve_detail_jobs(limit=1, listing_ids=[listing_id]), [])
|
|
|
|
self.assertEqual(
|
|
self.persistence.enqueue_detail_jobs([f"mobile.de:{listing_id}"], priority=9, reset=True),
|
|
{"ready": 1, "deferred": 0, "missing": 0},
|
|
)
|
|
retry = self.persistence.reserve_detail_jobs(limit=1, listing_ids=[listing_id])
|
|
self.assertEqual(len(retry), 1)
|
|
self.assertTrue(self.persistence.complete_detail_job(listing_id, lease_owner=str(retry[0]["lease_owner"])))
|
|
|
|
with self.persistence.session_scope() as session:
|
|
job = session.execute(select(DetailJob).where(DetailJob.listing_id == listing_id)).scalar_one()
|
|
self.assertEqual(job.status, "completed")
|
|
self.assertEqual(job.priority, 9)
|
|
self.assertEqual(job.last_http_status, 200)
|
|
self.assertIsNone(job.lease_owner)
|
|
self.assertIsNone(job.error_kind)
|
|
|
|
def test_expired_detail_lease_is_recovered(self) -> None:
|
|
self.persistence.upsert_car(self._search_record("mobile.de:expired"))
|
|
self.persistence.enqueue_detail_jobs(["mobile.de:expired"])
|
|
lease = self.persistence.reserve_detail_jobs(limit=1)[0]
|
|
|
|
with self.persistence.session_scope() as session:
|
|
job = session.execute(select(DetailJob).where(DetailJob.listing_id == "expired")).scalar_one()
|
|
job.lease_expires_at = datetime.now(timezone.utc) - timedelta(minutes=1)
|
|
|
|
recovered = self.persistence.reserve_detail_jobs(limit=1, listing_ids=["expired"])
|
|
|
|
self.assertEqual(len(recovered), 1)
|
|
self.assertNotEqual(recovered[0]["lease_owner"], lease["lease_owner"])
|
|
self.assertEqual(recovered[0]["attempts"], 2)
|
|
|
|
def test_start_sync_run_keeps_parallel_active_runs_running(self) -> None:
|
|
first_run_id = self.persistence.start_sync_run("lane-a")
|
|
second_run_id = self.persistence.start_sync_run("lane-b")
|
|
|
|
self.assertNotEqual(first_run_id, second_run_id)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
first = session.get(SyncRun, first_run_id)
|
|
second = session.get(SyncRun, second_run_id)
|
|
|
|
self.assertEqual(first.status, "running")
|
|
self.assertIsNone(first.finished_at)
|
|
self.assertEqual(second.status, "running")
|
|
|
|
def test_start_sync_run_marks_only_old_running_runs_as_failed(self) -> None:
|
|
first_run_id = self.persistence.start_sync_run("lane-a")
|
|
with self.persistence.session_scope() as session:
|
|
first = session.get(SyncRun, first_run_id)
|
|
first.started_at = datetime(2020, 1, 1, tzinfo=timezone.utc)
|
|
|
|
self.persistence.start_sync_run("lane-b")
|
|
|
|
with self.persistence.session_scope() as session:
|
|
first = session.get(SyncRun, first_run_id)
|
|
self.assertEqual(first.status, "failed")
|
|
self.assertIsNotNone(first.finished_at)
|
|
|
|
def test_individual_upsert_fallback_reports_failed_records(self) -> None:
|
|
first = self._record("mobile.de:ok")
|
|
second = self._record("mobile.de:failed")
|
|
with patch.object(
|
|
self.persistence,
|
|
"upsert_car",
|
|
side_effect=[{"action": "inserted", "images_upserted": 0}, RuntimeError("db failure")],
|
|
):
|
|
result = self.persistence._upsert_cars_individually([first, second])
|
|
|
|
self.assertEqual(result["inserted"], 1)
|
|
self.assertEqual(result["failed"], 1)
|
|
self.assertEqual(result["failed_origin_ids"], ["mobile.de:failed"])
|
|
|
|
def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
|
|
first = self._record("OLD-ID")
|
|
first.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
|
|
self.persistence.upsert_car(first)
|
|
|
|
second = self._record("NEW-ID")
|
|
second.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
|
|
result = self.persistence.upsert_car(second)
|
|
|
|
self.assertEqual(result["action"], "updated")
|
|
with self.persistence.session_scope() as session:
|
|
cars = session.execute(select(Car)).scalars().all()
|
|
|
|
self.assertEqual(len(cars), 1)
|
|
self.assertEqual(cars[0].origin_id, "NEW-ID")
|
|
|
|
def test_upsert_preserves_first_seen_and_reactivates_seen_car(self) -> None:
|
|
first_seen = datetime(2026, 5, 1, tzinfo=timezone.utc)
|
|
sold_at = datetime(2026, 5, 2, tzinfo=timezone.utc)
|
|
seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
|
|
first = self._record("mobile.de:777", price=1000)
|
|
first.first_seen_at = first_seen
|
|
first.last_seen_at = first_seen
|
|
self.persistence.upsert_car(first)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
|
|
car.is_sold = True
|
|
car.sold_at = sold_at
|
|
|
|
updated = self._record("mobile.de:777", price=1500)
|
|
updated.first_seen_at = seen_at
|
|
updated.last_seen_at = seen_at
|
|
updated.is_sold = False
|
|
updated.sold_at = None
|
|
self.persistence.upsert_car(updated)
|
|
|
|
with self.persistence.session_scope() as session:
|
|
car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
|
|
|
|
self.assertEqual(self._as_utc(car.first_seen_at), first_seen)
|
|
self.assertEqual(self._as_utc(car.last_seen_at), seen_at)
|
|
self.assertFalse(car.is_sold)
|
|
self.assertIsNone(car.sold_at)
|
|
self.assertEqual(car.price, 1500)
|
|
|
|
def test_mark_sold_not_seen_since_sets_sold_at_to_cycle_seen_at(self) -> None:
|
|
seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
|
|
old = self._record("mobile.de:old")
|
|
old.first_seen_at = seen_at - timedelta(days=2)
|
|
old.last_seen_at = seen_at - timedelta(days=1)
|
|
current = self._record("mobile.de:current")
|
|
current.first_seen_at = seen_at
|
|
current.last_seen_at = seen_at
|
|
self.persistence.upsert_car(old)
|
|
self.persistence.upsert_car(current)
|
|
|
|
marked = self.persistence.mark_sold_not_seen_since(seen_at)
|
|
|
|
self.assertEqual(marked, 1)
|
|
with self.persistence.session_scope() as session:
|
|
cars = session.execute(select(Car).order_by(Car.origin_id.asc())).scalars().all()
|
|
|
|
sold_map = {car.origin_id: (car.is_sold, self._as_utc(car.sold_at)) for car in cars}
|
|
self.assertEqual(sold_map["mobile.de:old"], (True, seen_at))
|
|
self.assertEqual(sold_map["mobile.de:current"], (False, None))
|
|
|
|
def test_mark_sold_not_seen_since_limits_candidates_to_scope_brands(self) -> None:
|
|
seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
|
|
bmw = self._record("mobile.de:bmw")
|
|
bmw.brand = "BMW"
|
|
bmw.last_seen_at = seen_at - timedelta(days=1)
|
|
toyota = self._record("mobile.de:toyota")
|
|
toyota.brand = "Toyota"
|
|
toyota.last_seen_at = seen_at - timedelta(days=1)
|
|
self.persistence.upsert_car(bmw)
|
|
self.persistence.upsert_car(toyota)
|
|
|
|
marked = self.persistence.mark_sold_not_seen_since(seen_at, brands=("BMW",))
|
|
|
|
self.assertEqual(marked, 1)
|
|
with self.persistence.session_scope() as session:
|
|
cars = {car.origin_id: car for car in session.execute(select(Car)).scalars().all()}
|
|
self.assertTrue(cars[bmw.origin_id].is_sold)
|
|
self.assertFalse(cars[toyota.origin_id].is_sold)
|
|
|
|
def test_get_active_cars_batch_for_image_enrich_selects_unfetched_galleries(self) -> None:
|
|
low = self._record("mobile.de:low")
|
|
low.details_confirmed = False
|
|
low.images = [
|
|
ImageRecord(
|
|
fullres_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-640.jpg",
|
|
preview_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-200.jpg",
|
|
order_index=0,
|
|
)
|
|
]
|
|
rich = self._record("mobile.de:rich")
|
|
rich.images = [
|
|
ImageRecord(
|
|
fullres_image=f"https://img.classistatic.de/api/v1/mo-prod/images/rich-{idx}?rule=mo-640.jpg",
|
|
preview_image=f"https://img.classistatic.de/api/v1/mo-prod/images/rich-{idx}?rule=mo-200.jpg",
|
|
order_index=idx,
|
|
)
|
|
for idx in range(3)
|
|
]
|
|
sold = self._record("mobile.de:sold")
|
|
sold.images = []
|
|
sold.is_sold = True
|
|
|
|
self.persistence.upsert_car(low)
|
|
self.persistence.upsert_car(rich)
|
|
self.persistence.upsert_car(sold)
|
|
|
|
self.persistence.replace_car_gallery(rich.origin_id, rich.images)
|
|
|
|
candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10, max_existing_images=1)
|
|
|
|
self.assertEqual([(origin_id, image_count) for _id, origin_id, _url, image_count in candidates], [("mobile.de:low", 1)])
|
|
|
|
def test_image_enrich_selector_prefers_unfetched_then_oldest_stale(self) -> None:
|
|
unfetched = self._record("mobile.de:unfetched")
|
|
unfetched.details_confirmed = False
|
|
unfetched_new = self._record("mobile.de:unfetched-new")
|
|
unfetched_new.details_confirmed = False
|
|
stale_old = self._record("mobile.de:stale-old")
|
|
stale_new = self._record("mobile.de:stale-new")
|
|
fresh = self._record("mobile.de:fresh")
|
|
for record in (unfetched, unfetched_new, stale_old, stale_new, fresh):
|
|
record.images = []
|
|
self.persistence.upsert_car(record)
|
|
|
|
now = datetime.now(timezone.utc)
|
|
with self.persistence.session_scope() as session:
|
|
cars = {
|
|
car.origin_id: car
|
|
for car in session.execute(select(Car)).scalars().all()
|
|
}
|
|
cars[unfetched.origin_id].first_seen_at = now - timedelta(hours=2)
|
|
cars[unfetched_new.origin_id].first_seen_at = now - timedelta(hours=1)
|
|
cars[stale_old.origin_id].gallery_fetched_at = now - timedelta(days=10)
|
|
cars[stale_new.origin_id].gallery_fetched_at = now - timedelta(days=8)
|
|
cars[fresh.origin_id].gallery_fetched_at = now - timedelta(days=1)
|
|
|
|
candidates = self.persistence.get_active_cars_batch_for_image_enrich(
|
|
limit=10,
|
|
max_existing_images=1,
|
|
stale_before=now - timedelta(days=7),
|
|
)
|
|
|
|
self.assertEqual(
|
|
[origin_id for _id, origin_id, _url, _count in candidates],
|
|
[unfetched_new.origin_id, unfetched.origin_id, stale_old.origin_id, stale_new.origin_id],
|
|
)
|
|
self.assertEqual(
|
|
self.persistence.get_active_cars_batch_for_image_enrich(
|
|
limit=0,
|
|
max_existing_images=1,
|
|
stale_before=now,
|
|
),
|
|
[],
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|
|
|