Track sold cars
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
@@ -48,6 +49,14 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
|
||||
],
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _as_utc(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
def test_insert_update_and_skip_flow(self) -> None:
|
||||
first = self._record("777", price=1000)
|
||||
inserted = self.persistence.upsert_car(first)
|
||||
@@ -122,6 +131,57 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
|
||||
self.assertEqual(len(cars), 1)
|
||||
self.assertEqual(cars[0].origin_id, "NEW-ID")
|
||||
|
||||
def test_upsert_preserves_first_seen_and_reactivates_seen_car(self) -> None:
|
||||
first_seen = datetime(2026, 5, 1, tzinfo=timezone.utc)
|
||||
sold_at = datetime(2026, 5, 2, tzinfo=timezone.utc)
|
||||
seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
|
||||
first = self._record("mobile.de:777", price=1000)
|
||||
first.first_seen_at = first_seen
|
||||
first.last_seen_at = first_seen
|
||||
self.persistence.upsert_car(first)
|
||||
|
||||
with self.persistence.session_scope() as session:
|
||||
car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
|
||||
car.is_sold = True
|
||||
car.sold_at = sold_at
|
||||
|
||||
updated = self._record("mobile.de:777", price=1500)
|
||||
updated.first_seen_at = seen_at
|
||||
updated.last_seen_at = seen_at
|
||||
updated.is_sold = False
|
||||
updated.sold_at = None
|
||||
self.persistence.upsert_car(updated)
|
||||
|
||||
with self.persistence.session_scope() as session:
|
||||
car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
|
||||
|
||||
self.assertEqual(self._as_utc(car.first_seen_at), first_seen)
|
||||
self.assertEqual(self._as_utc(car.last_seen_at), seen_at)
|
||||
self.assertFalse(car.is_sold)
|
||||
self.assertIsNone(car.sold_at)
|
||||
self.assertEqual(car.price, 1500)
|
||||
|
||||
def test_mark_sold_not_seen_since_sets_sold_at_to_cycle_seen_at(self) -> None:
|
||||
seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
|
||||
old = self._record("mobile.de:old")
|
||||
old.first_seen_at = seen_at - timedelta(days=2)
|
||||
old.last_seen_at = seen_at - timedelta(days=1)
|
||||
current = self._record("mobile.de:current")
|
||||
current.first_seen_at = seen_at
|
||||
current.last_seen_at = seen_at
|
||||
self.persistence.upsert_car(old)
|
||||
self.persistence.upsert_car(current)
|
||||
|
||||
marked = self.persistence.mark_sold_not_seen_since(seen_at)
|
||||
|
||||
self.assertEqual(marked, 1)
|
||||
with self.persistence.session_scope() as session:
|
||||
cars = session.execute(select(Car).order_by(Car.origin_id.asc())).scalars().all()
|
||||
|
||||
sold_map = {car.origin_id: (car.is_sold, self._as_utc(car.sold_at)) for car in cars}
|
||||
self.assertEqual(sold_map["mobile.de:old"], (True, seen_at))
|
||||
self.assertEqual(sold_map["mobile.de:current"], (False, None))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
|
||||
from mobilede_scraper.mobile_de.scraper import MobileDeScraper
|
||||
@@ -9,6 +10,7 @@ from mobilede_scraper.mobile_de.scraper import MobileDeScraper
|
||||
class _FakePersistence:
|
||||
def __init__(self) -> None:
|
||||
self.upsert_calls: list[list[str]] = []
|
||||
self.upsert_records: list[object] = []
|
||||
self.finish_payload: dict[str, object] | None = None
|
||||
|
||||
def create_tables(self) -> None:
|
||||
@@ -22,6 +24,7 @@ class _FakePersistence:
|
||||
|
||||
def upsert_cars_batch(self, records):
|
||||
self.upsert_calls.append([record.origin_id for record in records])
|
||||
self.upsert_records.extend(records)
|
||||
return {
|
||||
"inserted": len(records),
|
||||
"updated": 0,
|
||||
@@ -82,6 +85,31 @@ class TestMobileDeScraperStreamingSync(unittest.TestCase):
|
||||
self.assertIsNotNone(persistence.finish_payload)
|
||||
self.assertEqual(int(persistence.finish_payload["cars_upserted"]), 3)
|
||||
|
||||
def test_sync_search_applies_one_seen_at_to_all_records_in_run(self) -> None:
|
||||
seen_at = datetime(2026, 5, 5, 12, 30, tzinfo=timezone.utc)
|
||||
pages = [
|
||||
MobileDeSearchPage(
|
||||
url="https://example.test/page-1",
|
||||
page_number=1,
|
||||
total_results=2,
|
||||
listings=[
|
||||
MobileDeListing(id="1", url="https://example.test/1", title="Car 1"),
|
||||
MobileDeListing(id="2", url="https://example.test/2", title="Car 2"),
|
||||
],
|
||||
),
|
||||
]
|
||||
persistence = _FakePersistence()
|
||||
scraper = MobileDeScraper(client=_FakeClient(pages), persistence=persistence)
|
||||
|
||||
scraper.sync_search(max_pages=1, seen_at=seen_at)
|
||||
|
||||
records = persistence.upsert_calls
|
||||
self.assertEqual(records, [["mobile.de:1", "mobile.de:2"]])
|
||||
self.assertTrue(all(record.first_seen_at == seen_at for record in persistence.upsert_records))
|
||||
self.assertTrue(all(record.last_seen_at == seen_at for record in persistence.upsert_records))
|
||||
self.assertTrue(all(record.is_sold is False for record in persistence.upsert_records))
|
||||
self.assertTrue(all(record.sold_at is None for record in persistence.upsert_records))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user