qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Track sold cars

Browse Source
This commit is contained in:
qananasikq
2026-05-06 21:07:24 +03:00
parent 81e99a41e8
commit 519686c0b7
8 changed files with 322 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
mobilede_scraper/mobile_de/scraper.py
View File

@@ -4,6 +4,7 @@ import logging
import os
from collections.abc import Callable
from dataclasses import asdict
from datetime import datetime, timezone
from typing import Any
import requests
@@ -19,6 +20,7 @@ logger = logging.getLogger("mobile_de.scraper")
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
MOBILEDE_LIGHT_REFRESH_EXISTING = os.getenv("MOBILEDE_LIGHT_REFRESH_EXISTING", "false").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_SEARCH_STRATEGY_NOTE = (
"mobile.de search pages return about 20 listings per page and are limited to about 50 pages; "
"for full coverage split into narrower segments and deduplicate by id."
@@ -102,13 +104,15 @@ class MobileDeScraper:
skipped_existing: int,
existing_streak: int,
new_records_kept: int,
existing_origin_ids: set[str] | None = None,
) -> tuple[list[CarRecord], int, int, int, bool]:
if not only_new or not page_records:
return page_records, skipped_existing, existing_streak, new_records_kept, False
existing_origin_ids = self.persistence.get_existing_origin_ids(
[record.origin_id for record in page_records if record.origin_id]
)
if existing_origin_ids is None:
existing_origin_ids = self.persistence.get_existing_origin_ids(
[record.origin_id for record in page_records if record.origin_id]
)
head_cut_triggered = False
should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
@@ -289,10 +293,14 @@ class MobileDeScraper:
mileage_max: str | None = None,
sort_by: str | None = None,
sort_order: str | None = None,
seen_at: datetime | None = None,
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
) -> dict[str, Any]:
self.persistence.create_tables()
run_id = self.persistence.start_sync_run(lane)
run_seen_at = seen_at or datetime.now(timezone.utc)
if run_seen_at.tzinfo is None:
run_seen_at = run_seen_at.replace(tzinfo=timezone.utc)
pages_payload: list[dict[str, Any]] = []
unique_ids: set[str] = set()
listing_count = 0
@@ -354,6 +362,17 @@ class MobileDeScraper:
page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings]
page_records = self._dedupe_page_records(page_records, seen_record_keys)
for record in page_records:
record.is_sold = False
record.first_seen_at = run_seen_at
record.last_seen_at = run_seen_at
record.sold_at = None
record.skip_image_sync = False
existing_origin_ids: set[str] = set()
if page_records and (only_new or MOBILEDE_LIGHT_REFRESH_EXISTING):
existing_origin_ids = self.persistence.get_existing_origin_ids(
[record.origin_id for record in page_records if record.origin_id]
)
page_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered = (
self._apply_only_new_page_policy(
page_records=page_records,
@@ -363,8 +382,13 @@ class MobileDeScraper:
skipped_existing=skipped_existing,
existing_streak=existing_streak,
new_records_kept=new_records_kept,
existing_origin_ids=existing_origin_ids,
)
)
if MOBILEDE_LIGHT_REFRESH_EXISTING and existing_origin_ids:
for record in page_records:
if record.origin_id and record.origin_id in existing_origin_ids:
record.skip_image_sync = True
if progress_callback is not None:
progress_callback(

185
mobilede_scraper/storage/db.py
View File

@@ -20,6 +20,7 @@ CAR_DB_FIELDS = {
col.key for col in Car.__table__.columns
if col.key not in ("id",)
}
CAR_UPDATE_FIELDS = CAR_DB_FIELDS - {"first_seen_at"}
_IN_CHUNK_SIZE = 5000
CAR_TABLE_NAME = Car.__tablename__
@@ -170,6 +171,16 @@ class PersistenceService:
payload = record.model_dump(mode="python")
return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
@staticmethod
def _apply_update_payload(car: Car, payload: dict[str, object]) -> None:
for key, value in payload.items():
if key in CAR_UPDATE_FIELDS:
setattr(car, key, value)
@staticmethod
def _skip_image_sync(record: CarRecord) -> bool:
return bool(getattr(record, "skip_image_sync", False))
def _is_postgres(self) -> bool:
return self.engine.dialect.name == "postgresql"
@@ -227,7 +238,7 @@ class PersistenceService:
def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
return {
key: getattr(insert_stmt.excluded, key)
for key in CAR_DB_FIELDS
for key in CAR_UPDATE_FIELDS
}
def _replace_images_for_car(
@@ -266,12 +277,16 @@ class PersistenceService:
images = [image.model_dump(mode="python") for image in record.images]
car_by_id = existing_by_id.get(record.origin_id)
car_by_url = existing_by_url.get(record.origin_url)
entry: dict[str, object] = {"record": record, "images": images, "car_id": None, "action": "inserted"}
entry: dict[str, object] = {
"record": record,
"images": images,
"car_id": None,
"action": "inserted",
"skip_image_sync": self._skip_image_sync(record),
}
if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
for key, value in payload.items():
setattr(car_by_url, key, value)
car_by_url.last_seen_at = record.last_seen_at
self._apply_update_payload(car_by_url, payload)
entry["car_id"] = int(car_by_url.id)
entry["action"] = "updated"
updated += 1
@@ -303,37 +318,54 @@ class PersistenceService:
raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
entry["car_id"] = car_id
car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
existing_images_map = self._load_existing_image_urls(session, car_ids)
images_by_car_id: dict[int, list[dict[str, object]]] = {}
replace_ids: list[int] = []
insert_images_by_car_id: dict[int, list[dict[str, object]]] = {}
updated_entries_needing_compare: list[dict[str, object]] = []
for entry in entries:
car_id = int(entry["car_id"])
action = str(entry.get("action") or "updated")
images = entry["images"]
if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and action == "updated":
skip_image_sync = bool(entry.get("skip_image_sync"))
if action == "updated" and (MOBILEDE_SKIP_IMAGES_FOR_UPDATED or skip_image_sync):
continue
new_image_urls = {
str(img.get("fullres_image", ""))
for img in images
if img.get("fullres_image")
}
old_image_urls = existing_images_map.get(car_id, set())
if new_image_urls != old_image_urls:
replace_ids.append(car_id)
images_by_car_id[car_id] = images
else:
images_total += len(old_image_urls)
if action == "inserted":
insert_images_by_car_id[car_id] = images
continue
updated_entries_needing_compare.append(entry)
if replace_ids:
for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
session.execute(delete(Image).where(Image.car_id.in_(chunk)))
for car_id in replace_ids:
images = images_by_car_id[car_id]
self._add_images(session, car_id, images)
images_total += len(images)
for car_id, images in insert_images_by_car_id.items():
self._add_images(session, car_id, images)
images_total += len(images)
if updated_entries_needing_compare:
car_ids = {int(entry["car_id"]) for entry in updated_entries_needing_compare}
existing_images_map = self._load_existing_image_urls(session, car_ids)
images_by_car_id: dict[int, list[dict[str, object]]] = {}
replace_ids: list[int] = []
for entry in updated_entries_needing_compare:
car_id = int(entry["car_id"])
images = entry["images"]
new_image_urls = {
str(img.get("fullres_image", ""))
for img in images
if img.get("fullres_image")
}
old_image_urls = existing_images_map.get(car_id, set())
if new_image_urls != old_image_urls:
replace_ids.append(car_id)
images_by_car_id[car_id] = images
else:
images_total += len(old_image_urls)
if replace_ids:
for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
session.execute(delete(Image).where(Image.car_id.in_(chunk)))
for car_id in replace_ids:
images = images_by_car_id[car_id]
self._add_images(session, car_id, images)
images_total += len(images)
return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
@@ -347,9 +379,7 @@ class PersistenceService:
select(Car).where(Car.origin_url == record.origin_url)
).scalar_one_or_none()
if car_by_url is not None and car_by_url.origin_id != record.origin_id:
for key, value in payload.items():
setattr(car_by_url, key, value)
car_by_url.last_seen_at = record.last_seen_at
self._apply_update_payload(car_by_url, payload)
session.flush()
car_id = int(car_by_url.id)
action = "updated"
@@ -380,9 +410,7 @@ class PersistenceService:
session.flush()
else:
action = "updated"
for key, value in payload.items():
setattr(car, key, value)
car.last_seen_at = record.last_seen_at
self._apply_update_payload(car, payload)
session.flush()
images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
@@ -436,18 +464,8 @@ class PersistenceService:
existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
# Предзагружаем изображения.
existing_car_ids = set()
for record in records:
car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
if car is not None:
existing_car_ids.add(int(car.id))
# Готовим map car_id -> image_urls.
existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
new_cars: list[tuple[Car, list[dict]]] = []
update_cars_needing_images: list[tuple[Car, list[dict]]] = []
update_image_candidates: list[tuple[Car, list[dict]]] = []
for record in records:
payload = self._car_payload(record)
@@ -460,18 +478,11 @@ class PersistenceService:
inserted += 1
new_cars.append((car, images))
else:
for key, value in payload.items():
setattr(car, key, value)
car.last_seen_at = record.last_seen_at
self._apply_update_payload(car, payload)
updated += 1
# Проверяем изменения картинок.
new_image_urls = {img.get("fullres_image", "") for img in images}
old_image_urls = existing_images_map.get(int(car.id), set())
if new_image_urls != old_image_urls:
update_cars_needing_images.append((car, images))
else:
images_total += len(old_image_urls)
if MOBILEDE_SKIP_IMAGES_FOR_UPDATED or self._skip_image_sync(record):
continue
update_image_candidates.append((car, images))
# Один flush.
session.flush()
@@ -481,6 +492,18 @@ class PersistenceService:
self._add_images(session, int(car.id), images)
images_total += len(images)
update_cars_needing_images: list[tuple[Car, list[dict]]] = []
if update_image_candidates:
update_ids = [int(car.id) for car, _ in update_image_candidates]
existing_images_map = self._load_existing_image_urls(session, set(update_ids))
for car, images in update_image_candidates:
new_image_urls = {img.get("fullres_image", "") for img in images}
old_image_urls = existing_images_map.get(int(car.id), set())
if new_image_urls != old_image_urls:
update_cars_needing_images.append((car, images))
else:
images_total += len(old_image_urls)
# Обновляем только изменённые картинки.
if update_cars_needing_images:
update_ids = [int(car.id) for car, _ in update_cars_needing_images]
@@ -521,7 +544,7 @@ class PersistenceService:
.where(Car.origin_id.notin_(active_origin_ids))
.where(Car.is_sold == False) # noqa: E712
.where(_origin_prefix_filter(Car.origin_id))
.values(is_sold=True)
.values(is_sold=True, sold_at=datetime.now(timezone.utc))
)
result = session.execute(stmt)
count = result.rowcount or 0
@@ -598,7 +621,7 @@ class PersistenceService:
# Массовая пометка sold.
result = session.execute(text("""
UPDATE {car_table}
SET is_sold = TRUE
SET is_sold = TRUE, sold_at = NOW()
FROM (
SELECT c.id
FROM {car_table} c
@@ -616,7 +639,7 @@ class PersistenceService:
update(Car)
.where(Car.is_sold == False) # noqa: E712
.where(_origin_prefix_filter(Car.origin_id))
.values(is_sold=True)
.values(is_sold=True, sold_at=datetime.now(timezone.utc))
)
# Загружаем active URL.
all_active = session.execute(
@@ -638,7 +661,7 @@ class PersistenceService:
for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True, sold_at=datetime.now(timezone.utc)))
count = len(mark_ids)
if count:
@@ -699,7 +722,7 @@ class PersistenceService:
.where(_origin_prefix_filter(Car.origin_id, prefixes))
.where(Car.is_sold == False) # noqa: E712
.where(Car.last_seen_at < since_ts)
.values(is_sold=True)
.values(is_sold=True, sold_at=since_ts)
)
count = int(result.rowcount or 0)
if count:
@@ -762,3 +785,41 @@ class PersistenceService:
).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
)
return [str(row[0]) for row in result if row and row[0]]
def get_active_cars_batch_for_sold_probe(
self,
prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
*,
limit: int = 200,
newest_first: bool = True,
) -> list[tuple[int, str, datetime]]:
prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
order_by = Car.last_seen_at.desc() if newest_first else Car.last_seen_at.asc()
with self.session_scope() as session:
result = session.execute(
select(Car.id, Car.origin_url, Car.last_seen_at).where(
_origin_prefix_filter(Car.origin_id, prefixes),
Car.is_sold == False, # noqa: E712
).order_by(order_by).limit(limit)
)
return [
(int(row[0]), str(row[1]), row[2])
for row in result
if row and row[0] and row[1] and row[2]
]
def mark_cars_sold_by_ids(self, car_ids: list[int], *, sold_at: datetime | None = None) -> int:
if not car_ids:
return 0
ts = sold_at or datetime.now(timezone.utc)
with self.session_scope() as session:
result = session.execute(
update(Car)
.where(Car.id.in_(car_ids))
.where(Car.is_sold == False) # noqa: E712
.values(is_sold=True, sold_at=ts)
)
count = int(result.rowcount or 0)
if count:
logger.info("Marked %d cars as sold by explicit id probe", count)
return count

2
mobilede_scraper/storage/models.py
View File

@@ -54,7 +54,9 @@ class Car(Base):
rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
slug: Mapped[str] = mapped_column(String(), nullable=False)
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
sold_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")

5
mobilede_scraper/storage/schemas.py
View File

@@ -37,7 +37,10 @@ class CarRecord(BaseModel):
rental: bool = False
repair_history: bool = False
slug: str
first_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
sold_at: datetime | None = None
skip_image_sync: bool = False
images: list[ImageRecord] = Field(default_factory=list)
@@ -82,6 +85,8 @@ class CarRead(BaseModel):
rental: bool = False
repair_history: bool = False
slug: str = ""
first_seen_at: datetime | None = None
last_seen_at: datetime | None = None
sold_at: datetime | None = None
images: list[ImageRead] = Field(default_factory=list)

65
mobilede_scraper/worker/celery_app.py
View File

@@ -11,6 +11,7 @@ from redis import Redis
from ..core.config import settings
from ..core.logs import setup_logging
from .constants import GLOBAL_DB_PROGRESS_TS_KEY, GLOBAL_PROGRESS_TS_KEY
logger = logging.getLogger("mobilede_scraper.worker.celery_app")
STARTUP_SYNC_DISPATCH_KEY = "mobilede:state:startup_sync_dispatched"
@@ -23,6 +24,9 @@ def _env_bool(name: str, default: bool) -> bool:
return raw in {"1", "true", "yes", "on"}
MOBILEDE_BEAT_SYNC_ENABLED = _env_bool("MOBILEDE_BEAT_SYNC_ENABLED", True)
def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool:
now = int(time.time())
try:
@@ -43,6 +47,18 @@ def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 18
return False
def _has_recent_global_progress(redis_client: Redis, *, max_age_seconds: int = 300) -> bool:
now = int(time.time())
try:
progress_ts = int(redis_client.get(GLOBAL_PROGRESS_TS_KEY) or 0)
db_progress_ts = int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY) or 0)
except Exception:
logger.warning("Failed to inspect global startup progress keys", exc_info=True)
return False
freshest_ts = max(progress_ts, db_progress_ts)
return freshest_ts > 0 and now - freshest_ts <= max_age_seconds
@celery_setup_logging.connect
def _configure_logging(loglevel=None, **kwargs):
# Перехватываем логирование Celery и пишем только в stderr (Docker logs).
@@ -85,6 +101,25 @@ if _hard > _max_hard:
)
_hard = _max_hard
beat_schedule = {}
if MOBILEDE_BEAT_SYNC_ENABLED:
beat_schedule = {
"periodic-mobilede-sync-search": {
"task": "mobilede.sync_runtime_segments",
"schedule": settings.celery.beat_sync_interval_minutes * 60.0,
"args": (),
"kwargs": {
"delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
"use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
"continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
},
"options": {
"queue": MOBILEDE_SYNC_QUEUE,
"expires": settings.celery.beat_sync_interval_minutes * 60.0,
},
}
}
celery_app.conf.update(
task_serializer="json",
accept_content=["json"],
@@ -107,22 +142,7 @@ celery_app.conf.update(
result_expires=86400,
worker_redirect_stdouts=False,
worker_hijack_root_logger=False,
beat_schedule={
"periodic-mobilede-sync-search": {
"task": "mobilede.sync_runtime_segments",
"schedule": settings.celery.beat_sync_interval_minutes * 60.0,
"args": (),
"kwargs": {
"delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
"use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
"continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
},
"options": {
"queue": MOBILEDE_SYNC_QUEUE,
"expires": settings.celery.beat_sync_interval_minutes * 60.0,
},
}
},
beat_schedule=beat_schedule,
task_routes={
"mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE},
"mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE},
@@ -153,17 +173,24 @@ def _on_worker_ready(**kwargs):
)
has_fresh_progress = _has_fresh_active_progress(redis_client)
has_recent_global_progress = _has_recent_global_progress(redis_client)
has_live_progress = bool(has_fresh_progress or has_recent_global_progress)
try:
queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0)
except Exception:
queue_len = 0
if queue_len > 0:
if queue_len > 0 and has_live_progress:
logger.info("Worker ready: MOBILEDE_sync queue already has %d task(s); skip startup dispatch", queue_len)
return
if queue_len > 0 and not has_live_progress:
logger.warning(
"Worker ready: MOBILEDE_sync queue has %d task(s), but no fresh progress is visible; forcing runtime sync dispatch",
queue_len,
)
should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
if not should_dispatch and not has_fresh_progress:
if not should_dispatch and not has_live_progress:
redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
if should_dispatch:
@@ -182,7 +209,7 @@ def _on_worker_ready(**kwargs):
logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
return
logger.info("Worker ready — dispatching initial mobile.de sync_search task")
logger.info("Worker ready - dispatching initial mobile.de sync_runtime_segments task")
celery_app.send_task(
"mobilede.sync_runtime_segments",
kwargs={