qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Track sold cars

Browse Source
This commit is contained in:
qananasikq
2026-05-06 21:07:24 +03:00
parent 81e99a41e8
commit 519686c0b7
8 changed files with 322 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
mobilede_scraper/storage/db.py
View File

@@ -20,6 +20,7 @@ CAR_DB_FIELDS = {
col.key for col in Car.__table__.columns
if col.key not in ("id",)
}
CAR_UPDATE_FIELDS = CAR_DB_FIELDS - {"first_seen_at"}
_IN_CHUNK_SIZE = 5000
CAR_TABLE_NAME = Car.__tablename__
@@ -170,6 +171,16 @@ class PersistenceService:
payload = record.model_dump(mode="python")
return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
@staticmethod
def _apply_update_payload(car: Car, payload: dict[str, object]) -> None:
for key, value in payload.items():
if key in CAR_UPDATE_FIELDS:
setattr(car, key, value)
@staticmethod
def _skip_image_sync(record: CarRecord) -> bool:
return bool(getattr(record, "skip_image_sync", False))
def _is_postgres(self) -> bool:
return self.engine.dialect.name == "postgresql"
@@ -227,7 +238,7 @@ class PersistenceService:
def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
return {
key: getattr(insert_stmt.excluded, key)
for key in CAR_DB_FIELDS
for key in CAR_UPDATE_FIELDS
}
def _replace_images_for_car(
@@ -266,12 +277,16 @@ class PersistenceService:
images = [image.model_dump(mode="python") for image in record.images]
car_by_id = existing_by_id.get(record.origin_id)
car_by_url = existing_by_url.get(record.origin_url)
entry: dict[str, object] = {"record": record, "images": images, "car_id": None, "action": "inserted"}
entry: dict[str, object] = {
"record": record,
"images": images,
"car_id": None,
"action": "inserted",
"skip_image_sync": self._skip_image_sync(record),
}
if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
for key, value in payload.items():
setattr(car_by_url, key, value)
car_by_url.last_seen_at = record.last_seen_at
self._apply_update_payload(car_by_url, payload)
entry["car_id"] = int(car_by_url.id)
entry["action"] = "updated"
updated += 1
@@ -303,37 +318,54 @@ class PersistenceService:
raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
entry["car_id"] = car_id
car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
existing_images_map = self._load_existing_image_urls(session, car_ids)
images_by_car_id: dict[int, list[dict[str, object]]] = {}
replace_ids: list[int] = []
insert_images_by_car_id: dict[int, list[dict[str, object]]] = {}
updated_entries_needing_compare: list[dict[str, object]] = []
for entry in entries:
car_id = int(entry["car_id"])
action = str(entry.get("action") or "updated")
images = entry["images"]
if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and action == "updated":
skip_image_sync = bool(entry.get("skip_image_sync"))
if action == "updated" and (MOBILEDE_SKIP_IMAGES_FOR_UPDATED or skip_image_sync):
continue
new_image_urls = {
str(img.get("fullres_image", ""))
for img in images
if img.get("fullres_image")
}
old_image_urls = existing_images_map.get(car_id, set())
if new_image_urls != old_image_urls:
replace_ids.append(car_id)
images_by_car_id[car_id] = images
else:
images_total += len(old_image_urls)
if action == "inserted":
insert_images_by_car_id[car_id] = images
continue
updated_entries_needing_compare.append(entry)
if replace_ids:
for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
session.execute(delete(Image).where(Image.car_id.in_(chunk)))
for car_id in replace_ids:
images = images_by_car_id[car_id]
self._add_images(session, car_id, images)
images_total += len(images)
for car_id, images in insert_images_by_car_id.items():
self._add_images(session, car_id, images)
images_total += len(images)
if updated_entries_needing_compare:
car_ids = {int(entry["car_id"]) for entry in updated_entries_needing_compare}
existing_images_map = self._load_existing_image_urls(session, car_ids)
images_by_car_id: dict[int, list[dict[str, object]]] = {}
replace_ids: list[int] = []
for entry in updated_entries_needing_compare:
car_id = int(entry["car_id"])
images = entry["images"]
new_image_urls = {
str(img.get("fullres_image", ""))
for img in images
if img.get("fullres_image")
}
old_image_urls = existing_images_map.get(car_id, set())
if new_image_urls != old_image_urls:
replace_ids.append(car_id)
images_by_car_id[car_id] = images
else:
images_total += len(old_image_urls)
if replace_ids:
for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
session.execute(delete(Image).where(Image.car_id.in_(chunk)))
for car_id in replace_ids:
images = images_by_car_id[car_id]
self._add_images(session, car_id, images)
images_total += len(images)
return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
@@ -347,9 +379,7 @@ class PersistenceService:
select(Car).where(Car.origin_url == record.origin_url)
).scalar_one_or_none()
if car_by_url is not None and car_by_url.origin_id != record.origin_id:
for key, value in payload.items():
setattr(car_by_url, key, value)
car_by_url.last_seen_at = record.last_seen_at
self._apply_update_payload(car_by_url, payload)
session.flush()
car_id = int(car_by_url.id)
action = "updated"
@@ -380,9 +410,7 @@ class PersistenceService:
session.flush()
else:
action = "updated"
for key, value in payload.items():
setattr(car, key, value)
car.last_seen_at = record.last_seen_at
self._apply_update_payload(car, payload)
session.flush()
images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
@@ -436,18 +464,8 @@ class PersistenceService:
existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
# Предзагружаем изображения.
existing_car_ids = set()
for record in records:
car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
if car is not None:
existing_car_ids.add(int(car.id))
# Готовим map car_id -> image_urls.
existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
new_cars: list[tuple[Car, list[dict]]] = []
update_cars_needing_images: list[tuple[Car, list[dict]]] = []
update_image_candidates: list[tuple[Car, list[dict]]] = []
for record in records:
payload = self._car_payload(record)
@@ -460,18 +478,11 @@ class PersistenceService:
inserted += 1
new_cars.append((car, images))
else:
for key, value in payload.items():
setattr(car, key, value)
car.last_seen_at = record.last_seen_at
self._apply_update_payload(car, payload)
updated += 1
# Проверяем изменения картинок.
new_image_urls = {img.get("fullres_image", "") for img in images}
old_image_urls = existing_images_map.get(int(car.id), set())
if new_image_urls != old_image_urls:
update_cars_needing_images.append((car, images))
else:
images_total += len(old_image_urls)
if MOBILEDE_SKIP_IMAGES_FOR_UPDATED or self._skip_image_sync(record):
continue
update_image_candidates.append((car, images))
# Один flush.
session.flush()
@@ -481,6 +492,18 @@ class PersistenceService:
self._add_images(session, int(car.id), images)
images_total += len(images)
update_cars_needing_images: list[tuple[Car, list[dict]]] = []
if update_image_candidates:
update_ids = [int(car.id) for car, _ in update_image_candidates]
existing_images_map = self._load_existing_image_urls(session, set(update_ids))
for car, images in update_image_candidates:
new_image_urls = {img.get("fullres_image", "") for img in images}
old_image_urls = existing_images_map.get(int(car.id), set())
if new_image_urls != old_image_urls:
update_cars_needing_images.append((car, images))
else:
images_total += len(old_image_urls)
# Обновляем только изменённые картинки.
if update_cars_needing_images:
update_ids = [int(car.id) for car, _ in update_cars_needing_images]
@@ -521,7 +544,7 @@ class PersistenceService:
.where(Car.origin_id.notin_(active_origin_ids))
.where(Car.is_sold == False) # noqa: E712
.where(_origin_prefix_filter(Car.origin_id))
.values(is_sold=True)
.values(is_sold=True, sold_at=datetime.now(timezone.utc))
)
result = session.execute(stmt)
count = result.rowcount or 0
@@ -598,7 +621,7 @@ class PersistenceService:
# Массовая пометка sold.
result = session.execute(text("""
UPDATE {car_table}
SET is_sold = TRUE
SET is_sold = TRUE, sold_at = NOW()
FROM (
SELECT c.id
FROM {car_table} c
@@ -616,7 +639,7 @@ class PersistenceService:
update(Car)
.where(Car.is_sold == False) # noqa: E712
.where(_origin_prefix_filter(Car.origin_id))
.values(is_sold=True)
.values(is_sold=True, sold_at=datetime.now(timezone.utc))
)
# Загружаем active URL.
all_active = session.execute(
@@ -638,7 +661,7 @@ class PersistenceService:
for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True, sold_at=datetime.now(timezone.utc)))
count = len(mark_ids)
if count:
@@ -699,7 +722,7 @@ class PersistenceService:
.where(_origin_prefix_filter(Car.origin_id, prefixes))
.where(Car.is_sold == False) # noqa: E712
.where(Car.last_seen_at < since_ts)
.values(is_sold=True)
.values(is_sold=True, sold_at=since_ts)
)
count = int(result.rowcount or 0)
if count:
@@ -762,3 +785,41 @@ class PersistenceService:
).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
)
return [str(row[0]) for row in result if row and row[0]]
def get_active_cars_batch_for_sold_probe(
self,
prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
*,
limit: int = 200,
newest_first: bool = True,
) -> list[tuple[int, str, datetime]]:
prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
order_by = Car.last_seen_at.desc() if newest_first else Car.last_seen_at.asc()
with self.session_scope() as session:
result = session.execute(
select(Car.id, Car.origin_url, Car.last_seen_at).where(
_origin_prefix_filter(Car.origin_id, prefixes),
Car.is_sold == False, # noqa: E712
).order_by(order_by).limit(limit)
)
return [
(int(row[0]), str(row[1]), row[2])
for row in result
if row and row[0] and row[1] and row[2]
]
def mark_cars_sold_by_ids(self, car_ids: list[int], *, sold_at: datetime | None = None) -> int:
if not car_ids:
return 0
ts = sold_at or datetime.now(timezone.utc)
with self.session_scope() as session:
result = session.execute(
update(Car)
.where(Car.id.in_(car_ids))
.where(Car.is_sold == False) # noqa: E712
.values(is_sold=True, sold_at=ts)
)
count = int(result.rowcount or 0)
if count:
logger.info("Marked %d cars as sold by explicit id probe", count)
return count

2
mobilede_scraper/storage/models.py
View File

@@ -54,7 +54,9 @@ class Car(Base):
rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
slug: Mapped[str] = mapped_column(String(), nullable=False)
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
sold_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")

5
mobilede_scraper/storage/schemas.py
View File

@@ -37,7 +37,10 @@ class CarRecord(BaseModel):
rental: bool = False
repair_history: bool = False
slug: str
first_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
sold_at: datetime | None = None
skip_image_sync: bool = False
images: list[ImageRecord] = Field(default_factory=list)
@@ -82,6 +85,8 @@ class CarRead(BaseModel):
rental: bool = False
repair_history: bool = False
slug: str = ""
first_seen_at: datetime | None = None
last_seen_at: datetime | None = None
sold_at: datetime | None = None
images: list[ImageRead] = Field(default_factory=list)