qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix parser identifiers and locale

Browse Source
This commit is contained in:
qananasikq
2026-08-05 18:13:30 +03:00
parent 2cebb78316
commit fba7299747
5 changed files with 185 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
mobilede_scraper/storage/db.py
View File

@@ -1,5 +1,6 @@
import logging
import os
import re
from contextlib import contextmanager
from datetime import datetime, timezone
from typing import Any, Iterator
@@ -25,6 +26,7 @@ CAR_UPDATE_FIELDS = CAR_DB_FIELDS - {"first_seen_at"}
_IN_CHUNK_SIZE = 5000
CAR_TABLE_NAME = Car.__tablename__
MOBILEDE_ORIGIN_PREFIXES = ("mobile.de:", "mobilede:")
PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
def _origin_prefix_filter(column, prefixes: tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES):
@@ -175,6 +177,8 @@ class PersistenceService:
def _apply_update_payload(car: Car, payload: dict[str, object]) -> None:
for key, value in payload.items():
if key in CAR_UPDATE_FIELDS:
if key == "parser_id" and PARSER_ID_RE.fullmatch(str(car.parser_id or "")):
continue
setattr(car, key, value)
@staticmethod
@@ -277,6 +281,8 @@ class PersistenceService:
images = [image.model_dump(mode="python") for image in record.images]
car_by_id = existing_by_id.get(record.origin_id)
car_by_url = existing_by_url.get(record.origin_url)
if car_by_id is not None and PARSER_ID_RE.fullmatch(str(car_by_id.parser_id or "")):
payload["parser_id"] = car_by_id.parser_id
entry: dict[str, object] = {
"record": record,
"images": images,
@@ -375,6 +381,11 @@ class PersistenceService:
images = [image.model_dump(mode="python") for image in record.images]
with self.session_scope() as session:
if self._is_postgres():
car_by_id = session.execute(
select(Car).where(Car.origin_id == record.origin_id)
).scalar_one_or_none()
if car_by_id is not None and PARSER_ID_RE.fullmatch(str(car_by_id.parser_id or "")):
payload["parser_id"] = car_by_id.parser_id
car_by_url = session.execute(
select(Car).where(Car.origin_url == record.origin_url)
).scalar_one_or_none()
@@ -384,9 +395,7 @@ class PersistenceService:
car_id = int(car_by_url.id)
action = "updated"
else:
existed = session.execute(
select(Car.id).where(Car.origin_id == record.origin_id)
).scalar_one_or_none() is not None
existed = car_by_id is not None
insert_stmt = pg_insert(Car).values(**payload)
upsert_stmt = insert_stmt.on_conflict_do_update(
index_elements=[Car.origin_id],