refactor mobile.de parser, fix country mapping, update README

mobilede_scraper/storage/db.py

@@ -21,6 +21,17 @@ CAR_DB_FIELDS = {
 
 _IN_CHUNK_SIZE = 5000
 CAR_TABLE_NAME = Car.__tablename__
+MOBILEDE_ORIGIN_PREFIXES = ("mobile.de:", "mobilede:")
+
+
+def _origin_prefix_filter(column, prefixes: tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES):
+	"""Match all supported mobile.de origin_id prefixes.
+
+	Older records were stored as ``mobile.de:<id>`` while some newer helper code
+	used ``mobilede:<id>``. Cleanup and refresh queries must include both to avoid
+	leaving stale active cars in the DB.
+	"""
+	return or_(*[column.like(f"{prefix}%") for prefix in prefixes])
 
 
 class PersistenceService:
@@ -502,7 +513,7 @@ class PersistenceService:
 				update(Car)
 				.where(Car.origin_id.notin_(active_origin_ids))
 				.where(Car.is_sold == False)  # noqa: E712
-				.where(Car.origin_id.like("mobilede:%"))
+				.where(_origin_prefix_filter(Car.origin_id))
 				.values(is_sold=True)
 			)
 			result = session.execute(stmt)
@@ -533,7 +544,12 @@ class PersistenceService:
 			if is_postgres:
 				# Считаем кандидатов на sold.
 				total_active = session.execute(
-					text(f"SELECT count(*) FROM {CAR_TABLE_NAME} WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%%'")
+					text(f"""
+						SELECT count(*)
+						FROM {CAR_TABLE_NAME}
+						WHERE is_sold = FALSE
+						AND (origin_id LIKE 'mobile.de:%%' OR origin_id LIKE 'mobilede:%%')
+					""")
 				).scalar() or 0
 
 				if total_active == 0:
@@ -561,7 +577,7 @@ class PersistenceService:
 					LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
 					WHERE a.url IS NULL
 					AND c.is_sold = FALSE
-					AND c.origin_id LIKE 'mobilede:%%'
+					AND (c.origin_id LIKE 'mobile.de:%%' OR c.origin_id LIKE 'mobilede:%%')
 				""".format(car_table=CAR_TABLE_NAME))).scalar() or 0
 
 				# Защита от аномалии.
@@ -582,7 +598,7 @@ class PersistenceService:
 						LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
 						WHERE a.url IS NULL
 						AND c.is_sold = FALSE
-						AND c.origin_id LIKE 'mobilede:%%'
+						AND (c.origin_id LIKE 'mobile.de:%%' OR c.origin_id LIKE 'mobilede:%%')
 					) sub
 					WHERE {car_table}.id = sub.id
 				""".format(car_table=CAR_TABLE_NAME)))
@@ -592,13 +608,13 @@ class PersistenceService:
 				stmt = (
 					update(Car)
 					.where(Car.is_sold == False)  # noqa: E712
-					.where(Car.origin_id.like("mobilede:%"))
+					.where(_origin_prefix_filter(Car.origin_id))
 					.values(is_sold=True)
 				)
 				# Загружаем active URL.
 				all_active = session.execute(
 					select(Car.id, Car.origin_url).where(
-						Car.is_sold == False, Car.origin_id.like("mobilede:%")  # noqa: E712
+						Car.is_sold == False, _origin_prefix_filter(Car.origin_id)  # noqa: E712
 					)
 				).all()
 				mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
@@ -622,35 +638,99 @@ class PersistenceService:
 				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
 			return count
 
-	def get_all_origin_ids_for_lane(self, prefix: str = "mobilede:") -> set[str]:
+	def mark_sold_not_seen_since(
+		self,
+		since_ts: datetime,
+		*,
+		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
+		safety_ratio: float = 0.8,
+	) -> int:
+		"""Mark active cars as sold when they were not seen during a full refresh cycle.
+
+		Any unsold mobile.de car with ``last_seen_at < since_ts`` is considered absent
+		from the latest completed refresh cycle and can be marked as sold.
+		A safety guard prevents anomalous bulk updates.
+		"""
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
+		with self.session_scope() as session:
+			total_active = int(
+				session.execute(
+					select(text("count(*)")).select_from(Car).where(
+						_origin_prefix_filter(Car.origin_id, prefixes),
+						Car.is_sold == False,  # noqa: E712
+					)
+				).scalar()
+				or 0
+			)
+			if total_active <= 0:
+				return 0
+
+			would_mark = int(
+				session.execute(
+					select(text("count(*)")).select_from(Car).where(
+						_origin_prefix_filter(Car.origin_id, prefixes),
+						Car.is_sold == False,  # noqa: E712
+						Car.last_seen_at < since_ts,
+					)
+				).scalar()
+				or 0
+			)
+			if would_mark <= 0:
+				return 0
+
+			if total_active > 100 and would_mark > int(total_active * max(0.0, min(1.0, safety_ratio))):
+				logger.error(
+					"mark_sold(last_seen) safety abort: would mark %d/%d (%.0f%%) as sold",
+					would_mark,
+					total_active,
+					(would_mark / max(1, total_active)) * 100,
+				)
+				return 0
+
+			result = session.execute(
+				update(Car)
+				.where(_origin_prefix_filter(Car.origin_id, prefixes))
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.last_seen_at < since_ts)
+				.values(is_sold=True)
+			)
+			count = int(result.rowcount or 0)
+			if count:
+				logger.info("Marked %d cars as sold by last_seen cutoff=%s", count, since_ts.isoformat())
+			return count
+
+	def get_all_origin_ids_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> set[str]:
 		"""Возвращает все известные origin_id для заданного префикса.
 
 		Использует yield_per для потоковой загрузки при большом количестве записей.
 		"""
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
 		with self.session_scope() as session:
 			result = session.execute(
-				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
+				select(Car.origin_id).where(_origin_prefix_filter(Car.origin_id, prefixes)).execution_options(yield_per=10000)
 			)
 			return {str(row[0]) for row in result if row and row[0]}
 
-	def get_all_active_origin_urls_for_lane(self, prefix: str = "mobilede:") -> set[str]:
+	def get_all_active_origin_urls_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> set[str]:
 		"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
 		with self.session_scope() as session:
 			result = session.execute(
 				select(Car.origin_url).where(
-					Car.origin_id.like(f"{prefix}%"),
+					_origin_prefix_filter(Car.origin_id, prefixes),
 					Car.is_sold == False,  # noqa: E712
 				).execution_options(yield_per=10000)
 			)
 			return {str(row[0]) for row in result if row and row[0]}
 
-	def count_active_cars_for_lane(self, prefix: str = "mobilede:") -> int:
+	def count_active_cars_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> int:
 		"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
 		from sqlalchemy import func as sa_func
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
 		with self.session_scope() as session:
 			result = session.execute(
 				select(sa_func.count()).select_from(Car).where(
-					Car.origin_id.like(f"{prefix}%"),
+					_origin_prefix_filter(Car.origin_id, prefixes),
 					Car.is_sold == False,  # noqa: E712
 				)
 			)
@@ -658,7 +738,7 @@ class PersistenceService:
 
 	def get_active_origin_urls_batch_for_refresh(
 		self,
-		prefix: str = "mobilede:",
+		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
 		offset: int = 0,
 		limit: int = 500,
 	) -> list[str]:
@@ -666,10 +746,11 @@ class PersistenceService:
 
 		Сортировка по last_seen_at ASC — давно не обновлённые идут первыми.
 		"""
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
 		with self.session_scope() as session:
 			result = session.execute(
 				select(Car.origin_url).where(
-					Car.origin_id.like(f"{prefix}%"),
+					_origin_prefix_filter(Car.origin_id, prefixes),
 					Car.is_sold == False,  # noqa: E712
 				).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
 			)