add tests

This commit is contained in:
qananasikq
2026-04-16 18:01:37 +03:00
parent 8140fb6f41
commit 80f7a1353d
5 changed files with 520 additions and 41 deletions

View File

@@ -1,4 +1,4 @@
from __future__ import annotations from __future__ import annotations
import tempfile import tempfile
import unittest import unittest
@@ -6,10 +6,10 @@ from pathlib import Path
from sqlalchemy import select from sqlalchemy import select
from iaai_scraper.core.config import Settings from encar_scraper.core.config import Settings
from iaai_scraper.storage.db import PersistenceService from encar_scraper.storage.db import PersistenceService
from iaai_scraper.storage.models import Car, Image, SyncRun from encar_scraper.storage.models import Car, Image, SyncRun
from iaai_scraper.storage.schemas import CarRecord, ImageRecord from encar_scraper.storage.schemas import CarRecord, ImageRecord
class TestPersistenceServiceIntegration(unittest.TestCase): class TestPersistenceServiceIntegration(unittest.TestCase):
@@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
@staticmethod @staticmethod
def _record(origin_id: str, *, price: int = 1000) -> CarRecord: def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
return CarRecord( return CarRecord(
parser_id=f"iaai:{origin_id}", parser_id=f"encar:{origin_id}",
brand="Toyota", brand="Toyota",
model="Camry", model="Camry",
year=2014, year=2014,
price=price, price=price,
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", origin_url=f"https://www.encar.com/dc/dc_cardetailview.do?carid={origin_id}",
origin_id=origin_id, origin_id=origin_id,
slug=f"toyota-camry-{origin_id}", slug=f"toyota-camry-{origin_id}",
images=[ images=[
ImageRecord( ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", fullres_image=f"https://www.encar.com/carpicture01/pic{origin_id}_001.jpg",
preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300", preview_image=f"https://www.encar.com/carpicture01/pic{origin_id}_001.jpg",
order_index=0, order_index=0,
) )
], ],
@@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
second = self._record("888") second = self._record("888")
second.images = [ second.images = [
ImageRecord( ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633", fullres_image="https://www.encar.com/carpicture01/pic888_002.jpg",
preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", preview_image="https://www.encar.com/carpicture01/pic888_002.jpg",
order_index=0, order_index=0,
) )
] ]
@@ -90,7 +90,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
images = session.execute(select(Image)).scalars().all() images = session.execute(select(Image)).scalars().all()
self.assertEqual(len(images), 1) self.assertEqual(len(images), 1)
self.assertIn("imageKeys=2", images[0].fullres_image) self.assertIn("pic888_002.jpg", images[0].fullres_image)
def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None: def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
first_run_id = self.persistence.start_sync_run("lane-a") first_run_id = self.persistence.start_sync_run("lane-a")
@@ -108,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None: def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
first = self._record("OLD-ID") first = self._record("OLD-ID")
first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" first.origin_url = "https://www.encar.com/dc/dc_cardetailview.do?carid=45089484"
self.persistence.upsert_car(first) self.persistence.upsert_car(first)
second = self._record("NEW-ID") second = self._record("NEW-ID")
second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" second.origin_url = "https://www.encar.com/dc/dc_cardetailview.do?carid=45089484"
result = self.persistence.upsert_car(second) result = self.persistence.upsert_car(second)
self.assertEqual(result["action"], "updated") self.assertEqual(result["action"], "updated")

62
tests/test_encar.py Normal file
View File

@@ -0,0 +1,62 @@
import unittest
from unittest.mock import MagicMock
from encar_scraper.encar import EncarMapper, EncarScraper, ENCAR_DETAIL_URL_TEMPLATE
class TestEncarMapper(unittest.TestCase):
def test_map_to_car_record(self):
payload = {
"Id": 41421262,
"Manufacturer": "Hyundai",
"Model": "G90 (RS4)",
"FormYear": "2023",
"Price": 8590,
"Mileage": 37923,
"Photos": [
{"location": "/carpicture01/pic4141/41412663_001.jpg", "ordering": 1},
{"location": "/carpicture01/pic4141/41412663_003.jpg", "ordering": 3},
],
}
record = EncarMapper().map_to_car_record(
ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="41421262"),
payload,
)
self.assertEqual(record.origin_id, "encar:41421262")
self.assertEqual(record.parser_id, "encar:41421262")
self.assertEqual(record.brand, "Hyundai")
self.assertEqual(record.model, "G90 (RS4)")
self.assertEqual(record.year, 2023)
self.assertEqual(record.price, 85900000) # 8590 만원 × 10000
self.assertEqual(record.mileage, 37923)
self.assertEqual(record.country, "KR")
self.assertEqual(record.currency, "KRW")
self.assertEqual(len(record.images), 2)
self.assertTrue(record.images[0].fullres_image.endswith("_001.jpg"))
def test_extract_vehicle_id_from_url(self):
url = ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="41421262")
self.assertEqual(EncarMapper()._extract_vehicle_id(url), "41421262")
class TestEncarScraper(unittest.TestCase):
def test_collect_listing_limits_results(self):
scraper = EncarScraper()
scraper._fetch_listing_page = MagicMock(side_effect=[
{"SearchResults": [{"Id": 1}, {"Id": 2}]},
])
result = scraper.collect_listing(limit=1, page_size=2)
self.assertEqual(result["items_collected"], 1)
self.assertEqual(result["vehicle_urls"], [ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="1")])
def test_scrape_vehicle_detail_raises_without_id(self):
scraper = EncarScraper()
with self.assertRaises(ValueError):
scraper.scrape_vehicle_detail("https://www.encar.com/invalid")
if __name__ == "__main__":
unittest.main()

414
tests/test_load.py Normal file
View File

@@ -0,0 +1,414 @@
"""
Нагрузочные тесты: API endpoints, batch upsert, конкурентный доступ к БД.
Запуск:
pytest tests/test_load.py -v -s
"""
from __future__ import annotations
import concurrent.futures
import time
import random
import string
import unittest
from datetime import datetime, timezone
from fastapi.testclient import TestClient
from encar_scraper.api.app import create_app
from encar_scraper.core.config import DatabaseConfig, RedisConfig, Settings
from encar_scraper.storage.db import PersistenceService
from encar_scraper.storage.schemas import CarRecord, ImageRecord
# ── helpers ──────────────────────────────────────────────────────────────
def _sqlite_settings() -> Settings:
return Settings(
database=DatabaseConfig(url="sqlite:///test_load.db", echo=False, auto_create_tables=True),
redis=RedisConfig(url="redis://localhost:6379/15"),
)
def _make_car(idx: int, *, images: int = 3) -> CarRecord:
uid = f"load-{idx}-{''.join(random.choices(string.ascii_lowercase, k=4))}"
return CarRecord(
parser_id=uid,
brand=random.choice(["Hyundai", "Kia", "BMW", "Mercedes", "Genesis"]),
model=f"Model-{idx % 50}",
year=random.randint(2010, 2025),
price=random.randint(5_000_000, 80_000_000),
currency="KRW",
mileage=random.randint(0, 300_000),
country="KR",
origin="ENCAR",
origin_url=f"https://encar.com/cars/{uid}",
origin_id=uid,
slug=uid,
selling_type="NA",
body_type="SEDAN",
last_seen_at=datetime.now(timezone.utc),
images=[
ImageRecord(
fullres_image=f"https://ci.encar.com/photo/{uid}/{j}.jpg",
preview_image=f"https://ci.encar.com/photo/{uid}/{j}_thumb.jpg",
order_index=j,
)
for j in range(images)
],
)
# ── DB load tests ───────────────────────────────────────────────────────
class TestDatabaseLoad(unittest.TestCase):
"""Тесты производительности PersistenceService на SQLite."""
@classmethod
def setUpClass(cls):
import os
if os.path.exists("test_load.db"):
os.remove("test_load.db")
cls.settings = _sqlite_settings()
cls.persistence = PersistenceService(cls.settings)
cls.persistence.create_tables()
@classmethod
def tearDownClass(cls):
cls.persistence.engine.dispose()
import os
if os.path.exists("test_load.db"):
os.remove("test_load.db")
def test_batch_upsert_500_cars(self):
"""Пакетная вставка 500 авто с изображениями — должна пройти за <10 сек."""
records = [_make_car(i, images=5) for i in range(500)]
t0 = time.perf_counter()
result = self.persistence.upsert_cars_batch(records)
elapsed = time.perf_counter() - t0
print(f"\n[batch_upsert_500] inserted={result['inserted']}, "
f"updated={result['updated']}, images={result['images_upserted']}, "
f"time={elapsed:.2f}s")
self.assertEqual(result["inserted"], 500)
self.assertEqual(result["images_upserted"], 2500)
self.assertLess(elapsed, 10, "Batch upsert 500 cars should complete in <10s")
def test_batch_upsert_update_existing(self):
"""Повторный upsert 200 существующих записей — обновление без дублей."""
records = [_make_car(i, images=3) for i in range(10000, 10200)]
self.persistence.upsert_cars_batch(records)
# Обновляем цену и перезаписываем
for r in records:
r.price = 99_999_999
t0 = time.perf_counter()
result = self.persistence.upsert_cars_batch(records)
elapsed = time.perf_counter() - t0
print(f"\n[batch_update_200] updated={result['updated']}, time={elapsed:.2f}s")
self.assertEqual(result["updated"], 200)
self.assertLess(elapsed, 10)
def test_concurrent_upsert_batches(self):
"""3 параллельных потока по 100 upsert-ов — нет deadlock / corruption."""
def worker(thread_id: int):
batch = [_make_car(thread_id * 10000 + i) for i in range(100)]
return self.persistence.upsert_cars_batch(batch)
t0 = time.perf_counter()
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
futures = [pool.submit(worker, tid) for tid in range(3)]
results = [f.result() for f in concurrent.futures.as_completed(futures)]
elapsed = time.perf_counter() - t0
total_inserted = sum(r["inserted"] for r in results)
print(f"\n[concurrent_upsert] total_inserted={total_inserted}, time={elapsed:.2f}s")
self.assertEqual(total_inserted, 300)
self.assertLess(elapsed, 30)
def test_single_upsert_throughput(self):
"""50 последовательных upsert_car — замер throughput."""
records = [_make_car(50000 + i, images=2) for i in range(50)]
t0 = time.perf_counter()
for r in records:
self.persistence.upsert_car(r)
elapsed = time.perf_counter() - t0
rps = 50 / elapsed if elapsed > 0 else 0
print(f"\n[single_upsert_50] time={elapsed:.2f}s, rps={rps:.1f}")
self.assertLess(elapsed, 15, "50 individual upserts should complete in <15s")
def test_mark_sold_large_set(self):
"""mark_sold на 300 origin_id — должен быть быстрым."""
records = [_make_car(60000 + i, images=1) for i in range(300)]
self.persistence.upsert_cars_batch(records)
active_ids = {r.origin_id for r in records[:150]}
t0 = time.perf_counter()
sold = self.persistence.mark_sold_not_in_listing(active_origin_ids=active_ids, lane="encar")
elapsed = time.perf_counter() - t0
print(f"\n[mark_sold_300] marked_sold={sold}, time={elapsed:.2f}s")
self.assertLess(elapsed, 5)
def test_sync_run_lifecycle(self):
"""start → finish sync_run 50 раз — измеряем overhead."""
t0 = time.perf_counter()
for _ in range(50):
run_id = self.persistence.start_sync_run("encar")
self.persistence.finish_sync_run(
run_id,
status="completed",
ids_fetched=1000,
cars_upserted=950,
cars_failed=50,
images_upserted=15000,
)
elapsed = time.perf_counter() - t0
print(f"\n[sync_run_lifecycle_50] time={elapsed:.2f}s")
self.assertLess(elapsed, 5)
# ── API load tests ──────────────────────────────────────────────────────
class TestAPILoad(unittest.TestCase):
"""Нагрузочные тесты на FastAPI endpoints через TestClient."""
@classmethod
def setUpClass(cls):
import os
if os.path.exists("test_api_load.db"):
os.remove("test_api_load.db")
settings = Settings(
database=DatabaseConfig(url="sqlite:///test_api_load.db", echo=False, auto_create_tables=True),
redis=RedisConfig(url="redis://localhost:6379/15"),
)
app = create_app(settings)
app.state.persistence.create_tables()
# Предзаполняем БД 200 авто
persistence: PersistenceService = app.state.persistence
persistence.upsert_cars_batch([_make_car(i, images=4) for i in range(200)])
cls.client = TestClient(app)
@classmethod
def tearDownClass(cls):
import os
if os.path.exists("test_api_load.db"):
os.remove("test_api_load.db")
def test_health_check_rapid(self):
"""100 GET /health подряд — все 200, <5 сек."""
t0 = time.perf_counter()
for _ in range(100):
r = self.client.get("/health")
self.assertEqual(r.status_code, 200)
elapsed = time.perf_counter() - t0
print(f"\n[health_100] time={elapsed:.2f}s, rps={100 / elapsed:.0f}")
self.assertLess(elapsed, 5)
def test_list_cars_paginated_rapid(self):
"""100 GET /api/v1/cars с разными страницами — все 200, <10 сек."""
t0 = time.perf_counter()
for page in range(1, 101):
r = self.client.get("/api/v1/cars", params={"page": (page % 10) + 1, "per_page": 20})
self.assertEqual(r.status_code, 200)
data = r.json()
self.assertIn("items", data)
elapsed = time.perf_counter() - t0
print(f"\n[list_cars_100] time={elapsed:.2f}s, rps={100 / elapsed:.0f}")
self.assertLess(elapsed, 10)
def test_list_cars_with_filters(self):
"""50 GET /api/v1/cars с brand/year фильтрами."""
t0 = time.perf_counter()
for i in range(50):
brand = random.choice(["Hyundai", "Kia", "BMW", "Mercedes", "Genesis"])
r = self.client.get(
"/api/v1/cars",
params={"brand": brand, "year_min": 2015, "year_max": 2023, "per_page": 50},
)
self.assertEqual(r.status_code, 200)
elapsed = time.perf_counter() - t0
print(f"\n[filtered_cars_50] time={elapsed:.2f}s, rps={50 / elapsed:.0f}")
self.assertLess(elapsed, 10)
def test_get_car_by_id_rapid(self):
"""50 GET /api/v1/cars/{id} — все 200."""
t0 = time.perf_counter()
for car_id in range(1, 51):
r = self.client.get(f"/api/v1/cars/{car_id}")
self.assertIn(r.status_code, (200, 404))
elapsed = time.perf_counter() - t0
print(f"\n[get_car_50] time={elapsed:.2f}s, rps={50 / elapsed:.0f}")
self.assertLess(elapsed, 10)
def test_stats_endpoint_rapid(self):
"""50 GET /api/v1/stats — агрегация по всей таблице."""
t0 = time.perf_counter()
for _ in range(50):
r = self.client.get("/api/v1/stats")
self.assertEqual(r.status_code, 200)
data = r.json()
self.assertGreater(data["total_cars"], 0)
elapsed = time.perf_counter() - t0
print(f"\n[stats_50] time={elapsed:.2f}s, rps={50 / elapsed:.0f}")
self.assertLess(elapsed, 10)
def test_concurrent_api_requests(self):
"""20 конкурентных запросов к разным endpoints — нет 500-х."""
def make_request(idx: int):
if idx % 3 == 0:
return self.client.get("/health")
elif idx % 3 == 1:
return self.client.get("/api/v1/cars", params={"page": 1, "per_page": 10})
else:
return self.client.get("/api/v1/stats")
t0 = time.perf_counter()
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool:
futures = [pool.submit(make_request, i) for i in range(20)]
responses = [f.result() for f in concurrent.futures.as_completed(futures)]
elapsed = time.perf_counter() - t0
for r in responses:
self.assertNotEqual(r.status_code, 500, f"Got 500: {r.text}")
print(f"\n[concurrent_api_20] time={elapsed:.2f}s, all_ok={all(r.status_code == 200 for r in responses)}")
self.assertLess(elapsed, 10)
def test_large_page_size(self):
"""GET /api/v1/cars с per_page=100 — проверяем сериализацию больших ответов."""
t0 = time.perf_counter()
for _ in range(20):
r = self.client.get("/api/v1/cars", params={"page": 1, "per_page": 100})
self.assertEqual(r.status_code, 200)
data = r.json()
# Каждый car содержит images — проверяем что сериализация прошла
for item in data["items"]:
self.assertIn("images", item)
elapsed = time.perf_counter() - t0
print(f"\n[large_page_20] time={elapsed:.2f}s, rps={20 / elapsed:.0f}")
self.assertLess(elapsed, 10)
# ── Data integrity under load ───────────────────────────────────────────
class TestDataIntegrity(unittest.TestCase):
"""Проверка целостности данных при массовых операциях."""
@classmethod
def setUpClass(cls):
import os
if os.path.exists("test_integrity.db"):
os.remove("test_integrity.db")
cls.settings = Settings(
database=DatabaseConfig(url="sqlite:///test_integrity.db", echo=False, auto_create_tables=True),
redis=RedisConfig(url="redis://localhost:6379/15"),
)
cls.persistence = PersistenceService(cls.settings)
cls.persistence.create_tables()
@classmethod
def tearDownClass(cls):
cls.persistence.engine.dispose()
import os
if os.path.exists("test_integrity.db"):
os.remove("test_integrity.db")
def test_no_duplicates_after_repeated_upserts(self):
"""3 раза upsert одного батча — ровно N уникальных записей в БД."""
records = [_make_car(90000 + i) for i in range(100)]
for _ in range(3):
self.persistence.upsert_cars_batch(records)
from sqlalchemy import select, func
from encar_scraper.storage.models import Car
with self.persistence.session_scope() as session:
count = session.execute(
select(func.count(Car.id)).where(Car.origin_id.like("load-90%"))
).scalar()
self.assertEqual(count, 100, f"Expected 100 unique cars, got {count}")
def test_images_replaced_not_accumulated(self):
"""При upsert с новыми images — старые удаляются, а не копятся."""
rec = _make_car(80001, images=5)
self.persistence.upsert_car(rec)
# Обновляем с 2 изображениями
rec.images = [
ImageRecord(fullres_image="https://new/1.jpg", preview_image="https://new/1t.jpg", order_index=0),
ImageRecord(fullres_image="https://new/2.jpg", preview_image="https://new/2t.jpg", order_index=1),
]
self.persistence.upsert_car(rec)
from sqlalchemy import select, func
from encar_scraper.storage.models import Car, Image
with self.persistence.session_scope() as session:
car = session.execute(select(Car).where(Car.origin_id == rec.origin_id)).scalar_one()
img_count = session.execute(
select(func.count(Image.id)).where(Image.car_id == car.id)
).scalar()
self.assertEqual(img_count, 2, f"Expected 2 images after replace, got {img_count}")
def test_dedup_within_batch(self):
"""Батч с дубликатами origin_id — дедупликация работает."""
base = _make_car(70001)
dupe = _make_car(70001)
dupe.parser_id = base.parser_id
dupe.origin_id = base.origin_id
dupe.origin_url = base.origin_url
dupe.price = 12345
records = [base, dupe]
result = self.persistence.upsert_cars_batch(records)
self.assertEqual(result["inserted"] + result["updated"], 1,
"Dedup should reduce 2 records to 1")
def test_batch_1000_cars_integrity(self):
"""Вставка 1000 авто — проверяем что ВСЕ записи в БД с верными images."""
records = [_make_car(200000 + i, images=3) for i in range(1000)]
result = self.persistence.upsert_cars_batch(records)
self.assertEqual(result["inserted"], 1000)
self.assertEqual(result["images_upserted"], 3000)
from sqlalchemy import select, func
from encar_scraper.storage.models import Car, Image
with self.persistence.session_scope() as session:
car_count = session.execute(
select(func.count(Car.id)).where(Car.origin_id.like("load-20____-%"))
).scalar()
img_count = session.execute(
select(func.count(Image.id)).where(
Image.car_id.in_(
select(Car.id).where(Car.origin_id.like("load-20____-%"))
)
)
).scalar()
self.assertEqual(car_count, 1000)
self.assertEqual(img_count, 3000)
if __name__ == "__main__":
unittest.main()

View File

@@ -1,8 +1,8 @@
from __future__ import annotations from __future__ import annotations
import unittest import unittest
from iaai_scraper.core.utils import deep_find_key from encar_scraper.core.utils import deep_find_key
class TestDeepFindKey(unittest.TestCase): class TestDeepFindKey(unittest.TestCase):

View File

@@ -1,9 +1,9 @@
from __future__ import annotations from __future__ import annotations
import unittest import unittest
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
from iaai_scraper.worker import tasks from encar_scraper.worker import tasks
class TestWorkerTaskLockHelpers(unittest.TestCase): class TestWorkerTaskLockHelpers(unittest.TestCase):
@@ -18,25 +18,26 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
def test_refresh_lock_if_owner_extends_ttl(self) -> None: def test_refresh_lock_if_owner_extends_ttl(self) -> None:
redis_client = MagicMock() redis_client = MagicMock()
redis_client.eval.return_value = 1 mock_script = MagicMock(return_value=1)
redis_client.register_script.return_value = mock_script
refreshed = tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120) refreshed = tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)
self.assertTrue(refreshed) self.assertTrue(refreshed)
redis_client.eval.assert_called_once() redis_client.register_script.assert_called_once()
mock_script.assert_called_once_with(keys=["lock:key"], args=["owner-token", 120])
def test_release_lock_if_owner_uses_owner_token(self) -> None: def test_release_lock_if_owner_uses_owner_token(self) -> None:
redis_client = MagicMock() redis_client = MagicMock()
mock_script = MagicMock(return_value=1)
redis_client.register_script.return_value = mock_script
tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token") tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
redis_client.eval.assert_called_once() redis_client.register_script.assert_called_once()
args = redis_client.eval.call_args[0] mock_script.assert_called_once_with(keys=["lock:key"], args=["owner-token"])
self.assertEqual(args[1], 1)
self.assertEqual(args[2], "lock:key")
self.assertEqual(args[3], "owner-token")
def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None: def test_encar_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
with patch.object(tasks, "_get_persistence") as get_persistence, \ with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=False): patch.object(tasks, "_acquire_lock", return_value=False):
@@ -44,30 +45,31 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
get_persistence.return_value = persistence get_persistence.return_value = persistence
get_redis.return_value = MagicMock() get_redis.return_value = MagicMock()
tasks.sync_listing_task.push_request(id="task-123") tasks.encar_sync_listing_task.push_request(id="task-123")
try: try:
result = tasks.sync_listing_task.run() result = tasks.encar_sync_listing_task.run()
finally: finally:
tasks.sync_listing_task.pop_request() tasks.encar_sync_listing_task.pop_request()
persistence.create_tables.assert_called_once() persistence.create_tables.assert_called_once()
self.assertEqual(result["status"], "skipped") self.assertEqual(result["status"], "skipped")
self.assertEqual(result["reason"], "sync_already_running") self.assertEqual(result["reason"], "sync_already_running")
def test_sync_listing_task_releases_owned_lock(self) -> None: def test_encar_sync_listing_task_releases_owned_lock(self) -> None:
mock_scraper_instance = MagicMock()
mock_scraper_instance.sync_listing.return_value = {
"total_available": 100,
"items_collected": 10,
"cars_synced": 8,
"cars_failed": 2,
}
with patch.object(tasks, "_get_persistence") as get_persistence, \ with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", return_value={ patch("encar_scraper.worker.tasks.EncarScraper", return_value=mock_scraper_instance):
"run_id": 7,
"cars_upserted": 2,
"cars_failed": 0,
"images_upserted": 4,
"skipped_existing": 1,
"elapsed_seconds": 1.25,
}):
persistence = MagicMock() persistence = MagicMock()
get_persistence.return_value = persistence get_persistence.return_value = persistence
redis_client = MagicMock() redis_client = MagicMock()
@@ -76,11 +78,12 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
heartbeat_thread = MagicMock() heartbeat_thread = MagicMock()
start_heartbeat.return_value = (stop_event, heartbeat_thread) start_heartbeat.return_value = (stop_event, heartbeat_thread)
tasks.sync_listing_task.push_request(id="task-123") tasks.encar_sync_listing_task.push_request(id="task-123")
try: try:
result = tasks.sync_listing_task.run(make="Toyota") with patch.object(tasks.encar_sync_listing_task, "update_state"):
result = tasks.encar_sync_listing_task.run(car_type="all")
finally: finally:
tasks.sync_listing_task.pop_request() tasks.encar_sync_listing_task.pop_request()
self.assertEqual(result["status"], "success") self.assertEqual(result["status"], "success")
stop_event.set.assert_called_once() stop_event.set.assert_called_once()