qananasikq/dubizzle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update scraper package

Browse Source
This commit is contained in:
qananasikq
2026-04-24 20:46:58 +03:00
parent 6aba4f0dbd
commit d484088e18
59 changed files with 2548 additions and 1272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
tests/test_self_heal.py
View File

@@ -3,7 +3,7 @@ from __future__ import annotations
import unittest
from unittest.mock import MagicMock, patch
from iaai_scraper.worker import self_heal
from dubizzle_scraper.worker import self_heal
class TestSelfHeal(unittest.TestCase):
@@ -20,14 +20,14 @@ class TestSelfHeal(unittest.TestCase):
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: {
self_heal.GLOBAL_PROGRESS_TS_KEY: None,
"iaai:state:task_progress:a": '{"ts": 100}',
"iaai:state:task_progress:b": '{"ts": 250}',
"iaai:state:task_progress:c": '{"ts": 150}',
"dubizzle:state:task_progress:a": '{"ts": 100}',
"dubizzle:state:task_progress:b": '{"ts": 250}',
"dubizzle:state:task_progress:c": '{"ts": 150}',
}.get(key)
redis_client.scan_iter.return_value = [
"iaai:state:task_progress:a",
"iaai:state:task_progress:b",
"iaai:state:task_progress:c",
"dubizzle:state:task_progress:a",
"dubizzle:state:task_progress:b",
"dubizzle:state:task_progress:c",
]
ts = self_heal._read_last_progress_ts(redis_client)
@@ -38,20 +38,20 @@ class TestSelfHeal(unittest.TestCase):
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: {
self_heal.GLOBAL_PROGRESS_TS_KEY: None,
"iaai:state:task_progress:a": "{bad-json}",
"iaai:state:task_progress:b": '{"foo": "bar"}',
"dubizzle:state:task_progress:a": "{bad-json}",
"dubizzle:state:task_progress:b": '{"foo": "bar"}',
}.get(key)
redis_client.scan_iter.return_value = [
"iaai:state:task_progress:a",
"iaai:state:task_progress:b",
"dubizzle:state:task_progress:a",
"dubizzle:state:task_progress:b",
]
ts = self_heal._read_last_progress_ts(redis_client)
self.assertIsNone(ts)
@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("iaai_scraper.worker.self_heal.os.kill")
@patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("dubizzle_scraper.worker.self_heal.os.kill")
@patch("builtins.open")
def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
open_mock.return_value.__enter__.return_value.read.return_value = "123"
@@ -64,8 +64,8 @@ class TestSelfHeal(unittest.TestCase):
self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("iaai_scraper.worker.self_heal.os.kill")
@patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("dubizzle_scraper.worker.self_heal.os.kill")
@patch("builtins.open")
def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
open_mock.return_value.__enter__.return_value.read.return_value = "123"