qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix self heal stale progress cleanup

Browse Source
This commit is contained in:
qananasikq
2026-04-27 13:09:34 +03:00
parent 331d8d350e
commit 993f692254
2 changed files with 134 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
tests/test_self_heal.py
View File

@@ -72,6 +72,98 @@ class TestSelfHeal(unittest.TestCase):
self.assertTrue(active)
@patch("iaai_scraper.worker.self_heal.time.time", return_value=5000)
def test_has_inflight_work_deletes_stale_terminal_progress_without_work(self, _time_mock) -> None:
redis_client = MagicMock()
redis_client.llen.return_value = 0
redis_client.get.side_effect = lambda key: {
self_heal.SYNC_LISTING_LOCK_KEY: None,
"iaai:state:task_progress:old": json.dumps(
{
"task_id": "old",
"stage": "segment_done",
"ts": 1000,
"segment_full_scan_completed": True,
}
),
}.get(key)
redis_client.scan_iter.return_value = ["iaai:state:task_progress:old"]
has_inflight, flags = self_heal._has_inflight_work(redis_client, self_heal.IAAI_SYNC_QUEUE)
self.assertFalse(has_inflight)
self.assertEqual(flags["has_task_progress"], 0)
redis_client.delete.assert_called_once_with("iaai:state:task_progress:old")
@patch("iaai_scraper.worker.self_heal.time.time", return_value=1010)
def test_has_inflight_work_keeps_recent_non_terminal_progress(self, _time_mock) -> None:
redis_client = MagicMock()
redis_client.llen.return_value = 0
redis_client.get.side_effect = lambda key: {
self_heal.SYNC_LISTING_LOCK_KEY: None,
"iaai:state:task_progress:active": json.dumps(
{
"task_id": "active",
"stage": "fast_listing_collected",
"ts": 1000,
}
),
}.get(key)
redis_client.scan_iter.return_value = ["iaai:state:task_progress:active"]
has_inflight, flags = self_heal._has_inflight_work(redis_client, self_heal.IAAI_SYNC_QUEUE)
self.assertTrue(has_inflight)
self.assertEqual(flags["has_task_progress"], 1)
redis_client.delete.assert_not_called()
@patch("iaai_scraper.worker.self_heal.time.time", return_value=5000)
def test_has_inflight_work_deletes_stale_non_terminal_progress_without_work(self, _time_mock) -> None:
redis_client = MagicMock()
redis_client.llen.return_value = 0
redis_client.get.side_effect = lambda key: {
self_heal.SYNC_LISTING_LOCK_KEY: None,
"iaai:state:task_progress:stale": json.dumps(
{
"task_id": "stale",
"stage": "segment_started",
"ts": 1000,
"segment_index": 11,
}
),
}.get(key)
redis_client.scan_iter.return_value = ["iaai:state:task_progress:stale"]
has_inflight, flags = self_heal._has_inflight_work(redis_client, self_heal.IAAI_SYNC_QUEUE)
self.assertFalse(has_inflight)
self.assertEqual(flags["has_task_progress"], 0)
self.assertEqual(flags["stale_progress_deleted"], 1)
redis_client.delete.assert_called_once_with("iaai:state:task_progress:stale")
@patch("iaai_scraper.worker.self_heal.time.time", return_value=5000)
def test_has_inflight_work_keeps_stale_progress_when_lock_exists(self, _time_mock) -> None:
redis_client = MagicMock()
redis_client.llen.return_value = 0
redis_client.get.side_effect = lambda key: {
self_heal.SYNC_LISTING_LOCK_KEY: "owner",
"iaai:state:task_progress:stale": json.dumps(
{
"task_id": "stale",
"stage": "segment_started",
"ts": 1000,
}
),
}.get(key)
redis_client.scan_iter.return_value = ["iaai:state:task_progress:stale"]
has_inflight, flags = self_heal._has_inflight_work(redis_client, self_heal.IAAI_SYNC_QUEUE)
self.assertTrue(has_inflight)
self.assertEqual(flags["has_lock"], 1)
self.assertEqual(flags["has_task_progress"], 1)
redis_client.delete.assert_not_called()
@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("iaai_scraper.worker.self_heal.os.kill")
@patch("builtins.open")