b11c39f3bf
- Add app/utils/retry.py with configurable async retry decorator - Update DeliveryLog model to track attempt_count and latency_seconds - Apply @http_retry to engine._exec_forward and _exec_notify methods - Update save_logs to record retry metadata - Add comprehensive unit tests for retry functionality - Support configuration via environment variables (RETRY_*) This improves reliability for downstream HTTP calls by automatically retrying transient failures with exponential backoff and jitter.
122 lines
4.0 KiB
Python
122 lines
4.0 KiB
Python
import pytest
|
|
import asyncio
|
|
from unittest.mock import AsyncMock, patch
|
|
from app.utils.retry import async_retry, http_retry, get_retry_config
|
|
|
|
|
|
class TestAsyncRetry:
|
|
@pytest.mark.asyncio
|
|
async def test_success_on_first_attempt(self):
|
|
"""Test that function succeeds on first attempt returns immediately."""
|
|
|
|
@async_retry(max_attempts=3)
|
|
async def test_func():
|
|
return "success"
|
|
|
|
result, metadata = await test_func()
|
|
|
|
assert result == "success"
|
|
assert metadata["attempts"] == 1
|
|
assert metadata["success"] is True
|
|
assert metadata["last_error"] is None
|
|
assert "total_latency" in metadata
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_on_failure_then_success(self):
|
|
"""Test retry mechanism when function fails then succeeds."""
|
|
|
|
call_count = 0
|
|
|
|
@async_retry(max_attempts=3, initial_delay=0.01)
|
|
async def test_func():
|
|
nonlocal call_count
|
|
call_count += 1
|
|
if call_count < 2:
|
|
raise ValueError("Temporary failure")
|
|
return "success"
|
|
|
|
result, metadata = await test_func()
|
|
|
|
assert result == "success"
|
|
assert metadata["attempts"] == 2
|
|
assert metadata["success"] is True
|
|
assert metadata["last_error"] is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_all_attempts_fail(self):
|
|
"""Test behavior when all retry attempts fail."""
|
|
|
|
@async_retry(max_attempts=2, initial_delay=0.01)
|
|
async def test_func():
|
|
raise ConnectionError("Persistent failure")
|
|
|
|
result, metadata = await test_func()
|
|
|
|
assert result is None
|
|
assert metadata["attempts"] == 2
|
|
assert metadata["success"] is False
|
|
assert metadata["last_error"] == "Persistent failure"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_on_specific_exceptions_only_value_error_retried(self):
|
|
"""Test that only ValueError triggers retry, RuntimeError does not."""
|
|
|
|
call_count = 0
|
|
|
|
@async_retry(max_attempts=3, retry_on=(ValueError,), initial_delay=0.01)
|
|
async def test_func():
|
|
nonlocal call_count
|
|
call_count += 1
|
|
if call_count == 1:
|
|
raise ValueError("Retry this")
|
|
elif call_count == 2:
|
|
raise RuntimeError("Don't retry this")
|
|
return "success"
|
|
|
|
# This should raise RuntimeError directly without retry
|
|
with pytest.raises(RuntimeError, match="Don't retry this"):
|
|
await test_func()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backoff_and_jitter(self):
|
|
"""Test exponential backoff with jitter."""
|
|
|
|
@async_retry(max_attempts=4, initial_delay=0.1, backoff_factor=2, jitter=True)
|
|
async def test_func():
|
|
raise ConnectionError("Always fail")
|
|
|
|
start_time = asyncio.get_event_loop().time()
|
|
result, metadata = await test_func()
|
|
end_time = asyncio.get_event_loop().time()
|
|
|
|
# Should take at least some time due to retries and delays
|
|
assert end_time - start_time >= 0.1 # At least initial delay
|
|
assert metadata["attempts"] == 4
|
|
assert metadata["success"] is False
|
|
|
|
def test_http_retry_decorator(self):
|
|
"""Test the http_retry convenience decorator."""
|
|
|
|
@http_retry(max_attempts=5)
|
|
async def test_func():
|
|
return "ok"
|
|
|
|
assert hasattr(test_func, '__name__')
|
|
# Should be awaitable
|
|
assert asyncio.iscoroutinefunction(test_func)
|
|
|
|
def test_get_retry_config(self):
|
|
"""Test configuration loading from environment."""
|
|
config = get_retry_config()
|
|
|
|
assert "max_attempts" in config
|
|
assert "initial_delay" in config
|
|
assert "backoff_factor" in config
|
|
assert "max_delay" in config
|
|
|
|
# Test with environment override
|
|
with patch.dict('os.environ', {'RETRY_MAX_ATTEMPTS': '5', 'RETRY_INITIAL_DELAY': '2.0'}):
|
|
config = get_retry_config()
|
|
assert config["max_attempts"] == 5
|
|
assert config["initial_delay"] == 2.0
|