feat: add async retry mechanism with exponential backoff

- Add app/utils/retry.py with configurable async retry decorator
- Update DeliveryLog model to track attempt_count and latency_seconds
- Apply @http_retry to engine._exec_forward and _exec_notify methods
- Update save_logs to record retry metadata
- Add comprehensive unit tests for retry functionality
- Support configuration via environment variables (RETRY_*)

This improves reliability for downstream HTTP calls by automatically
retrying transient failures with exponential backoff and jitter.

tests/test_retry.py

@@ -0,0 +1,121 @@
+import pytest
+import asyncio
+from unittest.mock import AsyncMock, patch
+from app.utils.retry import async_retry, http_retry, get_retry_config
+
+
+class TestAsyncRetry:
+    @pytest.mark.asyncio
+    async def test_success_on_first_attempt(self):
+        """Test that function succeeds on first attempt returns immediately."""
+
+        @async_retry(max_attempts=3)
+        async def test_func():
+            return "success"
+
+        result, metadata = await test_func()
+
+        assert result == "success"
+        assert metadata["attempts"] == 1
+        assert metadata["success"] is True
+        assert metadata["last_error"] is None
+        assert "total_latency" in metadata
+
+    @pytest.mark.asyncio
+    async def test_retry_on_failure_then_success(self):
+        """Test retry mechanism when function fails then succeeds."""
+
+        call_count = 0
+
+        @async_retry(max_attempts=3, initial_delay=0.01)
+        async def test_func():
+            nonlocal call_count
+            call_count += 1
+            if call_count < 2:
+                raise ValueError("Temporary failure")
+            return "success"
+
+        result, metadata = await test_func()
+
+        assert result == "success"
+        assert metadata["attempts"] == 2
+        assert metadata["success"] is True
+        assert metadata["last_error"] is None
+
+    @pytest.mark.asyncio
+    async def test_all_attempts_fail(self):
+        """Test behavior when all retry attempts fail."""
+
+        @async_retry(max_attempts=2, initial_delay=0.01)
+        async def test_func():
+            raise ConnectionError("Persistent failure")
+
+        result, metadata = await test_func()
+
+        assert result is None
+        assert metadata["attempts"] == 2
+        assert metadata["success"] is False
+        assert metadata["last_error"] == "Persistent failure"
+
+    @pytest.mark.asyncio
+    async def test_retry_on_specific_exceptions_only_value_error_retried(self):
+        """Test that only ValueError triggers retry, RuntimeError does not."""
+
+        call_count = 0
+
+        @async_retry(max_attempts=3, retry_on=(ValueError,), initial_delay=0.01)
+        async def test_func():
+            nonlocal call_count
+            call_count += 1
+            if call_count == 1:
+                raise ValueError("Retry this")
+            elif call_count == 2:
+                raise RuntimeError("Don't retry this")
+            return "success"
+
+        # This should raise RuntimeError directly without retry
+        with pytest.raises(RuntimeError, match="Don't retry this"):
+            await test_func()
+
+    @pytest.mark.asyncio
+    async def test_backoff_and_jitter(self):
+        """Test exponential backoff with jitter."""
+
+        @async_retry(max_attempts=4, initial_delay=0.1, backoff_factor=2, jitter=True)
+        async def test_func():
+            raise ConnectionError("Always fail")
+
+        start_time = asyncio.get_event_loop().time()
+        result, metadata = await test_func()
+        end_time = asyncio.get_event_loop().time()
+
+        # Should take at least some time due to retries and delays
+        assert end_time - start_time >= 0.1  # At least initial delay
+        assert metadata["attempts"] == 4
+        assert metadata["success"] is False
+
+    def test_http_retry_decorator(self):
+        """Test the http_retry convenience decorator."""
+
+        @http_retry(max_attempts=5)
+        async def test_func():
+            return "ok"
+
+        assert hasattr(test_func, '__name__')
+        # Should be awaitable
+        assert asyncio.iscoroutinefunction(test_func)
+
+    def test_get_retry_config(self):
+        """Test configuration loading from environment."""
+        config = get_retry_config()
+
+        assert "max_attempts" in config
+        assert "initial_delay" in config
+        assert "backoff_factor" in config
+        assert "max_delay" in config
+
+        # Test with environment override
+        with patch.dict('os.environ', {'RETRY_MAX_ATTEMPTS': '5', 'RETRY_INITIAL_DELAY': '2.0'}):
+            config = get_retry_config()
+            assert config["max_attempts"] == 5
+            assert config["initial_delay"] == 2.0