diff --git a/maintenance_service_http_monitoring/README.md b/maintenance_service_http_monitoring/README.md
index 3b9bbc7..ea8ae8a 100644
--- a/maintenance_service_http_monitoring/README.md
+++ b/maintenance_service_http_monitoring/README.md
@@ -39,13 +39,19 @@ By default, maintenance mode lasts 4 hours. To change this:
## Cron Jobs
-Two scheduled actions are installed:
+Three scheduled actions are installed:
1. **HTTP Service Monitoring: check all services**
- - Runs every 15 minutes
- - Checks HTTP status of all active service instances with URLs
+ - Runs every 10 minutes
+ - Discovery sweep: checks HTTP status of all active service instances with URLs
+ and flags the start of a KO streak
-2. **HTTP Service Monitoring: deactivate expired maintenance mode**
+2. **HTTP Service Monitoring: confirm KO services**
+ - Runs every 2 minutes
+ - Re-checks only the services currently flagged KO (without an open request) and
+ creates a maintenance request once the outage is confirmed
+
+3. **HTTP Service Monitoring: deactivate expired maintenance mode**
- Runs every 15 minutes
- Automatically disables maintenance mode when the end time is reached
@@ -98,14 +104,18 @@ On service instances, you can see:
## Automatic Maintenance Requests
When a service fails HTTP checks:
+- The 10-minute discovery sweep flags the outage (``http_first_ko_at``) but does
+ **not** create a request yet
+- The 2-minute confirmation cron re-checks the flagged services only. A request is
+ created once the service has been continuously KO for at least 2 minutes
+ (``HTTP_KO_CONFIRMATION_DELAY``), so a short transient outage is not flagged
- A corrective maintenance request is created per failing service, named
``[HTTP KO] {service_url}``
- The request description includes the error detail: the HTTP status code,
or a network error label (timeout / DNS / SSL) when no HTTP response was received
- No duplicate is created as long as an open request already exists for that service
-- A **double-check** is performed before creating the request: the service is retested
- after 2 seconds. A maintenance request is only created if the service fails **both**
- checks, reducing noise from transient HTTP errors
+- The confirmation cron skips services that already have an open request; those are
+ handled by the discovery sweep
When a service recovers (returns HTTP 200 after having an open request):
- The open maintenance request is automatically moved to the first **done** stage
diff --git a/maintenance_service_http_monitoring/data/cron.xml b/maintenance_service_http_monitoring/data/cron.xml
index 1a50515..5528817 100644
--- a/maintenance_service_http_monitoring/data/cron.xml
+++ b/maintenance_service_http_monitoring/data/cron.xml
@@ -8,6 +8,14 @@
10
minutes
+
+ HTTP Service Monitoring : confirm KO services
+
+ code
+ model.cron_confirm_http_ko_services()
+ 2
+ minutes
+
= HTTP_KO_CONFIRMATION_DELAY
diff --git a/maintenance_service_http_monitoring/tests/test_http_monitoring.py b/maintenance_service_http_monitoring/tests/test_http_monitoring.py
index f291bc8..83b101d 100644
--- a/maintenance_service_http_monitoring/tests/test_http_monitoring.py
+++ b/maintenance_service_http_monitoring/tests/test_http_monitoring.py
@@ -12,8 +12,8 @@ EQUIPMENT_HTTP_REQUESTS = (
".models.maintenance_equipment.http_requests"
)
-# Comfortably past HTTP_KO_CONFIRMATION_DELAY (currently 10 minutes).
-PAST_CONFIRMATION_DELAY = timedelta(minutes=20)
+# Comfortably past HTTP_KO_CONFIRMATION_DELAY (currently 2 minutes).
+PAST_CONFIRMATION_DELAY = timedelta(minutes=10)
def _mock_response(status_code):
@@ -42,7 +42,7 @@ class TestHttpMonitoring(TransactionCase):
)
def _backdate_first_ko(self, *service_instances):
- """Simulate that the current KO streak started long enough ago to be confirmed."""
+ """Simulate a KO streak old enough ago to be confirmed."""
for service_instance in service_instances:
service_instance.write(
{"http_first_ko_at": fields.Datetime.now() - PAST_CONFIRMATION_DELAY}
@@ -81,7 +81,7 @@ class TestHttpMonitoring(TransactionCase):
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
- self.env["service.instance"].cron_check_http_services()
+ self.env["service.instance"].cron_confirm_http_ko_services()
request = self.service_instance.http_maintenance_request
self.assertTrue(request)
@@ -118,16 +118,19 @@ class TestHttpMonitoring(TransactionCase):
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
- self.env["service.instance"].cron_check_http_services() # confirmed, request created
+ # confirmed -> request created
+ self.env["service.instance"].cron_confirm_http_ko_services()
request_1 = self.service_instance.http_maintenance_request
self.assertTrue(request_1)
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
- mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
- self.env["service.instance"].cron_check_http_services() # still KO
+ # already handled -> skipped
+ self.env["service.instance"].cron_confirm_http_ko_services()
+ # Service already has an open request -> the confirmation cron skips it
+ mock_requests.get.assert_not_called()
self.assertEqual(self.service_instance.http_maintenance_request, request_1)
self.assertEqual(
self.env["maintenance.request"].search_count(
@@ -222,7 +225,7 @@ class TestHttpMonitoring(TransactionCase):
):
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
- self.env["service.instance"].cron_check_http_services()
+ self.env["service.instance"].cron_confirm_http_ko_services()
mock_http.post.assert_called_once()
call_kwargs = mock_http.post.call_args
@@ -249,7 +252,7 @@ class TestHttpMonitoring(TransactionCase):
):
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
- self.env["service.instance"].cron_check_http_services()
+ self.env["service.instance"].cron_confirm_http_ko_services()
self.assertTrue(self.service_instance.http_maintenance_request)
mock_http.post.assert_not_called()
@@ -283,7 +286,7 @@ class TestHttpMonitoring(TransactionCase):
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
mock_requests.get.return_value = _mock_response(200)
mock_requests.exceptions.RequestException = Exception
- self.env["service.instance"].cron_check_http_services()
+ self.env["service.instance"].cron_confirm_http_ko_services()
self.assertTrue(self.service_instance.http_status_ok)
self.assertEqual(self.service_instance.last_http_status_code, 200)
@@ -314,7 +317,7 @@ class TestHttpMonitoring(TransactionCase):
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
mock_requests.get.return_value = _mock_response(503)
mock_requests.exceptions.RequestException = Exception
- self.env["service.instance"].cron_check_http_services()
+ self.env["service.instance"].cron_confirm_http_ko_services()
self.assertEqual(mock_requests.get.call_count, 1)
self.assertFalse(self.service_instance.http_status_ok)
@@ -345,7 +348,7 @@ class TestHttpMonitoring(TransactionCase):
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
- self.env["service.instance"].cron_check_http_services()
+ self.env["service.instance"].cron_confirm_http_ko_services()
req1 = self.service_instance.http_maintenance_request
req2 = service_instance2.http_maintenance_request
@@ -374,11 +377,11 @@ class TestHttpMonitoring(TransactionCase):
self._backdate_first_ko(self.service_instance)
- # Second cron run: streak confirmed -> request created
+ # Confirmation cron run: streak confirmed -> request created
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
- self.env["service.instance"].cron_check_http_services()
+ self.env["service.instance"].cron_confirm_http_ko_services()
request = self.service_instance.http_maintenance_request
self.assertTrue(request)
@@ -411,3 +414,55 @@ class TestHttpMonitoring(TransactionCase):
# Calling close directly must not raise
self.service_instance._close_http_maintenance_request()
self.assertFalse(self.service_instance.http_maintenance_request)
+
+ # ------------------------------------------------------------------
+ # Test 17 -- Confirmation cron ignores services without a KO streak
+ # ------------------------------------------------------------------
+ def test_confirmation_cron_ignores_unflagged_services(self):
+ with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
+ mock_requests.get.return_value = _mock_response(200)
+ mock_requests.exceptions.RequestException = Exception
+ self.env["service.instance"].cron_confirm_http_ko_services()
+
+ mock_requests.get.assert_not_called()
+ self.assertFalse(self.service_instance.http_maintenance_request)
+
+ # ------------------------------------------------------------------
+ # Test 18 -- Confirmation cron does not create a request before
+ # HTTP_KO_CONFIRMATION_DELAY has elapsed
+ # ------------------------------------------------------------------
+ def test_confirmation_cron_waits_for_confirmation_delay(self):
+ with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
+ mock_requests.get.return_value = _mock_response(500)
+ mock_requests.exceptions.RequestException = Exception
+ self.env["service.instance"].cron_check_http_services()
+
+ self.assertTrue(self.service_instance.http_first_ko_at)
+
+ with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
+ mock_requests.get.return_value = _mock_response(500)
+ mock_requests.exceptions.RequestException = Exception
+ self.env["service.instance"].cron_confirm_http_ko_services()
+
+ self.assertFalse(self.service_instance.http_maintenance_request)
+
+ # ------------------------------------------------------------------
+ # Test 19 -- Confirmation cron ignores services whose equipment is in
+ # maintenance mode
+ # ------------------------------------------------------------------
+ def test_confirmation_cron_skips_maintenance_mode(self):
+ with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
+ mock_requests.get.return_value = _mock_response(500)
+ mock_requests.exceptions.RequestException = Exception
+ self.env["service.instance"].cron_check_http_services()
+
+ self.assertTrue(self.service_instance.http_first_ko_at)
+ self.equipment.write({"maintenance_mode": True})
+
+ with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
+ mock_requests.get.return_value = _mock_response(500)
+ mock_requests.exceptions.RequestException = Exception
+ self.env["service.instance"].cron_confirm_http_ko_services()
+
+ mock_requests.get.assert_not_called()
+ self.assertFalse(self.service_instance.http_maintenance_request)