From 0e32152278032cfbbdf72c2138375fb37f20dd48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phan=20Sainl=C3=A9ger?= Date: Fri, 18 Sep 2026 23:13:26 +0200 Subject: [PATCH] [IMP] maintenance_service_http_monitoring : speed up KO confirmation The previous design required two passes of the 10-minute sweep before a maintenance request was created, so an outage took at least ~10 extra minutes (up to 20) to be reported. Split the monitoring into two crons with distinct responsibilities: - the 10-minute sweep only discovers outages, flags the KO streak via ``http_first_ko_at`` and auto-resolves recovered services; - a new 2-minute confirmation cron re-checks only the flagged services and creates a request once the service has been continuously KO for ``HTTP_KO_CONFIRMATION_DELAY`` (lowered from 5 to 2 minutes). This keeps the transient-outage filter while removing the extra sweep latency. Services already under an open request are skipped by the confirmation cron and remain handled by the sweep. --- maintenance_service_http_monitoring/README.md | 24 ++++-- .../data/cron.xml | 8 ++ .../models/service_instance.py | 44 ++++++++-- .../tests/test_http_monitoring.py | 83 +++++++++++++++---- 4 files changed, 130 insertions(+), 29 deletions(-) diff --git a/maintenance_service_http_monitoring/README.md b/maintenance_service_http_monitoring/README.md index 3b9bbc7..ea8ae8a 100644 --- a/maintenance_service_http_monitoring/README.md +++ b/maintenance_service_http_monitoring/README.md @@ -39,13 +39,19 @@ By default, maintenance mode lasts 4 hours. To change this: ## Cron Jobs -Two scheduled actions are installed: +Three scheduled actions are installed: 1. **HTTP Service Monitoring: check all services** - - Runs every 15 minutes - - Checks HTTP status of all active service instances with URLs + - Runs every 10 minutes + - Discovery sweep: checks HTTP status of all active service instances with URLs + and flags the start of a KO streak -2. **HTTP Service Monitoring: deactivate expired maintenance mode** +2. **HTTP Service Monitoring: confirm KO services** + - Runs every 2 minutes + - Re-checks only the services currently flagged KO (without an open request) and + creates a maintenance request once the outage is confirmed + +3. **HTTP Service Monitoring: deactivate expired maintenance mode** - Runs every 15 minutes - Automatically disables maintenance mode when the end time is reached @@ -98,14 +104,18 @@ On service instances, you can see: ## Automatic Maintenance Requests When a service fails HTTP checks: +- The 10-minute discovery sweep flags the outage (``http_first_ko_at``) but does + **not** create a request yet +- The 2-minute confirmation cron re-checks the flagged services only. A request is + created once the service has been continuously KO for at least 2 minutes + (``HTTP_KO_CONFIRMATION_DELAY``), so a short transient outage is not flagged - A corrective maintenance request is created per failing service, named ``[HTTP KO] {service_url}`` - The request description includes the error detail: the HTTP status code, or a network error label (timeout / DNS / SSL) when no HTTP response was received - No duplicate is created as long as an open request already exists for that service -- A **double-check** is performed before creating the request: the service is retested - after 2 seconds. A maintenance request is only created if the service fails **both** - checks, reducing noise from transient HTTP errors +- The confirmation cron skips services that already have an open request; those are + handled by the discovery sweep When a service recovers (returns HTTP 200 after having an open request): - The open maintenance request is automatically moved to the first **done** stage diff --git a/maintenance_service_http_monitoring/data/cron.xml b/maintenance_service_http_monitoring/data/cron.xml index 1a50515..5528817 100644 --- a/maintenance_service_http_monitoring/data/cron.xml +++ b/maintenance_service_http_monitoring/data/cron.xml @@ -8,6 +8,14 @@ 10 minutes + + HTTP Service Monitoring : confirm KO services + + code + model.cron_confirm_http_ko_services() + 2 + minutes + request created + self.env["service.instance"].cron_confirm_http_ko_services() request_1 = self.service_instance.http_maintenance_request self.assertTrue(request_1) with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: - mock_requests.get.return_value = _mock_response(500) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() # still KO + # already handled -> skipped + self.env["service.instance"].cron_confirm_http_ko_services() + # Service already has an open request -> the confirmation cron skips it + mock_requests.get.assert_not_called() self.assertEqual(self.service_instance.http_maintenance_request, request_1) self.assertEqual( self.env["maintenance.request"].search_count( @@ -222,7 +225,7 @@ class TestHttpMonitoring(TransactionCase): ): mock_requests.get.return_value = _mock_response(500) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() + self.env["service.instance"].cron_confirm_http_ko_services() mock_http.post.assert_called_once() call_kwargs = mock_http.post.call_args @@ -249,7 +252,7 @@ class TestHttpMonitoring(TransactionCase): ): mock_requests.get.return_value = _mock_response(500) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() + self.env["service.instance"].cron_confirm_http_ko_services() self.assertTrue(self.service_instance.http_maintenance_request) mock_http.post.assert_not_called() @@ -283,7 +286,7 @@ class TestHttpMonitoring(TransactionCase): with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: mock_requests.get.return_value = _mock_response(200) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() + self.env["service.instance"].cron_confirm_http_ko_services() self.assertTrue(self.service_instance.http_status_ok) self.assertEqual(self.service_instance.last_http_status_code, 200) @@ -314,7 +317,7 @@ class TestHttpMonitoring(TransactionCase): with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: mock_requests.get.return_value = _mock_response(503) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() + self.env["service.instance"].cron_confirm_http_ko_services() self.assertEqual(mock_requests.get.call_count, 1) self.assertFalse(self.service_instance.http_status_ok) @@ -345,7 +348,7 @@ class TestHttpMonitoring(TransactionCase): with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: mock_requests.get.return_value = _mock_response(500) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() + self.env["service.instance"].cron_confirm_http_ko_services() req1 = self.service_instance.http_maintenance_request req2 = service_instance2.http_maintenance_request @@ -374,11 +377,11 @@ class TestHttpMonitoring(TransactionCase): self._backdate_first_ko(self.service_instance) - # Second cron run: streak confirmed -> request created + # Confirmation cron run: streak confirmed -> request created with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: mock_requests.get.return_value = _mock_response(500) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() + self.env["service.instance"].cron_confirm_http_ko_services() request = self.service_instance.http_maintenance_request self.assertTrue(request) @@ -411,3 +414,55 @@ class TestHttpMonitoring(TransactionCase): # Calling close directly must not raise self.service_instance._close_http_maintenance_request() self.assertFalse(self.service_instance.http_maintenance_request) + + # ------------------------------------------------------------------ + # Test 17 -- Confirmation cron ignores services without a KO streak + # ------------------------------------------------------------------ + def test_confirmation_cron_ignores_unflagged_services(self): + with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: + mock_requests.get.return_value = _mock_response(200) + mock_requests.exceptions.RequestException = Exception + self.env["service.instance"].cron_confirm_http_ko_services() + + mock_requests.get.assert_not_called() + self.assertFalse(self.service_instance.http_maintenance_request) + + # ------------------------------------------------------------------ + # Test 18 -- Confirmation cron does not create a request before + # HTTP_KO_CONFIRMATION_DELAY has elapsed + # ------------------------------------------------------------------ + def test_confirmation_cron_waits_for_confirmation_delay(self): + with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: + mock_requests.get.return_value = _mock_response(500) + mock_requests.exceptions.RequestException = Exception + self.env["service.instance"].cron_check_http_services() + + self.assertTrue(self.service_instance.http_first_ko_at) + + with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: + mock_requests.get.return_value = _mock_response(500) + mock_requests.exceptions.RequestException = Exception + self.env["service.instance"].cron_confirm_http_ko_services() + + self.assertFalse(self.service_instance.http_maintenance_request) + + # ------------------------------------------------------------------ + # Test 19 -- Confirmation cron ignores services whose equipment is in + # maintenance mode + # ------------------------------------------------------------------ + def test_confirmation_cron_skips_maintenance_mode(self): + with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: + mock_requests.get.return_value = _mock_response(500) + mock_requests.exceptions.RequestException = Exception + self.env["service.instance"].cron_check_http_services() + + self.assertTrue(self.service_instance.http_first_ko_at) + self.equipment.write({"maintenance_mode": True}) + + with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: + mock_requests.get.return_value = _mock_response(500) + mock_requests.exceptions.RequestException = Exception + self.env["service.instance"].cron_confirm_http_ko_services() + + mock_requests.get.assert_not_called() + self.assertFalse(self.service_instance.http_maintenance_request) -- 2.49.1