diff --git a/maintenance_service_http_monitoring/README.md b/maintenance_service_http_monitoring/README.md index 3b9bbc7..ea8ae8a 100644 --- a/maintenance_service_http_monitoring/README.md +++ b/maintenance_service_http_monitoring/README.md @@ -39,13 +39,19 @@ By default, maintenance mode lasts 4 hours. To change this: ## Cron Jobs -Two scheduled actions are installed: +Three scheduled actions are installed: 1. **HTTP Service Monitoring: check all services** - - Runs every 15 minutes - - Checks HTTP status of all active service instances with URLs + - Runs every 10 minutes + - Discovery sweep: checks HTTP status of all active service instances with URLs + and flags the start of a KO streak -2. **HTTP Service Monitoring: deactivate expired maintenance mode** +2. **HTTP Service Monitoring: confirm KO services** + - Runs every 2 minutes + - Re-checks only the services currently flagged KO (without an open request) and + creates a maintenance request once the outage is confirmed + +3. **HTTP Service Monitoring: deactivate expired maintenance mode** - Runs every 15 minutes - Automatically disables maintenance mode when the end time is reached @@ -98,14 +104,18 @@ On service instances, you can see: ## Automatic Maintenance Requests When a service fails HTTP checks: +- The 10-minute discovery sweep flags the outage (``http_first_ko_at``) but does + **not** create a request yet +- The 2-minute confirmation cron re-checks the flagged services only. A request is + created once the service has been continuously KO for at least 2 minutes + (``HTTP_KO_CONFIRMATION_DELAY``), so a short transient outage is not flagged - A corrective maintenance request is created per failing service, named ``[HTTP KO] {service_url}`` - The request description includes the error detail: the HTTP status code, or a network error label (timeout / DNS / SSL) when no HTTP response was received - No duplicate is created as long as an open request already exists for that service -- A **double-check** is performed before creating the request: the service is retested - after 2 seconds. A maintenance request is only created if the service fails **both** - checks, reducing noise from transient HTTP errors +- The confirmation cron skips services that already have an open request; those are + handled by the discovery sweep When a service recovers (returns HTTP 200 after having an open request): - The open maintenance request is automatically moved to the first **done** stage diff --git a/maintenance_service_http_monitoring/data/cron.xml b/maintenance_service_http_monitoring/data/cron.xml index 1a50515..5528817 100644 --- a/maintenance_service_http_monitoring/data/cron.xml +++ b/maintenance_service_http_monitoring/data/cron.xml @@ -8,6 +8,14 @@ 10 minutes + + HTTP Service Monitoring : confirm KO services + + code + model.cron_confirm_http_ko_services() + 2 + minutes + request created + self.env["service.instance"].cron_confirm_http_ko_services() request_1 = self.service_instance.http_maintenance_request self.assertTrue(request_1) with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: - mock_requests.get.return_value = _mock_response(500) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() # still KO + # already handled -> skipped + self.env["service.instance"].cron_confirm_http_ko_services() + # Service already has an open request -> the confirmation cron skips it + mock_requests.get.assert_not_called() self.assertEqual(self.service_instance.http_maintenance_request, request_1) self.assertEqual( self.env["maintenance.request"].search_count( @@ -222,7 +225,7 @@ class TestHttpMonitoring(TransactionCase): ): mock_requests.get.return_value = _mock_response(500) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() + self.env["service.instance"].cron_confirm_http_ko_services() mock_http.post.assert_called_once() call_kwargs = mock_http.post.call_args @@ -249,7 +252,7 @@ class TestHttpMonitoring(TransactionCase): ): mock_requests.get.return_value = _mock_response(500) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() + self.env["service.instance"].cron_confirm_http_ko_services() self.assertTrue(self.service_instance.http_maintenance_request) mock_http.post.assert_not_called() @@ -283,7 +286,7 @@ class TestHttpMonitoring(TransactionCase): with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: mock_requests.get.return_value = _mock_response(200) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() + self.env["service.instance"].cron_confirm_http_ko_services() self.assertTrue(self.service_instance.http_status_ok) self.assertEqual(self.service_instance.last_http_status_code, 200) @@ -314,7 +317,7 @@ class TestHttpMonitoring(TransactionCase): with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: mock_requests.get.return_value = _mock_response(503) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() + self.env["service.instance"].cron_confirm_http_ko_services() self.assertEqual(mock_requests.get.call_count, 1) self.assertFalse(self.service_instance.http_status_ok) @@ -345,7 +348,7 @@ class TestHttpMonitoring(TransactionCase): with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: mock_requests.get.return_value = _mock_response(500) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() + self.env["service.instance"].cron_confirm_http_ko_services() req1 = self.service_instance.http_maintenance_request req2 = service_instance2.http_maintenance_request @@ -374,11 +377,11 @@ class TestHttpMonitoring(TransactionCase): self._backdate_first_ko(self.service_instance) - # Second cron run: streak confirmed -> request created + # Confirmation cron run: streak confirmed -> request created with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: mock_requests.get.return_value = _mock_response(500) mock_requests.exceptions.RequestException = Exception - self.env["service.instance"].cron_check_http_services() + self.env["service.instance"].cron_confirm_http_ko_services() request = self.service_instance.http_maintenance_request self.assertTrue(request) @@ -411,3 +414,55 @@ class TestHttpMonitoring(TransactionCase): # Calling close directly must not raise self.service_instance._close_http_maintenance_request() self.assertFalse(self.service_instance.http_maintenance_request) + + # ------------------------------------------------------------------ + # Test 17 -- Confirmation cron ignores services without a KO streak + # ------------------------------------------------------------------ + def test_confirmation_cron_ignores_unflagged_services(self): + with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: + mock_requests.get.return_value = _mock_response(200) + mock_requests.exceptions.RequestException = Exception + self.env["service.instance"].cron_confirm_http_ko_services() + + mock_requests.get.assert_not_called() + self.assertFalse(self.service_instance.http_maintenance_request) + + # ------------------------------------------------------------------ + # Test 18 -- Confirmation cron does not create a request before + # HTTP_KO_CONFIRMATION_DELAY has elapsed + # ------------------------------------------------------------------ + def test_confirmation_cron_waits_for_confirmation_delay(self): + with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: + mock_requests.get.return_value = _mock_response(500) + mock_requests.exceptions.RequestException = Exception + self.env["service.instance"].cron_check_http_services() + + self.assertTrue(self.service_instance.http_first_ko_at) + + with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: + mock_requests.get.return_value = _mock_response(500) + mock_requests.exceptions.RequestException = Exception + self.env["service.instance"].cron_confirm_http_ko_services() + + self.assertFalse(self.service_instance.http_maintenance_request) + + # ------------------------------------------------------------------ + # Test 19 -- Confirmation cron ignores services whose equipment is in + # maintenance mode + # ------------------------------------------------------------------ + def test_confirmation_cron_skips_maintenance_mode(self): + with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: + mock_requests.get.return_value = _mock_response(500) + mock_requests.exceptions.RequestException = Exception + self.env["service.instance"].cron_check_http_services() + + self.assertTrue(self.service_instance.http_first_ko_at) + self.equipment.write({"maintenance_mode": True}) + + with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests: + mock_requests.get.return_value = _mock_response(500) + mock_requests.exceptions.RequestException = Exception + self.env["service.instance"].cron_confirm_http_ko_services() + + mock_requests.get.assert_not_called() + self.assertFalse(self.service_instance.http_maintenance_request)