18.0-improve-http-checks #7
@@ -107,6 +107,12 @@ When a service fails HTTP checks:
|
|||||||
after 2 seconds. A maintenance request is only created if the service fails **both**
|
after 2 seconds. A maintenance request is only created if the service fails **both**
|
||||||
checks, reducing noise from transient HTTP errors
|
checks, reducing noise from transient HTTP errors
|
||||||
|
|
||||||
|
When a service recovers (returns HTTP 200 after having an open request):
|
||||||
|
- The open maintenance request is automatically moved to the first **done** stage
|
||||||
|
- A chatter note is posted on the request by OdooBot to record the automatic closure
|
||||||
|
- The link between the service and the request is cleared, allowing a new request to
|
||||||
|
be created if the service fails again in the future
|
||||||
|
|
||||||
## Webhook notifications
|
## Webhook notifications
|
||||||
|
|
||||||
When a new maintenance request is created (HTTP check failure), the module can
|
When a new maintenance request is created (HTTP check failure), the module can
|
||||||
|
|||||||
@@ -74,13 +74,43 @@ class ServiceInstance(models.Model):
|
|||||||
ko_records |= rec
|
ko_records |= rec
|
||||||
return ko_records
|
return ko_records
|
||||||
|
|
||||||
|
def _close_http_maintenance_request(self):
|
||||||
|
"""
|
||||||
|
Close the open maintenance.request for each recovered service.
|
||||||
|
|
||||||
|
Moves the request to the first done stage, posts a chatter note as OdooBot, and
|
||||||
|
clears http_maintenance_request on the service instance.
|
||||||
|
"""
|
||||||
|
done_stage = self.env["maintenance.stage"].search(
|
||||||
|
[("done", "=", True)], limit=1
|
||||||
|
)
|
||||||
|
if not done_stage:
|
||||||
|
return
|
||||||
|
odoobot = self.env.ref("base.partner_root")
|
||||||
|
for rec in self:
|
||||||
|
request = rec.http_maintenance_request
|
||||||
|
if not request or request.stage_id.done:
|
||||||
|
continue
|
||||||
|
request.sudo().write({"stage_id": done_stage.id})
|
||||||
|
request.sudo().message_post(
|
||||||
|
body=(
|
||||||
|
f"Service {rec.service_url} is back online. "
|
||||||
|
"This request has been automatically closed by the monitoring cron."
|
||||||
|
),
|
||||||
|
author_id=odoobot.id,
|
||||||
|
message_type="comment",
|
||||||
|
subtype_xmlid="mail.mt_note",
|
||||||
|
)
|
||||||
|
rec.http_maintenance_request = False
|
||||||
|
|
||||||
@api.model
|
@api.model
|
||||||
def cron_check_http_services(self):
|
def cron_check_http_services(self):
|
||||||
"""
|
"""
|
||||||
Check all active services with a URL, with one retry on failure.
|
Check all active services with a URL, with one retry on failure.
|
||||||
|
|
||||||
Pass 1: test every eligible service.
|
Pass 1: test every eligible service.
|
||||||
If any fail, wait HTTP_RETRY_DELAY seconds then retest only the KO ones.
|
- Services that had an open request and are now OK are auto-resolved.
|
||||||
|
- Services still KO after pass 1 are retested after HTTP_RETRY_DELAY seconds.
|
||||||
maintenance.request is created only for services that fail both passes,
|
maintenance.request is created only for services that fail both passes,
|
||||||
reducing noise from transient HTTP errors.
|
reducing noise from transient HTTP errors.
|
||||||
"""
|
"""
|
||||||
@@ -93,8 +123,19 @@ class ServiceInstance(models.Model):
|
|||||||
lambda s: not s.equipment_id.maintenance_mode
|
lambda s: not s.equipment_id.maintenance_mode
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Snapshot services that currently have an open request before pass 1
|
||||||
|
services_with_open_request = services.filtered(
|
||||||
|
lambda s: s.http_maintenance_request
|
||||||
|
and not s.http_maintenance_request.stage_id.done
|
||||||
|
)
|
||||||
|
|
||||||
ko_after_pass1 = services.check_http_status()
|
ko_after_pass1 = services.check_http_status()
|
||||||
|
|
||||||
|
# Auto-resolve services that recovered during pass 1
|
||||||
|
recovered = services_with_open_request.filtered(lambda s: s.http_status_ok)
|
||||||
|
if recovered:
|
||||||
|
recovered._close_http_maintenance_request()
|
||||||
|
|
||||||
if not ko_after_pass1:
|
if not ko_after_pass1:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|||||||
@@ -312,3 +312,49 @@ class TestHttpMonitoring(TransactionCase):
|
|||||||
),
|
),
|
||||||
2,
|
2,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Test 15 -- Service recovery closes the open request and posts a note
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
def test_service_recovery_closes_request(self):
|
||||||
|
# First cron run: service is KO -> request created
|
||||||
|
with (
|
||||||
|
patch(SERVICE_INSTANCE_REQUESTS) as mock_requests,
|
||||||
|
patch(SERVICE_INSTANCE_SLEEP),
|
||||||
|
):
|
||||||
|
mock_requests.get.return_value = _mock_response(500)
|
||||||
|
mock_requests.exceptions.RequestException = Exception
|
||||||
|
self.env["service.instance"].cron_check_http_services()
|
||||||
|
|
||||||
|
request = self.service_instance.http_maintenance_request
|
||||||
|
self.assertTrue(request)
|
||||||
|
self.assertFalse(request.stage_id.done)
|
||||||
|
|
||||||
|
# Second cron run: service is back OK -> request auto-closed
|
||||||
|
with (
|
||||||
|
patch(SERVICE_INSTANCE_REQUESTS) as mock_requests,
|
||||||
|
patch(SERVICE_INSTANCE_SLEEP),
|
||||||
|
):
|
||||||
|
mock_requests.get.return_value = _mock_response(200)
|
||||||
|
mock_requests.exceptions.RequestException = Exception
|
||||||
|
self.env["service.instance"].cron_check_http_services()
|
||||||
|
|
||||||
|
# Request must be in a done stage
|
||||||
|
self.assertTrue(request.stage_id.done)
|
||||||
|
# http_maintenance_request must be cleared on the service instance
|
||||||
|
self.assertFalse(self.service_instance.http_maintenance_request)
|
||||||
|
# A chatter note must have been posted mentioning the service URL
|
||||||
|
notes = request.message_ids.filtered(
|
||||||
|
lambda m: self.service_instance.service_url in (m.body or "")
|
||||||
|
)
|
||||||
|
self.assertTrue(notes)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Test 16 -- No open request -> _close_http_maintenance_request is a no-op
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
def test_no_close_when_no_open_request(self):
|
||||||
|
# Service is OK from the start, no request exists
|
||||||
|
self.assertFalse(self.service_instance.http_maintenance_request)
|
||||||
|
# Calling close directly must not raise
|
||||||
|
self.service_instance._close_http_maintenance_request()
|
||||||
|
self.assertFalse(self.service_instance.http_maintenance_request)
|
||||||
|
|||||||
Reference in New Issue
Block a user