18.0-improve-http-checks #7

Merged
borisgallet merged 6 commits from 18.0-improve-http-checks into 18.0 2026-06-17 13:06:53 +00:00
8 changed files with 496 additions and 65 deletions

View File

@@ -98,10 +98,20 @@ On service instances, you can see:
## Automatic Maintenance Requests
When a service fails HTTP checks:
- A corrective maintenance request is created with prefix "[HTTP KO]"
- The request is linked to the equipment
- Only one request per equipment per day is created
- The request description lists all failing services
- A corrective maintenance request is created per failing service, named
``[HTTP KO] {service_url}``
- The request description includes the error detail: the HTTP status code,
or a network error label (timeout / DNS / SSL) when no HTTP response was received
- No duplicate is created as long as an open request already exists for that service
- A **double-check** is performed before creating the request: the service is retested
after 2 seconds. A maintenance request is only created if the service fails **both**
checks, reducing noise from transient HTTP errors
When a service recovers (returns HTTP 200 after having an open request):
- The open maintenance request is automatically moved to the first **done** stage
- A chatter note is posted on the request by OdooBot to record the automatic closure
- The link between the service and the request is cleared, allowing a new request to
be created if the service fails again in the future
## Webhook notifications

View File

@@ -5,7 +5,7 @@
"license": "AGPL-3",
"category": "Tools",
"summary": "Monitor HTTP availability of services",
"depends": ["base", "maintenance", "maintenance_server_data"],
"depends": ["base", "maintenance", "hr_maintenance", "maintenance_server_data"],
"external_dependencies": {"python": ["requests"]},
"data": [
"data/ir_config_parameter.xml",

View File

@@ -3,7 +3,6 @@ from datetime import timedelta
from odoo import api, fields, models
try:
import requests as http_requests
except ImportError:
@@ -13,28 +12,21 @@ _logger = logging.getLogger(__name__)
WEBHOOK_TIMEOUT = 10 # seconds
class MaintenanceEquipment(models.Model):
_inherit = "maintenance.equipment"
maintenance_mode = fields.Boolean(
string="Maintenance Mode",
default=False,
tracking=True,
)
maintenance_mode_start = fields.Datetime(
string="Maintenance Mode Start",
readonly=True,
)
maintenance_mode_end = fields.Datetime(
string="Maintenance Mode End",
readonly=True,
help="Computed from start + configured duration",
)
http_maintenance_request = fields.Many2one(
"maintenance.request",
string="HTTP Maintenance Request",
readonly=True,
)
def action_activate_maintenance_mode(self):
for rec in self:
@@ -75,31 +67,30 @@ class MaintenanceEquipment(models.Model):
)
expired.action_deactivate_maintenance_mode()
def create_http_maintenance_request(self, ko_services):
def create_http_maintenance_request(self, ko_service):
"""
Create or return the open maintenance.request for a single KO service.
Deduplication: if ko_service already has an open (non-done) request,
return it without creating a new one.
"""
self.ensure_one()
today = fields.Date.context_today(self)
name = f"[HTTP KO] {self.name}"
domain = [
("name", "=", name),
("equipment_id", "=", self.id),
("maintenance_type", "=", "corrective"),
("create_date", ">=", f"{today} 00:00:00"),
("create_date", "<=", f"{today} 23:59:59"),
]
existing = self.env["maintenance.request"].search(domain, limit=1)
# Check if a task with same name already exist for the day, if its the case : skip
if existing:
self.http_maintenance_request = existing.id
existing = ko_service.http_maintenance_request
if existing and not existing.stage_id.done:
return existing
request = self.http_maintenance_request
if request and not request.stage_id.done:
return request
status_code = ko_service.last_http_status_code
if status_code == -1:
error_detail = "Erreur réseau (timeout / DNS / SSL)"
else:
error_detail = f"HTTP {status_code}"
name = f"[HTTP KO] {ko_service.service_url}"
description = f"Service KO: {ko_service.service_url}\n{error_detail}"
vals = {
"name": name,
"equipment_id": self.id,
"priority": "2",
"maintenance_type": "corrective",
"description": self._build_ko_services_description(ko_services),
"description": description,
}
if self.employee_id:
vals["employee_id"] = self.employee_id.id
@@ -112,35 +103,36 @@ class MaintenanceEquipment(models.Model):
if team:
vals["maintenance_team_id"] = team.id
request = self.env["maintenance.request"].create(vals)
self.http_maintenance_request = request.id
self._notify_webhook(request, ko_services)
ko_service.http_maintenance_request = request.id
self._notify_webhook(request, ko_service)
return request
def _notify_webhook(self, request, ko_services):
"""Send a webhook notification when a new maintenance request is created."""
ICP = self.env['ir.config_parameter'].sudo()
def _notify_webhook(self, request, ko_service):
"""
Send a webhook notification when a new maintenance request is created.
"""
ICP = self.env["ir.config_parameter"].sudo()
webhook_url = ICP.get_param(
'maintenance_service_http_monitoring.webhook_url', ''
"maintenance_service_http_monitoring.webhook_url", ""
)
if not webhook_url:
return
webhook_user = ICP.get_param(
'maintenance_service_http_monitoring.webhook_user', ''
"maintenance_service_http_monitoring.webhook_user", ""
)
webhook_password = ICP.get_param(
'maintenance_service_http_monitoring.webhook_password', ''
"maintenance_service_http_monitoring.webhook_password", ""
)
base_url = ICP.get_param('web.base.url', '')
base_url = ICP.get_param("web.base.url", "")
link = (
f"{base_url}/web#id={request.id}"
f"&model=maintenance.request&view_type=form"
f"{base_url}/web#id={request.id}&model=maintenance.request&view_type=form"
)
payload = {
'id': request.id,
'name': request.name,
'description': request.description or '',
'equipment': self.name,
'link': link,
"id": request.id,
"name": request.name,
"description": request.description or "",
"equipment": self.name,
"link": link,
}
auth = None
if webhook_user and webhook_password:
@@ -155,9 +147,6 @@ class MaintenanceEquipment(models.Model):
except Exception as e:
_logger.warning(
"Webhook notification failed for maintenance request %s: %s",
request.id, e,
request.id,
e,
)
def _build_ko_services_description(self, ko_services):
lines = [f"Service KO: {s.service_url or s.name}" for s in ko_services]
return "\n".join(lines)

View File

@@ -1,4 +1,5 @@
import logging
import time
from odoo import api, fields, models
@@ -10,6 +11,7 @@ except ImportError:
_logger = logging.getLogger(__name__)
HTTP_CHECK_TIMEOUT = 10 # seconds
HTTP_RETRY_DELAY = 2 # seconds between pass 1 and pass 2
class ServiceInstance(models.Model):
@@ -29,13 +31,25 @@ class ServiceInstance(models.Model):
readonly=True,
default=True,
)
http_maintenance_request = fields.Many2one(
"maintenance.request",
string="HTTP Maintenance Request",
readonly=True,
)
def check_http_status(self):
"""
Perform HTTP check for each record and return the KO recordset.
Writes last_http_status_code, last_http_check_date and http_status_ok on every
checked record. Does NOT create maintenance.request — that decision belongs to
the caller (cron) after optional retry logic.
"""
ko_records = self.browse()
for rec in self:
if not rec.service_url or not rec.equipment_id:
continue
equipment = rec.equipment_id
if getattr(equipment, "maintenance_mode", False):
if rec.equipment_id.maintenance_mode:
continue
status_ok = False
status_code = -1
@@ -57,20 +71,77 @@ class ServiceInstance(models.Model):
}
)
if not status_ok:
# Delegate maintenance.request creation to equipment
if hasattr(equipment, "create_http_maintenance_request"):
equipment.create_http_maintenance_request([rec])
ko_records |= rec
return ko_records
def _close_http_maintenance_request(self):
"""
Close the open maintenance.request for each recovered service.
Moves the request to the first done stage, posts a chatter note as OdooBot, and
clears http_maintenance_request on the service instance.
"""
done_stage = self.env["maintenance.stage"].search(
[("done", "=", True)], limit=1
)
if not done_stage:
return
odoobot = self.env.ref("base.partner_root")
for rec in self:
request = rec.http_maintenance_request
if not request or request.stage_id.done:
continue
request.sudo().write({"stage_id": done_stage.id})
request.sudo().message_post(
body=(
f"Service {rec.service_url} is back online. "
"This request has been automatically closed by the monitoring cron."
),
author_id=odoobot.id,
message_type="comment",
subtype_xmlid="mail.mt_note",
)
rec.http_maintenance_request = False
@api.model
def cron_check_http_services(self):
"""
Check all active services with a URL, with one retry on failure.
Pass 1: test every eligible service.
- Services that had an open request and are now OK are auto-resolved.
- Services still KO after pass 1 are retested after HTTP_RETRY_DELAY seconds.
maintenance.request is created only for services that fail both passes,
reducing noise from transient HTTP errors.
"""
domain = [
("active", "=", True),
("service_url", "!=", False),
("equipment_id", "!=", False),
]
services = self.search(domain)
for service in services:
equipment = service.equipment_id
if getattr(equipment, "maintenance_mode", False):
continue
service.check_http_status()
services = self.search(domain).filtered(
lambda s: not s.equipment_id.maintenance_mode
)
# Snapshot services that currently have an open request before pass 1
services_with_open_request = services.filtered(
lambda s: s.http_maintenance_request
and not s.http_maintenance_request.stage_id.done
)
ko_after_pass1 = services.check_http_status()
# Auto-resolve services that recovered during pass 1
recovered = services_with_open_request.filtered(lambda s: s.http_status_ok)
if recovered:
recovered._close_http_maintenance_request()
if not ko_after_pass1:
return
time.sleep(HTTP_RETRY_DELAY)
ko_confirmed = ko_after_pass1.check_http_status()
for service in ko_confirmed:
service.equipment_id.create_http_maintenance_request(service)

View File

@@ -0,0 +1 @@
from . import test_http_monitoring

View File

@@ -0,0 +1,360 @@
from datetime import timedelta
from unittest.mock import MagicMock, patch
from odoo import fields
from odoo.tests.common import TransactionCase
SERVICE_INSTANCE_REQUESTS = (
"odoo.addons.maintenance_service_http_monitoring.models.service_instance.requests"
)
EQUIPMENT_HTTP_REQUESTS = (
"odoo.addons.maintenance_service_http_monitoring"
".models.maintenance_equipment.http_requests"
)
SERVICE_INSTANCE_SLEEP = (
"odoo.addons.maintenance_service_http_monitoring.models.service_instance.time.sleep"
)
def _mock_response(status_code):
response = MagicMock()
response.status_code = status_code
return response
class TestHttpMonitoring(TransactionCase):
def setUp(self):
super().setUp()
team = self.env["maintenance.team"].search([], limit=1)
self.equipment = self.env["maintenance.equipment"].create(
{
"name": "Test Server",
"maintenance_team_id": team.id if team else False,
}
)
self.service = self.env["service"].create({"name": "Test Service"})
self.service_instance = self.env["service.instance"].create(
{
"equipment_id": self.equipment.id,
"service_id": self.service.id,
"service_url": "https://example.com",
}
)
# ------------------------------------------------------------------
# Test 1 -- HTTP 200 -> service marked OK
# ------------------------------------------------------------------
def test_http_200_sets_status_ok(self):
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
mock_requests.get.return_value = _mock_response(200)
mock_requests.exceptions.RequestException = Exception
self.service_instance.check_http_status()
self.assertTrue(self.service_instance.http_status_ok)
self.assertEqual(self.service_instance.last_http_status_code, 200)
self.assertIsNotNone(self.service_instance.last_http_check_date)
# ------------------------------------------------------------------
# Test 2 -- Two KO passes -> maintenance.request created on the service
# ------------------------------------------------------------------
def test_http_500_creates_maintenance_request(self):
with (
patch(SERVICE_INSTANCE_REQUESTS) as mock_requests,
patch(SERVICE_INSTANCE_SLEEP),
):
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
self.env["service.instance"].cron_check_http_services()
self.assertFalse(self.service_instance.http_status_ok)
self.assertEqual(self.service_instance.last_http_status_code, 500)
request = self.service_instance.http_maintenance_request
self.assertTrue(request)
self.assertEqual(request.name, f"[HTTP KO] {self.service_instance.service_url}")
self.assertEqual(request.priority, "2")
self.assertEqual(request.maintenance_type, "corrective")
self.assertIn("HTTP 500", request.description)
self.assertIn(self.service_instance.service_url, request.description)
# ------------------------------------------------------------------
# Test 3 -- Network error -> KO with code -1
# ------------------------------------------------------------------
def test_network_error_sets_status_ko_and_minus_one(self):
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
mock_requests.get.side_effect = Exception("connection refused")
mock_requests.exceptions.RequestException = Exception
self.service_instance.check_http_status()
self.assertFalse(self.service_instance.http_status_ok)
self.assertEqual(self.service_instance.last_http_status_code, -1)
# ------------------------------------------------------------------
# Test 4 -- Two consecutive cron runs KO -> no duplicate request
# ------------------------------------------------------------------
def test_no_duplicate_request_on_repeated_failure(self):
with (
patch(SERVICE_INSTANCE_REQUESTS) as mock_requests,
patch(SERVICE_INSTANCE_SLEEP),
):
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
self.env["service.instance"].cron_check_http_services()
request_1 = self.service_instance.http_maintenance_request
self.assertTrue(request_1)
with (
patch(SERVICE_INSTANCE_REQUESTS) as mock_requests,
patch(SERVICE_INSTANCE_SLEEP),
):
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
self.env["service.instance"].cron_check_http_services()
self.assertEqual(self.service_instance.http_maintenance_request, request_1)
self.assertEqual(
self.env["maintenance.request"].search_count(
[("equipment_id", "=", self.equipment.id)]
),
1,
)
# ------------------------------------------------------------------
# Test 5 -- Equipment in maintenance mode -> cron skips it
# ------------------------------------------------------------------
def test_maintenance_mode_skips_http_check(self):
self.equipment.write({"maintenance_mode": True})
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
self.env["service.instance"].cron_check_http_services()
mock_requests.get.assert_not_called()
self.assertFalse(self.service_instance.last_http_check_date)
# ------------------------------------------------------------------
# Test 6 -- Expired maintenance mode -> cron deactivates it
# ------------------------------------------------------------------
def test_maintenance_mode_auto_expiry(self):
past = fields.Datetime.now() - timedelta(hours=1)
self.equipment.write(
{
"maintenance_mode": True,
"maintenance_mode_start": past - timedelta(hours=4),
"maintenance_mode_end": past,
}
)
self.assertTrue(self.equipment.maintenance_mode)
self.env["maintenance.equipment"].cron_deactivate_expired_maintenance_mode()
self.assertFalse(self.equipment.maintenance_mode)
self.assertFalse(self.equipment.maintenance_mode_start)
self.assertFalse(self.equipment.maintenance_mode_end)
# ------------------------------------------------------------------
# Test 7 -- Service without URL -> ignored by cron
# ------------------------------------------------------------------
def test_service_without_url_is_ignored(self):
self.service_instance.write({"service_url": False})
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
mock_requests.get.return_value = _mock_response(200)
mock_requests.exceptions.RequestException = Exception
self.env["service.instance"].cron_check_http_services()
mock_requests.get.assert_not_called()
self.assertFalse(self.service_instance.last_http_check_date)
# ------------------------------------------------------------------
# Test 8 -- HTTP 404 (non-exception) -> KO with correct code
# ------------------------------------------------------------------
def test_http_non_200_non_exception(self):
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
mock_requests.get.return_value = _mock_response(404)
mock_requests.exceptions.RequestException = Exception
self.service_instance.check_http_status()
self.assertFalse(self.service_instance.http_status_ok)
self.assertEqual(self.service_instance.last_http_status_code, 404)
# ------------------------------------------------------------------
# Test 9 -- Webhook called when a new maintenance.request is created
# ------------------------------------------------------------------
def test_webhook_called_on_new_request(self):
self.env["ir.config_parameter"].sudo().set_param(
"maintenance_service_http_monitoring.webhook_url",
"https://webhook.example.com/hook",
)
with (
patch(SERVICE_INSTANCE_REQUESTS) as mock_requests,
patch(SERVICE_INSTANCE_SLEEP),
patch(EQUIPMENT_HTTP_REQUESTS) as mock_http,
):
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
self.env["service.instance"].cron_check_http_services()
mock_http.post.assert_called_once()
call_kwargs = mock_http.post.call_args
payload = call_kwargs.kwargs.get("json") or call_kwargs[1].get("json")
self.assertEqual(payload["equipment"], self.equipment.name)
# ------------------------------------------------------------------
# Test 10 -- Webhook skipped when no URL configured
# ------------------------------------------------------------------
def test_webhook_skipped_when_no_url(self):
self.env["ir.config_parameter"].sudo().set_param(
"maintenance_service_http_monitoring.webhook_url", ""
)
with (
patch(SERVICE_INSTANCE_REQUESTS) as mock_requests,
patch(SERVICE_INSTANCE_SLEEP),
patch(EQUIPMENT_HTTP_REQUESTS) as mock_http,
):
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
self.env["service.instance"].cron_check_http_services()
mock_http.post.assert_not_called()
# ------------------------------------------------------------------
# Test 11 -- Service without equipment -> check_http_status skips it
# ------------------------------------------------------------------
def test_service_without_equipment_is_ignored(self):
self.service_instance.write({"equipment_id": False})
with patch(SERVICE_INSTANCE_REQUESTS) as mock_requests:
mock_requests.get.return_value = _mock_response(200)
mock_requests.exceptions.RequestException = Exception
self.service_instance.check_http_status()
mock_requests.get.assert_not_called()
self.assertFalse(self.service_instance.last_http_check_date)
# ------------------------------------------------------------------
# Test 12 -- Transient failure (KO pass 1, OK pass 2) -> no request created
# ------------------------------------------------------------------
def test_transient_failure_no_request_created(self):
with (
patch(SERVICE_INSTANCE_REQUESTS) as mock_requests,
patch(SERVICE_INSTANCE_SLEEP),
):
mock_requests.get.side_effect = [
_mock_response(500), # pass 1: KO
_mock_response(200), # pass 2 (retry): OK
]
mock_requests.exceptions.RequestException = Exception
self.env["service.instance"].cron_check_http_services()
self.assertTrue(self.service_instance.http_status_ok)
self.assertEqual(self.service_instance.last_http_status_code, 200)
self.assertFalse(self.service_instance.http_maintenance_request)
self.assertEqual(
self.env["maintenance.request"].search_count(
[("equipment_id", "=", self.equipment.id)]
),
0,
)
# ------------------------------------------------------------------
# Test 13 -- Confirmed failure (KO pass 1 and 2) -> request created
# ------------------------------------------------------------------
def test_confirmed_failure_creates_request(self):
with (
patch(SERVICE_INSTANCE_REQUESTS) as mock_requests,
patch(SERVICE_INSTANCE_SLEEP) as mock_sleep,
):
mock_requests.get.return_value = _mock_response(503)
mock_requests.exceptions.RequestException = Exception
self.env["service.instance"].cron_check_http_services()
mock_sleep.assert_called_once_with(2)
self.assertEqual(mock_requests.get.call_count, 2)
self.assertFalse(self.service_instance.http_status_ok)
self.assertEqual(self.service_instance.last_http_status_code, 503)
self.assertTrue(self.service_instance.http_maintenance_request)
# ------------------------------------------------------------------
# Test 14 -- 2 KO services on same equipment -> 2 distinct requests
# ------------------------------------------------------------------
def test_two_ko_services_same_equipment_create_two_requests(self):
service2 = self.env["service"].create({"name": "Test Service 2"})
service_instance2 = self.env["service.instance"].create(
{
"equipment_id": self.equipment.id,
"service_id": service2.id,
"service_url": "https://other.example.com",
}
)
with (
patch(SERVICE_INSTANCE_REQUESTS) as mock_requests,
patch(SERVICE_INSTANCE_SLEEP),
):
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
self.env["service.instance"].cron_check_http_services()
req1 = self.service_instance.http_maintenance_request
req2 = service_instance2.http_maintenance_request
self.assertTrue(req1)
self.assertTrue(req2)
self.assertNotEqual(req1, req2)
self.assertEqual(req1.name, f"[HTTP KO] {self.service_instance.service_url}")
self.assertEqual(req2.name, f"[HTTP KO] {service_instance2.service_url}")
self.assertEqual(
self.env["maintenance.request"].search_count(
[("equipment_id", "=", self.equipment.id)]
),
2,
)
# ------------------------------------------------------------------
# Test 15 -- Service recovery closes the open request and posts a note
# ------------------------------------------------------------------
def test_service_recovery_closes_request(self):
# First cron run: service is KO -> request created
with (
patch(SERVICE_INSTANCE_REQUESTS) as mock_requests,
patch(SERVICE_INSTANCE_SLEEP),
):
mock_requests.get.return_value = _mock_response(500)
mock_requests.exceptions.RequestException = Exception
self.env["service.instance"].cron_check_http_services()
request = self.service_instance.http_maintenance_request
self.assertTrue(request)
self.assertFalse(request.stage_id.done)
# Second cron run: service is back OK -> request auto-closed
with (
patch(SERVICE_INSTANCE_REQUESTS) as mock_requests,
patch(SERVICE_INSTANCE_SLEEP),
):
mock_requests.get.return_value = _mock_response(200)
mock_requests.exceptions.RequestException = Exception
self.env["service.instance"].cron_check_http_services()
# Request must be in a done stage
self.assertTrue(request.stage_id.done)
# http_maintenance_request must be cleared on the service instance
self.assertFalse(self.service_instance.http_maintenance_request)
# A chatter note must have been posted mentioning the service URL
notes = request.message_ids.filtered(
lambda m: self.service_instance.service_url in (m.body or "")
)
self.assertTrue(notes)
# ------------------------------------------------------------------
# Test 16 -- No open request -> _close_http_maintenance_request is a no-op
# ------------------------------------------------------------------
def test_no_close_when_no_open_request(self):
# Service is OK from the start, no request exists
self.assertFalse(self.service_instance.http_maintenance_request)
# Calling close directly must not raise
self.service_instance._close_http_maintenance_request()
self.assertFalse(self.service_instance.http_maintenance_request)

View File

@@ -36,7 +36,6 @@
<field name="maintenance_mode" />
<field name="maintenance_mode_start" />
<field name="maintenance_mode_end" />
<field name="http_maintenance_request" readonly="1" />
</group>
</page>
</xpath>

View File

@@ -17,6 +17,7 @@
<field name="last_http_check_date" />
<field name="last_http_status_code" />
<field name="http_status_ok" />
<field name="http_maintenance_request" optional="hide" />
</field>
</field>
</record>