Elabore/maintenance-tools
8
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

[IMP] maintenance_server_monitoring:

Browse Source 
* add cron
* ssh in other module
* new maintenance request if error
* disk usage test
This commit is contained in:
clementthomas
2024-04-03 18:13:02 +02:00
parent 5f9119c4e8
commit b18940fe56
11 changed files with 445 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
maintenance_server_monitoring/__manifest__.py
View File

@@ -14,6 +14,7 @@
"depends": [
"base",
"maintenance",
"maintenance_server_ssh"
],
"qweb": [
# "static/src/xml/*.xml",
@@ -24,6 +25,7 @@
# always loaded
"data": [
"views/maintenance_equipment_views.xml",
"data/cron.xml",
],
# only loaded in demonstration mode
"demo": [],

12
maintenance_server_monitoring/data/cron.xml Normal file
View File

@@ -0,0 +1,12 @@
<odoo>
<record id="ir_cron_server_monitoring" model="ir.cron">
<field name="name">Server Monitoring : check all equipments</field>
<field name="model_id" ref="model_maintenance_equipment"/>
<field name="state">code</field>
<field name="code">model.cron_monitoring_test()</field>
<field name="interval_number">1</field>
<field name="interval_type">minutes</field>
<field name="numbercall">-1</field>
<field eval="False" name="doall"/>
</record>
</odoo>

313
maintenance_server_monitoring/models/maintenance_equipment.py
View File

@@ -1,46 +1,303 @@
from odoo import fields, models
from odoo import fields, models, api
import subprocess
import sys
import psutil
from io import StringIO
AVAILABLE_MEMORY_PERCENT_COMMAND = "free | grep Mem | awk '{print $3/$2 * 100.0}'"
MIN_AVAILABLE_MEMORY_PERCENT_WARNING = 20
MIN_AVAILABLE_MEMORY_PERCENT_ERROR = 5
USED_DISK_SPACE_COMMAND = "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 5 -d ' ' | cut -f 1 -d %"
MAX_USED_DISK_SPACE_WARNING = 70
MAX_USED_DISK_SPACE_ERROR = 90
MAX_PING_MS_WARNING = 1000
MAX_PING_MS_ERROR = 5000
"""
if you want to add a new test :
* add new field to MaintenanceEquipment (named {fieldname} below)
* add a new function named test_{fieldname} which return a filled MonitoringTest class with :
-> log = logs you want to appear in logs
-> result = value which will be set to {fieldname}
-> error = MonitoringTest.ERROR or MonitoringTest.WARNING to generate maintenance request
* add requirements if necessary in install_dependencies function
* call your function in monitoring_test() with a simple launch_test({fieldname}, *args)
if needed, *args can be passed by parameters to your test function
"""
class MaintenanceEquipment(models.Model):
_inherit = 'maintenance.equipment'
server_domain = fields.Char('Server Domain')
ssh_private_key_path = fields.Char("SSH private key path", default="/opt/odoo/auto/dev/ssh_keys/id_rsa")
last_monitoring_test_date = fields.Datetime('Date of last monitoring test', readonly=True)
ping_ok = fields.Boolean("Ping ok", readonly=True)
available_memory_percent = fields.Float('Percent of available memory', readonly=True)
used_disk_space = fields.Float('Percent of used disk space', readonly=True)
log = fields.Html("Log", readonly=True)
error_maintenance_request = fields.Many2one('maintenance.request', "Error maintenance request")
warning_maintenance_request = fields.Many2one('maintenance.request', "Warning maintenance request")
class MonitoringTest:
"""Class to make the tests
"""
WARNING = "warning"
ERROR = "error"
def __init__(self, name):
self.name = name #name of the test
self.result = 0 #result of the test
self.log = "" #logs of the test
self.date = fields.Datetime.now() #date of the test
self.error = "" #errors of the test
def add_to_log(self, text):
"""
add a new line to logs composed with DATE > TEST NAME > WHAT TO LOG
"""
self.log += f"{self.date} > {self.name} > {text}\n"
@api.model
def cron_monitoring_test(self):
"""cron launch test on all equipments
"""
self.search([]).monitoring_test()
def monitoring_test(self):
def launch_test(attribute, *test_function_args):
"""run test function with name = test_[attribute]
associate result of test to equipment
write logs of test
Args:
attribute (string): attribute of MaintenanceEquipment we want to test
Returns:
MonitoringTest: returned by test function
"""
test_function = getattr(equipment,"test_"+attribute)
test = test_function(*test_function_args)
setattr(equipment, attribute, test.result)
log.write(test.log)
tests.append(test)
return test
for equipment in self:
#clear log
log = StringIO() #we use StingIO instead of string to use mutable object
tests = []
#PIP dependant test
log.write(equipment.install_dependencies().log) # launch_test is not used, only logs are necessary
#run ping test
launch_test("ping_ok")
#SSH dependant test
try:
ssh = self.get_ssh_connection() #ssh connection given by maintenance_server_ssh module
except Exception as e:
ssh = False
log.write(f"{fields.Datetime.now()} > SSH > connection failed {e}\n")
if ssh:
#test available memory
launch_test("available_memory_percent", ssh)
#test disk usage
launch_test("used_disk_space", ssh)
else:
equipment.available_memory_percent = -1 #set -1 by convention if error
equipment.used_disk_space = -1 #set -1 by convention if error
#set test date
equipment.last_monitoring_test_date = fields.Datetime.now()
#write logs
log.seek(0) #log is a StringIO so seek to beginning before read
new_log = f'📣 {fields.Datetime.now()}\n{log.read()}\n'
new_log = new_log.replace("\n","<br />") # log field is HTML, so format lines
equipment.log = f'{new_log}<br />{equipment.log}'[:10000] #limit logs to 10000 characters
#if error create maintenance request
error = warning =False
if any(test.error == test.ERROR for test in tests):
error = True # if any arror in tests
elif any(test.error == test.WARNING for test in tests):
warning = True # if any warning in tests
if error or warning:
# check if error or warning request already exists before creating a new one
# if only a warning exists, error request will be created anyway
if (error and not equipment.error_maintenance_request) \
or (warning and not equipment.warning_maintenance_request and not equipment.error_maintenance_request):
maintenance_request = self.env['maintenance.request'].create({
"name":f'[{"ERROR" if error else "WARNING"}] {equipment.name}',
"equipment_id":equipment.id,
"employee_id":equipment.employee_id,
"user_id":equipment.technician_user_id,
"maintenance_team_id":equipment.maintenance_team_id.id or self.env["maintenance.team"].search([], limit=1),
"priority":'2' if error else '3',
"maintenance_type":"corrective" if error else "preventive",
"description":new_log
})
if error:
equipment.error_maintenance_request = maintenance_request
else:
equipment.warning_maintenance_request = maintenance_request
def install_dependencies(self):
if "ping3" not in sys.modules:
command = ['pip','install',"ping3"]
response = subprocess.call(command)
return response
"""
install dependencies needed to do all tests, as python or shell programs
def monitoring_test(self):
self.install_dependencies()
self.test_ping()
self.test_available_memory_percent()
self.last_monitoring_test_date = fields.Datetime.now()
return
Returns:
MonitoringTest: representing current test with result=0 if not error
"""
monitoring_test = self.MonitoringTest("install dependencies")
if "ping3" in sys.modules:
monitoring_test.add_to_log("ping3 already satisfied")
monitoring_test.result = 0
else:
error = True
try:
command = ['pip','install',"ping3"]
response = subprocess.call(command) # run "pip install ping3" command
if response == 0:
error = False
except Exception as e:
error = str(e)
if error:
monitoring_test.add_to_log(f"🚨 ping3 : unable to install : {error}")
monitoring_test.result = -1
monitoring_test.error = monitoring_test.ERROR
else:
monitoring_test.add_to_log("ping3 installation successful")
monitoring_test.result = 0
return monitoring_test
def test_available_memory_percent(self):
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(self.server_domain, username="root", key_filename=self.ssh_private_key_path)
_stdin, stdout, _stderr = ssh.exec_command("free | grep Mem | awk '{print $3/$2 * 100.0}'")
self.available_memory_percent = float(stdout.read().decode())
def test_ping(self):
from ping3 import ping
def test_available_memory_percent(self, ssh):
"""
test available memory with a bash command called by ssh
Args:
ssh (paramiko.SSHClient): ssh client
Returns:
MonitoringTest: representing current test with :
* result = -2 if error
* result = percent of available memory if no error
* error defined with ERROR or WARNING depending on result comparaison
with MIN_AVAILABLE_MEMORY_PERCENT_WARNING and MIN_AVAILABLE_MEMORY_PERCENT_ERROR
* log file
"""
try:
test = self.MonitoringTest("Available memory percent")
_stdin, stdout, _stderr = ssh.exec_command(AVAILABLE_MEMORY_PERCENT_COMMAND)
test.result = float(stdout.read().decode())
if test.result > MIN_AVAILABLE_MEMORY_PERCENT_WARNING:
test.add_to_log(f"OK : {test.result}% available")
elif test.result > MIN_AVAILABLE_MEMORY_PERCENT_ERROR: #memory between warning and error step
test.add_to_log(f"🔥 WARNING : {test.result}% available")
test.error = test.WARNING
else:
test.add_to_log(f"🚨 ERROR : {test.result}% available") #memory available lower than error step
test.error = test.ERROR
except Exception as e:
test.result = -2
test.add_to_log(f"🚨 ERROR : {e}")
return test
def test_used_disk_space(self, ssh):
"""
test Used disk space with a bash command called by ssh
Args:
ssh (paramiko.SSHClient): ssh client
Returns:
MonitoringTest: representing current test with :
* result = -2 if error
* result = percent of Used disk space if no error
* error defined with ERROR or WARNING depending on result comparaison
with MAX_USED_DISK_SPACE_WARNING and MAX_USED_DISK_SPACE_ERROR
* log file
"""
try:
test = self.MonitoringTest("Used disk space")
_stdin, stdout, _stderr = ssh.exec_command(USED_DISK_SPACE_COMMAND)
test.result = float(stdout.read().decode())
if test.result < MAX_USED_DISK_SPACE_WARNING:
test.add_to_log(f"OK : {test.result}% used")
elif test.result < MAX_USED_DISK_SPACE_ERROR:
test.add_to_log(f"🔥 WARNING : {test.result}% used") # disk usage between WARNING and ERROR steps
test.error = test.WARNING
else:
test.add_to_log(f"🚨 ERROR : {test.result}% used") # disk usage higher than ERROR steps
test.error = test.ERROR
except Exception as e:
test.result = -2
test.add_to_log(f"🚨 ERROR : {e}")
return test
def test_ping_ok(self):
"""
test PING with ping3 library
Returns:
MonitoringTest: representing current test with :
* result = False if error
* result = True if no error
* error defined with ERROR or WARNING depending on ping time comparaison
with MAX_PING_MS_WARNING and MAX_PING_MS_ERROR
* log file
"""
test = self.MonitoringTest("Ping")
try:
from ping3 import ping
except Exception as e:
test.result = False
test.add_to_log(f"🚨 ping3 dependencie not satisfied : {e}")
test.error = test.ERROR
return
hostname = self.server_domain
r = ping(hostname)
try:
r = ping(hostname)
except Exception as e:
test.result = False
test.error = test.ERROR
test.add_to_log(f"🚨 unable to call ping ! > {e}")
if r:
self.ping_ok = True
test.result = True
ping_ms = int(r*1000)
if ping_ms < MAX_PING_MS_WARNING:
test.add_to_log("PING OK in "+str(ping_ms)+"ms")
elif ping_ms < MAX_PING_MS_ERROR:
test.add_to_log("🔥 WARNING : PING OK in "+str(ping_ms)+"ms")
test.error = test.WARNING
else:
test.add_to_log("🚨 ERROR : PING OK in "+str(ping_ms)+"ms")
test.error = test.ERROR
else:
self.ping_ok = False
return
test.result = False
test.error = test.ERROR
test.add_to_log("🚨 PING FAILED")
return test

17
maintenance_server_monitoring/views/maintenance_equipment_views.xml
View File

@@ -6,17 +6,17 @@
<field name="inherit_id" ref="maintenance.hr_equipment_view_form" />
<field name="arch" type="xml">
<xpath expr="//notebook" position="inside">
<page name="monitoring" string="Monitoring">
<group name="monitoring" string="Configuration">
<field name="server_domain" />
<field name="ssh_private_key_path" />
</group>
<page name="monitoring" string="Monitoring">
<group name="monitoring_test" string="Test">
<field name="last_monitoring_test_date" />
<field name="ping_ok" />
<field name="available_memory_percent" />
<field name="available_memory_percent" />
<field name="used_disk_space" />
<button name="monitoring_test" type="object" string="Test" />
</group>
<group name="monitoring_log" string="Log">
<field name="log" />
</group>
</page>
</xpath>
</field>
@@ -28,9 +28,10 @@
<field name="model">maintenance.equipment</field>
<field name="inherit_id" ref="maintenance.hr_equipment_view_tree" />
<field name="arch" type="xml">
<xpath expr="//field[@name='category_id']" position="after">
<field name="server_domain" optional="hide" />
<xpath expr="//field[@name='category_id']" position="after">
<field name="ping_ok" optional="hide" />
<field name="available_memory_percent" optional="hide" />
<field name="used_disk_space" optional="hide" />
</xpath>
</field>
</record>

2
maintenance_server_ssh/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
*.*~
*pyc

44
maintenance_server_ssh/README.rst Normal file
View File

@@ -0,0 +1,44 @@
======================================
maintenance_server_ssh
======================================
Create an SSH remote connection for maintenance equipment, usable for other modules
Installation
============
Use Odoo normal module installation procedure to install
``maintenance_server_ssh``.
Known issues / Roadmap
======================
None yet.
Bug Tracker
===========
Bugs are tracked on `our issues website <https://github.com/elabore-coop/maintenance-tools/issues>`_. In case of
trouble, please check there if your issue has already been
reported. If you spotted it first, help us smashing it by providing a
detailed and welcomed feedback.
Credits
=======
Contributors
------------
* Clément Thomas
Funders
-------
The development of this module has been financially supported by:
* Elabore (https://elabore.coop)
Maintainer
----------
This module is maintained by Elabore.

3
maintenance_server_ssh/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
# -*- coding: utf-8 -*-
from . import models

37
maintenance_server_ssh/__manifest__.py Normal file
View File

@@ -0,0 +1,37 @@
# Copyright 2023 Stéphan Sainléger (Elabore)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
{
"name": "maintenance_server_ssh",
"version": "16.0.1.0.0",
"author": "Elabore",
"website": "https://elabore.coop",
"maintainer": "Clément Thomas",
"license": "AGPL-3",
"category": "Tools",
"summary": "Monitor some data on remote hosts",
# any module necessary for this one to work correctly
"depends": [
"base",
"maintenance",
],
"qweb": [
# "static/src/xml/*.xml",
],
"external_dependencies": {
"python": [],
},
# always loaded
"data": [
"views/maintenance_equipment_views.xml",
],
# only loaded in demonstration mode
"demo": [],
"js": [],
"css": [],
"installable": True,
# Install this module automatically if all dependency have been previously
# and independently installed. Used for synergetic or glue modules.
"auto_install": False,
"application": False,
}

1
maintenance_server_ssh/models/__init__.py Normal file
View File

@@ -0,0 +1 @@
from . import maintenance_equipment

20
maintenance_server_ssh/models/maintenance_equipment.py Normal file
View File

@@ -0,0 +1,20 @@
from odoo import fields, models
import subprocess
import sys
import psutil
class MaintenanceEquipment(models.Model):
_inherit = 'maintenance.equipment'
server_domain = fields.Char('Server Domain')
ssh_private_key_path = fields.Char("SSH private key path", default="/opt/odoo/auto/dev/ssh_keys/id_rsa")
def get_ssh_connection(self):
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(self.server_domain, username="root", key_filename=self.ssh_private_key_path)
return ssh

30
maintenance_server_ssh/views/maintenance_equipment_views.xml Normal file
View File

@@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<odoo>
<record id="equipment_view_form_server_inherit" model="ir.ui.view">
<field name="name">equipment.form.server.inherit</field>
<field name="model">maintenance.equipment</field>
<field name="inherit_id" ref="maintenance.hr_equipment_view_form" />
<field name="arch" type="xml">
<xpath expr="//notebook" position="inside">
<page name="ssh" string="SSH">
<group name="ssh_connection" string="SSH Connection">
<field name="server_domain" />
<field name="ssh_private_key_path" />
</group>
</page>
</xpath>
</field>
</record>
<record id="equipment_view_tree_server_inherit" model="ir.ui.view">
<field name="name">equipment.tree.server.inherit</field>
<field name="model">maintenance.equipment</field>
<field name="inherit_id" ref="maintenance.hr_equipment_view_tree" />
<field name="arch" type="xml">
<xpath expr="//field[@name='category_id']" position="after">
<field name="server_domain" optional="hide" />
</xpath>
</field>
</record>
</odoo>