[IMP] maintenance_server_monitoring:

* add cron
* ssh in other module
* new maintenance request if error
* disk usage test

maintenance_server_monitoring/__manifest__.py

@@ -14,6 +14,7 @@
     "depends": [
         "base",
         "maintenance",
+        "maintenance_server_ssh"
     ],
     "qweb": [
         # "static/src/xml/*.xml",
@@ -24,6 +25,7 @@
     # always loaded
     "data": [        
         "views/maintenance_equipment_views.xml",        
+        "data/cron.xml",        
     ],
     # only loaded in demonstration mode
     "demo": [],

maintenance_server_monitoring/data/cron.xml

@@ -0,0 +1,12 @@
+<odoo>
+    <record id="ir_cron_server_monitoring" model="ir.cron">
+        <field name="name">Server Monitoring : check all equipments</field>
+        <field name="model_id" ref="model_maintenance_equipment"/>
+        <field name="state">code</field>
+        <field name="code">model.cron_monitoring_test()</field>
+        <field name="interval_number">1</field>
+        <field name="interval_type">minutes</field>
+        <field name="numbercall">-1</field>
+        <field eval="False" name="doall"/>
+    </record>
+</odoo>

maintenance_server_monitoring/models/maintenance_equipment.py

@@ -1,46 +1,303 @@
-from odoo import fields, models
+from odoo import fields, models, api
 import subprocess
 import sys
 import psutil
+from io import StringIO
+
+AVAILABLE_MEMORY_PERCENT_COMMAND = "free | grep Mem | awk '{print $3/$2 * 100.0}'"
+MIN_AVAILABLE_MEMORY_PERCENT_WARNING = 20
+MIN_AVAILABLE_MEMORY_PERCENT_ERROR = 5
+
+USED_DISK_SPACE_COMMAND = "df /srv -h | tail -n +2 | sed -r 's/ +/ /g' | cut -f 5 -d ' ' | cut -f 1 -d %"
+MAX_USED_DISK_SPACE_WARNING = 70
+MAX_USED_DISK_SPACE_ERROR = 90
+
+MAX_PING_MS_WARNING = 1000
+MAX_PING_MS_ERROR = 5000
+
+
+"""
+if you want to add a new test : 
+    * add new field to MaintenanceEquipment (named {fieldname} below)
+    * add a new function named test_{fieldname} which return a filled MonitoringTest class with :
+        -> log = logs you want to appear in logs
+        -> result = value which will be set to {fieldname}
+        -> error = MonitoringTest.ERROR or MonitoringTest.WARNING to generate maintenance request
+    * add requirements if necessary in install_dependencies function
+    * call your function in monitoring_test() with a simple launch_test({fieldname}, *args)
+        if needed, *args can be passed by parameters to your test function
+
+
+"""
 
 
 class MaintenanceEquipment(models.Model):
     _inherit = 'maintenance.equipment'
-
+    
-    server_domain = fields.Char('Server Domain')    
-    ssh_private_key_path = fields.Char("SSH private key path", default="/opt/odoo/auto/dev/ssh_keys/id_rsa")
     last_monitoring_test_date = fields.Datetime('Date of last monitoring test', readonly=True)
     ping_ok = fields.Boolean("Ping ok", readonly=True)
     available_memory_percent = fields.Float('Percent of available memory', readonly=True)
+    used_disk_space = fields.Float('Percent of used disk space', readonly=True)
+    log = fields.Html("Log", readonly=True)
+    error_maintenance_request = fields.Many2one('maintenance.request', "Error maintenance request")
+    warning_maintenance_request = fields.Many2one('maintenance.request', "Warning maintenance request")
 
+
+    class MonitoringTest:        
+        """Class to make the tests
+        """
+        WARNING = "warning"
+        ERROR = "error"
+
+        def __init__(self, name):
+            self.name = name #name of the test
+            self.result = 0 #result of the test
+            self.log = "" #logs of the test
+            self.date = fields.Datetime.now() #date of the test
+            self.error = "" #errors of the test
+
+        def add_to_log(self, text):   
+            """
+                add a new line to logs composed with DATE > TEST NAME > WHAT TO LOG
+            """     
+            self.log += f"{self.date} > {self.name} > {text}\n"
+        
+    @api.model
+    def cron_monitoring_test(self):
+        """cron launch test on all equipments
+        """
+        self.search([]).monitoring_test()
+    
+    def monitoring_test(self):    
+
+        def launch_test(attribute, *test_function_args):
+            """run test function with name = test_[attribute]
+            associate result of test to equipment
+            write logs of test
+
+
+            Args:
+                attribute (string): attribute of MaintenanceEquipment we want to test
+
+            Returns:
+                MonitoringTest: returned by test function
+            """
+            test_function = getattr(equipment,"test_"+attribute)
+            test = test_function(*test_function_args)            
+            setattr(equipment, attribute, test.result)
+            log.write(test.log)            
+            tests.append(test)
+            return test
+
+
+        for equipment in self:
+            #clear log            
+            log = StringIO() #we use StingIO instead of string to use mutable object 
+
+            tests = []
+
+            #PIP dependant test
+            log.write(equipment.install_dependencies().log) # launch_test is not used, only logs are necessary
+
+            #run ping test
+            launch_test("ping_ok")
+
+            #SSH dependant test
+            try:
+                ssh = self.get_ssh_connection() #ssh connection given by maintenance_server_ssh module
+            except Exception as e:
+                ssh = False
+                log.write(f"{fields.Datetime.now()} > SSH > connection failed {e}\n")
+                
+            
+            if ssh:
+                #test available memory
+                launch_test("available_memory_percent", ssh)         
+
+                #test disk usage
+                launch_test("used_disk_space", ssh)
+            else:
+                equipment.available_memory_percent = -1 #set -1 by convention if error
+                equipment.used_disk_space = -1 #set -1 by convention if error         
+
+            #set test date
+            equipment.last_monitoring_test_date = fields.Datetime.now()
+            
+            #write logs
+            log.seek(0) #log is a StringIO so seek to beginning before read
+            new_log = f'📣 {fields.Datetime.now()}\n{log.read()}\n'   
+            new_log = new_log.replace("\n","<br />") # log field is HTML, so format lines 
+            equipment.log = f'{new_log}<br />{equipment.log}'[:10000] #limit logs to 10000 characters
+
+            #if error create maintenance request
+            error = warning =False
+            if any(test.error == test.ERROR for test in tests):
+                error = True # if any arror in tests
+            elif any(test.error == test.WARNING for test in tests):
+                warning = True # if any warning in tests
+
+            if error or warning:                
+                # check if error or warning request already exists before creating a new one
+                # if only a warning exists, error request will be created anyway
+                if (error and not equipment.error_maintenance_request) \
+                    or (warning and not equipment.warning_maintenance_request and not equipment.error_maintenance_request):
+                    maintenance_request = self.env['maintenance.request'].create({
+                        "name":f'[{"ERROR" if error else "WARNING"}] {equipment.name}',
+                        "equipment_id":equipment.id, 
+                        "employee_id":equipment.employee_id,
+                        "user_id":equipment.technician_user_id,
+                        "maintenance_team_id":equipment.maintenance_team_id.id or self.env["maintenance.team"].search([], limit=1),
+                        "priority":'2' if error else '3',
+                        "maintenance_type":"corrective" if error else "preventive", 
+                        "description":new_log
+                    })
+                    if error:
+                        equipment.error_maintenance_request = maintenance_request
+                    else:
+                        equipment.warning_maintenance_request = maintenance_request
+                
+
+    
     def install_dependencies(self):
-        if "ping3" not in sys.modules:
+        """
-            command = ['pip','install',"ping3"]
+            install dependencies needed to do all tests, as python or shell programs
-            response = subprocess.call(command)
-            return response
-    
 
-    def monitoring_test(self):
+            Returns:
-        self.install_dependencies()
+                MonitoringTest: representing current test with result=0 if not error
-        self.test_ping()
+        """        
-        self.test_available_memory_percent()
+        monitoring_test = self.MonitoringTest("install dependencies")
-        self.last_monitoring_test_date = fields.Datetime.now()
+        if "ping3" in sys.modules:
-        return
+            monitoring_test.add_to_log("ping3 already satisfied")
+            monitoring_test.result = 0
+        else:
+            error = True
+            try:
+                command = ['pip','install',"ping3"]
+                response = subprocess.call(command) # run "pip install ping3" command
+                if response == 0:
+                    error = False
+            except Exception as e:
+                error = str(e)
+                
+            if error:
+                monitoring_test.add_to_log(f"🚨 ping3 : unable to install : {error}")
+                monitoring_test.result = -1
+                monitoring_test.error = monitoring_test.ERROR
+            else:
+                monitoring_test.add_to_log("ping3 installation successful")
+                monitoring_test.result = 0
+        
+        return monitoring_test
     
-    def test_available_memory_percent(self):
-        import paramiko
-        ssh = paramiko.SSHClient()
-        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        ssh.connect(self.server_domain, username="root", key_filename=self.ssh_private_key_path)
-        _stdin, stdout, _stderr = ssh.exec_command("free | grep Mem | awk '{print $3/$2 * 100.0}'")
-        self.available_memory_percent = float(stdout.read().decode())        
     
-    def test_ping(self):                
+    def test_available_memory_percent(self, ssh):    
-        from ping3 import ping
+        """
+            test available memory with a bash command called by ssh
+
+            Args:
+                ssh (paramiko.SSHClient): ssh client
+
+            Returns:
+                MonitoringTest: representing current test with :
+                    * result = -2 if error
+                    * result = percent of available memory if no error
+                    * error defined with ERROR or WARNING depending on result comparaison 
+                        with MIN_AVAILABLE_MEMORY_PERCENT_WARNING and MIN_AVAILABLE_MEMORY_PERCENT_ERROR
+                    * log file
+        """      
+        try:      
+            test = self.MonitoringTest("Available memory percent")      
+            _stdin, stdout, _stderr = ssh.exec_command(AVAILABLE_MEMORY_PERCENT_COMMAND)
+            test.result = float(stdout.read().decode())
+            if test.result > MIN_AVAILABLE_MEMORY_PERCENT_WARNING:
+                test.add_to_log(f"OK : {test.result}% available")
+            elif test.result > MIN_AVAILABLE_MEMORY_PERCENT_ERROR: #memory between warning and error step
+                test.add_to_log(f"🔥 WARNING : {test.result}% available")
+                test.error = test.WARNING                
+            else:
+                test.add_to_log(f"🚨 ERROR : {test.result}% available") #memory available lower than error step
+                test.error = test.ERROR
+        except Exception as e:
+            test.result = -2
+            test.add_to_log(f"🚨 ERROR : {e}")
+        return test
+
+        
+    def test_used_disk_space(self, ssh):        
+        """
+            test Used disk space with a bash command called by ssh
+
+            Args:
+                ssh (paramiko.SSHClient): ssh client
+
+            Returns:
+                MonitoringTest: representing current test with :
+                    * result = -2 if error
+                    * result = percent of Used disk space if no error
+                    * error defined with ERROR or WARNING depending on result comparaison 
+                        with MAX_USED_DISK_SPACE_WARNING and MAX_USED_DISK_SPACE_ERROR
+                    * log file
+        """      
+        try:
+            test = self.MonitoringTest("Used disk space")      
+            _stdin, stdout, _stderr = ssh.exec_command(USED_DISK_SPACE_COMMAND)
+            test.result = float(stdout.read().decode())                    
+            if test.result < MAX_USED_DISK_SPACE_WARNING:
+                test.add_to_log(f"OK : {test.result}% used")
+            elif test.result < MAX_USED_DISK_SPACE_ERROR:
+                test.add_to_log(f"🔥 WARNING : {test.result}% used") # disk usage between WARNING and ERROR steps
+                test.error = test.WARNING
+            else:
+                test.add_to_log(f"🚨 ERROR : {test.result}% used") # disk usage higher than ERROR steps
+                test.error = test.ERROR
+                
+        except Exception as e:
+            test.result = -2
+            test.add_to_log(f"🚨 ERROR : {e}")            
+        return test
+        
+    
+    def test_ping_ok(self):
+        """
+            test PING with ping3 library
+
+            Returns:
+                MonitoringTest: representing current test with :
+                    * result = False if error
+                    * result = True if no error
+                    * error defined with ERROR or WARNING depending on ping time comparaison 
+                        with MAX_PING_MS_WARNING and MAX_PING_MS_ERROR
+                    * log file
+        """   
+        test = self.MonitoringTest("Ping")
+        try:               
+            from ping3 import ping
+        except Exception as e:            
+            test.result = False
+            test.add_to_log(f"🚨 ping3 dependencie not satisfied : {e}")
+            test.error = test.ERROR
+            return
+        
         hostname = self.server_domain
-        r = ping(hostname)
+        try:
+            r = ping(hostname)
+        except Exception as e: 
+            test.result = False
+            test.error = test.ERROR
+            test.add_to_log(f"🚨 unable to call ping ! > {e}")  
         if r:
-            self.ping_ok = True
+            test.result = True
+            ping_ms = int(r*1000)
+            if ping_ms < MAX_PING_MS_WARNING:
+                test.add_to_log("PING OK in "+str(ping_ms)+"ms")
+            elif ping_ms < MAX_PING_MS_ERROR:
+                test.add_to_log("🔥 WARNING : PING OK in "+str(ping_ms)+"ms")            
+                test.error = test.WARNING
+            else:
+                test.add_to_log("🚨 ERROR : PING OK in "+str(ping_ms)+"ms")
+                test.error = test.ERROR
         else:   
-            self.ping_ok = False
+            test.result = False
-        return
+            test.error = test.ERROR
+            test.add_to_log("🚨 PING FAILED")        
+        return test

maintenance_server_monitoring/views/maintenance_equipment_views.xml

@@ -6,17 +6,17 @@
         <field name="inherit_id" ref="maintenance.hr_equipment_view_form" />
         <field name="arch" type="xml">            
             <xpath expr="//notebook" position="inside">
-                <page name="monitoring" string="Monitoring">
+                <page name="monitoring" string="Monitoring">                    
-                    <group name="monitoring" string="Configuration">
-                        <field name="server_domain" />
-                        <field name="ssh_private_key_path" />                                                
-                    </group>
                     <group name="monitoring_test" string="Test">
                         <field name="last_monitoring_test_date" />
                         <field name="ping_ok" />  
-                        <field name="available_memory_percent" />                          
+                        <field name="available_memory_percent" />    
+                        <field name="used_disk_space" />    
                         <button name="monitoring_test" type="object" string="Test" />
                     </group>
+                    <group name="monitoring_log" string="Log">
+                        <field name="log" />
+                    </group>
                 </page>
             </xpath>
         </field>
@@ -28,9 +28,10 @@
         <field name="model">maintenance.equipment</field>
         <field name="inherit_id" ref="maintenance.hr_equipment_view_tree" />
         <field name="arch" type="xml">
-            <xpath expr="//field[@name='category_id']" position="after">
+            <xpath expr="//field[@name='category_id']" position="after">                
-                <field name="server_domain" optional="hide" />
                 <field name="ping_ok" optional="hide" />  
+                <field name="available_memory_percent" optional="hide" />    
+                <field name="used_disk_space" optional="hide" />    
             </xpath>
         </field>
     </record>

maintenance_server_ssh/.gitignore

@@ -0,0 +1,2 @@
+*.*~
+*pyc

maintenance_server_ssh/README.rst

@@ -0,0 +1,44 @@
+======================================
+maintenance_server_ssh
+======================================
+
+Create an SSH remote connection for maintenance equipment, usable for other modules
+
+Installation
+============
+
+Use Odoo normal module installation procedure to install
+``maintenance_server_ssh``.
+
+Known issues / Roadmap
+======================
+
+None yet.
+
+Bug Tracker
+===========
+
+Bugs are tracked on `our issues website <https://github.com/elabore-coop/maintenance-tools/issues>`_. In case of
+trouble, please check there if your issue has already been
+reported. If you spotted it first, help us smashing it by providing a
+detailed and welcomed feedback.
+
+Credits
+=======
+
+Contributors
+------------
+
+* Clément Thomas
+
+Funders
+-------
+
+The development of this module has been financially supported by:
+* Elabore (https://elabore.coop)
+
+
+Maintainer
+----------
+
+This module is maintained by Elabore.

maintenance_server_ssh/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from . import models

maintenance_server_ssh/__manifest__.py

@@ -0,0 +1,37 @@
+# Copyright 2023 Stéphan Sainléger (Elabore)
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+{
+    "name": "maintenance_server_ssh",
+    "version": "16.0.1.0.0",
+    "author": "Elabore",
+    "website": "https://elabore.coop",
+    "maintainer": "Clément Thomas",
+    "license": "AGPL-3",
+    "category": "Tools",
+    "summary": "Monitor some data on remote hosts",
+    # any module necessary for this one to work correctly
+    "depends": [
+        "base",
+        "maintenance",
+    ],
+    "qweb": [
+        # "static/src/xml/*.xml",
+    ],
+    "external_dependencies": {
+        "python": [],
+    },
+    # always loaded
+    "data": [        
+        "views/maintenance_equipment_views.xml",        
+    ],
+    # only loaded in demonstration mode
+    "demo": [],
+    "js": [],
+    "css": [],
+    "installable": True,
+    # Install this module automatically if all dependency have been previously
+    # and independently installed.  Used for synergetic or glue modules.
+    "auto_install": False,
+    "application": False,
+}

maintenance_server_ssh/models/__init__.py

@@ -0,0 +1 @@
+from . import maintenance_equipment

maintenance_server_ssh/models/maintenance_equipment.py

@@ -0,0 +1,20 @@
+from odoo import fields, models
+import subprocess
+import sys
+import psutil
+
+
+class MaintenanceEquipment(models.Model):
+    _inherit = 'maintenance.equipment'
+
+    server_domain = fields.Char('Server Domain')    
+    ssh_private_key_path = fields.Char("SSH private key path", default="/opt/odoo/auto/dev/ssh_keys/id_rsa")
+            
+    def get_ssh_connection(self):
+        import paramiko
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(self.server_domain, username="root", key_filename=self.ssh_private_key_path)        
+        return ssh
+    
+    

maintenance_server_ssh/views/maintenance_equipment_views.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<odoo>
+    <record id="equipment_view_form_server_inherit" model="ir.ui.view">
+        <field name="name">equipment.form.server.inherit</field>
+        <field name="model">maintenance.equipment</field>
+        <field name="inherit_id" ref="maintenance.hr_equipment_view_form" />
+        <field name="arch" type="xml">            
+            <xpath expr="//notebook" position="inside">
+                <page name="ssh" string="SSH">
+                    <group name="ssh_connection" string="SSH Connection">
+                        <field name="server_domain" />
+                        <field name="ssh_private_key_path" />                                                
+                    </group>                    
+                </page>
+            </xpath>
+        </field>
+    </record>
+
+    <record id="equipment_view_tree_server_inherit" model="ir.ui.view">
+        <field name="name">equipment.tree.server.inherit</field>
+        <field name="model">maintenance.equipment</field>
+        <field name="inherit_id" ref="maintenance.hr_equipment_view_tree" />
+        <field name="arch" type="xml">
+            <xpath expr="//field[@name='category_id']" position="after">
+                <field name="server_domain" optional="hide" />                
+            </xpath>
+        </field>
+    </record>
+
+</odoo>