[IMP] hr_employee_stats_sheet : lire en une fois les données de tous les jours calculés #25

Open
vlab wants to merge 5 commits from vlab/hr_employee_stats_sheet-batch-reads AGit into 16.0
4 changed files with 283 additions and 102 deletions

View File

@@ -23,6 +23,14 @@ class HrEmployee(models.Model):
("date_end", ">=", date_start), # date de fin après le début
],
)
return self._select_stats_calendar(calendars)
def _select_stats_calendar(self, calendars):
"""
get the ressource calendar to use, given the hr_employee_calendar
lines running over the period
"""
self.ensure_one()
if len(calendars) > 1:
raise UserError(
_("There is a calendar starting during the timesheet sheet time period for the employee %s "

View File

@@ -1,10 +1,12 @@
import logging
import pytz
from collections import defaultdict
from odoo import api, fields, models, _
from datetime import timedelta
from pytz import utc
from odoo.exceptions import UserError
from odoo.osv import expression
class HrEmployeeStats(models.Model):
@@ -46,14 +48,28 @@ class HrEmployeeStats(models.Model):
else:
return False
def _compute_timesheet_line_ids(self):
def _dates_by_employee(self):
dates = defaultdict(set)
for stat in self:
stat.timesheet_line_ids = self.env["account.analytic.line"].search(
[
("employee_id", "=", stat.employee_id.id),
("date", "=", stat.date),
]
)
dates[stat.employee_id].add(stat.date)
return dates
def _timesheet_lines_by_day(self):
"""Return {(employee, date): lines} with a single search. Each day
keeps the search order (date desc, id desc), so sums are unchanged."""
if not self:
return {}
lines = self.env["account.analytic.line"].search(self.get_total_hours_domain())
line_ids = defaultdict(list)
for line in lines:
line_ids[line.employee_id, line.date].append(line.id)
return {day: lines.browse(ids) for day, ids in line_ids.items()}
def _compute_timesheet_line_ids(self):
lines = self._timesheet_lines_by_day()
no_lines = self.env["account.analytic.line"]
for stat in self:
stat.timesheet_line_ids = lines.get((stat.employee_id, stat.date), no_lines)
def _get_intersects(
self, datetime1_start, datetime1_end, datetime2_start, datetime2_end
@@ -64,102 +80,124 @@ class HrEmployeeStats(models.Model):
return max(0, delta)
def get_total_hours_domain(self):
return [
("employee_id", "=", self.employee_id.id),
("date", "=", self.date),
]
"""Timesheet lines counted in the total hours of these days."""
return expression.OR([
[("employee_id", "=", employee.id), ("date", "in", list(dates))]
for employee, dates in self._dates_by_employee().items()
])
def _total_hours_by_stat(self):
days = self.filtered(lambda stat: stat.date and stat.employee_id)
lines = days._timesheet_lines_by_day()
no_lines = self.env["account.analytic.line"]
return {
stat: sum(lines.get((stat.employee_id, stat.date), no_lines).mapped("unit_amount"))
for stat in days
}
@api.depends("timesheet_line_ids")
def _get_total_hours(self):
self.ensure_one()
total_hours = 0
timesheet_line = self.env["account.analytic.line"]
if self.date and self.employee_id:
timesheet_line_ids = timesheet_line.search(
self.get_total_hours_domain()
)
total_hours = sum(timesheet_line_ids.mapped("unit_amount"))
return total_hours
return self._total_hours_by_stat().get(self, 0)
def _planned_hours_by_stat(self):
"""Return {stat: planned hours} from the calendar each employee had
that day. Public holidays and days without employee or date are left
out: they have no planned hours."""
days = self.filtered(lambda stat: stat.employee_id and stat.date and not stat.is_public_holiday)
dates_by_employee = days._dates_by_employee()
if not dates_by_employee:
return {}
histories = self.env["hr.employee.calendar"].search(expression.OR([
[
("employee_id", "=", employee.id),
("date_start", "<=", max(dates)),
"|", ("date_end", "=", False), ("date_end", ">=", min(dates)),
]
for employee, dates in dates_by_employee.items()
]))
calendars = {}
for employee, dates in dates_by_employee.items():
employee_histories = histories.filtered(lambda history: history.employee_id == employee)
for date in dates:
# same lines as _get_calendar_in_progress_during_a_time_period(date, date)
calendars[employee, date] = employee._select_stats_calendar(employee_histories.filtered(
lambda history: history.date_start <= date
and (not history.date_end or date <= history.date_end)
))
attendances = self.env["resource.calendar.attendance"].search(
[("calendar_id", "in", [calendar.id for calendar in set(calendars.values())])]
)
hours = defaultdict(float)
for attendance in attendances:
week_type = attendance.week_type if attendance.calendar_id.two_weeks_calendar else None
key = (attendance.calendar_id, attendance.dayofweek, week_type)
hours[key] += attendance.hour_to - attendance.hour_from
planned_hours = {}
for stat in days:
calendar = calendars[stat.employee_id, stat.date]
week_type = str(stat.date.isocalendar()[1] % 2) if calendar.two_weeks_calendar else None
planned_hours[stat] = hours.get((calendar, str(stat.date.weekday()), week_type), 0)
return planned_hours
def _get_total_planned_hours(self):
self.ensure_one()
total_planned_hours = 0
if self.employee_id and self.date and not self.is_public_holiday:
dayofweek = int(self.date.strftime("%u")) - 1
calendar_id = self.employee_id._get_calendar_in_progress_during_a_time_period(self.date,self.date)
week_number = self.date.isocalendar()[1] % 2
if calendar_id.two_weeks_calendar:
hours = calendar_id.attendance_ids.search(
[
("dayofweek", "=", dayofweek),
("calendar_id", "=", calendar_id.id),
("week_type", "=", week_number),
]
)
else:
hours = calendar_id.attendance_ids.search(
[
("dayofweek", "=", dayofweek),
("calendar_id", "=", calendar_id.id),
]
)
total_planned_hours = sum(
hours.mapped(lambda r: r.hour_to - r.hour_from)
)
return total_planned_hours
return self._planned_hours_by_stat().get(self, 0)
def _get_total_recovery_hours(self):
#TODO: duplicated code with _get_total_leave_hours, to refactor later
self.ensure_one()
recovery = self.env["hr.leave"]
total_recovery_hours = 0
if self.date and self.employee_id and self._get_holiday_status_id():
recovery_ids = recovery.search(
def _validated_leaves_by_day(self):
"""Return {(employee, date): validated leaves} with a single search,
in its order (date_from desc)."""
dates_by_employee = self.filtered(lambda stat: stat.employee_id and stat.date)._dates_by_employee()
if not dates_by_employee:
return {}
leaves = self.env["hr.leave"].search(expression.AND([
[("state", "=", "validate"), ("active", "=", True)],
expression.OR([
[
("employee_id", "=", self.employee_id.id),
("holiday_status_id", "=", self._get_holiday_status_id()),
("request_date_from", "<=", self.date),
("request_date_to", ">=", self.date),
("state", '=', 'validate'),
("active", '=', True),
],
)
for recovery_id in recovery_ids:
if recovery_id.request_date_from == recovery_id.request_date_to:
("employee_id", "=", employee.id),
("request_date_from", "<=", max(dates)),
("request_date_to", ">=", min(dates)),
]
for employee, dates in dates_by_employee.items()
]),
]))
leaves_by_day = defaultdict(list)
for leave in leaves:
for date in dates_by_employee[leave.employee_id]:
if leave.request_date_from <= date <= leave.request_date_to:
leaves_by_day[leave.employee_id, date].append(leave)
return leaves_by_day
def _absence_hours_by_stat(self, planned_hours):
"""Return {stat: (recovery hours, other leave hours)}."""
leaves = self._validated_leaves_by_day()
absence_hours = {}
for stat in self:
recovery_type = stat._get_holiday_status_id()
recovery_hours = leave_hours = 0
for leave in leaves.get((stat.employee_id, stat.date), ()):
if leave.request_date_from == leave.request_date_to:
# single-day request (hours or half-day):
# Odoo already computed the hours from the calendar
total_recovery_hours += recovery_id.number_of_hours_display
hours = leave.number_of_hours_display
else:
# multi-day request: we take the planned hours
# of THAT specific day
total_recovery_hours += self._get_total_planned_hours()
return total_recovery_hours
hours = planned_hours.get(stat, 0)
if recovery_type and leave.holiday_status_id.id == recovery_type:
recovery_hours += hours
else:
leave_hours += hours
absence_hours[stat] = (recovery_hours, leave_hours)
return absence_hours
def _get_total_recovery_hours(self):
self.ensure_one()
return self._absence_hours_by_stat(self._planned_hours_by_stat())[self][0]
def _get_total_leave_hours(self):
self.ensure_one()
leave = self.env["hr.leave"]
total_leave_hours = 0
if self.date and self.employee_id:
leave_ids = leave.search(
[
("employee_id", "=", self.employee_id.id),
("holiday_status_id", "!=", self._get_holiday_status_id()),
("request_date_from", "<=", self.date),
("request_date_to", ">=", self.date),
("state", '=', 'validate'),
("active", '=', True),
],
)
for leave_id in leave_ids:
if leave_id.request_date_from == leave_id.request_date_to:
# single-day request (hours or half-day):
# Odoo already computed the hours from the calendar
total_leave_hours += leave_id.number_of_hours_display
else:
# multi-day request: we take the planned hours
# of THAT specific day
total_leave_hours += self._get_total_planned_hours()
return total_leave_hours
return self._absence_hours_by_stat(self._planned_hours_by_stat())[self][1]
@api.depends("employee_id", "date")
def _compute_name(self):
@@ -168,13 +206,16 @@ class HrEmployeeStats(models.Model):
@api.depends("date","employee_id")
def _compute_dayofweek(self):
public_holidays = self._public_holidays_by_day()
for stat in self:
if not stat.date:
stat.dayofweek = None
stat.is_public_holiday = False
continue
stat.dayofweek = int(stat.date.strftime("%u")) - 1
stat.is_public_holiday = stat._is_public_holiday_accordig_to_employe_tz()
stat.is_public_holiday = stat._matches_public_holiday(
public_holidays.get((stat.employee_id, stat.date))
)
def _convert_to_employee_tz(self, date):
"""Convert a UTC datetime to the employee's timezone datetime."""
@@ -186,14 +227,28 @@ class HrEmployeeStats(models.Model):
dt = pytz.utc.localize(date)
return dt.astimezone(employee_tz)
def _public_holidays_by_day(self):
"""Return {(employee, date): public holidays} with one search per
employee: the holidays _get_public_holidays(date, date) returns."""
public_holidays = {}
days = self.filtered(lambda stat: stat.employee_id and stat.date)
for employee, dates in days._dates_by_employee().items():
holidays = employee._get_public_holidays(min(dates), max(dates))
for date in dates:
# the search compares the UTC datetimes with the whole day
public_holidays[employee, date] = holidays.filtered(
lambda holiday: holiday.date_from.date() <= date <= holiday.date_to.date()
)
return public_holidays
def _is_public_holiday_accordig_to_employe_tz(self):
self.ensure_one()
if not self.date or not self.employee_id:
return False
#get public holidays for the employee
public_holidays = self.employee_id._get_public_holidays(
self.date, self.date
return self._matches_public_holiday(
self._public_holidays_by_day().get((self.employee_id, self.date))
)
def _matches_public_holiday(self, public_holidays):
self.ensure_one()
if not public_holidays:
return False
if len(public_holidays) > 1:
@@ -230,11 +285,14 @@ class HrEmployeeStats(models.Model):
"timesheet_line_ids",
)
def _compute_hours(self):
# read what the days need once for all of them, not day by day
hours = self._total_hours_by_stat()
planned_hours = self._planned_hours_by_stat()
absence_hours = self._absence_hours_by_stat(planned_hours)
for stat in self:
total_hours = stat._get_total_hours()
total_planned_hours = stat._get_total_planned_hours()
total_recovery_hours = stat._get_total_recovery_hours()
total_leave_hours = stat._get_total_leave_hours()
total_hours = hours.get(stat, 0)
total_planned_hours = planned_hours.get(stat, 0)
total_recovery_hours, total_leave_hours = absence_hours[stat]
stat.total_hours = total_hours
stat.total_planned_hours = total_planned_hours
stat.gap_hours = stat._get_gap_hours(total_hours, total_recovery_hours, total_leave_hours, total_planned_hours)

View File

@@ -26,18 +26,19 @@ class HrTimesheetSheet(models.Model):
@api.depends("employee_stats_ids.gap_hours", "employee_id")
def _compute_hours(self):
# compute the days of all the sheets together, not sheet by sheet
self._stats_through_today().mapped("gap_hours")
for sheet in self:
sheet.timesheet_sheet_gap_hours = sheet._get_timesheet_sheet_gap_hours()
sheet.timesheet_sheet_recovery_hours = sheet._get_timesheet_sheet_recovery_hours()
def _stats_through_today(self):
today = fields.Date.today()
return self.mapped("employee_stats_ids").filtered(lambda stat: stat.date <= today)
def _get_timesheet_sheet_gap_hours(self):
self.ensure_one()
timesheet_sheet_gap_hours = sum(
self.employee_stats_ids.filtered(
lambda stat: stat.date <= fields.Date.today()
).mapped("gap_hours")
)
return timesheet_sheet_gap_hours
return sum(self._stats_through_today().mapped("gap_hours"))
def _get_timesheet_sheet_recovery_hours(self):
self.ensure_one()

View File

@@ -455,6 +455,104 @@ class TestHrEmployeeStatsRecovery(TransactionCase):
self.assertEqual(len(recovery_allocation), 1, "Il doit y avoir une allocation de récupération générée")
self.assertAlmostEqual(recovery_allocation.number_of_hours_display, 8.75, msg="The recovery allocation should display 8.75 hours")
# --- Reading many days at once -----------------------------------------
def _create_days(self, *days):
return self.env['hr.employee.stats'].create([
{'employee_id': self.employee.id, 'date': day} for day in days
])
def _count_queries(self, records, fnames):
"""SQL queries needed to read fnames of records from an empty cache."""
self.env.invalidate_all()
records.read(fnames) # fills the caches that invalidate_all keeps
self.env.invalidate_all()
before = self.env.cr.sql_log_count
records.read(fnames)
return self.env.cr.sql_log_count - before
def test_query_count_does_not_grow_with_days(self):
start = Date.to_date("2025-03-03")
self.env['hr.employee.calendar'].create({
'employee_id': self.employee.id,
'date_start': start,
'calendar_id': self.base_calendar.id,
})
days = self._create_days(*(start + timedelta(days=i) for i in range(14)))
self.env['account.analytic.line'].create([{
'employee_id': self.employee.id, 'date': day.date, 'unit_amount': 3,
'account_id': self.analytic_account.id, 'name': 'Work Entry',
} for day in days])
fnames = ['total_hours', 'total_planned_hours', 'total_leave_hours',
'total_recovery_hours', 'gap_hours', 'is_public_holiday']
self.assertLessEqual(self._count_queries(days, fnames),
self._count_queries(days[:2], fnames))
self.assertEqual(days.mapped('total_hours'), [3] * 14)
def test_query_count_does_not_grow_with_sheets(self):
start = Date.to_date("2025-03-03")
self.env['hr.employee.calendar'].create({
'employee_id': self.employee.id,
'date_start': start,
'calendar_id': self.base_calendar.id,
})
sheets = self.env['hr_timesheet.sheet'].concat(*(
self._create_timesheet_sheet(start + timedelta(weeks=week)) for week in range(6)
))
fnames = ['timesheet_sheet_gap_hours', 'timesheet_sheet_recovery_hours']
self.assertLessEqual(self._count_queries(sheets, fnames),
self._count_queries(sheets[:2], fnames))
self.assertEqual(sheets.mapped('timesheet_sheet_gap_hours'), [-35] * 6)
def test_public_holidays_read_together(self):
# Stored in UTC, this holiday is May 1st 00:00 to May 2nd 01:00 in
# Paris. A day counts only if it is in the UTC range AND in the
# employee's range: April 30 and May 2 do not count.
self.env['resource.calendar.leaves'].create({
'name': 'Late finish',
'date_from': '2025-04-30 22:00:00',
'date_to': '2025-05-01 23:00:00',
})
days = self._create_days('2025-04-30', '2025-05-01', '2025-05-02')
self.assertEqual(days.mapped('is_public_holiday'), [False, True, False])
# April 30 now has two holidays in its UTC range: that is an anomaly.
self.env['resource.calendar.leaves'].create({
'name': 'Earlier',
'date_from': '2025-04-30 08:00:00',
'date_to': '2025-04-30 09:00:00',
})
self.env.invalidate_all()
with self.assertRaisesRegex(UserError, '2025-04-30'):
days.mapped('is_public_holiday')
def test_planned_hours_read_together_follow_calendar_history(self):
two_weeks = self.env['resource.calendar'].create({
'name': 'Alternating Mondays',
'two_weeks_calendar': True,
'attendance_ids': [
(0, 0, {'name': 'Even Monday', 'dayofweek': '0', 'week_type': '0',
'hour_from': 9, 'hour_to': 12}),
(0, 0, {'name': 'Odd Monday', 'dayofweek': '0', 'week_type': '1',
'hour_from': 9, 'hour_to': 14}),
],
})
mondays = ['2025-03-03', '2025-03-10', '2025-03-17', '2025-03-24']
self.env['hr.employee.calendar'].create({
'employee_id': self.employee.id, 'calendar_id': two_weeks.id,
'date_start': mondays[0], 'date_end': mondays[1],
})
one_day = self.env['hr.employee.calendar'].create({
'employee_id': self.employee.id, 'calendar_id': self.base_calendar.id,
'date_start': mondays[2], 'date_end': mondays[2],
})
days = self._create_days(*mondays)
# even week, odd week, base calendar, company calendar (8h) after the history
self.assertEqual(days.mapped('total_planned_hours'), [3, 5, 7, 8])
one_day.date_start = mondays[1] # two calendars on March 10: an anomaly
self.env.invalidate_all()
with self.assertRaisesRegex(UserError, 'calendar starting'):
days.mapped('total_planned_hours')
@tagged("post_install", "-at_install")
class TestHrEmployeeStatsLeaveAndRecoveryUnit(TransactionCase):
@@ -671,6 +769,22 @@ class TestHrEmployeeStatsLeaveAndRecoveryUnit(TransactionCase):
self.assertEqual(stat.total_leave_hours, 0, "recovery must not count as a leave")
self.assertEqual(stat.total_recovery_hours, 7, "it must count as a recovery")
def test_leaves_read_together(self):
leave_type = self._create_leave_type('hour')
self.recovery_type.request_unit = 'half_day'
wednesday = self.tuesday + timedelta(days=1)
thursday = self.tuesday + timedelta(days=2)
self._create_leave(self.recovery_type, self.monday, self.monday, half=True, period='am')
self._create_leave(leave_type, self.monday, self.monday, hours=True, hour_from='13', hour_to='15')
self._create_leave(self._create_leave_type('day'), self.tuesday, wednesday)
days = self.env['hr.employee.stats'].create([
{'employee_id': self.employee.id, 'date': day}
for day in (self.monday, self.tuesday, wednesday, thursday)
])
self.env.invalidate_all()
self.assertEqual(days.mapped('total_recovery_hours'), [3, 0, 0, 0])
self.assertEqual(days.mapped('total_leave_hours'), [2, 7, 7, 0])