From 718b367e86553b7a4b5b5d2eed9ee8471a2afe2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phan=20Sainl=C3=A9ger?= Date: Mon, 13 Jul 2026 15:23:26 +0200 Subject: [PATCH] [ADD] various: deduplicate soon-to-be-unique fields before migration Some target versions add a ``UNIQUE`` constraint on a field that allowed duplicates in the source version (e.g. ``utm.source.name`` in 16.0). The unique index creation then aborts the whole registry load with a ``psycopg2.errors.UniqueViolation``. Declare such constraints per version in ``known_changes.yaml`` under the ``new_unique_constraints`` key (only ``model`` and ``fields`` needed). During ``prepare_db.sh``, ``dedup_unique_constraints.py`` aggregates the declarations of every traversed version and, on the source database, renames duplicates by suffixing `` []`` while preserving foreign keys. The actual PostgreSQL column type is used (not the ORM ``translate`` attribute) so both ``varchar`` and already-converted ``jsonb`` columns are handled: ``jsonb`` duplicates are compared and renamed on every language key (indexed ``en_US`` value). Renamed records are reported in a timestamped JSON file and summarized at the end of ``migration.log``. --- README.md | 55 ++++- lib/common.sh | 63 +++++ lib/python/dedup_unique_constraints.py | 304 +++++++++++++++++++++++++ scripts/prepare_db.sh | 24 ++ upgrade.sh | 16 ++ versions/16.0/known_changes.yaml | 9 + 6 files changed, 467 insertions(+), 4 deletions(-) create mode 100644 lib/python/dedup_unique_constraints.py diff --git a/README.md b/README.md index 9e7115e..23c07fb 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,11 @@ cd 0k-odoo-upgrade ├── lib/ │ ├── common.sh # Shared bash functions │ └── python/ # Python utility scripts -│ ├── check_views.py # View analysis (pre-migration) -│ ├── validate_views.py # View validation (post-migration) -│ ├── fix_duplicated_views.py # Fix duplicated views -│ └── cleanup_modules.py # Obsolete module cleanup +│ ├── check_views.py # View analysis (pre-migration) +│ ├── validate_views.py # View validation (post-migration) +│ ├── fix_duplicated_views.py # Fix duplicated views +│ ├── dedup_unique_constraints.py # Dedup soon-to-be-unique fields (pre-migration) +│ └── cleanup_modules.py # Obsolete module cleanup │ ├── scripts/ │ ├── prepare_db.sh # Database preparation before migration @@ -84,6 +85,8 @@ The script performs a **step-by-step migration** between each major version. For 3. **Database Preparation** (`scripts/prepare_db.sh`) - Neutralization: disable mail servers and cron jobs - Detection of installed modules missing in the target version + - Deduplication of fields that become `UNIQUE` in a traversed version + (see [Unique Constraints](#unique-constraints-new_unique_constraints)) - View state verification - User confirmation prompt @@ -356,6 +359,48 @@ Version scripts have access to functions defined in `lib/common.sh`: | `log_info`, `log_warn`, `log_error` | Logging functions | | `log_step "title"` | Display a section header | +### Unique Constraints (`new_unique_constraints`) + +Some Odoo versions add a `UNIQUE` constraint on a field that previously allowed +duplicates. A well-known case is `utm.source.name` in 16.0. When the migration +rebuilds the model, PostgreSQL tries to create the unique index and aborts the +whole registry load if the legacy data contains duplicates: + +``` +psycopg2.errors.UniqueViolation: could not create unique index "utm_source_unique_name" +DETAIL: Key (name)=(Webinaire gouvernance (copie)) is duplicated. +``` + +To prevent this, declare the newly-introduced unique constraints in the +`known_changes.yaml` of the version that introduces them, under the +`new_unique_constraints` key. Only `model` and `fields` are needed — the table, +the column type and the translatable (jsonb) flag are resolved automatically +from the ORM: + +```yaml +new_unique_constraints: + - model: utm.source + fields: [name] + - model: utm.medium + fields: [name] + - model: utm.campaign + fields: [name] +``` + +During `prepare_db.sh`, the `dedup_unique_constraints.py` checker aggregates the +declarations of every traversed version and, on the **source** database: + +- detects duplicate groups (for translatable/jsonb fields it compares the + indexed `en_US` value, not the raw jsonb); +- keeps the record with the smallest `id` untouched and renames each other + duplicate by appending ` []` to every language key (jsonb) or to the + plain value (varchar) — all foreign keys are preserved; +- writes a timestamped report to + `/tmp/dedup_unique_constraints__.json`. + +Renamed records are summarized at the end of `migration.log` (see the final +`UPGRADE PROCESS ENDED WITH SUCCESS` section). + ### Adding a New Version To add support for a new version (e.g., 19.0): @@ -368,6 +413,8 @@ cp versions/18.0/*.sh versions/19.0/ # - Change references from ou18 → ou19 # - Change the port from -p 8018:8069 → -p 8019:8069 # - Add SQL fixes specific to this migration +# - Declare any new UNIQUE constraints in versions/19.0/known_changes.yaml +# under `new_unique_constraints` (see "Unique Constraints" above) ``` ## Troubleshooting diff --git a/lib/common.sh b/lib/common.sh index 1d94aef..0e55052 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -100,6 +100,27 @@ exec_python_script_in_odoo_shell() { run_compose --debug run "$service_name" shell -d "$db_name" --no-http --stop-after-init < "$python_script" } +# Same as exec_python_script_in_odoo_shell but prepends a Python preamble read +# from stdin (heredoc). Useful to inject environment values into the Odoo shell +# when the compose wrapper does not reliably propagate host env vars into the +# container. The preamble runs before the script body in the same interpreter. +# +# Usage: +# exec_python_script_in_odoo_shell_with_preamble svc db /path/script.py <<'PY' +# import os; os.environ['FOO'] = 'bar' +# PY +exec_python_script_in_odoo_shell_with_preamble() { + local service_name="$1" + local db_name="$2" + local python_script="$3" + + local preamble + preamble="$(cat)" + + { printf '%s\n' "$preamble"; cat "$python_script"; } \ + | run_compose --debug run "$service_name" shell -d "$db_name" --no-http --stop-after-init +} + # Classifies missing modules into 4 categories based on the known_changes.yaml # files from each traversed version (from ORIGIN_VERSION+1 to FINAL_VERSION). # The following global arrays are populated: @@ -175,8 +196,50 @@ classify_missing_addons() { done } +# Aggregates the `new_unique_constraints` sections from the known_changes.yaml +# files of every traversed version (from ORIGIN_VERSION+1 to FINAL_VERSION) into +# a single JSON array printed on stdout, e.g.: +# [{"model":"utm.source","fields":["name"]},{"model":"utm.medium","fields":["name"]}] +# Duplicate (model, fields) entries are collapsed. Prints "[]" when none. +# +# Prerequisites: ORIGIN_VERSION and FINAL_VERSION must be exported. +collect_new_unique_constraints() { + local versions_path="${PROJECT_ROOT}/versions" + local v + local -a json_parts=() + local -A seen=() + + for v in $(seq $((ORIGIN_VERSION + 1)) "$FINAL_VERSION"); do + local yaml_file="${versions_path}/${v}.0/known_changes.yaml" + [[ -f "$yaml_file" ]] || continue + + local count + count=$(yq '.new_unique_constraints | length' "$yaml_file" 2>/dev/null) + [[ "$count" =~ ^[0-9]+$ && "$count" -gt 0 ]] || continue + + local i + for ((i = 0; i < count; i++)); do + local model fields entry + model=$(yq -o=json ".new_unique_constraints[$i].model" "$yaml_file" 2>/dev/null) + fields=$(yq -o=json -I=0 ".new_unique_constraints[$i].fields" "$yaml_file" 2>/dev/null) + [[ -n "$model" && "$model" != "null" ]] || continue + [[ -n "$fields" && "$fields" != "null" ]] || continue + entry="{\"model\":${model},\"fields\":${fields}}" + if [[ -z "${seen[$entry]:-}" ]]; then + seen["$entry"]=1 + json_parts+=("$entry") + fi + done + done + + local IFS=',' + printf '[%s]' "${json_parts[*]}" +} + export PROJECT_ROOT DATASTORE_PATH FILESTORE_SUBPATH export -f log_info log_warn log_error log_step confirm_or_exit export -f check_required_commands export -f query_postgres_container copy_database copy_filestore run_compose exec_python_script_in_odoo_shell +export -f exec_python_script_in_odoo_shell_with_preamble export -f classify_missing_addons +export -f collect_new_unique_constraints diff --git a/lib/python/dedup_unique_constraints.py b/lib/python/dedup_unique_constraints.py new file mode 100644 index 0000000..a15ac19 --- /dev/null +++ b/lib/python/dedup_unique_constraints.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +""" +Pre-Migration Unique Constraint Deduplication Script for Odoo + +Some Odoo versions introduce new UNIQUE constraints on fields that previously +allowed duplicates (e.g. utm.source.name in 16.0). When the migration rebuilds +the model, the unique index creation raises +`psycopg2.errors.UniqueViolation: could not create unique index ...` +and aborts the whole registry load. + +This script runs BEFORE the migration loop, on the SOURCE database, and renames +duplicate records so the target-version unique index can be built successfully. + +For each declared constraint (model + fields), it: + - resolves the table, column type and translatable flag from the ORM; + - detects duplicate groups (for translatable/jsonb fields the comparison is + done on the value actually indexed by Odoo, i.e. the `en_US` key); + - keeps the record with the smallest id untouched and renames every other + duplicate by appending ` []` to ALL language keys (jsonb) or to the + plain value (varchar), preserving all foreign keys. + +Constraints to check are passed as JSON via the DEDUP_UNIQUE_CONSTRAINTS +environment variable, e.g.: + [{"model": "utm.source", "fields": ["name"]}, ...] + +A machine-readable report of every rename is written to +/tmp/dedup_unique_constraints__.json when DEDUP_REPORT is set. + +Usage: + compose run shell -d --no-http --stop-after-init \ + < dedup_unique_constraints.py + +Exit codes: + 0 - Completed (duplicates, if any, were deduplicated) + 2 - Technical failure (see traceback) +""" + +import os +import sys +import json +from datetime import datetime +from collections import defaultdict + + +class Colors: + RED = '\033[91m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + BLUE = '\033[94m' + BOLD = '\033[1m' + END = '\033[0m' + + +def _c(color, text): + return f"{color}{text}{Colors.END}" + + +def load_constraints(): + """Load the constraints catalog from the environment.""" + raw = os.environ.get('DEDUP_UNIQUE_CONSTRAINTS', '').strip() + if not raw: + return [] + try: + data = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"DEDUP_UNIQUE_CONSTRAINTS is not valid JSON: {exc}") + if not isinstance(data, list): + raise ValueError("DEDUP_UNIQUE_CONSTRAINTS must be a JSON list") + return data + + +def resolve_constraint(env, spec): + """Resolve a {model, fields} spec into concrete DB metadata. + + Returns a dict with table, list of (field_name, column, is_jsonb) or + None if the model/fields are not present in this database (e.g. module + not installed in the source version). + """ + model_name = spec.get('model') + field_names = spec.get('fields') or [] + if not model_name or not field_names: + return None + if model_name not in env: + return None + + model = env[model_name] + table = model._table + + # The physical table must exist in the source DB. + env.cr.execute("SELECT to_regclass(%s)", (table,)) + if not env.cr.fetchone()[0]: + return None + + columns = [] + for fname in field_names: + field = model._fields.get(fname) + if field is None: + return None + column = field.name + # Column must physically exist (skip non-stored/computed fields). + # We rely on the ACTUAL PostgreSQL column type, not the ORM `translate` + # attribute: in the source version a translatable field may still be a + # plain `character varying` column (it only becomes `jsonb` during the + # migration to the version that introduces translation storage). + env.cr.execute( + """ + SELECT data_type FROM information_schema.columns + WHERE table_name = %s AND column_name = %s + """, + (table, column), + ) + row = env.cr.fetchone() + if not row: + return None + data_type = row[0] + columns.append({ + 'field': fname, + 'column': column, + 'is_jsonb': data_type == 'jsonb', + }) + + return {'model': model_name, 'table': table, 'columns': columns} + + +def key_sql(column, is_jsonb): + """SQL expression yielding the value Odoo compares for the unique index. + + For jsonb (translated) columns Odoo indexes the value; the semantically + relevant duplicate is on the `en_US` key. For plain columns, the value. + """ + if is_jsonb: + return f"({column} ->> 'en_US')" + return column + + +def find_duplicate_groups(env, meta): + """Return duplicate groups as list of dicts: {value, ids (sorted)}.""" + table = meta['table'] + key_exprs = [key_sql(c['column'], c['is_jsonb']) for c in meta['columns']] + key_concat = " || '\x1f' || ".join(f"COALESCE({e}, '')" for e in key_exprs) + + query = f""" + SELECT {key_concat} AS dup_key, array_agg(id ORDER BY id) AS ids + FROM {table} + GROUP BY {key_concat} + HAVING COUNT(*) > 1 + """ + env.cr.execute(query) + groups = [] + for dup_key, ids in env.cr.fetchall(): + groups.append({'value': dup_key, 'ids': list(ids)}) + return groups + + +def rename_record(env, meta, record_id): + """Suffix ` []` on every language key (jsonb) / plain value (varchar). + + Returns a list of {field, old_value, new_value} describing the change. + """ + table = meta['table'] + changes = [] + suffix = f" [{record_id}]" + + for col in meta['columns']: + column = col['column'] + env.cr.execute( + f"SELECT {column} FROM {table} WHERE id = %s", (record_id,) + ) + row = env.cr.fetchone() + if not row: + continue + old_value = row[0] + + if col['is_jsonb']: + # jsonb: append suffix to each language value + if not isinstance(old_value, dict): + # psycopg may already decode jsonb to dict; if not, load it. + try: + old_value = json.loads(old_value) if old_value else {} + except (TypeError, ValueError): + old_value = {} + new_value = { + lang: (val or '') + suffix for lang, val in old_value.items() + } + env.cr.execute( + f"UPDATE {table} SET {column} = %s::jsonb WHERE id = %s", + (json.dumps(new_value), record_id), + ) + else: + new_value = (old_value or '') + suffix + env.cr.execute( + f"UPDATE {table} SET {column} = %s WHERE id = %s", + (new_value, record_id), + ) + + changes.append({ + 'field': col['field'], + 'old_value': old_value, + 'new_value': new_value, + }) + + return changes + + +def main(): + print("\n" + "=" * 80) + print(_c(Colors.BOLD, "PRE-MIGRATION - UNIQUE CONSTRAINT DEDUPLICATION")) + print("=" * 80 + "\n") + + try: + specs = load_constraints() + except ValueError as exc: + print(_c(Colors.RED, f"[ERROR] {exc}")) + sys.exit(2) + + if not specs: + print(_c(Colors.YELLOW, "No unique constraints declared. Nothing to do.")) + return + + renamed_records = [] # flat report entries + checked = 0 + skipped = [] + + for spec in specs: + meta = resolve_constraint(env, spec) + label = f"{spec.get('model')}({', '.join(spec.get('fields', []))})" + if meta is None: + skipped.append(label) + print(_c(Colors.YELLOW, f"[SKIP] {label} - not present in this database")) + continue + + checked += 1 + groups = find_duplicate_groups(env, meta) + if not groups: + print(_c(Colors.GREEN, f"[OK] {label} - no duplicates")) + continue + + total_dups = sum(len(g['ids']) - 1 for g in groups) + print(_c(Colors.BOLD, + f"[FIX] {label} - {len(groups)} duplicate group(s), " + f"{total_dups} record(s) to rename")) + + for group in groups: + ids = group['ids'] + keep_id, dup_ids = ids[0], ids[1:] + print(f" value={group['value']!r} keep id={keep_id} " + f"rename ids={dup_ids}") + for dup_id in dup_ids: + changes = rename_record(env, meta, dup_id) + for ch in changes: + renamed_records.append({ + 'model': meta['model'], + 'table': meta['table'], + 'id': dup_id, + 'field': ch['field'], + 'old_value': ch['old_value'], + 'new_value': ch['new_value'], + }) + + if renamed_records: + env.cr.commit() + + # ---- Summary --------------------------------------------------------- + print("\n" + "=" * 80) + print(_c(Colors.BOLD, "DEDUPLICATION SUMMARY")) + print("=" * 80) + print(f" Constraints checked ... {checked}") + print(f" Constraints skipped ... {len(skipped)}") + print(f" Records renamed ....... {len(renamed_records)}") + if renamed_records: + print("") + for r in renamed_records: + old_display = r['old_value'] + new_display = r['new_value'] + print(f" - {r['model']} #{r['id']} [{r['field']}]: " + f"{old_display!r} -> {new_display!r}") + print("=" * 80 + "\n") + + # ---- JSON report ----------------------------------------------------- + if os.environ.get('DEDUP_REPORT') and renamed_records: + ts = datetime.now().strftime('%Y%m%d_%H%M%S') + db = env.cr.dbname + path = f"/tmp/dedup_unique_constraints_{db}_{ts}.json" + report = { + 'type': 'dedup_unique_constraints', + 'timestamp': datetime.now().isoformat(), + 'database': db, + 'constraints_checked': checked, + 'constraints_skipped': skipped, + 'renamed_count': len(renamed_records), + 'renamed': renamed_records, + } + with open(path, 'w') as fh: + json.dump(report, fh, indent=2, default=str) + print(_c(Colors.BLUE, f"[REPORT] JSON written to {path}")) + + +try: + main() +except Exception as exc: # noqa: BLE001 + print(_c(Colors.RED, f"[ERROR] Deduplication script failed: {exc}")) + import traceback + traceback.print_exc() + sys.exit(2) diff --git a/scripts/prepare_db.sh b/scripts/prepare_db.sh index c8e5f0d..a74e0a6 100755 --- a/scripts/prepare_db.sh +++ b/scripts/prepare_db.sh @@ -87,6 +87,30 @@ else fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +############################################# +## Deduplicate soon-to-be-unique fields ## +############################################# +# Some target versions add UNIQUE constraints on fields that allowed duplicates +# in the source version (e.g. utm.source.name in 16.0). Left untouched, the +# unique index creation aborts the migration with a UniqueViolation. We rename +# duplicates in the source DB now, before the migration loop. +log_step "UNIQUE CONSTRAINTS DEDUP" +UNIQUE_CONSTRAINTS_JSON=$(collect_new_unique_constraints) +if [[ "$UNIQUE_CONSTRAINTS_JSON" == "[]" || -z "$UNIQUE_CONSTRAINTS_JSON" ]]; then + log_info "No new unique constraints declared for the traversed versions — skipping." +else + log_info "Checking unique constraints: $UNIQUE_CONSTRAINTS_JSON" + DEDUP_SCRIPT="${SCRIPT_DIR}/lib/python/dedup_unique_constraints.py" + # Inject the catalog via a Python preamble because the compose wrapper does + # not reliably propagate host environment variables into the container. + exec_python_script_in_odoo_shell_with_preamble "$DB_NAME" "$DB_NAME" "$DEDUP_SCRIPT" </dev/null || true) +if [[ ${#dedup_reports[@]} -gt 0 ]]; then + latest_report="${dedup_reports[0]}" + renamed_count=$(yq -p=json '.renamed_count' "$latest_report" 2>/dev/null || echo "?") + log_info "Unique-constraint deduplication: ${renamed_count} record(s) renamed in the source DB before migration." + log_info "Details: ${latest_report}" + if [[ "$renamed_count" =~ ^[0-9]+$ && "$renamed_count" -gt 0 ]]; then + yq -p=json -r '.renamed[] | " - \(.model) #\(.id) [\(.field)]: \(.old_value) -> \(.new_value)"' \ + "$latest_report" 2>/dev/null || true + fi +fi + log_info "Full logs available at: ${LOG_FILE}" diff --git a/versions/16.0/known_changes.yaml b/versions/16.0/known_changes.yaml index 56ce329..f52eaff 100644 --- a/versions/16.0/known_changes.yaml +++ b/versions/16.0/known_changes.yaml @@ -21,3 +21,12 @@ renamed: new: server_action_mass_edit - old: crm_project new: crm_lead_to_task + +# Unique constraints newly introduced in 16.0 +new_unique_constraints: + - model: utm.source + fields: [name] + - model: utm.medium + fields: [name] + - model: utm.campaign + fields: [name]