[ADD] various: deduplicate soon-to-be-unique fields before migration

Some target versions add a ``UNIQUE`` constraint on a field that allowed
duplicates in the source version (e.g. ``utm.source.name`` in 16.0). The
unique index creation then aborts the whole registry load with a
``psycopg2.errors.UniqueViolation``.

Declare such constraints per version in ``known_changes.yaml`` under the
``new_unique_constraints`` key (only ``model`` and ``fields`` needed).
During ``prepare_db.sh``, ``dedup_unique_constraints.py`` aggregates the
declarations of every traversed version and, on the source database,
renames duplicates by suffixing `` [<id>]`` while preserving foreign keys.

The actual PostgreSQL column type is used (not the ORM ``translate``
attribute) so both ``varchar`` and already-converted ``jsonb`` columns are
handled: ``jsonb`` duplicates are compared and renamed on every language
key (indexed ``en_US`` value). Renamed records are reported in a
timestamped JSON file and summarized at the end of ``migration.log``.

README.md

@@ -42,6 +42,7 @@ cd 0k-odoo-upgrade
 │       ├── check_views.py                 # View analysis (pre-migration)
 │       ├── validate_views.py              # View validation (post-migration)
 │       ├── fix_duplicated_views.py        # Fix duplicated views
+│       ├── dedup_unique_constraints.py    # Dedup soon-to-be-unique fields (pre-migration)
 │       └── cleanup_modules.py             # Obsolete module cleanup
 │
 ├── scripts/
@@ -84,6 +85,8 @@ The script performs a **step-by-step migration** between each major version. For
 3. **Database Preparation** (`scripts/prepare_db.sh`)
    - Neutralization: disable mail servers and cron jobs
    - Detection of installed modules missing in the target version
+   - Deduplication of fields that become `UNIQUE` in a traversed version
+     (see [Unique Constraints](#unique-constraints-new_unique_constraints))
    - View state verification
    - User confirmation prompt
 
@@ -356,6 +359,48 @@ Version scripts have access to functions defined in `lib/common.sh`:
 | `log_info`, `log_warn`, `log_error` | Logging functions |
 | `log_step "title"` | Display a section header |
 
+### Unique Constraints (`new_unique_constraints`)
+
+Some Odoo versions add a `UNIQUE` constraint on a field that previously allowed
+duplicates. A well-known case is `utm.source.name` in 16.0. When the migration
+rebuilds the model, PostgreSQL tries to create the unique index and aborts the
+whole registry load if the legacy data contains duplicates:
+
+```
+psycopg2.errors.UniqueViolation: could not create unique index "utm_source_unique_name"
+DETAIL:  Key (name)=(Webinaire gouvernance (copie)) is duplicated.
+```
+
+To prevent this, declare the newly-introduced unique constraints in the
+`known_changes.yaml` of the version that introduces them, under the
+`new_unique_constraints` key. Only `model` and `fields` are needed — the table,
+the column type and the translatable (jsonb) flag are resolved automatically
+from the ORM:
+
+```yaml
+new_unique_constraints:
+  - model: utm.source
+    fields: [name]
+  - model: utm.medium
+    fields: [name]
+  - model: utm.campaign
+    fields: [name]
+```
+
+During `prepare_db.sh`, the `dedup_unique_constraints.py` checker aggregates the
+declarations of every traversed version and, on the **source** database:
+
+- detects duplicate groups (for translatable/jsonb fields it compares the
+  indexed `en_US` value, not the raw jsonb);
+- keeps the record with the smallest `id` untouched and renames each other
+  duplicate by appending ` [<id>]` to every language key (jsonb) or to the
+  plain value (varchar) — all foreign keys are preserved;
+- writes a timestamped report to
+  `/tmp/dedup_unique_constraints_<source_db>_<timestamp>.json`.
+
+Renamed records are summarized at the end of `migration.log` (see the final
+`UPGRADE PROCESS ENDED WITH SUCCESS` section).
+
 ### Adding a New Version
 
 To add support for a new version (e.g., 19.0):
@@ -368,6 +413,8 @@ cp versions/18.0/*.sh versions/19.0/
 # - Change references from ou18 → ou19
 # - Change the port from -p 8018:8069 → -p 8019:8069
 # - Add SQL fixes specific to this migration
+# - Declare any new UNIQUE constraints in versions/19.0/known_changes.yaml
+#   under `new_unique_constraints` (see "Unique Constraints" above)
 ```
 
 ## Troubleshooting

lib/common.sh

@@ -100,6 +100,27 @@ exec_python_script_in_odoo_shell() {
     run_compose --debug run "$service_name" shell -d "$db_name" --no-http --stop-after-init < "$python_script"
 }
 
+# Same as exec_python_script_in_odoo_shell but prepends a Python preamble read
+# from stdin (heredoc). Useful to inject environment values into the Odoo shell
+# when the compose wrapper does not reliably propagate host env vars into the
+# container. The preamble runs before the script body in the same interpreter.
+#
+# Usage:
+#   exec_python_script_in_odoo_shell_with_preamble svc db /path/script.py <<'PY'
+#   import os; os.environ['FOO'] = 'bar'
+#   PY
+exec_python_script_in_odoo_shell_with_preamble() {
+    local service_name="$1"
+    local db_name="$2"
+    local python_script="$3"
+
+    local preamble
+    preamble="$(cat)"
+
+    { printf '%s\n' "$preamble"; cat "$python_script"; } \
+        | run_compose --debug run "$service_name" shell -d "$db_name" --no-http --stop-after-init
+}
+
 # Classifies missing modules into 4 categories based on the known_changes.yaml
 # files from each traversed version (from ORIGIN_VERSION+1 to FINAL_VERSION).
 # The following global arrays are populated:
@@ -175,8 +196,50 @@ classify_missing_addons() {
     done
 }
 
+# Aggregates the `new_unique_constraints` sections from the known_changes.yaml
+# files of every traversed version (from ORIGIN_VERSION+1 to FINAL_VERSION) into
+# a single JSON array printed on stdout, e.g.:
+#   [{"model":"utm.source","fields":["name"]},{"model":"utm.medium","fields":["name"]}]
+# Duplicate (model, fields) entries are collapsed. Prints "[]" when none.
+#
+# Prerequisites: ORIGIN_VERSION and FINAL_VERSION must be exported.
+collect_new_unique_constraints() {
+    local versions_path="${PROJECT_ROOT}/versions"
+    local v
+    local -a json_parts=()
+    local -A seen=()
+
+    for v in $(seq $((ORIGIN_VERSION + 1)) "$FINAL_VERSION"); do
+        local yaml_file="${versions_path}/${v}.0/known_changes.yaml"
+        [[ -f "$yaml_file" ]] || continue
+
+        local count
+        count=$(yq '.new_unique_constraints | length' "$yaml_file" 2>/dev/null)
+        [[ "$count" =~ ^[0-9]+$ && "$count" -gt 0 ]] || continue
+
+        local i
+        for ((i = 0; i < count; i++)); do
+            local model fields entry
+            model=$(yq -o=json ".new_unique_constraints[$i].model" "$yaml_file" 2>/dev/null)
+            fields=$(yq -o=json -I=0 ".new_unique_constraints[$i].fields" "$yaml_file" 2>/dev/null)
+            [[ -n "$model" && "$model" != "null" ]] || continue
+            [[ -n "$fields" && "$fields" != "null" ]] || continue
+            entry="{\"model\":${model},\"fields\":${fields}}"
+            if [[ -z "${seen[$entry]:-}" ]]; then
+                seen["$entry"]=1
+                json_parts+=("$entry")
+            fi
+        done
+    done
+
+    local IFS=','
+    printf '[%s]' "${json_parts[*]}"
+}
+
 export PROJECT_ROOT DATASTORE_PATH FILESTORE_SUBPATH
 export -f log_info log_warn log_error log_step confirm_or_exit
 export -f check_required_commands
 export -f query_postgres_container copy_database copy_filestore run_compose exec_python_script_in_odoo_shell
+export -f exec_python_script_in_odoo_shell_with_preamble
 export -f classify_missing_addons
+export -f collect_new_unique_constraints

lib/python/dedup_unique_constraints.py

@@ -0,0 +1,304 @@
+#!/usr/bin/env python3
+"""
+Pre-Migration Unique Constraint Deduplication Script for Odoo
+
+Some Odoo versions introduce new UNIQUE constraints on fields that previously
+allowed duplicates (e.g. utm.source.name in 16.0). When the migration rebuilds
+the model, the unique index creation raises
+`psycopg2.errors.UniqueViolation: could not create unique index ...`
+and aborts the whole registry load.
+
+This script runs BEFORE the migration loop, on the SOURCE database, and renames
+duplicate records so the target-version unique index can be built successfully.
+
+For each declared constraint (model + fields), it:
+  - resolves the table, column type and translatable flag from the ORM;
+  - detects duplicate groups (for translatable/jsonb fields the comparison is
+    done on the value actually indexed by Odoo, i.e. the `en_US` key);
+  - keeps the record with the smallest id untouched and renames every other
+    duplicate by appending ` [<id>]` to ALL language keys (jsonb) or to the
+    plain value (varchar), preserving all foreign keys.
+
+Constraints to check are passed as JSON via the DEDUP_UNIQUE_CONSTRAINTS
+environment variable, e.g.:
+    [{"model": "utm.source", "fields": ["name"]}, ...]
+
+A machine-readable report of every rename is written to
+/tmp/dedup_unique_constraints_<db>_<timestamp>.json when DEDUP_REPORT is set.
+
+Usage:
+    compose run <service> shell -d <db> --no-http --stop-after-init \
+        < dedup_unique_constraints.py
+
+Exit codes:
+    0 - Completed (duplicates, if any, were deduplicated)
+    2 - Technical failure (see traceback)
+"""
+
+import os
+import sys
+import json
+from datetime import datetime
+from collections import defaultdict
+
+
+class Colors:
+    RED = '\033[91m'
+    GREEN = '\033[92m'
+    YELLOW = '\033[93m'
+    BLUE = '\033[94m'
+    BOLD = '\033[1m'
+    END = '\033[0m'
+
+
+def _c(color, text):
+    return f"{color}{text}{Colors.END}"
+
+
+def load_constraints():
+    """Load the constraints catalog from the environment."""
+    raw = os.environ.get('DEDUP_UNIQUE_CONSTRAINTS', '').strip()
+    if not raw:
+        return []
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"DEDUP_UNIQUE_CONSTRAINTS is not valid JSON: {exc}")
+    if not isinstance(data, list):
+        raise ValueError("DEDUP_UNIQUE_CONSTRAINTS must be a JSON list")
+    return data
+
+
+def resolve_constraint(env, spec):
+    """Resolve a {model, fields} spec into concrete DB metadata.
+
+    Returns a dict with table, list of (field_name, column, is_jsonb) or
+    None if the model/fields are not present in this database (e.g. module
+    not installed in the source version).
+    """
+    model_name = spec.get('model')
+    field_names = spec.get('fields') or []
+    if not model_name or not field_names:
+        return None
+    if model_name not in env:
+        return None
+
+    model = env[model_name]
+    table = model._table
+
+    # The physical table must exist in the source DB.
+    env.cr.execute("SELECT to_regclass(%s)", (table,))
+    if not env.cr.fetchone()[0]:
+        return None
+
+    columns = []
+    for fname in field_names:
+        field = model._fields.get(fname)
+        if field is None:
+            return None
+        column = field.name
+        # Column must physically exist (skip non-stored/computed fields).
+        # We rely on the ACTUAL PostgreSQL column type, not the ORM `translate`
+        # attribute: in the source version a translatable field may still be a
+        # plain `character varying` column (it only becomes `jsonb` during the
+        # migration to the version that introduces translation storage).
+        env.cr.execute(
+            """
+            SELECT data_type FROM information_schema.columns
+            WHERE table_name = %s AND column_name = %s
+            """,
+            (table, column),
+        )
+        row = env.cr.fetchone()
+        if not row:
+            return None
+        data_type = row[0]
+        columns.append({
+            'field': fname,
+            'column': column,
+            'is_jsonb': data_type == 'jsonb',
+        })
+
+    return {'model': model_name, 'table': table, 'columns': columns}
+
+
+def key_sql(column, is_jsonb):
+    """SQL expression yielding the value Odoo compares for the unique index.
+
+    For jsonb (translated) columns Odoo indexes the value; the semantically
+    relevant duplicate is on the `en_US` key. For plain columns, the value.
+    """
+    if is_jsonb:
+        return f"({column} ->> 'en_US')"
+    return column
+
+
+def find_duplicate_groups(env, meta):
+    """Return duplicate groups as list of dicts: {value, ids (sorted)}."""
+    table = meta['table']
+    key_exprs = [key_sql(c['column'], c['is_jsonb']) for c in meta['columns']]
+    key_concat = " || '\x1f' || ".join(f"COALESCE({e}, '')" for e in key_exprs)
+
+    query = f"""
+        SELECT {key_concat} AS dup_key, array_agg(id ORDER BY id) AS ids
+        FROM {table}
+        GROUP BY {key_concat}
+        HAVING COUNT(*) > 1
+    """
+    env.cr.execute(query)
+    groups = []
+    for dup_key, ids in env.cr.fetchall():
+        groups.append({'value': dup_key, 'ids': list(ids)})
+    return groups
+
+
+def rename_record(env, meta, record_id):
+    """Suffix ` [<id>]` on every language key (jsonb) / plain value (varchar).
+
+    Returns a list of {field, old_value, new_value} describing the change.
+    """
+    table = meta['table']
+    changes = []
+    suffix = f" [{record_id}]"
+
+    for col in meta['columns']:
+        column = col['column']
+        env.cr.execute(
+            f"SELECT {column} FROM {table} WHERE id = %s", (record_id,)
+        )
+        row = env.cr.fetchone()
+        if not row:
+            continue
+        old_value = row[0]
+
+        if col['is_jsonb']:
+            # jsonb: append suffix to each language value
+            if not isinstance(old_value, dict):
+                # psycopg may already decode jsonb to dict; if not, load it.
+                try:
+                    old_value = json.loads(old_value) if old_value else {}
+                except (TypeError, ValueError):
+                    old_value = {}
+            new_value = {
+                lang: (val or '') + suffix for lang, val in old_value.items()
+            }
+            env.cr.execute(
+                f"UPDATE {table} SET {column} = %s::jsonb WHERE id = %s",
+                (json.dumps(new_value), record_id),
+            )
+        else:
+            new_value = (old_value or '') + suffix
+            env.cr.execute(
+                f"UPDATE {table} SET {column} = %s WHERE id = %s",
+                (new_value, record_id),
+            )
+
+        changes.append({
+            'field': col['field'],
+            'old_value': old_value,
+            'new_value': new_value,
+        })
+
+    return changes
+
+
+def main():
+    print("\n" + "=" * 80)
+    print(_c(Colors.BOLD, "PRE-MIGRATION - UNIQUE CONSTRAINT DEDUPLICATION"))
+    print("=" * 80 + "\n")
+
+    try:
+        specs = load_constraints()
+    except ValueError as exc:
+        print(_c(Colors.RED, f"[ERROR] {exc}"))
+        sys.exit(2)
+
+    if not specs:
+        print(_c(Colors.YELLOW, "No unique constraints declared. Nothing to do."))
+        return
+
+    renamed_records = []  # flat report entries
+    checked = 0
+    skipped = []
+
+    for spec in specs:
+        meta = resolve_constraint(env, spec)
+        label = f"{spec.get('model')}({', '.join(spec.get('fields', []))})"
+        if meta is None:
+            skipped.append(label)
+            print(_c(Colors.YELLOW, f"[SKIP]  {label} - not present in this database"))
+            continue
+
+        checked += 1
+        groups = find_duplicate_groups(env, meta)
+        if not groups:
+            print(_c(Colors.GREEN, f"[OK]    {label} - no duplicates"))
+            continue
+
+        total_dups = sum(len(g['ids']) - 1 for g in groups)
+        print(_c(Colors.BOLD,
+                 f"[FIX]   {label} - {len(groups)} duplicate group(s), "
+                 f"{total_dups} record(s) to rename"))
+
+        for group in groups:
+            ids = group['ids']
+            keep_id, dup_ids = ids[0], ids[1:]
+            print(f"        value={group['value']!r} keep id={keep_id} "
+                  f"rename ids={dup_ids}")
+            for dup_id in dup_ids:
+                changes = rename_record(env, meta, dup_id)
+                for ch in changes:
+                    renamed_records.append({
+                        'model': meta['model'],
+                        'table': meta['table'],
+                        'id': dup_id,
+                        'field': ch['field'],
+                        'old_value': ch['old_value'],
+                        'new_value': ch['new_value'],
+                    })
+
+    if renamed_records:
+        env.cr.commit()
+
+    # ---- Summary ---------------------------------------------------------
+    print("\n" + "=" * 80)
+    print(_c(Colors.BOLD, "DEDUPLICATION SUMMARY"))
+    print("=" * 80)
+    print(f"  Constraints checked ... {checked}")
+    print(f"  Constraints skipped ... {len(skipped)}")
+    print(f"  Records renamed ....... {len(renamed_records)}")
+    if renamed_records:
+        print("")
+        for r in renamed_records:
+            old_display = r['old_value']
+            new_display = r['new_value']
+            print(f"  - {r['model']} #{r['id']} [{r['field']}]: "
+                  f"{old_display!r} -> {new_display!r}")
+    print("=" * 80 + "\n")
+
+    # ---- JSON report -----------------------------------------------------
+    if os.environ.get('DEDUP_REPORT') and renamed_records:
+        ts = datetime.now().strftime('%Y%m%d_%H%M%S')
+        db = env.cr.dbname
+        path = f"/tmp/dedup_unique_constraints_{db}_{ts}.json"
+        report = {
+            'type': 'dedup_unique_constraints',
+            'timestamp': datetime.now().isoformat(),
+            'database': db,
+            'constraints_checked': checked,
+            'constraints_skipped': skipped,
+            'renamed_count': len(renamed_records),
+            'renamed': renamed_records,
+        }
+        with open(path, 'w') as fh:
+            json.dump(report, fh, indent=2, default=str)
+        print(_c(Colors.BLUE, f"[REPORT] JSON written to {path}"))
+
+
+try:
+    main()
+except Exception as exc:  # noqa: BLE001
+    print(_c(Colors.RED, f"[ERROR] Deduplication script failed: {exc}"))
+    import traceback
+    traceback.print_exc()
+    sys.exit(2)

scripts/prepare_db.sh

@@ -87,6 +87,30 @@ else
 fi
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+#############################################
+## Deduplicate soon-to-be-unique fields    ##
+#############################################
+# Some target versions add UNIQUE constraints on fields that allowed duplicates
+# in the source version (e.g. utm.source.name in 16.0). Left untouched, the
+# unique index creation aborts the migration with a UniqueViolation. We rename
+# duplicates in the source DB now, before the migration loop.
+log_step "UNIQUE CONSTRAINTS DEDUP"
+UNIQUE_CONSTRAINTS_JSON=$(collect_new_unique_constraints)
+if [[ "$UNIQUE_CONSTRAINTS_JSON" == "[]" || -z "$UNIQUE_CONSTRAINTS_JSON" ]]; then
+    log_info "No new unique constraints declared for the traversed versions — skipping."
+else
+    log_info "Checking unique constraints: $UNIQUE_CONSTRAINTS_JSON"
+    DEDUP_SCRIPT="${SCRIPT_DIR}/lib/python/dedup_unique_constraints.py"
+    # Inject the catalog via a Python preamble because the compose wrapper does
+    # not reliably propagate host environment variables into the container.
+    exec_python_script_in_odoo_shell_with_preamble "$DB_NAME" "$DB_NAME" "$DEDUP_SCRIPT" <<PY
+import os
+os.environ['DEDUP_UNIQUE_CONSTRAINTS'] = ${UNIQUE_CONSTRAINTS_JSON@Q}
+os.environ['DEDUP_REPORT'] = '1'
+PY
+fi
+
 PYTHON_SCRIPT="${SCRIPT_DIR}/lib/python/check_views.py"
 echo "Check views with script $PYTHON_SCRIPT ..."
 exec_python_script_in_odoo_shell "$DB_NAME" "$DB_NAME" "$PYTHON_SCRIPT"

upgrade.sh

@@ -232,4 +232,20 @@ log_step "POST-UPGRADE PROCESSES"
 "${SCRIPT_DIR}/scripts/finalize_db.sh" "$FINALE_DB_NAME" "$FINALE_SERVICE_NAME"
 
 log_step "UPGRADE PROCESS ENDED WITH SUCCESS"
+
+# Report any records that were renamed to satisfy new unique constraints.
+# The deduplication runs on the source DB (ou${ORIGIN_VERSION}) during
+# prepare_db.sh and writes a timestamped JSON report to /tmp.
+readarray -t dedup_reports < <(ls -1t /tmp/dedup_unique_constraints_"${COPY_DB_NAME}"_*.json 2>/dev/null || true)
+if [[ ${#dedup_reports[@]} -gt 0 ]]; then
+    latest_report="${dedup_reports[0]}"
+    renamed_count=$(yq -p=json '.renamed_count' "$latest_report" 2>/dev/null || echo "?")
+    log_info "Unique-constraint deduplication: ${renamed_count} record(s) renamed in the source DB before migration."
+    log_info "Details: ${latest_report}"
+    if [[ "$renamed_count" =~ ^[0-9]+$ && "$renamed_count" -gt 0 ]]; then
+        yq -p=json -r '.renamed[] | "  - \(.model) #\(.id) [\(.field)]: \(.old_value) -> \(.new_value)"' \
+            "$latest_report" 2>/dev/null || true
+    fi
+fi
+
 log_info "Full logs available at: ${LOG_FILE}"

versions/16.0/known_changes.yaml

@@ -21,3 +21,12 @@ renamed:
     new: server_action_mass_edit
   - old: crm_project
     new: crm_lead_to_task
+
+# Unique constraints newly introduced in 16.0
+new_unique_constraints:
+  - model: utm.source
+    fields: [name]
+  - model: utm.medium
+    fields: [name]
+  - model: utm.campaign
+    fields: [name]