#!/usr/bin/env python3
"""
Exp file filter and import utility for single CTM migrations.
Handles parsing, filtering, and direct PostgreSQL import of exp files.
Uses the EM db_access module for database connectivity.
"""

import sys
import argparse
import json
import os
import platform
import re
import subprocess
import tempfile
from io import StringIO

em_home = os.environ.get("EM_HOME", os.path.expanduser("~"))
sys.path.append(os.path.join(em_home, "bin", "python", "db_access", "em"))


def get_db_connection():
    """Get raw DB connection from EM db_access module"""
    from em_db_commands import EmDbCommands
    return EmDbCommands().db_mngr.db_access.db_conn


class ExpFileFilterImporter:
    """Handles filtering and importing exp files to PostgreSQL database"""
    
    # Exp file format delimiters
    FLD_DELIM = '\x1C\x1B\x1D'
    REC_DELIM = '\x1E\x1B\x1F'
    
    def __init__(self, db_connection):
        """
        Initialize with an existing database connection
        db_connection: a DB-API 2.0 connection object (e.g. from psycopg2 or db_access)
        """
        self.conn = db_connection
        self.cursor = self.conn.cursor()
        # {child_table_name: {child_col_name: {old_val: new_val}}}
        self._id_remaps: dict = {}
        # Set by build_auto_config; used to derive DB table name from exp table name
        self._net_type_prefix: str = ''
    
    def parse_exp_file(self, exp_file_path):
        """
        Parse exp file into table structures
        Returns: dict of {table_name: [rows]}
        """
        tables = {}
        
        with open(exp_file_path, 'r', encoding='iso-8859-1') as f:
            # Skip header until data section
            for line in f:
                if "DATA : Do not delete this line" in line:
                    break
            
            # Parse data records
            current_record = ''
            for line in f:
                if line.startswith("# CONTROL-M/Enterprise Manager : END OF DUMP"):
                    break
                
                current_record += line
                
                # Python 3.13+ treats \x1C-\x1F as whitespace, so only strip actual newlines
                if line.rstrip('\n').rstrip('\r').endswith(self.REC_DELIM):
                    record = current_record.rstrip('\n\r')
                    if record.endswith(self.REC_DELIM):
                        record = record[:-len(self.REC_DELIM)]
                    fields = [field.strip('\n\r') for field in record.split(self.FLD_DELIM)]
                    
                    if len(fields) > 0:
                        table_name = fields[0]
                        if table_name not in tables:
                            tables[table_name] = []
                        tables[table_name].append(fields)
                    
                    current_record = ''
        
        return tables
    
    def get_table_columns(self, table_name):
        """Get column names for a table from database schema"""
        self.cursor.execute("""
            SELECT column_name 
            FROM information_schema.columns 
            WHERE table_schema = 'public' 
            AND table_name = %s
            ORDER BY ordinal_position
        """, (table_name.lower(),))
        
        return [row[0] for row in self.cursor.fetchall()]

    def get_timestamp_columns(self, table_name):
        """Get set of column names that are timestamp types"""
        self.cursor.execute("""
            SELECT column_name
            FROM information_schema.columns
            WHERE table_schema = 'public'
            AND table_name = %s
            AND data_type IN ('timestamp without time zone', 'timestamp with time zone', 'timestamp')
            ORDER BY ordinal_position
        """, (table_name.lower(),))
        return {row[0] for row in self.cursor.fetchall()}

    def _get_primary_key_columns(self, db_table_name):
        """Return all PK columns for a table ordered by ordinal position, or an empty list."""
        self.cursor.execute("""
            SELECT kcu.column_name
            FROM information_schema.table_constraints tc
            JOIN information_schema.key_column_usage kcu
              ON tc.constraint_name = kcu.constraint_name
             AND tc.table_schema    = kcu.table_schema
            WHERE tc.constraint_type = 'PRIMARY KEY'
              AND tc.table_schema    = 'public'
              AND tc.table_name      = %s
            ORDER BY kcu.ordinal_position
        """, (db_table_name.lower(),))
        return [row[0] for row in self.cursor.fetchall()]

    def _get_primary_key_column(self, db_table_name):
        """Return the first PK column for a table, or None (kept for i_serials usage)."""
        cols = self._get_primary_key_columns(db_table_name)
        return cols[0] if cols else None

    def _find_serial_column(self, db_table_name, db_columns, use_i_serials=False):
        """Check if db_table_name is tracked in i_serials.

        The serial column is auto-detected from the table's primary key so no
        explicit configuration is needed (works for both 'table_id' and 'service_id'
        or any other PK column name).

        If use_i_serials=True and the table is NOT found in i_serials, a new row is
        inserted automatically so the table can be managed from this point on.

        Returns the serial column name if i_serials should be used, or None otherwise.
        """
        serial_col = self._get_primary_key_column(db_table_name)
        if serial_col is None:
            print(f"Warning: Could not detect primary key for '{db_table_name}', skipping i_serials rewrite")
            return None

        self.cursor.execute("SELECT cnt FROM i_serials WHERE name = %s", (db_table_name.upper(),))
        row = self.cursor.fetchone()

        if row is None:
            if not use_i_serials:
                # Table not in i_serials and caller did not request i_serials management.
                return None
            # Table not in i_serials but caller requested i_serials management.
            # Insert a new entry seeded from the current MAX of the serial column.
            if serial_col not in db_columns:
                print(f"Warning: '{db_table_name}' not in i_serials and PK column "
                      f"'{serial_col}' not found in DB columns, skipping i_serials rewrite")
                return None
            self.cursor.execute(
                f"SELECT COALESCE(MAX({serial_col}), 0) FROM {db_table_name.lower()}"
            )
            current_max = int(self.cursor.fetchone()[0])
            self.cursor.execute(
                "INSERT INTO i_serials (name, cnt) VALUES (%s, %s)",
                (db_table_name.upper(), current_max)
            )
            self.conn.commit()
            print(f"Table '{db_table_name}': inserted into i_serials with cnt={current_max}")
        if serial_col not in db_columns:
            print(f"Warning: Table '{db_table_name}' is in i_serials but PK column "
                  f"'{serial_col}' not found in DB columns, skipping i_serials rewrite")
            return None
        return serial_col
    
    def filter_table(self, table_rows, filter_value, filter_column_names, all_columns):
        """
        Filter table rows by checking if filter_value exists in specified columns.
        filter_value can be a single string or a list/set of strings (match any).
        Returns (matching_rows, non_matching_rows).
        """
        match_set = set(filter_value) if isinstance(filter_value, list) else {filter_value}

        filtered = []
        non_filtered = []

        filter_indices = []
        for col_name in filter_column_names:
            try:
                idx = all_columns.index(col_name)
                filter_indices.append(idx)
            except ValueError:
                print(f"Warning: Column '{col_name}' not found in table columns", file=sys.stderr)
        
        if not filter_indices:
            print(f"Warning: No valid filter columns, checking all columns", file=sys.stderr)
        
        for row in table_rows:
            matches = False
            
            if filter_indices:
                for idx in filter_indices:
                    if idx + 1 < len(row) and row[idx + 1] in match_set:
                        matches = True
                        break
            else:
                if match_set.intersection(row):
                    matches = True
            
            if matches:
                filtered.append(row)
            else:
                non_filtered.append(row)

        return filtered, non_filtered
    
    def import_to_postgres(self, table_name, rows, columns, chunk_size=10000, timestamp_cols=None):
        """
        Import rows to PostgreSQL using COPY FROM STDIN with chunked streaming.
        On chunk failure, falls back to row-by-row INSERT to skip bad rows and report them.
        Returns: (imported_count, failed_rows) tuple
        """
        if not rows:
            print(f"No rows to import for table {table_name}")
            return 0, []
        
        total_rows = len(rows)
        total_imported = 0
        all_failed_rows = []
        
        chunks = [rows] if total_rows < chunk_size else [
            rows[i:min(i + chunk_size, total_rows)]
            for i in range(0, total_rows, chunk_size)
        ]
        
        if len(chunks) > 1:
            print(f"Large table detected ({total_rows} rows), using chunked import ({len(chunks)} chunks)")
        
        for chunk_idx, chunk in enumerate(chunks):
            offset = chunk_idx * chunk_size
            
            if len(chunks) > 1:
                print(f"Importing chunk {chunk_idx+1}/{len(chunks)}: rows {offset+1} to {offset+len(chunk)}...")
            
            imported, failed = self._import_chunk_with_fallback(table_name, chunk, columns, offset, timestamp_cols)
            total_imported += imported
            all_failed_rows.extend(failed)
            
            if len(chunks) > 1:
                print(f"Progress: {total_imported}/{total_rows} rows imported ({int(total_imported/total_rows*100)}%)")
        
        if all_failed_rows:
            print(f"Table '{table_name}': {total_imported}/{total_rows} rows imported, {len(all_failed_rows)} rows failed")
        else:
            print(f"Table '{table_name}': all {total_imported} rows imported successfully")
        
        return total_imported, all_failed_rows
    
    @staticmethod
    def _normalize_value(value, is_timestamp=False):
        """
        Normalize a single exp file field value for PostgreSQL COPY:
        - Empty string or literal 'null' -> NULL
        - 14-digit string in a timestamp column -> YYYY-MM-DD HH:MM:SS
        - Escape special COPY characters otherwise
        """
        if value == '' or value == 'null':
            return '\\N'
        if is_timestamp and len(value) == 14 and value.isdigit():
            try:
                return f"{value[0:4]}-{value[4:6]}-{value[6:8]} {value[8:10]}:{value[10:12]}:{value[12:14]}"
            except Exception:
                pass
        return value.replace('\\', '\\\\').replace('\t', '\\t').replace('\n', '\\n').replace('\r', '\\r')

    def _build_copy_buffer(self, rows, columns, timestamp_cols=None):
        """Build a tab-delimited StringIO buffer for COPY."""
        ts_cols = timestamp_cols or set()
        data_buffer = StringIO()
        for row in rows:
            values = []
            for i in range(1, len(columns) + 1):
                if i < len(row):
                    col_name = columns[i - 1]
                    values.append(self._normalize_value(row[i], is_timestamp=(col_name in ts_cols)))
                else:
                    values.append('\\N')
            data_buffer.write('\t'.join(values))
            data_buffer.write('\n')
        data_buffer.seek(0)
        return data_buffer
    
    def _import_chunk_with_fallback(self, table_name, rows, columns, offset, timestamp_cols=None):
        """
        Try bulk COPY first. On failure, rollback and fall back to row-by-row INSERT
        to identify and skip the failing rows.
        Returns: (imported_count, failed_rows_list)
        """
        column_list = ', '.join(columns)
        copy_sql = f"COPY {table_name.lower()} ({column_list}) FROM STDIN WITH (FORMAT text, DELIMITER E'\\t')"
        
        try:
            data_buffer = self._build_copy_buffer(rows, columns, timestamp_cols)
            self.cursor.copy_expert(copy_sql, data_buffer)
            self.conn.commit()
            return len(rows), []
        except Exception as bulk_err:
            self.conn.rollback()
            print(f"Bulk COPY failed for {table_name} (chunk at offset {offset}): {bulk_err}")
            print(f"Falling back to row-by-row INSERT to skip bad rows...")
            return self._insert_rows_one_by_one(table_name, rows, columns, offset, timestamp_cols)
    
    def _insert_rows_one_by_one(self, table_name, rows, columns, offset, timestamp_cols=None):
        """
        Insert rows one at a time. Skip and report failing rows.
        Returns: (imported_count, failed_rows_list)
        """
        column_list = ', '.join(columns)
        placeholders = ', '.join(['%s'] * len(columns))
        insert_sql = f"INSERT INTO {table_name.lower()} ({column_list}) VALUES ({placeholders})"
        
        imported = 0
        failed_rows = []
        
        ts_cols = timestamp_cols or set()
        for i, row in enumerate(rows):
            values = []
            for col_idx in range(1, len(columns) + 1):
                col_name = columns[col_idx - 1]
                if col_idx < len(row):
                    val = row[col_idx]
                    if val == '' or val == 'null':
                        values.append(None)
                    elif col_name in ts_cols and len(val) == 14 and val.isdigit():
                        try:
                            values.append(f"{val[0:4]}-{val[4:6]}-{val[6:8]} {val[8:10]}:{val[10:12]}:{val[12:14]}")
                        except Exception:
                            values.append(val)
                    else:
                        values.append(val)
                else:
                    values.append(None)
            
            try:
                self.cursor.execute(insert_sql, values)
                self.conn.commit()
                imported += 1
            except Exception as row_err:
                self.conn.rollback()
                row_num = offset + i + 1
                error_msg = str(row_err).split('\n')[0]
                failed_rows.append({
                    "row_number": row_num,
                    "error": error_msg,
                    "key_values": values[:3]
                })
                print(f"  Row {row_num} failed: {error_msg}", file=sys.stderr)
        
        print(f"Row-by-row fallback: {imported} imported, {len(failed_rows)} failed")
        return imported, failed_rows
    
    def _parse_table_config(self, config_value):
        """
        Parse per-table config, supporting both old and new formats.
        Old format: ["col1", "col2"]  ->  {"filter_columns": ["col1", "col2"]}
        New format (auto_increment_id): {"filter_columns": [...], "auto_increment_id": "id_col"}
            Assigns IDs starting from MAX(id_col)+1 in the target table.
            If filter_columns is omitted, all exp file rows are imported (no ctm_name filtering).
        New format (i_serials_id): {"filter_columns": [...], "i_serials_id": "id_col"}
            Assigns IDs starting from i_serials.cnt+1 for this table, then increments the counter.
            auto_increment_id and i_serials_id are mutually exclusive.
        No-filter mode (i_serials_id without filter_columns): {"i_serials_id": "id_col"}
            Imports ALL rows from the exp file, assigning i_serials IDs. No ctm_name filtering.
            Mutually exclusive with auto_increment_id.
        remap_dependents (optional, used with i_serials_id): {"child_table": "fk_column", ...}
            After rewriting parent IDs, registers old->new map so that when child tables are
            imported their foreign key column is remapped to the new parent IDs.
            Parent table must appear before the child table in table_configs.
        db_table_name (optional): override the DB table name used for column lookup and INSERT.
            Use when the exp file table name differs from the actual DB table name.
            Example: exp table "JOB" maps to DB table "def_ver_job".
        filter_by_remap (optional): column name to use for filtering rows by the registered
            remap map from a parent table. Rows are kept only if their value for this column
            exists as a key in the parent's old->new ID map. Applied before remap rewriting.
        force_column_values (optional): {"column": "value"} — overwrite column when value differs
            (e.g. df_calendar link_id -> -1).
        """
        if isinstance(config_value, list):
            return {"filter_columns": config_value}
        return config_value
    
    def _rewrite_auto_increment_ids(self, table_name, rows, id_column_name, db_columns):
        """
        Rewrite the ID column in rows with auto-incrementing values starting from MAX(id)+1 in DB.
        """
        id_col_idx = db_columns.index(id_column_name)
        
        self.cursor.execute(f"SELECT COALESCE(MAX({id_column_name}), 0) FROM {table_name.lower()}")
        current_max = int(self.cursor.fetchone()[0])
        print(f"Table '{table_name}': current MAX({id_column_name}) = {current_max}, rewriting IDs starting from {current_max + 1}")
        
        for i, row in enumerate(rows):
            new_id = current_max + i + 1
            row[id_col_idx + 1] = str(new_id)
        
        return rows
    
    def _rewrite_i_serials_ids(self, table_name, rows, id_column_name, db_columns):
        """
        Rewrite the ID column in rows using the i_serials serial counter.
        IDs are assigned starting from old_cnt + 1.
        Returns: (rows, old_cnt) — caller must call _update_i_serials() after import.
        Raises ValueError if table_name is not found in i_serials.
        """
        id_col_idx = db_columns.index(id_column_name)

        self.cursor.execute("SELECT cnt FROM i_serials WHERE name = %s", (table_name.upper(),))
        row = self.cursor.fetchone()
        if row is None:
            raise ValueError(f"Table '{table_name}' not found in i_serials — cannot assign serial IDs")
        
        old_cnt = int(row[0])

        self.cursor.execute(f"SELECT COALESCE(MAX({id_column_name}), 0) FROM {table_name.lower()}")
        max_in_db = int(self.cursor.fetchone()[0])

        if max_in_db > old_cnt:
            print(f"Warning: Table '{table_name}': i_serials.cnt={old_cnt} is behind MAX({id_column_name})={max_in_db} in DB — using DB max as base", file=sys.stderr)

        effective_base = max(old_cnt, max_in_db)
        print(f"Table '{table_name}': effective base = {effective_base}, assigning IDs {effective_base + 1} to {effective_base + len(rows)}")


        old_ids = [data_row[id_col_idx + 1] for data_row in rows]
        for i, data_row in enumerate(rows):
            data_row[id_col_idx + 1] = str(effective_base + i + 1)
        new_ids = [data_row[id_col_idx + 1] for data_row in rows]
        id_map = dict(zip(old_ids, new_ids))

        return rows, effective_base, id_map
    
    def _update_i_serials(self, table_name, base, imported_count):
        """
        Set the i_serials counter for table_name to base + imported_count.
        Uses an absolute value (not increment) so that if i_serials was behind
        the actual DB max, the counter is correctly fast-forwarded.
        Should be called after a successful import when i_serials_id is used.
        """
        new_cnt = base + imported_count
        self.cursor.execute(
            "UPDATE i_serials SET cnt = %s WHERE name = %s",
            (new_cnt, table_name.upper())
        )
        self.conn.commit()
        print(f"Table '{table_name}': i_serials.cnt updated to {new_cnt}")


    def _apply_id_remaps(self, table_name, rows, db_columns):
        """
        Apply registered old→new ID remaps to foreign key columns in rows.
        Remaps are registered by parent tables via remap_dependents config.
        Only values present in the map are rewritten; unrecognised values are left unchanged.
        """
        remaps = self._id_remaps.get(table_name, {})
        for col_name, id_map in remaps.items():
            if col_name not in db_columns:
                print(f"Warning: remap column '{col_name}' not found in '{table_name}', skipping remap", file=sys.stderr)
                continue
            col_idx = db_columns.index(col_name)
            remapped = 0
            for row in rows:
                old_val = row[col_idx + 1]
                if old_val in id_map:
                    row[col_idx + 1] = id_map[old_val]
                    remapped += 1
            print(f"Table '{table_name}': remapped {remapped}/{len(rows)} values in column '{col_name}'")

        return rows

    def _resolve_exp_name(self, table_name, all_tables):
        """Find the exp file record key for this config table name.
        Tries exact match first, then strips known prefixes (e.g. DEF_VER_).
        Returns the matching exp key, or None if not found.
        """
        if table_name in all_tables:
            return table_name
        for prefix in ('DEF_VER_',):
            if table_name.upper().startswith(prefix):
                short = table_name[len(prefix):]
                if short in all_tables:
                    return short
        return None

    @staticmethod
    def _resolve_em_bin():
        """Return path to em CLI (em.bat on Windows, em on Unix)."""
        em_home = os.environ.get("EM_HOME")
        if platform.system() == "Windows":
            if not em_home:
                return "em.bat"
            return os.path.join(em_home, "bin", "em.bat")
        if not em_home:
            return "em"
        return os.path.join(em_home, "bin", "em")

    @staticmethod
    def _run_single_table_export(em_bin, table_name, stdout_f):
        """Invoke em util -single_table_export (em.bat on Windows needs shell/cmd)."""
        if platform.system() == "Windows":
            cmd = f'"{em_bin}" util -single_table_export {table_name}'
            subprocess.run(
                cmd,
                stdout=stdout_f,
                stderr=subprocess.PIPE,
                check=True,
                text=True,
                shell=True,
            )
            return
        subprocess.run(
            [em_bin, "util", "-single_table_export", table_name],
            stdout=stdout_f,
            stderr=subprocess.PIPE,
            check=True,
            text=True,
        )

    @staticmethod
    def _parse_exp_table(table_name):
        em_bin = ExpFileFilterImporter._resolve_em_bin()

        with tempfile.NamedTemporaryFile(mode="w", suffix=".exp", delete=False) as tf:
            tmp_path = tf.name
        try:
            try:
                with open(tmp_path, "w", encoding="iso-8859-1") as out_f:
                    ExpFileFilterImporter._run_single_table_export(em_bin, table_name, out_f)
            except (subprocess.CalledProcessError, FileNotFoundError) as e:
                stderr = getattr(e, "stderr", "") or ""
                print(f"Error: '{em_bin} util -single_table_export {table_name}' failed: "
                      f"{e}{(' ' + stderr) if stderr else ''}",
                      file=sys.stderr)
                return [], []

            fld_delim = None
            rec_delim = None
            with open(tmp_path, "r", encoding="iso-8859-1") as f:
                for line in f:
                    if line.startswith("FLD_DELIM"):
                        fld_delim = line[len("FLD_DELIM"):].strip()
                    elif line.startswith("REC_DELIM"):
                        rec_delim = line[len("REC_DELIM"):].strip()
                    if "DATA : Do not delete this line" in line:
                        break
                else:
                    print(f"Error: malformed export for table '{table_name}': "
                          f"DATA marker not found", file=sys.stderr)
                    return [], []
                if not fld_delim:
                    print(f"Error: malformed export for table '{table_name}': "
                          f"FLD_DELIM not found", file=sys.stderr)
                    return [], []
                data = f.read()

            end_marker = "# CONTROL-M/Enterprise Manager : END OF DUMP"
            if end_marker in data:
                data = data.split(end_marker, 1)[0]

            # Split into records. If REC_DELIM is absent or just a
            # line terminator, fall back to one record per line.
            if rec_delim and rec_delim not in ("", "\n", "\r", "\r\n"):
                raw_records = data.split(rec_delim)
            else:
                raw_records = data.splitlines()
            records = [r.strip("\n\r") for r in raw_records if r.strip("\n\r")]
            if not records:
                return [], []

            # First record is the header:
            #     <table_name><FLD_DELIM><col1><FLD_DELIM><col2>...
            header_fields = records[0].split(fld_delim)
            data_records = [rec.split(fld_delim) for rec in records[1:]]
            return header_fields, data_records
        finally:
            try:
                os.unlink(tmp_path)
            except OSError:
                pass

    @staticmethod
    def _column_to_array(table_name, col_name, unique=False):
        header_fields, records = ExpFileFilterImporter._parse_exp_table(table_name)
        if not header_fields or col_name not in header_fields[1:]:
            return []
        col_idx = header_fields.index(col_name)

        values = [fields[col_idx] for fields in records if col_idx < len(fields)]

        if not unique:
            return values
        seen = set()
        out = []
        for v in values:
            if v in seen:
                continue
            seen.add(v)
            out.append(v)
        return out

    @staticmethod
    def _table_to_rows(table_name):
        """
        Run `em util -single_table_export TABLE_NAME` and return rows as a list
        of dicts keyed by LOWERCASED column names (mirrors the EM DB row
        shape). All values are strings (the .exp format is text-only, so
        NULL/None becomes empty string and numerics are not coerced).
        Returns [] on failure or empty table.
        """
        header_fields, records = ExpFileFilterImporter._parse_exp_table(table_name)
        if len(header_fields) < 2:
            return []
        column_names = [c.lower() for c in header_fields[1:]]
        rows = []
        for fields in records:
            # Data records are [<table_name>, <col1_value>, <col2_value>, ...]
            values = fields[1:]
            row = {col: (values[i] if i < len(values) else "")
                   for i, col in enumerate(column_names)}
            rows.append(row)
        return rows

    @staticmethod
    def get_data_centers():
        return ExpFileFilterImporter._column_to_array("COMM", "DATA_CENTER", unique=True)

    def _apply_exclude_values(self, table_name, rows, exclude_values, db_columns):
        """Remove rows where a column matches any of the excluded values.
        exclude_values: {column_name: [value1, value2, ...]}
        Returns (kept_rows, excluded_rows).
        """
        if not exclude_values:
            return rows, []
        excluded = []
        for col_name, values_to_skip in exclude_values.items():
            if col_name not in db_columns:
                print(f"Warning: exclude column '{col_name}' not found in '{table_name}', skipping exclude", file=sys.stderr)
                continue
            before = len(rows)
            col_idx = db_columns.index(col_name)
            skip_set = set(values_to_skip)
            kept = []
            for row in rows:
                if col_idx + 1 < len(row) and row[col_idx + 1] in skip_set:
                    excluded.append(row)
                else:
                    kept.append(row)
            rows = kept
            print(f"Table '{table_name}': {before} -> {len(rows)} rows after excluding {col_name} in {values_to_skip}")
        return rows, excluded

    def _apply_replace_values(self, table_name, rows, replace_values, db_columns):
        """Replace column values in-place.
        replace_values: [{"column": "col", "old_value": "X", "new_value": "Y"}, ...]
        """
        if not replace_values:
            return rows
        for rule in replace_values:
            col_name = rule["column"]
            old_val = rule["old_value"]
            new_val = rule["new_value"]
            if col_name not in db_columns:
                print(f"Warning: replace column '{col_name}' not found in '{table_name}', skipping", file=sys.stderr)
                continue
            col_idx = db_columns.index(col_name) + 1
            replaced = 0
            for row in rows:
                if col_idx < len(row) and row[col_idx] == old_val:
                    row[col_idx] = new_val
                    replaced += 1
            if replaced > 0:
                print(f"Table '{table_name}': replaced {replaced} values in '{col_name}': '{old_val}' -> '{new_val}'")
            else:
                print(f"Warning: Table '{table_name}': replace_values found 0 matches for '{col_name}'='{old_val}' "
                      f"(checked {len(rows)} rows — value may differ in exp file or column ordering may be unexpected)")
        return rows

    def _filter_rows(self, table_name, table_rows, filter_value,
                     filter_by_remap, filter_column_names,
                     exclude_values, replace_values, auto_increment_id, db_columns):
        """Filter table rows according to the configured filter mode.
        filter_value: the value to match in filter_columns (e.g. CTM name).
                      Comes from the per-table config 'filter_value' key.
        exclude_values: {column_name: [values]} — rows matching any excluded value are dropped.
        Returns (filtered_rows, excluded_rows), or (None, []) if the table should be skipped.
        excluded_rows contains every row removed by exclude_values (for skip logging).
        """
        if filter_by_remap:
            remap_map = self._id_remaps.get(table_name, {}).get(filter_by_remap, {})
            if not remap_map:
                print(f"Warning: No remap registered for '{table_name}.{filter_by_remap}', skipping")
                return None, []
            if filter_by_remap not in db_columns:
                print(f"Error: filter_by_remap column '{filter_by_remap}' not found in '{table_name}', skipping")
                return None, []
            col_idx = db_columns.index(filter_by_remap)
            filtered     = [row for row in table_rows if row[col_idx + 1] in remap_map]
            non_filtered = [row for row in table_rows if row[col_idx + 1] not in remap_map]
            print(f"Table '{table_name}': {len(table_rows)} -> {len(filtered)} rows after remap-filter on '{filter_by_remap}'")
            filtered, excluded = self._apply_exclude_values(table_name, filtered, exclude_values, db_columns)
            return self._apply_replace_values(table_name, filtered, replace_values, db_columns), non_filtered + excluded

        if not filter_column_names and not filter_by_remap:
            print(f"Table '{table_name}': importing all {len(table_rows)} rows (no-filter mode)")
            rows, excluded = self._apply_exclude_values(table_name, list(table_rows), exclude_values, db_columns)
            return self._apply_replace_values(table_name, rows, replace_values, db_columns), excluded

        if not filter_value:
            print(f"Error: Table '{table_name}' has filter_columns but no filter_value in config, skipping")
            return None, []

        filtered, non_filtered = self.filter_table(table_rows, filter_value, filter_column_names, db_columns)
        print(f"Table '{table_name}': {len(table_rows)} -> {len(filtered)} rows after filtering by '{filter_value}'")
        filtered, excluded = self._apply_exclude_values(table_name, filtered, exclude_values, db_columns)
        return self._apply_replace_values(table_name, filtered, replace_values, db_columns), non_filtered + excluded

    def _rewrite_ids(self, table_name, db_table_name, rows, auto_increment_id,
                     db_columns, remap_dependents, use_i_serials=False):
        """Rewrite the ID column if needed, and register child remaps.

        Priority:
          1. auto_increment_id in config → rewrite using MAX(col)+1
          2. Table found in i_serials    → rewrite using i_serials counter (PK auto-detected)
          3. Neither                     → no ID rewriting

        Returns (rows, effective_base, i_serials_col) on success,
                or None if the table should be skipped.
        effective_base and i_serials_col are None when i_serials is not used.
        """
        if auto_increment_id:
            if auto_increment_id not in db_columns:
                print(f"Error: auto_increment_id column '{auto_increment_id}' not found in '{table_name}', skipping")
                return None
            rows = self._rewrite_auto_increment_ids(table_name, rows, auto_increment_id, db_columns)
            return rows, None, None

        # i_serials management is controlled explicitly via the per-table "use_i_serials" config flag.
        # The serial column is auto-detected from the table's PK — no explicit config needed.
        detected_serial_col = self._find_serial_column(
            db_table_name, db_columns,
            use_i_serials=use_i_serials
        ) if use_i_serials else None
        if detected_serial_col:
            print(f"Table '{table_name}': detected i_serials tracking on column '{detected_serial_col}'")
            try:
                rows, effective_base, id_map = self._rewrite_i_serials_ids(db_table_name, rows, detected_serial_col, db_columns)
            except ValueError as e:
                print(f"Error: {e}, skipping table '{table_name}'")
                return None
            for child_table, child_col in remap_dependents.items():
                if child_table not in self._id_remaps:
                    self._id_remaps[child_table] = {}
                self._id_remaps[child_table][child_col] = id_map
            return rows, effective_base, detected_serial_col

        # No ID rewriting: no auto_increment_id and not in i_serials.
        # Still register identity remap for any remap_dependents so children
        # can be filtered by the parent's existing IDs without any rewriting.
        if remap_dependents:
            id_col = next(iter(remap_dependents.values()), None)
            if id_col and id_col in db_columns:
                col_idx = db_columns.index(id_col)
                id_map = {row[col_idx + 1]: row[col_idx + 1] for row in rows}
                for child_table, child_col in remap_dependents.items():
                    if child_table not in self._id_remaps:
                        self._id_remaps[child_table] = {}
                    self._id_remaps[child_table][child_col] = id_map
                print(f"Table '{table_name}': registered identity remap for {list(remap_dependents.keys())} "
                      f"({len(id_map)} IDs, no i_serials rewrite needed)")

        return rows, None, None

    def _append_suffix_on_conflict(self, db_table_name, rows, col_name, suffix, db_columns):
        """
        For rows whose col_name value already exists in the target DB table,
        append suffix to make the value unique before import.
        Intended for PK/unique-name columns (e.g. DF_PERIODIC_STATS.name) where
        a collision would cause a duplicate-key error.
        """
        if col_name not in db_columns:
            print(f"Warning: append_suffix_on_conflict column '{col_name}' not found in "
                  f"'{db_table_name}', skipping", file=sys.stderr)
            return rows

        self.cursor.execute(
            f"SELECT {col_name} FROM public.{db_table_name.lower()}"
        )
        existing = {row[0] for row in self.cursor.fetchall()}

        col_idx = db_columns.index(col_name) + 1  # +1: row[0] is the exp table-name field
        renamed = 0
        for row in rows:
            if col_idx < len(row) and row[col_idx] in existing:
                row[col_idx] = row[col_idx] + suffix
                renamed += 1

        if renamed:
            print(f"Table '{db_table_name}': appended suffix '{suffix}' to {renamed} "
                  f"conflicting '{col_name}' value(s)")
        else:
            print(f"Table '{db_table_name}': no '{col_name}' conflicts found, no suffix appended")
        return rows

    def _skip_on_conflict(self, db_table_name, rows, col_name, db_columns):
        """
        Remove rows whose col_name value(s) already exist in the target DB table.
        col_name may be a single column name (str) or a list of column names for a
        composite unique / PK constraint check.

        The conflict check uses col_name, but the value written to skipped_pks is always
        the table's primary-key representation (auto-detected from the DB schema) so that
        the skip log consistently shows PKs regardless of what the skip criteria was.
        For composite PKs the representation is 'val1|val2|...' (pipe-separated).

        Returns (filtered_rows, skipped_pks).
        """
        # Normalise conflict columns to a list.
        if isinstance(col_name, str):
            col_names = [col_name]
        else:
            col_names = list(col_name)

        for cn in col_names:
            if cn not in db_columns:
                print(f"Warning: skip_on_conflict column '{cn}' not found in "
                      f"'{db_table_name}', skipping check", file=sys.stderr)
                return rows, []

        # Build the set of already-existing conflict-key values from the DB.
        # +1 offset: row[0] is the exp file table-name field, not a data value.
        conflict_indices = [db_columns.index(cn) + 1 for cn in col_names]
        select_cols = ', '.join(col_names)
        self.cursor.execute(f"SELECT {select_cols} FROM public.{db_table_name.lower()}")

        if len(col_names) == 1:
            existing = {row[0] for row in self.cursor.fetchall()}
        else:
            existing = {
                tuple(str(v) if v is not None else '' for v in row)
                for row in self.cursor.fetchall()
            }

        # Auto-detect PK columns so the log always shows the PK, not the conflict column.
        pk_cols = self._get_primary_key_columns(db_table_name)
        pk_indices = [db_columns.index(c) + 1 for c in pk_cols if c in db_columns]

        kept_rows   = []
        skipped_pks = []

        for row in rows:
            # Determine whether this row conflicts.
            if len(col_names) == 1:
                conflict_val = row[conflict_indices[0]] if conflict_indices[0] < len(row) else ''
                conflicts = conflict_val in existing
            else:
                conflict_vals = tuple(row[idx] if idx < len(row) else '' for idx in conflict_indices)
                conflicts = conflict_vals in existing

            if conflicts:
                # Log the PK of the skipped row (not the conflict column value).
                if pk_indices:
                    pk_repr = '|'.join(str(row[idx]) if idx < len(row) else '' for idx in pk_indices)
                else:
                    # Fallback: use conflict column value when no PK is detectable.
                    if len(col_names) == 1:
                        pk_repr = str(conflict_val)
                    else:
                        pk_repr = '|'.join(str(v) for v in conflict_vals)
                skipped_pks.append(pk_repr)
            else:
                kept_rows.append(row)

        skipped = len(rows) - len(kept_rows)
        if skipped:
            print(f"Table '{db_table_name}': skipped {skipped} row(s) whose '{col_name}' "
                  f"already exists in target")
        else:
            print(f"Table '{db_table_name}': no '{col_name}' conflicts found, all rows kept")
        return kept_rows, skipped_pks

    def _overwrite_if_empty(self, db_table_name, rows, name_column, name_value,
                            xml_column, empty_xml_max_length, db_columns):
        """
        Handle the special entry identified by name_column=name_value (e.g. SCENARIOS/<SYSTEM>).

        The entry is guaranteed to exist in the target DB.  Two cases:
          * Existing XML length <= empty_xml_max_length  →  the entry still holds the factory
            default, so overwrite every column with the import-file values via UPDATE.
          * Existing XML length >  empty_xml_max_length  →  the tenant has customised it;
            leave it untouched (skip the import row).

        In both cases the row is removed from the returned list so that the subsequent
        skip_on_conflict step (which would unconditionally skip it) never sees it.

        If the entry does not exist in the DB at all, the row is kept for normal INSERT.

        Returns the (possibly modified) rows list.
        """
        if name_column not in db_columns:
            print(f"Warning: overwrite_if_empty name_column '{name_column}' not found in "
                  f"'{db_table_name}', skipping special handling", file=sys.stderr)
            return rows
        if xml_column not in db_columns:
            print(f"Warning: overwrite_if_empty xml_column '{xml_column}' not found in "
                  f"'{db_table_name}', skipping special handling", file=sys.stderr)
            return rows

        self.cursor.execute(
            f"SELECT {xml_column} FROM public.{db_table_name.lower()} WHERE {name_column} = %s",
            (name_value,)
        )
        existing = self.cursor.fetchone()

        if existing is None:
            print(f"Table '{db_table_name}': '{name_value}' not found in DB — will be inserted normally")
            return rows

        existing_xml = existing[0] or ""
        xml_len = len(existing_xml)

        name_col_idx = db_columns.index(name_column) + 1  # +1: row[0] is exp table-name field

        # Locate the import row for this special entry
        target_rows  = [r for r in rows if name_col_idx < len(r) and r[name_col_idx] == name_value]
        other_rows   = [r for r in rows if not (name_col_idx < len(r) and r[name_col_idx] == name_value)]

        if not target_rows:
            print(f"Table '{db_table_name}': '{name_value}' not present in import file, nothing to do")
            return rows

        import_row = target_rows[0]

        if xml_len <= empty_xml_max_length:
            # Factory-default XML → overwrite all columns via UPDATE
            set_clause  = ', '.join(f"{col} = %s" for col in db_columns)
            ts_cols     = self.get_timestamp_columns(db_table_name)
            values = []
            for i, col in enumerate(db_columns):
                raw = import_row[i + 1] if (i + 1) < len(import_row) else ''
                if raw == '':
                    values.append(None)
                elif col in ts_cols and len(raw) == 14 and raw.isdigit():
                    values.append(
                        f"{raw[0:4]}-{raw[4:6]}-{raw[6:8]} {raw[8:10]}:{raw[10:12]}:{raw[12:14]}"
                    )
                else:
                    values.append(raw)
            self.cursor.execute(
                f"UPDATE public.{db_table_name.lower()} SET {set_clause} WHERE {name_column} = %s",
                values + [name_value]
            )
            self.conn.commit()
            print(f"Table '{db_table_name}': '{name_value}' had default XML (len={xml_len}), "
                  f"overwritten with import-file values")
        else:
            print(f"Table '{db_table_name}': '{name_value}' has customised XML (len={xml_len} > "
                  f"{empty_xml_max_length}), keeping existing entry (skipped)")

        return other_rows

    def _replace_xml_tag(self, db_table_name, rows, col_name, tag, new_value, db_columns):
        """
        In an XML column, replace the text content of <tag>...</tag> with new_value
        for every row. Handles any existing value: specific name, wildcard (*), comma-
        separated list, prefix wildcard (e.g. Tlv*) — all are overwritten unconditionally.
        Rows where the tag is absent are left unchanged.

        Example: <CtmName>OldCTM,Tlv*</CtmName>  ->  <CtmName>MY_CTM</CtmName>
        """
        if col_name not in db_columns:
            print(f"Warning: replace_xml_tag column '{col_name}' not found in "
                  f"'{db_table_name}', skipping", file=sys.stderr)
            return rows

        col_idx     = db_columns.index(col_name) + 1  # +1: row[0] is the exp table-name field
        pattern     = re.compile(rf'<{re.escape(tag)}>[^<]*</{re.escape(tag)}>')
        replacement = f'<{tag}>{new_value}</{tag}>'
        updated     = 0

        for row in rows:
            if col_idx < len(row) and row[col_idx]:
                new_xml = pattern.sub(replacement, row[col_idx])
                if new_xml != row[col_idx]:
                    row[col_idx] = new_xml
                    updated += 1

        print(f"Table '{db_table_name}': replaced <{tag}> content with '{new_value}' "
              f"in {updated}/{len(rows)} '{col_name}' XML value(s)")
        return rows

    def _apply_force_column_values(self, table_name, rows, force_values, db_columns):
        """Set column(s) to a fixed value when the current value differs (e.g. link_id -> -1)."""
        if not force_values:
            return rows
        for col_name, target_value in force_values.items():
            if col_name not in db_columns:
                print(f"Warning: force_column_values column '{col_name}' not found in '{table_name}', skipping",
                      file=sys.stderr)
                continue
            col_idx = db_columns.index(col_name) + 1
            changed = 0
            for row in rows:
                if col_idx >= len(row):
                    continue
                if row[col_idx] != target_value:
                    row[col_idx] = target_value
                    changed += 1
            if changed:
                print(f"Table '{table_name}': force_column_values set '{col_name}' to '{target_value}' on {changed} rows")
        return rows

    def _row_pk_reprs(self, rows, pk_cols, db_columns):
        """Return PK string representations for a list of rows.
        For composite PKs the values are pipe-joined: 'val1|val2|...'.
        Rows whose PK columns are missing are silently skipped.
        """
        col_indices = [db_columns.index(c) + 1 for c in pk_cols if c in db_columns]
        if not col_indices:
            return []
        result = []
        for row in rows:
            vals = [row[idx] if idx < len(row) else '' for idx in col_indices]
            result.append('|'.join(str(v) for v in vals))
        return result

    def _import_table_rows(self, table_name, db_table_name, rows, db_columns):
        """Apply FK remaps, detect timestamp columns, and import rows to PostgreSQL.
        Returns (count, failed_rows).
        """
        rows = self._apply_id_remaps(table_name, rows, db_columns)
        ts_cols = self.get_timestamp_columns(db_table_name)
        count, failed_rows = self.import_to_postgres(db_table_name, rows, db_columns, timestamp_cols=ts_cols)
        return count, failed_rows

    def filter_and_import(self, exp_file, table_configs, use_i_serials=False):
        """
        Main method: Filter and import exp file based on table configurations.

        Args:
            exp_file: Path to exp file
            table_configs: Dict of {table_name: config}
                Each config dict may include:
                  filter_value      – value to match in filter_columns (e.g. CTM name).
                                      Required for tables that use filter_columns.
                  filter_columns    – list of column names to filter on.
                  auto_increment_id – column whose PK is rewritten from MAX(col)+1.
                  i_serials_id      – column whose PK is rewritten via i_serials counter.
                  filter_by_remap   – FK column: keep only rows whose value is in parent remap.
                  remap_dependents  – {child_table: fk_col} to register remap for children.
                  db_table_name     – override DB table name (default: key.lower()).
                  exclude_values    – {col_name: [val1, val2]} rows matching any value are dropped
                                      after the main filter is applied.
                  replace_values    – [{"column": "col", "old_value": "X", "new_value": "Y"}, ...]
                                      replaces matching values in-place after filtering.
                  force_column_values – {"col_name": "value"} overwrites column values when
                                      different, e.g. {"link_id": "-1"}.
                  overwrite_if_empty – {"name_column": "col", "name_value": "<SYSTEM>",
                                        "xml_column": "data", "empty_xml_max_length": 500}
                                      For the single row identified by name_column=name_value:
                                      if the existing DB row's xml_column length is within
                                      empty_xml_max_length (factory default), overwrite the row
                                      via UPDATE; otherwise skip it.  All other conflicting
                                      names are handled by skip_on_conflict as usual.
                                      Applied BEFORE skip_on_conflict.
                  use_i_serials     – boolean: enable i_serials ID management for this table.
                                      The PK/serial column is auto-detected from the DB schema
                                      so no explicit column name is needed.
                                      Overrides the global --use_i_serials flag.
        """
        print(f"Starting exp file import")
        print(f"Exp file: {exp_file}")
        print(f"Tables to process: {list(table_configs.keys())}")
        
        print("Parsing exp file...")
        all_tables = self.parse_exp_file(exp_file)
        print(f"Parsed {len(all_tables)} tables from exp file")
        
        total_imported = 0
        table_results = {}
        skipped_tables = []
        all_failed = {}
        skipped_records = {}  # {table_name: [pk_str, ...]} — rows dropped by skip_on_conflict
        
        for table_name, raw_config in table_configs.items():
            config = self._parse_table_config(raw_config)
            filter_column_names = config.get("filter_columns", [])
            filter_value        = config.get("filter_values") or config.get("filter_value")
            auto_increment_id   = config.get("auto_increment_id")
            remap_dependents    = config.get("remap_dependents", {})
            db_table_name       = config.get("db_table_name") or table_name.lower()
            filter_by_remap     = config.get("filter_by_remap")
            exclude_values      = config.get("exclude_values", {})
            replace_values      = config.get("replace_values", [])
            force_column_values       = config.get("force_column_values", {})
            skip_on_conflict          = config.get("skip_on_conflict")
            overwrite_if_empty        = config.get("overwrite_if_empty")
            replace_xml_tag           = config.get("replace_xml_tag")
            append_suffix_on_conflict = config.get("append_suffix_on_conflict")
            table_use_i_serials       = config.get("use_i_serials", use_i_serials)

            # When skip_on_conflict is the boolean True, auto-detect the table's PK columns
            # from the DB schema so no explicit column list is needed in the config.
            if skip_on_conflict is True:
                pk_cols = self._get_primary_key_columns(db_table_name)
                if pk_cols:
                    print(f"Table '{table_name}': auto-detected PK columns for skip_on_conflict: {pk_cols}")
                    skip_on_conflict = pk_cols
                else:
                    print(f"Warning: Table '{table_name}': skip_on_conflict=true but no PK found in DB, "
                          f"conflict check skipped", file=sys.stderr)
                    skip_on_conflict = None

            # 1. Resolve exp file name (strips DEF_VER_ prefix if needed)
            exp_name = self._resolve_exp_name(table_name, all_tables)
            if exp_name is None:
                print(f"Warning: Table '{table_name}' not found in exp file (tried with/without prefix), skipping")
                skipped_tables.append(table_name)
                continue

            table_rows = all_tables[exp_name]
            print(f"Table '{table_name}' (exp: '{exp_name}', db: '{db_table_name}'): {len(table_rows)} rows in exp file")

            # 2. Get DB columns
            db_columns = self.get_table_columns(db_table_name)
            if not db_columns:
                print(f"Error: Could not get columns for db table '{db_table_name}', skipping")
                skipped_tables.append(table_name)
                continue

            # 3. Filter rows
            filtered_rows, excluded_rows = self._filter_rows(
                table_name, table_rows, filter_value,
                filter_by_remap, filter_column_names,
                exclude_values, replace_values, auto_increment_id, db_columns
            )
            if filtered_rows is None:
                skipped_tables.append(table_name)
                continue

            # Log rows removed by exclude_values as skipped records (auto-detect PK).
            if excluded_rows:
                pk_cols = self._get_primary_key_columns(db_table_name)
                if pk_cols:
                    excluded_pks = self._row_pk_reprs(excluded_rows, pk_cols, db_columns)
                    if excluded_pks:
                        existing = skipped_records.get(table_name, [])
                        skipped_records[table_name] = existing + excluded_pks

            if not filtered_rows:
                print(f"No rows to import for table '{table_name}'")
                table_results[table_name] = 0
                continue

            # 3a. Overwrite the special system entry if its XML is still the factory default;
            #     otherwise skip it.  Must run BEFORE skip_on_conflict so that the entry
            #     is removed from the rows list before the generic conflict check.
            if overwrite_if_empty:
                ow_name_col   = overwrite_if_empty.get("name_column")
                ow_name_val   = overwrite_if_empty.get("name_value")
                ow_xml_col    = overwrite_if_empty.get("xml_column")
                ow_max_len    = overwrite_if_empty.get("empty_xml_max_length", 500)
                if ow_name_col and ow_name_val and ow_xml_col:
                    filtered_rows = self._overwrite_if_empty(
                        db_table_name, filtered_rows,
                        ow_name_col, ow_name_val, ow_xml_col, ow_max_len, db_columns
                    )
                else:
                    print(f"Warning: overwrite_if_empty for '{table_name}' missing "
                          f"'name_column', 'name_value', or 'xml_column', skipping",
                          file=sys.stderr)

            # 3b. Drop rows whose PK/unique column already exists in the target DB (skip mode)
            if skip_on_conflict:
                filtered_rows, skipped_pks = self._skip_on_conflict(
                    db_table_name, filtered_rows, skip_on_conflict, db_columns
                )
                if skipped_pks:
                    skipped_records[table_name] = skipped_pks
                if not filtered_rows:
                    print(f"No new rows to import for table '{table_name}' (all skipped as duplicates)")
                    table_results[table_name] = 0
                    continue

            # 3d. Replace <tag> content inside an XML column (e.g. <CtmName> in job_filter)
            if replace_xml_tag:
                col_name  = replace_xml_tag.get("column")
                tag       = replace_xml_tag.get("tag")
                tag_value = replace_xml_tag.get("value")
                if col_name and tag and tag_value:
                    filtered_rows = self._replace_xml_tag(
                        db_table_name, filtered_rows, col_name, tag, tag_value, db_columns
                    )
                else:
                    print(f"Warning: replace_xml_tag for '{table_name}' missing 'column', "
                          f"'tag', or 'value', skipping", file=sys.stderr)

            # 3e. Append suffix to PK/unique-name values that already exist in the target DB
            if append_suffix_on_conflict:
                col_name = append_suffix_on_conflict.get("column")
                suffix   = append_suffix_on_conflict.get("suffix", "")
                if col_name and suffix:
                    filtered_rows = self._append_suffix_on_conflict(
                        db_table_name, filtered_rows, col_name, suffix, db_columns
                    )
                else:
                    print(f"Warning: append_suffix_on_conflict for '{table_name}' missing "
                          f"'column' or 'suffix', skipping", file=sys.stderr)

            # 4. Rewrite IDs and register child remaps
            rewrite_result = self._rewrite_ids(
                table_name, db_table_name, filtered_rows,
                auto_increment_id, db_columns, remap_dependents,
                use_i_serials=table_use_i_serials
            )
            if rewrite_result is None:
                skipped_tables.append(table_name)
                continue
            filtered_rows, effective_base, i_serials_col = rewrite_result

            # 5. Optional forced column values (e.g. link_id -> -1)
            if force_column_values:
                filtered_rows = self._apply_force_column_values(
                    table_name, filtered_rows, force_column_values, db_columns
                )

            # 6. Import (apply FK remaps + COPY to PostgreSQL)
            count, failed_rows = self._import_table_rows(
                table_name, db_table_name, filtered_rows, db_columns
            )
            total_imported += count
            table_results[table_name] = count
            if failed_rows:
                all_failed[table_name] = failed_rows

            # 7. Update i_serials counter after successful import
            if i_serials_col and count > 0:
                self._update_i_serials(db_table_name, effective_base, count)
        
        total_failed = sum(len(v) for v in all_failed.values())
        print(f"Total imported: {total_imported} rows, {total_failed} rows failed")

        result = {
            "total_imported": total_imported,
            "total_failed": total_failed,
            "tables": table_results,
            "skipped": skipped_tables,
            "failed_rows": {t: [{"row": r["row_number"], "error": r["error"]} for r in rows] for t, rows in all_failed.items()},
            "skipped_records": skipped_records,
        }
        return result
    
    def build_auto_config(self, exp_file, net_type_prefix, filter_value, filter_col='data_center', id_col='table_id'):
        from collections import OrderedDict
        prefix = net_type_prefix.lower()  # e.g. "def_ver_"

        # --- parse exp file header for NET_TYPE and table row counts ---
        tables_in_exp = {}
        with open(exp_file, 'r', encoding='iso-8859-1') as f:
            for line in f:
                if 'DATA : Do not delete this line' in line:
                    break
                line = line.strip()
                if line.startswith('NR '):
                    parts = line.split()
                    if len(parts) >= 3:
                        tables_in_exp[parts[1]] = int(parts[2])

        print(f"Auto-config: prefix='{prefix}', tables in exp header: {list(tables_in_exp.keys())}")

        # --- classify each table ---
        parent = None          # (short_name, db_name)
        children = []          # [(short_name, db_name), ...]

        for short_name in tables_in_exp:
            db_name = prefix + short_name.lower()
            db_cols = self.get_table_columns(db_name)
            if not db_cols:
                print(f"  {short_name}: DB table '{db_name}' not found, skipping")
                continue

            has_filter_col = filter_col in db_cols
            has_id_col     = id_col in db_cols

            if has_filter_col and has_id_col:
                self.cursor.execute("SELECT cnt FROM i_serials WHERE name = %s", (db_name.upper(),))
                if self.cursor.fetchone() is not None:
                    print(f"  {short_name}: PARENT  (db={db_name})")
                    parent = (short_name, db_name)
                else:
                    print(f"  {short_name}: skipping (has {filter_col}+{id_col} but not in i_serials)")
            elif has_id_col:
                print(f"  {short_name}: CHILD   (db={db_name})")
                children.append((short_name, db_name))
            else:
                print(f"  {short_name}: skipping (no {id_col} column in DB)")

        if not parent:
            raise ValueError(
                f"Could not find a parent table for prefix '{prefix}' "
                f"(needs {filter_col}+{id_col}+i_serials entry)"
            )

        parent_short, parent_db = parent

        # Store prefix so filter_and_import can derive DB names automatically.
        # key  = exp file table name  (e.g. "JOB")
        # DB name = prefix + key.lower()  (e.g. "def_ver_job")  — no explicit db_table_name needed.
        self._net_type_prefix = prefix

        # --- build ordered config: parent first, children after ---
        config = OrderedDict()

        config[parent_short] = {
            'filter_columns':   [filter_col],
            'filter_value':     filter_value,
            'remap_dependents': {short: id_col for short, _ in children},
        }

        for short_name, _ in children:
            config[short_name] = {
                'filter_by_remap': id_col,
            }

        print(f"Auto-config built: parent={parent_short}, children={[s for s, _ in children]}")
        return config

    def close(self):
        """Close cursor (connection is managed by db_access)"""
        if self.cursor:
            self.cursor.close()


def main():
    parser = argparse.ArgumentParser(description='Filter and import exp file to PostgreSQL')
    parser.add_argument('--exp_file',    required=True,  help='Path to exp file')
    parser.add_argument('--tables',      required=False, help='JSON dict of {table_name: config} (manual mode). '
                                                              'Each table config may include filter_value for CTM filtering.')
    parser.add_argument('--auto_config', required=False, help='Net-type prefix for dynamic config, e.g. DEF_VER_')
    parser.add_argument('--ctm_name',    required=False, help='CTM name — required with --auto_config, '
                                                              'ignored with --tables (use filter_value per table instead)')
    parser.add_argument('--filter_col',    default='data_center', help='Parent filter column for --auto_config (default: data_center)')
    parser.add_argument('--id_col',        default='table_id',    help='Parent/child ID column for --auto_config (default: table_id)')
    parser.add_argument('--use_i_serials', action='store_true',   help='Enable i_serials ID management. '
                                                                        'If a table is not yet in i_serials, a new row is inserted automatically. '
                                                                        'Without this flag, parent IDs are not rewritten (identity remap used instead).')

    args = parser.parse_args()

    if not args.tables and not args.auto_config:
        print("Error: one of --tables or --auto_config is required", file=sys.stderr)
        sys.exit(1)

    if args.auto_config and not args.ctm_name:
        print("Error: --ctm_name is required when using --auto_config", file=sys.stderr)
        sys.exit(1)

    importer = None
    try:
        conn = get_db_connection()
        importer = ExpFileFilterImporter(conn)

        if args.auto_config:
            table_configs = importer.build_auto_config(
                args.exp_file, args.auto_config, args.ctm_name,
                args.filter_col, args.id_col
            )
            # In auto_config mode, propagate use_i_serials via filter_and_import
        else:
            try:
                table_configs = json.loads(args.tables)
            except json.JSONDecodeError as e:
                print(f"Error parsing table configs JSON: {e}", file=sys.stderr)
                sys.exit(1)

        result = importer.filter_and_import(args.exp_file, table_configs, use_i_serials=args.use_i_serials)

        print(f"\n=== SUCCESS ===")
        print(f"Total rows imported: {result['total_imported']}")
        print(f"RESULT:{json.dumps(result)}")
        sys.exit(0)

    except Exception as e:
        print(f"\n=== ERROR ===", file=sys.stderr)
        print(f"Failed to import exp file: {e}", file=sys.stderr)
        import traceback
        traceback.print_exc()
        sys.exit(1)

    finally:
        if importer:
            importer.close()


if __name__ == "__main__":
    main()
