#!/usr/bin/env python3
"""
Export script to retrieve all rows from job_info tables using the export_job_info function.
Uses psql command-line tool for PostgreSQL or sqlplus for Oracle.
Supports parallel execution with configurable thread pool.
Every call to the stored procedure returns 100MB (into txt file) and the python decide if ask for additional 100MB or to compress gz.
Every call pass skip_rows to the stored procedure to mark from where to continue the export.

main()
├── JobInfoExporter.__init__()
│   ├── extract_db_info()
│   │   └── clean_url()
│   └── is_pg()
│       └── is_windows()
│
├── get_db_size()
│   └── run_query()
│       ├── execute_psql() [if PostgreSQL]
│       └── execute_sqlplus() [if Oracle]
│
├── initialize_database_functions()
│   ├── is_em_ge_v21()
│   │   └── run_query()
│   ├── is_db_version_supported()
│   │   ├── run_query()
│   │   └── is_local_pg()
│   │       ├── em_home()
│   │       └── is_windows()
│   └── execute_psql() [if PostgreSQL]
│   └── execute_sqlplus() [if Oracle]
│
└── export_all_tables_parallel()
    ├── get_existing_job_info_tables()
    │   ├── count_job_info_tables()
    │   │   └── run_query()
    │   ├── tar_file_exists()
    │   ├── get_table_row_count()
    │   │   └── run_query()
    │   └── log()
    │
    ├── count_used_space()
    │   ├── count_used_space_windows() [if Windows]
    │   └── count_used_space_unix() [if Unix/Linux]
    │
    ├── get_available_space()
    │   ├── get_available_space_windows() [if Windows]
    │   └── get_available_space_unix() [if Unix/Linux]
    │
    ├── disk_usage_monitor() [background thread]
    │   ├── count_used_space()
    │   └── thread_safe_print()
    │       └── log()
    │
    └── ThreadPoolExecutor.submit()
        └── export_table_wrapper() [multiple parallel threads]
            ├── get_last_skip_rows()
            │   └── thread_safe_print()
            │
            ├── export_all_rows()
            │   ├── get_table_row_count()
            │   ├── execute_export_query()
            │   │   ├── execute_psql() [if PostgreSQL]
            │   │   └── execute_sqlplus() [if Oracle]
            │   ├── validate_record()
            │   └── thread_safe_print()
            │
            ├── create_rules_json_file()
            │   ├── get_distinct_rule_names()
            │   │   └── run_query()
            │   └── thread_safe_print()
            │
            └── create_tar_and_cleanup()
                └── thread_safe_print()
"""

import os
import subprocess
import sys
import gzip
import shutil
import tarfile
import argparse
import json
import time
import threading
import getpass
import re
import signal
from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError
from threading import Lock, Event
from datetime import datetime
try:
    from colorama import Style, init as colorama_init
    colorama_init(autoreset=True)
    BOLD = Style.BRIGHT
    RESET = Style.RESET_ALL
except ImportError:
    # Fallback if colorama is not installed
    BOLD = ''
    RESET = ''


class JobInfoExporter:
    """Class to handle exporting job_info tables data using psql or sqlplus command."""
    
    # Delimiter constants used for data export
    DELIMITER_START = '#@45>'
    DELIMITER_END = '<45@#'
    DELIMITER_EOL = '[#@!]'
    
    def __init__(self, host, port, database, user, password, db_type=None, log_dir=None, output_dir=None, skip_validations=False, file_size_limit=100000000, combined_file_size_limit=100, max_lines_per_file=30000):
        """
        Initialize the exporter with database connection parameters.
        
        Args:
            host: Database host (optional - will try to extract from jdbc.properties)
            port: Database port (optional - will try to extract from jdbc.properties)
            database: Database name (for PostgreSQL) or SID (for Oracle) (optional - will try to extract from jdbc.properties)
            user: Database user (optional - will try to extract from jdbc.properties)
            password: Database password (required)
            db_type: Database type - 'postgresql' or 'oracle' (default: auto-detect using is_pg())
            log_dir: Directory to store log files (default: <current_working_dir>/logs)
            output_dir: Output directory for export files
            skip_validations: Skip EM version and DB version validation checks (default: False)
            file_size_limit: Maximum number of characters per batch (default: 100000000 = ~100MB)
            combined_file_size_limit: Size limit in MB for combined files before compression (default: 100)
            max_lines_per_file: Maximum number of lines (records) per batch file (default: 30000)
        """
        # Extract database info from jdbc.properties file
        file_db_host, file_db_port, file_db_name, file_db_user = self.extract_db_info()
        
        # Use provided parameters, or fall back to jdbc.properties values
        self.host = host if host else file_db_host
        self.port = port if port else file_db_port
        self.database = database if database else file_db_name
        self.user = user if user else file_db_user
        self.password = password
        self.skip_validations = skip_validations
        self.file_size_limit = file_size_limit
        self.combined_file_size_limit = combined_file_size_limit
        self.max_lines_per_file = max_lines_per_file
        self.PROGRESS_FILE = 'progress-export.json'
        
        # Validate mandatory parameters
        if not self.host:
            print("ERROR: Database host is mandatory. Provide via --host parameter or ensure jdbc.properties file exists.")
            sys.exit(1)
        
        if not self.port:
            print("ERROR: Database port is mandatory. Provide via --port parameter or ensure jdbc.properties file exists.")
            sys.exit(1)
        
        if not self.database:
            print("ERROR: Database name/SID is mandatory. Provide via --database parameter or ensure jdbc.properties file exists.")
            sys.exit(1)
        
        if not self.user:
            print("ERROR: Database user is mandatory. Provide via --user parameter or ensure jdbc.properties file exists.")
            sys.exit(1)
        
        if not self.password:
            # Prompt for password if not provided
            self.password = getpass.getpass("Type your Workload Archiving database password: ")
            if not self.password:
                print("ERROR: Database password is mandatory.")
                sys.exit(1)
        
        # Auto-detect database type if not specified
        if db_type is None:
            self.db_type = 'postgresql' if self.is_pg() else 'oracle'
        else:
            self.db_type = db_type.lower()
        
        self.print_lock = Lock()  # Thread-safe printing
        self.abort_event = Event()  # Event to signal abort
        self.abort_reason = None  # Reason for abort
        self.table_progress = {}  # Dictionary to track progress per table {table_name: {'current': 0, 'total': 0}}
        self.progress_lock = Lock()  # Lock for thread-safe progress updates
        self.active_processes = []  # Track active subprocesses for cleanup
        self.process_lock = Lock()  # Lock for thread-safe process list updates
        
        # Register signal handlers for cleanup
        signal.signal(signal.SIGTERM, self._signal_handler)
        signal.signal(signal.SIGINT, self._signal_handler)
        
        # Create log directory if it doesn't exist (use absolute path)
        if log_dir is None:
            # If no log_dir provided, default to current working directory/logs
            self.log_dir = os.path.join(os.getcwd(), 'logs')
        elif not os.path.isabs(log_dir):
            # If relative path provided, make it relative to current working directory
            self.log_dir = os.path.join(os.getcwd(), log_dir)
        else:
            self.log_dir = log_dir
        
        if not os.path.exists(self.log_dir):
            os.makedirs(self.log_dir)
        
        # Create log file with timestamp
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        self.log_file = os.path.join(self.log_dir, f'export_{timestamp}.log')
        
        # Initialize log file
        now = datetime.now()
        session_start_time = now.strftime('%Y-%m-%d %H:%M:%S') + f'.{now.microsecond // 1000:03d}'
        self.log(f"Export session started at {session_start_time}")
        self.log(f"Database: {self.database} ({self.db_type})")
        self.log(f"Host: {self.host}:{self.port}")
        
        # Database size queries
        self.GET_DB_SIZE_PG = f"SELECT pg_database_size('{self.database}')"
        self.GET_DB_SIZE_ORCL = "SELECT NVL(SUM(bytes), 0) FROM user_segments"
        
        # EM and DB version queries
        self.GET_EM_DB_VERSION_PG = "SELECT pvalue FROM params WHERE pname = 'db_version'"
        self.GET_EM_DB_VERSION_ORCL = "SELECT pvalue FROM params WHERE pname = 'db_version'"
        self.GET_PG_VERSION = "SELECT current_setting('server_version')"
        
        # Job info table count query (works for both PostgreSQL and Oracle)
        self.COUNT_JOB_INFO_TABLES_QUERY = "SELECT COUNT(*) FROM partitions WHERE partition_type = 'job_info'"
        
        # Get and log database size
        db_size = self.get_db_size()
        if db_size:
            db_size_gb = db_size / (1024 * 1024 * 1024)
            self.log(f"Database Size: {db_size:,} bytes ({db_size_gb:.2f} GB)")
        
        # Perform validations unless skipped
        if not self.skip_validations:
            if not self.is_em_ge_v21():
                error_msg = 'EM Version must be greater than or equals to 21.'
                self.log(error_msg)
                print(f"ERROR: {error_msg}")
                sys.exit(1)
            
            if not self.is_db_version_supported():
                error_msg = 'Database version is not supported'
                self.log(error_msg)
                print(f"ERROR: {error_msg}")
                sys.exit(1)
        else:
            self.log("Skipping EM and DB version validations")
        
        self.log("="*80)
    
    def load_progress_from_file(self):
        """Load progress from progress-export.json file.
        
        Returns:
            Dictionary with table completion status, or empty dict if file doesn't exist
        """
        progress = {}
        try:
            if os.path.exists(self.PROGRESS_FILE):
                with open(self.PROGRESS_FILE, 'r') as f:
                    progress = json.load(f)
                    self.log(f"{self.PROGRESS_FILE} content loaded: {progress}")
        except Exception as e:
            self.log(f"Error loading {self.PROGRESS_FILE}: {e}. Running export from scratch")
        return progress
    
    def save_progress_to_file(self, progress):
        """Save progress to progress-export.json file.
        
        Args:
            progress: Dictionary with table completion status
        """
        try:
            with open(self.PROGRESS_FILE, 'w') as f:
                json.dump(progress, f, indent=2)
            self.log(f"Progress saved to {self.PROGRESS_FILE}: {progress}")
        except Exception as e:
            self.log(f"Error saving progress to {self.PROGRESS_FILE}: {e}")
    
    def mark_table_completed(self, table_name):
        """Mark a table as completed in the progress file.
        
        Args:
            table_name: Name of the table to mark as completed
        """
        progress = self.load_progress_from_file()
        progress[table_name] = 'COMPLETED'
        self.save_progress_to_file(progress)
    
    def is_table_completed(self, table_name):
        """Check if a table is marked as completed in the progress file.
        
        Args:
            table_name: Name of the table to check
            
        Returns:
            True if table is marked as COMPLETED, False otherwise
        """
        progress = self.load_progress_from_file()
        return progress.get(table_name) == 'COMPLETED'
    
    def _signal_handler(self, signum, frame):
        """Handle termination signals by killing active Oracle processes"""
        self.log(f"Received signal {signum}, cleaning up active processes...")
        self._cleanup_processes()
        sys.exit(1)
    
    def _cleanup_processes(self):
        """Kill all active subprocesses"""
        with self.process_lock:
            for proc in self.active_processes:
                try:
                    if proc.poll() is None:  # Process still running
                        self.log(f"Terminating process {proc.pid}")
                        proc.terminate()
                        try:
                            proc.wait(timeout=5)
                        except subprocess.TimeoutExpired:
                            self.log(f"Force killing process {proc.pid}")
                            proc.kill()
                except Exception as e:
                    self.log(f"Error cleaning up process: {e}")
            self.active_processes.clear()
    
    def initialize_database_functions(self):
        """
        Install the export_job_info function in the database.
        For PostgreSQL: runs psql -f pg_export.sql
        For Oracle: runs sqlplus @orcl_export.sql
        
        Returns:
            True if successful, False otherwise
        """
        # Get the script directory where the SQL files are located
        script_dir = os.path.dirname(os.path.abspath(__file__))
        
        if self.db_type == 'postgresql':
            sql_file = os.path.join(script_dir, 'pg_export.sql')
            if not os.path.exists(sql_file):
                self.log(f"ERROR: SQL file not found: {sql_file}")
                return False
            
            self.log(f"Installing PostgreSQL export function from {sql_file}")
            result = self.execute_psql(sql_file=sql_file)
            
            if result.returncode != 0:
                self.log(f"ERROR installing PostgreSQL function: {result.stderr}")
                return False
            
            self.log("PostgreSQL export function installed successfully")
            return True
            
        elif self.db_type == 'oracle':
            sql_file = os.path.join(script_dir, 'orcl_export.sql')
            if not os.path.exists(sql_file):
                self.log(f"ERROR: SQL file not found: {sql_file}")
                return False
            
            self.log(f"Installing Oracle export function from {sql_file}")
            
            try:
                # Use a reasonable timeout for installation
                result = self.execute_sqlplus(sql_file=sql_file, timeout=60)
            except subprocess.TimeoutExpired as e:
                # If timeout, log error and suggest manual intervention
                error_msg = f"Installation timed out after 60 seconds. There may be stuck sessions holding locks."
                self.log(f"ERROR: {error_msg}")
                self.log("Manual intervention required:")
                self.log("1. Find stuck sessions: SELECT sid, serial#, username, program, status FROM v$session WHERE username = 'ARCHIVE' AND program LIKE '%sqlplus%';")
                self.log("2. Kill stuck sessions: ALTER SYSTEM KILL SESSION 'sid,serial#' IMMEDIATE;")
                self.log("3. Retry the export")
                print(f"\nERROR: {error_msg}")
                print("Check the log file for manual steps to resolve this issue.")
                return False
            except Exception as e:
                self.log(f"Installation failed: {e}")
                return False
            
            if result.returncode != 0:
                self.log(f"ERROR installing Oracle function: {result.stderr}")
                return False
            
            # Log any output from the installation
            if result.stdout:
                self.log(f"Installation output: {result.stdout}")
            
            # Check for compilation errors in function
            error_check_query = "SELECT line || ' ' || position || ' ' || text FROM user_errors WHERE name = 'EXPORT_JOB_INFO' AND type = 'PROCEDURE' ORDER BY sequence"
            
            errors_result = self.run_query(error_check_query)
            if errors_result and errors_result.strip():
                self.log(f"Oracle function has compilation errors:\n{errors_result}")
                return False
            
            self.log("Oracle export function installed successfully")
            return True
        
        return False
    
    def execute_psql(self, query=None, sql_file=None, tuples_only=False):
        """
        Execute a psql command with proper Windows directory handling.
        
        Args:
            query: SQL query to execute (use -c flag)
            sql_file: SQL file to execute (use -f flag)
            tuples_only: If True, add -t flag for tuples-only output
            
        Returns:
            subprocess.CompletedProcess result
            
        Note:
            Either query or sql_file must be provided, but not both.
        """
        if not query and not sql_file:
            raise ValueError("Either query or sql_file must be provided")
        if query and sql_file:
            raise ValueError("Cannot specify both query and sql_file")
        
        # On Windows, use full path to psql instead of changing directories
        psql_cmd = 'psql'
        if self.is_windows():
            psql_folder = self.em_home() + "\\archive\\pgsql\\bin"
            psql_cmd = f'"{psql_folder}\\psql.exe"'
        
        try:
            env = os.environ.copy()
            env['PGPASSWORD'] = self.password
            env['PGCLIENTENCODING'] = 'UTF8'  # Force UTF-8 output to prevent encoding issues on Windows

            # Build command
            cmd_parts = [f'{psql_cmd} -d{self.database} -h{self.host} -p{self.port} -U{self.user}']
            
            if tuples_only:
                # -t: tuples only (no headers/footers)
                # -A: unaligned output mode (no padding, preserves actual delimiters)
                cmd_parts.append('-t -A')
            
            if sql_file:
                cmd_parts.append(f'-f "{sql_file}"')
            elif query:
                cmd_parts.append(f'-c "{query}"')
            
            command = ' '.join(cmd_parts)
            
            result = subprocess.run(
                command,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                text=True,
                encoding='utf-8',
                errors='replace',  # Replace invalid UTF-8 bytes with replacement character
                env=env
            )
            
            return result
            
        except Exception as e:
            raise Exception(f"psql command failed: {e}")
    
    def execute_sqlplus(self, query=None, sql_file=None, timeout=300):
        """
        Execute a sqlplus command.
        
        Args:
            query: SQL query to execute
            sql_file: SQL file to execute (use @ syntax)
            timeout: Maximum time in seconds to wait (default: 300 = 5 minutes)
            
        Returns:
            subprocess.CompletedProcess result
            
        Note:
            Either query or sql_file must be provided, but not both.
        """
        if not query and not sql_file:
            raise ValueError("Either query or sql_file must be provided")
        if query and sql_file:
            raise ValueError("Cannot specify both query and sql_file")
        
        connection_string = f"{self.user}/{self.password}@{self.host}:{self.port}/{self.database}"
        
        # Set up environment to force UTF-8 output (prevents encoding issues on Windows)
        env = os.environ.copy()
        env['NLS_LANG'] = '.AL32UTF8'  # Character set only, no language/territory

        if sql_file:
            # For SQL files, pipe exit command after the file execution to ensure sqlplus terminates
            command = f'(echo "@{sql_file}"; echo "exit;") | arc sqlplus -s {connection_string}'
        else:
            # Format the query with proper SQL*Plus settings for clean output
            formatted_query = f"set pagesize 0\nset feedback off\nset heading off\nset linesize 32767\n{query}\nexit;"
            command = f'echo "{formatted_query}" | arc sqlplus -s {connection_string}'
        
        try:
            result = subprocess.run(
                command,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                text=True,
                encoding='utf-8',
                errors='replace',  # Replace invalid UTF-8 bytes with replacement character
                timeout=timeout,
                env=env
            )
            return result
        except subprocess.TimeoutExpired:
            error_msg = f"sqlplus command timed out after {timeout} seconds. There may be a stuck session holding locks."
            self.log(f"ERROR: {error_msg}")
            self.log("Suggestion: Check for and kill any stuck sqlplus sessions with:")
            self.log("  SELECT sid, serial#, username, program, status FROM v$session WHERE username='ARCHIVE' AND program LIKE 'sqlplus%' AND status='ACTIVE';")
            self.log("  ALTER SYSTEM KILL SESSION 'sid,serial#' IMMEDIATE;")
            raise Exception(error_msg)
    
    def get_available_space(self, output_folder):
        """
        Get available disk space on the output folder machine.
        
        Args:
            output_folder: Path to the output folder
            
        Returns:
            Available space in GB, or None if command fails
        """
        if self.is_windows():
            return self.get_available_space_windows(output_folder)
        else:
            return self.get_available_space_unix(output_folder)
    
    def get_available_space_unix(self, output_folder):
        """
        Get available disk space on Unix/Linux systems.
        
        Args:
            output_folder: Path to the output folder
            
        Returns:
            Available space in GB, or None if command fails
        """
        try:
            # Command to get available space (works on Linux/Unix)
            space_cmd = f"df -H {output_folder} | tail -1 | awk '{{print $4}}'"
            
            result = subprocess.run(
                space_cmd,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                text=True
            )
            
            if result.returncode != 0:
                self.log(f"Error getting available space: {result.stderr}")
                return None
            
            space_str = result.stdout.strip()
            # Parse space string (e.g., "100G" or "500M")
            if space_str.endswith('G'):
                return float(space_str.rstrip('G'))
            elif space_str.endswith('M'):
                return float(space_str.rstrip('M')) / 1024
            elif space_str.endswith('T'):
                return float(space_str.rstrip('T')) * 1024
            else:
                # Assume it's in bytes
                return float(space_str) / (1024 * 1024 * 1024)
            
        except Exception as e:
            self.log(f"Error getting available space: {e}")
            return None
    
    def get_available_space_windows(self, output_folder):
        """
        Get available disk space on Windows systems.
        
        Args:
            output_folder: Path to the output folder
            
        Returns:
            Available space in GB, or None if command fails
        """
        try:
            # Handle network paths (UNC paths starting with \\)
            if output_folder.startswith('\\\\'):
                # For network paths, use the root share
                total, used, free = shutil.disk_usage(output_folder)
            else:
                # For local paths
                total, used, free = shutil.disk_usage(output_folder)
            
            # Convert free space from bytes to GB
            free_gb = free / (1024 * 1024 * 1024)
            return free_gb
            
        except Exception as e:
            self.log(f"Error getting available space on Windows: {e}")
            return None
    
    def count_used_space(self, output_folder):
        """
        Get used disk space percentage on the output folder machine.
        
        Args:
            output_folder: Path to the output folder
            
        Returns:
            Used space percentage as integer (e.g., 45), or None if command fails
        """
        if self.is_windows():
            return self.count_used_space_windows(output_folder)
        else:
            return self.count_used_space_unix(output_folder)
    
    def count_used_space_unix(self, output_folder):
        """
        Get used disk space percentage on Unix/Linux systems.
        
        Args:
            output_folder: Path to the output folder
            
        Returns:
            Used space percentage as integer (e.g., 45), or None if command fails
        """
        try:
            # Command to get used space percentage
            space_cmd = f"df -H {output_folder} | tail -1 | awk '{{print $5}}'"
            
            result = subprocess.run(
                space_cmd,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                text=True
            )
            
            if result.returncode != 0:
                self.log(f"Error getting used space: {result.stderr}")
                return None
            
            used_space = result.stdout.strip()
            # Parse percentage (e.g., "45%" -> 45)
            return int(used_space.rstrip('%'))
            
        except Exception as e:
            self.log(f"Error getting used space: {e}")
            return None
    
    def count_used_space_windows(self, output_folder):
        """
        Get used disk space percentage on Windows systems.
        
        Args:
            output_folder: Path to the output folder
            
        Returns:
            Used space percentage as integer (e.g., 45), or None if command fails
        """
        try:
            # Handle network paths (UNC paths starting with \\)
            if output_folder.startswith('\\\\'):
                # For network paths, use the root share
                total, used, free = shutil.disk_usage(output_folder)
            else:
                # For local paths
                total, used, free = shutil.disk_usage(output_folder)
            
            # Calculate percentage used
            percent_used = (used / total) * 100
            return int(percent_used)
            
        except Exception as e:
            self.log(f"Error getting used space on Windows: {e}")
            return None
    
    def is_windows(self):
        """Check if running on Windows platform."""
        return os.name == 'nt'
    
    def is_pg(self):
        """
        Check if the database type is PostgreSQL.
        Returns True for PostgreSQL, False for Oracle.
        
        Detection logic:
        1. If running on Windows, assume PostgreSQL
        2. If file ~/ctm_em/archive/.PGenv.csh exists, assume PostgreSQL
        3. Otherwise, assume Oracle
        """
        if self.is_windows():
            return True
        if os.path.exists(os.getenv("HOME") + "/ctm_em/archive/.PGenv.csh"):
            return True
        else:
            return False
    
    def em_home(self):
        """Get the EM_HOME environment variable."""
        return os.getenv("EM_HOME")
    
    def clean_url(self, url):
        """Clean JDBC URL by removing escape characters."""
        return url.replace('\\:', ':').replace('\\\\', '')
    
    def extract_db_info(self):
        """
        Extract database connection information from jdbc.properties file.
        
        Returns:
            Tuple of (host, port, db_name, db_user) or (None, None, None, None) if extraction fails
        """
        try:
            em_home_path = self.em_home()
            if not em_home_path:
                # EM_HOME not set, cannot extract from jdbc.properties
                return None, None, None, None
            
            file_path = os.path.join(em_home_path, "archive", "config", "jdbc.properties")
            
            if not os.path.exists(file_path):
                # jdbc.properties file not found
                return None, None, None, None
            
            host = None
            port = None
            db_name = None
            db_user = None
            
            with open(file_path, 'r') as file:
                for line in file:
                    line = line.strip()
                    if line.startswith('jdbc.url='):
                        url = self.clean_url(line.split('=', 1)[1])
                        if '//' in url:
                            parts = url.split('//')[1].split('/')
                            host_port = parts[0]
                            db_name = parts[1] if len(parts) > 1 else None
                            if ':' in host_port:
                                host, port = host_port.split(':')
                    elif line.startswith('jdbc.user.name='):
                        db_user = line.split('=', 1)[1]
            
            if host and port and db_name and db_user:
                print(f"Extracted from jdbc.properties: host={host}, port={port}, database={db_name}, user={db_user}")
            
            return host, port, db_name, db_user
            
        except Exception as e:
            # Failed to extract from jdbc.properties
            return None, None, None, None
    
    def run_query(self, query):
        """
        Execute a SQL query and return the result.
        Simplified version for validation queries.
        
        Args:
            query: SQL query to execute
            
        Returns:
            Query output as string (stripped), or None if query fails
        """
        try:
            if self.db_type == 'postgresql':
                result = self.execute_psql(query=query, tuples_only=True)
            elif self.db_type == 'oracle':
                # execute_sqlplus handles formatting, just pass the query with semicolon
                result = self.execute_sqlplus(query=query + ";")
            else:
                return None
            
            if result.returncode != 0:
                # Check for specific database connection errors
                stderr_lower = result.stderr.lower()
                stdout_lower = result.stdout.lower() if result.stdout else ""
                
                # Authentication errors (wrong password or user)
                if 'password authentication failed' in stderr_lower or 'authentication failed' in stderr_lower:
                    err_msg = f"Authentication failed. Please check your database password."
                    self.log(f"ERROR: {err_msg}")
                    print(f"ERROR: {err_msg}")
                    sys.exit(1)
                
                # Invalid user
                elif 'role' in stderr_lower and 'does not exist' in stderr_lower:
                    err_msg = f"Database user '{self.user}' does not exist. Please check the username."
                    self.log(f"ERROR: {err_msg}")
                    print(f"ERROR: {err_msg}")
                    sys.exit(1)
                
                # Oracle specific: invalid username/password
                elif 'ora-01017' in stderr_lower or 'ora-01017' in stdout_lower:
                    err_msg = f"Invalid username or password. Please check your credentials."
                    self.log(f"ERROR: {err_msg}")
                    print(f"ERROR: {err_msg}")
                    sys.exit(1)
                
                # Database does not exist
                elif 'database' in stderr_lower and 'does not exist' in stderr_lower:
                    err_msg = f"Database '{self.database}' does not exist. Please check the database name."
                    self.log(f"ERROR: {err_msg}")
                    print(f"ERROR: {err_msg}")
                    sys.exit(1)
                
                # Oracle specific: unknown SID
                elif 'ora-12154' in stderr_lower or 'ora-12154' in stdout_lower or 'tns:could not resolve' in stderr_lower:
                    err_msg = f"Could not resolve database SID '{self.database}'. Please check the database name/SID."
                    self.log(f"ERROR: {err_msg}")
                    print(f"ERROR: {err_msg}")
                    sys.exit(1)
                
                # Connection refused (wrong host or port, or service not running)
                elif 'could not connect' in stderr_lower or 'connection refused' in stderr_lower:
                    err_msg = f"Could not connect to database at {self.host}:{self.port}. Please check host, port, and that the database service is running."
                    self.log(f"ERROR: {err_msg}")
                    print(f"ERROR: {err_msg}")
                    sys.exit(1)
                
                # Oracle specific: listener errors (wrong host/port)
                elif 'ora-12541' in stderr_lower or 'ora-12541' in stdout_lower or 'tns:no listener' in stderr_lower:
                    err_msg = f"No listener found at {self.host}:{self.port}. Please check the host and port."
                    self.log(f"ERROR: {err_msg}")
                    print(f"ERROR: {err_msg}")
                    sys.exit(1)
                
                # Host not found
                elif 'could not translate host name' in stderr_lower or 'name or service not known' in stderr_lower or 'nodename nor servname provided' in stderr_lower:
                    err_msg = f"Could not resolve hostname '{self.host}'. Please check the host address."
                    self.log(f"ERROR: {err_msg}")
                    print(f"ERROR: {err_msg}")
                    sys.exit(1)
                
                # Timeout errors
                elif 'timeout' in stderr_lower or 'timed out' in stderr_lower:
                    err_msg = f"Connection timeout to {self.host}:{self.port}. Please check network connectivity and firewall settings."
                    self.log(f"ERROR: {err_msg}")
                    print(f"ERROR: {err_msg}")
                    sys.exit(1)
                
                # Generic connection error (fallback)
                self.log(f"Error executing query: {result.stderr}")
                return None
            
            return result.stdout.strip()
            
        except Exception as e:
            self.log(f"Error running query: {e}")
            return None
    
    def is_em_ge_v21(self):
        """
        Check if EM version is greater than or equal to 21.
        
        Returns:
            True if EM version >= 21, False otherwise
        """
        try:
            if self.db_type == 'postgresql':
                em_version = self.run_query(self.GET_EM_DB_VERSION_PG)
            else:
                em_version = self.run_query(self.GET_EM_DB_VERSION_ORCL)
            
            if not em_version:
                self.log("Failed to retrieve EM version")
                return False
            
            self.log(f"EM version: {em_version}")
            
            # Parse version string (format: X.Y.Z)
            version_parts = em_version.split('.')
            if len(version_parts) >= 3:
                major_version = int(version_parts[2])
                return major_version >= 21
            else:
                self.log(f"Invalid EM version format: {em_version}")
                return False
                
        except Exception as e:
            self.log(f"Error checking EM version: {e}")
            return False
    
    def is_db_version_supported(self):
        """
        Check if database version is supported.
        PostgreSQL: >= 11 for local PG, >= 14 for external PG
        Oracle: Always returns True
        
        Returns:
            True if DB version is supported, False otherwise
        """
        try:
            if self.db_type == 'postgresql':
                pg_version = self.run_query(self.GET_PG_VERSION)
                
                if not pg_version:
                    self.log("Failed to retrieve PostgreSQL version")
                    return False
                
                self.log(f"PostgreSQL version: {pg_version}")
                
                # Parse version (could be "15.3" or "15.3 (Debian...)")
                version_str = pg_version.split()[0]  # Get first part
                version_float = float(version_str)
                
                if self.is_local_pg():
                    self.log("Local PostgreSQL detected")
                    return version_float >= 11
                else:
                    self.log("External PostgreSQL detected")
                    return version_float >= 14
            else:
                # Oracle - always supported
                return True
                
        except Exception as e:
            self.log(f"Error checking DB version: {e}")
            return False
    
    def is_local_pg(self):
        """
        Check if PostgreSQL is running locally (embedded in EM).
        
        Returns:
            True if local PostgreSQL, False otherwise
        """
        try:
            em_home_path = self.em_home()
            if not em_home_path:
                return False
            
            if self.is_windows():
                postmaster_path = os.path.join(em_home_path, 'archive', 'pgsql', 'data', 'postmaster.pid')
            else:
                postmaster_path = os.path.join(em_home_path, 'archive', 'pgsql', 'data', 'postmaster.pid')
            
            return os.path.exists(postmaster_path)
            
        except Exception as e:
            self.log(f"Error checking if local PG: {e}")
            return False
    
    def count_job_info_tables(self):
        """
        Get the count of existing job_info tables from the partitions table.
        
        Returns:
            Number of job_info tables, or 0 if query fails
        """
        try:
            count_str = self.run_query(self.COUNT_JOB_INFO_TABLES_QUERY)
            
            if not count_str:
                self.log("Failed to get job_info table count")
                return 0
            
            count = int(count_str.strip())
            self.log(f"Found {count} job_info tables in partitions table")
            return count
            
        except Exception as e:
            self.log(f"Error counting job_info tables: {e}")
            return 0
    
    def get_existing_job_info_tables(self, output_dir=None):
        """
        Get list of existing non-empty job_info tables.
        Excludes tables that already have tar files in the output directory.
        
        Args:
            output_dir: Output directory to check for existing tar files (optional)
        
        Returns:
            List of table indices (1-based) that exist and have data
        """
        existing_tables = []
        
        # First, get the count from partitions table
        total_tables = self.count_job_info_tables()
        
        if total_tables == 0:
            self.log("No job_info tables found in partitions table")
            return existing_tables
        
        self.log(f"Checking which of {total_tables} job_info tables have data...")
        
        # Check each table to see if it exists and has rows
        for i in range(1, total_tables + 1):
            table_name = f"job_info_{i}"
            
            # Check if tar file already exists for this table
            if output_dir and self.tar_file_exists(output_dir, table_name):
                # tar_file_exists already prints the message to user
                continue
            
            row_count = self.get_table_row_count(table_name)
            
            if row_count is not None and row_count > 0:
                existing_tables.append(i)
                self.log(f"[{table_name}] Found table with {row_count:,} rows")
            elif row_count == 0:
                self.log(f"[{table_name}] Table exists but is empty - skipping")
            else:
                self.log(f"[{table_name}] Table does not exist or error accessing - skipping")
        
        self.log(f"Found {len(existing_tables)} non-empty job_info tables to export: {existing_tables}")
        return existing_tables
    
    def tar_file_exists(self, output_dir, table_name):
        """
        Check if a tar file exists for the given table or if table is marked as completed.
        Matches patterns like job_info_10.tar or job_info_10_part1.tar
        
        Args:
            output_dir: Directory to check for tar files
            table_name: Name of the table (e.g., "job_info_10")
            
        Returns:
            True if tar file exists or table is marked as completed, False otherwise
        """
        try:
            # First check if marked as completed in progress file
            if self.is_table_completed(table_name):
                progress_file_path = os.path.abspath(self.PROGRESS_FILE)
                skip_msg = f"[{table_name}] Skipping table that is marked as completed in {progress_file_path}"
                self.log(skip_msg)
                print(skip_msg)
                return True
            
            # Then check for physical tar files
            if not os.path.exists(output_dir):
                return False
            
            # Pattern to match: job_info_10.tar or job_info_10_part*.tar
            pattern = re.compile(rf'^{re.escape(table_name)}(_part\d+)?\.tar$')
            
            for filename in os.listdir(output_dir):
                if pattern.match(filename):
                    full_path = os.path.join(output_dir, filename)
                    skip_msg = f"[{table_name}] Skipping Tar file that already exists: {full_path}"
                    self.log(skip_msg)
                    print(skip_msg)
                    return True
            
            return False
            
        except Exception as e:
            self.log(f"[{table_name}] Error checking for tar file: {e}")
            return False
    
    def log(self, message):
        """Write message to log file in a thread-safe manner."""
        with self.print_lock:
            with open(self.log_file, 'a', encoding='utf-8', errors='replace') as f:
                now = datetime.now()
                timestamp = now.strftime('%Y-%m-%d %H:%M:%S') + f'.{now.microsecond // 1000:03d}'
                f.write(f"[{timestamp}] {message}\n")
                f.flush()  # Ensure immediate write
    
    def thread_safe_print(self, message):
        """Thread-safe print function - now redirects to log file."""
        self.log(message)
    
    def get_db_size(self):
        """
        Get the database size in bytes.
        
        Returns:
            Database size in bytes, or None if query fails
        """
        try:
            if self.db_type == 'postgresql':
                query = self.GET_DB_SIZE_PG
            elif self.db_type == 'oracle':
                query = self.GET_DB_SIZE_ORCL
            else:
                return None
            
            size_str = self.run_query(query)
            if not size_str:
                return None
            
            # Clean up the string - remove spaces, commas, etc.
            size_str = size_str.strip().replace(',', '').replace(' ', '')
            
            # Try to parse as integer or float
            try:
                # Handle scientific notation and decimals
                size_value = float(size_str)
                return int(size_value)
            except ValueError:
                self.log(f"Failed to parse database size: '{size_str}'")
                return None
            
        except Exception as e:
            self.log(f"Error getting database size: {e}")
            return None
    
    def run_psql_query(self, query):
        """
        Execute a SQL query using psql command (PostgreSQL).
        
        Args:
            query: SQL query to execute
            
        Returns:
            Query output as string
        """
        try:
            result = self.execute_psql(query=query, tuples_only=True)
            
            if result.returncode != 0:
                self.log(f"Error executing query: {result.stderr}")
                return None
            
            return result.stdout.strip()
            
        except Exception as e:
            self.log(f"Error running psql command: {e}")
            return None
    
    def run_sqlplus_query(self, query):
        """
        Execute a SQL query using sqlplus command (Oracle).
        
        Args:
            query: SQL query to execute
            
        Returns:
            Query output as string
        """
        # Format the sqlplus command with connection string
        connection_string = f"{self.user}/{self.password}@{self.host}:{self.port}/{self.database}"
        command = f'echo "set pagesize 0\n set feedback off\n set heading off\n {query};" | arc sqlplus -s {connection_string}'
        
        try:
            result = subprocess.run(
                command,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                text=True
            )
            
            if result.returncode != 0:
                self.log(f"Error executing query: {result.stderr}")
                return None
            
            return result.stdout.strip()
            
        except Exception as e:
            self.log(f"Error running sqlplus command: {e}")
            return None
    
    def get_table_row_count(self, table_name):
        """
        Get the total number of rows in the table that will be exported.
        Only counts rows where rule_name exists in the rules table.
        
        Args:
            table_name: Name of the table
            
        Returns:
            Total row count, or None if query fails
        """
        try:
            query = f"SELECT COUNT(*) FROM {table_name} t WHERE EXISTS (SELECT 1 FROM rules r WHERE r.name = t.rule_name)"
            
            count_str = self.run_query(query)
            return int(count_str) if count_str and count_str.isdigit() else None
            
        except Exception as e:
            self.thread_safe_print(f"[{table_name}] Error getting row count: {e}")
            return None
    
    def validate_record(self, record, output_file):
        """
        Validate that a record has the correct number of delimiters.
        
        Args:
            record: The record string to validate
            output_file: The output file name (for error messages)
            
        Raises:
            SystemExit: If validation fails and skip_validations is False
        """
        if self.skip_validations:
            return
        
        delimiter_start_count = record.count(self.DELIMITER_START)
        delimiter_end_count = record.count(self.DELIMITER_END)
        delimiter_eol_count = record.count(self.DELIMITER_EOL)
        
        # Verify each record has the correct number of start delimiters
        if delimiter_start_count != 45:
            self.thread_safe_print(f"[{output_file}] delimiter_start_count != 45: {record}")
            print(f"[{output_file}] delimiter_start_count != 45: {record}")
            sys.exit(1)
        
        # Verify each record has the correct number of end delimiters
        if delimiter_end_count != 45:
            self.thread_safe_print(f"[{output_file}] delimiter_end_count != 45: {record}")
            print(f"[{output_file}] delimiter_end_count != 45: {record}")
            sys.exit(1)
        
        # Verify each record has exactly one EOL delimiter
        if delimiter_eol_count != 1:
            self.thread_safe_print(f"[{output_file}] delimiter_eol_count != 1: {record}")
            print(f"[{output_file}] delimiter_eol_count != 1: {record}")
            sys.exit(1)
        
        # Verify each record ends with EOL delimiter
        if not record.endswith(self.DELIMITER_EOL):
            self.thread_safe_print(f"[{output_file}] Record missing endline delimiter: {record}")
            print(f"[{output_file}] Record missing endline delimiter: {record}")
            sys.exit(1)
    
    def execute_export_query(self, table_name, skip_rows, file_size_limit, max_lines_per_file):
        """
        Execute export query using the appropriate database command (psql or sqlplus).
        
        Args:
            table_name: Name of the table to export
            skip_rows: Number of rows to skip
            file_size_limit: Maximum number of characters per batch
            max_lines_per_file: Maximum number of lines (records) per batch
            
        Returns:
            subprocess.CompletedProcess result
        """
        if self.db_type == 'postgresql':
            query = f"SELECT * FROM export_job_info('{table_name}', {skip_rows}, {file_size_limit}, {max_lines_per_file})"
            return self.execute_psql(query=query, tuples_only=True)
        elif self.db_type == 'oracle':
            # Create a temporary SQL file to call the procedure
            import tempfile
            temp_sql_fd, temp_sql_file = tempfile.mkstemp(suffix='.sql', text=True)
            
            try:
                # Write the PL/SQL block to call the procedure
                sql_block = f"""set pagesize 0
set feedback off
set heading off
set linesize 32767
set trimspool on
set tab off
set serveroutput on size unlimited format truncated

BEGIN
    export_job_info('{table_name}', {skip_rows}, {file_size_limit}, {max_lines_per_file});
END;
/
exit;
"""
                
                with os.fdopen(temp_sql_fd, 'w') as f:
                    f.write(sql_block)
                
                # Execute the SQL file
                connection_string = f"{self.user}/{self.password}@{self.host}:{self.port}/{self.database}"
                command = f'arc sqlplus -s {connection_string} @"{temp_sql_file}"'
                
                # Set up environment to force UTF-8 output (prevents encoding issues on Windows)
                env = os.environ.copy()
                env['NLS_LANG'] = '.AL32UTF8'  # Character set only, no language/territory

                # Use Popen to track the process
                proc = subprocess.Popen(
                    command,
                    shell=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    text=True,
                    encoding='utf-8',
                    errors='replace',  # Replace invalid UTF-8 bytes with replacement character
                    env=env
                )
                
                # Track the process
                with self.process_lock:
                    self.active_processes.append(proc)
                
                try:
                    # Use a timeout for the export query (5 minutes per batch)
                    stdout, stderr = proc.communicate(timeout=300)
                    result = subprocess.CompletedProcess(
                        args=command,
                        returncode=proc.returncode,
                        stdout=stdout,
                        stderr=stderr
                    )
                    return result
                except subprocess.TimeoutExpired:
                    # Kill the process if it times out
                    proc.kill()
                    stdout, stderr = proc.communicate()
                    error_msg = f"Export query timed out after 300 seconds for table {table_name}"
                    self.log(f"ERROR: {error_msg}")
                    # Return a failed result
                    return subprocess.CompletedProcess(
                        args=command,
                        returncode=1,
                        stdout=stdout,
                        stderr=error_msg
                    )
                finally:
                    # Remove from active processes
                    with self.process_lock:
                        if proc in self.active_processes:
                            self.active_processes.remove(proc)
            finally:
                # Clean up the temporary file
                if os.path.exists(temp_sql_file):
                    os.remove(temp_sql_file)
        else:
            raise ValueError(f"Unsupported database type: {self.db_type}")
    
    def export_all_rows(self, table_dir, table_name, file_size_limit=100000000, starting_skip_rows=0):
        """
        Export all rows using the export_job_info function (PostgreSQL or Oracle).
        Appends to existing output file until it reaches 200MB, then compresses it.
        
        Args:
            table_dir: Directory to store output files
            table_name: Name of the table to export
            file_size_limit: Maximum number of characters per batch (default: 100MB)
            starting_skip_rows: Starting skip_rows value (for resume functionality)
        """
        skip_rows = starting_skip_rows
        total_written = 0
        record_index = skip_rows
        
        # Get total row count for progress tracking
        total_rows = self.get_table_row_count(table_name)
        if total_rows is not None:
            self.thread_safe_print(f"[{table_name}] Total rows in table: {total_rows:,}")
            # Initialize progress tracking
            with self.progress_lock:
                self.table_progress[table_name] = {'current': starting_skip_rows, 'total': total_rows}
        
        max_file_size_mb = self.combined_file_size_limit  # Maximum file size before compression (configurable via --combined-file-size-limit)
        
        # Track the current output file
        current_output_file = None
        current_file_from = None
        current_file_to = None
        current_file_lines = 0  # Track number of lines in current file
        
        # On resume, if there are any existing .txt files, compress them first
        # (they should have been compressed, but may have been left from interrupted run)
        if starting_skip_rows > 0 and os.path.exists(table_dir):
            txt_files = [f for f in os.listdir(table_dir) if f.endswith('.txt')]
            for txt_file in txt_files:
                txt_path = os.path.join(table_dir, txt_file)
                self.thread_safe_print(f"[{table_name}] Found uncompressed file from previous run: {txt_file}, compressing...")
                gz_path = txt_path.replace('.txt', '.gz')
                with open(txt_path, 'rb') as f_in:
                    with gzip.open(gz_path, 'wb') as f_out:
                        shutil.copyfileobj(f_in, f_out)
                os.remove(txt_path)
                self.thread_safe_print(f"[{table_name}] Compressed to {os.path.basename(gz_path)}")
        
        try:
            while True:
                # Check if abort has been signaled
                if self.abort_event.is_set():
                    self.thread_safe_print(f"[{table_name}] Export aborted")
                    break
                
                # Output file will be named with record range: job_info_<table_index>_<from>_<to>.txt
                output_from = record_index
                
                # Execute the export query using the appropriate database command
                result = self.execute_export_query(table_name, skip_rows, file_size_limit, self.max_lines_per_file)
                
                if result.returncode != 0:
                    self.log(f"Error executing query: {result.stderr}")
                    db_cmd = 'psql' if self.db_type == 'postgresql' else 'sqlplus'
                    raise Exception(f"{db_cmd} command failed: {result.stderr}")
                
                # Check for PostgreSQL errors in stderr even if returncode is 0
                if self.db_type == 'postgresql' and result.stderr and ('ERROR:' in result.stderr or 'EXCEPTION:' in result.stderr):
                    self.log(f"PostgreSQL error detected: {result.stderr}")
                    raise Exception(f"PostgreSQL error: {result.stderr}")
                
                # Check for Oracle errors in stderr (connection issues, file errors, etc.)
                # Note: We don't check stdout because procedures can legitimately output ORA- or ERROR in DBMS_OUTPUT
                if self.db_type == 'oracle' and result.stderr and result.stderr.strip():
                    self.log(f"Oracle error detected in stderr: {result.stderr}")
                    raise Exception(f"Oracle error: {result.stderr}")
                
                output = result.stdout.strip()

                # If no output, we've reached the end
                if not output:
                    break
                
                # Count how many times DELIMITER_EOL appears (each occurrence = one record)
                batch_count = output.count(self.DELIMITER_EOL)
                    
                # Split by DELIMITER_EOL to get individual records
                records = [rec.strip() + self.DELIMITER_EOL for rec in output.split(self.DELIMITER_EOL) if rec.strip()]
                
                # If no actual rows were found, we've reached the end
                if batch_count == 0:
                    break
                if batch_count != len(records):
                    self.thread_safe_print(f"[{batch_count}] len(batch_count) != len(records) {len(records)}")        

                output_to = output_from + batch_count - 1
                
                # Write records to file (create new or append to existing)
                if current_output_file is None:
                    # No current file - create new one
                    current_file_from = output_from
                    current_file_to = output_to
                    current_output_file = os.path.join(table_dir, f"{table_name}_{current_file_from}_{current_file_to}.txt")
                    
                    # Sanity check: ensure file doesn't already exist
                    if os.path.exists(current_output_file):
                        error_msg = f"File {current_output_file} already exists! Possible duplicate export process."
                        self.thread_safe_print(f"[{table_name}] ERROR: {error_msg}")
                        raise Exception(error_msg)
                    
                    # Write records to new file
                    with open(current_output_file, 'w', encoding='utf-8', errors='replace') as f:
                        for record in records:
                            self.validate_record(record, current_output_file)
                            f.write(record + '\n')
                            total_written += 1
                    current_file_lines = batch_count
                else:
                    # Append to existing file
                    with open(current_output_file, 'a', encoding='utf-8', errors='replace') as f:
                        for record in records:
                            self.validate_record(record, current_output_file)
                            f.write(record + '\n')
                            total_written += 1
                    current_file_lines += batch_count
                    
                    # Update the range and rename file
                    current_file_to = output_to
                    new_filename = os.path.join(table_dir, f"{table_name}_{current_file_from}_{current_file_to}.txt")
                    if new_filename != current_output_file:
                        os.rename(current_output_file, new_filename)
                        current_output_file = new_filename
                
                # Check file size and line count (for both new and existing files)
                file_size = os.path.getsize(current_output_file)
                file_size_mb = file_size / (1024 * 1024)
                
                # Compress if file size exceeds limit OR line count exceeds limit
                should_compress = file_size_mb >= max_file_size_mb or current_file_lines >= self.max_lines_per_file
                
                if should_compress:
                    # Compress the file
                    if file_size_mb >= max_file_size_mb:
                        self.thread_safe_print(f"[{table_name}] File {os.path.basename(current_output_file)} reached {file_size_mb:.2f} MB, compressing...")
                    else:
                        self.thread_safe_print(f"[{table_name}] File {os.path.basename(current_output_file)} reached {current_file_lines} lines, compressing...")
                    output_gz = current_output_file.replace('.txt', '.gz')
                    with open(current_output_file, 'rb') as f_in:
                        with gzip.open(output_gz, 'wb') as f_out:
                            shutil.copyfileobj(f_in, f_out)
                    os.remove(current_output_file)
                    self.thread_safe_print(f"[{table_name}] Compressed to {os.path.basename(output_gz)}")
                    
                    # Reset current file
                    current_output_file = None
                    current_file_from = None
                    current_file_to = None
                    current_file_lines = 0
                
                # Update progress tracking
                with self.progress_lock:
                    if table_name in self.table_progress:
                        self.table_progress[table_name]['current'] = output_to + 1
                
                # Calculate and display progress
                if total_rows is not None and total_rows > 0:
                    progress_pct = (output_to + 1) / total_rows * 100
                    self.thread_safe_print(f"[{table_name}] Written batch: {batch_count} rows (Total: {total_written}, Progress: {output_to + 1}/{total_rows} = {progress_pct:.2f}%)")
                else:
                    self.thread_safe_print(f"[{table_name}] Written batch: {batch_count} rows (Total: {total_written})")
                
                # Move to next batch
                skip_rows += batch_count
                record_index += batch_count
            
            # Compress any remaining file
            if current_output_file and os.path.exists(current_output_file):
                self.thread_safe_print(f"[{table_name}] Compressing final file {os.path.basename(current_output_file)}...")
                output_gz = current_output_file.replace('.txt', '.gz')
                with open(current_output_file, 'rb') as f_in:
                    with gzip.open(output_gz, 'wb') as f_out:
                        shutil.copyfileobj(f_in, f_out)
                os.remove(current_output_file)
                self.thread_safe_print(f"[{table_name}] Compressed to {os.path.basename(output_gz)}")
            
            # Check for error: table has rows but we exported nothing
            if total_rows is not None and total_rows > 0 and total_written == 0:
                error_msg = f"Table has {total_rows} rows but exported 0 rows. This likely indicates a delimiter mismatch (expected '{self.DELIMITER_EOL}')."
                self.log(f"[{table_name}] ERROR: {error_msg}")
                raise Exception(error_msg)
            
            # Get final accurate row count from database to verify export
            final_count = self.get_table_row_count(table_name)
            if final_count is not None:
                self.thread_safe_print(f"[{table_name}] Export completed. Total rows exported in this activation: {total_written}, Current table rows: {final_count:,}")
                
                # Validate: starting_skip_rows + total_written should match final_count
                expected_total = starting_skip_rows + total_written
                if expected_total != final_count:
                    error_msg = f"Export validation failed: starting_skip_rows ({starting_skip_rows}) + total_written ({total_written}) = {expected_total}, but final_count = {final_count}. Data may have changed during export or export is incomplete."
                    self.log(f"[{table_name}] ERROR: {error_msg}")
                    if not self.skip_validations:
                        raise Exception(error_msg)
                else:
                    self.thread_safe_print(f"[{table_name}] Export validation passed: {starting_skip_rows} + {total_written} = {final_count}")
                
                # Mark as complete in progress tracking only after all validations pass
                with self.progress_lock:
                    if table_name in self.table_progress:
                        self.table_progress[table_name]['total'] = final_count
                        self.table_progress[table_name]['current'] = final_count
            else:
                self.thread_safe_print(f"[{table_name}] Export completed. Total rows: {total_written}")
                # Mark as complete even without validation
                with self.progress_lock:
                    if table_name in self.table_progress and self.table_progress[table_name]['total'] > 0:
                        self.table_progress[table_name]['current'] = self.table_progress[table_name]['total']
            
        except Exception as e:
            self.thread_safe_print(f"[{table_name}] Error exporting to file: {e}")
            raise  # Re-raise the exception so it's caught by export_table_wrapper
    
    def get_distinct_rule_names(self, table_name):
        """
        Get distinct rule_name values from the job_info table.
        
        Args:
            table_name: Name of the table to query
            
        Returns:
            List of distinct rule names, or empty list if query fails
        """
        try:
            query = f"SELECT DISTINCT rule_name FROM {table_name}"
            
            output = self.run_query(query)
            
            if not output:
                return []
            
            # Parse output and clean up lines
            lines = output.splitlines()
            cleaned_lines = [line.strip() for line in lines if line.strip()]
            
            return cleaned_lines
            
        except Exception as e:
            self.thread_safe_print(f"[{table_name}] Error getting distinct rule names: {e}")
            return []
    
    def create_rules_json_file(self, table_dir, table_name):
        """
        Create a rules.json file containing distinct rule_name values from the table.
        
        Args:
            table_dir: Directory where the rules.json file will be created
            table_name: Name of the table to query
            
        Returns:
            Path to the created rules.json file, or None if creation failed
        """
        try:
            # Get distinct rule names
            rule_names = self.get_distinct_rule_names(table_name)
            
            if not rule_names:
                self.thread_safe_print(f"[{table_name}] No rule names found or query failed")
                return None
            
            # Create rules.json file
            rules_file = os.path.join(table_dir, "rules.json")
            
            with open(rules_file, 'w', encoding='utf-8', errors='replace') as f:
                json.dump(rule_names, f, indent=2)
            
            self.thread_safe_print(f"[{table_name}] Created rules.json with {len(rule_names)} distinct rule names")
            
            return rules_file
            
        except Exception as e:
            self.thread_safe_print(f"[{table_name}] Error creating rules.json file: {e}")
            return None
    
    def create_tar_and_cleanup(self, table_dir, output_dir, table_name, max_tar_size=5*1024*1024*1024*0.98):
        """
        Create tar files from all .gz files and rules.json in the table directory and clean up.
        Splits into multiple tar files if total size exceeds max_tar_size.
        
        Args:
            table_dir: Directory containing the .gz files
            output_dir: Parent directory where tar files will be created
            table_name: Name of the table (used for tar filename)
            max_tar_size: Maximum size per tar file in bytes (default: 5GB)
        """
        try:
            # Get all .gz files sorted by name (which includes skip_rows number)
            gz_files = sorted([f for f in os.listdir(table_dir) if f.endswith('.gz')])
            
            # Check for rules.json file
            rules_file = os.path.join(table_dir, "rules.json")
            has_rules_file = os.path.exists(rules_file)
            
            if not gz_files and not has_rules_file:
                self.thread_safe_print(f"[{table_name}] No files found to archive")
                return
            
            # Split files into groups based on size limit
            tar_groups = []
            current_group = []
            current_size = 0
            
            for gz_file in gz_files:
                file_path = os.path.join(table_dir, gz_file)
                file_size = os.path.getsize(file_path)
                
                # If adding this file would exceed the limit, start a new group
                if current_size + file_size > max_tar_size and current_group:
                    tar_groups.append(current_group)
                    current_group = []
                    current_size = 0
                
                current_group.append((gz_file, file_path, file_size))
                current_size += file_size
            
            # Add the last group if it has files
            if current_group:
                tar_groups.append(current_group)
            
            # Create tar files for each group
            for idx, group in enumerate(tar_groups):
                if len(tar_groups) == 1:
                    # Single tar file - no suffix needed
                    tar_file = os.path.join(output_dir, f"{table_name}.tar")
                else:
                    # Multiple tar files - add part number
                    tar_file = os.path.join(output_dir, f"{table_name}_part{idx + 1}.tar")
                
                total_size = sum(f[2] for f in group)
                self.thread_safe_print(f"[{table_name}] Creating tar archive: {tar_file} ({len(group)} files, {total_size / (1024*1024*1024):.2f} GB)")
                
                # Create tar file with files from this group
                with tarfile.open(tar_file, 'w') as tar:
                    # Add rules.json to all tar files
                    if has_rules_file:
                        tar.add(rules_file, arcname="rules.json")
                        rules_size = os.path.getsize(rules_file)
                        self.thread_safe_print(f"[{table_name}] Added rules.json ({rules_size / 1024:.2f} KB) to {os.path.basename(tar_file)}")
                    
                    for gz_file, file_path, file_size in group:
                        # Add file to tar with just the filename (no path)
                        tar.add(file_path, arcname=gz_file)
                        self.thread_safe_print(f"[{table_name}] Added {gz_file} ({file_size / (1024*1024):.2f} MB) to {os.path.basename(tar_file)}")
            
            # Remove the table directory and all its contents
            shutil.rmtree(table_dir)
            
            if len(tar_groups) == 1:
                self.thread_safe_print(f"[{table_name}] Created tar archive: {tar_file}")
            else:
                self.thread_safe_print(f"[{table_name}] Created {len(tar_groups)} tar archives (split due to 5GB size limit)")
            
            self.thread_safe_print(f"[{table_name}] Cleaned up directory: {table_dir}")
            
        except Exception as e:
            self.thread_safe_print(f"[{table_name}] Error creating tar file: {e}")
    
    def get_last_skip_rows(self, table_dir, table_name):
        """
        Get the highest _to value from existing .gz and .txt files in the table directory.
        This allows resuming an interrupted export.
        
        Args:
            table_dir: Directory containing the files
            table_name: Name of the table
        Returns:
            The highest _to value found, or 0 if no files exist
        """
        import re
        
        if not os.path.exists(table_dir):
            return 0
        
        max_to = 0
        # Match both .gz and .txt files with range: job_info_<table_index>_<from>_<to>.gz or .txt
        pattern = re.compile(rf'{re.escape(table_name)}_(\d+)_(\d+)\.(?:gz|txt)')
        try:
            for item in os.listdir(table_dir):
                match = pattern.match(item)
                if match:
                    to_value = int(match.group(2))
                    if to_value > max_to:
                        max_to = to_value
            if max_to > 0:
                self.thread_safe_print(f"[{table_name}] Found existing files. Resuming from record {max_to + 1}")
            return max_to + 1 if max_to > 0 else 0
        except Exception as e:
            self.thread_safe_print(f"[{table_name}] Error reading existing files: {e}")
            return 0
    
    def export_table_wrapper(self, table_index, output_dir):
        """
        Wrapper function for exporting a single table (used by thread pool).
        
        Args:
            table_index: Index of the job_info table (1-based)
            output_dir: Directory to store output files
            
        Returns:
            Tuple of (table_name, success, message)
        """
        table_name = f"job_info_{table_index}"
        
        # Check if abort has been signaled
        if self.abort_event.is_set():
            return (table_name, False, f"Aborted: {self.abort_reason}")
        
        # Create a folder for this table within the output directory
        table_dir = os.path.join(output_dir, table_name)
        if not os.path.exists(table_dir):
            os.makedirs(table_dir)
        
        # Check for multiple .txt files - indicates parallel export conflict
        txt_files = [f for f in os.listdir(table_dir) if f.endswith('.txt') and f.startswith(table_name)]
        if len(txt_files) >= 2:
            error_msg = f"Found {len(txt_files)} .txt files in {table_dir}. This indicates duplicate/parallel exports. Files: {txt_files}"
            self.thread_safe_print(f"[{table_name}] ERROR: {error_msg}")
            return (table_name, False, error_msg)
        
        self.thread_safe_print(f"[{table_name}] Starting export...")
        
        try:
            # Check if there are existing files and get the last skip_rows value
            starting_skip_rows = self.get_last_skip_rows(table_dir, table_name)
            
            # Export all rows starting from the last checkpoint
            # Periodically check for abort signal
            if self.abort_event.is_set():
                return (table_name, False, f"Aborted: {self.abort_reason}")
            
            self.export_all_rows(table_dir, table_name, self.file_size_limit, starting_skip_rows)
            
            # Check abort after export - don't create tar files if aborted
            if self.abort_event.is_set():
                self.thread_safe_print(f"[{table_name}] Export aborted - skipping tar file creation")
                return (table_name, False, f"Aborted: {self.abort_reason}")
            
            # Create rules.json file with distinct rule names
            self.create_rules_json_file(table_dir, table_name)
            
            # Check abort before creating tar
            if self.abort_event.is_set():
                self.thread_safe_print(f"[{table_name}] Export aborted - skipping tar file creation")
                return (table_name, False, f"Aborted: {self.abort_reason}")
            
            # Create tar file from all .gz files and rules.json, then cleanup the directory
            self.create_tar_and_cleanup(table_dir, output_dir, table_name)
            
            # Mark table as completed in progress file
            self.mark_table_completed(table_name)
            
            # Sleep for 15 seconds after tar creation to allow progress monitor to catch up
            self.thread_safe_print(f"[{table_name}] Waiting 15 seconds for progress update...")
            time.sleep(15)
            
            return (table_name, True, "Export completed successfully")
        except KeyboardInterrupt:
            # Ctrl+C was pressed - set abort event and don't create tar files
            self.abort_reason = "Interrupted by user (Ctrl+C)"
            self.abort_event.set()
            self.thread_safe_print(f"[{table_name}] Export interrupted by user - skipping tar file creation")
            # Don't mark as complete - leave progress at actual failure point
            return (table_name, False, "Interrupted by user")
        except Exception as e:
            # Don't mark table as complete in progress tracking on failure
            # The progress should remain at whatever point it failed
            return (table_name, False, f"Export failed: {str(e)}")
    
    def disk_usage_monitor(self, output_dir, max_disk_usage_percent, check_interval=15, stop_event=None):
        """
        Monitor disk usage in a separate thread and signal abort if limit is exceeded.
        Also prints progress every check_interval seconds.
        
        Args:
            output_dir: Directory to monitor
            max_disk_usage_percent: Maximum allowed disk usage percentage
            check_interval: Interval in seconds between checks (default: 15)
            stop_event: Event to signal when monitoring should stop
        """
        self.thread_safe_print(f"Disk usage monitor started (checking every {check_interval} seconds)")
        
        while not (stop_event and stop_event.is_set()):
            try:
                used_percent = self.count_used_space(output_dir)
                if used_percent is not None:
                    # Log disk usage percentage every 15 seconds
                    self.thread_safe_print(f"Disk usage: {used_percent}%")
                    
                    if used_percent >= max_disk_usage_percent:
                        error_msg = f"Your export file sizes exceed the maximum amount of allowed disk space ({max_disk_usage_percent}%)."
                        self.abort_reason = error_msg
                        self.abort_event.set()
                        self.thread_safe_print(f"CRITICAL: {error_msg}. Aborting all export operations!")
                        return
                
                # Build progress JSON for all active tables
                progress = {}
                with self.progress_lock:
                    for table_name, stats in self.table_progress.items():
                        total = stats.get('total', 0)
                        current = stats.get('current', 0)
                        if total > 0:
                            percentage = int((current / total) * 100)
                            progress[table_name] = f"{percentage}%"
                        else:
                            progress[table_name] = "0%"
                
                # Sort progress by table name for consistent output
                sorted_progress = dict(sorted(progress.items()))
                
                # Print progress to console (like archive_export.py)
                if sorted_progress:
                    print(f"\rProgress: {sorted_progress}", end="", flush=True)
                    # Also log progress to the log file
                    self.thread_safe_print(f"Progress: {sorted_progress}")
            
            except Exception as e:
                self.thread_safe_print(f"Error in disk usage monitor: {e}")
            
            # Sleep for the check interval, but check stop_event more frequently
            for _ in range(check_interval):
                if stop_event and stop_event.is_set():
                    break
                time.sleep(1)
        
        self.thread_safe_print("Disk usage monitor stopped")
    
    def export_all_tables_parallel(self, max_workers=10, output_dir="export_output", max_disk_usage_percent=90):
        """
        Export multiple job_info tables in parallel using a thread pool.
        Only exports existing non-empty tables.
        
        Args:
            max_workers: Maximum number of concurrent threads (default: 10)
            output_dir: Directory to store output files (default: 'export_output')
            max_disk_usage_percent: Maximum allowed disk usage percentage before aborting (default: 90)
            
        Returns:
            Dictionary with summary of results
        """
        # Create output directory if it doesn't exist
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)
            self.thread_safe_print(f"Created output directory: {output_dir}")
        else:
            self.thread_safe_print(f"Using existing output directory: {output_dir}")
        
        # Get list of existing non-empty job_info tables (excluding those with existing tar files)
        existing_tables = self.get_existing_job_info_tables(output_dir)
        
        if not existing_tables:
            error_msg = "No existing non-empty job_info tables found. Nothing to export."
            self.thread_safe_print(error_msg)
            return {
                'total': 0,
                'successful': 0,
                'failed': 0,
                'successful_tables': [],
                'failed_tables': [],
                'aborted': False,
                'abort_reason': None
            }
        
        num_tables = len(existing_tables)
        
        # Reset abort event
        self.abort_event.clear()
        self.abort_reason = None
        
        # Start disk usage monitoring thread (only on non-Windows systems)
        monitor_stop_event = Event()
        monitor_thread = None
        
        # Check available space on output folder (works on both Windows and Unix/Linux)
        used_space = self.count_used_space(output_dir)
        if used_space is not None:
            self.thread_safe_print(f"Disk space used on output folder: {used_space}%")
            
            # Check if disk usage exceeds the limit immediately
            if used_space >= max_disk_usage_percent:
                error_msg = f"ABORT: Disk usage ({used_space}%) exceeds maximum allowed ({max_disk_usage_percent}%). Cannot proceed with export."
                self.thread_safe_print(error_msg)
                self.thread_safe_print("="*80)
                return {
                    'total': num_tables,
                    'successful': 0,
                    'failed': num_tables,
                    'successful_tables': [],
                    'failed_tables': [f"job_info_{i}" for i in existing_tables],
                    'aborted': True,
                    'abort_reason': error_msg
                }
        
        available_space = self.get_available_space(output_dir)
        if available_space:
            self.thread_safe_print(f"Available disk space on output folder: {available_space:.2f} GB")
        
        # Start monitoring thread (only on non-Windows systems due to df command)
        if not self.is_windows():
            monitor_thread = threading.Thread(
                target=self.disk_usage_monitor,
                args=(output_dir, max_disk_usage_percent, 15, monitor_stop_event),
                daemon=True
            )
            monitor_thread.start()
        else:
            # On Windows, we can still monitor but using shutil.disk_usage
            monitor_thread = threading.Thread(
                target=self.disk_usage_monitor,
                args=(output_dir, max_disk_usage_percent, 15, monitor_stop_event),
                daemon=True
            )
            monitor_thread.start()
        
        self.thread_safe_print(f"\n{'='*80}")
        self.thread_safe_print(f"Starting parallel export of {num_tables} non-empty job_info tables with {max_workers} workers")
        self.thread_safe_print(f"Tables to export: {existing_tables}")
        self.thread_safe_print(f"Output directory: {output_dir}")
        self.thread_safe_print(f"File size limit per batch: {self.file_size_limit:,} characters")
        self.thread_safe_print(f"Max lines per file: {self.max_lines_per_file:,}")
        self.thread_safe_print(f"Maximum disk usage allowed: {max_disk_usage_percent}%")
        self.thread_safe_print(f"{'='*80}\n")
        
        successful_exports = []
        failed_exports = []
        
        executor = None
        try:
            # Use ThreadPoolExecutor to manage the thread pool
            executor = ThreadPoolExecutor(max_workers=max_workers)
            # Submit all tasks to the thread pool - only for existing tables
            future_to_table = {
                executor.submit(self.export_table_wrapper, i, output_dir): i 
                for i in existing_tables
            }
            
            # Process completed tasks as they finish
            # Use a set to track which futures are still pending
            pending = set(future_to_table.keys())
            
            while pending:
                # Use as_completed with timeout to allow interrupt checking
                done = set()
                try:
                    # Check for completed futures with a short timeout
                    for future in as_completed(pending, timeout=0.5):
                        done.add(future)
                        
                        table_index = future_to_table[future]
                        try:
                            table_name, success, message = future.result()
                            if success:
                                successful_exports.append(table_name)
                                self.thread_safe_print(f"[{table_name}] ✓ {message}")
                            else:
                                failed_exports.append(table_name)
                                self.thread_safe_print(f"[{table_name}] ✗ {message}")
                        except Exception as e:
                            table_name = f"job_info_{table_index}"
                            failed_exports.append(table_name)
                            self.thread_safe_print(f"[{table_name}] ✗ Exception: {str(e)}")
                except TimeoutError:
                    # Timeout is normal - just means no futures completed in 0.5 seconds
                    # This allows us to check for abort/interrupt
                    pass
                except Exception as ex:
                    # Catch any other unexpected exceptions from as_completed()
                    self.thread_safe_print(f"Warning: Unexpected exception in as_completed: {ex}")
                    pass
                
                # Remove completed futures from pending
                pending -= done
                
                # Check for abort AFTER processing any completed futures
                if self.abort_event.is_set():
                    break
            
            # After loop exits, ensure ALL futures have been processed
            # This includes any that completed right at the end
            for f in list(pending):
                if f.done():
                    table_index = future_to_table[f]
                    try:
                        table_name, success, message = f.result()
                        if success:
                            successful_exports.append(table_name)
                            self.thread_safe_print(f"[{table_name}] ✓ {message}")
                        else:
                            failed_exports.append(table_name)
                            self.thread_safe_print(f"[{table_name}] ✗ {message}")
                    except Exception as e:
                        table_name = f"job_info_{table_index}"
                        failed_exports.append(table_name)
                        self.thread_safe_print(f"[{table_name}] ✗ Exception: {str(e)}")
                    pending.remove(f)
            
            # Check if we still have pending futures due to abort
            if self.abort_event.is_set() and pending:
                self.thread_safe_print("Abort detected - waiting for running tasks to finish...")
                # Don't cancel - let running tasks finish gracefully
                # They will detect abort_event and exit cleanly
                # Wait for all remaining tasks to complete
                for f in pending:
                    table_index = future_to_table[f]
                    table_name = f"job_info_{table_index}"
                    self.thread_safe_print(f"[{table_name}] Waiting for task to complete...")
                    try:
                        # Wait for this future to complete (no timeout - must finish)
                        result_table_name, success, message = f.result()
                        if success:
                            successful_exports.append(result_table_name)
                            self.thread_safe_print(f"[{result_table_name}] ✓ {message}")
                        else:
                            failed_exports.append(result_table_name)
                            self.thread_safe_print(f"[{result_table_name}] ✗ {message}")
                    except Exception as e:
                        failed_exports.append(table_name)
                        self.thread_safe_print(f"[{table_name}] ✗ Exception: {str(e)}")
        
        except KeyboardInterrupt:
            # User pressed Ctrl+C - signal abort and cancel all tasks
            self.abort_reason = "Interrupted by user (Ctrl+C)"
            self.abort_event.set()
            self.thread_safe_print("\n" + "="*80)
            self.thread_safe_print("INTERRUPT RECEIVED - Stopping all exports...")
            self.thread_safe_print("="*80)
            # Cancel all futures
            for f in future_to_table.keys():
                f.cancel()
            # Give threads a moment to see the abort event
            time.sleep(1)
            raise  # Re-raise to be caught by main()
        
        except MemoryError as e:
            error_msg = "CRITICAL: Out of memory error during export!"
            self.thread_safe_print(error_msg)
            self.thread_safe_print("="*80)
            self.thread_safe_print("RESOURCE EXHAUSTION DETECTED")
            self.thread_safe_print("="*80)
            self.thread_safe_print("The system ran out of memory during the export process.")
            self.thread_safe_print("Please try the following to reduce memory usage:")
            self.thread_safe_print(f"   1. Move away completed tar files from the output folder {output-dir}")
            self.thread_safe_print(f"   2. Rerun with less parallelism --max-workers=2 (current: {max_workers})")
            self.thread_safe_print(f"   3. Rerun with higher abort percentages threshold --max-disk-usage=95 (current: {max_disk_usage_percent}%")
            self.thread_safe_print("="*80)
            self.abort_reason = error_msg
            self.abort_event.set()
        
        finally:
            # Shutdown executor if it was created
            if executor:
                try:
                    executor.shutdown(wait=True)
                except Exception as shutdown_error:
                    # Ignore shutdown errors - they're not critical
                    self.thread_safe_print(f"Note: Executor shutdown encountered an issue: {shutdown_error}")
        
        # Stop the disk usage monitoring thread
        if monitor_thread:
            monitor_stop_event.set()
            monitor_thread.join(timeout=5)
        
        # Check if export was aborted
        was_aborted = self.abort_event.is_set()
        
        # Print summary
        self.thread_safe_print(f"\n{'='*80}")
        self.thread_safe_print(f"EXPORT SUMMARY")
        self.thread_safe_print(f"{'='*80}")
        
        if was_aborted:
            self.thread_safe_print(f"STATUS: ABORTED - {self.abort_reason}")
        
        self.thread_safe_print(f"Total tables: {num_tables}")
        self.thread_safe_print(f"Successful: {len(successful_exports)}")
        self.thread_safe_print(f"Failed: {len(failed_exports)}")
        
        if failed_exports:
            self.thread_safe_print(f"\nFailed tables: {', '.join(failed_exports)}")
        
        self.thread_safe_print(f"{'='*80}\n")
        
        return {
            'total': num_tables,
            'successful': len(successful_exports),
            'failed': len(failed_exports),
            'successful_tables': successful_exports,
            'failed_tables': failed_exports,
            'aborted': was_aborted,
            'abort_reason': self.abort_reason if was_aborted else None
        }


def signal_handler(signum, frame):
    """Handle termination signals and provide helpful error message."""
    print("\n" + "="*80)
    print("PROCESS TERMINATED - LIKELY RESOURCE EXHAUSTION")
    print("="*80)
    print(f"Signal received: {signum}")
    print("This usually indicates the system killed the process due to:")
    print("  - Out of memory (OOM)")
    print("  - Resource limits exceeded")
    print("")
    print("To fix this, try reducing resource usage:")
    print("  1. Move away completed tar files from the output folder")
    print("  2. Rerun with less parallelism --max-workers=2 (default is 5)")
    print("  3. Rerun with higher abort limit --max-disk-usage=95 (default is 90)")
    print("="*80)
    sys.exit(1)


def main():
    """Main function to run the export."""
    
    # Register signal handlers for graceful termination
    signal.signal(signal.SIGTERM, signal_handler)
    # Note: SIGKILL cannot be caught on any platform, and is not available on Windows
    
    # Set up argument parser
    parser = argparse.ArgumentParser(
        description='Export job_info tables from PostgreSQL or Oracle database',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog='''
Examples:
  # PostgreSQL export with all parameters
  python export.py --host dev2-alm8-dist-1 --port 5432 --database archivedb --user postgres --password mypass --output-dir /path/to/output

  # Using jdbc.properties for connection info (only password and output-dir required)
  python export.py --password mypass --output-dir ./exports

  # Custom parameters
  python export.py --host myhost --port 5432 --database mydb --user myuser --password mypass --output-dir ./exports --max-workers 5
        '''
    )
    
    # Database connection parameters (optional - can be extracted from jdbc.properties)
    parser.add_argument('--host', required=False, help='Database host (optional if jdbc.properties exists)')
    parser.add_argument('--port', required=False, help='Database port (optional if jdbc.properties exists)')
    parser.add_argument('--database', required=False, help='Database name for PostgreSQL or SID for Oracle (optional if jdbc.properties exists)')
    parser.add_argument('--user', required=False, help='Database user (optional if jdbc.properties exists)')
    parser.add_argument('--password', required=False, help='Database password (optional - will prompt if not provided)')
    parser.add_argument('--output-dir', required=False, help='Output directory for export files (will prompt if not provided)')
    
    # Optional parameters
    parser.add_argument('--max-workers', type=int, default=5,
                        help='Maximum number of parallel threads (default: 5)')
    parser.add_argument('--log-dir', default=None,
                        help='Log directory (default: <current_working_dir>/logs)')
    parser.add_argument('--file-size-limit', type=int, default=100000000,
                        help='Maximum characters per SP return value (default: 100000000 = ~100MB)')
    parser.add_argument('--combined-file-size-limit', type=int, default=500,
                        help='Size limit in MB for combined files before python compression (default: 500)')
    parser.add_argument('--max-lines-per-file', type=int, default=30000,
                        help='Maximum number of lines (records) per batch file (default: 30000)')
    parser.add_argument('--max-disk-usage', type=int, default=90,
                        help='Maximum disk usage percentage before aborting (default: 90)')
    parser.add_argument('--skip-validations', action='store_true',
                        help='Skip EM version and DB version validation checks')
    
    args = parser.parse_args()
    
    # Prompt for output directory if not provided
    output_dir = args.output_dir
    if not output_dir:
        output_dir = input("Enter output directory for export files: ").strip()
        if not output_dir:
            print("ERROR: Output directory is mandatory.")
            sys.exit(1)
    
    # Create database configuration from arguments
    DB_CONFIG = {
        'host': args.host,
        'port': args.port,
        'database': args.database,
        'user': args.user,
        'password': args.password,
        'db_type': None,  # Auto-detect using is_pg() method
        'log_dir': args.log_dir,
        'output_dir': output_dir,
        'skip_validations': args.skip_validations,
        'file_size_limit': args.file_size_limit,
        'combined_file_size_limit': args.combined_file_size_limit,
        'max_lines_per_file': args.max_lines_per_file
    }
    
    # Create exporter instance
    exporter = JobInfoExporter(**DB_CONFIG)
    
    print(f"Logging In To: {exporter.log_file}")
    
    # Get and display database size
    db_size = exporter.get_db_size()
    if db_size is not None:
        db_size_gb = db_size / (1024 * 1024 * 1024)
        db_size_msg = f"Database Size: {db_size_gb:.2f} GB ({db_size:,} bytes)"
        print(db_size_msg)
        exporter.log(db_size_msg)
    else:
        db_size_msg = "Database Size: Unable to determine"
        print(db_size_msg)
        exporter.log(db_size_msg)
    
    # Initialize database functions (install export_job_info function)
    exporter.log("Initializing database export functions...")
    if not exporter.initialize_database_functions():
        print("ERROR: Failed to initialize database export functions. Check log file for details.")
        sys.exit(1)
    exporter.log("Database export functions initialized successfully")
    
    # Warn if high resource usage is configured
    if args.max_workers > 10:
        print(f"WARNING: High number of workers ({args.max_workers}) may cause memory issues.")
        print("         Consider using --max-workers=5 or less for large tables.")
    
    if args.file_size_limit > 1000000000:
        print(f"WARNING: High file size limit ({args.file_size_limit:,}) may cause memory issues.")
        print("         Consider using --file-size-limit=100000000 or less.")
    
    # PARALLEL EXPORT MODE
    # Export all existing non-empty job_info tables using parallel threads
    try:
        results = exporter.export_all_tables_parallel(
            max_workers=args.max_workers,
            output_dir=output_dir,
            max_disk_usage_percent=args.max_disk_usage
        )
    except KeyboardInterrupt:
        # User pressed Ctrl+C - signal all threads to stop
        exporter.abort_reason = "Interrupted by user (Ctrl+C)"
        exporter.abort_event.set()
        print("\n" + "="*80)
        print("EXPORT INTERRUPTED BY USER")
        print("="*80)
        print("You can resume the export by running the same command again.")
        print("Already completed tables (with .tar files) will be skipped.")
        print("Incomplete exports will be resumed from the last checkpoint.")
        print("="*80)
        sys.exit(130)
    except Exception as e:
        print("\n" + "="*80)
        print("CRITICAL ERROR DURING EXPORT")
        print("="*80)
        print(f"Error: {str(e)}")
        print("")
        print("If this is a resource issue, try:")
        print(f"  1. Reduce max workers: --max-workers=2 (current: {args.max_workers})")
        print(f"  2. Reduce file size limit: --file-size-limit=50000000 (current: {args.file_size_limit:,})")
        print("  3. Export tables one at a time: --max-workers=1")
        print("="*80)
        sys.exit(1)
    
    # Check if export was aborted and print appropriate message
    if results.get('aborted', False):
        print("\n" + "="*80)
        print("EXPORT ABORTED")
        print("="*80)
        print(f"Reason: {results.get('abort_reason', 'Unknown')}")
        print("")
        print("To increase the maximum allowed disk usage rerun the export script with the [--max-disk-usage MAX_DISK_USAGE] command, ")
        print("or temporarily move the TAR files that have finished exporting to  different folder.")
        print(f"Check log file for details: {exporter.log_file}")
        print("="*80)
        sys.exit(2)  # Exit code 2 for aborted
    elif results['failed'] > 0:
        print(f"\nExport completed with {results['failed']} failed table(s). Check log file: {exporter.log_file}")
        sys.exit(1)
    else:
        print(f"\nThe export completed successfully.")
        print(f"The export log file is saved here: {exporter.log_file}")
        print(f"You can now run the import script ({BOLD}run_archive_migration_import{RESET}) to import the exported Workload Archiving data into Control-M SaaS.")
        sys.exit(0)


if __name__ == '__main__':
    main()
