diff --git a/employee_data_merger.py b/employee_data_merger.py new file mode 100644 index 0000000..a9de40b --- /dev/null +++ b/employee_data_merger.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +""" +Employee Data Merger and Deduplication Script +============================================ + +This script combines employee data from multiple SQL files and removes duplicates +based on intelligent business rules while maintaining data integrity. + +Features: +- Parses SQL INSERT statements from multiple files +- Intelligent duplicate detection by employee ID +- Quality-based record selection for deduplication +- Comprehensive logging of merge operations +- Generates clean combined SQL output +""" + +import re +import os +from typing import List, Dict, Set, Tuple +from dataclasses import dataclass +from datetime import datetime + +@dataclass +class Employee: + """Employee record structure""" + index: int + id: int + firstName: str + lastName: str + title: str + contractId: int + source: str = "" + quality_score: float = 0.0 + +class EmployeeDataMerger: + """Handles merging and deduplication of employee data from multiple sources""" + + def __init__(self): + self.employees: List[Employee] = [] + self.duplicate_stats = { + 'total_duplicates': 0, + 'kept_gov': 0, + 'kept_lt': 0, + 'removed_dummy': 0, + 'quality_upgrades': 0 + } + + def parse_sql_file(self, filename: str, source_name: str) -> List[Employee]: + """Parse employee data from SQL INSERT file""" + print(f"šŸ“ Parsing {filename} (Source: {source_name})") + + try: + with open(filename, 'r', encoding='utf-8') as f: + content = f.read() + except FileNotFoundError: + print(f"āŒ File {filename} not found") + return [] + + # Extract VALUES section from INSERT statement + values_match = re.search(r'VALUES\s*(.*?)(?:;|$)', content, re.DOTALL) + if not values_match: + print(f"āŒ No VALUES section found in {filename}") + return [] + + values_text = values_match.group(1) + employees = [] + + # Split records by ),( pattern + record_pattern = r'\((\d+),\s*(\d+),\s*\'([^\']*)\',\s*\'([^\']*)\',\s*(NULL|\'[^\']*\'),\s*(\d+)\)' + matches = re.findall(record_pattern, values_text) + + for match in matches: + index, emp_id, first_name, last_name, title, contract_id = match + + # Clean title field + title_clean = None if title == 'NULL' else title.strip("'") + + employee = Employee( + index=int(index), + id=int(emp_id), + firstName=first_name.strip(), + lastName=last_name.strip(), + title=title_clean, + contractId=int(contract_id), + source=source_name + ) + employees.append(employee) + + print(f"āœ… Parsed {len(employees)} employees from {filename}") + return employees + + def is_dummy_record(self, emp: Employee) -> bool: + """Check if employee record appears to be dummy/test data""" + dummy_patterns = [ + r'^(no\s*(name|id)|pending|test|unknown|n\/?a|do\s*not|enter|dummy|xxx?|zzz?)', + r'^(incorrect|missing|wrong|employee)', + r'^[0-9]+$', # Only numbers + r'^[a-z]{1,2}$', # Single letters + r'^(a|b|x|z)\s*(a|b|x|z)$' # Single letter combinations + ] + + full_name = f"{emp.firstName} {emp.lastName}".lower().strip() + first_name = emp.firstName.lower().strip() + last_name = emp.lastName.lower().strip() + + for pattern in dummy_patterns: + if (re.search(pattern, full_name, re.IGNORECASE) or + re.search(pattern, first_name, re.IGNORECASE) or + re.search(pattern, last_name, re.IGNORECASE)): + return True + + return False + + def calculate_quality_score(self, emp: Employee) -> float: + """Calculate quality score for record selection""" + score = 0.0 + + # Major penalty for dummy records + if self.is_dummy_record(emp): + score -= 1000 + + # Reward complete data + if emp.title and emp.title.strip(): + score += 10 + + # Prefer higher contract IDs (usually more recent) + score += emp.contractId * 0.1 + + # Reward longer, more descriptive names + score += len(emp.firstName) + len(emp.lastName) + + # Slight preference for GOV source (appears more authoritative) + if emp.source == 'GOV': + score += 5 + + # Penalty for empty or very short names + if len(emp.firstName) <= 2 or len(emp.lastName) <= 2: + score -= 50 + + # Reward normal name patterns (letters, spaces, common punctuation) + name_pattern = re.compile(r'^[A-Za-z\s\.\-\']+$') + if name_pattern.match(emp.firstName) and name_pattern.match(emp.lastName): + score += 20 + + return score + + def deduplicate_employees(self, employees: List[Employee]) -> List[Employee]: + """Remove duplicates based on employee ID with intelligent selection""" + print("\nšŸ” Starting deduplication process...") + + # Group employees by ID + id_groups: Dict[int, List[Employee]] = {} + for emp in employees: + if emp.id not in id_groups: + id_groups[emp.id] = [] + id_groups[emp.id].append(emp) + + # Calculate quality scores + for emp in employees: + emp.quality_score = self.calculate_quality_score(emp) + + deduplicated = [] + + for emp_id, group in id_groups.items(): + if len(group) == 1: + # No duplicates + deduplicated.append(group[0]) + else: + # Multiple records - select best one + self.duplicate_stats['total_duplicates'] += len(group) - 1 + + # Sort by quality score (highest first) + group.sort(key=lambda x: x.quality_score, reverse=True) + best_record = group[0] + + # Track statistics + if best_record.source == 'GOV': + self.duplicate_stats['kept_gov'] += 1 + else: + self.duplicate_stats['kept_lt'] += 1 + + # Count dummy records removed + dummy_removed = sum(1 for emp in group[1:] if self.is_dummy_record(emp)) + self.duplicate_stats['removed_dummy'] += dummy_removed + + # Check for quality upgrades + sources = [emp.source for emp in group] + if len(set(sources)) > 1: + self.duplicate_stats['quality_upgrades'] += 1 + + deduplicated.append(best_record) + + # Log decision for significant duplicates + if len(group) > 2 or any(not self.is_dummy_record(emp) for emp in group): + print(f"šŸ“‹ Employee ID {emp_id}:") + print(f" āœ… KEPT: {best_record.source} - {best_record.firstName} {best_record.lastName} (Score: {best_record.quality_score:.1f})") + for removed in group[1:]: + status = "DUMMY" if self.is_dummy_record(removed) else "LOWER_QUALITY" + print(f" āŒ {status}: {removed.source} - {removed.firstName} {removed.lastName} (Score: {removed.quality_score:.1f})") + + return deduplicated + + def generate_combined_sql(self, employees: List[Employee], output_filename: str = 'combined_employees.sql'): + """Generate combined SQL file with deduplicated data""" + print(f"\nšŸ“ Generating combined SQL file: {output_filename}") + + # Sort employees by index for consistency + employees.sort(key=lambda x: x.index) + + # Reassign consecutive indices + for i, emp in enumerate(employees, 1): + emp.index = i + + sql_content = """-- Combined and Deduplicated Employee Data +-- Generated on: {timestamp} +-- Total Records: {total} +-- Sources: Government employee data + LT employee data +-- Deduplication: Intelligent quality-based selection + +CREATE TABLE IF NOT EXISTS `employee` ( + `index` bigint NOT NULL AUTO_INCREMENT, + `id` bigint NOT NULL, + `firstName` varchar(50) NOT NULL, + `lastName` varchar(50) NOT NULL, + `title` varchar(20) DEFAULT NULL, + `contractId` bigint NOT NULL DEFAULT '1', + UNIQUE KEY `index_2` (`index`), + KEY `index` (`index`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; + +-- Clear existing data +TRUNCATE TABLE `employee`; + +-- Insert combined and deduplicated data +INSERT INTO `employee` (`index`, `id`, `firstName`, `lastName`, `title`, `contractId`) VALUES +""".format( + timestamp=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + total=len(employees) +) + + # Generate VALUES entries + values_entries = [] + for emp in employees: + title_value = 'NULL' if emp.title is None else f"'{emp.title}'" + values_entries.append( + f"({emp.index}, {emp.id}, '{emp.firstName}', '{emp.lastName}', {title_value}, {emp.contractId})" + ) + + sql_content += ',\n'.join(values_entries) + ';\n' + + # Add statistics as comments + sql_content += f""" +-- DEDUPLICATION STATISTICS +-- ======================== +-- Original records processed: {self.duplicate_stats['total_duplicates'] + len(employees)} +-- Final unique records: {len(employees)} +-- Duplicates removed: {self.duplicate_stats['total_duplicates']} +-- Records kept from GOV source: {self.duplicate_stats['kept_gov']} +-- Records kept from LT source: {self.duplicate_stats['kept_lt']} +-- Dummy/test records removed: {self.duplicate_stats['removed_dummy']} +-- Quality upgrades performed: {self.duplicate_stats['quality_upgrades']} +""" + + # Write to file + with open(output_filename, 'w', encoding='utf-8') as f: + f.write(sql_content) + + print(f"āœ… Combined SQL file generated successfully!") + return output_filename + + def merge_files(self, file_configs: List[Tuple[str, str]], output_filename: str = 'combined_employees.sql'): + """Main method to merge multiple employee data files""" + print("šŸš€ Starting Employee Data Merger") + print("=" * 50) + + all_employees = [] + + # Parse all source files + for filename, source_name in file_configs: + employees = self.parse_sql_file(filename, source_name) + all_employees.extend(employees) + + print(f"\nšŸ“Š SUMMARY BEFORE DEDUPLICATION") + print(f"Total records loaded: {len(all_employees)}") + + # Perform deduplication + deduplicated_employees = self.deduplicate_employees(all_employees) + + print(f"\nšŸ“Š DEDUPLICATION RESULTS") + print("=" * 30) + print(f"Original records: {len(all_employees)}") + print(f"Final records: {len(deduplicated_employees)}") + print(f"Duplicates removed: {self.duplicate_stats['total_duplicates']}") + print(f"Kept from GOV: {self.duplicate_stats['kept_gov']}") + print(f"Kept from LT: {self.duplicate_stats['kept_lt']}") + print(f"Dummy records removed: {self.duplicate_stats['removed_dummy']}") + print(f"Quality upgrades: {self.duplicate_stats['quality_upgrades']}") + + # Generate combined SQL + output_file = self.generate_combined_sql(deduplicated_employees, output_filename) + + print(f"\nāœ… MERGE COMPLETED SUCCESSFULLY!") + print(f"šŸ“ Output file: {output_file}") + print(f"šŸ“ˆ Final record count: {len(deduplicated_employees)}") + + return output_file, deduplicated_employees + +def main(): + """Main execution function""" + merger = EmployeeDataMerger() + + # Configure source files + file_configs = [ + ('gov_employee.sql', 'GOV'), # Government employee data + ('lt_employee.sql', 'LT') # LT employee data + ] + + # Verify files exist + missing_files = [f for f, _ in file_configs if not os.path.exists(f)] + if missing_files: + print(f"āŒ Missing files: {missing_files}") + print("Please ensure all SQL files are in the current directory.") + return + + # Perform merge + try: + output_file, final_employees = merger.merge_files(file_configs, 'combined_employees.sql') + + # Additional validation + print(f"\nšŸ” VALIDATION CHECKS") + print("=" * 20) + + # Check for remaining duplicates + ids = [emp.id for emp in final_employees] + duplicate_ids = set([id for id in ids if ids.count(id) > 1]) + + if duplicate_ids: + print(f"āš ļø Warning: {len(duplicate_ids)} employee IDs still have duplicates: {duplicate_ids}") + else: + print("āœ… No duplicate employee IDs found") + + # Check data quality + dummy_count = sum(1 for emp in final_employees if merger.is_dummy_record(emp)) + print(f"šŸ“Š Data quality: {len(final_employees) - dummy_count}/{len(final_employees)} real records ({dummy_count} dummy records remaining)") + + # Contract ID distribution + contract_distribution = {} + for emp in final_employees: + contract_distribution[emp.contractId] = contract_distribution.get(emp.contractId, 0) + 1 + + print(f"šŸ“‹ Contract ID distribution:") + for contract_id, count in sorted(contract_distribution.items()): + print(f" Contract {contract_id}: {count} employees") + + print(f"\nšŸŽ‰ Employee data merge completed successfully!") + print(f"šŸ“ Use {output_file} for your employee synchronization script.") + + except Exception as e: + print(f"āŒ Error during merge: {e}") + import traceback + traceback.print_exc() + +if __name__ == "__main__": + main() \ No newline at end of file