Update import when source files don't have Name column

time_attendance_import_service.py

@@ -197,57 +197,129 @@ class TimeAttendanceImportService:
                 # Already a number
                 distance_float = float(distance_value)
             
-            # Validate reasonable range (0 to 100 miles)
+            # Validate the distance is reasonable (0 to 1000 miles)
-            if distance_float < 0:
+            if 0 <= distance_float <= 1000:
+                return distance_float
+            else:
                 if self.logger:
-                    self.logger.logger.warning(f"Negative distance value {distance_float} converted to positive")
+                    self.logger.logger.warning(f"Distance value {distance_float} is out of reasonable range")
-                distance_float = abs(distance_float)
+                return None
-            
-            if distance_float > 100:
-                if self.logger:
-                    self.logger.logger.warning(f"Distance value {distance_float} exceeds 100 miles, may be invalid")
-            
-            return round(distance_float, 4)  # Round to 4 decimal places
                 
         except (ValueError, TypeError) as e:
             if self.logger:
-                self.logger.logger.debug(f"Could not parse distance value '{distance_value}': {e}")
+                self.logger.logger.warning(f"Could not parse distance value '{distance_value}': {e}")
             return None
     
-    def _process_recorded_address(self, row):
+    def _process_recorded_address(self, row) -> Optional[str]:
         """
-        Process recorded address field, handling Excel HYPERLINK formulas
+        Process Recorded Address field from Excel - handles HYPERLINK formulas
         
         Args:
             row: DataFrame row containing the 'Recorded Address' column
             
         Returns:
-            Parsed address string or None
+            Cleaned address text, or None if not present/invalid
         """
-        recorded_address_value = row.get('Recorded Address')
+        # Check if Recorded Address column exists
+        if 'Recorded Address' not in row.index:
+            return None
+        
+        address_value = row.get('Recorded Address')
         
         # Check if value exists and is not NaN
-        if pd.notna(recorded_address_value):
+        if pd.isna(address_value):
-            # Convert to string
-            address_str = str(recorded_address_value).strip()
-            
-            # Skip if empty or the string "nan"
-            if not address_str or address_str.lower() == 'nan':
             return None
         
-            # Parse the value (handles both formulas and plain text)
+        # Parse HYPERLINK formula if present, or return raw value
-            parsed_address = self._parse_excel_hyperlink(address_str)
+        parsed_address = self._parse_excel_hyperlink(address_value)
         
-            if parsed_address and parsed_address.strip():
+        # Clean and return
-                if self.logger:
+        if parsed_address:
-                    self.logger.logger.debug(f"Processed Recorded Address: '{address_str[:60]}...' -> '{parsed_address}'")
+            return str(parsed_address).strip()
-                return parsed_address.strip()
         else:
-                if self.logger:
-                    self.logger.logger.warning(f"Empty result after parsing Recorded Address: '{address_str[:60]}...'")
             return None
     
-        return None
+    def _generate_record_hash(self, record_data: Dict[str, Any]) -> str:
+        """
+        Generate unique hash for a time attendance record
+        
+        Args:
+            record_data: Dictionary containing record data
+            
+        Returns:
+            SHA-256 hash string
+        """
+        hash_string = (
+            f"{record_data['employee_id']}-"
+            f"{record_data['attendance_date']}-"
+            f"{record_data['attendance_time']}-"
+            f"{record_data['location_name']}-"
+            f"{record_data['action_description']}"
+        )
+        
+        return hashlib.sha256(hash_string.encode()).hexdigest()
+    
+    def _get_existing_record_hashes(self) -> set:
+        """
+        Get hashes of all existing time attendance records
+        
+        Returns:
+            Set of hash strings
+        """
+        try:
+            from models.time_attendance import TimeAttendance
+            
+            records = TimeAttendance.query.all()
+            hashes = set()
+            
+            for record in records:
+                record_data = {
+                    'employee_id': record.employee_id,
+                    'attendance_date': record.attendance_date,
+                    'attendance_time': record.attendance_time,
+                    'location_name': record.location_name,
+                    'action_description': record.action_description
+                }
+                hashes.add(self._generate_record_hash(record_data))
+            
+            return hashes
+            
+        except Exception as e:
+            if self.logger:
+                self.logger.logger.error(f"Failed to get existing record hashes: {e}")
+            return set()
+    
+    def _get_existing_record_hashes_with_data(self) -> Dict[str, Dict]:
+        """
+        Get hashes with corresponding record data for duplicate comparison
+        
+        Returns:
+            Dictionary mapping hash to record data
+        """
+        try:
+            from models.time_attendance import TimeAttendance
+            
+            records = TimeAttendance.query.all()
+            hash_map = {}
+            
+            for record in records:
+                record_data = {
+                    'employee_id': record.employee_id,
+                    'employee_name': record.employee_name,
+                    'attendance_date': record.attendance_date,
+                    'attendance_time': record.attendance_time,
+                    'location_name': record.location_name,
+                    'action_description': record.action_description
+                }
+                record_hash = self._generate_record_hash(record_data)
+                hash_map[record_hash] = record_data
+            
+            return hash_map
+            
+        except Exception as e:
+            if self.logger:
+                self.logger.logger.error(f"Failed to get existing record hashes with data: {e}")
+            return {}
     
     def analyze_for_duplicates(self, file_path: str) -> Dict[str, Any]:
         """
@@ -351,7 +423,8 @@ class TimeAttendanceImportService:
                         new_records_count += 1
                         
                 except Exception as e:
-                    analysis_result['errors'].append(f"Row {index + 2}: {str(e)}")
+                    if self.logger:
+                        self.logger.logger.warning(f"Error analyzing row {index + 2}: {e}")
                     continue
             
             analysis_result['success'] = True
@@ -407,33 +480,26 @@ class TimeAttendanceImportService:
             df = df.dropna(how='all')
             analysis_result['total_rows'] = len(df)
             
-            # Process each row and collect invalid ones
+            # Analyze each row
             invalid_list = []
             valid_count = 0
             
             for index, row in df.iterrows():
                 row_errors = []
                 row_data = {
-                    'employee_id': None,
+                    'row_number': index + 2,
-                    'employee_name': None,
+                    'employee_id': self._clean_employee_id(row['ID']) if pd.notna(row['ID']) else None,
-                    'attendance_date': None,
-                    'attendance_time': None,
-                    'location_name': None,
-                    'action_description': None
                 }
                 
-                # Validate ID (required)
+                # Check ID
                 if pd.isna(row['ID']):
-                    row_errors.append('Missing ID')
+                    row_errors.append("Missing ID")
                 else:
-                    # Clean the employee ID (handles float issues)
                     row_data['employee_id'] = self._clean_employee_id(row['ID'])
-                
+                    # Get employee name
-                # Get Name (optional - lookup if not provided)
                     if 'Name' in df.columns and pd.notna(row.get('Name')):
                         row_data['employee_name'] = str(row['Name']).strip()
-                elif row_data['employee_id']:
+                    else:
-                    # Lookup employee name from employee table using cleaned ID
                         row_data['employee_name'] = self._get_employee_name(row_data['employee_id'])
                 
                 # Check and parse Date
@@ -718,36 +784,64 @@ class TimeAttendanceImportService:
         return import_results
     
     def _parse_time_field(self, time_value) -> time:
-        """Parse time field with multiple format support"""
+        """
+        Parse various time formats from Excel
+        
+        Handles:
+        - datetime.time objects
+        - datetime.datetime objects
+        - String formats (HH:MM, HH:MM:SS, HH:MM AM/PM)
+        
+        Args:
+            time_value: Time value from Excel
+            
+        Returns:
+            time object
+        """
         if pd.isna(time_value):
             raise ValueError("Time value is empty")
         
+        # If already a time object
         if isinstance(time_value, time):
             return time_value
         
-        time_str = str(time_value).strip()
+        # If datetime object, extract time
+        if isinstance(time_value, datetime):
+            return time_value.time()
         
-        time_formats = [
+        # If string, parse it
-            '%H:%M:%S',
+        if isinstance(time_value, str):
-            '%H:%M',
+            time_str = time_value.strip()
-            '%I:%M:%S %p',
-            '%I:%M %p',
-        ]
             
-        for fmt in time_formats:
+            # Try parsing with pandas
             try:
-                return datetime.strptime(time_str, fmt).time()
+                dt = pd.to_datetime(time_str)
-            except ValueError:
+                return dt.time()
-                continue
+            except:
-        
+                # Try manual parsing for common formats
                 try:
-            return pd.to_datetime(time_value).time()
+                    # Format: HH:MM or HH:MM:SS
+                    parts = time_str.split(':')
+                    if len(parts) >= 2:
+                        hour = int(parts[0])
+                        minute = int(parts[1])
+                        second = int(parts[2]) if len(parts) > 2 else 0
+                        return time(hour, minute, second)
                 except:
                     pass
         
-        raise ValueError(f"Unable to parse time value: {time_value}")
+        raise ValueError(f"Could not parse time value: {time_value}")
     
     def _get_employee_name(self, employee_id: str) -> str:
+        """
+        Get employee name from database, formatted as 'lastname, firstname'
+        
+        Args:
+            employee_id: Employee ID to lookup
+            
+        Returns:
+            Employee name in 'lastname, firstname' format
+        """
         try:
             from models.employee import Employee
             
@@ -766,7 +860,8 @@ class TimeAttendanceImportService:
             # Now lookup the employee
             employee = Employee.get_by_employee_id(int(cleaned_id))
             if employee:
-                return employee.full_name
+                # Format name as "lastname, firstname"
+                return f"{employee.lastName}, {employee.firstName}"
             else:
                 if self.logger:
                     self.logger.logger.warning(f"Employee ID {cleaned_id} not found in employee table")
@@ -782,6 +877,15 @@ class TimeAttendanceImportService:
                 return f"Employee {employee_id}"
             
     def _clean_employee_id(self, employee_id) -> str:
+        """
+        Clean employee ID to handle various formats
+        
+        Args:
+            employee_id: Raw employee ID value
+            
+        Returns:
+            Cleaned employee ID string
+        """
         try:
             # Convert to string first
             id_str = str(employee_id).strip()
@@ -799,123 +903,31 @@ class TimeAttendanceImportService:
                 self.logger.logger.warning(f"Could not clean employee ID '{employee_id}': {e}")
             return str(employee_id).strip()
     
-    def _generate_record_hash(self, record_data: Dict) -> str:
-        """Generate unique hash for a record to detect duplicates"""
-        hash_string = (
-            f"{record_data['employee_id']}_"
-            f"{record_data['attendance_date']}_"
-            f"{record_data['attendance_time']}_"
-            f"{record_data['location_name']}_"
-            f"{record_data['action_description']}"
-        )
-        return hashlib.md5(hash_string.encode()).hexdigest()
-    
-    def _get_existing_record_hashes(self) -> set:
-        """Get hashes of existing records (hash only)"""
-        try:
-            from models.time_attendance import TimeAttendance
-            
-            existing_records = TimeAttendance.query.all()
-            hashes = set()
-            
-            for record in existing_records:
-                record_data = {
-                    'employee_id': record.employee_id,
-                    'attendance_date': record.attendance_date,
-                    'attendance_time': record.attendance_time,
-                    'location_name': record.location_name,
-                    'action_description': record.action_description
-                }
-                hashes.add(self._generate_record_hash(record_data))
-            
-            return hashes
-        except Exception as e:
-            if self.logger:
-                self.logger.logger.warning(f"Failed to get existing record hashes: {e}")
-            return set()
-    
-    def _get_existing_record_hashes_with_data(self) -> Dict[str, Dict]:
-        """Get hashes with full existing record data for comparison"""
-        try:
-            from models.time_attendance import TimeAttendance
-            
-            existing_records = TimeAttendance.query.all()
-            hash_map = {}
-            
-            for record in existing_records:
-                record_data = {
-                    'employee_id': record.employee_id,
-                    'attendance_date': record.attendance_date,
-                    'attendance_time': record.attendance_time,
-                    'location_name': record.location_name,
-                    'action_description': record.action_description
-                }
-                record_hash = self._generate_record_hash(record_data)
-                
-                hash_map[record_hash] = {
-                    'id': record.id,
-                    'employee_id': record.employee_id,
-                    'employee_name': record.employee_name,
-                    'platform': record.platform,
-                    'attendance_date': record.attendance_date,
-                    'attendance_time': record.attendance_time,
-                    'location_name': record.location_name,
-                    'action_description': record.action_description,
-                    'event_description': record.event_description,
-                    'recorded_address': record.recorded_address,
-                    'distance': getattr(record, 'distance', None),
-                    'import_batch_id': record.import_batch_id,
-                    'import_date': record.import_date,
-                    'import_source': record.import_source
-                }
-            
-            return hash_map
-        except Exception as e:
-            if self.logger:
-                self.logger.logger.warning(f"Failed to get existing records with data: {e}")
-            return {}
-    
     def validate_excel_file(self, file_path: str) -> Dict[str, Any]:
-        """Enhanced Excel file validation with detailed analysis"""
+        """
+        Validate Excel file structure and content before import
+        
+        Args:
+            file_path: Path to the Excel file
+            
+        Returns:
+            Dictionary containing validation results
+        """
         validation_results = {
             'valid': False,
-            'total_rows': 0,
-            'valid_rows': 0,
-            'invalid_rows': 0,
-            'columns': [],
-            'sample_data': [],
             'errors': [],
             'warnings': [],
-            'file_info': {}
+            'total_rows': 0,
+            'valid_rows': 0,
+            'invalid_rows': 0
         }
         
         try:
-            import os
+            # Try to read the Excel file
-            file_stats = os.stat(file_path)
+            df = pd.read_excel(file_path)
-            validation_results['file_info'] = {
-                'size': file_stats.st_size,
-                'size_mb': round(file_stats.st_size / (1024 * 1024), 2)
-            }
-            
-            excel_file = pd.ExcelFile(file_path)
-            sheet_name = excel_file.sheet_names[0]
-            df = pd.read_excel(file_path, sheet_name=sheet_name)
-            
-            original_row_count = len(df)
-            df = df.dropna(how='all')
-            
             validation_results['total_rows'] = len(df)
-            validation_results['columns'] = df.columns.tolist()
             
-            if original_row_count > len(df):
+            # Check required columns
-                validation_results['warnings'].append(
-                    f"Removed {original_row_count - len(df)} completely empty rows"
-                )
-            
-            sample_rows = df.head(5).to_dict('records')
-            validation_results['sample_data'] = sample_rows
-
-            # Define ONLY truly required columns (ID, Name, Date, Time, Location Name, Action Description)
             required_columns = ['ID', 'Date', 'Time', 'Location Name', 'Action Description']
             missing_columns = [col for col in required_columns if col not in df.columns]
             
@@ -924,7 +936,17 @@ class TimeAttendanceImportService:
                     f"Missing required columns: {', '.join(missing_columns)}"
                 )
             
-            # Count valid rows by checking if ALL required fields have values
+            # Check optional columns
+            optional_columns = ['Name', 'Platform', 'Event Description', 'Recorded Address', 'Distance']
+            present_optional = [col for col in optional_columns if col in df.columns]
+            
+            if present_optional:
+                validation_results['warnings'].append(
+                    f"Optional columns found: {', '.join(present_optional)}"
+                )
+            
+            # Validate data in rows
+            if not missing_columns:
                 valid_row_count = 0
                 invalid_row_details = []
                 
@@ -932,7 +954,6 @@ class TimeAttendanceImportService:
                     is_valid = True
                     missing_fields = []
                     
-                # Check each required column
                     for col in required_columns:
                         if col in df.columns:
                             if pd.isna(row[col]):
@@ -1131,4 +1152,3 @@ class TimeAttendanceImportService:
                 self.logger.logger.error(f"Failed to delete batch {batch_id}: {e}")
         
         return result
-