Sep 16 - Optimize code, part 2

time_attendance_import_service.py

@@ -6,8 +6,9 @@ Added functionality to detect and present duplicates for user review.
 """
 
 import pandas as pd
+import re
 import uuid
-from datetime import datetime, time
+from datetime import datetime, date, time
 from sqlalchemy.exc import SQLAlchemyError
 from typing import Dict, List, Any, Optional, Tuple
 import traceback
@@ -433,7 +434,7 @@ class TimeAttendanceImportService:
                         employee_name = self._get_employee_name(clean_id)
                     
                     # Parse date and time
-                    attendance_date = pd.to_datetime(row['Date']).date()
+                    attendance_date = self._parse_date_field(row['Date'])
                     attendance_time = self._parse_time_field(row['Time'])
                     
                     # Prepare record data
@@ -593,10 +594,10 @@ class TimeAttendanceImportService:
                     row_errors.append("Missing Date")
                 else:
                     try:
-                        attendance_date = pd.to_datetime(row['Date']).date()
+                        attendance_date = self._parse_date_field(row['Date'])
                         row_data['attendance_date'] = attendance_date
-                    except Exception:
-                        row_errors.append(f"Invalid date format: {row['Date']}")
+                    except Exception as date_error:
+                        row_errors.append(f"Invalid date format: {row['Date']} ({date_error})")
                 
                 # Check and parse Time
                 if pd.isna(row['Time']):
@@ -832,7 +833,7 @@ class TimeAttendanceImportService:
                     
                     # Validate and parse date
                     try:
-                        attendance_date = pd.to_datetime(row['Date']).date()
+                        attendance_date = self._parse_date_field(row['Date'])
                     except Exception as date_error:
                         import_results['failed_records'] += 1
                         import_results['errors'].append(f"Row {index + 2}: Invalid date format - {str(date_error)}")
@@ -936,17 +937,20 @@ class TimeAttendanceImportService:
                 )
             
         except SQLAlchemyError as e:
-            self.db.session.rollback()
             error_msg = f"Database error during import: {str(e)}"
             import_results['errors'].append(error_msg)
-            
+            # Rows commit in batches of 50 — drop the partial batch (all-or-nothing)
+            self._rollback_partial_batch(batch_id, import_results)
+            import_results['success'] = False
+
             if self.logger:
                 self.logger.log_database_error('time_attendance_import', e)
-        
+
         except Exception as e:
-            self.db.session.rollback()
             error_msg = f"Unexpected error during import: {str(e)}"
             import_results['errors'].append(error_msg)
+            self._rollback_partial_batch(batch_id, import_results)
+            import_results['success'] = False
             import_results['traceback'] = traceback.format_exc()
             
             if self.logger:
@@ -955,6 +959,111 @@ class TimeAttendanceImportService:
         
         return import_results
     
+    # Excel stores dates as days since 1899-12-30. A column formatted as General
+    # arrives as that number instead of a date, and pd.to_datetime() then read it
+    # as nanoseconds since 1970 — every such row imported as 1970-01-01.
+    _EXCEL_EPOCH = '1899-12-30'
+    _EXCEL_SERIAL_MIN = 20000   # 1954-10-03
+    _EXCEL_SERIAL_MAX = 60000   # 2064-04-05
+
+    def _parse_date_field(self, date_value) -> date:
+        """
+        Parse the Date column into a date.
+
+        Handles:
+        - real date / datetime / pandas Timestamp cells
+        - Excel serial numbers (45123 -> 2023-07-04), the General-format case
+        - text dates, read **month-first** (US time clocks): "03/04/2026" is
+          4 March 2026. When the first number cannot be a month ("13/04/2026")
+          it is read day-first instead.
+
+        Raises ValueError with a readable message — the caller records the row as
+        failed instead of importing a wrong date.
+        """
+        if date_value is None or (isinstance(date_value, float) and pd.isna(date_value)):
+            raise ValueError('Missing date')
+
+        # Already a date/datetime (openpyxl / pandas parsed the cell)
+        if isinstance(date_value, datetime):
+            return date_value.date()
+        if isinstance(date_value, date):
+            return date_value
+        to_pydatetime = getattr(date_value, 'to_pydatetime', None)  # pandas Timestamp
+        if callable(to_pydatetime):
+            return to_pydatetime().date()
+
+        # Excel serial number, as a number or as text
+        serial = None
+        if isinstance(date_value, bool):
+            raise ValueError(f"'{date_value}' is not a date")
+        if isinstance(date_value, (int, float)):
+            serial = float(date_value)
+        else:
+            text_value = str(date_value).strip()
+            if re.fullmatch(r'\d{4,6}(\.\d+)?', text_value):
+                serial = float(text_value)
+
+        if serial is not None:
+            if not (self._EXCEL_SERIAL_MIN <= serial <= self._EXCEL_SERIAL_MAX):
+                raise ValueError(f"'{date_value}' is not a valid date (number out of range)")
+            return pd.to_datetime(serial, unit='D', origin=self._EXCEL_EPOCH).date()
+
+        text_value = str(date_value).strip()
+        if not text_value:
+            raise ValueError('Missing date')
+
+        # Month-first unless the first component cannot be a month
+        day_first = False
+        parts = re.match(r'^(\d{1,2})[/\-.](\d{1,2})[/\-.](\d{2,4})$', text_value)
+        if parts and int(parts.group(1)) > 12:
+            day_first = True
+
+        parsed = pd.to_datetime(text_value, dayfirst=day_first, errors='coerce')
+        if pd.isna(parsed):
+            raise ValueError(f"'{date_value}' is not a recognised date")
+        return parsed.date()
+
+    def _rollback_partial_batch(self, batch_id, import_results=None):
+        """
+        Remove rows already committed for a failed import batch.
+
+        The row loop commits every 50 records to keep memory flat, so a failure
+        part-way through used to leave a partial batch in the table with nothing
+        to show it was incomplete. An import is all-or-nothing from the user's
+        point of view, so the batch is deleted and the failure reported.
+        """
+        try:
+            from models.time_attendance import TimeAttendance
+
+            self.db.session.rollback()
+            removed = TimeAttendance.query.filter_by(import_batch_id=batch_id).delete(
+                synchronize_session=False)
+            self.db.session.commit()
+
+            if removed and self.logger:
+                self.logger.logger.warning(
+                    f"Import batch {batch_id} failed — removed {removed} partially imported records"
+                )
+            if import_results is not None:
+                import_results['imported_records'] = 0
+                import_results['rolled_back_records'] = removed
+                import_results['errors'].append(
+                    f"Import failed — {removed} partially imported records were removed. "
+                    f"Nothing from this file was kept."
+                )
+            return removed
+        except Exception as cleanup_error:
+            if self.logger:
+                self.logger.logger.error(
+                    f"Could not remove partial import batch {batch_id}: {cleanup_error}", exc_info=True
+                )
+            if import_results is not None:
+                import_results['errors'].append(
+                    f"Import failed and the partial batch {batch_id} could not be removed "
+                    f"automatically — delete it from the import history."
+                )
+            return 0
+
     def _parse_time_field(self, time_value) -> time:
         """
         Parse various time formats from Excel
@@ -1050,30 +1159,48 @@ class TimeAttendanceImportService:
             
     def _clean_employee_id(self, employee_id) -> str:
         """
-        Clean employee ID to handle various formats
-        
-        Args:
-            employee_id: Raw employee ID value
-            
-        Returns:
-            Cleaned employee ID string
+        Normalise an employee ID from an Excel cell into the form the app stores.
+
+            1234 / "1234" ............. "1234"     (unchanged)
+            1234.0 (Excel numeric) .... "1234"
+            "1759.PW", "1759 - PW" .... "1759PW"   canonical work-type spelling
+            "1234.5" .................. "1234.5"   NOT an ID — never truncated
+            "01234" ................... "01234"    zero padding preserved
+
+        Work-type IDs are stored canonically so the calculator, the exports and
+        the report filters all read the same spelling (§13). Before this, an
+        imported "1759.PW" stayed as-is and the exports counted it as a separate
+        REGULAR employee called "1759.PW" instead of PW hours for employee 1759.
         """
-        try:
-            # Convert to string first
-            id_str = str(employee_id).strip()
-            
-            # Handle float values like 1234.0 or '1234.0'
-            if '.' in id_str:
-                # Convert to float, then to int, then back to string
-                # This removes the decimal part: 1234.0 -> 1234
-                id_str = str(int(float(id_str)))
-            
+        from working_hours_calculator import parse_employee_id_for_work_type
+
+        id_str = str(employee_id).strip()
+        if not id_str:
             return id_str
-        except (ValueError, TypeError) as e:
-            # If conversion fails, return original string
-            if self.logger:
-                self.logger.logger.warning(f"Could not clean employee ID '{employee_id}': {e}")
-            return str(employee_id).strip()
+
+        # Excel hands whole numbers over as floats: 1234.0 -> "1234". A real
+        # fraction is not an employee ID, so keep it as typed and let validation
+        # flag it rather than silently importing hours for employee 1234.
+        if '.' in id_str:
+            try:
+                numeric = float(id_str)
+            except (TypeError, ValueError):
+                numeric = None
+            if numeric is not None:
+                if numeric.is_integer():
+                    return str(int(numeric))
+                if self.logger:
+                    self.logger.logger.warning(
+                        f"Employee ID '{id_str}' is not a whole number — imported unchanged"
+                    )
+                return id_str
+
+        # "1759.PW" / "1759 - PW" / "PW.1759" -> "1759PW"
+        base_id, work_type = parse_employee_id_for_work_type(id_str)
+        if work_type != 'regular' and base_id.isdigit():
+            return f"{base_id}{work_type}"
+
+        return id_str
     
     def validate_excel_file(self, file_path: str) -> Dict[str, Any]:
         """
@@ -1167,8 +1294,8 @@ class TimeAttendanceImportService:
                 for idx, date_val in df['Date'].items():
                     if pd.notna(date_val):
                         try:
-                            pd.to_datetime(date_val)
-                        except:
+                            self._parse_date_field(date_val)
+                        except Exception:
                             invalid_dates += 1
                 
                 if invalid_dates > 0: