diff --git a/address_normalization_fix.py b/address_normalization_fix.py new file mode 100644 index 0000000..be2882c --- /dev/null +++ b/address_normalization_fix.py @@ -0,0 +1,208 @@ +""" +Address Normalization Fix for Distance Calculation Issues +========================================================== + +This module fixes the issue where nearly identical addresses are geocoded to +different coordinates, causing incorrect distance calculations. + +Issue: +- "7100 Gordon Rd" vs "7100 Gordons Rd, USA" → 1.5 miles apart (WRONG!) +- "3402 South Glebe Road" vs "3402, South Glebe Road, Aurora Hills" → 0.7 miles (WRONG!) + +Root Cause: +- Google Maps/OSM geocodes slightly different address strings to different coordinates +- Minor variations (plurals, commas, neighborhoods, "USA") create false distance + +Solution: +- Normalize addresses before geocoding +- Use fuzzy matching to detect identical locations +- Prevent re-geocoding of essentially the same address +""" + +import re +from difflib import SequenceMatcher + + +def normalize_address(address): + """ + Normalize address string for better matching and geocoding accuracy + This helps prevent geocoding nearly identical addresses to different coordinates + + Args: + address: Raw address string + + Returns: + Normalized address string + + Examples: + "7100 Gordon Rd, Falls Church, VA 22043" + "7100 Gordons Rd, Falls Church, VA 22043, USA" + Both normalize to: "7100 gordon rd, falls church, va 22043" + + "3402 South Glebe Road Arlington VA 22202" + "3402, South Glebe Road, Aurora Hills, Arlington VA 22202" + Both normalize to: "3402 s glebe rd, arlington, va 22202" + """ + if not address or not isinstance(address, str): + return address + + # Convert to lowercase for consistent comparison + normalized = address.lower().strip() + + # Remove extra whitespace and normalize separators + normalized = re.sub(r'\s+', ' ', normalized) # Multiple spaces to single space + normalized = re.sub(r'\s*,\s*', ', ', normalized) # Normalize comma spacing + + # Standardize common street abbreviations to short forms + street_abbrev = { + r'\broad\b': 'rd', + r'\broads\b': 'rd', # Handle plural form (Gordon Rd vs Gordons Rd) + r'\bstreet\b': 'st', + r'\bavenue\b': 'ave', + r'\bdrive\b': 'dr', + r'\blane\b': 'ln', + r'\bcourt\b': 'ct', + r'\bboulevard\b': 'blvd', + r'\bparkway\b': 'pkwy', + r'\bcircle\b': 'cir', + r'\bplace\b': 'pl', + r'\bterrace\b': 'ter', + r'\bhighway\b': 'hwy' + } + + for full_form, abbrev in street_abbrev.items(): + normalized = re.sub(full_form, abbrev, normalized) + + # Standardize directionals to single letter + directionals = { + r'\bnorth\b': 'n', + r'\bsouth\b': 's', + r'\beast\b': 'e', + r'\bwest\b': 'w', + r'\bnortheast\b': 'ne', + r'\bnorthwest\b': 'nw', + r'\bsoutheast\b': 'se', + r'\bsouthwest\b': 'sw' + } + + for full_form, abbrev in directionals.items(): + normalized = re.sub(full_form, abbrev, normalized) + + # Remove neighborhood/district names that aren't essential for location + # Examples: "Aurora Hills", "Downtown", etc. + parts = [p.strip() for p in normalized.split(',')] + + # Keep: street address, city, state, zip + # Remove: neighborhood names, building names, country suffixes + filtered_parts = [] + + # Known neighborhood keywords to remove (these don't affect geocoding) + neighborhood_keywords = ['hills', 'heights', 'park', 'village', 'estates', + 'manor', 'gardens', 'terrace', 'commons', 'plaza'] + + for i, part in enumerate(parts): + part_clean = part.strip() + + # Always keep first part (street address) + if i == 0: + filtered_parts.append(part_clean) + continue + + # Skip empty parts + if not part_clean: + continue + + # Skip country suffixes + if part_clean in ['usa', 'us', 'united states']: + continue + + # Skip if it's a neighborhood name (contains neighborhood keywords but no numbers) + is_neighborhood = False + for keyword in neighborhood_keywords: + if keyword in part_clean and not re.search(r'\d', part_clean): + is_neighborhood = True + print(f" Removing neighborhood: '{part_clean}'") + break + + if is_neighborhood: + continue + + # Keep if it looks like state (2 letter abbrev) + if re.match(r'^[a-z]{2}$', part_clean): + filtered_parts.append(part_clean) + continue + + # Keep if it looks like zip code + if re.match(r'^\d{5}(-\d{4})?$', part_clean): + filtered_parts.append(part_clean) + continue + + # Keep if it's likely a city name (reasonable length, no special patterns) + if 3 <= len(part_clean) <= 30: + filtered_parts.append(part_clean) + + # Reconstruct address + normalized = ', '.join(filtered_parts) + + # Remove common country suffixes that don't affect location + normalized = re.sub(r',?\s*(usa|united states|us)$', '', normalized) + + # Final cleanup: remove trailing commas and spaces + normalized = normalized.strip(', ') + + print(f"šŸ”§ Address normalization:") + print(f" Original: {address}") + print(f" Normalized: {normalized}") + + return normalized + + +def addresses_are_similar(addr1, addr2, threshold=0.90): + """ + Check if two addresses are similar enough to be considered the same location + Uses fuzzy string matching to handle minor variations + + Args: + addr1: First address string + addr2: Second address string + threshold: Similarity threshold (0-1), default 0.90 (90% similar) + + Returns: + Boolean indicating if addresses are similar + + Examples: + addresses_are_similar( + "7100 Gordon Rd, Falls Church, VA 22043", + "7100 Gordons Rd, Falls Church, VA 22043, USA" + ) → True (same location, minor spelling difference) + + addresses_are_similar( + "3402 South Glebe Road Arlington VA 22202", + "3402, South Glebe Road, Aurora Hills, Arlington VA 22202" + ) → True (same location, extra neighborhood name) + """ + if not addr1 or not addr2: + return False + + # Normalize both addresses + norm1 = normalize_address(addr1) + norm2 = normalize_address(addr2) + + # Exact match after normalization + if norm1 == norm2: + print(f"āœ… Addresses match exactly after normalization") + return True + + # Calculate similarity using difflib SequenceMatcher + similarity = SequenceMatcher(None, norm1, norm2).ratio() + + is_similar = similarity >= threshold + + print(f"šŸ“Š Address similarity check:") + print(f" Address 1 (normalized): {norm1}") + print(f" Address 2 (normalized): {norm2}") + print(f" Similarity score: {similarity:.2%}") + print(f" Threshold: {threshold:.2%}") + print(f" Result: {'āœ… SIMILAR (same location)' if is_similar else 'āŒ DIFFERENT (different locations)'}") + + return is_similar diff --git a/app.py b/app.py index 13aa303..73b919f 100644 --- a/app.py +++ b/app.py @@ -26,6 +26,7 @@ load_dotenv() from turnstile_utils import turnstile_utils from db_performance_optimization import initialize_performance_optimizations from app_performance_middleware import PerformanceMonitor +from address_normalization_fix import normalize_address, addresses_are_similar # Initialize Flask application app = Flask(__name__) @@ -413,15 +414,18 @@ def get_coordinates_from_address_enhanced(address): Returns (latitude, longitude, accuracy_level) """ if not address or address.strip() == "": - print("āš ļø Empty address provided for geocoding") return None, None, None - + address = address.strip() print(f"šŸŒ Enhanced geocoding for: {address}") - - # Check cache first - cached_lat, cached_lng, cached_accuracy = get_cached_coordinates(address) + + # STEP 1: Normalize address before any processing + normalized_address = normalize_address(address) + + # STEP 2: Check cache using normalized address + cached_lat, cached_lng, cached_accuracy = get_cached_coordinates(normalized_address) if cached_lat is not None: + print(f"āœ… Using cached coordinates for normalized address") return cached_lat, cached_lng, cached_accuracy # Log enhanced geocoding action @@ -470,7 +474,7 @@ def get_coordinates_from_address_enhanced(address): print(f" Place types: {place_types[:3]}") # Show first 3 types # Cache the result - cache_coordinates(address, lat, lng, accuracy) + cache_coordinates(normalized_address, lat, lng, accuracy) # Log successful enhanced geocoding try: @@ -526,7 +530,7 @@ def get_coordinates_from_address_enhanced(address): print(f" Accuracy: {accuracy} (fallback)") # Cache the fallback result - cache_coordinates(address, lat, lng, accuracy) + cache_coordinates(normalized_address, lat, lng, accuracy) # Log fallback enhanced geocoding try: @@ -849,6 +853,12 @@ def calculate_location_accuracy_enhanced(qr_address, checkin_address, checkin_la # Step 1: Get coordinates for QR address using enhanced geocoding print(f"\nšŸ“ Step 1: Geocoding QR address...") try: + # STEP 0: Check if addresses are essentially the same + if addresses_are_similar(qr_address, checkin_address, threshold=0.90): + print(f"šŸŽÆ Addresses are essentially identical - returning near-zero distance") + # Return very small distance (within 50 feet / ~0.01 miles) + return 0.01 + qr_lat, qr_lng, qr_accuracy = get_coordinates_from_address_enhanced(qr_address) print(f" Geocoding result: lat={qr_lat}, lng={qr_lng}, accuracy={qr_accuracy}")