04/24 Fixed bulk import check for duplicate
This commit is contained in:
@@ -608,15 +608,17 @@ class BulkImportDialog(tk.Toplevel):
|
||||
tree_frame = tk.Frame(self, bg=C["bg"])
|
||||
tree_frame.pack(fill="both", expand=True, padx=24)
|
||||
|
||||
cols = ("name", "url", "check_type", "note", "visibility")
|
||||
widths = [160, 240, 90, 180, 90]
|
||||
cols = ("name", "url", "check_type", "note", "visibility", "status")
|
||||
widths = [150, 210, 80, 150, 80, 110]
|
||||
self._tree = ttk.Treeview(tree_frame, columns=cols,
|
||||
show="headings", selectmode="none", height=12)
|
||||
for col, w in zip(cols, widths):
|
||||
self._tree.heading(col, text=col)
|
||||
self._tree.heading(col, text=col.capitalize())
|
||||
self._tree.column(col, width=w, anchor="w")
|
||||
self._tree.tag_configure("skip", foreground=C["text_dim"])
|
||||
self._tree.tag_configure("valid", foreground=C["text"])
|
||||
self._tree.tag_configure("skip", foreground=C["text_dim"])
|
||||
self._tree.tag_configure("valid", foreground=C["text"])
|
||||
self._tree.tag_configure("dup_url", foreground=C["warning"])
|
||||
self._tree.tag_configure("dup_name", foreground=C["warning"])
|
||||
|
||||
vsb = ttk.Scrollbar(tree_frame, orient="vertical",
|
||||
command=self._tree.yview)
|
||||
@@ -672,8 +674,18 @@ class BulkImportDialog(tk.Toplevel):
|
||||
text="No data rows found.", fg=COLOURS["warning"])
|
||||
return
|
||||
|
||||
# Fetch existing URLs and names once for the entire preview pass
|
||||
try:
|
||||
from models import get_existing_website_urls, get_existing_website_names
|
||||
existing_urls = get_existing_website_urls()
|
||||
existing_names = get_existing_website_names()
|
||||
except Exception:
|
||||
existing_urls = set()
|
||||
existing_names = set()
|
||||
|
||||
valid_count = 0
|
||||
skip_count = 0
|
||||
dup_count = 0
|
||||
for row in rows[:200]: # preview cap
|
||||
name = (row.get("name") or "").strip()
|
||||
url = (row.get("url") or "").strip()
|
||||
@@ -687,27 +699,47 @@ class BulkImportDialog(tk.Toplevel):
|
||||
if vis not in self._ALLOWED_VIS:
|
||||
vis = "all"
|
||||
|
||||
skip = not name or not url
|
||||
tag = "skip" if skip else "valid"
|
||||
# Classify the row
|
||||
skip = not name or not url
|
||||
dup_url = not skip and url.lower() in existing_urls
|
||||
dup_name = not skip and not dup_url and name.lower() in existing_names
|
||||
|
||||
if skip:
|
||||
tag = "skip"
|
||||
status = "⚠ Missing field"
|
||||
skip_count += 1
|
||||
elif dup_url:
|
||||
tag = "dup_url"
|
||||
status = "⚠ Duplicate URL"
|
||||
dup_count += 1
|
||||
elif dup_name:
|
||||
tag = "dup_name"
|
||||
status = "⚠ Duplicate name"
|
||||
dup_count += 1
|
||||
else:
|
||||
tag = "valid"
|
||||
status = "✔ Will import"
|
||||
valid_count += 1
|
||||
|
||||
self._tree.insert("", "end", tags=(tag,),
|
||||
values=(name, url, ct, note[:60], vis))
|
||||
values=(name, url, ct, note[:50], vis, status))
|
||||
self._rows.append({
|
||||
"name": name, "url": url,
|
||||
"check_type": ct, "note": note, "visibility": vis,
|
||||
"_skip": skip,
|
||||
"_skip": skip or dup_url or dup_name,
|
||||
})
|
||||
|
||||
total = len(rows)
|
||||
shown = min(total, 200)
|
||||
more = f" ({total - shown} more not shown)" if total > 200 else ""
|
||||
parts = [f"✔ {valid_count} will import"]
|
||||
if dup_count:
|
||||
parts.append(f"⚠ {dup_count} duplicate(s) skipped")
|
||||
if skip_count:
|
||||
parts.append(f"✕ {skip_count} missing required field(s)")
|
||||
self._status_lbl.config(
|
||||
text=(f"Preview: {valid_count} valid row(s), "
|
||||
f"{skip_count} skipped (missing name/url).{more}"),
|
||||
text="Preview: " + ", ".join(parts) + more + ". "
|
||||
"(Duplicates are highlighted in orange.)",
|
||||
fg=COLOURS["text_dim"],
|
||||
)
|
||||
if valid_count:
|
||||
@@ -762,20 +794,55 @@ class BulkImportDialog(tk.Toplevel):
|
||||
# ── Import ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _do_import(self):
|
||||
"""Insert all valid rows via create_website(). Skips duplicates by URL."""
|
||||
from models import create_website
|
||||
import re
|
||||
"""Insert all valid, non-duplicate rows via create_website().
|
||||
|
||||
Duplicate detection is done in the application layer by fetching
|
||||
existing URLs and names from the DB immediately before inserting.
|
||||
This is necessary because there is no UNIQUE constraint on the
|
||||
websites table, so relying on a DB error (1062) would not work.
|
||||
"""
|
||||
from models import (
|
||||
create_website,
|
||||
get_existing_website_urls,
|
||||
get_existing_website_names,
|
||||
)
|
||||
|
||||
to_import = [r for r in self._rows if not r["_skip"]]
|
||||
if not to_import:
|
||||
show_error("No valid rows to import.")
|
||||
show_error("No valid rows to import.\n\n"
|
||||
"All rows are either missing required fields "
|
||||
"or are duplicates of existing websites.")
|
||||
return
|
||||
|
||||
inserted = 0
|
||||
skipped = 0
|
||||
errors = []
|
||||
# Re-fetch existing state at import time — user may have been on
|
||||
# the preview screen for a while; another admin could have added
|
||||
# a site in the interim.
|
||||
try:
|
||||
existing_urls = get_existing_website_urls()
|
||||
existing_names = get_existing_website_names()
|
||||
except Exception as e:
|
||||
show_error(f"Could not verify existing websites:\n{e}")
|
||||
return
|
||||
|
||||
inserted = 0
|
||||
dup_url = 0
|
||||
dup_name = 0
|
||||
errors = []
|
||||
|
||||
for r in to_import:
|
||||
url_key = r["url"].strip().lower()
|
||||
name_key = r["name"].strip().lower()
|
||||
|
||||
if url_key in existing_urls:
|
||||
dup_url += 1
|
||||
logger.info(f"[IMPORT] Skipped — duplicate URL: {r['url']}")
|
||||
continue
|
||||
|
||||
if name_key in existing_names:
|
||||
dup_name += 1
|
||||
logger.info(f"[IMPORT] Skipped — duplicate name: {r['name']}")
|
||||
continue
|
||||
|
||||
try:
|
||||
create_website(
|
||||
admin_id=self.current_user["id"],
|
||||
@@ -786,26 +853,24 @@ class BulkImportDialog(tk.Toplevel):
|
||||
credentials=[],
|
||||
visibility=r["visibility"],
|
||||
)
|
||||
# Track inserted URL/name so intra-file duplicates are
|
||||
# also caught (e.g. the same URL appears twice in the CSV).
|
||||
existing_urls.add(url_key)
|
||||
existing_names.add(name_key)
|
||||
inserted += 1
|
||||
logger.info(
|
||||
f"[IMPORT] Website '{r['name']}' ({r['url']}) imported "
|
||||
f"by admin_id={self.current_user['id']}."
|
||||
)
|
||||
except Exception as e:
|
||||
err_str = str(e)
|
||||
# Duplicate entry (MySQL error 1062) → count as skip, not error
|
||||
if "1062" in err_str or "Duplicate" in err_str:
|
||||
skipped += 1
|
||||
logger.info(
|
||||
f"[IMPORT] Skipped duplicate URL: {r['url']}")
|
||||
else:
|
||||
errors.append(f"{r['name']}: {e}")
|
||||
logger.error(
|
||||
f"[IMPORT] Failed to import '{r['name']}': {e}")
|
||||
errors.append(f"{r['name']}: {e}")
|
||||
logger.error(f"[IMPORT] Failed to import '{r['name']}': {e}")
|
||||
|
||||
msg = f"Import complete.\n\n✔ {inserted} website(s) imported."
|
||||
if skipped:
|
||||
msg += f"\n⏭ {skipped} duplicate URL(s) skipped."
|
||||
if dup_url:
|
||||
msg += f"\n⚠ {dup_url} row(s) skipped — URL already exists."
|
||||
if dup_name:
|
||||
msg += f"\n⚠ {dup_name} row(s) skipped — name already exists."
|
||||
if errors:
|
||||
msg += f"\n✕ {len(errors)} error(s):\n" + "\n".join(errors[:5])
|
||||
show_info(msg)
|
||||
|
||||
Reference in New Issue
Block a user