Update memory_logic.py

memory_logic.py

@@ -1,4 +1,3 @@
-# memory_logic.py
 import os
 import json
 import time
@@ -7,7 +6,6 @@ import logging
 import re
 import threading
 
-# Conditionally import heavy dependencies
 try:
     from sentence_transformers import SentenceTransformer
     import faiss
@@ -30,38 +28,34 @@ except ImportError:
 
 
 logger = logging.getLogger(__name__)
-# Suppress verbose logs from dependencies
 for lib_name in ["sentence_transformers", "faiss", "datasets", "huggingface_hub"]:
-    if logging.getLogger(lib_name): # Check if logger exists
+    if logging.getLogger(lib_name):
         logging.getLogger(lib_name).setLevel(logging.WARNING)
 
 
-# --- Configuration (Read directly from environment variables) ---
-STORAGE_BACKEND = os.getenv("STORAGE_BACKEND", "HF_DATASET").upper() #HF_DATASET, RAM, SQLITE
-SQLITE_DB_PATH = os.getenv("SQLITE_DB_PATH", "app_data/ai_memory.db") # Changed default path
+STORAGE_BACKEND = os.getenv("STORAGE_BACKEND", "HF_DATASET").upper()
+SQLITE_DB_PATH = os.getenv("SQLITE_DB_PATH", "app_data/ai_memory.db")
 HF_TOKEN = os.getenv("HF_TOKEN")
-HF_MEMORY_DATASET_REPO = os.getenv("HF_MEMORY_DATASET_REPO", "broadfield-dev/ai-brain") # Example
-HF_RULES_DATASET_REPO = os.getenv("HF_RULES_DATASET_REPO", "broadfield-dev/ai-rules")     # Example
+HF_MEMORY_DATASET_REPO = os.getenv("HF_MEMORY_DATASET_REPO", "broadfield-dev/ai-brain")
+HF_RULES_DATASET_REPO = os.getenv("HF_RULES_DATASET_REPO", "broadfield-dev/ai-rules")
 
-# --- Globals for RAG within this module ---
 _embedder = None
-_dimension = 384 # Default, will be set by embedder
+_dimension = 384
 _faiss_memory_index = None
-_memory_items_list = []  # Stores JSON strings of memory objects for RAM, or loaded from DB/HF
+_memory_items_list = []
 _faiss_rules_index = None
-_rules_items_list = []   # Stores rule text strings
+_rules_items_list = []
 
 _initialized = False
 _init_lock = threading.Lock()
 
-# --- Helper: SQLite Connection ---
 def _get_sqlite_connection():
     if not sqlite3:
         raise ImportError("sqlite3 module is required for SQLite backend but not found.")
     db_dir = os.path.dirname(SQLITE_DB_PATH)
     if db_dir and not os.path.exists(db_dir):
         os.makedirs(db_dir, exist_ok=True)
-    return sqlite3.connect(SQLITE_DB_PATH, timeout=10) # Added timeout
+    return sqlite3.connect(SQLITE_DB_PATH, timeout=10)
 
 def _init_sqlite_tables():
     if STORAGE_BACKEND != "SQLITE" or not sqlite3:
@@ -69,22 +63,17 @@ def _init_sqlite_tables():
     try:
         with _get_sqlite_connection() as conn:
             cursor = conn.cursor()
-            # Stores JSON string of the memory object
             cursor.execute("""
                 CREATE TABLE IF NOT EXISTS memories (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     memory_json TEXT NOT NULL,
-                    # Optionally add embedding here if not using separate FAISS index
-                    # embedding BLOB, 
                     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
                 )
             """)
-            # Stores the rule text directly
             cursor.execute("""
                 CREATE TABLE IF NOT EXISTS rules (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     rule_text TEXT NOT NULL UNIQUE,
-                    # embedding BLOB,
                     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
                 )
             """)
@@ -93,7 +82,6 @@ def _init_sqlite_tables():
     except Exception as e:
         logger.error(f"SQLite table initialization error: {e}", exc_info=True)
 
-# --- Initialization ---
 def initialize_memory_system():
     global _initialized, _embedder, _dimension, _faiss_memory_index, _memory_items_list, _faiss_rules_index, _rules_items_list
     
@@ -105,10 +93,9 @@ def initialize_memory_system():
         logger.info(f"Initializing memory system with backend: {STORAGE_BACKEND}")
         init_start_time = time.time()
 
-        # 1. Load Sentence Transformer Model (always needed for semantic operations)
         if not SentenceTransformer or not faiss or not np:
             logger.error("Core RAG libraries (SentenceTransformers, FAISS, NumPy) not available. Cannot initialize semantic memory.")
-            _initialized = False # Mark as not properly initialized
+            _initialized = False
             return
         
         if not _embedder:
@@ -120,17 +107,15 @@ def initialize_memory_system():
             except Exception as e:
                 logger.critical(f"FATAL: Error loading SentenceTransformer: {e}", exc_info=True)
                 _initialized = False
-                return # Cannot proceed without embedder
+                return
 
-        # 2. Initialize SQLite if used
         if STORAGE_BACKEND == "SQLITE":
             _init_sqlite_tables()
 
-        # 3. Load Memories
         logger.info("Loading memories...")
         temp_memories_json = []
         if STORAGE_BACKEND == "RAM":
-            _memory_items_list = [] # Start fresh for RAM backend
+            _memory_items_list = []
         elif STORAGE_BACKEND == "SQLITE" and sqlite3:
             try:
                 with _get_sqlite_connection() as conn:
@@ -139,8 +124,8 @@ def initialize_memory_system():
         elif STORAGE_BACKEND == "HF_DATASET" and HF_TOKEN and Dataset and load_dataset:
             try:
                 logger.info(f"Attempting to load memories from HF Dataset: {HF_MEMORY_DATASET_REPO}")
-                dataset = load_dataset(HF_MEMORY_DATASET_REPO, token=HF_TOKEN, trust_remote_code=True) # Add download_mode if needed
-                if "train" in dataset and "memory_json" in dataset["train"].column_names: # Assuming 'memory_json' column
+                dataset = load_dataset(HF_MEMORY_DATASET_REPO, token=HF_TOKEN, trust_remote_code=True)
+                if "train" in dataset and "memory_json" in dataset["train"].column_names:
                     temp_memories_json = [m_json for m_json in dataset["train"]["memory_json"] if isinstance(m_json, str)]
                 else: logger.warning(f"HF Dataset {HF_MEMORY_DATASET_REPO} for memories not found or 'memory_json' column missing.")
             except Exception as e: logger.error(f"Error loading memories from HF Dataset ({HF_MEMORY_DATASET_REPO}): {e}")
@@ -148,16 +133,13 @@ def initialize_memory_system():
         _memory_items_list = temp_memories_json
         logger.info(f"Loaded {len(_memory_items_list)} memory items from {STORAGE_BACKEND}.")
 
-        # 4. Build/Load FAISS Memory Index
         _faiss_memory_index = faiss.IndexFlatL2(_dimension)
         if _memory_items_list:
             logger.info(f"Building FAISS index for {len(_memory_items_list)} memories...")
-            # Extract text to embed from memory JSON objects
             texts_to_embed_mem = []
             for mem_json_str in _memory_items_list:
                 try:
                     mem_obj = json.loads(mem_json_str)
-                    # Consistent embedding strategy: user input + bot response + takeaway
                     text = f"User: {mem_obj.get('user_input','')}\nAI: {mem_obj.get('bot_response','')}\nTakeaway: {mem_obj.get('metrics',{}).get('takeaway','N/A')}"
                     texts_to_embed_mem.append(text)
                 except json.JSONDecodeError:
@@ -165,7 +147,7 @@ def initialize_memory_system():
             
             if texts_to_embed_mem:
                 try:
-                    embeddings = _embedder.encode(texts_to_embed_mem, convert_to_tensor=False, show_progress_bar=False) # convert_to_numpy=True
+                    embeddings = _embedder.encode(texts_to_embed_mem, convert_to_tensor=False, show_progress_bar=False)
                     embeddings_np = np.array(embeddings, dtype=np.float32)
                     if embeddings_np.ndim == 2 and embeddings_np.shape[0] == len(texts_to_embed_mem) and embeddings_np.shape[1] == _dimension:
                         _faiss_memory_index.add(embeddings_np)
@@ -173,8 +155,6 @@ def initialize_memory_system():
                 except Exception as e_faiss_mem: logger.error(f"Error building FAISS memory index: {e_faiss_mem}")
         logger.info(f"FAISS memory index built. Total items: {_faiss_memory_index.ntotal if _faiss_memory_index else 'N/A'}")
 
-
-        # 5. Load Rules
         logger.info("Loading rules...")
         temp_rules_text = []
         if STORAGE_BACKEND == "RAM":
@@ -193,14 +173,13 @@ def initialize_memory_system():
                 else: logger.warning(f"HF Dataset {HF_RULES_DATASET_REPO} for rules not found or 'rule_text' column missing.")
             except Exception as e: logger.error(f"Error loading rules from HF Dataset ({HF_RULES_DATASET_REPO}): {e}")
         
-        _rules_items_list = sorted(list(set(temp_rules_text))) # Ensure unique and sorted
+        _rules_items_list = sorted(list(set(temp_rules_text)))
         logger.info(f"Loaded {len(_rules_items_list)} rule items from {STORAGE_BACKEND}.")
 
-        # 6. Build/Load FAISS Rules Index
         _faiss_rules_index = faiss.IndexFlatL2(_dimension)
         if _rules_items_list:
             logger.info(f"Building FAISS index for {len(_rules_items_list)} rules...")
-            if _rules_items_list: # Check again in case it became empty after filtering
+            if _rules_items_list:
                 try:
                     embeddings = _embedder.encode(_rules_items_list, convert_to_tensor=False, show_progress_bar=False)
                     embeddings_np = np.array(embeddings, dtype=np.float32)
@@ -214,9 +193,7 @@ def initialize_memory_system():
         logger.info(f"Memory system initialization complete in {time.time() - init_start_time:.2f}s")
 
 
-# --- Memory Operations (Semantic) ---
 def add_memory_entry(user_input: str, metrics: dict, bot_response: str) -> tuple[bool, str]:
-    """Adds a memory entry to the configured backend and FAISS index."""
     global _memory_items_list, _faiss_memory_index
     if not _initialized: initialize_memory_system()
     if not _embedder or not _faiss_memory_index:
@@ -240,31 +217,25 @@ def add_memory_entry(user_input: str, metrics: dict, bot_response: str) -> tuple
             logger.error(f"Memory embedding shape error: {embedding_np.shape}. Expected (1, {_dimension})")
             return False, "Embedding shape error."
 
-        # Add to FAISS
         _faiss_memory_index.add(embedding_np)
         
-        # Add to in-memory list
         _memory_items_list.append(memory_json_str)
         
-        # Add to persistent storage
         if STORAGE_BACKEND == "SQLITE" and sqlite3:
             with _get_sqlite_connection() as conn:
                 conn.execute("INSERT INTO memories (memory_json) VALUES (?)", (memory_json_str,))
                 conn.commit()
         elif STORAGE_BACKEND == "HF_DATASET" and HF_TOKEN and Dataset:
-            # This can be slow, consider batching or async push
             logger.info(f"Pushing {len(_memory_items_list)} memories to HF Hub: {HF_MEMORY_DATASET_REPO}")
-            Dataset.from_dict({"memory_json": list(_memory_items_list)}).push_to_hub(HF_MEMORY_DATASET_REPO, token=HF_TOKEN, private=True) # Ensure 'private' as needed
+            Dataset.from_dict({"memory_json": list(_memory_items_list)}).push_to_hub(HF_MEMORY_DATASET_REPO, token=HF_TOKEN, private=True)
         
         logger.info(f"Added memory. RAM: {len(_memory_items_list)}, FAISS: {_faiss_memory_index.ntotal}")
         return True, "Memory added successfully."
     except Exception as e:
         logger.error(f"Error adding memory entry: {e}", exc_info=True)
-        # TODO: Potential rollback logic if FAISS add succeeded but backend failed (complex)
         return False, f"Error adding memory: {e}"
 
 def retrieve_memories_semantic(query: str, k: int = 3) -> list[dict]:
-    """Retrieves k most relevant memories using semantic search."""
     if not _initialized: initialize_memory_system()
     if not _embedder or not _faiss_memory_index or _faiss_memory_index.ntotal == 0:
         logger.debug("Cannot retrieve memories: Embedder, FAISS index not ready, or index is empty.")
@@ -297,9 +268,7 @@ def retrieve_memories_semantic(query: str, k: int = 3) -> list[dict]:
         return []
 
 
-# --- Rule (Insight) Operations (Semantic) ---
 def add_rule_entry(rule_text: str) -> tuple[bool, str]:
-    """Adds a rule if valid and not a duplicate. Updates backend and FAISS."""
     global _rules_items_list, _faiss_rules_index
     if not _initialized: initialize_memory_system()
     if not _embedder or not _faiss_rules_index:
@@ -335,15 +304,9 @@ def add_rule_entry(rule_text: str) -> tuple[bool, str]:
         return True, "Rule added successfully."
     except Exception as e:
         logger.error(f"Error adding rule entry: {e}", exc_info=True)
-        # Basic rollback if FAISS add succeeded
-        if rule_text in _rules_items_list and _faiss_rules_index.ntotal > 0: # Crude check
-             # A full rollback would involve rebuilding FAISS index from _rules_items_list before append.
-             # For simplicity, this is omitted here. State could be inconsistent on error.
-             pass
         return False, f"Error adding rule: {e}"
 
 def retrieve_rules_semantic(query: str, k: int = 5) -> list[str]:
-    """Retrieves k most relevant rules using semantic search."""
     if not _initialized: initialize_memory_system()
     if not _embedder or not _faiss_rules_index or _faiss_rules_index.ntotal == 0:
         return []
@@ -362,35 +325,31 @@ def retrieve_rules_semantic(query: str, k: int = 5) -> list[str]:
         return []
 
 def remove_rule_entry(rule_text_to_delete: str) -> bool:
-    """Removes a rule from backend and rebuilds FAISS for rules."""
     global _rules_items_list, _faiss_rules_index
     if not _initialized: initialize_memory_system()
     if not _embedder or not _faiss_rules_index: return False
 
     rule_text_to_delete = rule_text_to_delete.strip()
     if rule_text_to_delete not in _rules_items_list:
-        return False # Not found
+        return False
 
     try:
         _rules_items_list.remove(rule_text_to_delete)
-        _rules_items_list.sort() # Maintain sorted order
+        _rules_items_list.sort()
 
-        # Rebuild FAISS index for rules (simplest way to ensure consistency after removal)
         new_faiss_rules_index = faiss.IndexFlatL2(_dimension)
         if _rules_items_list:
             embeddings = _embedder.encode(_rules_items_list, convert_to_tensor=False)
             embeddings_np = np.array(embeddings, dtype=np.float32)
             if embeddings_np.ndim == 2 and embeddings_np.shape[0] == len(_rules_items_list) and embeddings_np.shape[1] == _dimension:
                  new_faiss_rules_index.add(embeddings_np)
-            else: # Should not happen if list is consistent
+            else:
                 logger.error("Error rebuilding FAISS for rules after removal: Embedding shape error. State might be inconsistent.")
-                # Attempt to revert _rules_items_list (add back the rule)
                 _rules_items_list.append(rule_text_to_delete)
                 _rules_items_list.sort()
-                return False # Indicate failure
+                return False
         _faiss_rules_index = new_faiss_rules_index
         
-        # Remove from persistent storage
         if STORAGE_BACKEND == "SQLITE" and sqlite3:
             with _get_sqlite_connection() as conn:
                 conn.execute("DELETE FROM rules WHERE rule_text = ?", (rule_text_to_delete,))
@@ -402,25 +361,21 @@ def remove_rule_entry(rule_text_to_delete: str) -> bool:
         return True
     except Exception as e:
         logger.error(f"Error removing rule entry: {e}", exc_info=True)
-        # Potential partial failure, state might be inconsistent.
         return False
 
-# --- Utility functions to get all data (for UI display, etc.) ---
 def get_all_rules_cached() -> list[str]:
     if not _initialized: initialize_memory_system()
     return list(_rules_items_list)
 
 def get_all_memories_cached() -> list[dict]:
     if not _initialized: initialize_memory_system()
-    # Convert JSON strings to dicts for easier use by UI
     mem_dicts = []
     for mem_json_str in _memory_items_list:
         try: mem_dicts.append(json.loads(mem_json_str))
-        except: pass # Ignore parse errors for display
+        except: pass
     return mem_dicts
 
 def clear_all_memory_data_backend() -> bool:
-    """Clears all memories from backend and resets in-memory FAISS/list."""
     global _memory_items_list, _faiss_memory_index
     if not _initialized: initialize_memory_system()
     
@@ -429,11 +384,10 @@ def clear_all_memory_data_backend() -> bool:
         if STORAGE_BACKEND == "SQLITE" and sqlite3:
             with _get_sqlite_connection() as conn: conn.execute("DELETE FROM memories"); conn.commit()
         elif STORAGE_BACKEND == "HF_DATASET" and HF_TOKEN and Dataset:
-            # Deleting from HF usually means pushing an empty dataset
             Dataset.from_dict({"memory_json": []}).push_to_hub(HF_MEMORY_DATASET_REPO, token=HF_TOKEN, private=True)
         
         _memory_items_list = []
-        if _faiss_memory_index: _faiss_memory_index.reset() # Clear FAISS index
+        if _faiss_memory_index: _faiss_memory_index.reset()
         logger.info("All memories cleared from backend and in-memory stores.")
     except Exception as e:
         logger.error(f"Error clearing all memory data: {e}")
@@ -441,7 +395,6 @@ def clear_all_memory_data_backend() -> bool:
     return success
 
 def clear_all_rules_data_backend() -> bool:
-    """Clears all rules from backend and resets in-memory FAISS/list."""
     global _rules_items_list, _faiss_rules_index
     if not _initialized: initialize_memory_system()
     
@@ -460,7 +413,231 @@ def clear_all_rules_data_backend() -> bool:
         success = False
     return success
 
-# Optional: Function to save FAISS indices to disk (from ai-learn, if needed for persistence between app runs with RAM backend)
+def load_rules_from_file(filepath: str | None):
+    if not filepath:
+        logger.info("LOAD_RULES_FILE environment variable not set. Skipping rules loading from file.")
+        return 0, 0, 0
+
+    if not os.path.exists(filepath):
+        logger.warning(f"LOAD_RULES: Specified rules file not found: {filepath}. Skipping loading.")
+        return 0, 0, 0
+
+    added_count, skipped_count, error_count = 0, 0, 0
+    potential_rules = []
+
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except Exception as e:
+        logger.error(f"LOAD_RULES: Error reading file {filepath}: {e}", exc_info=False)
+        return 0, 0, 1
+
+    if not content.strip():
+        logger.info(f"LOAD_RULES: File {filepath} is empty. Skipping loading.")
+        return 0, 0, 0
+
+    file_name_lower = filepath.lower()
+
+    if file_name_lower.endswith(".txt"):
+        potential_rules = content.split("\n\n---\n\n")
+        if len(potential_rules) == 1 and "\n" in content:
+             potential_rules = [r.strip() for r in content.splitlines() if r.strip()]
+    elif file_name_lower.endswith(".jsonl"):
+        for line_num, line in enumerate(content.splitlines()):
+            line = line.strip()
+            if line:
+                try:
+                    rule_text_in_json_string = json.loads(line)
+                    if isinstance(rule_text_in_json_string, str):
+                        potential_rules.append(rule_text_in_json_string)
+                    else:
+                        logger.warning(f"LOAD_RULES (JSONL): Line {line_num+1} in {filepath} did not contain a string value. Got: {type(rule_text_in_json_string)}")
+                        error_count +=1
+                except json.JSONDecodeError:
+                    logger.warning(f"LOAD_RULES (JSONL): Line {line_num+1} in {filepath} failed to parse as JSON: {line[:100]}")
+                    error_count +=1
+    else:
+        logger.error(f"LOAD_RULES: Unsupported file type for rules: {filepath}. Must be .txt or .jsonl")
+        return 0, 0, 1
+
+    valid_potential_rules = [r.strip() for r in potential_rules if r.strip()]
+    total_to_process = len(valid_potential_rules)
+
+    if total_to_process == 0 and error_count == 0:
+        logger.info(f"LOAD_RULES: No valid rule segments found in {filepath} to process.")
+        return 0, 0, 0
+    elif total_to_process == 0 and error_count > 0:
+         logger.warning(f"LOAD_RULES: No valid rule segments found to process. Encountered {error_count} parsing/format errors in {filepath}.")
+         return 0, 0, error_count
+
+    logger.info(f"LOAD_RULES: Attempting to add {total_to_process} potential rules from {filepath}...")
+    for idx, rule_text in enumerate(valid_potential_rules):
+        success, status_msg = add_rule_entry(rule_text)
+        if success:
+            added_count += 1
+        elif status_msg == "duplicate":
+            skipped_count += 1
+        else:
+            logger.warning(f"LOAD_RULES: Failed to add rule from {filepath} (segment {idx+1}): '{rule_text[:50]}...'. Status: {status_msg}")
+            error_count += 1
+
+    logger.info(f"LOAD_RULES: Finished processing {filepath}. Added: {added_count}, Skipped (duplicates): {skipped_count}, Errors: {error_count}.")
+    return added_count, skipped_count, error_count
+
+def load_memories_from_file(filepath: str | None):
+    if not filepath:
+        logger.info("LOAD_MEMORIES_FILE environment variable not set. Skipping memories loading from file.")
+        return 0, 0, 0
+
+    if not os.path.exists(filepath):
+        logger.warning(f"LOAD_MEMORIES: Specified memories file not found: {filepath}. Skipping loading.")
+        return 0, 0, 0
+
+    added_count, format_error_count, save_error_count = 0, 0, 0
+    memory_objects_to_process = []
+
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except Exception as e:
+        logger.error(f"LOAD_MEMORIES: Error reading file {filepath}: {e}", exc_info=False)
+        return 0, 1, 0
+
+    if not content.strip():
+        logger.info(f"LOAD_MEMORIES: File {filepath} is empty. Skipping loading.")
+        return 0, 0, 0
+
+    file_ext = os.path.splitext(filepath.lower())[1]
+
+    if file_ext == ".json":
+        try:
+            parsed_json = json.loads(content)
+            if isinstance(parsed_json, list):
+                memory_objects_to_process = parsed_json
+            elif isinstance(parsed_json, dict):
+                 memory_objects_to_process = [parsed_json]
+            else:
+                logger.warning(f"LOAD_MEMORIES (.json): File content is not a JSON list or object in {filepath}. Type: {type(parsed_json)}")
+                format_error_count = 1
+        except json.JSONDecodeError as e:
+            logger.warning(f"LOAD_MEMORIES (.json): Invalid JSON file {filepath}. Error: {e}")
+            format_error_count = 1
+    elif file_ext == ".jsonl":
+        for line_num, line in enumerate(content.splitlines()):
+            line = line.strip()
+            if line:
+                try:
+                    memory_objects_to_process.append(json.loads(line))
+                except json.JSONDecodeError:
+                    logger.warning(f"LOAD_MEMORIES (.jsonl): Line {line_num+1} in {filepath} parse error: {line[:100]}")
+                    format_error_count += 1
+    else:
+        logger.error(f"LOAD_MEMORIES: Unsupported file type for memories: {filepath}. Must be .json or .jsonl")
+        return 0, 1, 0
+
+    total_to_process = len(memory_objects_to_process)
+
+    if total_to_process == 0 and format_error_count > 0 :
+         logger.warning(f"LOAD_MEMORIES: File parsing failed for {filepath}. Found {format_error_count} format errors and no processable objects.")
+         return 0, format_error_count, 0
+    elif total_to_process == 0:
+         logger.info(f"LOAD_MEMORIES: No memory objects found in {filepath} after parsing.")
+         return 0, 0, 0
+
+    logger.info(f"LOAD_MEMORIES: Attempting to add {total_to_process} memory objects from {filepath}...")
+    for idx, mem_data in enumerate(memory_objects_to_process):
+        if isinstance(mem_data, dict) and all(k in mem_data for k in ["user_input", "bot_response", "metrics"]):
+            success, _ = add_memory_entry(mem_data["user_input"], mem_data["metrics"], mem_data["bot_response"])
+            if success:
+                added_count += 1
+            else:
+                logger.warning(f"LOAD_MEMORIES: Failed to save memory object from {filepath} (segment {idx+1}). Data: {str(mem_data)[:100]}")
+                save_error_count += 1
+        else:
+            logger.warning(f"LOAD_MEMORIES: Skipped invalid memory object structure in {filepath} (segment {idx+1}): {str(mem_data)[:100]}")
+            format_error_count += 1
+
+    logger.info(f"LOAD_MEMORIES: Finished processing {filepath}. Added: {added_count}, Format/Structure Errors: {format_error_count}, Save Errors: {save_error_count}.")
+    return added_count, format_error_count, save_error_count
+
+
+def process_rules_from_text_blob(rules_text: str, progress_callback=None) -> dict:
+    if not rules_text.strip():
+        return {"added": 0, "skipped": 0, "errors": 0, "total": 0}
+    
+    potential_rules = rules_text.split("\n\n---\n\n")
+    if len(potential_rules) == 1 and "\n" in rules_text:
+        potential_rules = [r.strip() for r in rules_text.splitlines() if r.strip()]
+    
+    unique_rules = sorted(list(set(filter(None, [r.strip() for r in potential_rules]))))
+    total_unique = len(unique_rules)
+    if total_unique == 0:
+        return {"added": 0, "skipped": 0, "errors": 0, "total": 0}
+    
+    stats = {"added": 0, "skipped": 0, "errors": 0, "total": total_unique}
+    for idx, rule_text in enumerate(unique_rules):
+        success, status_msg = add_rule_entry(rule_text)
+        if success:
+            stats["added"] += 1
+        elif status_msg == "duplicate":
+            stats["skipped"] += 1
+        else:
+            stats["errors"] += 1
+        
+        if progress_callback is not None:
+            progress_callback((idx + 1) / total_unique, desc=f"Processed {idx+1}/{total_unique} rules...")
+            
+    return stats
+
+
+def import_kb_from_kv_dict(kv_dict: dict, progress_callback=None) -> dict:
+    rules_to_add, memories_to_add = [], []
+    for key, value in kv_dict.items():
+        if key.startswith("rule_"):
+            try:
+                rules_to_add.append(json.loads(value))
+            except:
+                logger.warning(f"KB Dict Import: Bad rule format for key {key}")
+        elif key.startswith("memory_"):
+            try:
+                mem_dict = json.loads(value)
+                if isinstance(mem_dict, dict) and all(k in mem_dict for k in ['user_input', 'bot_response', 'metrics']):
+                    memories_to_add.append(mem_dict)
+            except:
+                logger.warning(f"KB Dict Import: Bad memory format for key {key}")
+
+    stats = {"rules_added": 0, "rules_skipped": 0, "rules_errors": 0, "mems_added": 0, "mems_errors": 0}
+    total_items = len(rules_to_add) + len(memories_to_add)
+    processed_items = 0
+
+    if progress_callback is not None:
+        progress_callback(0, desc=f"Importing {total_items} items...")
+
+    for rule in rules_to_add:
+        s, m = add_rule_entry(rule)
+        if s:
+            stats["rules_added"] += 1
+        elif m == "duplicate":
+            stats["rules_skipped"] += 1
+        else:
+            stats["rules_errors"] += 1
+        processed_items += 1
+        if progress_callback is not None and total_items > 0:
+            progress_callback(processed_items / total_items, desc=f"Processing item {processed_items}/{total_items}...")
+
+    for mem in memories_to_add:
+        s, _ = add_memory_entry(mem['user_input'], mem['metrics'], mem['bot_response'])
+        if s:
+            stats["mems_added"] += 1
+        else:
+            stats["mems_errors"] += 1
+        processed_items += 1
+        if progress_callback is not None and total_items > 0:
+            progress_callback(processed_items / total_items, desc=f"Processing item {processed_items}/{total_items}...")
+            
+    return stats
+
+
 FAISS_MEMORY_PATH = os.path.join(os.getenv("FAISS_STORAGE_PATH", "app_data/faiss_indices"), "memory_index.faiss")
 FAISS_RULES_PATH = os.path.join(os.getenv("FAISS_STORAGE_PATH", "app_data/faiss_indices"), "rules_index.faiss")
 
@@ -486,12 +663,11 @@ def load_faiss_indices_from_disk():
     global _faiss_memory_index, _faiss_rules_index
     if not _initialized or not faiss: return
 
-    if os.path.exists(FAISS_MEMORY_PATH) and _faiss_memory_index: # Check if index object exists
+    if os.path.exists(FAISS_MEMORY_PATH) and _faiss_memory_index:
         try:
             logger.info(f"Loading memory FAISS index from {FAISS_MEMORY_PATH}...")
             _faiss_memory_index = faiss.read_index(FAISS_MEMORY_PATH)
             logger.info(f"Memory FAISS index loaded ({_faiss_memory_index.ntotal} items).")
-            # Consistency check: FAISS ntotal vs len(_memory_items_list)
             if _faiss_memory_index.ntotal != len(_memory_items_list) and len(_memory_items_list) > 0:
                  logger.warning(f"Memory FAISS index count ({_faiss_memory_index.ntotal}) differs from loaded texts ({len(_memory_items_list)}). Consider rebuilding FAISS.")
         except Exception as e: logger.error(f"Error loading memory FAISS index: {e}. Will use fresh index.")
@@ -503,6 +679,4 @@ def load_faiss_indices_from_disk():
             logger.info(f"Rules FAISS index loaded ({_faiss_rules_index.ntotal} items).")
             if _faiss_rules_index.ntotal != len(_rules_items_list) and len(_rules_items_list) > 0:
                  logger.warning(f"Rules FAISS index count ({_faiss_rules_index.ntotal}) differs from loaded texts ({len(_rules_items_list)}). Consider rebuilding FAISS.")
-        except Exception as e: logger.error(f"Error loading rules FAISS index: {e}. Will use fresh index.")
-
-
+        except Exception as e: logger.error(f"Error loading rules FAISS index: {e}. Will use fresh index.")