Model Management

# backend/_src/model_manager.py
async def select_model(model_id: str):
    # 1. Acquire switching lock (blocks concurrent switches)
    async with self._switching_lock:
        # 2. Validate model ID and configuration
        model_config = self._get_model_config(model_id)
 
        # 3. Unload current model
        await self._unload_current_model()
 
        # 4. VRAM cleanup
        torch.cuda.empty_cache()
        gc.collect()
        await asyncio.sleep(0.5)  # Allow cleanup
 
        # 5. Load new model
        await self._load_model(model_config)
 
        # 6. Test generation (5 tokens)
        await self._test_model()
 
        # 7. Update RAGEngine reference
        # 8. Clear cache (old model answers incompatible)
        # 9. Release lock

# backend/_src/model_manager.py
MODEL_PROFILES = [
    {
        "id": "llama-8b-q5",
        "name": "Llama 3.1 8B Q5_K_M",
        "backend": "llamacpp",
        "path": "./models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf",
        "vram": "5.4GB",
        "speed": "80-100 tok/s",
        "context_window": 8192,
        "gpu_layers": 33,
        "description": "Fast, balanced quality. Default model."
    },
    {
        "id": "qwen-14b-q8",
        "name": "Qwen 2.5 14B Q8_0",
        "backend": "llamacpp",
        "path": "./models/qwen2.5-14b-instruct-q8_0.gguf",
        "vram": "14.8GB",
        "speed": "40-50 tok/s",
        "context_window": 8192,
        "gpu_layers": 33,
        "description": "High quality, slower. Best for complex queries."
    },
    {
        "id": "ollama",
        "name": "Ollama (qwen2.5:14b)",
        "backend": "ollama",
        "vram": "Variable",
        "speed": "8-12 tok/s",
        "description": "Fallback HTTP backend. Slower but more compatible."
    }
]

POST http://localhost:8000/api/models/select
Content-Type: application/json
 
{
  "model_id": "qwen-14b-q8"
}

{
  "success": true,
  "current_model": {
    "id": "qwen-14b-q8",
    "name": "Qwen 2.5 14B Q8_0",
    "backend": "llamacpp",
    "vram": "14.8GB",
    "speed": "40-50 tok/s"
  },
  "previous_model": "llama-8b-q5",
  "switch_time": 18.3
}

{
  "success": false,
  "error": "Model 'invalid-model' not found in registry"
}

// src/components/Settings/ModelSelector.tsx
const handleModelSwitch = async (modelId: string) => {
  setIsSwitching(true);
 
  try {
    const response = await fetch('http://localhost:8000/api/models/select', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ model_id: modelId })
    });
 
    const result = await response.json();
 
    if (result.success) {
      toast.success(`Switched to ${result.current_model.name}`);
    } else {
      toast.error(`Switch failed: ${result.error}`);
    }
  } catch (error) {
    toast.error('Failed to switch model');
  } finally {
    setIsSwitching(false);
  }
};

// src-tauri/src/commands.rs
#[tauri::command]
async fn switch_model(model_id: String) -> Result<ModelInfo, String> {
    let client = reqwest::Client::new();
 
    let response = client
        .post("http://localhost:8000/api/models/select")
        .json(&serde_json::json!({ "model_id": model_id }))
        .send()
        .await
        .map_err(|e| format!("Switch failed: {}", e))?;
 
    let result: ModelSwitchResponse = response
        .json()
        .await
        .map_err(|e| format!("Failed to parse response: {}", e))?;
 
    Ok(result.current_model)
}

# backend/_src/llm_engine_llamacpp.py
LLM_CONFIG = {
    "model_path": "./models/llama-3.1-8b-instruct.Q5_K_M.gguf",
    "n_gpu_layers": 33,        # Offload all layers to GPU (faster)
    "n_ctx": 8192,             # Context window size
    "n_batch": 512,            # Batch size for prompt processing
    "temperature": 0.0,        # Deterministic (no randomness)
    "max_tokens": 512,         # Max response length
    "use_mlock": True,         # Lock model in RAM (prevents swapping)
    "use_mmap": True,          # Memory-map model file (faster load)
    "verbose": False           # Disable llama.cpp logging
}

# Llama 3.1 8B (Recommended)
wget https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf
 
# Qwen 2.5 14B (High Quality)
wget https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-GGUF/resolve/main/qwen2.5-14b-instruct-q8_0.gguf

# backend/_src/llm_engine_llamacpp.py
try:
    self.llm = Llama(
        model_path=config.model_path,
        n_gpu_layers=config.n_gpu_layers,  # Try full GPU offload
        n_ctx=config.n_ctx
    )
except Exception as e:
    logger.warning(f"GPU offload failed: {e}. Falling back to CPU.")
 
    # Fallback: Reduce GPU layers
    self.llm = Llama(
        model_path=config.model_path,
        n_gpu_layers=0,  # CPU-only
        n_ctx=config.n_ctx
    )

Fast (100 tok/s) ←─────────────────→ Quality (40 tok/s)
        ↑                                    ↑
   Llama 8B Q4                         Qwen 14B Q8
        |                                    |
        └─ Best for: High throughput        └─ Best for: Accuracy-critical
           Simple queries                       Complex reasoning
           Real-time chat                       Production RAG

Solution:
- Verify VRAM available: nvidia-smi
- Switch to smaller quantization (Q8 → Q5)
- Reduce GPU layers in config

Solution:
- Check GPU layers: Should be 33 for full offload
- Verify CUDA build: llama-cpp-python must be built with CUDA
- Check batch size: Increase n_batch to 512

Solution:
- This is normal for 14B+ models
- Pre-cache models in Docker to speed up first load
- Use SSD for model storage (not HDD)

Solution:
- Cache was not cleared. Manually clear:
   POST /api/conversation/clear
- Restart frontend to reset state

# List available models
curl http://localhost:8000/api/models
 
# Get current model
curl http://localhost:8000/api/models/current
 
# Check VRAM usage
nvidia-smi --query-gpu=memory.used,memory.total --format=csv
 
# Monitor model switching
docker-compose -f backend/docker-compose.atlas.yml logs -f atlas-backend