Reworked dev.py, and made changes related to the rework

.gitignore

@@ -3,6 +3,7 @@ __pycache__/
 *$py.class
 venv/
 .env
+dev/.env
 .idea/
 .vscode/
 .agent/

README.md

@@ -104,14 +104,44 @@ Configure via environment variables.
 
 ## Known Issues
 
-- [ ] When asking a follow up question the previous output disappears
-- [ ] Parts of the UI are not theme aware resulting in a unpolished look when not using a dark theme
-- [ ] When SearXNG provides a info blob for a search it appears on top of the overview i.e. `Wikipedia` or `Linux` 
+- [ ] Update README with all updates
+
+- [x] When asking a follow up question the previous output disappears
+
+- [x] Parts of the UI are not theme aware resulting in a unpolished look when not using a dark theme
+
+- [x] When SearXNG provides a info blob for a search it appears on top of the overview i.e. `Wikipedia` or `Linux` 
 
 For any issues not stated here please create an issue ticket on [Gitea](https://git.tysstech.com/TySS-Dev/ollama-ai-answers-searxng/issues) or [GitHub](https://github.com/TySP-Dev/ollama-ai-answers-searxng/issues) and add the `bug` tag.
 
 ## Roadmap
 
+### Dev Server
+
+- [x] Stream viewer — show tokens arriving in real time in the debug panel as they come out of Valkey, so you can see exactly what the LLM is generating chunk by chunk
+
+- [x] TF-IDF score visualizer — show a table of which URLs were fetched, their scores, and which chunks were selected for context
+
+- [ ] Intent detection display — show what intent was detected and which system prompt was used for each query
+
+- [ ] Saved queries — save/load test queries so you can quickly re-run the same set of searches after making changes to the plugin
+
+- [ ] A/B model comparison — run the same query against two different models simultaneously and show both responses side by side
+
+- [ ] Response time breakdown — show how long each phase took: SearXNG fetch, page fetching, TF-IDF scoring, LLM stream start, stream complete
+
+- [ ] Context inspector — show the full assembled context string that gets sent to the LLM, so you can see exactly what it's working with
+
+- [ ] Prompt viewer — show the full system prompt + user prompt that gets sent to Ollama
+
+- [ ] Export button — copy the full context + prompt + response as a JSON blob for bug reports
+
+- [ ] Per-intent system prompt editor — edit the system prompts for each intent type live without restarting
+
+- [ ] Token counter — show estimated token count of the context being sent
+
+### Plugin
+
 - [ ] Working on feature plans
 
 ## Architecture

ollama_answers.py

@@ -1,12 +1,11 @@
-import json, os, logging, base64, time, hashlib, re, http.client, ssl, concurrent.futures, threading, math
+import json, os, logging, base64, typing, time, hashlib, re, http.client, ssl, concurrent.futures, threading, math
 from collections import Counter
 from urllib.parse import urlparse
-from searx import network
 try:
     from searx.network import get_network
 except ImportError:
     get_network = None
-from flask import Response, request, abort, jsonify
+from flask import request, abort, jsonify
 from searx.plugins import Plugin, PluginInfo
 from searx.result_types import EngineResults
 from searx import settings
@@ -24,7 +23,6 @@ except ImportError:
     logger.warning("AI Answers: valkey package not found. Streaming via Valkey unavailable.")
 
 TOKEN_EXPIRY_SEC = 3600
-STREAM_CHUNK_SIZE = 512
 STREAM_TIMEOUT_SEC = 60
 CONV_TTL = 1800
 
@@ -276,17 +274,17 @@ INTERACTIVE_CSS = '''
                             width: 32px;
                             height: 32px;
                             padding: 0;
-                            border: none;
+                            border: 1px solid var(--color-result-border, rgba(0,0,0,0.1));
                             border-radius: 4px;
-                            background: var(--color-sidebar-bg, #424247);
-                            color: var(--color-search-url, #bbb);
+                            background: var(--color-base-background-hover, rgba(0,0,0,0.06));
+                            color: var(--color-base-font, inherit);
                             cursor: pointer;
                             vertical-align: middle;
                             line-height: 1.4;
                         }
                         .sxng-btn:hover {
-                            background: var(--color-search-url, #303033);
-                            color: var(--color-sidebar-bg, #bbb);
+                            background: var(--color-result-border, rgba(0,0,0,0.15));
+                            color: var(--color-base-font, inherit);
                         }
                         .sxng-btn svg { width: 18px; height: 18px; fill: currentColor; }
                         .sxng-input-wrapper {
@@ -300,9 +298,9 @@ INTERACTIVE_CSS = '''
                         .sxng-input {
                             width: 100%;
                             height: -webkit-fill-available;
-                            background: var(--color-sidebar-bg, #424247);
-                            border: none;
-                            color: var(--color-base-font, #cdd6f4);
+                            background: var(--color-base-background-hover, rgba(0,0,0,0.06));
+                            border: 1px solid var(--color-result-border, rgba(0,0,0,0.15));
+                            color: var(--color-base-font, inherit);
                             font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
                             font-size: 0.78em;
                             padding: 3px 8px;
@@ -311,7 +309,7 @@ INTERACTIVE_CSS = '''
                             vertical-align: middle;
                         }
                         .sxng-input:focus { outline: none; }
-                        .sxng-input::placeholder { color: var(--color-base-font, #333); opacity: 0.35; }
+                        .sxng-input::placeholder { color: var(--color-base-font, inherit); opacity: 0.4; }
                         .sxng-input-line {
                             position: absolute;
                             bottom: 0;
@@ -344,14 +342,15 @@ INTERACTIVE_CSS = '''
                             appearance: none;
                             -webkit-appearance: none;
                             background: url("data:image/svg+xml;charset=UTF-8,%3C%3Fxml%20version%3D%221.0%22%20encoding%3D%22UTF-8%22%3F%3E%0A%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20512%20512%22%3E%0A%3Cg%20fill%3D%22%23aaa%22%3E%0A%3Cpolygon%20points%3D%22128%2C192%20256%2C320%20384%2C192%22%2F%3E%3C%2Fg%3E%0A%3C%2Fsvg%3E") calc(100% + 2rem) / 1rem no-repeat content-box border-box;
-                            background-color: #424247;
+                            background-color: var(--color-base-background-hover, rgba(0,0,0,0.06));
                             text-overflow: ellipsis;
-                            border-width: 0 2rem 0 0;
-                            border-color: transparent;
+                            border: 0px solid var(--color-result-border, rgba(0,0,0,0.1));
+                            border-right-width: 2rem;
+                            border-right-color: transparent;
                             border-radius: 5px;
                             outline: none;
                             height: 25px;
-                            color: var(--color-search-url, #bbb);
+                            color: var(--color-base-font, inherit);
                             font-size: .9rem;
                             padding: 1px 10px 1px 10px !important;
                             margin: 0;
@@ -361,8 +360,7 @@ INTERACTIVE_CSS = '''
                             vertical-align: middle;
                         }
                         .sxng-model-select:hover {
-                            background-color: #303033;
-                            color: var(--color-search-url, #bbb);
+                            background-color: var(--color-result-border, rgba(0,0,0,0.15));
                         }
                         .sxng-reasoning {
                             margin: 0.5rem 0; padding: 0.5rem;
@@ -385,7 +383,8 @@ INTERACTIVE_CSS = '''
                             font-size: 0.75em;
                             color: var(--color-result-link, #5e81ac);
                             text-decoration: none;
-                            opacity: 0.75;
+                            opacity: 1;
+                            font-weight: 600;
                         }
                         .sxng-citation-item a:hover {
                             opacity: 1;
@@ -395,18 +394,18 @@ INTERACTIVE_CSS = '''
                             margin-bottom: 0.75rem;
                             padding: 0.5rem;
                             border-left: 2px solid var(--color-result-link, #5e81ac);
-                            opacity: 0.6;
+                            opacity: 0.85;
                             font-size: 0.85em;
                         }
                         .sxng-prior-history summary {
                             cursor: pointer;
                             color: var(--color-result-link, #5e81ac);
-                            font-weight: 600;
+                            font-weight: 700;
                         }
                         .sxng-prior-answer {
                             margin: 0.25rem 0;
                             padding-left: 0.5rem;
-                            color: var(--color-base-font, #cdd6f4);
+                            color: var(--color-base-font, inherit);
                         }
                         .sxng-md-content {
                             line-height: 1.6;
@@ -600,23 +599,6 @@ INTERACTIVE_JS = r'''
                                 _ms.appendChild(_o);
                             }
                         }
-                        if (window.getComputedStyle && box) {
-                            try {
-                                const docStyles = getComputedStyle(document.documentElement);
-                                let accent = docStyles.getPropertyValue('--color-result-link').trim();
-                                if (!accent) {
-                                    const a = document.createElement('a');
-                                    document.body.appendChild(a);
-                                    accent = getComputedStyle(a).color;
-                                    document.body.removeChild(a);
-                                }
-                                if (accent) {
-                                    box.style.setProperty('--color-result-link', accent);
-                                    box.style.setProperty('--sxng-ai-accent', accent);
-                                }
-                            } catch(e) {}
-                        }
-
                         // conversation saved as base64 URL fragment.
                         const updateState = () => {
                             try {
@@ -686,7 +668,6 @@ INTERACTIVE_JS = r'''
                                         }
                                     });
                                     box.style.display = 'block';
-                                    if(wrapper) wrapper.style.display = '';
                                     if(footer && is_interactive) footer.style.display = 'flex';
                                     restored = true;
                                 }
@@ -871,13 +852,89 @@ FRONTEND_JS_TEMPLATE = r"""
     const conversation = {
         originalQuery: q_init,
         originalContext: new TextDecoder().decode(Uint8Array.from(atob(b64_init), c => c.charCodeAt(0))),
-        originalSources: [...urls],
         turns: [{role: 'user', content: q_init, ts: Date.now()}]
     };
     const box = document.getElementById('sxng-stream-box');
     const data = document.getElementById('sxng-stream-data');
-    const wrapper = box.closest('.answer');
-    if (wrapper) wrapper.style.display = 'none';
+
+    (function applyTheme() {
+        try {
+            const root = document.documentElement;
+            const s = getComputedStyle(root);
+            const get = (v, fallback) => s.getPropertyValue(v).trim() || fallback;
+
+            const theme = {
+                '--color-answer-background': get('--color-answer-background', '#313338'),
+                '--color-answer-font':       get('--color-answer-font', '#fff'),
+                '--color-result-link':       get('--color-result-link', '#8aacf7'),
+                '--color-base-font':         get('--color-base-font', '#cdd6f4'),
+                '--color-sidebar-bg':        get('--color-sidebar-bg', '#424247'),
+                '--color-result-hover':      get('--color-result-hover', '#303033'),
+                '--color-base-background':   get('--color-base-background', '#2a2a2e'),
+                '--color-search-font':       get('--color-search-font', '#bbb'),
+                '--color-result-border':     get('--color-result-border', '#4c566a'),
+                '--color-result-description':get('--color-result-description', '#d8dee9'),
+                '--color-toolkit-select-background': get('--color-toolkit-select-background', '#313338'),
+            };
+
+            // Apply to box and any ai-answers container
+            const targets = [box, document.getElementById('ai-answers')].filter(Boolean);
+            targets.forEach(el => {
+                Object.entries(theme).forEach(([k, v]) => {
+                    if (v) el.style.setProperty(k, v);
+                });
+            });
+
+        } catch(e) {}
+    })();
+
+    // Move AI Overview outside #answers, place it before #results
+    (function relocateBox() {
+        const answersDiv = document.getElementById('answers');
+
+        if (!box || !answersDiv) return;
+
+        // Create our own container
+        const aiContainer = document.createElement('div');
+        aiContainer.id = 'ai-answers';
+        const rootStyle = getComputedStyle(document.documentElement);
+        const getVar = (v, fb) => rootStyle.getPropertyValue(v).trim() || fb;
+        const bg = getVar('--color-answer-background', '');
+        const answerFont = getVar('--color-answer-font', '');
+        // Detect light mode by checking if answer font is dark
+        const isLight = answerFont && (answerFont.includes('0,0,0') ||
+            answerFont.includes('#000') || answerFont.includes('#333') ||
+            answerFont.includes('#444') || answerFont.includes('rgb(0') ||
+            answerFont.includes('rgb(3') || answerFont.includes('rgb(4') ||
+            answerFont.includes('rgb(5') || answerFont.includes('rgb(6'));
+        const containerBg = isLight
+            ? 'rgba(0,0,0,0.06)'
+            : (bg || 'var(--color-answer-background, #313338)');
+        aiContainer.style.cssText = [
+            `background: ${containerBg}`,
+            'padding: 1rem',
+            'margin: 0 0 1rem 0',
+            `color: ${getVar('--color-answer-font', 'var(--color-answer-font, #fff)')}`,
+            'border-radius: 8px',
+            'box-sizing: border-box',
+            'width: 100%'
+        ].join('; ');
+
+        // Move our box into the new container
+        aiContainer.appendChild(box);
+
+        const resultsGrid = document.getElementById('results');
+        if (resultsGrid) {
+            // Insert as first child of #results grid so grid-area:answers applies
+            resultsGrid.insertBefore(aiContainer, resultsGrid.firstChild);
+        } else {
+            answersDiv.parentNode.insertBefore(aiContainer, answersDiv);
+        }
+
+        // Hide #answers entirely since our box is now elsewhere
+        answersDiv.style.display = 'none';
+    })();
+
     let restored = false;
     let isStreaming = false;
 
@@ -947,7 +1004,6 @@ FRONTEND_JS_TEMPLATE = r"""
         isStreaming = true;
         try {
             const ctx = auxContext || conversation.originalContext;
-            if (wrapper) wrapper.style.display = '';
             box.style.display = 'block';
 
             const controller = new AbortController();
@@ -1002,6 +1058,11 @@ FRONTEND_JS_TEMPLATE = r"""
                 data.appendChild(cursor);
             }
 
+            const streamContainer = document.createElement('div');
+            streamContainer.className = 'sxng-stream-container';
+            if (cursor) cursor.before(streamContainer);
+            else data.appendChild(streamContainer);
+
             let buffer = '';
             let fullText = '';
             const flushBuffer = (force = false) => {
@@ -1009,8 +1070,7 @@ FRONTEND_JS_TEMPLATE = r"""
 
                 if (force) {
                     const fragment = renderCitations(buffer, urls);
-                    if (cursor) cursor.before(fragment);
-                    else data.appendChild(fragment);
+                    streamContainer.appendChild(fragment);
                     buffer = '';
                     return;
                 }
@@ -1025,12 +1085,12 @@ FRONTEND_JS_TEMPLATE = r"""
                         const s = document.createElement('span');
                         s.className = 'sxng-chunk';
                         s.textContent = preText;
-                        cursor.before(s);
+                        streamContainer.appendChild(s);
                     }
 
                     const citationText = match[0];
                     const fragment = renderCitations(citationText, urls);
-                    cursor.before(fragment);
+                    streamContainer.appendChild(fragment);
 
                     buffer = buffer.substring(match.index + match[0].length);
                 }
@@ -1041,7 +1101,7 @@ FRONTEND_JS_TEMPLATE = r"""
                         const s = document.createElement('span');
                         s.className = 'sxng-chunk';
                         s.textContent = buffer;
-                        cursor.before(s);
+                        streamContainer.appendChild(s);
                         buffer = '';
                     }
                 } else {
@@ -1050,7 +1110,7 @@ FRONTEND_JS_TEMPLATE = r"""
                         const s = document.createElement('span');
                         s.className = 'sxng-chunk';
                         s.textContent = safeChunk;
-                        cursor.before(s);
+                        streamContainer.appendChild(s);
                     }
                     buffer = buffer.substring(openIdx);
 
@@ -1058,7 +1118,7 @@ FRONTEND_JS_TEMPLATE = r"""
                         const s = document.createElement('span');
                         s.className = 'sxng-chunk';
                         s.textContent = buffer[0];
-                        cursor.before(s);
+                        streamContainer.appendChild(s);
                         buffer = buffer.substring(1);
                     }
                 }
@@ -1120,11 +1180,9 @@ FRONTEND_JS_TEMPLATE = r"""
                 }
             }
 
+            streamContainer.remove();
             if (cursor) cursor.remove();
 
-            // Remove only the streamed chunk spans added during this stream
-            Array.from(data.querySelectorAll('.sxng-chunk')).forEach(node => node.remove());
-
             const rendered = parseMarkdown(fullText.trim());
             const mdDiv = document.createElement('div');
             mdDiv.className = 'sxng-md-content';
@@ -1288,8 +1346,6 @@ INTENT_CONFIGS = {
     },
 }
 
-
-import typing
 if typing.TYPE_CHECKING:
     from searx.search import SearchWithPlugins
     from searx.extended_types import SXNG_Request
@@ -1669,6 +1725,16 @@ class SXNGPlugin(Plugin):
 
             job_id = hashlib.sha256(f"{time.time()}{q}".encode()).hexdigest()[:16]
 
+            # Persist intent for dev UI
+            logger.warning(f"INTENT BEFORE PERSIST: {repr(intent)}")
+            logger.warning(f"JOB_ID BEFORE PERSIST: {repr(job_id)}")
+            try:
+                vk = _get_valkey()
+                vk.setex(f"ai:job:{job_id}:intent", 3600, intent)
+                logger.debug(f"{PLUGIN_NAME}: persisted intent '{intent}' for job {job_id}")
+            except Exception:
+                logger.exception(f"{PLUGIN_NAME}: failed to persist intent")
+
             payload_dict = {
                 "model": effective_model,
                 "messages": [
@@ -1982,6 +2048,17 @@ class SXNGPlugin(Plugin):
             detected_intent = _detect_intent(q_clean)
             js_intent = safe_json(detected_intent)
 
+            # Persist intent for dev tooling / UI
+            try:
+                vk = _get_valkey()
+                vk.setex(
+                    f"ai:job:{job_id}:intent",
+                    1800,
+                    detected_intent
+                )
+            except Exception as e:
+                logger.debug(f"{PLUGIN_NAME}: failed to persist intent: {e}")
+
             b64_context = base64.b64encode(context_str.encode('utf-8')).decode('utf-8')
             total_context_count = self.context_deep_count + self.context_shallow_count
 
@@ -2028,7 +2105,7 @@ class SXNGPlugin(Plugin):
                 .replace("__JS_Q__", js_q)
 
             html_payload = f'''
-                <article id="sxng-stream-box" class="answer" style="display:none; margin-top: 0; margin-bottom: 0;">
+                <article id="sxng-stream-box" class="answer" style="display:none; margin: 0; padding: 0;">
                     <style>
                         @keyframes sxng-fade-pulse {{
                             0%, 100% {{ opacity: 0.1; }}
@@ -2078,11 +2155,11 @@ class SXNGPlugin(Plugin):
                     </style>
                     <div class="sxng-ai-header">
                         <span class="sxng-ai-label">
-                            <span style="color:#4a9eff;font-size:1.1em;">✦</span> AI Overview
+                            <span style="color:var(--color-result-link, #4a9eff);font-size:1.1em;">✦</span> AI Overview
                         </span>
                         <select id="sxng-model-select" class="sxng-model-select" title="Select model"></select>
                     </div>
-                    <p id="sxng-stream-data" style="white-space: pre-wrap; color: var(--color-result-description); font-size: 0.95rem; margin:0;"><span class="sxng-cursor"></span></p>
+                    <p id="sxng-stream-data" style="white-space: pre-wrap; color: var(--color-answer-font, var(--color-result-description, inherit)); font-size: 0.95rem; margin:0;"><span class="sxng-cursor"></span></p>
                     {interactive_html}
                     <script>
                     {js_code}

requirements.txt

@@ -1,3 +1,4 @@
 flask
 flask-babel
 certifi
+python-dotenv