import express from 'express';
import { WebSocketServer } from 'ws';
import { createServer } from 'http';
import { join, dirname, basename } from 'path';
import { fileURLToPath } from 'url';
import { readFile, writeFile, stat } from 'fs/promises';
import { EventEmitter } from 'events';
import { readFileSync, writeFileSync } from 'fs';
import {
  setBroadcast, addEvent, getEvents, getAllMemories, getSkillsFromDb, toggleSkill,
  deleteEventsUpTo, deleteAllChatEvents, getChatCount, deleteMemory, getDailyUsage, getDailyTotalCost, getTasks, markProcessed,
  // v3.0 — KG
  addEntity, getEntity, findEntities, getEntitiesByType, getRelations, getOutgoingRelations, addRelation, getEntityCount, getRelationCount, getDb,
  // v3.0 — Workspaces
  addWorkspace, getWorkspace, listWorkspacesFromDb, setActiveWorkspace, getActiveWorkspace, deleteWorkspaceFromDb,
  // v3.0 — Ingestion
  addIngestionItem, getIngestionQueue,
  // v3.0 — Cognitive
  getCognitiveLogs,
  // v4.x — Procedural Memory
  getProceduralMemories
} from './db.mjs';
import { getConfig, updateConfig, getAgentRoot } from './config.mjs';
import { getWorkspaceRoot, listWorkspaceFiles, writeWorkspaceBuffer } from './workspace.mjs';
import { isSTTAvailable, speechToText, transcribeViaApi, ensureWhisperBinary, ensureWhisperModel, resetSTTCache } from './multimodal/speech.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));

let wss;
const clients = new Set();
export const serverEvents = new EventEmitter();
let isShuttingDown = false;
let runtimeApi = {
  getTaskSnapshot: () => null,
  cancelTask: () => false,
  retryTask: () => null,
  setLoopEnabled: () => false,
  getLoopEnabled: () => true,
  getAgentState: () => ({ conversation_active: false, autonomy_paused: false })
};

export function setRuntimeApi(handlers = {}) {
  runtimeApi = { ...runtimeApi, ...handlers };
}

function _isNewerVersion(remote, local) {
  const parse = v => String(v || '0.0.0').split('.').map(n => parseInt(n, 10) || 0);
  const r = parse(remote);
  const l = parse(local);
  for (let i = 0; i < Math.max(r.length, l.length); i++) {
    const rv = r[i] || 0;
    const lv = l[i] || 0;
    if (rv > lv) return true;
    if (rv < lv) return false;
  }
  return false;
}

export function startServer() {
  const app = express();
  app.use(express.json({ limit: '10mb' }));

  // Serve index.html (v4 UI) with auth token injected
  app.get('/', async (req, res) => {
    try {
      const html = await readFile(join(__dirname, 'public', 'index.html'), 'utf-8');
      const { authToken } = getConfig();
      const tokenMeta = authToken ? `<meta name="auth-token" content="${authToken}">` : '';
      res.type('html').send(html.replace('</head>', `${tokenMeta}\n</head>`));
    } catch (err) {
      res.status(500).send('Could not load UI');
    }
  });

  app.use(express.static(join(__dirname, 'public'), { maxAge: 0, etag: false }));
  app.use('/lib', express.static(join(__dirname, 'node_modules/marked/lib'), { maxAge: '1h' }));

  // Version — no auth required
  app.get('/api/version', async (req, res) => {
    try {
      const content = await readFile(join(__dirname, 'version.json'), 'utf-8');
      res.json(JSON.parse(content));
    } catch {
      res.json({ version: 'unknown' });
    }
  });

  // Health check — no auth required
  app.get('/api/health', async (req, res) => {
    let version = 'unknown';
    try {
      const content = await readFile(join(__dirname, 'version.json'), 'utf-8');
      version = JSON.parse(content).version || 'unknown';
    } catch {}
    const mem = process.memoryUsage();
    res.json({
      status: 'ok',
      version,
      uptime: process.uptime(),
      memory: { rss: mem.rss, heapUsed: mem.heapUsed, heapTotal: mem.heapTotal },
      timestamp: Date.now()
    });
  });

  // Reject new requests during graceful shutdown
  app.use('/api', (req, res, next) => {
    if (isShuttingDown) return res.status(503).json({ error: 'Server is shutting down' });
    next();
  });

  // Auth middleware for API routes
  app.use('/api', (req, res, next) => {
    const { authToken } = getConfig();
    if (!authToken) return next();
    const bearer = req.headers.authorization?.replace('Bearer ', '');
    const queryToken = req.query.token;
    if (bearer === authToken || queryToken === authToken) return next();
    res.status(401).json({ error: 'Unauthorized' });
  });

  // Serve agent workspace files — always fresh (maxAge: 0)
  app.get('/workspace-files/:name', (req, res) => {
    const safeName = basename(req.params.name || '');
    if (!safeName) {
      return res.status(400).send('Invalid file name');
    }
    const root = getWorkspaceRoot();
    res.sendFile(join(root, safeName), { maxAge: 0 }, (err) => {
      if (err) {
        if (!res.headersSent) res.status(404).send('File not found');
      }
    });
  });

  // Events timeline
  app.get('/api/events', (req, res) => {
    const { type, limit = 50, offset = 0, since } = req.query;
    const safeLimit = Math.max(1, Math.min(500, parseInt(limit, 10) || 50));
    const safeOffset = Math.max(0, parseInt(offset, 10) || 0);
    const events = getEvents({
      type: type || undefined,
      limit: safeLimit,
      offset: safeOffset,
      since: since || undefined
    });
    res.json(events);
  });

  // Log a custom event (e.g. user feedback)
  app.post('/api/events', (req, res) => {
    const { type = 'custom', source = 'user', content = '', metadata } = req.body || {};
    const event = addEvent(String(type), String(source), String(content), metadata || undefined);
    res.json(event);
  });

  // Memories
  app.get('/api/memories', (req, res) => {
    const { category } = req.query;
    res.json(getAllMemories(category || undefined));
  });

  // Procedural Memory
  app.get('/api/memories/procedural', (req, res) => {
    const { skill, limit } = req.query;
    const safeLimit = Math.max(1, Math.min(200, parseInt(limit, 10) || 50));
    res.json(getProceduralMemories(skill || undefined, safeLimit));
  });

  // Skills
  app.get('/api/skills', (req, res) => {
    res.json(getSkillsFromDb());
  });

  app.post('/api/skills/:name/toggle', (req, res) => {
    const { enabled } = req.body;
    toggleSkill(req.params.name, enabled);
    res.json({ ok: true });
  });

  // Chat — user sends a message
  app.post('/api/chat', (req, res) => {
    const { message } = req.body;
    if (!message || !message.trim()) {
      return res.status(400).json({ error: 'Message required' });
    }
    const event = addEvent('chat', 'user', message.trim());
    res.json(event);
    // Notify agent to process immediately
    serverEvents.emit('new-chat');
  });

  // Clear chat messages up to and including a given event ID
  app.post('/api/chat/clear', (req, res) => {
    const { upToId } = req.body;
    if (!upToId) return res.status(400).json({ error: 'upToId required' });
    const deleted = deleteEventsUpTo(upToId);
    // Broadcast so all clients refresh
    broadcastToClients({ type: 'chat_cleared', data: { upToId, deleted } });
    res.json({ ok: true, deleted });
  });

  // Clear all chat messages (New Chat)
  app.post('/api/chat/clear-all', (req, res) => {
    const deleted = deleteAllChatEvents();
    broadcastToClients({ type: 'chat_cleared', data: { all: true, deleted } });
    res.json({ ok: true, deleted });
  });

  // Get chat message count
  app.get('/api/chat/count', (req, res) => {
    res.json({ count: getChatCount() });
  });

  // File upload (always to workspace)
  app.post('/api/upload', async (req, res) => {
    try {
      const { filename, content, encoding } = req.body;
      if (!filename || !content) {
        return res.status(400).json({ error: 'filename and content required' });
      }
      const buf = encoding === 'base64' ? Buffer.from(content, 'base64') : Buffer.from(content, 'utf-8');
      const written = await writeWorkspaceBuffer(filename, buf);
      const event = addEvent('file', 'user', `Uploaded file: ${written.name}`, {
        path: written.publicPath,
        size: buf.length,
        destination: 'workspace'
      });
      res.json(event);
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // List workspace files
  app.get('/api/workspace-files', async (req, res) => {
    try {
      const files = await listWorkspaceFiles();
      res.json(files);
    } catch {
      res.json([]);
    }
  });

  // Get workspace root path (for "open folder" feature)
  app.get('/api/workspace-root', (req, res) => {
    res.json({ path: getWorkspaceRoot() });
  });

  // Download a workspace file with Content-Disposition: attachment
  app.get('/api/workspace-files/:name/download', (req, res) => {
    const safeName = basename(req.params.name || '');
    if (!safeName) return res.status(400).send('Invalid file name');
    const root = getWorkspaceRoot();
    res.download(join(root, safeName), safeName, (err) => {
      if (err && !res.headersSent) res.status(404).send('File not found');
    });
  });

  app.get('/api/journal', async (req, res) => {
    try {
      const content = await readFile(join(__dirname, 'journal.md'), 'utf-8');
      res.json({ content, exists: true });
    } catch {
      res.json({ content: '', exists: false });
    }
  });

  // Heartbeat status + meta mode
  const metaFlagPath = join(__dirname, 'workspaces', '.meta-mode');

  app.get('/api/status', async (req, res) => {
    try {
      const hbPath = join(__dirname, '.heartbeat');
      const hbStat = await stat(hbPath);
      const ageSeconds = (Date.now() - hbStat.mtimeMs) / 1000;
      let metaMode = false;
      try {
        const flag = await readFile(metaFlagPath, 'utf-8');
        metaMode = flag.trim() === 'enabled';
      } catch {}
      const config = getConfig();
      const queue = runtimeApi.getTaskSnapshot ? runtimeApi.getTaskSnapshot() : null;
      const agentState = runtimeApi.getAgentState ? runtimeApi.getAgentState() : {};
      const loopEnabled = runtimeApi.getLoopEnabled ? !!runtimeApi.getLoopEnabled() : true;
      res.json({
        alive: ageSeconds < 600,
        heartbeat_age: Math.round(ageSeconds),
        meta_mode: metaMode,
        agentName: config.agentName,
        conversation_active: !!agentState.conversation_active,
        autonomy_paused: !!agentState.autonomy_paused,
        loop_enabled: loopEnabled,
        queue_depths: queue
      });
    } catch {
      res.json({ alive: false, heartbeat_age: null, meta_mode: false });
    }
  });

  app.get('/api/meta-mode', async (req, res) => {
    try {
      const flag = await readFile(metaFlagPath, 'utf-8');
      res.json({ enabled: flag.trim() === 'enabled' });
    } catch {
      res.json({ enabled: false });
    }
  });

  // Config endpoints
  app.get('/api/config', (req, res) => {
    const config = getConfig();
    const { authToken, ...safe } = config;
    // Mask API keys
    if (safe.agentic && safe.agentic.apiKey) {
      safe.agentic = { ...safe.agentic, apiKey: safe.agentic.apiKey.slice(0, 8) + '****' };
    }
    if (safe.voice && safe.voice.xaiApiKey) {
      safe.voice = { ...safe.voice, xaiApiKey: safe.voice.xaiApiKey.slice(0, 8) + '****' };
    }
    // Expose firstRun flag: true until user explicitly completes onboarding wizard
    // Also re-trigger if userName was never set (upgrade from pre-v5.2.5)
    safe.firstRun = config.firstRun !== false || !config.userName;
    res.json(safe);
  });

  app.put('/api/config', (req, res) => {
    try {
      const updates = req.body;
      const allowed = ['cli', 'port', 'loopMinutes', 'compactThreshold', 'compactKeepRecent', 'agentName', 'models', 'dailyBudget', 'workspaceRoot', 'maxUserTaskConcurrency', 'autonomyActiveChatPauseMs', 'desktopHotkey',
        /* v3.0 */ 'activeWorkspace', 'workspaces', 'ingestion', 'cognitive', 'multimodal', 'kgEnabled',
        /* v3.1 */ 'voice',
        /* v4.0 */ 'persona', 'vector',
        /* v4.2 */ 'agentic',
        /* v4.2 */ 'firstRun',
        /* v5.2 */ 'userName', 'autonomy', 'schedules', 'notifications', 'logLevel', 'accentColor'];
      const filtered = {};
      for (const key of allowed) {
        if (key in updates) filtered[key] = updates[key];
      }
      const newConfig = updateConfig(filtered);
      const { authToken, ...safe } = newConfig;
      res.json({ ok: true, config: safe, restartRequired: false });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // Delete a memory
  app.delete('/api/memories/:key', (req, res) => {
    deleteMemory(decodeURIComponent(req.params.key));
    res.json({ ok: true });
  });

  // Usage tracking
  app.get('/api/usage', (req, res) => {
    const { date } = req.query;
    const usage = getDailyUsage(date);
    const total = getDailyTotalCost(date);
    const { dailyBudget } = getConfig();
    res.json({ usage, total, budget: dailyBudget });
  });

  app.get('/api/tasks', (req, res) => {
    const limit = Math.max(1, Math.min(200, parseInt(req.query.limit || '100', 10) || 100));
    res.json(getTasks({ limit }));
  });

  app.post('/api/tasks/:id/cancel', (req, res) => {
    const taskId = parseInt(req.params.id, 10);
    if (!taskId) return res.status(400).json({ error: 'invalid task id' });
    const ok = runtimeApi.cancelTask ? runtimeApi.cancelTask(taskId) : false;
    if (!ok) return res.status(404).json({ error: 'task not cancelable' });
    res.json({ ok: true });
  });

  app.post('/api/tasks/:id/retry', (req, res) => {
    const taskId = parseInt(req.params.id, 10);
    if (!taskId) return res.status(400).json({ error: 'invalid task id' });
    const newTaskId = runtimeApi.retryTask ? runtimeApi.retryTask(taskId) : null;
    if (!newTaskId) return res.status(404).json({ error: 'task not retriable' });
    res.json({ ok: true, taskId: newTaskId });
  });

  app.get('/api/loop', (req, res) => {
    const enabled = runtimeApi.getLoopEnabled ? !!runtimeApi.getLoopEnabled() : true;
    res.json({ enabled });
  });

  app.post('/api/loop', (req, res) => {
    if (!runtimeApi.setLoopEnabled) {
      return res.status(503).json({ error: 'Loop control unavailable' });
    }
    const enabled = !!req.body?.enabled;
    const updated = runtimeApi.setLoopEnabled(enabled);
    res.json({ ok: true, enabled: !!updated });
  });

  app.post('/api/loop/pause', (req, res) => {
    if (!runtimeApi.setLoopEnabled) {
      return res.status(503).json({ error: 'Loop control unavailable' });
    }
    const updated = runtimeApi.setLoopEnabled(false);
    res.json({ ok: true, enabled: !!updated });
  });

  app.post('/api/loop/resume', (req, res) => {
    if (!runtimeApi.setLoopEnabled) {
      return res.status(503).json({ error: 'Loop control unavailable' });
    }
    const updated = runtimeApi.setLoopEnabled(true);
    res.json({ ok: true, enabled: !!updated });
  });

  // ── v3.0 Workspace endpoints ─────────────────────────────────

  app.get('/api/workspaces', async (req, res) => {
    try {
      const workspaces = listWorkspacesFromDb();
      // Enrich each workspace with a live file count from the filesystem
      const enriched = await Promise.all(workspaces.map(async (ws) => {
        try {
          const files = await listWorkspaceFiles(ws.id);
          return { ...ws, fileCount: files.length };
        } catch {
          return { ...ws, fileCount: 0 };
        }
      }));
      res.json(enriched);
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.post('/api/workspaces', (req, res) => {
    try {
      const { name, displayName, rootPath, description } = req.body;
      if (!name) return res.status(400).json({ error: 'name required' });
      addWorkspace(name, displayName || name, rootPath || '', description || '');
      res.json({ ok: true, workspace: getWorkspace(name) });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.put('/api/workspaces/:id/activate', (req, res) => {
    try {
      const id = req.params.id;
      const ws = getWorkspace(id);
      if (!ws) return res.status(404).json({ error: 'workspace not found' });
      setActiveWorkspace(id);
      updateConfig({ activeWorkspace: id });
      res.json({ ok: true, active: id });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.delete('/api/workspaces/:id', (req, res) => {
    try {
      const id = req.params.id;
      if (id === 'default') return res.status(400).json({ error: 'cannot delete default workspace' });
      deleteWorkspaceFromDb(id);
      res.json({ ok: true });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // ── v3.0 Knowledge Graph endpoints ─────────────────────────

  app.get('/api/kg/entities', (req, res) => {
    try {
      const { q, type, workspace, limit } = req.query;
      const safeLimit = Math.max(1, Math.min(500, parseInt(limit, 10) || 50));
      if (q) {
        res.json(findEntities(q, { workspaceId: workspace, type, limit: safeLimit }));
      } else if (type) {
        res.json(getEntitiesByType(type, workspace));
      } else {
        // Return all entities up to limit
        res.json(findEntities('%', { workspaceId: workspace, limit: safeLimit }));
      }
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.get('/api/kg/entities/:id/relations', (req, res) => {
    try {
      const id = parseInt(req.params.id, 10);
      if (!id) return res.status(400).json({ error: 'invalid entity id' });
      res.json(getRelations(id));
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.post('/api/kg/entities', (req, res) => {
    try {
      const { name, type, workspace, metadata } = req.body;
      if (!name || !type) return res.status(400).json({ error: 'name and type required' });
      const id = addEntity(name, type, workspace || 'default', metadata);
      res.json({ ok: true, id, entity: getEntity(id) });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // Graph endpoint — returns all entities + relations for force-directed visualization
  app.get('/api/kg/graph', (req, res) => {
    try {
      const { workspace, limit } = req.query;
      const safeLimit = Math.max(1, Math.min(200, parseInt(limit, 10) || 150));
      const entities = findEntities('%', { workspaceId: workspace, limit: safeLimit });
      const entityIds = new Set(entities.map(e => e.id));
      const relations = getDb().prepare(`
        SELECT r.id, r.source_id, r.target_id, r.relation, r.weight,
               s.name AS sourceName, t.name AS targetName
        FROM kg_relations r
        JOIN kg_entities s ON r.source_id = s.id
        JOIN kg_entities t ON r.target_id = t.id
      `).all().filter(r => entityIds.has(r.source_id) && entityIds.has(r.target_id));
      res.json({ entities, relations });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.get('/api/kg/stats', (req, res) => {
    try {
      const typeRows = getDb().prepare('SELECT type, COUNT(*) as count FROM kg_entities GROUP BY type').all();
      const types = Object.fromEntries(typeRows.map(r => [r.type, r.count]));
      res.json({
        totalEntities: getEntityCount(),
        totalRelations: getRelationCount(),
        types
      });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.get('/api/kg/export', (req, res) => {
    try {
      const db = getDb();
      const entities = db.prepare('SELECT * FROM kg_entities').all();
      const relations = db.prepare('SELECT * FROM kg_relations').all();
      res.json({ entities, relations });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.post('/api/kg/import', (req, res) => {
    try {
      const { entities = [], relations = [] } = req.body;
      if (!Array.isArray(entities) || !Array.isArray(relations)) {
        return res.status(400).json({ error: 'entities and relations must be arrays' });
      }
      const db = getDb();
      const upsertEntity = db.prepare(`
        INSERT INTO kg_entities (id, name, type, workspace_id, metadata, summary, updated_at)
        VALUES (@id, @name, @type, @workspace_id, @metadata, @summary, @updated_at)
        ON CONFLICT(id) DO UPDATE SET
          name=excluded.name, type=excluded.type, workspace_id=excluded.workspace_id,
          metadata=excluded.metadata, summary=excluded.summary, updated_at=excluded.updated_at
      `);
      const upsertRelation = db.prepare(`
        INSERT INTO kg_relations (id, source_id, target_id, relation, weight, metadata, created_at)
        VALUES (@id, @source_id, @target_id, @relation, @weight, @metadata, @created_at)
        ON CONFLICT(id) DO UPDATE SET
          source_id=excluded.source_id, target_id=excluded.target_id, relation=excluded.relation,
          weight=excluded.weight, metadata=excluded.metadata
      `);
      const importAll = db.transaction(() => {
        for (const e of entities) upsertEntity.run(e);
        for (const r of relations) upsertRelation.run(r);
      });
      importAll();
      res.json({ ok: true, entitiesImported: entities.length, relationsImported: relations.length });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // ── v3.0 Ingestion endpoints ───────────────────────────────

  app.post('/api/ingest', (req, res) => {
    try {
      const { sourceType, sourcePath, workspaceId } = req.body;
      if (!sourceType || !sourcePath) return res.status(400).json({ error: 'sourceType and sourcePath required' });
      const id = addIngestionItem(sourceType, sourcePath, workspaceId || 'default');
      res.json({ ok: true, id });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.get('/api/ingest/queue', (req, res) => {
    try {
      const { state } = req.query;
      res.json(getIngestionQueue(state));
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // ── v3.0 Tools Registry endpoints ──────────────────────────

  app.get('/api/tools', (req, res) => {
    try {
      const toolsPath = join(getAgentRoot(), 'tools.md');
      const content = readFileSync(toolsPath, 'utf-8');
      res.json({ content });
    } catch (err) {
      res.json({ content: '', error: err.message });
    }
  });

  app.put('/api/tools', (req, res) => {
    try {
      const { content } = req.body;
      if (typeof content !== 'string') return res.status(400).json({ error: 'content required' });
      const toolsPath = join(getAgentRoot(), 'tools.md');
      writeFileSync(toolsPath, content, 'utf-8');
      res.json({ ok: true });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // ── v3.0 Cognitive Log endpoint ────────────────────────────

  app.get('/api/cognitive/logs', (req, res) => {
    try {
      const { module: mod, limit } = req.query;
      const safeLimit = Math.max(1, Math.min(200, parseInt(limit, 10) || 50));
      res.json(getCognitiveLogs(mod, safeLimit));
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // ── v3.1 Voice endpoints ────────────────────────────────────

  app.get('/api/voice/status', (req, res) => {
    const state = runtimeApi.getVoiceState ? runtimeApi.getVoiceState() : { active: false, state: 'idle' };
    res.json(state);
  });

  app.post('/api/voice/toggle', async (req, res) => {
    const { enabled } = req.body;
    if (!runtimeApi.toggleVoice) {
      return res.status(503).json({ error: 'Voice control unavailable' });
    }
    try {
      const active = await runtimeApi.toggleVoice(!!enabled);
      res.json({ ok: true, active });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.post('/api/voice/listen', (req, res) => {
    if (!runtimeApi.voiceHotkey) {
      return res.status(503).json({ error: 'Voice control unavailable' });
    }
    try {
      runtimeApi.voiceHotkey();
      res.json({ ok: true });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // Transcribe audio — tries OpenAI Whisper API first, then local whisper-cli (auto-downloads model)
  app.post('/api/voice/transcribe', async (req, res) => {
    const { audio, mimeType } = req.body;
    if (!audio) return res.status(400).json({ error: 'audio (base64) required' });

    try {
      const { writeFile: writeFileAsync, unlink } = await import('fs/promises');
      const { tmpdir } = await import('os');
      const ext = (mimeType || '').includes('webm') ? '.webm'
        : (mimeType || '').includes('ogg') ? '.ogg'
        : (mimeType || '').includes('mp4') ? '.m4a'
        : '.wav';
      const tmpPath = join(tmpdir(), `aido-voice-${Date.now()}${ext}`);
      const buf = Buffer.from(audio, 'base64');
      await writeFileAsync(tmpPath, buf);

      let transcript = null;

      // Strategy 1: OpenAI-compatible Whisper API (if API key available)
      const config = getConfig();
      const provider = config.agentic?.provider;
      const apiKey = config.agentic?.apiKey;
      if (apiKey && (provider === 'openai' || provider === 'groq')) {
        const baseUrl = provider === 'groq'
          ? 'https://api.groq.com/openai'
          : (config.agentic?.baseUrl || 'https://api.openai.com');
        transcript = await transcribeViaApi(tmpPath, { apiKey, baseUrl });
        if (transcript) {
          console.log(`[voice] Transcribed via ${provider} API: "${transcript.slice(0, 80)}"`);
        }
      }

      // Strategy 2: Local whisper-cli (auto-downloads binary + model on first use)
      if (!transcript) {
        // If no STT backend yet, try to auto-download whisper.cpp
        if (!isSTTAvailable()) {
          try {
            broadcastToClients({ type: 'voice', data: { state: 'downloading', detail: 'Downloading speech recognition engine...' } });
            await ensureWhisperBinary((msg) => {
              broadcastToClients({ type: 'voice', data: { state: 'downloading', detail: msg } });
            });
            broadcastToClients({ type: 'voice', data: { state: 'downloading', detail: 'Downloading speech recognition model (~142 MB)...' } });
            await ensureWhisperModel();
            resetSTTCache(); // re-check availability with new binary
          } catch (dlErr) {
            console.warn(`[voice] Whisper auto-download failed: ${dlErr.message}`);
          }
        }

        if (isSTTAvailable()) {
          broadcastToClients({ type: 'voice', data: { state: 'processing', detail: 'Transcribing audio...' } });
          transcript = await speechToText(tmpPath);
          if (transcript) {
            console.log(`[voice] Transcribed via local whisper: "${transcript.slice(0, 80)}"`);
          }
        }
      }

      // Clean up temp file
      try { await unlink(tmpPath); } catch {}

      if (!transcript) {
        return res.status(503).json({
          error: 'No speech-to-text backend available. Configure OpenAI or Groq API key in Settings, or install whisper-cli locally.'
        });
      }

      res.json({ ok: true, transcript });
    } catch (err) {
      console.error('[voice] Transcription error:', err.message);
      res.status(500).json({ error: err.message });
    }
  });

  // ── Voice Agent ephemeral token (xAI Grok + OpenAI) ──────────
  app.post('/api/voice/grok-token', async (req, res) => {
    const config = getConfig();
    const voiceProvider = config.voice?.provider || 'xai';

    // Resolve instructions for the voice session
    let instructions = '';
    try {
      const { getPersonaForPrompt } = await import('./persona.mjs');
      instructions = await getPersonaForPrompt();
    } catch {
      instructions = 'You are a helpful AI assistant.';
    }

    // Try direct API key first (user's own key)
    const apiKey = voiceProvider === 'openai'
      ? config.voice?.openaiApiKey
      : config.voice?.xaiApiKey;

    // Fall back to worker proxy for voice
    if (!apiKey) {
      const worker = config.worker;
      if (worker?.url && worker?.token) {
        return res.json({
          ok: true,
          useWorkerProxy: true,
          workerUrl: worker.url + '/api/v1/voice/realtime',
          workerToken: worker.token,
          voiceProvider,
          instructions,
          voice: config.voice?.xaiVoice || 'Sal'
        });
      }
      return res.status(400).json({
        error: `No ${voiceProvider === 'openai' ? 'OpenAI' : 'xAI'} API key configured. Set it in Settings > Voice.`
      });
    }

    try {
      // Provider-specific token endpoint
      const tokenUrl = voiceProvider === 'openai'
        ? 'https://api.openai.com/v1/realtime/sessions'
        : 'https://api.x.ai/v1/realtime/client_secrets';

      const response = await fetch(tokenUrl, {
        method: 'POST',
        headers: {
          'Authorization': `Bearer ${apiKey}`,
          'Content-Type': 'application/json'
        },
        body: JSON.stringify({ expires_after: { seconds: 300 } })
      });

      if (!response.ok) {
        const text = await response.text();
        return res.status(response.status).json({ error: `${voiceProvider} API error: ${text}` });
      }

      const data = await response.json();
      console.log(`[voice] ${voiceProvider} token response:`, JSON.stringify(data));

      res.json({
        ok: true,
        token: data,
        voiceProvider,
        instructions,
        voice: config.voice?.xaiVoice || 'Sal'
      });
    } catch (err) {
      console.error('[voice] Token error:', err.message);
      res.status(500).json({ error: err.message });
    }
  });

  // ── Grok Voice → Chat sync ───────────────────────────────────
  // Persists a safe plain-text summary of the Grok voice conversation into
  // the chat log (never raw assistant text, which may contain HTML/code).
  // Then triggers skill detection with the full context so the agent can
  // take action on what was discussed.
  app.post('/api/voice/grok-sync', async (req, res) => {
    const { conversation } = req.body;
    if (!Array.isArray(conversation) || conversation.length === 0) {
      return res.status(400).json({ error: 'conversation array required' });
    }

    try {
      const userMsgs = conversation.filter(e => e.role === 'user' && e.text?.trim());

      // Build a concise summary of what the user discussed
      const topicHints = userMsgs.map(e => e.text.trim()).join('; ').slice(0, 300);
      const summaryText = `Voice conversation ended. The user said: ${topicHints}`;

      // Persist a single summary event — safe plain text only.
      // Note: We do NOT trigger skill detection here. Grok's tools
      // (delegate_task, send_message) already handle action delegation
      // during the voice session. This sync just records the summary.
      addEvent('chat', 'agent', summaryText, {
        source: 'grok-voice-summary',
        turnCount: conversation.length
      });

      res.json({ ok: true, synced: conversation.length });
    } catch (err) {
      console.error('[voice/grok-sync] Error:', err.message);
      res.status(500).json({ error: err.message });
    }
  });

  // ── Grok Voice tool execution ────────────────────────────────
  // Executes tools called by Grok during a voice session.
  // Tools let Grok delegate actions (build page, create file, etc.)
  // to the main agent instead of generating code/HTML in voice.
  app.post('/api/voice/grok-tool', async (req, res) => {
    const { tool, args } = req.body;
    if (!tool) {
      return res.status(400).json({ error: 'tool name required' });
    }

    try {
      switch (tool) {
        case 'delegate_task': {
          // Queue a specific skill by name
          const skillName = args?.skill_name;
          const description = args?.description || '';
          if (!skillName) {
            return res.json({ ok: false, error: 'skill_name required' });
          }
          if (runtimeApi.queueSkill) {
            const result = runtimeApi.queueSkill(skillName, { topic: description });
            // Post a visible notification to main chat so the user knows
            // something is being built — this broadcasts via WebSocket.
            addEvent('chat', 'agent',
              `🔧 Task queued from voice: **${skillName}** — ${description.slice(0, 200)}`,
              { source: 'grok-voice-task', skillName, taskId: result.taskId }
            );
            return res.json({ ok: true, message: `Task queued: ${skillName} (#${result.taskId})`, taskId: result.taskId });
          }
          return res.json({ ok: false, error: 'Agent not ready' });
        }

        case 'send_message': {
          // Inject a message into the main chat for context continuity.
          // Always use source='grok-voice' so the agent loop's
          // getUnprocessedChats() (which filters source='user') never
          // picks these up. Individual voice utterances should NOT
          // trigger separate LLM calls — the sync summary handles that.
          const message = args?.message;
          if (!message) {
            return res.json({ ok: false, error: 'message required' });
          }
          const displayRole = args?.role === 'user' ? 'user' : 'agent';
          addEvent('chat', 'grok-voice', message.trim(), {
            source: 'grok-voice',
            displayRole
          }, { silent: true });
          return res.json({ ok: true, message: 'Message added to chat' });
        }

        default:
          return res.json({ ok: false, error: `Unknown tool: ${tool}` });
      }
    } catch (err) {
      console.error('[voice/grok-tool] Error:', err.message);
      res.status(500).json({ error: err.message });
    }
  });

  // ── v4.0 SSE Streaming Chat endpoint ─────────────────────────

  app.post('/api/chat/stream', async (req, res) => {
    const { message, sessionId: requestSessionId } = req.body;
    if (!message || !message.trim()) {
      return res.status(400).json({ error: 'Message required' });
    }

    // Track client disconnect so we can abort server-side processing.
    // IMPORTANT: Use res.on('close'), NOT req.on('close'). Express has
    // already consumed the request body, so req may emit 'close' immediately.
    let clientDisconnected = false;
    res.on('close', () => { clientDisconnected = true; });

    // v4.2 — Session management: create or reuse a session
    let sessionId = requestSessionId;
    try {
      const sessionStore = await import('./session-store.mjs');
      if (!sessionId) {
        sessionId = sessionStore.createSession('chat');
      } else {
        // Verify session exists; create if not
        const existing = sessionStore.getSession(sessionId);
        if (!existing) {
          sessionId = sessionStore.createSession('chat');
        }
      }
      // Append user message to session store
      sessionStore.appendMessage(sessionId, 'user', message.trim());
    } catch (err) {
      console.error('[server] Session store error (non-fatal):', err.message);
    }

    // Record user message as event and immediately mark processed so the
    // agent loop doesn't also pick it up (avoiding duplicate skill execution).
    const userEvent = addEvent('chat', 'user', message.trim());
    markProcessed([userEvent.id]);
    // Note: we do NOT emit 'new-chat' — this endpoint handles the response
    // itself, then kicks off skill detection below to avoid duplication.

    // Set up SSE headers
    res.writeHead(200, {
      'Content-Type': 'text/event-stream',
      'Cache-Control': 'no-cache',
      'Connection': 'keep-alive',
      'X-Accel-Buffering': 'no'
    });

    try {
      let lastText = '';

      console.log('[server/stream] Building prompt...');
      const { streamLLM, streamChatLLM, hasStreamChat, pickModel } = await import('./llm.mjs');

      let prompt;
      if (runtimeApi.buildChatPrompt) {
        try {
          prompt = await runtimeApi.buildChatPrompt(message.trim());
        } catch (promptErr) {
          console.error('[server/stream] buildChatPrompt failed, using fallback:', promptErr.message);
          prompt = message.trim();
        }
      } else {
        prompt = message.trim();
      }

      // Smart chat streaming: Claude can read files, search code, and browse
      // the web — but cannot write files, run commands, or deploy anything.
      // MCP tools (Cloudflare, etc.) are always blocked.
      // Skill execution happens separately via detectAndQueueSkill after the
      // stream completes.
      const useStreamChat = hasStreamChat();
      const CHAT_TOOLS = ['Read', 'Glob', 'Grep', 'WebFetch', 'WebSearch'];
      const streamFn = useStreamChat
        ? () => streamChatLLM(prompt, { model: pickModel('chat'), allowedTools: CHAT_TOOLS })
        : () => streamLLM(prompt, { model: pickModel('chat') });

      console.log(`[server/stream] Prompt ready (${prompt.length} chars). Starting ${useStreamChat ? 'streamChatLLM' : 'streamLLM'}...`);
      if (clientDisconnected) { console.log('[server/stream] Client disconnected before stream'); res.end(); return; }

      let chunkCount = 0;
      for await (const chunk of streamFn()) {
        if (clientDisconnected) { console.log('[server/stream] Client disconnected during stream'); break; }
        chunkCount++;
        lastText = chunk.text;
        res.write(`data: ${JSON.stringify({ text: chunk.text, sessionId })}\n\n`);
        broadcastToClients({ type: 'stream', data: { text: chunk.text } });
      }
      console.log(`[server/stream] Stream complete. ${chunkCount} chunks, ${lastText.length} chars total.`);

      // Record agent response
      if (lastText) {
        const agentEvent = addEvent('chat', 'agent', lastText, { reply_to: userEvent.id, streamed: true });

        // Save assistant response to session store
        try {
          const sessionStore = await import('./session-store.mjs');
          sessionStore.appendMessage(sessionId, 'assistant', lastText);
        } catch {}

        if (!clientDisconnected) {
          res.write(`data: ${JSON.stringify({ id: agentEvent.id, sessionId, done: true })}\n\n`);
        }
      }

      if (!clientDisconnected) res.end();

      // Extract entities from user message into KG (non-blocking)
      try {
        const { getConfig: _getConfig } = await import('./config.mjs');
        if (_getConfig().kgEnabled !== false && message.trim().length > 30) {
          const { extractAndStore } = await import('./entity-extraction.mjs');
          extractAndStore(message.trim()).catch(() => {});
        }
      } catch {}

      // Detect and queue skill in the background (after stream completes)
      // Pass agent reply so skill detector has full conversation context
      if (runtimeApi.detectAndQueueSkill) {
        runtimeApi.detectAndQueueSkill(message.trim(), userEvent.id, lastText || '').catch(err => {
          console.error('[server/stream] Skill detection after stream failed:', err.message);
        });
      }
    } catch (err) {
      console.error('[server/stream] SSE stream error:', err.message, err.stack);
      if (!clientDisconnected) {
        res.write(`data: ${JSON.stringify({ error: err.message })}\n\n`);
        res.end();
      }
    }
  });

  // ── v4.0 Persona endpoints ──────────────────────────────────

  app.get('/api/persona', async (req, res) => {
    try {
      const { getPersonaConfig } = await import('./persona.mjs');
      res.json(getPersonaConfig());
    } catch (err) {
      // Fallback to config defaults
      const config = getConfig();
      res.json(config.persona || { name: 'AI-Do', greeting: 'Hello!', avatarStyle: 'default', traits: {} });
    }
  });

  app.put('/api/persona', async (req, res) => {
    try {
      const { updatePersona } = await import('./persona.mjs');
      const updated = await updatePersona(req.body);
      res.json({ ok: true, persona: updated });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // ── Update check endpoint ────────────────────────────────────

  const UPDATE_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000;
  let _updateCheckCache = null; // { result, checkedAt }

  app.get('/api/update-check', async (req, res) => {
    const force = req.query.force === 'true';
    try {
      // Return cached result if within 24 hours and not forced
      if (!force && _updateCheckCache && (Date.now() - _updateCheckCache.checkedAt < UPDATE_CHECK_INTERVAL_MS)) {
        return res.json(_updateCheckCache.result);
      }

      const versionPath = join(__dirname, 'version.json');
      let localVersion = '0.0.0';
      try {
        const raw = await readFile(versionPath, 'utf-8');
        localVersion = JSON.parse(raw).version || '0.0.0';
      } catch {}

      const config = getConfig();
      const siteUrl = (config.siteUrl || 'https://ai-do-site.pages.dev').replace(/\/$/, '');

      let remoteVersion = null;
      let changelog = '';
      try {
        const fetchRes = await fetch(`${siteUrl}/version.json`, {
          signal: AbortSignal.timeout(8000)
        });
        if (fetchRes.ok) {
          const remote = await fetchRes.json();
          remoteVersion = remote.version || null;
          changelog = remote.changelog || '';
        }
      } catch (fetchErr) {
        console.warn('[server] Update check fetch failed:', fetchErr.message);
      }

      const available = remoteVersion ? _isNewerVersion(remoteVersion, localVersion) : false;
      const result = { available, localVersion, remoteVersion, changelog, checkedAt: Date.now() };
      _updateCheckCache = { result, checkedAt: Date.now() };
      res.json(result);
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // ── v4.0 Skill Palette endpoint ─────────────────────────────

  app.get('/api/skills/palette', (req, res) => {
    try {
      const dbSkills = getSkillsFromDb();
      // Internal skills hidden from the user palette
      const HIDDEN_SKILLS = new Set(['configure-agent']);
      // Categorize skills for the function palette
      const categoryMap = {
        'research': 'Search', 'analyze': 'Analyze', 'vision': 'Analyze',
        'build-page': 'Create', 'write-doc': 'Create', 'create-file': 'Create',
        'app-build': 'Create', 'diagram': 'Create', 'mockup': 'Create',
        'journal': 'Journal', 'ingest': 'Retrieve',
        'self-improve': 'Evolve', 'summarize': 'Analyze'
      };
      const iconMap = {
        'Search': 'search', 'Analyze': 'analyze', 'Create': 'create',
        'Journal': 'journal', 'Retrieve': 'retrieve', 'Configure': 'configure',
        'Evolve': 'evolve', 'Visualize': 'vision'
      };

      const palette = dbSkills
        .filter(s => !HIDDEN_SKILLS.has(s.name))
        .map(s => ({
          name: s.name,
          description: s.description,
          enabled: !!s.enabled,
          category: categoryMap[s.name] || 'Other',
          icon: iconMap[categoryMap[s.name]] || 'skills',
          displayName: s.name.replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase())
        }));

      res.json(palette);
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // ── v4.2 Background Task endpoints ──────────────────────────

  app.get('/api/tasks/background/questions', async (req, res) => {
    try {
      const { getPendingQuestions } = await import('./askuser-bridge.mjs');
      res.json(getPendingQuestions());
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.get('/api/tasks/background', async (req, res) => {
    try {
      const { listTasks } = await import('./background-queue.mjs');
      const { status, limit } = req.query;
      const safeLimit = Math.max(1, Math.min(200, parseInt(limit, 10) || 50));
      res.json(listTasks({ status: status || undefined, limit: safeLimit }));
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.post('/api/tasks/background', async (req, res) => {
    try {
      const { enqueueTask } = await import('./background-queue.mjs');
      const { subject, description, scheduledAt } = req.body;
      if (!subject) return res.status(400).json({ error: 'subject required' });
      const taskId = enqueueTask({ subject, description, scheduledAt });
      res.json({ ok: true, taskId });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.get('/api/tasks/background/:id', async (req, res) => {
    try {
      const { getTask } = await import('./background-queue.mjs');
      const id = parseInt(req.params.id, 10);
      if (!id) return res.status(400).json({ error: 'invalid task id' });
      const task = getTask(id);
      if (!task) return res.status(404).json({ error: 'task not found' });
      res.json(task);
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.post('/api/tasks/background/:id/answer', async (req, res) => {
    try {
      const { resolveAnswer } = await import('./askuser-bridge.mjs');
      const { questionId, answer } = req.body;
      if (!questionId || !answer) return res.status(400).json({ error: 'questionId and answer required' });
      const resolved = resolveAnswer(questionId, answer);
      if (!resolved) return res.status(404).json({ error: 'question not found or already answered' });
      res.json({ ok: true });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  // ── Scheduled Tasks endpoints ─────────────────────────────────

  app.get('/api/schedules', (req, res) => {
    res.json(getConfig().schedules || []);
  });

  app.post('/api/schedules', async (req, res) => {
    try {
      const { createSchedule, addSchedule } = await import('./scheduler.mjs');
      const { name, skill, interval, enabled } = req.body || {};
      if (!name || !skill || !interval) {
        return res.status(400).json({ error: 'name, skill, and interval are required' });
      }
      const schedule = createSchedule({ name, skill, interval, enabled });
      addSchedule(schedule);
      res.json(schedule);
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.put('/api/schedules/:id', async (req, res) => {
    try {
      const { updateSchedule } = await import('./scheduler.mjs');
      const updated = updateSchedule(req.params.id, req.body || {});
      if (!updated) return res.status(404).json({ error: 'schedule not found' });
      res.json(updated);
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  app.delete('/api/schedules/:id', async (req, res) => {
    try {
      const { deleteSchedule } = await import('./scheduler.mjs');
      deleteSchedule(req.params.id);
      res.json({ ok: true });
    } catch (err) {
      res.status(500).json({ error: err.message });
    }
  });

  const server = createServer(app);

  // WebSocket
  wss = new WebSocketServer({ server, path: '/ws', verifyClient: (info) => {
    const { authToken } = getConfig();
    if (!authToken) return true;
    const url = new URL(info.req.url, `http://${info.req.headers.host}`);
    return url.searchParams.get('token') === authToken;
  }});
  wss.on('connection', (ws) => {
    clients.add(ws);
    ws.on('close', () => clients.delete(ws));
    ws.on('error', () => clients.delete(ws));

    // Handle incoming WebSocket messages
    ws.on('message', async (raw) => {
      try {
        const msg = JSON.parse(raw.toString());
        if (msg.type === 'user_answer' && msg.questionId && msg.answer) {
          if (runtimeApi.resolveUserAnswer) {
            runtimeApi.resolveUserAnswer(msg.questionId, msg.answer);
          } else {
            try {
              const { resolveAnswer } = await import('./askuser-bridge.mjs');
              resolveAnswer(msg.questionId, msg.answer);
            } catch {}
          }
        }
      } catch {
        // Ignore malformed messages
      }
    });
  });

  // Wire up broadcast for db.mjs
  setBroadcast(broadcastToClients);

  const { port } = getConfig();
  server.listen(port, () => {
    console.log(`[server] Web UI: http://localhost:${port}`);
  });

  return server;
}

function broadcastToClients(msg) {
  const data = JSON.stringify(msg);
  for (const client of clients) {
    if (client.readyState === 1) {
      client.send(data);
    }
  }
}

/**
 * Broadcast a partial streaming response to all connected WebSocket clients.
 * Used by agent.mjs to push real-time chat updates.
 */
export function broadcastStream(msgId, partialText) {
  broadcastToClients({ type: 'stream', data: { id: msgId, text: partialText } });
}

/**
 * Broadcast agent status updates to all connected WebSocket clients.
 * Used to show real-time activity in the UI.
 */
export function broadcastStatus(status) {
  broadcastToClients({ type: 'status', data: status });
}

/**
 * Broadcast an activity event to all connected WebSocket clients.
 * Used by the v4 UI activity feed.
 */
export function broadcastActivity(kind, source, content, metadata = null) {
  broadcastToClients({
    type: 'activity',
    data: { kind, source, content, metadata, timestamp: Date.now() }
  });
}

/**
 * Broadcast skill lifecycle events to all connected WebSocket clients.
 */
export function broadcastSkillStart(name, args) {
  broadcastToClients({ type: 'skill_start', data: { name, args, timestamp: Date.now() } });
}

export function broadcastSkillComplete(name, result) {
  broadcastToClients({ type: 'skill_complete', data: { name, result, timestamp: Date.now() } });
}

/**
 * Broadcast voice activation state changes to all connected WebSocket clients.
 * Sends { type: 'voice', data: { state, ... } } so the browser can update the voice orb
 * and react to wake word detections.
 */
export function broadcastVoice(state, extra = {}) {
  broadcastToClients({ type: 'voice', data: { state, ...extra, timestamp: Date.now() } });
}

/**
 * Initiate graceful shutdown of the WebSocket server.
 * Sets the shutdown flag (new API requests get 503), broadcasts a shutdown
 * message to all connected clients, then closes all WS connections.
 */
export function shutdownServer() {
  isShuttingDown = true;
  broadcastToClients({ type: 'shutdown', data: { message: 'Server shutting down' } });
  if (wss) {
    for (const client of clients) {
      try { client.close(1001, 'Server shutting down'); } catch {}
    }
    wss.close();
  }
}
