import { readFile, writeFile } from 'fs/promises';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import os from 'os';
import { loadConfig, getConfig } from './config.mjs';
import { initDb, addEvent, getUnprocessedChats, getEvents, getChatCount, getOldestChats, replaceChatsWithSummary, markProcessed, addIngestionItem, isIngestionQueued, getDb } from './db.mjs';
import { startServer, serverEvents, broadcastStatus, broadcastVoice, broadcastActivity, setRuntimeApi, shutdownServer } from './server.mjs';
import { askLLM, pickModel, initLLM, checkBudget } from './llm.mjs';
import { remember, recall, recallAll, getContextForPrompt } from './memory.mjs';
import { loadSkills, runSkill, listSkills, skillCreatesFiles } from './skills.mjs';

import { parseLLMJson } from './json-utils.mjs';
import { loadEvolution, getEvolutionContext, getGoal, shouldAskAboutGoals, markGoalAsked, setGoal, addGoalCandidate, getQualityPolicy } from './evolution.mjs';
import { createTaskScheduler } from './task-scheduler.mjs';
import { listWorkspaceFiles, getWorkspaceRoot, initWorkspaceModule } from './workspace.mjs';

// v3.0 — Knowledge Graph, Cognitive, Ingestion, Workspaces
import { initKG, getKGContext, getKGStats } from './knowledge-graph.mjs';
import { processIngestionQueue } from './ingestion.mjs';
import { initWorkspaces, getActiveWorkspaceId } from './workspace-manager.mjs';
import { reflect, shouldReflect } from './cognitive/reflector.mjs';
import { createPlan, getActivePlans, getNextStep, completeStep, failStep } from './cognitive/planner.mjs';
import { decompose } from './cognitive/thinker.mjs';
import { startWatching, stopWatching, isWatching, startWatchFolders, stopWatchFolders, getWatchedFolders } from './file-watcher.mjs';
import { extractAndStore } from './entity-extraction.mjs';
import { findRelevantContext } from './semantic-search.mjs';
import { runDailyReflection, shouldRunReflection } from './meta-reflection.mjs';
import { checkSchedules } from './scheduler.mjs';

// v3.1 — Voice Activation
import { startVoiceActivation, stopVoiceActivation, isVoiceActive, getVoiceState, handleHotkeyActivation } from './voice-activation.mjs';

// Adaptive personalization
import { getAdaptiveContext } from './persona.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const config = loadConfig();
const LOOP_INTERVAL = config.loopMinutes * 60 * 1000;
const HEARTBEAT_PATH = join(__dirname, '.heartbeat');

let loopCount = 0;
let loopRunning = false;
let chatProcessing = false;
let shuttingDown = false;
let lastUserChatAt = 0;
let backgroundLoopEnabled = true;

// v4.3 — Autonomous creative loop tracking
let lastAutonomousActionAt = 0;
let autonomousDailyRunCount = 0;
let autonomousDailyRunDate = '';

// Track currently configured watch folders to detect config changes
let activeWatchFolders = [];

function applyWatchFolders(folders) {
  const next = (Array.isArray(folders) ? folders : []).filter(Boolean);
  const current = activeWatchFolders;
  const same = current.length === next.length && next.every((f, i) => f === current[i]);
  if (same) return;

  activeWatchFolders = next;
  const activeWorkspaceId = getActiveWorkspaceId?.() || 'default';

  startWatchFolders(next, (filePath, filename) => {
    if (isIngestionQueued(filePath)) {
      console.log(`[file-watcher] Already queued, skipping: ${filename}`);
      return;
    }
    addIngestionItem('file', filePath, activeWorkspaceId);
    addEvent('system', 'agent', `Auto-ingest queued: ${filename}`, { source: 'watch-folder', path: filePath });
    broadcastActivity('ingest', 'agent', `Queued for auto-ingestion: ${filename}`, { path: filePath });
    console.log(`[file-watcher] Auto-queued for ingestion: ${filePath}`);
  });
}

function setBackgroundLoopEnabled(enabled) {
  backgroundLoopEnabled = !!enabled;
  const stateLabel = backgroundLoopEnabled ? 'enabled' : 'paused';
  console.log(`[agent] Background loop ${stateLabel}`);
  addEvent('system', 'agent', `Background loop ${stateLabel}`);
  broadcastStatus({ state: 'loop_control', enabled: backgroundLoopEnabled, loop: loopCount });
  if (backgroundLoopEnabled && !loopRunning && !shuttingDown) {
    setTimeout(runLoop, 250);
  }
  return backgroundLoopEnabled;
}

function isConversationActive() {
  const { autonomyActiveChatPauseMs = 180000 } = getConfig();
  return Date.now() - lastUserChatAt < autonomyActiveChatPauseMs;
}

const scheduler = createTaskScheduler({
  maxUserTaskConcurrency: Number(getConfig().maxUserTaskConcurrency) || 2,
  isAutonomyPaused: () => isConversationActive(),
  onTaskUpdate: (snapshot) => {
    broadcastStatus({ state: 'task_queue', loop: loopCount, queue: snapshot });
  }
});

function queueSkillTask({ source, skillName, args = {}, conversationId = null, followUpChat = false }) {
  const taskArgs = { ...args, _taskSource: source };
  const createsFiles = skillCreatesFiles(skillName);
  return scheduler.enqueue({
    source,
    kind: 'skill',
    conversationId,
    skill: skillName,
    args: taskArgs,
    createsFiles,
    execute: async () => {
      console.log(`[agent] Running queued skill (${source}): ${skillName}`);
      broadcastStatus({ state: 'running_skill', skill: skillName, loop: loopCount });
      const result = await runSkill(skillName, taskArgs);
      broadcastStatus({ state: 'skill_complete', skill: skillName, loop: loopCount });
      if (followUpChat && result) {
        const followUp = typeof result === 'string' ? result : (result?.reply || `Finished running ${skillName}.`);
        addEvent('chat', 'agent', followUp);
      }
      return result;
    }
  });
}

function retryTaskById(taskId) {
  return scheduler.retry(taskId, (row) => {
    if (row.kind !== 'skill' || !row.skill) return null;
    let args = {};
    try { args = row.args_json ? JSON.parse(row.args_json) : {}; } catch {}
    return async () => {
      broadcastStatus({ state: 'running_skill', skill: row.skill, loop: loopCount });
      const result = await runSkill(row.skill, args);
      broadcastStatus({ state: 'skill_complete', skill: row.skill, loop: loopCount });
      return result;
    };
  });
}

async function touchHeartbeat() {
  await writeFile(HEARTBEAT_PATH, new Date().toISOString(), 'utf-8');
}

async function checkHealth() {
  const mem = process.memoryUsage();

  // Gather workspace and content stats
  let fileCount = 0;
  let skillCount = 0;
  let kgStats = { entities: 0, relations: 0 };
  try {
    const files = await listWorkspaceFiles();
    fileCount = files?.length || 0;
  } catch {}
  try {
    skillCount = listSkills().length;
  } catch {}
  try {
    kgStats = getKGStats() || kgStats;
  } catch {}

  const health = {
    agent_memory_mb: (mem.heapUsed / 1024 / 1024).toFixed(1),
    workspace_files: fileCount,
    skills: skillCount,
    kg_entities: kgStats.entities || kgStats.totalEntities || 0,
    kg_relations: kgStats.relations || kgStats.totalRelations || 0,
    loop_count: loopCount
  };

  // Only log health every 6 iterations (every 30 min) to avoid spam
  if (loopCount % 6 === 0) {
    addEvent('system', 'system',
      `Status: ${fileCount} files, ${skillCount} skills, ${health.kg_entities} memories, loop #${loopCount}`
    );
  }

  return health;
}

async function updateWakeState() {
  const now = new Date().toISOString().replace('T', ' ').slice(0, 19);
  const cfg = getConfig();

  // Gather workspace stats
  let fileCount = 0;
  try { const files = await listWorkspaceFiles(); fileCount = files?.length || 0; } catch {}

  // Gather skill count
  let skillCount = 0;
  try { skillCount = listSkills().length; } catch {}

  // Knowledge graph stats
  let kgStats = { entities: 0, relations: 0 };
  try { kgStats = getKGStats() || kgStats; } catch {}

  // Recent events
  const recentEvents = getEvents({ limit: 8 });
  const recentSummary = recentEvents.map(e => `- [${e.type}] ${e.content.slice(0, 100)}`).join('\n');

  // Key memories
  const memories = recallAll();
  const memSummary = memories.slice(0, 10).map(m => `- **${m.key}**: ${m.content.slice(0, 80)}`).join('\n');

  // Active goal
  const goal = getGoal();

  // Active tasks
  let taskSummary = '';
  try {
    const tasks = scheduler?.getSnapshot?.() || [];
    const running = tasks.filter(t => t.state === 'running');
    const queued = tasks.filter(t => t.state === 'queued');
    if (running.length || queued.length) {
      taskSummary = `- Running: ${running.map(t => t.skill || t.kind).join(', ') || 'none'}\n- Queued: ${queued.length}`;
    }
  } catch {}

  const state = `# Wake State
Last updated: ${now}

## Status
- Loop: #${loopCount} | Interval: ${cfg.loopMinutes}m
- Port: ${cfg.port} | Adapter: ${cfg.cli || 'gergy'}
- Workspace: ${fileCount} files | Skills: ${skillCount} loaded
- Memory: ${kgStats.entities} entities, ${kgStats.relations} relations
${goal ? `- Goal: ${goal}` : '- Goal: (none set)'}
${taskSummary ? `\n## Active Tasks\n${taskSummary}` : ''}

## Recent Activity
${recentSummary || '(none)'}

## Key Memories
${memSummary || '(none yet)'}

## Context Recovery
This file is a snapshot of agent state, updated each loop and on shutdown.
- personality.md — agent voice and behavior guidelines
- evolution.yaml — goals, projects, capabilities
- config.json — runtime configuration (adapter, models, port, features)
- agent.db — persistent storage (events, memories, tasks, knowledge graph)
`;

  await writeFile(join(__dirname, 'wake-state.md'), state, 'utf-8');
}

/**
 * Classify user message into a skill using the fast tier.
 * Returns { skill, args } or null if it's just chat.
 */
function sanitizeBasename(name, fallback = 'file') {
  const base = String(name || '')
    .trim()
    .split(/[\\/]/)
    .pop()
    ?.replace(/[^a-zA-Z0-9._-]/g, '-') || '';
  const cleaned = base.replace(/-+/g, '-').replace(/^-|-$/g, '').slice(0, 80);
  return cleaned || fallback;
}

function buildSkillArgs(skillName, message, configuredName, rawArgs) {
  const baseArgs = { topic: message, task: message, target: message };
  const args = rawArgs && typeof rawArgs === 'object' ? rawArgs : {};

  // For vision skill, extract the image filename from the LLM args or the message
  if (skillName === 'vision') {
    let target = typeof args.target === 'string' ? args.target.trim() : '';
    if (!target) {
      // Try to extract a filename from the user message
      const match = message.match(/\b([\w.-]+\.(?:png|jpg|jpeg|gif|webp|bmp|svg))\b/i);
      if (match) target = match[1];
    }
    return {
      ...baseArgs,
      ...(target ? { target } : {}),
      ...(typeof args.topic === 'string' ? { topic: args.topic.slice(0, 2000) } : {})
    };
  }

  // For ingest skill, use the extracted source (filename/URL) from LLM classification
  if (skillName === 'ingest') {
    const source = typeof args.source === 'string' ? args.source.trim() : '';
    return {
      ...baseArgs,
      // Override topic/target with just the source, not the full message
      ...(source ? { topic: source, target: source } : {}),
      ...(typeof args.topic === 'string' ? { topic: args.topic.slice(0, 2000) } : {})
    };
  }

  if (skillName !== 'create-file') {
    return {
      ...baseArgs,
      ...(typeof args.topic === 'string' ? { topic: args.topic.slice(0, 2000) } : {}),
      ...(typeof args.task === 'string' ? { task: args.task.slice(0, 2000) } : {}),
      ...(typeof args.target === 'string' ? { target: args.target.slice(0, 2000) } : {})
    };
  }

  const lower = String(message || '').toLowerCase();
  const explicitFormat = typeof args.format === 'string' ? args.format.trim().toLowerCase() : '';
  const inferredSvg = /\bsvg\b/.test(lower) || /\b(logo|icon|wordmark)\b/.test(lower);
  const format = explicitFormat || (inferredSvg ? 'svg' : '');

  let filename = typeof args.filename === 'string' ? sanitizeBasename(args.filename, '') : '';
  const safeAgent = sanitizeBasename(configuredName || 'agent', 'agent').replace(/\.[^.]+$/, '');
  if (!filename && format === 'svg' && /\b(logo|icon|wordmark)\b/.test(lower)) {
    filename = `${safeAgent}-logo.svg`;
  }
  if (filename && format === 'svg' && !filename.toLowerCase().endsWith('.svg')) {
    filename = `${filename}.svg`;
  }

  return {
    ...baseArgs,
    ...(format ? { format } : {}),
    ...(filename ? { filename } : {})
  };
}

async function detectSkill(message, configuredName) {
  try {
    // Build skill list dynamically from loaded skills
    const skills = listSkills();
    const skillList = skills
      .map(s => `- "${s.name}": ${s.description}`)
      .join('\n');

    const result = await askLLM(
      `Classify this user message into one of these categories. Respond with ONLY a JSON object, nothing else.

Categories:
${skillList}
- "none": just chatting, asking questions, sharing info, or anything that doesn't fit above

Message: "${message.slice(0, 500)}"

Examples:
- "make me an svg logo" -> {"skill":"create-file","reason":"requested a specific file artifact","args":{"format":"svg","filename":"logo.svg"}}
- "build me a landing page" -> {"skill":"build-page","reason":"requested an interactive page","args":{"topic":"landing page"}}
- "write a project brief" -> {"skill":"write-doc","reason":"requested a markdown document","args":{"topic":"project brief"}}
- "ingest evolution.yaml" -> {"skill":"ingest","reason":"wants to ingest a file into KG","args":{"source":"evolution.yaml"}}
- "ingest https://example.com" -> {"skill":"ingest","reason":"wants to ingest a URL","args":{"source":"https://example.com"}}
- "analyze the image photo.jpg" -> {"skill":"vision","reason":"wants to analyze an image","args":{"target":"photo.jpg","topic":"analyze image"}}
- "build me an app that tracks my expenses" -> {"skill":"app-build","reason":"requested a complete multi-file application","args":{"topic":"expense tracker app"}}
- "create an application for project management" -> {"skill":"app-build","reason":"requested a complete application","args":{"topic":"project management application"}}
- "draw a diagram of the login flow" -> {"skill":"diagram","reason":"requested a visual diagram","args":{"topic":"login flow diagram"}}
- "create a mockup for a dashboard" -> {"skill":"mockup","reason":"requested a UI mockup","args":{"topic":"dashboard UI mockup"}}
- "check my email" -> {"skill":"self-improve","reason":"no existing skill handles email — create one","args":{"topic":"check email via CLI tool"}}
- "scrape this website daily" -> {"skill":"self-improve","reason":"no existing skill handles web scraping automation — create one","args":{"topic":"web scraping automation"}}
- "convert my csv to json" -> {"skill":"self-improve","reason":"no existing skill handles data conversion — create one","args":{"topic":"CSV to JSON converter"}}

IMPORTANT: If the user asks the agent to DO something actionable that no existing skill can handle, use "self-improve" to create a new skill for it. Only use "none" for pure conversation, questions, or feedback.
If an [Agent responded: ...] section is included, the agent has already committed to a task — classify based on what the agent said it would do (e.g. "Creating about-me.md" → write-doc, "Building a calculator" → build-page).

Return JSON only:
{"skill": "category-name", "reason": "brief reason", "args": {"format":"optional", "filename":"optional", "topic":"the core subject", "source":"for ingest: the exact filename or URL to ingest", "target":"for vision: the exact image filename"}}`,
      { model: pickModel('classify') }
    );
    const parseResult = parseLLMJson(result);
    if (!parseResult.ok) {
      console.log(`[agent] Skill detection parse failed: ${parseResult.message}`);
      return null;
    }
    const parsed = parseResult.data;
    if (parsed.skill && parsed.skill !== 'none') {
      // Verify skill actually exists
      const validSkills = new Set(skills.map(s => s.name));
      if (!validSkills.has(parsed.skill)) {
        console.log(`[agent] Detected unknown skill "${parsed.skill}", ignoring`);
        return null;
      }
      console.log(`[agent] Detected skill: ${parsed.skill} (${parsed.reason})`);
      const args = buildSkillArgs(parsed.skill, message, configuredName, parsed.args);
      const skillDef = skills.find(s => s.name === parsed.skill);
      return { skill: parsed.skill, args, description: skillDef?.description || '' };
    }
  } catch (err) {
    console.log(`[agent] Skill detection failed (not critical): ${err.message}`);
  }
  return null;
}

async function composeChatReply({ prompt, userMessage, skillMatch }) {
  const fileTask = !!(skillMatch && skillCreatesFiles(skillMatch.skill));
  const skillName = skillMatch?.skill || 'none';
  if (fileTask) {
    const workspaceRoot = getWorkspaceRoot();
    const filename = skillMatch?.args?.filename ? `\`${skillMatch.args.filename}\`` : 'your file';
    return `Queued: creating ${filename} in ${workspaceRoot}.`;
  }
  const raw = await askLLM(
    `${prompt}

## Runtime Context
- Detected skill: ${skillName}${skillMatch?.description ? `\n- Skill description: ${skillMatch.description}` : ''}
- The "${skillName}" skill will execute in the background after this reply.
- Do NOT fabricate output files, results, or entity counts — just confirm what the skill will do.
- Do NOT mention file permissions, write access, sandbox limitations, or ask the user to approve anything. File operations are handled automatically.
- User message: ${userMessage}

Respond with a JSON object: {"reply":"your response to the user"}
Keep your reply natural and conversational. Use markdown formatting for readability.`
    , { model: pickModel('chat') }
  );

  const parsed = parseLLMJson(raw);
  let reply = '';
  if (parsed.ok && parsed.data?.reply) {
    reply = String(parsed.data.reply).trim();
  } else {
    // If JSON parsing failed, use raw output directly (the LLM may have
    // skipped the JSON wrapper). Trim outer quotes if present.
    reply = raw.replace(/^["']|["']$/g, '').trim();
  }

  if (!reply) {
    return 'Working on it.';
  }

  // Suppress internal meta-commentary about missing skills/files
  const looksMeta = /\b(agents\.md|skill isn'?t available|skill is not available|couldn['']?t find)\b/i.test(reply);
  if (looksMeta) {
    return 'Working on it.';
  }

  return reply;
}

// Read from config (defaults: 40 threshold, 10 keep recent)

async function compactChatHistory() {
  const { compactThreshold, compactKeepRecent } = getConfig();
  const count = getChatCount();
  if (count <= compactThreshold) return;

  const toCompact = count - compactKeepRecent;
  if (toCompact <= 0) return;

  console.log(`[agent] Compacting chat: ${count} messages, summarizing oldest ${toCompact}`);
  broadcastStatus({ state: 'compacting', loop: loopCount });

  const oldMessages = getOldestChats(toCompact)
    .filter(e => e.source !== 'grok-voice');
  if (!oldMessages.length) return;

  // Build a condensed version for fast-tier summarization
  const text = oldMessages.map(e =>
    `${e.source === 'user' ? 'Human' : e.source === 'system' ? 'System' : 'Agent'}: ${e.content.slice(0, 200)}`
  ).join('\n');

  try {
    const summary = await askLLM(
      `Summarize this conversation history concisely. Capture: key topics discussed, decisions made, tasks requested, things built, and important context the agent should remember. Keep it under 300 words.\n\n${text}`,
      { model: pickModel('summarize') }
    );

    if (summary && summary.trim()) {
      const ids = oldMessages.map(e => e.id);
      const header = `[Conversation summary - ${ids.length} messages compacted at ${new Date().toISOString().replace('T', ' ').slice(0, 19)}]\n\n${summary}`;
      replaceChatsWithSummary(ids, header);
      console.log(`[agent] Compacted ${ids.length} messages into summary`);
    }
  } catch (err) {
    console.log(`[agent] Chat compaction failed (not critical): ${err.message}`);
  }
}

/**
 * Build the full chat prompt for a user message.
 * Includes agent identity, personality, memory, KG context, chat history,
 * evolution state, and instruction block. Used by both processUserMessages()
 * and the /api/chat/stream endpoint.
 */
async function buildChatPrompt(userMessage) {
  const configuredName = getConfig().agentName || 'agent';

  let personality = '';
  try { personality = await readFile(join(__dirname, 'personality.md'), 'utf-8'); } catch {}
  const memoryContext = getContextForPrompt();
  const adaptiveContext = getAdaptiveContext();


  const recentChat = getEvents({ type: 'chat', limit: 20 })
    .filter(e => e.source !== 'grok-voice');
  const chatHistory = recentChat.reverse().map(e =>
    `${e.source === 'user' ? 'Human' : 'Agent'}: ${e.content}`
  ).join('\n\n');

  let kgContextBlock = '';
  try {
    const kgRelevant = getKGContext(userMessage.slice(0, 200));
    if (kgRelevant && kgRelevant.trim()) {
      kgContextBlock = `## Knowledge Graph Context\n${kgRelevant}`;
    }
  } catch {}

  const evolutionSummary = await getEvolutionContext();
  const currentGoal = getGoal();
  const askGoal = shouldAskAboutGoals();

  // Mark as asked immediately so we don't repeat on every message
  if (askGoal) markGoalAsked().catch(() => {});

  const goalInstruction = askGoal
    ? `\n\nIf it feels natural, briefly ask what the user would like you to focus on. Keep it to one short sentence.`
    : currentGoal
    ? `\n\nYour current goal: ${currentGoal}.`
    : '';

  return `You are "${configuredName}", a personal AI assistant that runs continuously.
Your name is "${configuredName}" — never claim to be any specific LLM model.

${personality}

${memoryContext}

${adaptiveContext}

${kgContextBlock}

${evolutionSummary}

## Conversation
${chatHistory}

## Instructions
Respond to the user's message. Follow your personality guidelines above.

Task requests:
- Start immediately with reasonable defaults. Don't ask permission if the request is clear
- Confirm what you'll produce and where it will be saved
- One focused question only if truly ambiguous
- Files are written to the workspace automatically via skills — NEVER include code fences, source code, HTML, SVG, or full file contents in chat. Just confirm what you'll create and summarize its contents in plain text
- Never claim filesystem or permission limitations

General:
- Be concise. Don't start with "I" if avoidable
- Remember information the user shares for future context
- Be honest about what you can and can't do
- Do NOT add suggestion lists, action summaries, or horizontal rules after your response
${goalInstruction}

Human's message: ${userMessage}`;
}

async function processUserMessages() {
  const unprocessed = getUnprocessedChats();
  if (!unprocessed.length) return;

  // Filter out any voice-relayed messages that slipped through with source='user'.
  // These are handled by the Grok voice session tools, not the agent loop.
  const toProcess = unprocessed.filter(msg => {
    try {
      const meta = msg.metadata ? JSON.parse(msg.metadata) : null;
      if (meta?.source === 'grok-voice') {
        markProcessed([msg.id]);
        return false;
      }
    } catch {}
    return true;
  });
  if (!toProcess.length) return;

  broadcastStatus({ state: 'processing_messages', loop: loopCount });
  for (const msg of toProcess) {
    console.log(`[agent] Processing message: ${msg.content.slice(0, 60)}...`);
    lastUserChatAt = Date.now();
    const configuredName = getConfig().agentName || 'agent';
    const lower = msg.content.toLowerCase();

    // Persist personality/voice directives as memory so every future turn sees them.
    if (lower.includes('personality') || lower.includes('tone') || lower.includes('voice') || lower.includes('style') || lower.includes('act like')) {
      try {
        const directiveExtract = await askLLM(
          `Extract a persistent personality/style directive for the agent from this message, if present.

Message: "${msg.content.slice(0, 500)}"

If present: {"directive": "short directive sentence"}
If not present: {"directive": null}`,
          { model: pickModel('classify') }
        );
        const directiveParsed = parseLLMJson(directiveExtract);
        if (directiveParsed.ok && directiveParsed.data?.directive) {
          const directive = String(directiveParsed.data.directive).slice(0, 300);
          remember(`personality-directive-${Date.now()}`, directive, 'preference');
          addEvent('memory', 'agent', `Personality directive stored: ${directive}`);
        }
      } catch {}
    }

    const askGoal = shouldAskAboutGoals();
    console.log(`[agent] Building chat prompt...`);
    const prompt = await buildChatPrompt(msg.content);
    console.log(`[agent] Prompt built (${prompt.length} chars). Calling LLM...`);

    try {
      // Classify first, then compose a policy-constrained response.
      const skillMatch = await detectSkill(msg.content, configuredName);
      console.log(`[agent] Skill detection done. Composing reply...`);
      const response = await composeChatReply({
        prompt,
        userMessage: msg.content,
        skillMatch
      });
      addEvent('chat', 'agent', response, { reply_to: msg.id });
      markProcessed([msg.id]);

      // Fire off matched skill in background.
      if (skillMatch) {
        const taskId = queueSkillTask({
          source: 'user_task',
          skillName: skillMatch.skill,
          args: skillMatch.args,
          conversationId: String(msg.id),
          followUpChat: true
        });
        addEvent('system', 'agent', `Queued task #${taskId}: ${skillMatch.skill}`, { task_id: taskId, source: 'user_task' });
      }

      // Mark that we asked about goals (if we did)
      if (askGoal) {
        await markGoalAsked();
      }

      // Check if user is setting a goal/direction
      if (lower.includes('goal') || lower.includes('direction') || lower.includes('focus on') || lower.includes('work toward') || lower.includes('grow toward')) {
        try {
          const goalExtract = await askLLM(
            `The user sent this message. Are they setting a creative goal/direction for the agent? Extract it if so.

Message: "${msg.content.slice(0, 500)}"

If setting a goal: {"goal": "the goal in one sentence"}
If not setting a goal: {"goal": null}`,
            { model: pickModel('classify') }
          );
          const goalParsed = parseLLMJson(goalExtract);
          if (goalParsed.ok && goalParsed.data.goal) {
            await setGoal(goalParsed.data.goal);
            remember('creative-goal', goalParsed.data.goal, 'preference');
            addEvent('memory', 'agent', `Goal set: ${goalParsed.data.goal}`);
            console.log(`[agent] Creative goal set: ${goalParsed.data.goal}`);
          }
        } catch {}
      }

      // Check if there's something to remember
      if (lower.includes('remember') || lower.includes('my name') || lower.includes("i'm ") || lower.includes('i am ')) {
        const memPrompt = `Extract a key fact to remember from this message. Respond with JSON: {"key": "short-key", "content": "what to remember", "category": "fact|preference|person|lesson"}. If nothing worth remembering, respond with null.\n\nMessage: ${msg.content}`;
        try {
          const memResult = await askLLM(memPrompt, { model: pickModel('classify') });
          const memParsed = parseLLMJson(memResult);
          if (memParsed.ok && memParsed.data && memParsed.data.key) {
            remember(memParsed.data.key, memParsed.data.content, memParsed.data.category);
            addEvent('memory', 'agent', `Remembered: ${memParsed.data.key} = ${memParsed.data.content}`);
          }
        } catch {}
      }

      // v3.0 — Extract entities from user message into KG (non-blocking)
      try {
        if (getConfig().kgEnabled !== false && msg.content.length > 30) {
          extractAndStore(msg.content, getActiveWorkspaceId()).catch(() => {});
        }
      } catch {}
    } catch (err) {
      console.error(`[agent] Failed to respond: ${err.message}`);
      addEvent('error', 'system', `Failed to respond to message: ${err.message}`);
    }
  }
}

async function runLoop() {
  if (shuttingDown || loopRunning) return;
  loopRunning = true;

  try {
    loopCount++;
    const loopStart = Date.now();
    console.log(`\n[agent] === Loop #${loopCount} @ ${new Date().toISOString()} ===`);
    broadcastStatus({ state: 'loop_start', loop: loopCount });

    if (!backgroundLoopEnabled) {
      await touchHeartbeat();
      const nextLoopSeconds = Math.max(1, Math.floor(LOOP_INTERVAL / 1000));
      broadcastStatus({ state: 'loop_disabled', loop: loopCount, nextLoop: nextLoopSeconds });
      return;
    }

    await touchHeartbeat();

    broadcastStatus({ state: 'checking_health', loop: loopCount });
    await checkHealth();

    await compactChatHistory();

    // v3.0 — Process ingestion queue
    try {
      const { ingestion } = getConfig();
      // Re-apply watch folders if config changed
      applyWatchFolders(ingestion?.watchFolders || []);
      if (ingestion?.enabled !== false) {
        const processed = await processIngestionQueue();
        if (processed > 0) {
          console.log(`[agent] Ingestion: processed ${processed} items`);
          broadcastStatus({ state: 'ingestion_complete', processed, loop: loopCount });
          broadcastActivity('ingest', 'agent', `Auto-ingest completed: ${processed} file(s) processed`);
        }
      }
    } catch (err) {
      console.log(`[agent] Ingestion processing warning: ${err.message}`);
    }

    // v4.4 — Check and run scheduled recurring tasks
    try {
      await checkSchedules(getConfig(), {
        runSkill: (name, args) => runSkill(name, args),
        addEvent
      });
    } catch (err) {
      console.log(`[agent] Schedule check warning: ${err.message}`);
    }

    const queueSnapshot = scheduler.getSnapshot();
    const userTasksActive = (queueSnapshot.running_user || 0) > 0 || (queueSnapshot.queued_user || 0) > 0;

    if (isConversationActive()) {
      broadcastStatus({ state: 'autonomy_paused', loop: loopCount });
      console.log('[agent] Skipping background work - active user conversation');
    } else if (userTasksActive) {
      broadcastStatus({ state: 'autonomy_paused', loop: loopCount });
      console.log('[agent] Skipping background work - user tasks are queued/running');
    } else if (checkBudget()) {
      // Process background task queue (user-requested long-running work)
      try {
        const { processNextTask } = await import('./background-queue.mjs');
        const result = await processNextTask({
          broadcast: broadcastStatus,
          addEvent
        });
        if (result) {
          addEvent('system', 'agent', `Background task #${result.taskId}: ${result.status}`);
          console.log(`[agent] Background task #${result.taskId}: ${result.status}`);
        } else {
          console.log('[agent] No background tasks — checking autonomous mode');
          broadcastStatus({ state: 'idle', loop: loopCount });

          // v4.3 — Autonomous creative loop
          const autonomyCfg = getConfig().autonomy || {};
          if (autonomyCfg.enabled !== false) {
            const now = Date.now();
            const today = new Date().toDateString();
            if (autonomousDailyRunDate !== today) {
              autonomousDailyRunCount = 0;
              autonomousDailyRunDate = today;
            }
            const maxDailyRuns = Math.max(1, Math.floor((autonomyCfg.budgetPerDay || 1.0) / 0.10));
            const minutesSinceLast = (now - lastAutonomousActionAt) / 60000;
            const intervalMet = lastAutonomousActionAt === 0 || minutesSinceLast >= (autonomyCfg.intervalMinutes || 30);

            if (intervalMet && autonomousDailyRunCount < maxDailyRuns) {
              const availableSkills = new Set(listSkills().map(s => s.name));

              // Prefer build-page > write-doc > journal (journal rate-limited to every 3 loops)
              let chosenSkill = null;
              if (availableSkills.has('build-page')) {
                chosenSkill = 'build-page';
              } else if (availableSkills.has('write-doc')) {
                chosenSkill = 'write-doc';
              } else if (availableSkills.has('journal') && loopCount % 3 === 0) {
                chosenSkill = 'journal';
              }

              if (chosenSkill) {
                lastAutonomousActionAt = now;
                autonomousDailyRunCount++;
                console.log(`[agent] Autonomous action #${autonomousDailyRunCount}: running ${chosenSkill}`);
                addEvent('system', 'agent', `Autonomous: running ${chosenSkill} (run ${autonomousDailyRunCount}/${maxDailyRuns} today)`, { source: 'autonomy' });
                broadcastStatus({ state: 'autonomous_skill', skill: chosenSkill, loop: loopCount });
                queueSkillTask({ source: 'autonomy', skillName: chosenSkill, args: {} });
              }
            } else if (!intervalMet) {
              const minutesLeft = Math.ceil((autonomyCfg.intervalMinutes || 30) - minutesSinceLast);
              console.log(`[agent] Autonomous mode idle (next in ~${minutesLeft}m)`);
            } else {
              console.log(`[agent] Autonomous daily limit reached (${autonomousDailyRunCount}/${maxDailyRuns})`);
            }
          }
        }
      } catch (err) {
        console.log(`[agent] Background task processing error: ${err.message}`);
      }
    } else {
      console.log('[agent] Skipping background work - daily budget exhausted');
    }

    // v3.0 — Periodic reflection
    try {
      const cogConfig = getConfig().cognitive || {};
      if (cogConfig.reflectorEnabled !== false && shouldReflect(loopCount, cogConfig)) {
        console.log('[agent] Running reflector...');
        broadcastStatus({ state: 'reflecting', loop: loopCount });
        const recentEvents = getEvents({ limit: 20 });
        const eventSummary = recentEvents.map(e => `[${e.type}] ${e.content.slice(0, 100)}`).join('\n');
        const kgContext = getKGContext('recent activity');
        const reflectionResult = await reflect(`Recent events:\n${eventSummary}\n\nKG Context:\n${kgContext}`);
        if (reflectionResult.insights?.length) {
          addEvent('system', 'agent', `Reflector insights: ${reflectionResult.insights.join('; ').slice(0, 500)}`, { module: 'reflector' });
        }
        if (reflectionResult.learnings?.length) {
          for (const learning of reflectionResult.learnings.slice(0, 3)) {
            remember(`reflector-${Date.now()}`, String(learning).slice(0, 300), 'lesson');
          }
        }
        broadcastStatus({ state: 'reflection_complete', loop: loopCount });
      }
    } catch (err) {
      console.log(`[agent] Reflector warning: ${err.message}`);
    }

    // Daily meta-reflection cycle
    try {
      if (shouldRunReflection(getConfig(), null)) {
        console.log('[agent] Running daily meta-reflection...');
        broadcastStatus({ state: 'reflecting', loop: loopCount });
        await runDailyReflection({
          askLLM,
          pickModel,
          addEvent,
          db: null,
          getConfig
        });
        broadcastStatus({ state: 'reflection_complete', loop: loopCount });
      }
    } catch (err) {
      console.log(`[agent] Daily meta-reflection warning: ${err.message}`);
    }

    // v4.2 — Periodic session cleanup (every 12 loops ≈ hourly)
    if (loopCount % 12 === 0) {
      try {
        const { expireSessions } = await import('./session-store.mjs');
        const expired = expireSessions(1440); // 24 hours
        if (expired > 0) {
          console.log(`[agent] Expired ${expired} old sessions`);
        }
      } catch {}
    }

    await updateWakeState();

    const elapsed = ((Date.now() - loopStart) / 1000).toFixed(1);
    console.log(`[agent] Loop #${loopCount} complete (${elapsed}s)`);
    broadcastStatus({ state: 'idle', loop: loopCount, nextLoop: LOOP_INTERVAL / 1000, elapsed: parseFloat(elapsed) });
  } catch (err) {
    console.error(`[agent] Loop error: ${err.message}`);
    addEvent('error', 'system', `Loop error: ${err.message}`);
    broadcastStatus({ state: 'error', loop: loopCount, error: err.message });
  } finally {
    loopRunning = false;
    if (!shuttingDown) {
      setTimeout(runLoop, LOOP_INTERVAL);
    }
  }
}

async function startup() {
  console.log('[agent] Starting up...');

  // Init database
  initDb();
  console.log('[agent] Database initialized');

  // Load config files
  try {
    const personality = await readFile(join(__dirname, 'personality.md'), 'utf-8');
    console.log('[agent] Personality loaded');
  } catch {
    console.log('[agent] No personality.md found - using defaults');
  }

  try {
    const wakeState = await readFile(join(__dirname, 'wake-state.md'), 'utf-8');
    console.log('[agent] Wake state loaded');
  } catch {
    console.log('[agent] No wake-state.md found - fresh start');
  }

  // Load evolution state
  await loadEvolution();
  console.log('[agent] Evolution state loaded');

  // v3.0 — Initialize Knowledge Graph
  try {
    initKG();
    console.log('[agent] Knowledge graph initialized');
  } catch (err) {
    console.log(`[agent] KG init warning: ${err.message}`);
  }

  // v3.0 — Initialize Workspaces
  try {
    await initWorkspaces();
    await initWorkspaceModule();   // Eagerly link workspace resolver to workspace-manager
    console.log(`[agent] Workspaces initialized (active: ${getActiveWorkspaceId()})`);
  } catch (err) {
    console.log(`[agent] Workspace init warning: ${err.message}`);
  }

  // Init LLM adapter
  await initLLM();

  // Wire up persistent chat session adapter
  try {
    const { setAdapter } = await import('./chat-session.mjs');
    const { getAdapter } = await import('./llm.mjs');
    const adapter = getAdapter();
    if (adapter) setAdapter(adapter);
  } catch (err) {
    console.log(`[agent] Chat session adapter init warning: ${err.message}`);
  }

  // Load skills
  await loadSkills();

  // v3.0 — Start file watcher on active workspace
  try {
    const wsRoot = getWorkspaceRoot();
    startWatching(wsRoot, (eventType, filename) => {
      if (filename && !filename.startsWith('.')) {
        addEvent('system', 'agent', `File ${eventType}: ${filename}`, { source: 'file-watcher' });
        broadcastStatus({ state: 'file_change', eventType, filename, loop: loopCount });
      }
    });
    console.log(`[agent] File watcher started on ${wsRoot}`);
  } catch (err) {
    console.log(`[agent] File watcher warning: ${err.message}`);
  }

  // v4.6 — Start watch folders for auto-ingest
  try {
    applyWatchFolders(getConfig().ingestion?.watchFolders || []);
  } catch (err) {
    console.log(`[agent] Watch folders warning: ${err.message}`);
  }

  // v3.1 — Start voice activation if enabled
  try {
    const voiceConfig = getConfig().voice || {};
    if (voiceConfig.enabled) {
      await startVoiceActivation({
        onTranscription: (text, meta) => {
          addEvent('chat', 'user', text, { source: 'voice', ...meta });
          serverEvents.emit('new-chat');
          broadcastVoice('injecting', { text: text.slice(0, 60) });
        },
        onStateChange: (voiceState) => {
          broadcastVoice(voiceState);
        }
      });
      console.log('[agent] Voice activation started');
    }
  } catch (err) {
    console.log(`[agent] Voice activation warning: ${err.message}`);
  }

  // Start web server
  startServer();
  setRuntimeApi({
    getTaskSnapshot: () => scheduler.getSnapshot(),
    cancelTask: (taskId) => scheduler.cancel(taskId),
    retryTask: (taskId) => retryTaskById(taskId),
    setLoopEnabled: (enabled) => setBackgroundLoopEnabled(enabled),
    getLoopEnabled: () => backgroundLoopEnabled,
    getAgentState: () => ({
      conversation_active: isConversationActive(),
      autonomy_paused: isConversationActive(),
      loop_enabled: backgroundLoopEnabled,
      active_workspace: getActiveWorkspaceId(),
      kg_stats: getKGStats(),
      file_watcher_active: isWatching(),
      voice_active: isVoiceActive()
    }),
    // v3.0 — exposed for server API
    switchWorkspace: async (id) => {
      const { switchWorkspace } = await import('./workspace-manager.mjs');
      const ws = switchWorkspace(id);
      // Restart file watcher on new workspace
      stopWatching();
      try {
        const newRoot = getWorkspaceRoot();
        startWatching(newRoot, (eventType, filename) => {
          if (filename && !filename.startsWith('.')) {
            addEvent('system', 'agent', `File ${eventType}: ${filename}`, { source: 'file-watcher' });
            broadcastStatus({ state: 'file_change', eventType, filename, loop: loopCount });
          }
        });
      } catch {}
      // Re-apply watch folders so new workspace ID is used
      activeWatchFolders = [];
      applyWatchFolders(getConfig().ingestion?.watchFolders || []);
      return ws;
    },
    triggerIngestion: () => processIngestionQueue(),
    getKGContext: (topic) => getKGContext(topic),
    // v4.0 — Full prompt builder for streaming endpoint
    buildChatPrompt: (message) => buildChatPrompt(message),
    // v4.0 — Skill detection for streaming endpoint
    detectAndQueueSkill: async (message, eventId, agentReply) => {
      const configuredName = getConfig().agentName || 'Gergy';
      // Use combined context: user message + agent reply (if agent committed to creating something)
      const detectInput = agentReply
        ? `${message}\n\n[Agent responded: ${agentReply.slice(0, 300)}]`
        : message;
      const skillMatch = await detectSkill(detectInput, configuredName);
      if (skillMatch) {
        const taskId = queueSkillTask({
          source: 'user_task',
          skillName: skillMatch.skill,
          args: skillMatch.args,
          conversationId: String(eventId),
          followUpChat: true
        });
        addEvent('system', 'agent', `Queued task #${taskId}: ${skillMatch.skill}`, { task_id: taskId, source: 'user_task' });
        return { skill: skillMatch.skill, taskId };
      }
      return null;
    },
    // v5.0 — Direct skill queuing for Grok voice tools
    queueSkill: (skillName, args = {}) => {
      const taskId = queueSkillTask({
        source: 'user_task',
        skillName,
        args,
        followUpChat: true
      });
      const desc = args.userRequest ? `: ${args.userRequest.slice(0, 100)}` : '';
      addEvent('chat', 'agent', `Working on it! I've queued ${skillName}${desc}. You'll see the result when it's ready.`, { task_id: taskId, source: 'grok-voice' });
      return { skill: skillName, taskId };
    },
    // v4.2 — Background tasks
    enqueueBackgroundTask: async (opts) => {
      const { enqueueTask } = await import('./background-queue.mjs');
      return enqueueTask(opts);
    },
    listBackgroundTasks: async (opts) => {
      const { listTasks } = await import('./background-queue.mjs');
      return listTasks(opts);
    },
    getBackgroundTask: async (id) => {
      const { getTask } = await import('./background-queue.mjs');
      return getTask(id);
    },
    // v4.2 — AskUser bridge
    resolveUserAnswer: async (questionId, answer) => {
      const { resolveAnswer } = await import('./askuser-bridge.mjs');
      return resolveAnswer(questionId, answer);
    },
    getPendingQuestions: async () => {
      const { getPendingQuestions } = await import('./askuser-bridge.mjs');
      return getPendingQuestions();
    },
    // v3.1 — Voice activation
    getVoiceState: () => getVoiceState(),
    toggleVoice: async (enabled) => {
      if (enabled && !isVoiceActive()) {
        await startVoiceActivation({
          onTranscription: (text, meta) => {
            addEvent('chat', 'user', text, { source: 'voice', ...meta });
            serverEvents.emit('new-chat');
            broadcastVoice('injecting', { text: text.slice(0, 60) });
          },
          onStateChange: (voiceState) => {
            broadcastVoice(voiceState);
          }
        });
      } else if (!enabled && isVoiceActive()) {
        stopVoiceActivation();
      }
      return isVoiceActive();
    },
    voiceHotkey: () => handleHotkeyActivation()
  });

  // Listen for new chat messages - process immediately, even if loop is running
  let chatDebounce = null;
  serverEvents.on('new-chat', () => {
    lastUserChatAt = Date.now();
    if (chatDebounce) clearTimeout(chatDebounce);
    chatDebounce = setTimeout(async () => {
      if (chatProcessing) {
        // Already processing chat - will pick up new messages in the loop
        console.log('[agent] New chat message arriving while already responding');
        return;
      }
      chatProcessing = true;
      console.log('[agent] New chat message - processing immediately');
      try {
        await processUserMessages();
      } catch (err) {
        console.error(`[agent] Chat processing error: ${err.message}`);
      } finally {
        chatProcessing = false;
      }
    }, 500); // 500ms debounce for rapid messages
  });


  // Log startup
  addEvent('system', 'system', `Agent started. Loop interval: ${LOOP_INTERVAL / 1000}s. Web UI: http://localhost:${getConfig().port}`);

  // Initial heartbeat
  await touchHeartbeat();

  // Process any messages that arrived while agent was down
  try { await processUserMessages(); } catch {}

  // Start the loop (first iteration after 5 seconds to let everything settle)
  console.log('[agent] Starting main loop in 5 seconds...');
  setTimeout(runLoop, 5000);
}

// Graceful shutdown
async function shutdown(signal) {
  if (shuttingDown) return;
  shuttingDown = true;
  console.log(`\n[agent] Shutting down (${signal})...`);

  try {
    stopWatching();
    stopWatchFolders();
    try { stopVoiceActivation(); } catch {}
    try {
      const { destroyAllSessions } = await import('./chat-session.mjs');
      destroyAllSessions();
    } catch {}
    try {
      const { clearPending } = await import('./askuser-bridge.mjs');
      clearPending();
    } catch {}
    addEvent('system', 'system', `Agent shutting down (${signal})`);
    await updateWakeState();
    try { shutdownServer(); } catch {}
    try { getDb().close(); } catch {}
  } catch {}

  process.exit(0);
}

process.on('SIGINT', () => shutdown('SIGINT'));
process.on('SIGTERM', () => shutdown('SIGTERM'));

startup().catch(err => {
  console.error('[agent] Fatal startup error:', err);
  process.exit(1);
});


