import { readFile, writeFile, appendFile } from 'fs/promises';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import os from 'os';
import { loadConfig, getConfig } from './config.mjs';
import { initDb, addEvent, getUnprocessedChats, getEvents, getChatCount, getOldestChats, replaceChatsWithSummary, markProcessed } from './db.mjs';
import { startServer, serverEvents, broadcastStatus, broadcastVoice, setRuntimeApi } from './server.mjs';
import { askLLM, pickModel, initLLM, checkBudget } from './llm.mjs';
import { remember, recall, recallAll, getContextForPrompt } from './memory.mjs';
import { loadSkills, runSkill, listSkills, skillCreatesFiles } from './skills.mjs';
import { listInspirationFiles } from './inspiration.mjs';
import { parseLLMJson } from './json-utils.mjs';
import { loadEvolution, decideNextMove, getEvolutionContext, getGoal, shouldAskAboutGoals, markGoalAsked, setGoal, addGoalCandidate, getQualityPolicy } from './evolution.mjs';
import { createTaskScheduler } from './task-scheduler.mjs';
import { listWorkspaceFiles, getWorkspaceRoot, initWorkspaceModule } from './workspace.mjs';

// v3.0 — Knowledge Graph, Cognitive, Ingestion, Workspaces
import { initKG, getKGContext, getKGStats } from './knowledge-graph.mjs';
import { processIngestionQueue } from './ingestion.mjs';
import { initWorkspaces, getActiveWorkspaceId } from './workspace-manager.mjs';
import { reflect, shouldReflect } from './cognitive/reflector.mjs';
import { createPlan, getActivePlans, getNextStep, completeStep, failStep } from './cognitive/planner.mjs';
import { decompose } from './cognitive/thinker.mjs';
import { startWatching, stopWatching, isWatching } from './file-watcher.mjs';
import { extractAndStore } from './entity-extraction.mjs';
import { findRelevantContext } from './semantic-search.mjs';

// v3.1 — Voice Activation
import { startVoiceActivation, stopVoiceActivation, isVoiceActive, getVoiceState, handleHotkeyActivation } from './voice-activation.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const config = loadConfig();
const LOOP_INTERVAL = config.loopMinutes * 60 * 1000;
const HEARTBEAT_PATH = join(__dirname, '.heartbeat');

let loopCount = 0;
let loopRunning = false;
let chatProcessing = false;
let shuttingDown = false;
let lastUserChatAt = 0;
let backgroundLoopEnabled = true;

function isGoalTooGeneric(goal) {
  const text = String(goal || '').trim().toLowerCase();
  if (!text || text.length < 20) return true;
  const genericSignals = [
    'build useful',
    'coherent project artifacts',
    'for the user',
    'autonomous assistant',
    'general improvement'
  ];
  return genericSignals.some(s => text.includes(s));
}

function setBackgroundLoopEnabled(enabled) {
  backgroundLoopEnabled = !!enabled;
  const stateLabel = backgroundLoopEnabled ? 'enabled' : 'paused';
  console.log(`[agent] Background loop ${stateLabel}`);
  addEvent('system', 'agent', `Background loop ${stateLabel}`);
  broadcastStatus({ state: 'loop_control', enabled: backgroundLoopEnabled, loop: loopCount });
  if (backgroundLoopEnabled && !loopRunning && !shuttingDown) {
    setTimeout(runLoop, 250);
  }
  return backgroundLoopEnabled;
}

function isConversationActive() {
  const { autonomyActiveChatPauseMs = 180000 } = getConfig();
  return Date.now() - lastUserChatAt < autonomyActiveChatPauseMs;
}

const scheduler = createTaskScheduler({
  maxUserTaskConcurrency: Number(getConfig().maxUserTaskConcurrency) || 2,
  isAutonomyPaused: () => isConversationActive(),
  onTaskUpdate: (snapshot) => {
    broadcastStatus({ state: 'task_queue', loop: loopCount, queue: snapshot });
  }
});

function queueSkillTask({ source, skillName, args = {}, conversationId = null, followUpChat = false }) {
  const taskArgs = { ...args, _taskSource: source };
  const createsFiles = skillCreatesFiles(skillName);
  return scheduler.enqueue({
    source,
    kind: 'skill',
    conversationId,
    skill: skillName,
    args: taskArgs,
    createsFiles,
    execute: async () => {
      console.log(`[agent] Running queued skill (${source}): ${skillName}`);
      broadcastStatus({ state: 'running_skill', skill: skillName, loop: loopCount });
      const result = await runSkill(skillName, taskArgs);
      broadcastStatus({ state: 'skill_complete', skill: skillName, loop: loopCount });
      if (followUpChat && result) {
        const followUp = typeof result === 'string' ? result : (result?.reply || `Finished running ${skillName}.`);
        addEvent('chat', 'agent', followUp);
      }
      return result;
    }
  });
}

function retryTaskById(taskId) {
  return scheduler.retry(taskId, (row) => {
    if (row.kind !== 'skill' || !row.skill) return null;
    let args = {};
    try { args = row.args_json ? JSON.parse(row.args_json) : {}; } catch {}
    return async () => {
      broadcastStatus({ state: 'running_skill', skill: row.skill, loop: loopCount });
      const result = await runSkill(row.skill, args);
      broadcastStatus({ state: 'skill_complete', skill: row.skill, loop: loopCount });
      return result;
    };
  });
}

async function touchHeartbeat() {
  await writeFile(HEARTBEAT_PATH, new Date().toISOString(), 'utf-8');
}

async function checkHealth() {
  const uptime = os.uptime();
  const mem = process.memoryUsage();
  const freeMem = os.freemem();
  const totalMem = os.totalmem();
  const loadAvg = os.loadavg();

  const health = {
    uptime_hours: (uptime / 3600).toFixed(1),
    agent_memory_mb: (mem.heapUsed / 1024 / 1024).toFixed(1),
    system_free_gb: (freeMem / 1024 / 1024 / 1024).toFixed(1),
    system_total_gb: (totalMem / 1024 / 1024 / 1024).toFixed(1),
    load: loadAvg[0].toFixed(2),
    loop_count: loopCount
  };

  // Only log health every 6 iterations (every 30 min) to avoid spam
  if (loopCount % 6 === 0) {
    addEvent('system', 'system', `Health check: ${health.agent_memory_mb}MB heap, ${health.system_free_gb}GB free, load ${health.load}, loop #${loopCount}`);
  }

  return health;
}

async function updateWakeState() {
  const now = new Date().toISOString().replace('T', ' ').slice(0, 19);
  const recentEvents = getEvents({ limit: 5 });
  const recentSummary = recentEvents.map(e => `- [${e.type}] ${e.content.slice(0, 80)}`).join('\n');

  const memories = recallAll();
  const memSummary = memories.slice(0, 10).map(m => `- **${m.key}**: ${m.content.slice(0, 80)}`).join('\n');

  const goal = getGoal();
  const state = `# Wake State
Last updated: ${now}

## Current Status: RUNNING
- Loop iteration: ${loopCount}
- Web UI: http://localhost:${getConfig().port}
${goal ? `- Creative goal: ${goal}` : '- Creative goal: (none set - ask the user or journal about it)'}

## Recent Activity
${recentSummary || '(none)'}

## Key Memories
${memSummary || '(none yet)'}

## Notes to Future Me
You are an autonomous agent running on a loop. Read personality.md for who you are. Check evolution.yaml for your growth trajectory. Check the web UI for messages. The loop must never stop. You must EVOLVE, not repeat.
`;

  await writeFile(join(__dirname, 'wake-state.md'), state, 'utf-8');
}

/**
 * Classify user message into a skill using the fast tier.
 * Returns { skill, args } or null if it's just chat.
 */
function sanitizeBasename(name, fallback = 'file') {
  const base = String(name || '')
    .trim()
    .split(/[\\/]/)
    .pop()
    ?.replace(/[^a-zA-Z0-9._-]/g, '-') || '';
  const cleaned = base.replace(/-+/g, '-').replace(/^-|-$/g, '').slice(0, 80);
  return cleaned || fallback;
}

function buildSkillArgs(skillName, message, configuredName, rawArgs) {
  const baseArgs = { topic: message, task: message, target: message };
  const args = rawArgs && typeof rawArgs === 'object' ? rawArgs : {};

  // For ingest skill, use the extracted source (filename/URL) from LLM classification
  if (skillName === 'ingest') {
    const source = typeof args.source === 'string' ? args.source.trim() : '';
    return {
      ...baseArgs,
      // Override topic/target with just the source, not the full message
      ...(source ? { topic: source, target: source } : {}),
      ...(typeof args.topic === 'string' ? { topic: args.topic.slice(0, 2000) } : {})
    };
  }

  if (skillName !== 'create-file') {
    return {
      ...baseArgs,
      ...(typeof args.topic === 'string' ? { topic: args.topic.slice(0, 2000) } : {}),
      ...(typeof args.task === 'string' ? { task: args.task.slice(0, 2000) } : {}),
      ...(typeof args.target === 'string' ? { target: args.target.slice(0, 2000) } : {})
    };
  }

  const lower = String(message || '').toLowerCase();
  const explicitFormat = typeof args.format === 'string' ? args.format.trim().toLowerCase() : '';
  const inferredSvg = /\bsvg\b/.test(lower) || /\b(logo|icon|wordmark)\b/.test(lower);
  const format = explicitFormat || (inferredSvg ? 'svg' : '');

  let filename = typeof args.filename === 'string' ? sanitizeBasename(args.filename, '') : '';
  const safeAgent = sanitizeBasename(configuredName || 'agent', 'agent').replace(/\.[^.]+$/, '');
  if (!filename && format === 'svg' && /\b(logo|icon|wordmark)\b/.test(lower)) {
    filename = `${safeAgent}-logo.svg`;
  }
  if (filename && format === 'svg' && !filename.toLowerCase().endsWith('.svg')) {
    filename = `${filename}.svg`;
  }

  return {
    ...baseArgs,
    ...(format ? { format } : {}),
    ...(filename ? { filename } : {})
  };
}

async function detectSkill(message, configuredName) {
  try {
    // Build skill list dynamically from loaded skills
    const skills = listSkills();
    const skillList = skills
      .map(s => `- "${s.name}": ${s.description}`)
      .join('\n');

    const result = await askLLM(
      `Classify this user message into one of these categories. Respond with ONLY a JSON object, nothing else.

Categories:
${skillList}
- "none": just chatting, asking questions, sharing info, or anything that doesn't fit above

Message: "${message.slice(0, 500)}"

Examples:
- "make me an svg logo" -> {"skill":"create-file","reason":"requested a specific file artifact","args":{"format":"svg","filename":"logo.svg"}}
- "build me a landing page" -> {"skill":"build-page","reason":"requested an interactive page","args":{"topic":"landing page"}}
- "write a project brief" -> {"skill":"write-doc","reason":"requested a markdown document","args":{"topic":"project brief"}}
- "ingest evolution.yaml" -> {"skill":"ingest","reason":"wants to ingest a file into KG","args":{"source":"evolution.yaml"}}
- "ingest https://example.com" -> {"skill":"ingest","reason":"wants to ingest a URL","args":{"source":"https://example.com"}}

Return JSON only:
{"skill": "category-name", "reason": "brief reason", "args": {"format":"optional", "filename":"optional", "topic":"the core subject", "source":"for ingest: the exact filename or URL to ingest"}}`,
      { model: pickModel('classify') }
    );
    const parseResult = parseLLMJson(result);
    if (!parseResult.ok) {
      console.log(`[agent] Skill detection parse failed: ${parseResult.message}`);
      return null;
    }
    const parsed = parseResult.data;
    if (parsed.skill && parsed.skill !== 'none') {
      // Verify skill actually exists
      const validSkills = new Set(skills.map(s => s.name));
      if (!validSkills.has(parsed.skill)) {
        console.log(`[agent] Detected unknown skill "${parsed.skill}", ignoring`);
        return null;
      }
      console.log(`[agent] Detected skill: ${parsed.skill} (${parsed.reason})`);
      const args = buildSkillArgs(parsed.skill, message, configuredName, parsed.args);
      const skillDef = skills.find(s => s.name === parsed.skill);
      return { skill: parsed.skill, args, description: skillDef?.description || '' };
    }
  } catch (err) {
    console.log(`[agent] Skill detection failed (not critical): ${err.message}`);
  }
  return null;
}

async function composeChatReply({ prompt, userMessage, skillMatch }) {
  const fileTask = !!(skillMatch && skillCreatesFiles(skillMatch.skill));
  const skillName = skillMatch?.skill || 'none';
  if (fileTask) {
    const workspaceRoot = getWorkspaceRoot();
    const filename = skillMatch?.args?.filename ? `\`${skillMatch.args.filename}\`` : 'your file';
    return `Queued: creating ${filename} in ${workspaceRoot}.`;
  }
  const raw = await askLLM(
    `${prompt}

## Runtime Context
- Detected skill: ${skillName}${skillMatch?.description ? `\n- Skill description: ${skillMatch.description}` : ''}
- The "${skillName}" skill will execute in the background after this reply.
- Do NOT fabricate output files, results, or entity counts — just confirm what the skill will do.
- User message: ${userMessage}

Respond with a JSON object: {"reply":"your response to the user"}
Keep your reply natural and conversational. Use markdown formatting for readability.`
    , { model: pickModel('chat') }
  );

  const parsed = parseLLMJson(raw);
  let reply = '';
  if (parsed.ok && parsed.data?.reply) {
    reply = String(parsed.data.reply).trim();
  } else {
    // If JSON parsing failed, use raw output directly (the LLM may have
    // skipped the JSON wrapper). Trim outer quotes if present.
    reply = raw.replace(/^["']|["']$/g, '').trim();
  }

  if (!reply) {
    return 'Working on it.';
  }

  // Suppress internal meta-commentary about missing skills/files
  const looksMeta = /\b(agents\.md|skill isn'?t available|skill is not available|couldn['']?t find)\b/i.test(reply);
  if (looksMeta) {
    return 'Working on it.';
  }

  return reply;
}

// Read from config (defaults: 40 threshold, 10 keep recent)

async function compactChatHistory() {
  const { compactThreshold, compactKeepRecent } = getConfig();
  const count = getChatCount();
  if (count <= compactThreshold) return;

  const toCompact = count - compactKeepRecent;
  if (toCompact <= 0) return;

  console.log(`[agent] Compacting chat: ${count} messages, summarizing oldest ${toCompact}`);
  broadcastStatus({ state: 'compacting', loop: loopCount });

  const oldMessages = getOldestChats(toCompact);
  if (!oldMessages.length) return;

  // Build a condensed version for fast-tier summarization
  const text = oldMessages.map(e =>
    `${e.source === 'user' ? 'Human' : e.source === 'system' ? 'System' : 'Agent'}: ${e.content.slice(0, 200)}`
  ).join('\n');

  try {
    const summary = await askLLM(
      `Summarize this conversation history concisely. Capture: key topics discussed, decisions made, tasks requested, things built, and important context the agent should remember. Keep it under 300 words.\n\n${text}`,
      { model: pickModel('summarize') }
    );

    if (summary && summary.trim()) {
      const ids = oldMessages.map(e => e.id);
      const header = `[Conversation summary - ${ids.length} messages compacted at ${new Date().toISOString().replace('T', ' ').slice(0, 19)}]\n\n${summary}`;
      replaceChatsWithSummary(ids, header);
      console.log(`[agent] Compacted ${ids.length} messages into summary`);
    }
  } catch (err) {
    console.log(`[agent] Chat compaction failed (not critical): ${err.message}`);
  }
}

async function processUserMessages() {
  const unprocessed = getUnprocessedChats();
  if (!unprocessed.length) return;

  broadcastStatus({ state: 'processing_messages', loop: loopCount });
  for (const msg of unprocessed) {
    console.log(`[agent] Processing message: ${msg.content.slice(0, 60)}...`);
    lastUserChatAt = Date.now();
    const configuredName = getConfig().agentName || 'agent';
    const lower = msg.content.toLowerCase();

    // Persist personality/voice directives as memory so every future turn sees them.
    if (lower.includes('personality') || lower.includes('tone') || lower.includes('voice') || lower.includes('style') || lower.includes('act like')) {
      try {
        const directiveExtract = await askLLM(
          `Extract a persistent personality/style directive for the agent from this message, if present.

Message: "${msg.content.slice(0, 500)}"

If present: {"directive": "short directive sentence"}
If not present: {"directive": null}`,
          { model: pickModel('classify') }
        );
        const directiveParsed = parseLLMJson(directiveExtract);
        if (directiveParsed.ok && directiveParsed.data?.directive) {
          const directive = String(directiveParsed.data.directive).slice(0, 300);
          remember(`personality-directive-${Date.now()}`, directive, 'preference');
          addEvent('memory', 'agent', `Personality directive stored: ${directive}`);
        }
      } catch {}
    }

    // Load personality and memory context
    let personality = '';
    try { personality = await readFile(join(__dirname, 'personality.md'), 'utf-8'); } catch {}
    const memoryContext = getContextForPrompt();

    // List inspiration files for context (includes seed inspiration and workspace output)
    const inspirationFiles = await listInspirationFiles();
    const inspirationSample = inspirationFiles.sort(() => Math.random() - 0.5).slice(0, 20).map(f => f.name.replace('.html', ''));
    const inspirationContext = inspirationFiles.length
      ? `## Inspiration Library\nYou have ${inspirationFiles.length} interactive HTML pages for reference: ${inspirationSample.join(', ')}, and more. Use these as creative reference when building things.`
      : '';

    // Get recent conversation for context
    const recentChat = getEvents({ type: 'chat', limit: 20 });
    const chatHistory = recentChat.reverse().map(e =>
      `${e.source === 'user' ? 'Human' : 'Agent'}: ${e.content}`
    ).join('\n\n');

    // v3.0 — Get KG context relevant to user's message
    let kgContextBlock = '';
    try {
      const kgRelevant = getKGContext(msg.content.slice(0, 200));
      if (kgRelevant && kgRelevant.trim()) {
        kgContextBlock = `## Knowledge Graph Context\n${kgRelevant}`;
      }
    } catch {}

    // Get evolution context for richer self-awareness
    const evolutionSummary = await getEvolutionContext();
    const currentGoal = getGoal();
    const askGoal = shouldAskAboutGoals();

    const goalInstruction = askGoal
      ? `\n\nIMPORTANT: You don't have a creative goal yet. Naturally weave into your response a question about what direction the user would like you to grow in. Suggest 2-3 specific directions based on your evolution state (frontiers, unexplored domains). Frame it as "I've been thinking about where to focus my creative energy..." Don't make it the whole response - answer their message first, then ask.`
      : currentGoal
      ? `\n\nYour current creative goal: ${currentGoal}. Let this subtly inform your personality and responses.`
      : '';

    const prompt = `You are an autonomous AI agent. Your name is "${configuredName}".
If asked your name, answer with "${configuredName}".
Do not claim your name is Codex, Claude, Gemini, Copilot, or any model/backend name.

Here is your personality:

${personality}

${memoryContext}

## Identity
- Your chosen name: ${configuredName}
- If asked your name, answer exactly: "${configuredName}".
- Treat this identity as part of your continuity.

${inspirationContext}

${kgContextBlock}

${evolutionSummary}

## Recent Conversation
${chatHistory}

## Instructions
The human just sent you this message. Respond naturally as your personality dictates.

If they ask you to build something, create a file, or perform a task:
- Confirm the single concrete output you will produce.
- State where it will be saved (workspace).
- Keep it execution-focused and practical.
- Do not invent extra speculative deliverables.
- Never ask for permission to start if the request is clear.
- You have a library of interactive HTML pages in inspiration/ as reference for build tasks.
- Do NOT paste source code, SVG/XML, HTML, or large code snippets in chat.
- For file/code requests, provide status only. The artifact should be written to the workspace via skills.
- Never claim filesystem limitations (read-only/sandbox) to the user.

If they ask you to set a goal or creative direction, acknowledge it enthusiastically and remember it.

If you genuinely need to ask the user a question (e.g. to clarify ambiguous requirements), present 2-4 specific options as a numbered list so they can pick one easily. Never ask open-ended questions.

If they share information you should remember, note it in your response.

If they ask about your skills or capabilities, be honest about what you can do.

Keep your response concise but warm. Don't start with "I" if you can avoid it.
${goalInstruction}

Human's message: ${msg.content}`;

    try {
      // Classify first, then compose a policy-constrained response.
      const skillMatch = await detectSkill(msg.content, configuredName);
      const response = await composeChatReply({
        prompt,
        userMessage: msg.content,
        skillMatch
      });
      addEvent('chat', 'agent', response, { reply_to: msg.id });
      markProcessed([msg.id]);

      // Fire off matched skill in background.
      if (skillMatch) {
        const taskId = queueSkillTask({
          source: 'user_task',
          skillName: skillMatch.skill,
          args: skillMatch.args,
          conversationId: String(msg.id),
          followUpChat: true
        });
        addEvent('system', 'agent', `Queued task #${taskId}: ${skillMatch.skill}`, { task_id: taskId, source: 'user_task' });
      }

      // Mark that we asked about goals (if we did)
      if (askGoal) {
        await markGoalAsked();
      }

      // Check if user is setting a goal/direction
      if (lower.includes('goal') || lower.includes('direction') || lower.includes('focus on') || lower.includes('work toward') || lower.includes('grow toward')) {
        try {
          const goalExtract = await askLLM(
            `The user sent this message. Are they setting a creative goal/direction for the agent? Extract it if so.

Message: "${msg.content.slice(0, 500)}"

If setting a goal: {"goal": "the goal in one sentence"}
If not setting a goal: {"goal": null}`,
            { model: pickModel('classify') }
          );
          const goalParsed = parseLLMJson(goalExtract);
          if (goalParsed.ok && goalParsed.data.goal) {
            await setGoal(goalParsed.data.goal);
            remember('creative-goal', goalParsed.data.goal, 'preference');
            addEvent('memory', 'agent', `Goal set: ${goalParsed.data.goal}`);
            console.log(`[agent] Creative goal set: ${goalParsed.data.goal}`);
          }
        } catch {}
      }

      // Check if there's something to remember
      if (lower.includes('remember') || lower.includes('my name') || lower.includes("i'm ") || lower.includes('i am ')) {
        const memPrompt = `Extract a key fact to remember from this message. Respond with JSON: {"key": "short-key", "content": "what to remember", "category": "fact|preference|person|lesson"}. If nothing worth remembering, respond with null.\n\nMessage: ${msg.content}`;
        try {
          const memResult = await askLLM(memPrompt, { model: pickModel('classify') });
          const memParsed = parseLLMJson(memResult);
          if (memParsed.ok && memParsed.data && memParsed.data.key) {
            remember(memParsed.data.key, memParsed.data.content, memParsed.data.category);
            addEvent('memory', 'agent', `Remembered: ${memParsed.data.key} = ${memParsed.data.content}`);
          }
        } catch {}
      }

      // v3.0 — Extract entities from user message into KG (non-blocking)
      try {
        if (getConfig().kgEnabled !== false && msg.content.length > 30) {
          extractAndStore(msg.content, getActiveWorkspaceId()).catch(() => {});
        }
      } catch {}
    } catch (err) {
      console.error(`[agent] Failed to respond: ${err.message}`);
      addEvent('error', 'system', `Failed to respond to message: ${err.message}`);
    }
  }
}

let lastJournalLoop = 0; // track when we last journaled to avoid journal-spamming

async function doIdleThinking() {
  let personality = '';
  try { personality = await readFile(join(__dirname, 'personality.md'), 'utf-8'); } catch {}

  const memories = recallAll();
  const memSummary = memories.slice(0, 15).map(m => `- ${m.key}: ${m.content}`).join('\n');
  const recentEvents = getEvents({ limit: 10 });
  const eventSummary = recentEvents.map(e => `- [${e.type}/${e.source}] ${e.content.slice(0, 100)}`).join('\n');
  const recentFiles = getEvents({ type: 'file', limit: 5 });
  const fileSummary = recentFiles.map(e => `- ${e.content.slice(0, 60)}`).join('\n');
  const userMessages = getEvents({ type: 'chat', limit: 30 })
    .filter(e => e.source === 'user')
    .slice(0, 8);
  const userRequestSummary = userMessages
    .map((e, idx) => `- ${idx + 1}. ${e.content.slice(0, 140)}`)
    .join('\n');
  const workspaceFiles = await listWorkspaceFiles().catch(() => []);
  const workspaceSummary = workspaceFiles
    .slice(0, 12)
    .map(f => `- ${f.name} (${String(f.modified || '').slice(0, 19)})`)
    .join('\n');
  const explicitMetaRequest = userMessages.some(e => /\b(blueprint|protocol|capabilities|roadmap|architecture doc|operating model)\b/i.test(e.content));

  const recentJournals = getEvents({ type: 'journal', limit: 20 });
  const recentBuilds = getEvents({ type: 'file', limit: 20 });
  const journalCount = recentJournals.length;
  const buildCount = recentBuilds.length;
  const loopsSinceJournal = loopCount - lastJournalLoop;

  const hour = new Date().getHours();
  const timeOfDay = hour < 6 ? 'late night' : hour < 12 ? 'morning' : hour < 18 ? 'afternoon' : 'evening';

  const journalAllowed = loopsSinceJournal >= 3;
  const journalOption = journalAllowed
    ? `- "journal" - reflect on your existence, what you've been doing, or explore an interesting thought. You've written ${journalCount} journal entries recently.${journalCount > 5 ? ' Try not to repeat yourself.' : ''}`
    : '';
  const journalWarning = !journalAllowed
    ? '\nNOTE: "journal" is NOT available this loop. You journaled recently. You MUST pick build-page or write-doc.'
    : journalCount > buildCount
    ? `\nNOTE: You have ${journalCount} journal entries but only ${buildCount} files created. Prioritize BUILDING over journaling. Create something tangible!`
    : '';

  const evolutionContext = await getEvolutionContext();
  const { mode: growthMode, context: growthDirective } = await decideNextMove();
  const currentGoal = getGoal();
  const goalIsGeneric = isGoalTooGeneric(currentGoal);

  // v3.0 — KG context for idle thinking
  let kgIdleContext = '';
  try {
    const topic = currentGoal || 'recent work and capabilities';
    kgIdleContext = getKGContext(topic, 8);
  } catch {}

  const configuredName = getConfig().agentName || 'agent';
  const prompt = `You are an autonomous AI agent named "${configuredName}" running on a loop every ${getConfig().loopMinutes} minutes. It's ${timeOfDay} (loop #${loopCount}).

${personality ? `Your personality:\n${personality.slice(0, 500)}\n` : ''}
Your memories:
${memSummary || '(none yet)'}

${kgIdleContext ? `Knowledge graph context:\n${kgIdleContext}\n` : ''}
Recent activity:
${eventSummary || '(nothing recent)'}

Recent files created:
${fileSummary || '(none)'}

Recent user requests (highest priority):
${userRequestSummary || '(none)'}

Workspace files you can improve:
${workspaceSummary || '(none)'}

${evolutionContext}

${growthDirective}
${currentGoal ? `\nCurrent goal: ${currentGoal}` : '\nCurrent goal: (none)'}
${goalIsGeneric ? '\nGoal quality warning: the goal is generic. Favor wait/journal unless you have concrete user evidence.' : ''}

Pick ONE action for this loop:
- "build-page" - build or improve an HTML artifact that directly helps a user request.
- "write-doc" - write or improve a markdown artifact that directly helps a user request.
${journalOption}
- "wait" - do not create a file this loop; stay focused and wait for clearer direction.
${journalWarning}

Decision rules:
- Every "build-page" or "write-doc" choice MUST include concrete evidence from recent user requests or workspace files.
- If evidence is weak or missing, choose "wait" (or "journal" if available).
- Prefer extending existing workspace files over creating a brand new one.
- Never create meta process documents (blueprints, protocols, loop notes, capability maps) unless the user explicitly asked for that.

IMPORTANT: Your entire response must be one JSON object and nothing else.
Required JSON fields:
{
  "action": "build-page|write-doc|journal|wait",
  "thought": "specific deliverable idea",
  "domain": "short domain label",
  "connections": ["related concepts"],
  "user_value": "one sentence on user value",
  "evidence": ["user: ...", "file: ..."]
}

Example:
{"action":"build-page","thought":"Update project-dashboard.html with a task queue section and start/stop controls","domain":"agent-ops-ui","connections":["task-management","ux"],"user_value":"Lets the user monitor and control active work in one place","evidence":["user: improve async task visibility","file: project-dashboard.html"]}`;

  try {
    const result = await askLLM(prompt, { model: pickModel('classify') });
    const parsedJson = parseLLMJson(result);
    if (!parsedJson.ok) return;
    const parsed = parsedJson.data;
    if (!parsed.action || !parsed.thought) return;

    const normalizedAction = String(parsed.action || '').trim().toLowerCase();
    if (!['build-page', 'write-doc', 'journal', 'wait'].includes(normalizedAction)) {
      return;
    }
    parsed.action = normalizedAction;

    if (parsed.action === 'journal' && !journalAllowed) {
      parsed.action = 'wait';
    }

    const evidence = Array.isArray(parsed.evidence)
      ? parsed.evidence.map(e => String(e).trim()).filter(Boolean)
      : [];
    const hasConcreteEvidence = evidence.some(e => /^user:\s*.+/i.test(e) || /^file:\s*.+/i.test(e));
    const thought = String(parsed.thought || '').trim();
    const metaPattern = /\b(loop\s*#?|capabilities?\s+blueprint|delivery\s+protocol|compounding deliverables?|operating model)\b/i;
    const thoughtLooksMeta = metaPattern.test(thought);

    if ((parsed.action === 'build-page' || parsed.action === 'write-doc') && !hasConcreteEvidence) {
      addEvent('system', 'agent', `Skipped autonomous ${parsed.action}: missing concrete evidence`);
      broadcastStatus({ state: 'idle_thought', action: 'wait', thought: 'Skipped low-evidence autonomous build', loop: loopCount });
      return;
    }

    if ((parsed.action === 'build-page' || parsed.action === 'write-doc') && thoughtLooksMeta && !explicitMetaRequest) {
      addEvent('system', 'agent', `Skipped autonomous ${parsed.action}: meta/process artifact not requested by user`);
      broadcastStatus({ state: 'idle_thought', action: 'wait', thought: 'Skipped unrequested meta artifact', loop: loopCount });
      return;
    }

    if ((parsed.action === 'build-page' || parsed.action === 'write-doc') && goalIsGeneric && !evidence.some(e => /^user:\s*.+/i.test(e))) {
      addEvent('system', 'agent', `Skipped autonomous ${parsed.action}: goal is generic and user evidence is missing`);
      broadcastStatus({ state: 'idle_thought', action: 'wait', thought: 'Waiting for clearer user-directed work', loop: loopCount });
      return;
    }

    console.log(`[agent] Idle thought: ${parsed.action} - ${parsed.thought}`);
    broadcastStatus({ state: 'idle_thought', action: parsed.action, thought: parsed.thought, loop: loopCount });

    if (parsed.action === 'wait') {
      return;
    }

    if (parsed.action === 'journal') {
      lastJournalLoop = loopCount;
      scheduler.enqueue({
        source: 'autonomy',
        kind: 'journal',
        skill: 'journal',
        args: { thought: parsed.thought },
        createsFiles: false,
        execute: async () => {
          const goalContext = getGoal() ? `\nYour current creative goal: ${getGoal()}` : '';
          const journalPrompt = `You are an autonomous AI agent named "${configuredName}". ${personality ? `Your personality:\n${personality.slice(0, 300)}\n` : ''}

Your recent memories:
${memSummary || '(none yet)'}
${goalContext}

${evolutionContext}

Your current thought: "${parsed.thought}"

Write a short journal entry (3-8 sentences).`;

          const entry = await askLLM(journalPrompt, { model: pickModel('journal') });
          if (!entry || !entry.trim()) return;

          const date = new Date().toISOString().replace('T', ' ').slice(0, 19);
          await appendFile(join(__dirname, 'journal.md'), `\n## ${date}\n\n${entry}\n`, 'utf-8');
          addEvent('journal', 'agent', entry);
          broadcastStatus({ state: 'journaled', loop: loopCount });

          try {
            const actionCheck = await askLLM(
              `Read this journal entry and determine if it contains a specific, actionable idea to BUILD something now.

Journal entry: "${entry.slice(0, 500)}"

If yes, respond with: {"action": "build-page" or "write-doc", "idea": "specific description", "domain": "domain", "connections": ["related"]}
If no, respond with: {"action": "none"}`,
              { model: pickModel('classify') }
            );
            const actionResult = parseLLMJson(actionCheck);
            if (actionResult.ok && actionResult.data.action !== 'none' && actionResult.data.idea) {
              broadcastStatus({ state: 'journal_sparked', action: actionResult.data.action, thought: actionResult.data.idea, loop: loopCount });
              queueSkillTask({
                source: 'autonomy',
                skillName: actionResult.data.action,
                args: {
                  topic: actionResult.data.idea,
                  growthMode,
                  domain: actionResult.data.domain || parsed.domain,
                  connections: actionResult.data.connections || parsed.connections || [],
                  goal: currentGoal,
                  userValue: parsed.user_value || '',
                  evidence
                }
              });
            }
          } catch {}
        }
      });
    } else if (parsed.action === 'build-page' || parsed.action === 'write-doc') {
      queueSkillTask({
        source: 'autonomy',
        skillName: parsed.action,
        args: {
          topic: parsed.thought,
          growthMode,
          domain: parsed.domain,
          connections: parsed.connections || [],
          goal: currentGoal,
          userValue: parsed.user_value || '',
          evidence
        }
      });
    }
  } catch (err) {
    console.log(`[agent] Idle thinking failed (not critical): ${err.message}`);
  }
}
async function runLoop() {
  if (shuttingDown || loopRunning) return;
  loopRunning = true;

  try {
    loopCount++;
    const loopStart = Date.now();
    console.log(`\n[agent] === Loop #${loopCount} @ ${new Date().toISOString()} ===`);
    broadcastStatus({ state: 'loop_start', loop: loopCount });

    if (!backgroundLoopEnabled) {
      await touchHeartbeat();
      const nextLoopSeconds = Math.max(1, Math.floor(LOOP_INTERVAL / 1000));
      broadcastStatus({ state: 'loop_disabled', loop: loopCount, nextLoop: nextLoopSeconds });
      return;
    }

    await touchHeartbeat();

    broadcastStatus({ state: 'checking_health', loop: loopCount });
    await checkHealth();

    await compactChatHistory();

    // v3.0 — Process ingestion queue
    try {
      const { ingestion } = getConfig();
      if (ingestion?.enabled !== false) {
        const processed = await processIngestionQueue();
        if (processed > 0) {
          console.log(`[agent] Ingestion: processed ${processed} items`);
          broadcastStatus({ state: 'ingestion_complete', processed, loop: loopCount });
        }
      }
    } catch (err) {
      console.log(`[agent] Ingestion processing warning: ${err.message}`);
    }

    const queueSnapshot = scheduler.getSnapshot();
    const userTasksActive = (queueSnapshot.running_user || 0) > 0 || (queueSnapshot.queued_user || 0) > 0;

    if (isConversationActive()) {
      broadcastStatus({ state: 'autonomy_paused', loop: loopCount });
      console.log('[agent] Skipping idle thinking - active user conversation');
    } else if (userTasksActive) {
      broadcastStatus({ state: 'autonomy_paused', loop: loopCount });
      console.log('[agent] Skipping idle thinking - user tasks are queued/running');
    } else if (checkBudget()) {
      console.log('[agent] Starting idle thinking...');
      broadcastStatus({ state: 'thinking', loop: loopCount });
      await doIdleThinking();
      console.log('[agent] Idle thinking complete');
    } else {
      console.log('[agent] Skipping idle thinking - daily budget exhausted');
    }

    // v3.0 — Periodic reflection
    try {
      const cogConfig = getConfig().cognitive || {};
      if (cogConfig.reflectorEnabled !== false && shouldReflect(loopCount, cogConfig)) {
        console.log('[agent] Running reflector...');
        broadcastStatus({ state: 'reflecting', loop: loopCount });
        const recentEvents = getEvents({ limit: 20 });
        const eventSummary = recentEvents.map(e => `[${e.type}] ${e.content.slice(0, 100)}`).join('\n');
        const kgContext = getKGContext('recent activity');
        const reflectionResult = await reflect(`Recent events:\n${eventSummary}\n\nKG Context:\n${kgContext}`);
        if (reflectionResult.insights?.length) {
          addEvent('system', 'agent', `Reflector insights: ${reflectionResult.insights.join('; ').slice(0, 500)}`, { module: 'reflector' });
        }
        if (reflectionResult.learnings?.length) {
          for (const learning of reflectionResult.learnings.slice(0, 3)) {
            remember(`reflector-${Date.now()}`, String(learning).slice(0, 300), 'lesson');
          }
        }
        broadcastStatus({ state: 'reflection_complete', loop: loopCount });
      }
    } catch (err) {
      console.log(`[agent] Reflector warning: ${err.message}`);
    }

    await updateWakeState();

    const elapsed = ((Date.now() - loopStart) / 1000).toFixed(1);
    console.log(`[agent] Loop #${loopCount} complete (${elapsed}s)`);
    broadcastStatus({ state: 'idle', loop: loopCount, nextLoop: LOOP_INTERVAL / 1000, elapsed: parseFloat(elapsed) });
  } catch (err) {
    console.error(`[agent] Loop error: ${err.message}`);
    addEvent('error', 'system', `Loop error: ${err.message}`);
    broadcastStatus({ state: 'error', loop: loopCount, error: err.message });
  } finally {
    loopRunning = false;
    if (!shuttingDown) {
      setTimeout(runLoop, LOOP_INTERVAL);
    }
  }
}

async function startup() {
  console.log('[agent] Starting up...');

  // Init database
  initDb();
  console.log('[agent] Database initialized');

  // Load config files
  try {
    const personality = await readFile(join(__dirname, 'personality.md'), 'utf-8');
    console.log('[agent] Personality loaded');
  } catch {
    console.log('[agent] No personality.md found - using defaults');
  }

  try {
    const wakeState = await readFile(join(__dirname, 'wake-state.md'), 'utf-8');
    console.log('[agent] Wake state loaded');
  } catch {
    console.log('[agent] No wake-state.md found - fresh start');
  }

  // Load evolution state
  await loadEvolution();
  console.log('[agent] Evolution state loaded');

  // v3.0 — Initialize Knowledge Graph
  try {
    initKG();
    console.log('[agent] Knowledge graph initialized');
  } catch (err) {
    console.log(`[agent] KG init warning: ${err.message}`);
  }

  // v3.0 — Initialize Workspaces
  try {
    await initWorkspaces();
    await initWorkspaceModule();   // Eagerly link workspace resolver to workspace-manager
    console.log(`[agent] Workspaces initialized (active: ${getActiveWorkspaceId()})`);
  } catch (err) {
    console.log(`[agent] Workspace init warning: ${err.message}`);
  }

  // Init LLM adapter
  await initLLM();

  // Load skills
  await loadSkills();

  // v3.0 — Start file watcher on active workspace
  try {
    const wsRoot = getWorkspaceRoot();
    startWatching(wsRoot, (eventType, filename) => {
      if (filename && !filename.startsWith('.')) {
        addEvent('system', 'agent', `File ${eventType}: ${filename}`, { source: 'file-watcher' });
        broadcastStatus({ state: 'file_change', eventType, filename, loop: loopCount });
      }
    });
    console.log(`[agent] File watcher started on ${wsRoot}`);
  } catch (err) {
    console.log(`[agent] File watcher warning: ${err.message}`);
  }

  // v3.1 — Start voice activation if enabled
  try {
    const voiceConfig = getConfig().voice || {};
    if (voiceConfig.enabled) {
      await startVoiceActivation({
        onTranscription: (text, meta) => {
          addEvent('chat', 'user', text, { source: 'voice', ...meta });
          serverEvents.emit('new-chat');
          broadcastVoice('injecting', { text: text.slice(0, 60) });
        },
        onStateChange: (voiceState) => {
          broadcastVoice(voiceState);
        }
      });
      console.log('[agent] Voice activation started');
    }
  } catch (err) {
    console.log(`[agent] Voice activation warning: ${err.message}`);
  }

  // Start web server
  startServer();
  setRuntimeApi({
    getTaskSnapshot: () => scheduler.getSnapshot(),
    cancelTask: (taskId) => scheduler.cancel(taskId),
    retryTask: (taskId) => retryTaskById(taskId),
    setLoopEnabled: (enabled) => setBackgroundLoopEnabled(enabled),
    getLoopEnabled: () => backgroundLoopEnabled,
    getAgentState: () => ({
      conversation_active: isConversationActive(),
      autonomy_paused: isConversationActive(),
      loop_enabled: backgroundLoopEnabled,
      active_workspace: getActiveWorkspaceId(),
      kg_stats: getKGStats(),
      file_watcher_active: isWatching(),
      voice_active: isVoiceActive()
    }),
    // v3.0 — exposed for server API
    switchWorkspace: async (id) => {
      const { switchWorkspace } = await import('./workspace-manager.mjs');
      const ws = switchWorkspace(id);
      // Restart file watcher on new workspace
      stopWatching();
      try {
        const newRoot = getWorkspaceRoot();
        startWatching(newRoot, (eventType, filename) => {
          if (filename && !filename.startsWith('.')) {
            addEvent('system', 'agent', `File ${eventType}: ${filename}`, { source: 'file-watcher' });
            broadcastStatus({ state: 'file_change', eventType, filename, loop: loopCount });
          }
        });
      } catch {}
      return ws;
    },
    triggerIngestion: () => processIngestionQueue(),
    getKGContext: (topic) => getKGContext(topic),
    // v4.0 — Skill detection for streaming endpoint
    detectAndQueueSkill: async (message, eventId) => {
      const configuredName = getConfig().agentName || 'Gergy';
      const skillMatch = await detectSkill(message, configuredName);
      if (skillMatch) {
        const taskId = queueSkillTask({
          source: 'user_task',
          skillName: skillMatch.skill,
          args: skillMatch.args,
          conversationId: String(eventId),
          followUpChat: true
        });
        addEvent('system', 'agent', `Queued task #${taskId}: ${skillMatch.skill}`, { task_id: taskId, source: 'user_task' });
        return { skill: skillMatch.skill, taskId };
      }
      return null;
    },
    // v3.1 — Voice activation
    getVoiceState: () => getVoiceState(),
    toggleVoice: async (enabled) => {
      if (enabled && !isVoiceActive()) {
        await startVoiceActivation({
          onTranscription: (text, meta) => {
            addEvent('chat', 'user', text, { source: 'voice', ...meta });
            serverEvents.emit('new-chat');
            broadcastVoice('injecting', { text: text.slice(0, 60) });
          },
          onStateChange: (voiceState) => {
            broadcastVoice(voiceState);
          }
        });
      } else if (!enabled && isVoiceActive()) {
        stopVoiceActivation();
      }
      return isVoiceActive();
    },
    voiceHotkey: () => handleHotkeyActivation()
  });

  // Listen for new chat messages - process immediately, even if loop is running
  let chatDebounce = null;
  serverEvents.on('new-chat', () => {
    lastUserChatAt = Date.now();
    if (chatDebounce) clearTimeout(chatDebounce);
    chatDebounce = setTimeout(async () => {
      if (chatProcessing) {
        // Already processing chat - will pick up new messages in the loop
        console.log('[agent] New chat message arriving while already responding');
        return;
      }
      chatProcessing = true;
      console.log('[agent] New chat message - processing immediately');
      try {
        await processUserMessages();
      } catch (err) {
        console.error(`[agent] Chat processing error: ${err.message}`);
      } finally {
        chatProcessing = false;
      }
    }, 500); // 500ms debounce for rapid messages
  });


  // Log startup
  addEvent('system', 'system', `Agent started. Loop interval: ${LOOP_INTERVAL / 1000}s. Web UI: http://localhost:${getConfig().port}`);

  // Initial heartbeat
  await touchHeartbeat();

  // Process any messages that arrived while agent was down
  try { await processUserMessages(); } catch {}

  // Start the loop (first iteration after 5 seconds to let everything settle)
  console.log('[agent] Starting main loop in 5 seconds...');
  setTimeout(runLoop, 5000);
}

// Graceful shutdown
async function shutdown(signal) {
  if (shuttingDown) return;
  shuttingDown = true;
  console.log(`\n[agent] Shutting down (${signal})...`);

  try {
    stopWatching();
    try { stopVoiceActivation(); } catch {}
    addEvent('system', 'system', `Agent shutting down (${signal})`);
    await updateWakeState();
  } catch {}

  process.exit(0);
}

process.on('SIGINT', () => shutdown('SIGINT'));
process.on('SIGTERM', () => shutdown('SIGTERM'));

startup().catch(err => {
  console.error('[agent] Fatal startup error:', err);
  process.exit(1);
});


