first draft of log explorer

vscode-appdaemon/src/extension.ts

@@ -4,6 +4,7 @@ import { ADClient } from './adClient';
 import { StatusBarManager } from './statusBar';
 import { EntityCompletionProvider, EntityHoverProvider } from './entityProvider';
 import { ErrorViewer } from './errorViewer';
+import { LogExplorer } from './logExplorer';
 
 let haClient: HAClient;
 let adClient: ADClient;
@@ -344,6 +345,8 @@ export async function activate(context: vscode.ExtensionContext) {
     haClient = new HAClient(outputChannel);
     adClient = new ADClient(outputChannel);
     errorViewer = new ErrorViewer();
+    const logExplorer = new LogExplorer(context);
+    context.subscriptions.push(logExplorer, vscode.commands.registerCommand('appdaemon.exploreLogs', () => logExplorer.show()));
     statusBar = new StatusBarManager(adClient);
 
     // Wire error count → status bar badge

vscode-appdaemon/src/logExplorer.ts

@@ -0,0 +1,183 @@
+import * as vscode from 'vscode';
+import * as fs from 'fs';
+import * as path from 'path';
+import { randomBytes } from 'crypto';
+import { LogEntry, parseLine, decodeLogLine } from './logModel';
+
+type Cursor = { offset: number; identity: string; pending: Buffer; last?: LogEntry; discard: boolean; generation: string };
+export class LogExplorer implements vscode.Disposable {
+    private panel?: vscode.WebviewPanel;
+    private view?: vscode.WebviewView;
+    private activeWebview?: vscode.Webview;
+    private messageListener?: vscode.Disposable;
+    private timer?: ReturnType<typeof setInterval>;
+    private entries = new Map<string, LogEntry>();
+    private cursors = new Map<string, Cursor>();
+    private imported = new Set<string>();
+    private warnings = new Set<string>();
+    private busy = false;
+    constructor(private context: vscode.ExtensionContext) {}
+    show() {
+        if (this.view) { this.view.show(); return; }
+        if (this.panel) { this.panel.reveal(); return; }
+        this.panel = vscode.window.createWebviewPanel('appdaemon.logs', 'AppDaemon · Logs', vscode.ViewColumn.One, {
+            enableScripts: true, retainContextWhenHidden: true,
+            localResourceRoots: [vscode.Uri.joinPath(this.context.extensionUri, 'media')]
+        });
+        this.configureWebview(this.panel.webview);
+        this.panel.onDidDispose(() => {
+            this.panel = undefined;
+            if (!this.view) { this.stopPolling(); this.activeWebview = undefined; }
+        });
+        this.startPolling();
+    }
+    resolveWebviewView(view: vscode.WebviewView) {
+        this.view = view;
+        view.webview.options = {
+            enableScripts: true,
+            localResourceRoots: [vscode.Uri.joinPath(this.context.extensionUri, 'media')]
+        };
+        this.configureWebview(view.webview);
+        const updateVisibility = () => view.visible ? this.startPolling() : this.stopPolling();
+        view.onDidChangeVisibility(updateVisibility, undefined, this.context.subscriptions);
+        view.onDidDispose(() => {
+            if (this.view === view) { this.view = undefined; }
+            if (!this.panel) { this.stopPolling(); this.activeWebview = undefined; }
+        }, undefined, this.context.subscriptions);
+        updateVisibility();
+    }
+    private configureWebview(webview: vscode.Webview) {
+        const nonce = randomBytes(16).toString('hex');
+        const media = (name: string) => webview.asWebviewUri(vscode.Uri.joinPath(this.context.extensionUri, 'media', name));
+        const template = fs.readFileSync(path.join(this.context.extensionPath, 'media', 'logExplorer.html'), 'utf8');
+        webview.html = template.replaceAll('{{CSP}}', webview.cspSource).replaceAll('{{NONCE}}', nonce)
+            .replace('{{CSS}}', media('logExplorer.css').toString()).replace('{{JS}}', media('logExplorer.js').toString());
+        this.messageListener?.dispose();
+        this.messageListener = webview.onDidReceiveMessage(async message => {
+            if (message.type === 'copy' && typeof message.text === 'string' && message.text.length <= 1024 * 1024) {
+                try { await vscode.env.clipboard.writeText(message.text); webview.postMessage({ type: 'copied' }); }
+                catch (error) { vscode.window.showWarningMessage(`Unable to copy: ${String(error)}`); }
+            }
+            if (message.type === 'ready') { this.publish(); }
+            if (message.type === 'open' && typeof message.id === 'string') {
+                const entry = this.entries.get(message.id);
+                if (!entry?.file) { return; }
+                try {
+                    const doc = await vscode.workspace.openTextDocument(vscode.Uri.file(entry.file));
+                    const editor = await vscode.window.showTextDocument(doc, { preview: true, viewColumn: vscode.ViewColumn.Beside });
+                    const index = doc.getText().indexOf(entry.raw || entry.message);
+                    if (index >= 0) {
+                        const pos = doc.positionAt(index);
+                        editor.selection = new vscode.Selection(pos, pos);
+                        editor.revealRange(new vscode.Range(pos, pos));
+                    } else { vscode.window.showInformationMessage('This entry has moved after log rotation.'); }
+                } catch (error) { vscode.window.showWarningMessage(`Unable to read log: ${String(error)}`); }
+            }
+            if (message.type === 'source' && typeof message.id === 'string') {
+                const source = this.entries.get(message.id)?.source;
+                const root = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath;
+                if (!source || !root || typeof source.path !== 'string' || !Number.isInteger(source.line)) { return; }
+                const target = source.path.startsWith('/conf/') ? path.resolve(root, source.path.slice(6)) : path.resolve(source.path);
+                const relative = path.relative(root, target);
+                if (relative.startsWith('..') || path.isAbsolute(relative)) { vscode.window.showInformationMessage('The source is outside the workspace.'); return; }
+                try {
+                    const doc = await vscode.workspace.openTextDocument(vscode.Uri.file(target));
+                    const editor = await vscode.window.showTextDocument(doc, { preview: true, viewColumn: vscode.ViewColumn.Beside });
+                    const position = new vscode.Position(Math.max(0, Math.min(doc.lineCount - 1, source.line - 1)), 0);
+                    editor.selection = new vscode.Selection(position, position);
+                    editor.revealRange(new vscode.Range(position, position));
+                } catch (error) { vscode.window.showWarningMessage(`Source unavailable: ${String(error)}`); }
+            }
+            if (message.type === 'import') {
+                const files = await vscode.window.showOpenDialog({ canSelectMany: true, filters: { Logs: ['log', '1', '2', '3'] } });
+                files?.forEach(uri => this.imported.add(uri.fsPath));
+                this.poll();
+            }
+        }, undefined, this.context.subscriptions);
+        this.activeWebview = webview;
+    }
+    private startPolling() {
+        if (this.timer) { return; }
+        this.poll();
+        this.timer = setInterval(() => this.poll(), 1000);
+    }
+    private stopPolling() {
+        if (this.timer) { clearInterval(this.timer); this.timer = undefined; }
+    }
+    private files(): string[] {
+        const folder = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath;
+        if (!folder) { return [...this.imported]; }
+        const config = vscode.workspace.getConfiguration('appdaemon');
+        const configured = config.get<string[]>('logExplorerPaths', ['logs/appdaemon.log', 'logs/error.log', 'logs/virtualsensors.log']);
+        return [...new Set([...configured.map(p => path.resolve(folder, p)), ...this.imported])];
+    }
+    private poll() {
+        if (this.busy) { return; }
+        this.busy = true;
+        let changed = false;
+        try {
+            const encoding = vscode.workspace.getConfiguration('appdaemon').get<string>('logEncoding', 'auto');
+            const zone = vscode.workspace.getConfiguration('appdaemon').get<string>('logTimezone', 'Europe/Berlin');
+            for (const file of this.files()) {
+                if (!fs.existsSync(file)) { continue; }
+                try {
+                    const stat = fs.statSync(file);
+                    const identity = `${stat.dev}:${stat.ino}`;
+                    let cursor = this.cursors.get(file);
+                    if (!cursor || cursor.identity !== identity || stat.size < cursor.offset) {
+                        const offset = Math.max(0, stat.size - 2 * 1024 * 1024);
+                        cursor = { offset, identity, pending: Buffer.alloc(0), discard: offset > 0, generation: randomBytes(8).toString('hex') };
+                        this.cursors.set(file, cursor);
+                    }
+                    if (stat.size <= cursor.offset) { continue; }
+                    const length = Math.min(512 * 1024, stat.size - cursor.offset);
+                    const buffer = Buffer.alloc(length);
+                    const fd = fs.openSync(file, 'r');
+                    let bytes: number;
+                    try { bytes = fs.readSync(fd, buffer, 0, length, cursor.offset); } finally { fs.closeSync(fd); }
+                    const start = cursor.offset - cursor.pending.length;
+                    cursor.offset += bytes;
+                    const content = Buffer.concat([cursor.pending, buffer.subarray(0, bytes)]);
+                    let from = 0;
+                    for (let end = content.indexOf(10); end >= 0; end = content.indexOf(10, from)) {
+                        const line = decodeLogLine(content.subarray(from, end), file.includes('jsonl') ? 'utf8' : encoding).replace(/\r$/, '');
+                        const position = start + from;
+                        from = end + 1;
+                        if (cursor.discard) { cursor.discard = false; continue; }
+                        const entry = parseLine(line, file, position, zone);
+                        if (entry) {
+                            if (!entry.trace_id && !line.startsWith('{')) { entry.event_id += ':' + cursor.generation; }
+                            const existing = this.entries.get(entry.event_id);
+                            if (!existing || entry.raw?.startsWith('{')) { this.entries.set(entry.event_id, entry); }
+                            cursor.last = this.entries.get(entry.event_id);
+                            changed = true;
+                        } else if (line.trim() && cursor.last && !file.includes('jsonl')) {
+                            // Traceback lines belong to the previous timestamped entry.
+                            if ((cursor.last.raw?.length || 0) < 64 * 1024) {
+                                cursor.last.message += '\n' + line;
+                                cursor.last.raw += '\n' + line;
+                                changed = true;
+                            }
+                        } else if (line.trim() && file.includes('jsonl')) { this.warnings.add(`Invalid event ignored: ${path.basename(file)}`); changed = true; }
+                    }
+                    cursor.pending = content.subarray(from);
+                    if (cursor.pending.length > 256 * 1024) { cursor.pending = Buffer.alloc(0); cursor.discard = true; this.warnings.add('An oversized log line was skipped.'); }
+                } catch (error) { this.warnings.add(`${path.basename(file)} : ${String(error)}`); changed = true; }
+            }
+            while (this.entries.size > 10000) { this.entries.delete(this.entries.keys().next().value!); }
+            if (changed) { this.publish(); }
+        } finally { this.busy = false; }
+    }
+    private publish() {
+        const entries = [...this.entries.values()].sort((a, b) => a.time - b.time);
+        // Legacy errors duplicated across main/error logs are merged only on exact match.
+        const seen = new Set<string>();
+        const unique = entries.filter(e => {
+            const key = e.trace_id ? e.event_id : `${e.timestamp}|${e.app}|${e.level}|${e.message}`;
+            if (seen.has(key)) { return false; } seen.add(key); return true;
+        });
+        const zone = vscode.workspace.getConfiguration('appdaemon').get<string>('logTimezone', 'Europe/Berlin');
+        this.activeWebview?.postMessage({ type: 'entries', entries: unique, warnings: [...this.warnings], limit: 10000, timezone: zone });
+    }
+    dispose() { this.stopPolling(); this.messageListener?.dispose(); this.panel?.dispose(); }
+}

vscode-appdaemon/src/logModel.ts

@@ -0,0 +1,70 @@
+export interface LogEntry {
+    event_id: string; timestamp: string; app: string; level: string; message: string;
+    kind: string; cause_ids: string[]; probable_cause_ids?: string[];
+    entity?: string; values?: Record<string, unknown>;
+    source?: { path: string; line: number };
+    initialization_id?: string; phase?: string; status?: string;
+    file?: string; raw?: string; time: number; trace_id?: string;
+    [key: string]: unknown;
+}
+const timeCache = new Map<string, number>();
+const formatters = new Map<string, Intl.DateTimeFormat>();
+export function rawTime(date: string, zone: string): number {
+    const cacheKey = zone + ':' + date;
+    const cached = timeCache.get(cacheKey);
+    if (cached !== undefined) { return cached; }
+    const m = date.match(/^(\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)/);
+    if (!m) { return NaN; }
+    const utc = Date.UTC(2000 + Number(m[3]), Number(m[2]) - 1, Number(m[1]), Number(m[4]), Number(m[5]), Number(m[6]));
+    try {
+        let formatter = formatters.get(zone);
+        if (!formatter) { formatter = new Intl.DateTimeFormat('en-GB', { timeZone: zone, year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit', hourCycle: 'h23' }); formatters.set(zone, formatter); }
+        const parts = formatter.formatToParts(utc);
+        const p = Object.fromEntries(parts.map(part => [part.type, part.value]));
+        const time = utc - (Date.UTC(+p.year, +p.month - 1, +p.day, +p.hour, +p.minute, +p.second) - utc);
+        timeCache.set(cacheKey, time);
+        if (timeCache.size > 4096) { timeCache.delete(timeCache.keys().next().value!); }
+        return time;
+    } catch { return utc; }
+}
+export function parseLine(line: string, file: string, offset: number, zone = 'Europe/Berlin'): LogEntry | undefined {
+    if (line.startsWith('{')) {
+        try {
+            const e = JSON.parse(line);
+            if (e.version !== 1 || typeof e.event_id !== 'string' || typeof e.message !== 'string' || typeof e.app !== 'string' || typeof e.timestamp !== 'string' || !Array.isArray(e.cause_ids) || !e.cause_ids.every((id: unknown) => typeof id === 'string')) { return undefined; }
+            return { ...e, level: String(e.level || 'INFO'), kind: String(e.kind || 'log'), file, raw: line, time: Date.parse(e.timestamp) };
+        } catch { return undefined; }
+    }
+    const m = line.match(/^(\d{2}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?)\s+(DEBUG|INFO|WARNING|ERROR|CRITICAL)\s+([^:]+):\s?(.*)$/);
+    if (!m) { return undefined; }
+    const init = m[4].match(/\[init=([\w-]+)\]/);
+    const id = m[4].match(/\[event=([\w-]+) trace=([\w-]+)\]/);
+    return { event_id: id?.[1] || `${file}:${offset}`, trace_id: id?.[2], initialization_id: init?.[1], phase: init ? 'initialization' : undefined, timestamp: m[1], time: rawTime(m[1], zone), app: m[3].trim(), level: m[2], message: m[4], kind: 'log', cause_ids: [], file, raw: line };
+}
+
+/** Preserve valid UTF-8 spans, including files containing mixed legacy bytes. */
+export function decodeLogLine(bytes: Buffer, encoding = 'auto'): string {
+    if (encoding === 'utf8') { return bytes.toString('utf8'); }
+    const legacy = new TextDecoder('windows-1252');
+    if (encoding === 'windows-1252') { return legacy.decode(bytes); }
+    try { return new TextDecoder('utf-8', { fatal: true }).decode(bytes); } catch { /* Mixed legacy log. */ }
+    const output: string[] = [];
+    let start = 0;
+    for (let i = 0; i < bytes.length;) {
+        const first = bytes[i];
+        const width = first < 0x80 ? 1 : first >= 0xc2 && first <= 0xdf ? 2 : first >= 0xe0 && first <= 0xef ? 3 : first >= 0xf0 && first <= 0xf4 ? 4 : 0;
+        let valid = width > 0 && i + width <= bytes.length;
+        for (let j = 1; valid && j < width; j++) { valid = bytes[i + j] >= 0x80 && bytes[i + j] <= 0xbf; }
+        if (valid && width > 1) {
+            const second = bytes[i + 1];
+            valid = !(first === 0xe0 && second < 0xa0 || first === 0xed && second > 0x9f || first === 0xf0 && second < 0x90 || first === 0xf4 && second > 0x8f);
+        }
+        if (valid) { i += width; } else {
+            if (i > start) { output.push(bytes.subarray(start, i).toString('utf8')); }
+            output.push(legacy.decode(bytes.subarray(i, i + 1)));
+            start = ++i;
+        }
+    }
+    if (start < bytes.length) { output.push(bytes.subarray(start).toString('utf8')); }
+    return output.join('');
+}

vscode-appdaemon/src/statusBar.ts

@@ -7,6 +7,7 @@ export class StatusBarManager {
     private productionModeItem: vscode.StatusBarItem;
     private errorItem: vscode.StatusBarItem;
     private contextualItem: vscode.StatusBarItem;
+    private logsItem: vscode.StatusBarItem;
 
     constructor(private adClient: ADClient) {
         // Reload AppDaemon apps
@@ -36,6 +37,13 @@ export class StatusBarManager {
         this.errorItem.tooltip = 'Show AppDaemon Error Log';
         this.errorItem.show();
 
+        // Log explorer shortcut, available even when no editor is open.
+        this.logsItem = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left, 96);
+        this.logsItem.command = 'appdaemon.exploreLogs';
+        this.logsItem.text = '$(list-tree) AD Logs';
+        this.logsItem.tooltip = 'Open AppDaemon Log Explorer';
+        this.logsItem.show();
+
         // Contextual per-file restart button (hidden by default)
         this.contextualItem = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left, 101);
         this.contextualItem.command = 'appdaemon.restartCurrentFileApps';
@@ -93,6 +101,7 @@ export class StatusBarManager {
         this.restartHAItem.dispose();
         this.productionModeItem.dispose();
         this.errorItem.dispose();
+        this.logsItem.dispose();
         this.contextualItem.dispose();
     }
 }