const builtin = @import("builtin"); const std = @import("std"); const io = std.io; const fs = std.fs; const process = std.process; const ChildProcess = std.ChildProcess; const warn = std.debug.warn; const mem = std.mem; const testing = std.testing; const max_doc_file_size = 10 * 1024 * 1024; const exe_ext = @as(std.build.Target, std.build.Target.Native).exeFileExt(); const obj_ext = @as(std.build.Target, std.build.Target.Native).oFileExt(); const tmp_dir_name = "docgen_tmp"; const test_out_path = tmp_dir_name ++ fs.path.sep_str ++ "test" ++ exe_ext; pub fn main() !void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const allocator = &arena.allocator; var args_it = process.args(); if (!args_it.skip()) @panic("expected self arg"); const zig_exe = try (args_it.next(allocator) orelse @panic("expected zig exe arg")); defer allocator.free(zig_exe); const in_file_name = try (args_it.next(allocator) orelse @panic("expected input arg")); defer allocator.free(in_file_name); const out_file_name = try (args_it.next(allocator) orelse @panic("expected output arg")); defer allocator.free(out_file_name); var in_file = try fs.File.openRead(in_file_name); defer in_file.close(); var out_file = try fs.File.openWrite(out_file_name); defer out_file.close(); var file_in_stream = in_file.inStream(); const input_file_bytes = try file_in_stream.stream.readAllAlloc(allocator, max_doc_file_size); var file_out_stream = out_file.outStream(); var buffered_out_stream = io.BufferedOutStream(fs.File.WriteError).init(&file_out_stream.stream); var tokenizer = Tokenizer.init(in_file_name, input_file_bytes); var toc = try genToc(allocator, &tokenizer); try fs.makePath(allocator, tmp_dir_name); defer fs.deleteTree(tmp_dir_name) catch {}; try genHtml(allocator, &tokenizer, &toc, &buffered_out_stream.stream, zig_exe); try buffered_out_stream.flush(); } const Token = struct { id: Id, start: usize, end: usize, const Id = enum { Invalid, Content, BracketOpen, TagContent, Separator, BracketClose, Eof, }; }; const Tokenizer = struct { buffer: []const u8, index: usize, state: State, source_file_name: []const u8, code_node_count: usize, const State = enum { Start, LBracket, Hash, TagName, Eof, }; fn init(source_file_name: []const u8, buffer: []const u8) Tokenizer { return Tokenizer{ .buffer = buffer, .index = 0, .state = State.Start, .source_file_name = source_file_name, .code_node_count = 0, }; } fn next(self: *Tokenizer) Token { var result = Token{ .id = Token.Id.Eof, .start = self.index, .end = undefined, }; while (self.index < self.buffer.len) : (self.index += 1) { const c = self.buffer[self.index]; switch (self.state) { State.Start => switch (c) { '{' => { self.state = State.LBracket; }, else => { result.id = Token.Id.Content; }, }, State.LBracket => switch (c) { '#' => { if (result.id != Token.Id.Eof) { self.index -= 1; self.state = State.Start; break; } else { result.id = Token.Id.BracketOpen; self.index += 1; self.state = State.TagName; break; } }, else => { result.id = Token.Id.Content; self.state = State.Start; }, }, State.TagName => switch (c) { '|' => { if (result.id != Token.Id.Eof) { break; } else { result.id = Token.Id.Separator; self.index += 1; break; } }, '#' => { self.state = State.Hash; }, else => { result.id = Token.Id.TagContent; }, }, State.Hash => switch (c) { '}' => { if (result.id != Token.Id.Eof) { self.index -= 1; self.state = State.TagName; break; } else { result.id = Token.Id.BracketClose; self.index += 1; self.state = State.Start; break; } }, else => { result.id = Token.Id.TagContent; self.state = State.TagName; }, }, State.Eof => unreachable, } } else { switch (self.state) { State.Start, State.LBracket, State.Eof => {}, else => { result.id = Token.Id.Invalid; }, } self.state = State.Eof; } result.end = self.index; return result; } const Location = struct { line: usize, column: usize, line_start: usize, line_end: usize, }; fn getTokenLocation(self: *Tokenizer, token: Token) Location { var loc = Location{ .line = 0, .column = 0, .line_start = 0, .line_end = 0, }; for (self.buffer) |c, i| { if (i == token.start) { loc.line_end = i; while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {} return loc; } if (c == '\n') { loc.line += 1; loc.column = 0; loc.line_start = i + 1; } else { loc.column += 1; } } return loc; } }; fn parseError(tokenizer: *Tokenizer, token: Token, comptime fmt: []const u8, args: var) anyerror { const loc = tokenizer.getTokenLocation(token); const args_prefix = .{ tokenizer.source_file_name, loc.line + 1, loc.column + 1 }; warn("{}:{}:{}: error: " ++ fmt ++ "\n", args_prefix ++ args); if (loc.line_start <= loc.line_end) { warn("{}\n", .{tokenizer.buffer[loc.line_start..loc.line_end]}); { var i: usize = 0; while (i < loc.column) : (i += 1) { warn(" ", .{}); } } { const caret_count = token.end - token.start; var i: usize = 0; while (i < caret_count) : (i += 1) { warn("~", .{}); } } warn("\n", .{}); } return error.ParseError; } fn assertToken(tokenizer: *Tokenizer, token: Token, id: Token.Id) !void { if (token.id != id) { return parseError(tokenizer, token, "expected {}, found {}", .{ @tagName(id), @tagName(token.id) }); } } fn eatToken(tokenizer: *Tokenizer, id: Token.Id) !Token { const token = tokenizer.next(); try assertToken(tokenizer, token, id); return token; } const HeaderOpen = struct { name: []const u8, url: []const u8, n: usize, }; const SeeAlsoItem = struct { name: []const u8, token: Token, }; const ExpectedOutcome = enum { Succeed, Fail, BuildFail, }; const Code = struct { id: Id, name: []const u8, source_token: Token, is_inline: bool, mode: builtin.Mode, link_objects: []const []const u8, target_str: ?[]const u8, link_libc: bool, const Id = union(enum) { Test, TestError: []const u8, TestSafety: []const u8, Exe: ExpectedOutcome, Obj: ?[]const u8, Lib, }; }; const Link = struct { url: []const u8, name: []const u8, token: Token, }; const Node = union(enum) { Content: []const u8, Nav, Builtin: Token, HeaderOpen: HeaderOpen, SeeAlso: []const SeeAlsoItem, Code: Code, Link: Link, Syntax: Token, }; const Toc = struct { nodes: []Node, toc: []u8, urls: std.StringHashMap(Token), }; const Action = enum { Open, Close, }; fn genToc(allocator: *mem.Allocator, tokenizer: *Tokenizer) !Toc { var urls = std.StringHashMap(Token).init(allocator); errdefer urls.deinit(); var header_stack_size: usize = 0; var last_action = Action.Open; var last_columns: ?u8 = null; var toc_buf = try std.Buffer.initSize(allocator, 0); defer toc_buf.deinit(); var toc_buf_adapter = io.BufferOutStream.init(&toc_buf); var toc = &toc_buf_adapter.stream; var nodes = std.ArrayList(Node).init(allocator); defer nodes.deinit(); try toc.writeByte('\n'); while (true) { const token = tokenizer.next(); switch (token.id) { Token.Id.Eof => { if (header_stack_size != 0) { return parseError(tokenizer, token, "unbalanced headers", .{}); } try toc.write(" \n"); break; }, Token.Id.Content => { try nodes.append(Node{ .Content = tokenizer.buffer[token.start..token.end] }); }, Token.Id.BracketOpen => { const tag_token = try eatToken(tokenizer, Token.Id.TagContent); const tag_name = tokenizer.buffer[tag_token.start..tag_token.end]; if (mem.eql(u8, tag_name, "nav")) { _ = try eatToken(tokenizer, Token.Id.BracketClose); try nodes.append(Node.Nav); } else if (mem.eql(u8, tag_name, "builtin")) { _ = try eatToken(tokenizer, Token.Id.BracketClose); try nodes.append(Node{ .Builtin = tag_token }); } else if (mem.eql(u8, tag_name, "header_open")) { _ = try eatToken(tokenizer, Token.Id.Separator); const content_token = try eatToken(tokenizer, Token.Id.TagContent); const content = tokenizer.buffer[content_token.start..content_token.end]; var columns: ?u8 = null; while (true) { const bracket_tok = tokenizer.next(); switch (bracket_tok.id) { .BracketClose => break, .Separator => continue, .TagContent => { const param = tokenizer.buffer[bracket_tok.start..bracket_tok.end]; if (mem.eql(u8, param, "3col")) { columns = 3; } else { return parseError( tokenizer, bracket_tok, "unrecognized header_open param: {}", .{param}, ); } }, else => return parseError(tokenizer, bracket_tok, "invalid header_open token", .{}), } } header_stack_size += 1; const urlized = try urlize(allocator, content); try nodes.append(Node{ .HeaderOpen = HeaderOpen{ .name = content, .url = urlized, .n = header_stack_size, }, }); if (try urls.put(urlized, tag_token)) |entry| { parseError(tokenizer, tag_token, "duplicate header url: #{}", .{urlized}) catch {}; parseError(tokenizer, entry.value, "other tag here", .{}) catch {}; return error.ParseError; } if (last_action == Action.Open) { try toc.writeByte('\n'); try toc.writeByteNTimes(' ', header_stack_size * 4); if (last_columns) |n| { try toc.print("