move the cache system from compiler to std lib

CMakeLists.txt

@@ -216,6 +216,9 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/atomic/stack.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/base64.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/buf_map.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Build.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Build/Cache.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Build/Cache/DepTokenizer.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/builtin.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/c.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/c/linux.zig"
@@ -523,9 +526,7 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/zig/tokenizer.zig"
     "${CMAKE_SOURCE_DIR}/src/Air.zig"
     "${CMAKE_SOURCE_DIR}/src/AstGen.zig"
-    "${CMAKE_SOURCE_DIR}/src/Cache.zig"
     "${CMAKE_SOURCE_DIR}/src/Compilation.zig"
-    "${CMAKE_SOURCE_DIR}/src/DepTokenizer.zig"
     "${CMAKE_SOURCE_DIR}/src/Liveness.zig"
     "${CMAKE_SOURCE_DIR}/src/Module.zig"
     "${CMAKE_SOURCE_DIR}/src/Package.zig"

lib/std/Build.zig

@@ -19,6 +19,8 @@ const NativeTargetInfo = std.zig.system.NativeTargetInfo;
 const Sha256 = std.crypto.hash.sha2.Sha256;
 const Build = @This();
 
+pub const Cache = @import("Build/Cache.zig");
+
 /// deprecated: use `CompileStep`.
 pub const LibExeObjStep = CompileStep;
 /// deprecated: use `Build`.

lib/std/Build/Cache.zig

@@ -2,6 +2,45 @@
 //! This is not a general-purpose cache. It is designed to be fast and simple,
 //! not to withstand attacks using specially-crafted input.
 
+pub const Directory = struct {
+    /// This field is redundant for operations that can act on the open directory handle
+    /// directly, but it is needed when passing the directory to a child process.
+    /// `null` means cwd.
+    path: ?[]const u8,
+    handle: std.fs.Dir,
+
+    pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
+        if (self.path) |p| {
+            // TODO clean way to do this with only 1 allocation
+            const part2 = try std.fs.path.join(allocator, paths);
+            defer allocator.free(part2);
+            return std.fs.path.join(allocator, &[_][]const u8{ p, part2 });
+        } else {
+            return std.fs.path.join(allocator, paths);
+        }
+    }
+
+    pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
+        if (self.path) |p| {
+            // TODO clean way to do this with only 1 allocation
+            const part2 = try std.fs.path.join(allocator, paths);
+            defer allocator.free(part2);
+            return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
+        } else {
+            return std.fs.path.joinZ(allocator, paths);
+        }
+    }
+
+    /// Whether or not the handle should be closed, or the path should be freed
+    /// is determined by usage, however this function is provided for convenience
+    /// if it happens to be what the caller needs.
+    pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
+        self.handle.close();
+        if (self.path) |p| gpa.free(p);
+        self.* = undefined;
+    }
+};
+
 gpa: Allocator,
 manifest_dir: fs.Dir,
 hash: HashHelper = .{},
@@ -14,9 +53,11 @@ mutex: std.Thread.Mutex = .{},
 /// are replaced with single-character indicators. This is not to save
 /// space but to eliminate absolute file paths. This improves portability
 /// and usefulness of the cache for advanced use cases.
-prefixes_buffer: [3]Compilation.Directory = undefined,
+prefixes_buffer: [3]Directory = undefined,
 prefixes_len: usize = 0,
 
+pub const DepTokenizer = @import("Cache/DepTokenizer.zig");
+
 const Cache = @This();
 const std = @import("std");
 const builtin = @import("builtin");
@@ -27,10 +68,9 @@ const testing = std.testing;
 const mem = std.mem;
 const fmt = std.fmt;
 const Allocator = std.mem.Allocator;
-const Compilation = @import("Compilation.zig");
 const log = std.log.scoped(.cache);
 
-pub fn addPrefix(cache: *Cache, directory: Compilation.Directory) void {
+pub fn addPrefix(cache: *Cache, directory: Directory) void {
     if (directory.path) |p| {
         log.debug("Cache.addPrefix {d} {s}", .{ cache.prefixes_len, p });
     }
@@ -49,7 +89,7 @@ pub fn obtain(cache: *Cache) Manifest {
     };
 }
 
-pub fn prefixes(cache: *const Cache) []const Compilation.Directory {
+pub fn prefixes(cache: *const Cache) []const Directory {
     return cache.prefixes_buffer[0..cache.prefixes_len];
 }
 
@@ -135,8 +175,6 @@ pub const File = struct {
 pub const HashHelper = struct {
     hasher: Hasher = hasher_init,
 
-    const EmitLoc = Compilation.EmitLoc;
-
     /// Record a slice of bytes as an dependency of the process being cached
     pub fn addBytes(hh: *HashHelper, bytes: []const u8) void {
         hh.hasher.update(mem.asBytes(&bytes.len));
@@ -148,15 +186,6 @@ pub const HashHelper = struct {
         hh.addBytes(optional_bytes orelse return);
     }
 
-    pub fn addEmitLoc(hh: *HashHelper, emit_loc: EmitLoc) void {
-        hh.addBytes(emit_loc.basename);
-    }
-
-    pub fn addOptionalEmitLoc(hh: *HashHelper, optional_emit_loc: ?EmitLoc) void {
-        hh.add(optional_emit_loc != null);
-        hh.addEmitLoc(optional_emit_loc orelse return);
-    }
-
     pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void {
         hh.add(list_of_bytes.len);
         for (list_of_bytes) |bytes| hh.addBytes(bytes);
@@ -308,24 +337,6 @@ pub const Manifest = struct {
         return self.files.items.len - 1;
     }
 
-    pub fn hashCSource(self: *Manifest, c_source: Compilation.CSourceFile) !void {
-        _ = try self.addFile(c_source.src_path, null);
-        // Hash the extra flags, with special care to call addFile for file parameters.
-        // TODO this logic can likely be improved by utilizing clang_options_data.zig.
-        const file_args = [_][]const u8{"-include"};
-        var arg_i: usize = 0;
-        while (arg_i < c_source.extra_flags.len) : (arg_i += 1) {
-            const arg = c_source.extra_flags[arg_i];
-            self.hash.addBytes(arg);
-            for (file_args) |file_arg| {
-                if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) {
-                    arg_i += 1;
-                    _ = try self.addFile(c_source.extra_flags[arg_i], null);
-                }
-            }
-        }
-    }
-
     pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void {
         self.hash.add(optional_file_path != null);
         const file_path = optional_file_path orelse return;
@@ -778,7 +789,7 @@ pub const Manifest = struct {
         var error_buf = std.ArrayList(u8).init(self.cache.gpa);
         defer error_buf.deinit();
 
-        var it: @import("DepTokenizer.zig") = .{ .bytes = dep_file_contents };
+        var it: DepTokenizer = .{ .bytes = dep_file_contents };
 
         // Skip first token: target.
         switch (it.next() orelse return) { // Empty dep file OK.

src/Compilation.zig

@@ -26,7 +26,7 @@ const wasi_libc = @import("wasi_libc.zig");
 const fatal = @import("main.zig").fatal;
 const clangMain = @import("main.zig").clangMain;
 const Module = @import("Module.zig");
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
 const translate_c = @import("translate_c.zig");
 const clang = @import("clang.zig");
 const c_codegen = @import("codegen/c.zig");
@@ -807,44 +807,7 @@ pub const AllErrors = struct {
     }
 };
 
-pub const Directory = struct {
-    /// This field is redundant for operations that can act on the open directory handle
-    /// directly, but it is needed when passing the directory to a child process.
-    /// `null` means cwd.
-    path: ?[]const u8,
-    handle: std.fs.Dir,
-
-    pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
-        if (self.path) |p| {
-            // TODO clean way to do this with only 1 allocation
-            const part2 = try std.fs.path.join(allocator, paths);
-            defer allocator.free(part2);
-            return std.fs.path.join(allocator, &[_][]const u8{ p, part2 });
-        } else {
-            return std.fs.path.join(allocator, paths);
-        }
-    }
-
-    pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
-        if (self.path) |p| {
-            // TODO clean way to do this with only 1 allocation
-            const part2 = try std.fs.path.join(allocator, paths);
-            defer allocator.free(part2);
-            return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
-        } else {
-            return std.fs.path.joinZ(allocator, paths);
-        }
-    }
-
-    /// Whether or not the handle should be closed, or the path should be freed
-    /// is determined by usage, however this function is provided for convenience
-    /// if it happens to be what the caller needs.
-    pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
-        self.handle.close();
-        if (self.path) |p| gpa.free(p);
-        self.* = undefined;
-    }
-};
+pub const Directory = Cache.Directory;
 
 pub const EmitLoc = struct {
     /// If this is `null` it means the file will be output to the cache directory.
@@ -854,6 +817,35 @@ pub const EmitLoc = struct {
     basename: []const u8,
 };
 
+pub const cache_helpers = struct {
+    pub fn addEmitLoc(hh: *Cache.HashHelper, emit_loc: EmitLoc) void {
+        hh.addBytes(emit_loc.basename);
+    }
+
+    pub fn addOptionalEmitLoc(hh: *Cache.HashHelper, optional_emit_loc: ?EmitLoc) void {
+        hh.add(optional_emit_loc != null);
+        addEmitLoc(hh, optional_emit_loc orelse return);
+    }
+
+    pub fn hashCSource(self: *Cache.Manifest, c_source: Compilation.CSourceFile) !void {
+        _ = try self.addFile(c_source.src_path, null);
+        // Hash the extra flags, with special care to call addFile for file parameters.
+        // TODO this logic can likely be improved by utilizing clang_options_data.zig.
+        const file_args = [_][]const u8{"-include"};
+        var arg_i: usize = 0;
+        while (arg_i < c_source.extra_flags.len) : (arg_i += 1) {
+            const arg = c_source.extra_flags[arg_i];
+            self.hash.addBytes(arg);
+            for (file_args) |file_arg| {
+                if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) {
+                    arg_i += 1;
+                    _ = try self.addFile(c_source.extra_flags[arg_i], null);
+                }
+            }
+        }
+    }
+};
+
 pub const ClangPreprocessorMode = enum {
     no,
     /// This means we are doing `zig cc -E -o <path>`.
@@ -1522,8 +1514,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
         cache.hash.add(link_libunwind);
         cache.hash.add(options.output_mode);
         cache.hash.add(options.machine_code_model);
-        cache.hash.addOptionalEmitLoc(options.emit_bin);
-        cache.hash.addOptionalEmitLoc(options.emit_implib);
+        cache_helpers.addOptionalEmitLoc(&cache.hash, options.emit_bin);
+        cache_helpers.addOptionalEmitLoc(&cache.hash, options.emit_implib);
         cache.hash.addBytes(options.root_name);
         if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model);
         // TODO audit this and make sure everything is in it
@@ -2636,11 +2628,11 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
         man.hash.addListOfBytes(key.src.extra_flags);
     }
 
-    man.hash.addOptionalEmitLoc(comp.emit_asm);
-    man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
-    man.hash.addOptionalEmitLoc(comp.emit_llvm_bc);
-    man.hash.addOptionalEmitLoc(comp.emit_analysis);
-    man.hash.addOptionalEmitLoc(comp.emit_docs);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_asm);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_ir);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_bc);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_analysis);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_docs);
 
     man.hash.addListOfBytes(comp.clang_argv);
 
@@ -3959,11 +3951,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P
     defer man.deinit();
 
     man.hash.add(comp.clang_preprocessor_mode);
-    man.hash.addOptionalEmitLoc(comp.emit_asm);
-    man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
-    man.hash.addOptionalEmitLoc(comp.emit_llvm_bc);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_asm);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_ir);
+    cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_bc);
 
-    try man.hashCSource(c_object.src);
+    try cache_helpers.hashCSource(&man, c_object.src);
 
     var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
     defer arena_allocator.deinit();

src/Module.zig

@@ -16,7 +16,7 @@ const Ast = std.zig.Ast;
 
 const Module = @This();
 const Compilation = @import("Compilation.zig");
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
 const Value = @import("value.zig").Value;
 const Type = @import("type.zig").Type;
 const TypedValue = @import("TypedValue.zig");

src/Package.zig

@@ -13,7 +13,7 @@ const Compilation = @import("Compilation.zig");
 const Module = @import("Module.zig");
 const ThreadPool = @import("ThreadPool.zig");
 const WaitGroup = @import("WaitGroup.zig");
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
 const build_options = @import("build_options");
 const Manifest = @import("Manifest.zig");
 

src/glibc.zig

@@ -11,7 +11,7 @@ const target_util = @import("target.zig");
 const Compilation = @import("Compilation.zig");
 const build_options = @import("build_options");
 const trace = @import("tracy.zig").trace;
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
 const Package = @import("Package.zig");
 
 pub const Lib = struct {

src/link.zig

@@ -10,7 +10,7 @@ const wasi_libc = @import("wasi_libc.zig");
 
 const Air = @import("Air.zig");
 const Allocator = std.mem.Allocator;
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
 const Compilation = @import("Compilation.zig");
 const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
 const Liveness = @import("Liveness.zig");

src/link/Coff/lld.zig

@@ -5,6 +5,7 @@ const assert = std.debug.assert;
 const fs = std.fs;
 const log = std.log.scoped(.link);
 const mem = std.mem;
+const Cache = std.Build.Cache;
 
 const mingw = @import("../../mingw.zig");
 const link = @import("../../link.zig");
@@ -13,7 +14,6 @@ const trace = @import("../../tracy.zig").trace;
 
 const Allocator = mem.Allocator;
 
-const Cache = @import("../../Cache.zig");
 const Coff = @import("../Coff.zig");
 const Compilation = @import("../../Compilation.zig");
 

src/link/Elf.zig

@@ -21,7 +21,7 @@ const trace = @import("../tracy.zig").trace;
 const Air = @import("../Air.zig");
 const Allocator = std.mem.Allocator;
 pub const Atom = @import("Elf/Atom.zig");
-const Cache = @import("../Cache.zig");
+const Cache = std.Build.Cache;
 const Compilation = @import("../Compilation.zig");
 const Dwarf = @import("Dwarf.zig");
 const File = link.File;

src/link/MachO.zig

@@ -28,7 +28,7 @@ const Air = @import("../Air.zig");
 const Allocator = mem.Allocator;
 const Archive = @import("MachO/Archive.zig");
 pub const Atom = @import("MachO/Atom.zig");
-const Cache = @import("../Cache.zig");
+const Cache = std.Build.Cache;
 const CodeSignature = @import("MachO/CodeSignature.zig");
 const Compilation = @import("../Compilation.zig");
 const Dwarf = File.Dwarf;

src/link/MachO/zld.zig

@@ -20,7 +20,7 @@ const trace = @import("../../tracy.zig").trace;
 const Allocator = mem.Allocator;
 const Archive = @import("Archive.zig");
 const Atom = @import("ZldAtom.zig");
-const Cache = @import("../../Cache.zig");
+const Cache = std.Build.Cache;
 const CodeSignature = @import("CodeSignature.zig");
 const Compilation = @import("../../Compilation.zig");
 const DwarfInfo = @import("DwarfInfo.zig");

src/link/Wasm.zig

@@ -20,7 +20,7 @@ const lldMain = @import("../main.zig").lldMain;
 const trace = @import("../tracy.zig").trace;
 const build_options = @import("build_options");
 const wasi_libc = @import("../wasi_libc.zig");
-const Cache = @import("../Cache.zig");
+const Cache = std.Build.Cache;
 const Type = @import("../type.zig").Type;
 const TypedValue = @import("../TypedValue.zig");
 const LlvmObject = @import("../codegen/llvm.zig").Object;

src/main.zig

@@ -20,7 +20,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
 const wasi_libc = @import("wasi_libc.zig");
 const translate_c = @import("translate_c.zig");
 const clang = @import("clang.zig");
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
 const target_util = @import("target.zig");
 const ThreadPool = @import("ThreadPool.zig");
 const crash_report = @import("crash_report.zig");
@@ -3607,7 +3607,7 @@ fn cmdTranslateC(comp: *Compilation, arena: Allocator, enable_cache: bool) !void
     defer if (enable_cache) man.deinit();
 
     man.hash.add(@as(u16, 0xb945)); // Random number to distinguish translate-c from compiling C objects
-    man.hashCSource(c_source_file) catch |err| {
+    Compilation.cache_helpers.hashCSource(&man, c_source_file) catch |err| {
         fatal("unable to process '{s}': {s}", .{ c_source_file.src_path, @errorName(err) });
     };
 

src/mingw.zig

@@ -8,7 +8,7 @@ const log = std.log.scoped(.mingw);
 const builtin = @import("builtin");
 const Compilation = @import("Compilation.zig");
 const build_options = @import("build_options");
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
 
 pub const CRTFile = enum {
     crt2_o,