zig build system improvements, add some std API

* add std.buf_map.BufMap
 * add std.buf_set.BufSet
 * add std.mem.split
 * zig build system improvements (See #204)
   - automatically parses NIX_CFLAGS_COMPILE and NIX_LDFLAGS
   - add builder.addCIncludePath
   - add builder.addRPath
   - add builder.addLibPath
   - add exe.linkLibrary

CMakeLists.txt

@@ -204,6 +204,8 @@ install(TARGETS zig DESTINATION bin)
 
 install(FILES ${C_HEADERS} DESTINATION ${C_HEADERS_DEST})
 
+install(FILES "${CMAKE_SOURCE_DIR}/std/buf_map.zig" DESTINATION "${ZIG_STD_DEST}")
+install(FILES "${CMAKE_SOURCE_DIR}/std/buf_set.zig" DESTINATION "${ZIG_STD_DEST}")
 install(FILES "${CMAKE_SOURCE_DIR}/std/build.zig" DESTINATION "${ZIG_STD_DEST}")
 install(FILES "${CMAKE_SOURCE_DIR}/std/c/darwin.zig" DESTINATION "${ZIG_STD_DEST}/c")
 install(FILES "${CMAKE_SOURCE_DIR}/std/c/index.zig" DESTINATION "${ZIG_STD_DEST}/c")

src/main.cpp

@@ -195,7 +195,7 @@ int main(int argc, char **argv) {
             fprintf(stderr, "\nBuild failed. Use the following command to reproduce the failure:\n");
             fprintf(stderr, "./build");
             for (size_t i = 0; i < args.length; i += 1) {
-                fprintf(stderr, " \"%s\"", args.at(i));
+                fprintf(stderr, " %s", args.at(i));
             }
             fprintf(stderr, "\n");
         }

std/buf_map.zig

@@ -0,0 +1,71 @@
+const HashMap = @import("hash_map.zig").HashMap;
+const mem = @import("mem.zig");
+const Allocator = mem.Allocator;
+
+/// BufMap copies keys and values before they go into the map, and
+/// frees them when they get removed.
+pub const BufMap = struct {
+    hash_map: BufMapHashMap,
+
+    const BufMapHashMap = HashMap([]const u8, []const u8, mem.hash_slice_u8, mem.eql_slice_u8);
+
+    pub fn init(allocator: &Allocator) -> BufMap {
+        var self = BufMap {
+            .hash_map = undefined,
+        };
+        self.hash_map.init(allocator);
+        return self;
+    }
+
+    pub fn deinit(self: &BufMap) {
+        var it = self.hash_map.entryIterator();
+        while (true) {
+            const entry = it.next() ?? break; 
+            self.free(entry.key);
+            self.free(entry.value);
+        }
+
+        self.hash_map.deinit();
+    }
+
+    pub fn set(self: &BufMap, key: []const u8, value: []const u8) -> %void {
+        if (const entry ?= self.hash_map.get(key)) {
+            const value_copy = %return self.copy(value);
+            %defer self.free(value_copy);
+            %return self.hash_map.put(key, value_copy);
+            self.free(entry.value);
+        } else {
+            const key_copy = %return self.copy(key);
+            %defer self.free(key_copy);
+            const value_copy = %return self.copy(value);
+            %defer self.free(value_copy);
+            %return self.hash_map.put(key_copy, value_copy);
+        }
+    }
+
+    pub fn delete(self: &BufMap, key: []const u8) {
+        const entry = self.hash_map.remove(key) ?? return;
+        self.free(entry.key);
+        self.free(entry.value);
+    }
+
+    pub fn count(self: &const BufMap) -> usize {
+        return self.hash_map.size;
+    }
+
+    pub fn iterator(self: &const BufMap) -> BufMapHashMap.Iterator {
+        return self.hash_map.entryIterator();
+    }
+
+    fn free(self: &BufMap, value: []const u8) {
+        // remove the const
+        const mut_value = @ptrcast(&u8, value.ptr)[0...value.len];
+        self.hash_map.allocator.free(mut_value);
+    }
+
+    fn copy(self: &BufMap, value: []const u8) -> %[]const u8 {
+        const result = %return self.hash_map.allocator.alloc(u8, value.len);
+        mem.copy(u8, result, value);
+        return result;
+    }
+};

std/buf_set.zig

@@ -0,0 +1,61 @@
+const HashMap = @import("hash_map.zig").HashMap;
+const mem = @import("mem.zig");
+const Allocator = mem.Allocator;
+
+pub const BufSet = struct {
+    hash_map: BufSetHashMap,
+
+    const BufSetHashMap = HashMap([]const u8, void, mem.hash_slice_u8, mem.eql_slice_u8);
+
+    pub fn init(allocator: &Allocator) -> BufSet {
+        var self = BufSet {
+            .hash_map = undefined,
+        };
+        self.hash_map.init(allocator);
+        return self;
+    }
+
+    pub fn deinit(self: &BufSet) {
+        var it = self.hash_map.entryIterator();
+        while (true) {
+            const entry = it.next() ?? break; 
+            self.free(entry.key);
+        }
+
+        self.hash_map.deinit();
+    }
+
+    pub fn put(self: &BufSet, key: []const u8) -> %void {
+        if (self.hash_map.get(key) == null) {
+            const key_copy = %return self.copy(key);
+            %defer self.free(key_copy);
+            %return self.hash_map.put(key_copy, {});
+        }
+    }
+
+    pub fn delete(self: &BufSet, key: []const u8) {
+        const entry = self.hash_map.remove(key) ?? return;
+        self.free(entry.key);
+    }
+
+    pub fn count(self: &const BufSet) -> usize {
+        return self.hash_map.size;
+    }
+
+    pub fn iterator(self: &const BufSet) -> BufSetHashMap.Iterator {
+        return self.hash_map.entryIterator();
+    }
+
+    fn free(self: &BufSet, value: []const u8) {
+        // remove the const
+        const mut_value = @ptrcast(&u8, value.ptr)[0...value.len];
+        self.hash_map.allocator.free(mut_value);
+    }
+
+    fn copy(self: &BufSet, value: []const u8) -> %[]const u8 {
+        const result = %return self.hash_map.allocator.alloc(u8, value.len);
+        mem.copy(u8, result, value);
+        return result;
+    }
+};
+

std/build.zig

@@ -6,6 +6,7 @@ const Allocator = @import("mem.zig").Allocator;
 const os = @import("os/index.zig");
 const StdIo = os.ChildProcess.StdIo;
 const Term = os.ChildProcess.Term;
+const BufSet = @import("buf_set.zig").BufSet;
 
 error ExtraArg;
 error UncleanExit;
@@ -14,13 +15,28 @@ pub const Builder = struct {
     zig_exe: []const u8,
     allocator: &Allocator,
     exe_list: List(&Exe),
+    lib_paths: List([]const u8),
+    include_paths: List([]const u8),
+    rpaths: List([]const u8),
 
     pub fn init(zig_exe: []const u8, allocator: &Allocator) -> Builder {
-        Builder {
+        var self = Builder {
             .zig_exe = zig_exe,
             .allocator = allocator,
             .exe_list = List(&Exe).init(allocator),
-        }
+            .lib_paths = List([]const u8).init(allocator),
+            .include_paths = List([]const u8).init(allocator),
+            .rpaths = List([]const u8).init(allocator),
+        };
+        self.processNixOSEnvVars();
+        return self;
+    }
+
+    pub fn deinit(self: &Builder) {
+        self.exe_list.deinit();
+        self.lib_paths.deinit();
+        self.include_paths.deinit();
+        self.rpaths.deinit();
     }
 
     pub fn addExe(self: &Builder, root_src: []const u8, name: []const u8) -> &Exe {
@@ -34,11 +50,24 @@ pub const Builder = struct {
             .name = name,
             .target = Target.Native,
             .linker_script = LinkerScript.None,
+            .link_libs = BufSet.init(self.allocator),
         };
         %return self.exe_list.append(exe);
         return exe;
     }
 
+    pub fn addCIncludePath(self: &Builder, path: []const u8) {
+        %%self.include_paths.append(path);
+    }
+
+    pub fn addRPath(self: &Builder, path: []const u8) {
+        %%self.rpaths.append(path);
+    }
+
+    pub fn addLibPath(self: &Builder, path: []const u8) {
+        %%self.lib_paths.append(path);
+    }
+
     pub fn make(self: &Builder, leftover_arg_index: usize) -> %void {
         var env_map = %return os.getEnvMap(self.allocator);
 
@@ -96,18 +125,85 @@ pub const Builder = struct {
                 },
             }
 
-            printInvocation(self.zig_exe, zig_args);
+            {
+                var it = exe.link_libs.iterator();
+                while (true) {
+                    const entry = it.next() ?? break;
+                    %return zig_args.append("--library"[0...]); // TODO issue #296
+                    %return zig_args.append(entry.key);
+                }
+            }
+
+            for (self.include_paths.toSliceConst()) |include_path| {
+                %return zig_args.append("-isystem"[0...]); // TODO issue #296
+                %return zig_args.append(include_path);
+            }
+
+            for (self.rpaths.toSliceConst()) |rpath| {
+                %return zig_args.append("-rpath"[0...]); // TODO issue #296
+                %return zig_args.append(rpath);
+            }
+
+            for (self.lib_paths.toSliceConst()) |lib_path| {
+                %return zig_args.append("--library-path"[0...]); // TODO issue #296
+                %return zig_args.append(lib_path);
+            }
+
             // TODO issue #301
             var child = %return os.ChildProcess.spawn(self.zig_exe, zig_args.toSliceConst(), &env_map,
                 StdIo.Ignore, StdIo.Inherit, StdIo.Inherit, self.allocator);
             const term = %return child.wait();
-            switch (term) {
+            const exe_result = switch (term) {
                 Term.Clean => |code| {
                     if (code != 0) {
+                        %%io.stderr.printf("\nCompile failed with code {}. To reproduce:\n", code);
+                        printInvocation(self.zig_exe, zig_args);
                         return error.UncleanExit;
                     }
                 },
-                else => return error.UncleanExit,
+                else => {
+                    %%io.stderr.printf("\nCompile crashed. To reproduce:\n");
+                    printInvocation(self.zig_exe, zig_args);
+                    return error.UncleanExit;
+                },
+            };
+        }
+    }
+
+    fn processNixOSEnvVars(self: &Builder) {
+        if (const nix_cflags_compile ?= os.getEnv("NIX_CFLAGS_COMPILE")) {
+            var it = mem.split(nix_cflags_compile, ' ');
+            while (true) {
+                const word = it.next() ?? break;
+                if (mem.eql(u8, word, "-isystem")) {
+                    const include_path = it.next() ?? {
+                        %%io.stderr.printf("Expected argument after -isystem in NIX_CFLAGS_COMPILE\n");
+                        break;
+                    };
+                    self.addCIncludePath(include_path);
+                } else {
+                    %%io.stderr.printf("Unrecognized C flag from NIX_CFLAGS_COMPILE: {}\n", word);
+                    break;
+                }
+            }
+        }
+        if (const nix_ldflags ?= os.getEnv("NIX_LDFLAGS")) {
+            var it = mem.split(nix_ldflags, ' ');
+            while (true) {
+                const word = it.next() ?? break;
+                if (mem.eql(u8, word, "-rpath")) {
+                    const rpath = it.next() ?? {
+                        %%io.stderr.printf("Expected argument after -rpath in NIX_LDFLAGS\n");
+                        break;
+                    };
+                    self.addRPath(rpath);
+                } else if (word.len > 2 and word[0] == '-' and word[1] == 'L') {
+                    const lib_path = word[2...];
+                    self.addLibPath(lib_path);
+                } else {
+                    %%io.stderr.printf("Unrecognized C flag from NIX_LDFLAGS: {}\n", word);
+                    break;
+                }
             }
         }
     }
@@ -135,6 +231,11 @@ const Exe = struct {
     name: []const u8,
     target: Target,
     linker_script: LinkerScript,
+    link_libs: BufSet,
+
+    fn deinit(self: &Exe) {
+        self.link_libs.deinit();
+    }
 
     fn setTarget(self: &Exe, target_arch: Arch, target_os: Os, target_environ: Environ) {
         self.target = Target.Cross {
@@ -155,6 +256,10 @@ const Exe = struct {
     fn setLinkerScriptPath(self: &Exe, path: []const u8) {
         self.linker_script = LinkerScript.Path { path };
     }
+
+    fn linkLibrary(self: &Exe, name: []const u8) {
+        %%self.link_libs.put(name);
+    }
 };
 
 fn handleErr(err: error) -> noreturn {

std/mem.zig

@@ -176,6 +176,49 @@ pub fn writeInt(buf: []u8, value: var, big_endian: bool) {
     assert(bits == 0);
 }
 
+
+pub fn hash_slice_u8(k: []const u8) -> u32 {
+    // FNV 32-bit hash
+    var h: u32 = 2166136261;
+    for (k) |b| {
+        h = (h ^ b) *% 16777619;
+    }
+    return h;
+}
+
+pub fn eql_slice_u8(a: []const u8, b: []const u8) -> bool {
+    return eql(u8, a, b);
+}
+
+pub fn split(s: []const u8, c: u8) -> SplitIterator {
+    SplitIterator {
+        .index = 0,
+        .s = s,
+        .c = c,
+    }
+}
+
+const SplitIterator = struct {
+    s: []const u8,
+    c: u8,
+    index: usize,
+
+    pub fn next(self: &SplitIterator) -> ?[]const u8 {
+        // move to beginning of token
+        while (self.index < self.s.len and self.s[self.index] == self.c; self.index += 1) {}
+        const start = self.index;
+        if (start == self.s.len) {
+            return null;
+        }
+
+        // move to end of token
+        while (self.index < self.s.len and self.s[self.index] != self.c; self.index += 1) {}
+        const end = self.index;
+
+        return self.s[start...end];
+    }
+};
+
 test "testStringEquality" {
     assert(eql(u8, "abcd", "abcd"));
     assert(!eql(u8, "abcdef", "abZdef"));
@@ -223,3 +266,4 @@ fn testWriteIntImpl() {
     writeInt(bytes[0...], u16(0x1234), false);
     assert(eql(u8, bytes, []u8{ 0x34, 0x12, 0x00, 0x00 }));
 }
+

std/os/child_process.zig

@@ -6,6 +6,7 @@ const Allocator = mem.Allocator;
 const errno = @import("errno.zig");
 const debug = @import("../debug.zig");
 const assert = debug.assert;
+const BufMap = @import("../buf_map.zig").BufMap;
 
 pub const ChildProcess = struct {
     pid: i32,
@@ -29,7 +30,7 @@ pub const ChildProcess = struct {
         Close,
     };
 
-    pub fn spawn(exe_path: []const u8, args: []const []const u8, env_map: &const os.EnvMap,
+    pub fn spawn(exe_path: []const u8, args: []const []const u8, env_map: &const BufMap,
         stdin: StdIo, stdout: StdIo, stderr: StdIo, allocator: &Allocator) -> %ChildProcess
     {
         switch (@compileVar("os")) {
@@ -96,7 +97,7 @@ pub const ChildProcess = struct {
         };
     }
 
-    fn spawnPosix(exe_path: []const u8, args: []const []const u8, env_map: &const os.EnvMap,
+    fn spawnPosix(exe_path: []const u8, args: []const []const u8, env_map: &const BufMap,
         stdin: StdIo, stdout: StdIo, stderr: StdIo, allocator: &Allocator) -> %ChildProcess
     {
         // TODO issue #295

std/os/index.zig

@@ -21,7 +21,7 @@ const c = @import("../c/index.zig");
 const mem = @import("../mem.zig");
 const Allocator = mem.Allocator;
 
-const HashMap = @import("../hash_map.zig").HashMap;
+const BufMap = @import("../buf_map.zig").BufMap;
 const cstr = @import("../cstr.zig");
 
 error Unexpected;
@@ -204,7 +204,7 @@ pub fn posixDup2(old_fd: i32, new_fd: i32) -> %void {
 /// It must also convert to KEY=VALUE\0 format for environment variables, and include null
 /// pointers after the args and after the environment variables.
 /// Also make the first arg equal to path.
-pub fn posixExecve(path: []const u8, argv: []const []const u8, env_map: &const EnvMap,
+pub fn posixExecve(path: []const u8, argv: []const []const u8, env_map: &const BufMap,
     allocator: &Allocator) -> %usize
 {
     const path_buf = %return allocator.alloc(u8, path.len + 1);
@@ -271,87 +271,8 @@ pub fn posixExecve(path: []const u8, argv: []const []const u8, env_map: &const E
 
 pub var environ_raw: []&u8 = undefined;
 
-pub const EnvMap = struct {
-    hash_map: EnvHashMap,
-
-    const EnvHashMap = HashMap([]const u8, []const u8, hash_slice_u8, eql_slice_u8);
-
-    pub fn init(allocator: &Allocator) -> EnvMap {
-        var self = EnvMap {
-            .hash_map = undefined,
-        };
-        self.hash_map.init(allocator);
-        return self;
-    }
-
-    pub fn deinit(self: &EnvMap) {
-        var it = self.hash_map.entryIterator();
-        while (true) {
-            const entry = it.next() ?? break; 
-            self.free(entry.key);
-            self.free(entry.value);
-        }
-
-        self.hash_map.deinit();
-    }
-
-    pub fn set(self: &EnvMap, key: []const u8, value: []const u8) -> %void {
-        if (const entry ?= self.hash_map.get(key)) {
-            const value_copy = %return self.copy(value);
-            %defer self.free(value_copy);
-            %return self.hash_map.put(key, value_copy);
-            self.free(entry.value);
-        } else {
-            const key_copy = %return self.copy(key);
-            %defer self.free(key_copy);
-            const value_copy = %return self.copy(value);
-            %defer self.free(value_copy);
-            %return self.hash_map.put(key_copy, value_copy);
-        }
-    }
-
-    pub fn delete(self: &EnvMap, key: []const u8) {
-        const entry = self.hash_map.remove(key) ?? return;
-        self.free(entry.key);
-        self.free(entry.value);
-    }
-
-    pub fn count(self: &const EnvMap) -> usize {
-        return self.hash_map.size;
-    }
-
-    pub fn iterator(self: &const EnvMap) -> EnvHashMap.Iterator {
-        return self.hash_map.entryIterator();
-    }
-
-    fn free(self: &EnvMap, value: []const u8) {
-        // remove the const
-        const mut_value = @ptrcast(&u8, value.ptr)[0...value.len];
-        self.hash_map.allocator.free(mut_value);
-    }
-
-    fn copy(self: &EnvMap, value: []const u8) -> %[]const u8 {
-        const result = %return self.hash_map.allocator.alloc(u8, value.len);
-        mem.copy(u8, result, value);
-        return result;
-    }
-
-    fn hash_slice_u8(k: []const u8) -> u32 {
-        // FNV 32-bit hash
-        var h: u32 = 2166136261;
-        for (k) |b| {
-            h = (h ^ b) *% 16777619;
-        }
-        return h;
-    }
-
-    fn eql_slice_u8(a: []const u8, b: []const u8) -> bool {
-        return mem.eql(u8, a, b);
-    }
-};
-
-pub fn getEnvMap(allocator: &Allocator) -> %EnvMap {
-    var result = EnvMap.init(allocator);
+pub fn getEnvMap(allocator: &Allocator) -> %BufMap {
+    var result = BufMap.init(allocator);
     %defer result.deinit();
 
     for (environ_raw) |ptr| {

std/special/build_runner.zig

@@ -20,6 +20,7 @@ pub fn main() -> %void {
     defer inc_allocator.deinit();
 
     var builder = Builder.init(zig_exe, &inc_allocator.allocator);
+    defer builder.deinit();
     root.build(&builder);
     %return builder.make(leftover_arg_index);
 }