Merge pull request #13748 from jacobly0/c-unaligned

cbe: use memcpy for underaligned loads and stores

src/codegen/c.zig

@@ -3017,30 +3017,31 @@ fn airArg(f: *Function) CValue {
 fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
     const ty_op = f.air.instructions.items(.data)[inst].ty_op;
     const ptr_info = f.air.typeOf(ty_op.operand).ptrInfo().data;
+    const src_ty = ptr_info.pointee_type;
 
-    const inst_ty = f.air.typeOfIndex(inst);
+    if (!src_ty.hasRuntimeBitsIgnoreComptime() or
-    if (!inst_ty.hasRuntimeBitsIgnoreComptime() or
         !ptr_info.@"volatile" and f.liveness.isUnused(inst))
         return CValue.none;
 
     const target = f.object.dg.module.getTarget();
-    const is_array = lowersToArray(inst_ty, target);
+    const is_aligned = ptr_info.@"align" == 0 or ptr_info.@"align" >= src_ty.abiAlignment(target);
+    const is_array = lowersToArray(src_ty, target);
+    const need_memcpy = !is_aligned or is_array;
     const operand = try f.resolveInst(ty_op.operand);
     const writer = f.object.writer();
 
-    // We need to separately initialize arrays with a memcpy so they must be mutable.
+    // We need to initialize arrays and unaligned loads with a memcpy so they must be mutable.
-    const local = try f.allocLocal(inst_ty, if (is_array) .Mut else .Const);
+    const local = try f.allocLocal(src_ty, if (need_memcpy) .Mut else .Const);
 
-    if (is_array) {
+    if (need_memcpy) {
-        // Insert a memcpy to initialize this array. The source operand is always a pointer
-        // and thus we only need to know size/type information from the local type/dest.
         try writer.writeAll(";\n");
         try writer.writeAll("memcpy(");
+        if (!is_array) try writer.writeByte('&');
         try f.writeCValue(writer, local, .FunctionArgument);
-        try writer.writeAll(", ");
+        try writer.writeAll(", (const char *)");
-        try f.writeCValue(writer, operand, .FunctionArgument);
+        try f.writeCValue(writer, operand, .Other);
         try writer.writeAll(", sizeof(");
-        try f.renderTypecast(writer, inst_ty);
+        try f.renderTypecast(writer, src_ty);
         try writer.writeAll("))");
     } else if (ptr_info.host_size != 0) {
         var host_pl = Type.Payload.Bits{
@@ -3063,12 +3064,12 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
 
         var field_pl = Type.Payload.Bits{
             .base = .{ .tag = .int_unsigned },
-            .data = @intCast(u16, inst_ty.bitSize(target)),
+            .data = @intCast(u16, src_ty.bitSize(target)),
         };
         const field_ty = Type.initPayload(&field_pl.base);
 
         try writer.writeAll(" = (");
-        try f.renderTypecast(writer, inst_ty);
+        try f.renderTypecast(writer, src_ty);
         try writer.writeAll(")zig_wrap_");
         try f.object.dg.renderTypeForBuiltinFnName(writer, field_ty);
         try writer.writeAll("((");
@@ -3244,8 +3245,13 @@ fn airStore(f: *Function, inst: Air.Inst.Index) !CValue {
         return try airStoreUndefined(f, ptr_info.pointee_type, ptr_val);
 
     const target = f.object.dg.module.getTarget();
+    const is_aligned = ptr_info.@"align" == 0 or
+        ptr_info.@"align" >= ptr_info.pointee_type.abiAlignment(target);
+    const is_array = lowersToArray(ptr_info.pointee_type, target);
+    const need_memcpy = !is_aligned or is_array;
     const writer = f.object.writer();
-    if (lowersToArray(ptr_info.pointee_type, target)) {
+
+    if (need_memcpy) {
         // For this memcpy to safely work we need the rhs to have the same
         // underlying type as the lhs (i.e. they must both be arrays of the same underlying type).
         assert(src_ty.eql(ptr_info.pointee_type, f.object.dg.module));
@@ -3262,9 +3268,10 @@ fn airStore(f: *Function, inst: Air.Inst.Index) !CValue {
             break :blk new_local;
         } else src_val;
 
-        try writer.writeAll("memcpy(");
+        try writer.writeAll("memcpy((char *)");
         try f.writeCValue(writer, ptr_val, .FunctionArgument);
         try writer.writeAll(", ");
+        if (!is_array) try writer.writeByte('&');
         try f.writeCValue(writer, array_src, .FunctionArgument);
         try writer.writeAll(", sizeof(");
         try f.renderTypecast(writer, src_ty);
@@ -4186,11 +4193,46 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
             extra_i += clobber.len / 4 + 1;
         }
     }
-    const asm_source = std.mem.sliceAsBytes(f.air.extra[extra_i..])[0..extra.data.source_len];
+    {
+        const asm_source = std.mem.sliceAsBytes(f.air.extra[extra_i..])[0..extra.data.source_len];
 
-    try writer.writeAll("__asm");
+        var stack = std.heap.stackFallback(256, f.object.dg.gpa);
-    if (is_volatile) try writer.writeAll(" volatile");
+        const allocator = stack.get();
-    try writer.print("({s}", .{fmtStringLiteral(asm_source)});
+        const fixed_asm_source = try allocator.alloc(u8, asm_source.len);
+        defer allocator.free(fixed_asm_source);
+
+        var src_i: usize = 0;
+        var dst_i: usize = 0;
+        while (src_i < asm_source.len) : (src_i += 1) {
+            fixed_asm_source[dst_i] = asm_source[src_i];
+            dst_i += 1;
+            if (asm_source[src_i] != '%' or src_i + 1 >= asm_source.len) continue;
+            src_i += 1;
+            if (asm_source[src_i] != '[') {
+                // This handles %%
+                fixed_asm_source[dst_i] = asm_source[src_i];
+                dst_i += 1;
+                continue;
+            }
+            const len = std.mem.indexOfScalar(u8, asm_source[src_i + 1 ..], ']') orelse
+                return f.fail("CBE: invalid inline asm string '{s}'", .{asm_source});
+            if (std.mem.indexOfScalar(u8, asm_source[src_i + 1 ..][0..len], ':')) |colon| {
+                const modifier = asm_source[src_i + 1 + colon + 1 .. src_i + 1 + len];
+                std.mem.copy(u8, fixed_asm_source[dst_i..], modifier);
+                dst_i += modifier.len;
+
+                const name = asm_source[src_i .. src_i + 1 + colon];
+                std.mem.copy(u8, fixed_asm_source[dst_i..], name);
+                dst_i += name.len;
+
+                src_i += len;
+            }
+        }
+
+        try writer.writeAll("__asm");
+        if (is_volatile) try writer.writeAll(" volatile");
+        try writer.print("({s}", .{fmtStringLiteral(fixed_asm_source[0..dst_i])});
+    }
 
     extra_i = constraints_extra_begin;
     var locals_index = locals_begin;

test/behavior/asm.zig

@@ -140,7 +140,6 @@ export fn derp() i32 {
 test "asm modifiers (AArch64)" {
     if (builtin.target.cpu.arch != .aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
 
     var x: u32 = 15;
     const double = asm ("add %[ret:w], %[in:w], %[in:w]"

test/behavior/basic.zig

@@ -754,7 +754,6 @@ fn maybe(x: bool) anyerror!?u32 {
 }
 
 test "pointer to thread local array" {
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
@@ -770,7 +769,6 @@ threadlocal var buffer: [11]u8 = undefined;
 
 test "auto created variables have correct alignment" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn foo(str: [*]const u8) u32 {

test/behavior/bitreverse.zig

@@ -155,7 +155,6 @@ fn vector0() !void {
 test "bitReverse vectors u0" {
     if (builtin.zig_backend == .stage1) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
 
     comptime try vector0();

test/behavior/byteswap.zig

@@ -121,7 +121,6 @@ fn vector0() !void {
 test "@byteSwap vectors u0" {
     if (builtin.zig_backend == .stage1) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
 
     comptime try vector0();

test/behavior/math.zig

@@ -1518,7 +1518,6 @@ fn testRound(comptime T: type, x: T) !void {
 }
 
 test "vector integer addition" {
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO

test/behavior/ptrcast.zig

@@ -78,7 +78,6 @@ fn testReinterpretBytesAsExternStruct() !void {
 
 test "reinterpret bytes of an extern struct (with under-aligned fields) into another" {
     if (builtin.zig_backend == .stage1) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO: Under-aligned fields are not yet supported in the CBE
 
     try testReinterpretExternStructAsExternStruct();
     comptime try testReinterpretExternStructAsExternStruct();

test/behavior/struct.zig

@@ -641,7 +641,6 @@ test "default struct initialization fields" {
 
 test "packed array 24bits" {
     if (builtin.zig_backend == .stage1) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;

test/behavior/vector.zig

@@ -75,7 +75,6 @@ test "vector bin compares with mem.eql" {
 
 test "vector int operators" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
@@ -139,7 +138,6 @@ test "vector bit operators" {
 
 test "implicit cast vector to array" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
@@ -158,7 +156,6 @@ test "implicit cast vector to array" {
 
 test "array to vector" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
@@ -177,7 +174,6 @@ test "array to vector" {
 
 test "tuple to vector" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
@@ -210,7 +206,6 @@ test "tuple to vector" {
 
 test "vector casts of sizes not divisible by 8" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
@@ -945,7 +940,6 @@ test "multiplication-assignment operator with an array operand" {
         return error.SkipZigTest;
     }
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
@@ -1194,7 +1188,6 @@ test "zero divisor" {
 
 test "zero multiplicand" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO