zig/lib/std/heap.zig

943 lines
38 KiB
Zig
Raw Normal View History

2019-03-02 21:46:04 +00:00
const std = @import("std.zig");
const debug = std.debug;
const assert = debug.assert;
const testing = std.testing;
const mem = std.mem;
const os = std.os;
const builtin = @import("builtin");
const c = std.c;
const maxInt = std.math.maxInt;
pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator;
const Allocator = mem.Allocator;
pub const c_allocator = &c_allocator_state;
var c_allocator_state = Allocator{
.reallocFn = cRealloc,
.shrinkFn = cShrink,
};
fn cRealloc(self: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
assert(new_align <= @alignOf(c_longdouble));
const old_ptr = if (old_mem.len == 0) null else @ptrCast(*c_void, old_mem.ptr);
const buf = c.realloc(old_ptr, new_size) orelse return error.OutOfMemory;
return @ptrCast([*]u8, buf)[0..new_size];
}
fn cShrink(self: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
const old_ptr = @ptrCast(*c_void, old_mem.ptr);
const buf = c.realloc(old_ptr, new_size) orelse return old_mem[0..new_size];
return @ptrCast([*]u8, buf)[0..new_size];
}
/// This allocator makes a syscall directly for every allocation and free.
/// Thread-safe and lock-free.
pub const page_allocator = if (std.Target.current.isWasm())
&wasm_page_allocator_state
else
&page_allocator_state;
var page_allocator_state = Allocator{
.reallocFn = PageAllocator.realloc,
.shrinkFn = PageAllocator.shrink,
};
var wasm_page_allocator_state = Allocator{
.reallocFn = WasmPageAllocator.realloc,
.shrinkFn = WasmPageAllocator.shrink,
};
/// Deprecated. Use `page_allocator`.
pub const direct_allocator = page_allocator;
const PageAllocator = struct {
breaking API changes to all readInt/writeInt functions & more * add `@bswap` builtin function. See #767 * comptime evaluation facilities are improved to be able to handle a `@ptrCast` with a backing array. * `@truncate` allows "truncating" a u0 value to any integer type, and the result is always comptime known to be `0`. * when specifying pointer alignment in a type expression, the alignment value of pointers which do not have addresses at runtime is ignored, and always has the default/ABI alignment * threw in a fix to freebsd/x86_64.zig to update syntax from language changes * some improvements are pending #863 closes #638 closes #1733 std lib API changes * io.InStream().readIntNe renamed to readIntNative * io.InStream().readIntLe renamed to readIntLittle * io.InStream().readIntBe renamed to readIntBig * introduced io.InStream().readIntForeign * io.InStream().readInt has parameter order changed * io.InStream().readVarInt has parameter order changed * io.InStream().writeIntNe renamed to writeIntNative * introduced io.InStream().writeIntForeign * io.InStream().writeIntLe renamed to writeIntLittle * io.InStream().writeIntBe renamed to writeIntBig * io.InStream().writeInt has parameter order changed * mem.readInt has different parameters and semantics * introduced mem.readIntNative * introduced mem.readIntForeign * mem.readIntBE renamed to mem.readIntBig and different API * mem.readIntLE renamed to mem.readIntLittle and different API * introduced mem.readIntSliceNative * introduced mem.readIntSliceForeign * introduced mem.readIntSliceLittle * introduced mem.readIntSliceBig * introduced mem.readIntSlice * mem.writeInt has different parameters and semantics * introduced mem.writeIntNative * introduced mem.writeIntForeign * mem.writeIntBE renamed to mem.readIntBig and different semantics * mem.writeIntLE renamed to mem.readIntLittle and different semantics * introduced mem.writeIntSliceForeign * introduced mem.writeIntSliceNative * introduced mem.writeIntSliceBig * introduced mem.writeIntSliceLittle * introduced mem.writeIntSlice * removed mem.endianSwapIfLe * removed mem.endianSwapIfBe * removed mem.endianSwapIf * added mem.littleToNative * added mem.bigToNative * added mem.toNative * added mem.nativeTo * added mem.nativeToLittle * added mem.nativeToBig
2018-12-13 01:19:46 +00:00
fn alloc(allocator: *Allocator, n: usize, alignment: u29) error{OutOfMemory}![]u8 {
if (n == 0) return &[0]u8{};
if (builtin.os == .windows) {
2019-05-25 17:07:44 +00:00
const w = os.windows;
// Although officially it's at least aligned to page boundary,
// Windows is known to reserve pages on a 64K boundary. It's
// even more likely that the requested alignment is <= 64K than
// 4K, so we're just allocating blindly and hoping for the best.
// see https://devblogs.microsoft.com/oldnewthing/?p=42223
const addr = w.VirtualAlloc(
null,
n,
w.MEM_COMMIT | w.MEM_RESERVE,
w.PAGE_READWRITE,
) catch return error.OutOfMemory;
// If the allocation is sufficiently aligned, use it.
if (@ptrToInt(addr) & (alignment - 1) == 0) {
return @ptrCast([*]u8, addr)[0..n];
}
// If it wasn't, actually do an explicitely aligned allocation.
w.VirtualFree(addr, 0, w.MEM_RELEASE);
const alloc_size = n + alignment;
const final_addr = while (true) {
// Reserve a range of memory large enough to find a sufficiently
// aligned address.
const reserved_addr = w.VirtualAlloc(
null,
2019-05-25 17:07:44 +00:00
alloc_size,
w.MEM_RESERVE,
w.PAGE_NOACCESS,
) catch return error.OutOfMemory;
const aligned_addr = mem.alignForward(@ptrToInt(reserved_addr), alignment);
// Release the reserved pages (not actually used).
w.VirtualFree(reserved_addr, 0, w.MEM_RELEASE);
// At this point, it is possible that another thread has
// obtained some memory space that will cause the next
// VirtualAlloc call to fail. To handle this, we will retry
// until it succeeds.
2019-05-27 04:48:56 +00:00
const ptr = w.VirtualAlloc(
2019-05-25 17:07:44 +00:00
@intToPtr(*c_void, aligned_addr),
n,
w.MEM_COMMIT | w.MEM_RESERVE,
w.PAGE_READWRITE,
2019-05-25 17:07:44 +00:00
) catch continue;
2019-05-27 04:48:56 +00:00
return @ptrCast([*]u8, ptr)[0..n];
2019-05-25 17:07:44 +00:00
};
2019-05-25 17:07:44 +00:00
return @ptrCast([*]u8, final_addr)[0..n];
}
2019-05-26 17:17:34 +00:00
const alloc_size = if (alignment <= mem.page_size) n else n + alignment;
2019-05-26 23:56:37 +00:00
const slice = os.mmap(
2019-05-25 17:07:44 +00:00
null,
2019-05-26 23:56:37 +00:00
mem.alignForward(alloc_size, mem.page_size),
2019-05-25 17:07:44 +00:00
os.PROT_READ | os.PROT_WRITE,
os.MAP_PRIVATE | os.MAP_ANONYMOUS,
-1,
0,
) catch return error.OutOfMemory;
2019-05-27 03:35:26 +00:00
if (alloc_size == n) return slice[0..n];
2019-05-25 17:07:44 +00:00
2019-05-26 23:56:37 +00:00
const aligned_addr = mem.alignForward(@ptrToInt(slice.ptr), alignment);
2019-05-25 17:07:44 +00:00
// Unmap the extra bytes that were only requested in order to guarantee
// that the range of memory we were provided had a proper alignment in
// it somewhere. The extra bytes could be at the beginning, or end, or both.
2019-05-26 23:56:37 +00:00
const unused_start_len = aligned_addr - @ptrToInt(slice.ptr);
2019-05-25 17:07:44 +00:00
if (unused_start_len != 0) {
2019-05-26 23:56:37 +00:00
os.munmap(slice[0..unused_start_len]);
2019-05-25 17:07:44 +00:00
}
2019-05-26 23:56:37 +00:00
const aligned_end_addr = mem.alignForward(aligned_addr + n, mem.page_size);
const unused_end_len = @ptrToInt(slice.ptr) + slice.len - aligned_end_addr;
2019-05-25 17:07:44 +00:00
if (unused_end_len != 0) {
2019-05-26 23:56:37 +00:00
os.munmap(@intToPtr([*]align(mem.page_size) u8, aligned_end_addr)[0..unused_end_len]);
}
2019-05-25 17:07:44 +00:00
return @intToPtr([*]u8, aligned_addr)[0..n];
}
2019-05-26 23:56:37 +00:00
fn shrink(allocator: *Allocator, old_mem_unaligned: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
const old_mem = @alignCast(mem.page_size, old_mem_unaligned);
if (builtin.os == .windows) {
2019-05-25 17:07:44 +00:00
const w = os.windows;
if (new_size == 0) {
// From the docs:
// "If the dwFreeType parameter is MEM_RELEASE, this parameter
// must be 0 (zero). The function frees the entire region that
// is reserved in the initial allocation call to VirtualAlloc."
// So we can only use MEM_RELEASE when actually releasing the
// whole allocation.
w.VirtualFree(old_mem.ptr, 0, w.MEM_RELEASE);
} else {
const base_addr = @ptrToInt(old_mem.ptr);
const old_addr_end = base_addr + old_mem.len;
const new_addr_end = base_addr + new_size;
2019-05-26 17:17:34 +00:00
const new_addr_end_rounded = mem.alignForward(new_addr_end, mem.page_size);
if (old_addr_end > new_addr_end_rounded) {
2019-05-25 17:07:44 +00:00
// For shrinking that is not releasing, we will only
// decommit the pages not needed anymore.
w.VirtualFree(
@intToPtr(*c_void, new_addr_end_rounded),
old_addr_end - new_addr_end_rounded,
w.MEM_DECOMMIT,
);
}
2019-05-25 17:07:44 +00:00
}
return old_mem[0..new_size];
}
const base_addr = @ptrToInt(old_mem.ptr);
const old_addr_end = base_addr + old_mem.len;
const new_addr_end = base_addr + new_size;
2019-05-26 17:17:34 +00:00
const new_addr_end_rounded = mem.alignForward(new_addr_end, mem.page_size);
2019-05-25 17:07:44 +00:00
if (old_addr_end > new_addr_end_rounded) {
2019-05-26 23:56:37 +00:00
const ptr = @intToPtr([*]align(mem.page_size) u8, new_addr_end_rounded);
os.munmap(ptr[0 .. old_addr_end - new_addr_end_rounded]);
}
2019-05-25 17:07:44 +00:00
return old_mem[0..new_size];
}
2019-05-26 23:56:37 +00:00
fn realloc(allocator: *Allocator, old_mem_unaligned: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
const old_mem = @alignCast(mem.page_size, old_mem_unaligned);
if (builtin.os == .windows) {
2019-05-25 17:07:44 +00:00
if (old_mem.len == 0) {
return alloc(allocator, new_size, new_align);
}
if (new_size <= old_mem.len and new_align <= old_align) {
return shrink(allocator, old_mem, old_align, new_size, new_align);
}
const w = os.windows;
const base_addr = @ptrToInt(old_mem.ptr);
if (new_align > old_align and base_addr & (new_align - 1) != 0) {
// Current allocation doesn't satisfy the new alignment.
// For now we'll do a new one no matter what, but maybe
// there is something smarter to do instead.
const result = try alloc(allocator, new_size, new_align);
2019-05-25 17:07:44 +00:00
assert(old_mem.len != 0);
@memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
w.VirtualFree(old_mem.ptr, 0, w.MEM_RELEASE);
2019-05-25 17:07:44 +00:00
return result;
}
const old_addr_end = base_addr + old_mem.len;
2019-05-26 17:17:34 +00:00
const old_addr_end_rounded = mem.alignForward(old_addr_end, mem.page_size);
2019-05-25 17:07:44 +00:00
const new_addr_end = base_addr + new_size;
2019-05-26 17:17:34 +00:00
const new_addr_end_rounded = mem.alignForward(new_addr_end, mem.page_size);
2019-05-25 17:07:44 +00:00
if (new_addr_end_rounded == old_addr_end_rounded) {
// The reallocation fits in the already allocated pages.
return @ptrCast([*]u8, old_mem.ptr)[0..new_size];
2019-05-25 17:07:44 +00:00
}
assert(new_addr_end_rounded > old_addr_end_rounded);
// We need to commit new pages.
const additional_size = new_addr_end - old_addr_end_rounded;
const realloc_addr = w.kernel32.VirtualAlloc(
@intToPtr(*c_void, old_addr_end_rounded),
additional_size,
w.MEM_COMMIT | w.MEM_RESERVE,
w.PAGE_READWRITE,
) orelse {
// Committing new pages at the end of the existing allocation
// failed, we need to try a new one.
const new_alloc_mem = try alloc(allocator, new_size, new_align);
@memcpy(new_alloc_mem.ptr, old_mem.ptr, old_mem.len);
w.VirtualFree(old_mem.ptr, 0, w.MEM_RELEASE);
return new_alloc_mem;
};
assert(@ptrToInt(realloc_addr) == old_addr_end_rounded);
return @ptrCast([*]u8, old_mem.ptr)[0..new_size];
}
if (new_size <= old_mem.len and new_align <= old_align) {
return shrink(allocator, old_mem, old_align, new_size, new_align);
}
const result = try alloc(allocator, new_size, new_align);
if (old_mem.len != 0) {
@memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
2019-05-26 23:56:37 +00:00
os.munmap(old_mem);
}
2019-05-25 17:07:44 +00:00
return result;
}
};
// TODO Exposed LLVM intrinsics is a bug
// See: https://github.com/ziglang/zig/issues/2291
extern fn @"llvm.wasm.memory.size.i32"(u32) u32;
extern fn @"llvm.wasm.memory.grow.i32"(u32, u32) i32;
/// TODO: make this re-use freed pages, and cooperate with other callers of these global intrinsics
/// by better utilizing the return value of grow()
const WasmPageAllocator = struct {
var start_ptr: [*]u8 = undefined;
var num_pages: usize = 0;
var end_index: usize = 0;
comptime {
if (builtin.arch != .wasm32) {
@compileError("WasmPageAllocator is only available for wasm32 arch");
}
}
fn alloc(allocator: *Allocator, size: usize, alignment: u29) ![]u8 {
const addr = @ptrToInt(start_ptr) + end_index;
const adjusted_addr = mem.alignForward(addr, alignment);
const adjusted_index = end_index + (adjusted_addr - addr);
const new_end_index = adjusted_index + size;
if (new_end_index > num_pages * mem.page_size) {
const required_memory = new_end_index - (num_pages * mem.page_size);
var inner_num_pages: usize = required_memory / mem.page_size;
if (required_memory % mem.page_size != 0) {
inner_num_pages += 1;
}
const prev_page = @"llvm.wasm.memory.grow.i32"(0, @intCast(u32, inner_num_pages));
if (prev_page == -1) {
return error.OutOfMemory;
}
num_pages += inner_num_pages;
}
const result = start_ptr[adjusted_index..new_end_index];
end_index = new_end_index;
return result;
}
// Check if memory is the last "item" and is aligned correctly
fn is_last_item(memory: []u8, alignment: u29) bool {
return memory.ptr == start_ptr + end_index - memory.len and mem.alignForward(@ptrToInt(memory.ptr), alignment) == @ptrToInt(memory.ptr);
}
fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
// Initialize start_ptr at the first realloc
if (num_pages == 0) {
start_ptr = @intToPtr([*]u8, @intCast(usize, @"llvm.wasm.memory.size.i32"(0)) * mem.page_size);
}
if (is_last_item(old_mem, new_align)) {
const start_index = end_index - old_mem.len;
const new_end_index = start_index + new_size;
if (new_end_index > num_pages * mem.page_size) {
_ = try alloc(allocator, new_end_index - end_index, new_align);
}
const result = start_ptr[start_index..new_end_index];
end_index = new_end_index;
return result;
} else if (new_size <= old_mem.len and new_align <= old_align) {
return error.OutOfMemory;
} else {
const result = try alloc(allocator, new_size, new_align);
mem.copy(u8, result, old_mem);
return result;
}
}
fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
return old_mem[0..new_size];
}
};
pub const HeapAllocator = switch (builtin.os) {
.windows => struct {
allocator: Allocator,
heap_handle: ?HeapHandle,
const HeapHandle = os.windows.HANDLE;
pub fn init() HeapAllocator {
return HeapAllocator{
.allocator = Allocator{
.reallocFn = realloc,
.shrinkFn = shrink,
},
.heap_handle = null,
};
}
pub fn deinit(self: *HeapAllocator) void {
if (self.heap_handle) |heap_handle| {
2019-05-27 05:35:58 +00:00
os.windows.HeapDestroy(heap_handle);
}
}
fn alloc(allocator: *Allocator, n: usize, alignment: u29) error{OutOfMemory}![]u8 {
const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);
if (n == 0) return &[0]u8{};
const amt = n + alignment + @sizeOf(usize);
const optional_heap_handle = @atomicLoad(?HeapHandle, &self.heap_handle, builtin.AtomicOrder.SeqCst);
const heap_handle = optional_heap_handle orelse blk: {
const options = if (builtin.single_threaded) os.windows.HEAP_NO_SERIALIZE else 0;
2019-05-27 05:35:58 +00:00
const hh = os.windows.kernel32.HeapCreate(options, amt, 0) orelse return error.OutOfMemory;
const other_hh = @cmpxchgStrong(?HeapHandle, &self.heap_handle, null, hh, builtin.AtomicOrder.SeqCst, builtin.AtomicOrder.SeqCst) orelse break :blk hh;
2019-05-27 05:35:58 +00:00
os.windows.HeapDestroy(hh);
break :blk other_hh.?; // can't be null because of the cmpxchg
};
2019-05-27 05:35:58 +00:00
const ptr = os.windows.kernel32.HeapAlloc(heap_handle, 0, amt) orelse return error.OutOfMemory;
const root_addr = @ptrToInt(ptr);
const adjusted_addr = mem.alignForward(root_addr, alignment);
const record_addr = adjusted_addr + n;
@intToPtr(*align(1) usize, record_addr).* = root_addr;
return @intToPtr([*]u8, adjusted_addr)[0..n];
}
fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
return realloc(allocator, old_mem, old_align, new_size, new_align) catch {
const old_adjusted_addr = @ptrToInt(old_mem.ptr);
const old_record_addr = old_adjusted_addr + old_mem.len;
const root_addr = @intToPtr(*align(1) usize, old_record_addr).*;
const old_ptr = @intToPtr(*c_void, root_addr);
const new_record_addr = old_record_addr - new_size + old_mem.len;
@intToPtr(*align(1) usize, new_record_addr).* = root_addr;
return old_mem[0..new_size];
};
}
fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
if (old_mem.len == 0) return alloc(allocator, new_size, new_align);
const self = @fieldParentPtr(HeapAllocator, "allocator", allocator);
const old_adjusted_addr = @ptrToInt(old_mem.ptr);
const old_record_addr = old_adjusted_addr + old_mem.len;
const root_addr = @intToPtr(*align(1) usize, old_record_addr).*;
const old_ptr = @intToPtr(*c_void, root_addr);
if (new_size == 0) {
2019-05-27 05:35:58 +00:00
os.windows.HeapFree(self.heap_handle.?, 0, old_ptr);
return old_mem[0..0];
}
const amt = new_size + new_align + @sizeOf(usize);
2019-05-27 05:35:58 +00:00
const new_ptr = os.windows.kernel32.HeapReAlloc(
self.heap_handle.?,
0,
old_ptr,
amt,
) orelse return error.OutOfMemory;
const offset = old_adjusted_addr - root_addr;
const new_root_addr = @ptrToInt(new_ptr);
var new_adjusted_addr = new_root_addr + offset;
const offset_is_valid = new_adjusted_addr + new_size + @sizeOf(usize) <= new_root_addr + amt;
const offset_is_aligned = new_adjusted_addr % new_align == 0;
if (!offset_is_valid or !offset_is_aligned) {
// If HeapReAlloc didn't happen to move the memory to the new alignment,
// or the memory starting at the old offset would be outside of the new allocation,
// then we need to copy the memory to a valid aligned address and use that
const new_aligned_addr = mem.alignForward(new_root_addr, new_align);
@memcpy(@intToPtr([*]u8, new_aligned_addr), @intToPtr([*]u8, new_adjusted_addr), std.math.min(old_mem.len, new_size));
new_adjusted_addr = new_aligned_addr;
}
const new_record_addr = new_adjusted_addr + new_size;
@intToPtr(*align(1) usize, new_record_addr).* = new_root_addr;
return @intToPtr([*]u8, new_adjusted_addr)[0..new_size];
}
},
else => @compileError("Unsupported OS"),
};
/// This allocator takes an existing allocator, wraps it, and provides an interface
/// where you can allocate without freeing, and then free it all together.
pub const ArenaAllocator = struct {
2019-10-21 19:22:08 +00:00
allocator: Allocator,
child_allocator: *Allocator,
buffer_list: std.SinglyLinkedList([]u8),
end_index: usize,
const BufNode = std.SinglyLinkedList([]u8).Node;
pub fn init(child_allocator: *Allocator) ArenaAllocator {
return ArenaAllocator{
.allocator = Allocator{
.reallocFn = realloc,
.shrinkFn = shrink,
},
.child_allocator = child_allocator,
.buffer_list = std.SinglyLinkedList([]u8).init(),
.end_index = 0,
};
}
pub fn deinit(self: *ArenaAllocator) void {
var it = self.buffer_list.first;
while (it) |node| {
// this has to occur before the free because the free frees node
2019-06-22 05:13:10 +00:00
const next_it = node.next;
self.child_allocator.free(node.data);
2019-06-22 05:13:10 +00:00
it = next_it;
}
}
fn createNode(self: *ArenaAllocator, prev_len: usize, minimum_size: usize) !*BufNode {
const actual_min_size = minimum_size + @sizeOf(BufNode);
var len = prev_len;
while (true) {
len += len / 2;
2019-05-26 17:17:34 +00:00
len += mem.page_size - @rem(len, mem.page_size);
if (len >= actual_min_size) break;
}
const buf = try self.child_allocator.alignedAlloc(u8, @alignOf(BufNode), len);
const buf_node_slice = @bytesToSlice(BufNode, buf[0..@sizeOf(BufNode)]);
const buf_node = &buf_node_slice[0];
buf_node.* = BufNode{
.data = buf,
.next = null,
};
self.buffer_list.prepend(buf_node);
self.end_index = 0;
return buf_node;
}
fn alloc(allocator: *Allocator, n: usize, alignment: u29) ![]u8 {
const self = @fieldParentPtr(ArenaAllocator, "allocator", allocator);
var cur_node = if (self.buffer_list.first) |first_node| first_node else try self.createNode(0, n + alignment);
while (true) {
const cur_buf = cur_node.data[@sizeOf(BufNode)..];
const addr = @ptrToInt(cur_buf.ptr) + self.end_index;
const adjusted_addr = mem.alignForward(addr, alignment);
const adjusted_index = self.end_index + (adjusted_addr - addr);
const new_end_index = adjusted_index + n;
if (new_end_index > cur_buf.len) {
cur_node = try self.createNode(cur_buf.len, n + alignment);
continue;
}
const result = cur_buf[adjusted_index..new_end_index];
self.end_index = new_end_index;
return result;
}
}
fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
if (new_size <= old_mem.len and new_align <= new_size) {
// We can't do anything with the memory, so tell the client to keep it.
return error.OutOfMemory;
} else {
const result = try alloc(allocator, new_size, new_align);
@memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
return result;
}
}
fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
return old_mem[0..new_size];
}
};
pub const FixedBufferAllocator = struct {
allocator: Allocator,
end_index: usize,
buffer: []u8,
pub fn init(buffer: []u8) FixedBufferAllocator {
return FixedBufferAllocator{
.allocator = Allocator{
.reallocFn = realloc,
.shrinkFn = shrink,
},
.buffer = buffer,
.end_index = 0,
};
2017-11-05 17:27:56 +00:00
}
fn alloc(allocator: *Allocator, n: usize, alignment: u29) ![]u8 {
const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);
2018-04-28 21:53:06 +00:00
const addr = @ptrToInt(self.buffer.ptr) + self.end_index;
const adjusted_addr = mem.alignForward(addr, alignment);
const adjusted_index = self.end_index + (adjusted_addr - addr);
const new_end_index = adjusted_index + n;
if (new_end_index > self.buffer.len) {
return error.OutOfMemory;
}
const result = self.buffer[adjusted_index..new_end_index];
self.end_index = new_end_index;
return result;
}
fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
const self = @fieldParentPtr(FixedBufferAllocator, "allocator", allocator);
assert(old_mem.len <= self.end_index);
if (old_mem.ptr == self.buffer.ptr + self.end_index - old_mem.len and
mem.alignForward(@ptrToInt(old_mem.ptr), new_align) == @ptrToInt(old_mem.ptr))
{
const start_index = self.end_index - old_mem.len;
const new_end_index = start_index + new_size;
if (new_end_index > self.buffer.len) return error.OutOfMemory;
const result = self.buffer[start_index..new_end_index];
self.end_index = new_end_index;
return result;
} else if (new_size <= old_mem.len and new_align <= old_align) {
// We can't do anything with the memory, so tell the client to keep it.
return error.OutOfMemory;
} else {
const result = try alloc(allocator, new_size, new_align);
@memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
return result;
}
}
fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
return old_mem[0..new_size];
}
2019-09-16 01:35:01 +00:00
2019-09-21 15:42:26 +00:00
pub fn reset(self: *FixedBufferAllocator) void {
2019-09-16 01:35:01 +00:00
self.end_index = 0;
}
};
pub const ThreadSafeFixedBufferAllocator = blk: {
if (builtin.single_threaded) {
break :blk FixedBufferAllocator;
} else {
// lock free
break :blk struct {
allocator: Allocator,
end_index: usize,
buffer: []u8,
pub fn init(buffer: []u8) ThreadSafeFixedBufferAllocator {
return ThreadSafeFixedBufferAllocator{
.allocator = Allocator{
.reallocFn = realloc,
.shrinkFn = shrink,
},
.buffer = buffer,
.end_index = 0,
};
}
2018-04-28 21:53:06 +00:00
fn alloc(allocator: *Allocator, n: usize, alignment: u29) ![]u8 {
const self = @fieldParentPtr(ThreadSafeFixedBufferAllocator, "allocator", allocator);
var end_index = @atomicLoad(usize, &self.end_index, builtin.AtomicOrder.SeqCst);
while (true) {
const addr = @ptrToInt(self.buffer.ptr) + end_index;
const adjusted_addr = mem.alignForward(addr, alignment);
const adjusted_index = end_index + (adjusted_addr - addr);
const new_end_index = adjusted_index + n;
if (new_end_index > self.buffer.len) {
return error.OutOfMemory;
}
end_index = @cmpxchgWeak(usize, &self.end_index, end_index, new_end_index, builtin.AtomicOrder.SeqCst, builtin.AtomicOrder.SeqCst) orelse return self.buffer[adjusted_index..new_end_index];
}
}
2018-04-28 21:53:06 +00:00
fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
if (new_size <= old_mem.len and new_align <= old_align) {
// We can't do anything useful with the memory, tell the client to keep it.
return error.OutOfMemory;
} else {
const result = try alloc(allocator, new_size, new_align);
@memcpy(result.ptr, old_mem.ptr, std.math.min(old_mem.len, result.len));
return result;
}
2018-04-28 21:53:06 +00:00
}
fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
return old_mem[0..new_size];
}
};
2018-04-28 21:53:06 +00:00
}
};
pub fn stackFallback(comptime size: usize, fallback_allocator: *Allocator) StackFallbackAllocator(size) {
return StackFallbackAllocator(size){
.buffer = undefined,
.fallback_allocator = fallback_allocator,
.fixed_buffer_allocator = undefined,
.allocator = Allocator{
.reallocFn = StackFallbackAllocator(size).realloc,
.shrinkFn = StackFallbackAllocator(size).shrink,
},
};
}
pub fn StackFallbackAllocator(comptime size: usize) type {
return struct {
const Self = @This();
buffer: [size]u8,
allocator: Allocator,
fallback_allocator: *Allocator,
fixed_buffer_allocator: FixedBufferAllocator,
pub fn get(self: *Self) *Allocator {
self.fixed_buffer_allocator = FixedBufferAllocator.init(self.buffer[0..]);
return &self.allocator;
}
fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 {
const self = @fieldParentPtr(Self, "allocator", allocator);
const in_buffer = @ptrToInt(old_mem.ptr) >= @ptrToInt(&self.buffer) and
@ptrToInt(old_mem.ptr) < @ptrToInt(&self.buffer) + self.buffer.len;
if (in_buffer) {
return FixedBufferAllocator.realloc(
&self.fixed_buffer_allocator.allocator,
old_mem,
old_align,
new_size,
new_align,
) catch {
const result = try self.fallback_allocator.reallocFn(
self.fallback_allocator,
&[0]u8{},
undefined,
new_size,
new_align,
);
mem.copy(u8, result, old_mem);
return result;
};
}
return self.fallback_allocator.reallocFn(
self.fallback_allocator,
old_mem,
old_align,
new_size,
new_align,
);
}
fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 {
const self = @fieldParentPtr(Self, "allocator", allocator);
const in_buffer = @ptrToInt(old_mem.ptr) >= @ptrToInt(&self.buffer) and
@ptrToInt(old_mem.ptr) < @ptrToInt(&self.buffer) + self.buffer.len;
if (in_buffer) {
return FixedBufferAllocator.shrink(
&self.fixed_buffer_allocator.allocator,
old_mem,
old_align,
new_size,
new_align,
);
}
return self.fallback_allocator.shrinkFn(
self.fallback_allocator,
old_mem,
old_align,
new_size,
new_align,
);
}
};
}
test "c_allocator" {
if (builtin.link_libc) {
var slice = try c_allocator.alloc(u8, 50);
defer c_allocator.free(slice);
slice = try c_allocator.realloc(slice, 100);
}
}
test "PageAllocator" {
const allocator = page_allocator;
try testAllocator(allocator);
try testAllocatorAligned(allocator, 16);
try testAllocatorLargeAlignment(allocator);
try testAllocatorAlignedShrink(allocator);
if (builtin.os == .windows) {
// Trying really large alignment. As mentionned in the implementation,
// VirtualAlloc returns 64K aligned addresses. We want to make sure
// PageAllocator works beyond that, as it's not tested by
// `testAllocatorLargeAlignment`.
const slice = try allocator.alignedAlloc(u8, 1 << 20, 128);
slice[0] = 0x12;
slice[127] = 0x34;
allocator.free(slice);
}
}
test "HeapAllocator" {
if (builtin.os == .windows) {
var heap_allocator = HeapAllocator.init();
defer heap_allocator.deinit();
const allocator = &heap_allocator.allocator;
try testAllocator(allocator);
try testAllocatorAligned(allocator, 16);
try testAllocatorLargeAlignment(allocator);
try testAllocatorAlignedShrink(allocator);
}
}
2019-06-22 05:13:10 +00:00
test "ArenaAllocator" {
var arena_allocator = ArenaAllocator.init(page_allocator);
2019-06-22 05:13:10 +00:00
defer arena_allocator.deinit();
try testAllocator(&arena_allocator.allocator);
try testAllocatorAligned(&arena_allocator.allocator, 16);
try testAllocatorLargeAlignment(&arena_allocator.allocator);
try testAllocatorAlignedShrink(&arena_allocator.allocator);
}
var test_fixed_buffer_allocator_memory: [80000 * @sizeOf(u64)]u8 = undefined;
test "FixedBufferAllocator" {
var fixed_buffer_allocator = FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]);
try testAllocator(&fixed_buffer_allocator.allocator);
try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16);
try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator);
try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);
}
2019-09-16 01:35:01 +00:00
test "FixedBufferAllocator.reset" {
var buf: [8]u8 align(@alignOf(u64)) = undefined;
2019-09-16 01:35:01 +00:00
var fba = FixedBufferAllocator.init(buf[0..]);
const X = 0xeeeeeeeeeeeeeeee;
const Y = 0xffffffffffffffff;
var x = try fba.allocator.create(u64);
x.* = X;
testing.expectError(error.OutOfMemory, fba.allocator.create(u64));
fba.reset();
var y = try fba.allocator.create(u64);
y.* = Y;
// we expect Y to have overwritten X.
testing.expect(x.* == y.*);
testing.expect(y.* == Y);
}
test "FixedBufferAllocator Reuse memory on realloc" {
var small_fixed_buffer: [10]u8 = undefined;
// check if we re-use the memory
{
var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]);
var slice0 = try fixed_buffer_allocator.allocator.alloc(u8, 5);
testing.expect(slice0.len == 5);
var slice1 = try fixed_buffer_allocator.allocator.realloc(slice0, 10);
testing.expect(slice1.ptr == slice0.ptr);
testing.expect(slice1.len == 10);
testing.expectError(error.OutOfMemory, fixed_buffer_allocator.allocator.realloc(slice1, 11));
}
// check that we don't re-use the memory if it's not the most recent block
{
var fixed_buffer_allocator = FixedBufferAllocator.init(small_fixed_buffer[0..]);
var slice0 = try fixed_buffer_allocator.allocator.alloc(u8, 2);
slice0[0] = 1;
slice0[1] = 2;
var slice1 = try fixed_buffer_allocator.allocator.alloc(u8, 2);
var slice2 = try fixed_buffer_allocator.allocator.realloc(slice0, 4);
testing.expect(slice0.ptr != slice2.ptr);
testing.expect(slice1.ptr != slice2.ptr);
testing.expect(slice2[0] == 1);
testing.expect(slice2[1] == 2);
}
}
2018-04-28 21:53:06 +00:00
test "ThreadSafeFixedBufferAllocator" {
var fixed_buffer_allocator = ThreadSafeFixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]);
try testAllocator(&fixed_buffer_allocator.allocator);
try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16);
2018-04-28 21:53:06 +00:00
try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator);
try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator);
2018-04-28 21:53:06 +00:00
}
fn testAllocator(allocator: *mem.Allocator) !void {
var slice = try allocator.alloc(*i32, 100);
testing.expect(slice.len == 100);
for (slice) |*item, i| {
item.* = try allocator.create(i32);
item.*.* = @intCast(i32, i);
}
slice = try allocator.realloc(slice, 20000);
testing.expect(slice.len == 20000);
for (slice[0..100]) |item, i| {
testing.expect(item.* == @intCast(i32, i));
allocator.destroy(item);
}
slice = allocator.shrink(slice, 50);
testing.expect(slice.len == 50);
slice = allocator.shrink(slice, 25);
testing.expect(slice.len == 25);
slice = allocator.shrink(slice, 0);
testing.expect(slice.len == 0);
slice = try allocator.realloc(slice, 10);
testing.expect(slice.len == 10);
allocator.free(slice);
}
fn testAllocatorAligned(allocator: *mem.Allocator, comptime alignment: u29) !void {
// initial
var slice = try allocator.alignedAlloc(u8, alignment, 10);
testing.expect(slice.len == 10);
// grow
slice = try allocator.realloc(slice, 100);
testing.expect(slice.len == 100);
// shrink
slice = allocator.shrink(slice, 10);
testing.expect(slice.len == 10);
// go to zero
slice = allocator.shrink(slice, 0);
testing.expect(slice.len == 0);
// realloc from zero
slice = try allocator.realloc(slice, 100);
testing.expect(slice.len == 100);
// shrink with shrink
slice = allocator.shrink(slice, 10);
testing.expect(slice.len == 10);
// shrink to zero
slice = allocator.shrink(slice, 0);
testing.expect(slice.len == 0);
}
fn testAllocatorLargeAlignment(allocator: *mem.Allocator) mem.Allocator.Error!void {
2018-05-29 00:23:55 +00:00
//Maybe a platform's page_size is actually the same as or
// very near usize?
2019-05-26 17:17:34 +00:00
if (mem.page_size << 2 > maxInt(usize)) return;
const USizeShift = @IntType(false, std.math.log2(usize.bit_count));
const large_align = @as(u29, mem.page_size << 2);
var align_mask: usize = undefined;
_ = @shlWithOverflow(usize, ~@as(usize, 0), @as(USizeShift, @ctz(u29, large_align)), &align_mask);
var slice = try allocator.alignedAlloc(u8, large_align, 500);
testing.expect(@ptrToInt(slice.ptr) & align_mask == @ptrToInt(slice.ptr));
slice = allocator.shrink(slice, 100);
testing.expect(@ptrToInt(slice.ptr) & align_mask == @ptrToInt(slice.ptr));
slice = try allocator.realloc(slice, 5000);
testing.expect(@ptrToInt(slice.ptr) & align_mask == @ptrToInt(slice.ptr));
slice = allocator.shrink(slice, 10);
testing.expect(@ptrToInt(slice.ptr) & align_mask == @ptrToInt(slice.ptr));
slice = try allocator.realloc(slice, 20000);
testing.expect(@ptrToInt(slice.ptr) & align_mask == @ptrToInt(slice.ptr));
allocator.free(slice);
}
fn testAllocatorAlignedShrink(allocator: *mem.Allocator) mem.Allocator.Error!void {
var debug_buffer: [1000]u8 = undefined;
const debug_allocator = &FixedBufferAllocator.init(&debug_buffer).allocator;
2019-05-26 17:17:34 +00:00
const alloc_size = mem.page_size * 2 + 50;
var slice = try allocator.alignedAlloc(u8, 16, alloc_size);
defer allocator.free(slice);
var stuff_to_free = std.ArrayList([]align(16) u8).init(debug_allocator);
// On Windows, VirtualAlloc returns addresses aligned to a 64K boundary,
// which is 16 pages, hence the 32. This test may require to increase
// the size of the allocations feeding the `allocator` parameter if they
// fail, because of this high over-alignment we want to have.
2019-05-26 17:17:34 +00:00
while (@ptrToInt(slice.ptr) == mem.alignForward(@ptrToInt(slice.ptr), mem.page_size * 32)) {
try stuff_to_free.append(slice);
slice = try allocator.alignedAlloc(u8, 16, alloc_size);
}
while (stuff_to_free.popOrNull()) |item| {
allocator.free(item);
}
slice[0] = 0x12;
slice[60] = 0x34;
// realloc to a smaller size but with a larger alignment
2019-05-26 17:17:34 +00:00
slice = try allocator.alignedRealloc(slice, mem.page_size * 32, alloc_size / 2);
testing.expect(slice[0] == 0x12);
testing.expect(slice[60] == 0x34);
}