1 /// 2 module spasm.rt.allocator; 3 4 nothrow: 5 6 import stdx.allocator.building_blocks.bitmapped_block : BitmappedBlock; 7 import stdx.allocator.building_blocks.allocator_list : AllocatorList; 8 import stdx.allocator.building_blocks.null_allocator : NullAllocator; 9 import stdx.allocator.internal : Ternary; 10 import stdx.allocator.common : chooseAtRuntime,reallocate,alignedReallocate,roundUpToMultipleOf,platformAlignment,divideRoundUp,trailingZeros,roundUpToAlignment; 11 import std.typecons : Flag, Yes, No; 12 13 alias Destructor = void function(void*); 14 15 version (WebAssembly) { 16 private __gshared void* begin, current, end; 17 struct WasmAllocator { 18 import spasm.intrinsics; 19 import spasm.rt.memory : wasmPageSize; 20 enum uint alignment = platformAlignment; 21 22 nothrow: 23 static __gshared typeof(this) instance = WasmAllocator(); 24 25 Ternary owns(void[] b) { 26 return Ternary(b.ptr >= begin); 27 } 28 29 @trusted static void init(uint heap_base) { 30 begin = cast(void*)(heap_base.roundUpToAlignment(alignment)); 31 current = begin; 32 end = cast(void*)(wasmMemorySize * wasmPageSize); 33 } 34 35 void[] allocate(size_t n) { 36 const rounded = n.roundUpToAlignment(alignment); 37 if (current + rounded > end) 38 grow(1 + rounded / wasmPageSize); 39 void* mem = current; 40 current += rounded; 41 return mem[0..rounded]; 42 } 43 44 // NOTE: temporary until we front this with a FreeTree 45 bool deallocate(void[] data) { 46 return true; 47 } 48 49 private void grow(size_t pages) { 50 auto currentPages = wasmMemoryGrow(0); 51 current = cast(void*)(currentPages * wasmPageSize); 52 wasmMemoryGrow(pages); 53 end = cast(void*)((currentPages + pages) * wasmPageSize); 54 } 55 } 56 } else version (D_BetterC) { 57 import stdx.allocator.mallocator; 58 alias WasmAllocator = Mallocator; 59 } else { 60 struct WasmAllocator { 61 enum uint alignment = platformAlignment; 62 63 static __gshared typeof(this) instance = WasmAllocator(); 64 65 nothrow: 66 auto owns(void[] b) { 67 return Ternary.yes; 68 } 69 70 bool deallocate(void[] b) nothrow { 71 return true; 72 } 73 74 void[] allocate(size_t n) nothrow { 75 return new ubyte[n]; 76 } 77 } 78 } 79 80 /** 81 Returns `true` if `ptr` is aligned at `alignment`. 82 */ 83 @nogc nothrow pure bool alignedAt(T)(T* ptr, uint alignment) 84 { 85 return cast(size_t) ptr % alignment == 0; 86 } 87 88 // NOTE: had to copy this from the BitmappedBlock 89 pure nothrow @safe @nogc 90 private auto totalBitmappedBlockAllocation(size_t capacity, size_t blockSize) { 91 import spasm.ct : tuple; 92 auto blocks = cast(uint)capacity.divideRoundUp(blockSize); 93 auto leadingUlongs = blocks.divideRoundUp(64); 94 import std.algorithm.comparison : min; 95 immutable initialAlignment = min(platformAlignment, 96 1U << min(31U, trailingZeros(leadingUlongs * 8))); 97 auto maxSlack = platformAlignment <= initialAlignment 98 ? 0 99 : platformAlignment - initialAlignment; 100 return tuple!("leadingUlongs", "blocks","bytes")(leadingUlongs, blocks, leadingUlongs * 8 + maxSlack + blockSize * blocks); 101 } 102 103 nothrow 104 auto initPool(size_t blockSize, size_t capacity) { 105 import spasm.ct : tuple; 106 auto poolSize = totalBitmappedBlockAllocation(capacity, blockSize); 107 size_t metaDataSize = (PoolAllocator.MetaData.sizeof + poolSize.leadingUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment); 108 return tuple!("markerUlongs", "blocks", "memory", "metaDataSize")(poolSize.leadingUlongs, poolSize.blocks, WasmAllocator.instance.allocate(metaDataSize + poolSize.bytes), metaDataSize); 109 } 110 111 struct PoolAllocatorBacking { 112 nothrow: 113 void* pool; 114 115 MarkResult mark(void* ptr) { 116 // TODO; we can get the marking a little bit faster by precalculating all this stuff 117 uint blockSize = (cast(uint*)pool)[0]; 118 uint blockCount = (cast(uint*)pool)[1]; 119 size_t leadingMarkerUlongs = blockCount.divideRoundUp(64); 120 size_t offset = (PoolAllocator.MetaData.sizeof + leadingMarkerUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment); 121 void* startOfBitmappedBlock = pool + offset; 122 auto vector = BitVector((cast(ulong*)(pool + PoolAllocator.MetaData.sizeof))[0..leadingMarkerUlongs]); 123 void* startOfBlocks = startOfBitmappedBlock + leadingMarkerUlongs * ulong.sizeof; 124 auto index = cast(uint)(ptr - startOfBlocks) / blockSize; 125 auto wasSet = vector.markBit(index); 126 // NOTE: it is actually a little faster not to take this information into account (but maybe not with bigger sized objects. For now we assume that is true.) 127 return wasSet ? MarkResult.AlreadyMarked : MarkResult.WasUnmarked; 128 } 129 130 void freeUnmarked() { 131 // TODO; we can get the marking a little bit faster by precalculating all this stuff 132 auto meta = cast(PoolAllocator.MetaData*)pool; 133 size_t leadingMarkerUlongs = meta.blocks.divideRoundUp(64); 134 size_t offset = (PoolAllocator.MetaData.sizeof + leadingMarkerUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment); 135 void* startOfBitmappedBlock = pool + offset; 136 ulong[] markers = (cast(ulong*)(pool + PoolAllocator.MetaData.sizeof))[0..leadingMarkerUlongs]; 137 ulong[] control = (cast(ulong*)startOfBitmappedBlock)[0..leadingMarkerUlongs]; 138 if (meta.destructor !is null) { 139 destruct(markers, control, meta.destructor, meta.blockSize, pool + offset + control.length*ulong.sizeof); 140 } 141 control[] = markers[]; 142 markers[] = 0; 143 } 144 145 private void destruct(ulong[] markers, ulong[] control, Destructor destructor, uint blockSize, void* offset) { 146 import mir.bitop : cttz; 147 for(uint i = 0; i < markers.length; i++) { 148 ulong toFree = markers[i] ^ control[i]; 149 while(toFree != 0) { 150 auto lsbset = cttz(toFree); 151 void* item = offset + blockSize * ((i*64) + (63 - lsbset)); 152 destructor(item); 153 toFree = toFree & (toFree-1); 154 } 155 } 156 } 157 } 158 159 enum MarkResult { 160 WasUnmarked, 161 AlreadyMarked 162 } 163 164 struct PoolAllocatorIndex { 165 nothrow: 166 void*[] addresses; 167 uint lastFree = 0; 168 169 private void ensureSpace() { 170 if (addresses.length > lastFree) 171 return; 172 if (addresses.length == 0) { 173 addresses = (cast(void**)WasmAllocator.instance.allocate(32 * (void*).sizeof).ptr)[0..32]; 174 return; 175 } 176 size_t n = (void*).sizeof * addresses.length * 2; 177 void*[] biggerList = (cast(void**)WasmAllocator.instance.allocate(n).ptr)[0..addresses.length*2]; 178 biggerList[0..lastFree] = addresses[0..lastFree]; 179 WasmAllocator.instance.deallocate(addresses); 180 addresses = biggerList; 181 } 182 183 bool owns(void* ptr) { 184 if (addresses.length == 0) 185 return false; 186 return addresses[0] <= ptr; 187 } 188 189 MarkResult mark(void* ptr) { 190 if (owns(ptr)) { 191 return findAllocator(ptr).mark(ptr); 192 } 193 return MarkResult.WasUnmarked; 194 } 195 196 auto findAllocator(void* ptr) { 197 import std.range : assumeSorted; 198 assert(addresses.length > 0); 199 auto lower = addresses[0..lastFree].assumeSorted.lowerBound(ptr); 200 return PoolAllocatorBacking(addresses[lower.length - 1]); 201 } 202 203 void freeUnmarked() { 204 import std.algorithm : map, each; 205 addresses[0..lastFree].map!(p => PoolAllocatorBacking(p)).each!(p => p.freeUnmarked()); 206 } 207 208 void add(void* start) { 209 import std.range : assumeSorted; 210 ensureSpace(); 211 212 if (lastFree == 0 || addresses[lastFree-1] < start) { 213 addresses[lastFree++] = start; 214 return; 215 } 216 auto lower = addresses[0..lastFree].assumeSorted.lowerBound(start); 217 218 if (lower.length == addresses.length) { 219 addresses[lastFree++] = start; 220 } else { 221 auto offset = lower.length; 222 addresses[offset+1 .. lastFree+1] = addresses[offset .. lastFree]; 223 addresses[offset] = start; 224 lastFree++; 225 } 226 } 227 228 void remove(void* start) { 229 import std.range : assumeSorted; 230 auto lower = addresses[0..lastFree].assumeSorted.lowerBound(start); 231 auto offset = lower.length; 232 if (offset == lastFree) 233 return; 234 addresses[offset .. lastFree-1] = addresses[offset+1 .. lastFree]; 235 lastFree--; 236 } 237 238 version (SPASM_GC_DEBUG) { 239 static auto getAllocatorStats(void* ptr) { 240 import spasm.ct : tuple; 241 uint blockSize = (cast(uint*)ptr)[0]; 242 uint blockCount = (cast(uint*)ptr)[1]; 243 uint leadingMarkerUlongs = blockCount.divideRoundUp(64); 244 uint offset = (PoolAllocator.MetaData.sizeof + leadingMarkerUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment); 245 void* startOfBitmappedBlock = ptr + offset; 246 ulong[] markers = (cast(ulong*)(ptr + PoolAllocator.MetaData.sizeof))[0..leadingMarkerUlongs]; 247 ulong[] control = (cast(ulong*)startOfBitmappedBlock)[0..leadingMarkerUlongs]; 248 return tuple!("blockSize", "blockCount", "markers", "control")(blockSize, blockCount, markers, control); 249 } 250 251 auto getStats() { 252 import std.algorithm : map; 253 return addresses[0..lastFree].map!(PoolAllocatorIndex.getAllocatorStats); 254 } 255 } 256 } 257 258 static __gshared auto poolAllocatorIndex = PoolAllocatorIndex(); 259 260 struct PoolAllocator { 261 nothrow: 262 static struct MetaData { 263 uint blockSize; 264 uint blocks; 265 Destructor destructor; 266 } 267 import stdx.allocator.common : chooseAtRuntime; 268 alias Block = BitmappedBlock!(chooseAtRuntime, platformAlignment, NullAllocator); 269 alias alignment = platformAlignment; 270 private Block block; 271 void[] memory; 272 this(uint blockSize, size_t capacity, Destructor destructor) { 273 auto pool = initPool(blockSize, capacity); 274 memory = pool.memory; 275 auto metaData = cast(MetaData*)memory.ptr; 276 metaData.blockSize = blockSize; 277 metaData.blocks = pool.blocks; 278 metaData.destructor = destructor; 279 block = Block(cast(ubyte[])pool.memory[pool.metaDataSize..$], blockSize); 280 poolAllocatorIndex.add(memory.ptr); 281 } 282 // TODO: need to figure out how to release this PoolAllocator when AllocatorList kills it 283 // this destructor doesn't work since this object gets moved somewhere 284 // ~this() { 285 // import spasm.types; 286 // doLog(-1); 287 // poolAllocatorIndex.remove(memory.ptr); 288 // } 289 void[] allocate(size_t n) { 290 return block.allocate(n); 291 } 292 bool deallocate(void[] b) { 293 return block.deallocate(b); 294 } 295 Ternary owns(void[] b) const { 296 return block.owns(b); 297 } 298 } 299 300 unittest { 301 import stdx.allocator.common : chooseAtRuntime; 302 alias Block = BitmappedBlock!(32, platformAlignment, NullAllocator); 303 auto block = Block(new ubyte[128]); 304 void[] mem = block.allocate(16); 305 assert(block.owns(mem) == Ternary.yes); 306 assert(block.deallocate(mem) == true); 307 } 308 309 unittest { 310 auto allocator = PoolAllocator(32, 128, null); 311 void[] mem = allocator.allocate(16); 312 assert(allocator.owns(mem) == Ternary(true)); 313 assert(allocator.deallocate(mem) == true); 314 } 315 316 auto getGoodCapacity(uint blockSize) { 317 return 8 * 1024; 318 } 319 320 static struct PoolAllocatorFactory { 321 private uint blockSize; 322 private Destructor destructor; 323 this(uint blockSize, Destructor destructor = null) { 324 this.blockSize = blockSize; 325 this.destructor = destructor; 326 } 327 auto opCall(size_t n) { 328 auto capacity = getGoodCapacity(blockSize); 329 return PoolAllocator(blockSize, capacity, destructor); 330 } 331 } 332 333 struct PoolAllocatorList(T) { 334 enum blockSize = T.sizeof; 335 static if (hasMember!(T,"__xdtor")) { 336 static void destructor(void* item) nothrow { 337 T* t = cast(T*)item; 338 t.__xdtor(); 339 } 340 auto allocator = AllocatorList!(PoolAllocatorFactory, WasmAllocator)(PoolAllocatorFactory(blockSize, &destructor)); 341 } else { 342 auto allocator = AllocatorList!(PoolAllocatorFactory, WasmAllocator)(PoolAllocatorFactory(blockSize, null)); 343 } 344 alias allocator this; 345 } 346 347 struct PoolAllocatorList(size_t blockSize) { 348 auto allocator = AllocatorList!(PoolAllocatorFactory, WasmAllocator)(PoolAllocatorFactory(blockSize)); 349 alias allocator this; 350 } 351 352 import std.traits : hasMember; 353 static assert(hasMember!(WasmAllocator, "deallocate")); 354 static assert(hasMember!(PoolAllocatorList!8, "deallocate")); 355 static assert(hasMember!(PoolAllocatorList!16, "empty")); 356 357 private struct BitVector 358 { 359 ulong[] _rep; 360 361 @safe pure nothrow @nogc: 362 363 this(ulong[] data) { _rep = data; } 364 365 auto markBit(ulong x) { 366 assert(x / 64 <= size_t.max); 367 immutable i = cast(size_t) (x / 64); 368 immutable j = 0x8000_0000_0000_0000UL >> (x % 64); 369 const wasSet = (_rep[i] & j) > 0; 370 371 _rep[i] |= j; 372 return wasSet; 373 } 374 }