1 /// 2 module spasm.rt.allocator; 3 4 nothrow: 5 6 import stdx.allocator.building_blocks.bitmapped_block : BitmappedBlock; 7 import stdx.allocator.building_blocks.allocator_list : AllocatorList; 8 import stdx.allocator.building_blocks.null_allocator : NullAllocator; 9 import stdx.allocator.internal : Ternary; 10 import stdx.allocator.common : chooseAtRuntime,reallocate,alignedReallocate,roundUpToMultipleOf,platformAlignment,divideRoundUp,trailingZeros; 11 import std.typecons : Flag, Yes, No; 12 13 alias Destructor = void function(void*); 14 15 version (WebAssembly) { 16 private __gshared void* begin, current, end; 17 struct WasmAllocator { 18 import spasm.intrinsics; 19 import spasm.rt.memory : wasmPageSize; 20 enum uint alignment = platformAlignment; 21 22 nothrow: 23 static __gshared typeof(this) instance = WasmAllocator(); 24 25 Ternary owns(void[] b) { 26 return Ternary(b.ptr >= begin); 27 } 28 29 @trusted static void init(uint heap_base) { 30 begin = cast(void*)heap_base; 31 current = begin; 32 end = cast(void*)(wasmMemorySize * wasmPageSize); 33 } 34 35 void[] allocate(size_t n) { 36 if (current + n > end) 37 grow(1 + n / wasmPageSize); 38 void* mem = current; 39 current += n; 40 return mem[0..n]; 41 } 42 43 // NOTE: temporary until we front this with a FreeTree 44 bool deallocate(void[] data) { 45 return true; 46 } 47 48 private void grow(size_t pages) { 49 auto currentPages = wasmMemoryGrow(0); 50 current = cast(void*)(currentPages * wasmPageSize); 51 wasmMemoryGrow(pages); 52 end = cast(void*)((currentPages + pages) * wasmPageSize); 53 } 54 } 55 } else version (D_BetterC) { 56 import stdx.allocator.mallocator; 57 alias WasmAllocator = Mallocator; 58 } else { 59 struct WasmAllocator { 60 enum uint alignment = platformAlignment; 61 62 static __gshared typeof(this) instance = WasmAllocator(); 63 64 nothrow: 65 auto owns(void[] b) { 66 return Ternary.yes; 67 } 68 69 bool deallocate(void[] b) nothrow { 70 return true; 71 } 72 73 void[] allocate(size_t n) nothrow { 74 return new ubyte[n]; 75 } 76 } 77 } 78 79 /** 80 Returns `true` if `ptr` is aligned at `alignment`. 81 */ 82 @nogc nothrow pure bool alignedAt(T)(T* ptr, uint alignment) 83 { 84 return cast(size_t) ptr % alignment == 0; 85 } 86 87 // NOTE: had to copy this from the BitmappedBlock 88 pure nothrow @safe @nogc 89 private auto totalBitmappedBlockAllocation(size_t capacity, size_t blockSize) { 90 import spasm.ct : tuple; 91 auto blocks = cast(uint)capacity.divideRoundUp(blockSize); 92 auto leadingUlongs = blocks.divideRoundUp(64); 93 import std.algorithm.comparison : min; 94 immutable initialAlignment = min(platformAlignment, 95 1U << min(31U, trailingZeros(leadingUlongs * 8))); 96 auto maxSlack = platformAlignment <= initialAlignment 97 ? 0 98 : platformAlignment - initialAlignment; 99 return tuple!("leadingUlongs", "blocks","bytes")(leadingUlongs, blocks, leadingUlongs * 8 + maxSlack + blockSize * blocks); 100 } 101 102 nothrow 103 auto initPool(size_t blockSize, size_t capacity) { 104 import spasm.ct : tuple; 105 auto poolSize = totalBitmappedBlockAllocation(capacity, blockSize); 106 size_t metaDataSize = (PoolAllocator.MetaData.sizeof + poolSize.leadingUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment); 107 return tuple!("markerUlongs", "blocks", "memory", "metaDataSize")(poolSize.leadingUlongs, poolSize.blocks, WasmAllocator.instance.allocate(metaDataSize + poolSize.bytes), metaDataSize); 108 } 109 110 struct PoolAllocatorBacking { 111 nothrow: 112 void* pool; 113 114 MarkResult mark(void* ptr) { 115 // TODO; we can get the marking a little bit faster by precalculating all this stuff 116 uint blockSize = (cast(uint*)pool)[0]; 117 uint blockCount = (cast(uint*)pool)[1]; 118 size_t leadingMarkerUlongs = blockCount.divideRoundUp(64); 119 size_t offset = (PoolAllocator.MetaData.sizeof + leadingMarkerUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment); 120 void* startOfBitmappedBlock = pool + offset; 121 auto vector = BitVector((cast(ulong*)(pool + PoolAllocator.MetaData.sizeof))[0..leadingMarkerUlongs]); 122 void* startOfBlocks = startOfBitmappedBlock + leadingMarkerUlongs * ulong.sizeof; 123 auto index = cast(uint)(ptr - startOfBlocks) / blockSize; 124 auto wasSet = vector.markBit(index); 125 // NOTE: it is actually a little faster not to take this information into account (but maybe not with bigger sized objects. For now we assume that is true.) 126 return wasSet ? MarkResult.AlreadyMarked : MarkResult.WasUnmarked; 127 } 128 129 void freeUnmarked() { 130 // TODO; we can get the marking a little bit faster by precalculating all this stuff 131 auto meta = cast(PoolAllocator.MetaData*)pool; 132 size_t leadingMarkerUlongs = meta.blocks.divideRoundUp(64); 133 size_t offset = (PoolAllocator.MetaData.sizeof + leadingMarkerUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment); 134 void* startOfBitmappedBlock = pool + offset; 135 ulong[] markers = (cast(ulong*)(pool + PoolAllocator.MetaData.sizeof))[0..leadingMarkerUlongs]; 136 ulong[] control = (cast(ulong*)startOfBitmappedBlock)[0..leadingMarkerUlongs]; 137 if (meta.destructor !is null) { 138 destruct(markers, control, meta.destructor, meta.blockSize, pool + offset + control.length*ulong.sizeof); 139 } 140 control[] = markers[]; 141 markers[] = 0; 142 } 143 144 private void destruct(ulong[] markers, ulong[] control, Destructor destructor, uint blockSize, void* offset) { 145 import mir.bitop : cttz; 146 for(uint i = 0; i < markers.length; i++) { 147 ulong toFree = markers[i] ^ control[i]; 148 while(toFree != 0) { 149 auto lsbset = cttz(toFree); 150 void* item = offset + blockSize * ((i*64) + (63 - lsbset)); 151 destructor(item); 152 toFree = toFree & (toFree-1); 153 } 154 } 155 } 156 } 157 158 enum MarkResult { 159 WasUnmarked, 160 AlreadyMarked 161 } 162 163 struct PoolAllocatorIndex { 164 nothrow: 165 void*[] addresses; 166 uint lastFree = 0; 167 168 private void ensureSpace() { 169 if (addresses.length > lastFree) 170 return; 171 if (addresses.length == 0) { 172 addresses = (cast(void**)WasmAllocator.instance.allocate(32 * (void*).sizeof).ptr)[0..32]; 173 return; 174 } 175 size_t n = (void*).sizeof * addresses.length * 2; 176 void*[] biggerList = (cast(void**)WasmAllocator.instance.allocate(n).ptr)[0..addresses.length*2]; 177 biggerList[0..lastFree] = addresses[0..lastFree]; 178 WasmAllocator.instance.deallocate(addresses); 179 addresses = biggerList; 180 } 181 182 bool owns(void* ptr) { 183 if (addresses.length == 0) 184 return false; 185 return addresses[0] <= ptr; 186 } 187 188 MarkResult mark(void* ptr) { 189 if (owns(ptr)) { 190 return findAllocator(ptr).mark(ptr); 191 } 192 return MarkResult.WasUnmarked; 193 } 194 195 auto findAllocator(void* ptr) { 196 import std.range : assumeSorted; 197 assert(addresses.length > 0); 198 auto lower = addresses[0..lastFree].assumeSorted.lowerBound(ptr); 199 return PoolAllocatorBacking(addresses[lower.length - 1]); 200 } 201 202 void freeUnmarked() { 203 import std.algorithm : map, each; 204 addresses[0..lastFree].map!(p => PoolAllocatorBacking(p)).each!(p => p.freeUnmarked()); 205 } 206 207 void add(void* start) { 208 import std.range : assumeSorted; 209 ensureSpace(); 210 211 if (lastFree == 0 || addresses[lastFree-1] < start) { 212 addresses[lastFree++] = start; 213 return; 214 } 215 auto lower = addresses[0..lastFree].assumeSorted.lowerBound(start); 216 217 if (lower.length == addresses.length) { 218 addresses[lastFree++] = start; 219 } else { 220 auto offset = lower.length; 221 addresses[offset+1 .. lastFree+1] = addresses[offset .. lastFree]; 222 addresses[offset] = start; 223 lastFree++; 224 } 225 } 226 227 void remove(void* start) { 228 import std.range : assumeSorted; 229 auto lower = addresses[0..lastFree].assumeSorted.lowerBound(start); 230 auto offset = lower.length; 231 if (offset == lastFree) 232 return; 233 addresses[offset .. lastFree-1] = addresses[offset+1 .. lastFree]; 234 lastFree--; 235 } 236 237 version (SPASM_GC_DEBUG) { 238 static auto getAllocatorStats(void* ptr) { 239 import spasm.ct : tuple; 240 uint blockSize = (cast(uint*)ptr)[0]; 241 uint blockCount = (cast(uint*)ptr)[1]; 242 uint leadingMarkerUlongs = blockCount.divideRoundUp(64); 243 uint offset = (PoolAllocator.MetaData.sizeof + leadingMarkerUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment); 244 void* startOfBitmappedBlock = ptr + offset; 245 ulong[] markers = (cast(ulong*)(ptr + PoolAllocator.MetaData.sizeof))[0..leadingMarkerUlongs]; 246 ulong[] control = (cast(ulong*)startOfBitmappedBlock)[0..leadingMarkerUlongs]; 247 return tuple!("blockSize", "blockCount", "markers", "control")(blockSize, blockCount, markers, control); 248 } 249 250 auto getStats() { 251 import std.algorithm : map; 252 return addresses[0..lastFree].map!(PoolAllocatorIndex.getAllocatorStats); 253 } 254 } 255 } 256 257 static __gshared auto poolAllocatorIndex = PoolAllocatorIndex(); 258 259 struct PoolAllocator { 260 nothrow: 261 static struct MetaData { 262 uint blockSize; 263 uint blocks; 264 Destructor destructor; 265 } 266 import stdx.allocator.common : chooseAtRuntime; 267 alias Block = BitmappedBlock!(chooseAtRuntime, platformAlignment, NullAllocator); 268 alias alignment = platformAlignment; 269 private Block block; 270 void[] memory; 271 this(uint blockSize, size_t capacity, Destructor destructor) { 272 auto pool = initPool(blockSize, capacity); 273 memory = pool.memory; 274 auto metaData = cast(MetaData*)memory.ptr; 275 metaData.blockSize = blockSize; 276 metaData.blocks = pool.blocks; 277 metaData.destructor = destructor; 278 block = Block(cast(ubyte[])pool.memory[pool.metaDataSize..$], blockSize); 279 poolAllocatorIndex.add(memory.ptr); 280 } 281 // TODO: need to figure out how to release this PoolAllocator when AllocatorList kills it 282 // this destructor doesn't work since this object gets moved somewhere 283 // ~this() { 284 // import spasm.types; 285 // doLog(-1); 286 // poolAllocatorIndex.remove(memory.ptr); 287 // } 288 void[] allocate(size_t n) { 289 return block.allocate(n); 290 } 291 bool deallocate(void[] b) { 292 return block.deallocate(b); 293 } 294 Ternary owns(void[] b) const { 295 return block.owns(b); 296 } 297 } 298 299 unittest { 300 import stdx.allocator.common : chooseAtRuntime; 301 alias Block = BitmappedBlock!(32, platformAlignment, NullAllocator); 302 auto block = Block(new ubyte[128]); 303 void[] mem = block.allocate(16); 304 assert(block.owns(mem) == Ternary.yes); 305 assert(block.deallocate(mem) == true); 306 } 307 308 unittest { 309 auto allocator = PoolAllocator(32, 128, null); 310 void[] mem = allocator.allocate(16); 311 assert(allocator.owns(mem) == Ternary(true)); 312 assert(allocator.deallocate(mem) == true); 313 } 314 315 auto getGoodCapacity(uint blockSize) { 316 return 8 * 1024; 317 } 318 319 static struct PoolAllocatorFactory { 320 private uint blockSize; 321 private Destructor destructor; 322 this(uint blockSize, Destructor destructor = null) { 323 this.blockSize = blockSize; 324 this.destructor = destructor; 325 } 326 auto opCall(size_t n) { 327 auto capacity = getGoodCapacity(blockSize); 328 return PoolAllocator(blockSize, capacity, destructor); 329 } 330 } 331 332 struct PoolAllocatorList(T) { 333 enum blockSize = T.sizeof; 334 static if (hasMember!(T,"__xdtor")) { 335 static void destructor(void* item) nothrow { 336 T* t = cast(T*)item; 337 t.__xdtor(); 338 } 339 auto allocator = AllocatorList!(PoolAllocatorFactory, WasmAllocator)(PoolAllocatorFactory(blockSize, &destructor)); 340 } else { 341 auto allocator = AllocatorList!(PoolAllocatorFactory, WasmAllocator)(PoolAllocatorFactory(blockSize, null)); 342 } 343 alias allocator this; 344 } 345 346 struct PoolAllocatorList(size_t blockSize) { 347 auto allocator = AllocatorList!(PoolAllocatorFactory, WasmAllocator)(PoolAllocatorFactory(blockSize)); 348 alias allocator this; 349 } 350 351 import std.traits : hasMember; 352 static assert(hasMember!(WasmAllocator, "deallocate")); 353 static assert(hasMember!(PoolAllocatorList!8, "deallocate")); 354 static assert(hasMember!(PoolAllocatorList!16, "empty")); 355 356 private struct BitVector 357 { 358 ulong[] _rep; 359 360 @safe pure nothrow @nogc: 361 362 this(ulong[] data) { _rep = data; } 363 364 auto markBit(ulong x) { 365 assert(x / 64 <= size_t.max); 366 immutable i = cast(size_t) (x / 64); 367 immutable j = 0x8000_0000_0000_0000UL >> (x % 64); 368 const wasSet = (_rep[i] & j) > 0; 369 370 _rep[i] |= j; 371 return wasSet; 372 } 373 }