1 ///
2 module spasm.rt.allocator;
3 
4 nothrow:
5 
6 import stdx.allocator.building_blocks.bitmapped_block : BitmappedBlock;
7 import stdx.allocator.building_blocks.allocator_list : AllocatorList;
8 import stdx.allocator.building_blocks.null_allocator : NullAllocator;
9 import stdx.allocator.internal : Ternary;
10 import stdx.allocator.common : chooseAtRuntime,reallocate,alignedReallocate,roundUpToMultipleOf,platformAlignment,divideRoundUp,trailingZeros,roundUpToAlignment;
11 import std.typecons : Flag, Yes, No;
12 
13 alias Destructor = void function(void*);
14 
15 version (WebAssembly) {
16   private __gshared void* begin, current, end;
17   struct WasmAllocator {
18     import spasm.intrinsics;
19     import spasm.rt.memory : wasmPageSize;
20     enum uint alignment = platformAlignment;
21 
22     nothrow:
23     static __gshared typeof(this) instance = WasmAllocator();
24 
25     Ternary owns(void[] b) {
26       return Ternary(b.ptr >= begin);
27     }
28 
29     @trusted static void init(uint heap_base) {
30       begin = cast(void*)(heap_base.roundUpToAlignment(alignment));
31       current = begin;
32       end = cast(void*)(wasmMemorySize * wasmPageSize);
33     }
34 
35     void[] allocate(size_t n) {
36       const rounded = n.roundUpToAlignment(alignment);
37       if (current + rounded > end)
38         grow(1 + rounded / wasmPageSize);
39       void* mem = current;
40       current += rounded;
41       return mem[0..rounded];
42     }
43 
44     // NOTE: temporary until we front this with a FreeTree
45     bool deallocate(void[] data) {
46       return true;
47     }
48 
49     private void grow(size_t pages) {
50       auto currentPages = wasmMemoryGrow(0);
51       current = cast(void*)(currentPages * wasmPageSize);
52       wasmMemoryGrow(pages);
53       end = cast(void*)((currentPages + pages) * wasmPageSize);
54     }
55   }
56 } else version (D_BetterC) {
57   import stdx.allocator.mallocator;
58   alias WasmAllocator = Mallocator;
59 } else {
60   struct WasmAllocator {
61     enum uint alignment = platformAlignment;
62 
63     static __gshared typeof(this) instance = WasmAllocator();
64 
65     nothrow:
66     auto owns(void[] b) {
67       return Ternary.yes;
68     }
69 
70     bool deallocate(void[] b) nothrow {
71       return true;
72     }
73 
74     void[] allocate(size_t n) nothrow {
75       return new ubyte[n];
76     }
77   }
78 }
79 
80 /**
81    Returns `true` if `ptr` is aligned at `alignment`.
82 */
83 @nogc nothrow pure bool alignedAt(T)(T* ptr, uint alignment)
84 {
85   return cast(size_t) ptr % alignment == 0;
86 }
87 
88 // NOTE: had to copy this from the BitmappedBlock
89 pure nothrow @safe @nogc
90 private auto totalBitmappedBlockAllocation(size_t capacity, size_t blockSize) {
91   import spasm.ct : tuple;
92   auto blocks = cast(uint)capacity.divideRoundUp(blockSize);
93   auto leadingUlongs = blocks.divideRoundUp(64);
94   import std.algorithm.comparison : min;
95   immutable initialAlignment = min(platformAlignment,
96                                    1U << min(31U, trailingZeros(leadingUlongs * 8)));
97   auto maxSlack = platformAlignment <= initialAlignment
98     ? 0
99     : platformAlignment - initialAlignment;
100   return tuple!("leadingUlongs", "blocks","bytes")(leadingUlongs, blocks, leadingUlongs * 8 + maxSlack + blockSize * blocks);
101 }
102 
103 nothrow
104 auto initPool(size_t blockSize, size_t capacity) {
105   import spasm.ct : tuple;
106   auto poolSize = totalBitmappedBlockAllocation(capacity, blockSize);
107   size_t metaDataSize = (PoolAllocator.MetaData.sizeof + poolSize.leadingUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment);
108   return tuple!("markerUlongs", "blocks", "memory", "metaDataSize")(poolSize.leadingUlongs, poolSize.blocks, WasmAllocator.instance.allocate(metaDataSize + poolSize.bytes), metaDataSize);
109 }
110 
111 struct PoolAllocatorBacking {
112   nothrow:
113   void* pool;
114 
115   MarkResult mark(void* ptr) {
116     // TODO; we can get the marking a little bit faster by precalculating all this stuff
117     uint blockSize = (cast(uint*)pool)[0];
118     uint blockCount = (cast(uint*)pool)[1];
119     size_t leadingMarkerUlongs = blockCount.divideRoundUp(64);
120     size_t offset = (PoolAllocator.MetaData.sizeof + leadingMarkerUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment);
121     void* startOfBitmappedBlock = pool + offset;
122     auto vector = BitVector((cast(ulong*)(pool + PoolAllocator.MetaData.sizeof))[0..leadingMarkerUlongs]);
123     void* startOfBlocks = startOfBitmappedBlock + leadingMarkerUlongs * ulong.sizeof;
124     auto index = cast(uint)(ptr - startOfBlocks) / blockSize;
125     auto wasSet = vector.markBit(index);
126     // NOTE: it is actually a little faster not to take this information into account (but maybe not with bigger sized objects. For now we assume that is true.)
127     return wasSet ? MarkResult.AlreadyMarked : MarkResult.WasUnmarked;
128   }
129 
130   void freeUnmarked() {
131     // TODO; we can get the marking a little bit faster by precalculating all this stuff
132     auto meta = cast(PoolAllocator.MetaData*)pool;
133     size_t leadingMarkerUlongs = meta.blocks.divideRoundUp(64);
134     size_t offset = (PoolAllocator.MetaData.sizeof + leadingMarkerUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment);
135     void* startOfBitmappedBlock = pool + offset;
136     ulong[] markers = (cast(ulong*)(pool + PoolAllocator.MetaData.sizeof))[0..leadingMarkerUlongs];
137     ulong[] control = (cast(ulong*)startOfBitmappedBlock)[0..leadingMarkerUlongs];
138     if (meta.destructor !is null) {
139       destruct(markers, control, meta.destructor, meta.blockSize, pool + offset + control.length*ulong.sizeof);
140     }
141     control[] = markers[];
142     markers[] = 0;
143   }
144 
145   private void destruct(ulong[] markers, ulong[] control, Destructor destructor, uint blockSize, void* offset) {
146     import mir.bitop : cttz;
147     for(uint i = 0; i < markers.length; i++) {
148       ulong toFree = markers[i] ^ control[i];
149       while(toFree != 0) {
150         auto lsbset = cttz(toFree);
151         void* item = offset + blockSize * ((i*64) + (63 - lsbset));
152         destructor(item);
153         toFree = toFree & (toFree-1);
154       }
155     }
156   }
157 }
158 
159 enum MarkResult {
160   WasUnmarked,
161   AlreadyMarked
162 }
163 
164 struct PoolAllocatorIndex {
165   nothrow:
166   void*[] addresses;
167   uint lastFree = 0;
168 
169   private void ensureSpace() {
170     if (addresses.length > lastFree)
171       return;
172     if (addresses.length == 0) {
173       addresses = (cast(void**)WasmAllocator.instance.allocate(32 * (void*).sizeof).ptr)[0..32];
174       return;
175     }
176     size_t n = (void*).sizeof * addresses.length * 2;
177     void*[] biggerList = (cast(void**)WasmAllocator.instance.allocate(n).ptr)[0..addresses.length*2];
178     biggerList[0..lastFree] = addresses[0..lastFree];
179     WasmAllocator.instance.deallocate(addresses);
180     addresses = biggerList;
181   }
182 
183   bool owns(void* ptr) {
184     if (addresses.length == 0)
185       return false;
186     return addresses[0] <= ptr;
187   }
188 
189   MarkResult mark(void* ptr) {
190     if (owns(ptr)) {
191       return findAllocator(ptr).mark(ptr);
192     }
193     return MarkResult.WasUnmarked;
194   }
195 
196   auto findAllocator(void* ptr) {
197     import std.range : assumeSorted;
198     assert(addresses.length > 0);
199     auto lower = addresses[0..lastFree].assumeSorted.lowerBound(ptr);
200     return PoolAllocatorBacking(addresses[lower.length - 1]);
201   }
202 
203   void freeUnmarked() {
204     import std.algorithm : map, each;
205     addresses[0..lastFree].map!(p => PoolAllocatorBacking(p)).each!(p => p.freeUnmarked());
206   }
207 
208   void add(void* start) {
209     import std.range : assumeSorted;
210     ensureSpace();
211 
212     if (lastFree == 0 || addresses[lastFree-1] < start) {
213       addresses[lastFree++] = start;
214       return;
215     }
216     auto lower = addresses[0..lastFree].assumeSorted.lowerBound(start);
217 
218     if (lower.length == addresses.length) {
219       addresses[lastFree++] = start;
220     } else {
221       auto offset = lower.length;
222       addresses[offset+1 .. lastFree+1] = addresses[offset .. lastFree];
223       addresses[offset] = start;
224       lastFree++;
225     }
226   }
227 
228   void remove(void* start) {
229     import std.range : assumeSorted;
230     auto lower = addresses[0..lastFree].assumeSorted.lowerBound(start);
231     auto offset = lower.length;
232     if (offset == lastFree)
233       return;
234     addresses[offset .. lastFree-1] = addresses[offset+1 .. lastFree];
235     lastFree--;
236   }
237 
238   version (SPASM_GC_DEBUG) {
239     static auto getAllocatorStats(void* ptr) {
240       import spasm.ct : tuple;
241       uint blockSize = (cast(uint*)ptr)[0];
242       uint blockCount = (cast(uint*)ptr)[1];
243       uint leadingMarkerUlongs = blockCount.divideRoundUp(64);
244       uint offset = (PoolAllocator.MetaData.sizeof + leadingMarkerUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment);
245       void* startOfBitmappedBlock = ptr + offset;
246       ulong[] markers = (cast(ulong*)(ptr + PoolAllocator.MetaData.sizeof))[0..leadingMarkerUlongs];
247       ulong[] control = (cast(ulong*)startOfBitmappedBlock)[0..leadingMarkerUlongs];
248       return tuple!("blockSize", "blockCount", "markers", "control")(blockSize, blockCount, markers, control);
249     }
250 
251     auto getStats() {
252       import std.algorithm : map;
253       return addresses[0..lastFree].map!(PoolAllocatorIndex.getAllocatorStats);
254     }
255   }
256 }
257 
258 static __gshared auto poolAllocatorIndex = PoolAllocatorIndex();
259 
260 struct PoolAllocator {
261   nothrow:
262   static struct MetaData {
263     uint blockSize;
264     uint blocks;
265     Destructor destructor;
266   }
267   import stdx.allocator.common : chooseAtRuntime;
268   alias Block = BitmappedBlock!(chooseAtRuntime, platformAlignment, NullAllocator);
269   alias alignment = platformAlignment;
270   private Block block;
271   void[] memory;
272   this(uint blockSize, size_t capacity, Destructor destructor) {
273     auto pool = initPool(blockSize, capacity);
274     memory = pool.memory;
275     auto metaData = cast(MetaData*)memory.ptr;
276     metaData.blockSize = blockSize;
277     metaData.blocks = pool.blocks;
278     metaData.destructor = destructor;
279     block = Block(cast(ubyte[])pool.memory[pool.metaDataSize..$], blockSize);
280     poolAllocatorIndex.add(memory.ptr);
281   }
282   // TODO: need to figure out how to release this PoolAllocator when AllocatorList kills it
283   // this destructor doesn't work since this object gets moved somewhere
284   // ~this() {
285   //   import spasm.types;
286   //   doLog(-1);
287   //   poolAllocatorIndex.remove(memory.ptr);
288   // }
289   void[] allocate(size_t n) {
290     return block.allocate(n);
291   }
292   bool deallocate(void[] b) {
293     return block.deallocate(b);
294   }
295   Ternary owns(void[] b) const {
296     return block.owns(b);
297   }
298 }
299 
300 unittest {
301   import stdx.allocator.common : chooseAtRuntime;
302   alias Block = BitmappedBlock!(32, platformAlignment, NullAllocator);
303   auto block = Block(new ubyte[128]);
304   void[] mem = block.allocate(16);
305   assert(block.owns(mem) == Ternary.yes);
306   assert(block.deallocate(mem) == true);
307 }
308 
309 unittest {
310   auto allocator = PoolAllocator(32, 128, null);
311   void[] mem = allocator.allocate(16);
312   assert(allocator.owns(mem) == Ternary(true));
313   assert(allocator.deallocate(mem) == true);
314 }
315 
316 auto getGoodCapacity(uint blockSize) {
317   return 8 * 1024;
318 }
319 
320 static struct PoolAllocatorFactory {
321   private uint blockSize;
322   private Destructor destructor;
323   this(uint blockSize, Destructor destructor = null) {
324     this.blockSize = blockSize;
325     this.destructor = destructor;
326   }
327   auto opCall(size_t n) {
328     auto capacity = getGoodCapacity(blockSize);
329     return PoolAllocator(blockSize, capacity, destructor);
330   }
331 }
332 
333 struct PoolAllocatorList(T) {
334   enum blockSize = T.sizeof;
335   static if (hasMember!(T,"__xdtor")) {
336     static void destructor(void* item) nothrow {
337       T* t = cast(T*)item;
338       t.__xdtor();
339     }
340     auto allocator = AllocatorList!(PoolAllocatorFactory, WasmAllocator)(PoolAllocatorFactory(blockSize, &destructor));
341   } else {
342     auto allocator = AllocatorList!(PoolAllocatorFactory, WasmAllocator)(PoolAllocatorFactory(blockSize, null));
343   }
344   alias allocator this;
345 }
346 
347 struct PoolAllocatorList(size_t blockSize) {
348   auto allocator = AllocatorList!(PoolAllocatorFactory, WasmAllocator)(PoolAllocatorFactory(blockSize));
349   alias allocator this;
350 }
351 
352 import std.traits : hasMember;
353 static assert(hasMember!(WasmAllocator, "deallocate"));
354 static assert(hasMember!(PoolAllocatorList!8, "deallocate"));
355 static assert(hasMember!(PoolAllocatorList!16, "empty"));
356 
357 private struct BitVector
358 {
359   ulong[] _rep;
360 
361   @safe pure nothrow @nogc:
362 
363   this(ulong[] data) { _rep = data; }
364 
365   auto markBit(ulong x) {
366     assert(x / 64 <= size_t.max);
367     immutable i = cast(size_t) (x / 64);
368     immutable j = 0x8000_0000_0000_0000UL >> (x % 64);
369     const wasSet = (_rep[i] & j) > 0;
370 
371     _rep[i] |= j;
372     return wasSet;
373   }
374 }