1 ///
2 module spasm.rt.allocator;
3 
4 nothrow:
5 
6 import stdx.allocator.building_blocks.bitmapped_block : BitmappedBlock;
7 import stdx.allocator.building_blocks.allocator_list : AllocatorList;
8 import stdx.allocator.building_blocks.null_allocator : NullAllocator;
9 import stdx.allocator.internal : Ternary;
10 import stdx.allocator.common : chooseAtRuntime,reallocate,alignedReallocate,roundUpToMultipleOf,platformAlignment,divideRoundUp,trailingZeros;
11 import std.typecons : Flag, Yes, No;
12 
13 alias Destructor = void function(void*);
14 
15 version (WebAssembly) {
16   private __gshared void* begin, current, end;
17   struct WasmAllocator {
18     import spasm.intrinsics;
19     import spasm.rt.memory : wasmPageSize;
20     enum uint alignment = platformAlignment;
21 
22     nothrow:
23     static __gshared typeof(this) instance = WasmAllocator();
24 
25     Ternary owns(void[] b) {
26       return Ternary(b.ptr >= begin);
27     }
28 
29     @trusted static void init(uint heap_base) {
30       begin = cast(void*)heap_base;
31       current = begin;
32       end = cast(void*)(wasmMemorySize * wasmPageSize);
33     }
34 
35     void[] allocate(size_t n) {
36       if (current + n > end)
37         grow(1 + n / wasmPageSize);
38       void* mem = current;
39       current += n;
40       return mem[0..n];
41     }
42 
43     // NOTE: temporary until we front this with a FreeTree
44     bool deallocate(void[] data) {
45       return true;
46     }
47 
48     private void grow(size_t pages) {
49       auto currentPages = wasmMemoryGrow(0);
50       current = cast(void*)(currentPages * wasmPageSize);
51       wasmMemoryGrow(pages);
52       end = cast(void*)((currentPages + pages) * wasmPageSize);
53     }
54   }
55 } else version (D_BetterC) {
56   import stdx.allocator.mallocator;
57   alias WasmAllocator = Mallocator;
58 } else {
59   struct WasmAllocator {
60     enum uint alignment = platformAlignment;
61 
62     static __gshared typeof(this) instance = WasmAllocator();
63 
64     nothrow:
65     auto owns(void[] b) {
66       return Ternary.yes;
67     }
68 
69     bool deallocate(void[] b) nothrow {
70       return true;
71     }
72 
73     void[] allocate(size_t n) nothrow {
74       return new ubyte[n];
75     }
76   }
77 }
78 
79 /**
80    Returns `true` if `ptr` is aligned at `alignment`.
81 */
82 @nogc nothrow pure bool alignedAt(T)(T* ptr, uint alignment)
83 {
84   return cast(size_t) ptr % alignment == 0;
85 }
86 
87 // NOTE: had to copy this from the BitmappedBlock
88 pure nothrow @safe @nogc
89 private auto totalBitmappedBlockAllocation(size_t capacity, size_t blockSize) {
90   import spasm.ct : tuple;
91   auto blocks = cast(uint)capacity.divideRoundUp(blockSize);
92   auto leadingUlongs = blocks.divideRoundUp(64);
93   import std.algorithm.comparison : min;
94   immutable initialAlignment = min(platformAlignment,
95                                    1U << min(31U, trailingZeros(leadingUlongs * 8)));
96   auto maxSlack = platformAlignment <= initialAlignment
97     ? 0
98     : platformAlignment - initialAlignment;
99   return tuple!("leadingUlongs", "blocks","bytes")(leadingUlongs, blocks, leadingUlongs * 8 + maxSlack + blockSize * blocks);
100 }
101 
102 nothrow
103 auto initPool(size_t blockSize, size_t capacity) {
104   import spasm.ct : tuple;
105   auto poolSize = totalBitmappedBlockAllocation(capacity, blockSize);
106   size_t metaDataSize = (PoolAllocator.MetaData.sizeof + poolSize.leadingUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment);
107   return tuple!("markerUlongs", "blocks", "memory", "metaDataSize")(poolSize.leadingUlongs, poolSize.blocks, WasmAllocator.instance.allocate(metaDataSize + poolSize.bytes), metaDataSize);
108 }
109 
110 struct PoolAllocatorBacking {
111   nothrow:
112   void* pool;
113 
114   MarkResult mark(void* ptr) {
115     // TODO; we can get the marking a little bit faster by precalculating all this stuff
116     uint blockSize = (cast(uint*)pool)[0];
117     uint blockCount = (cast(uint*)pool)[1];
118     size_t leadingMarkerUlongs = blockCount.divideRoundUp(64);
119     size_t offset = (PoolAllocator.MetaData.sizeof + leadingMarkerUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment);
120     void* startOfBitmappedBlock = pool + offset;
121     auto vector = BitVector((cast(ulong*)(pool + PoolAllocator.MetaData.sizeof))[0..leadingMarkerUlongs]);
122     void* startOfBlocks = startOfBitmappedBlock + leadingMarkerUlongs * ulong.sizeof;
123     auto index = cast(uint)(ptr - startOfBlocks) / blockSize;
124     auto wasSet = vector.markBit(index);
125     // NOTE: it is actually a little faster not to take this information into account (but maybe not with bigger sized objects. For now we assume that is true.)
126     return wasSet ? MarkResult.AlreadyMarked : MarkResult.WasUnmarked;
127   }
128 
129   void freeUnmarked() {
130     // TODO; we can get the marking a little bit faster by precalculating all this stuff
131     auto meta = cast(PoolAllocator.MetaData*)pool;
132     size_t leadingMarkerUlongs = meta.blocks.divideRoundUp(64);
133     size_t offset = (PoolAllocator.MetaData.sizeof + leadingMarkerUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment);
134     void* startOfBitmappedBlock = pool + offset;
135     ulong[] markers = (cast(ulong*)(pool + PoolAllocator.MetaData.sizeof))[0..leadingMarkerUlongs];
136     ulong[] control = (cast(ulong*)startOfBitmappedBlock)[0..leadingMarkerUlongs];
137     if (meta.destructor !is null) {
138       destruct(markers, control, meta.destructor, meta.blockSize, pool + offset + control.length*ulong.sizeof);
139     }
140     control[] = markers[];
141     markers[] = 0;
142   }
143 
144   private void destruct(ulong[] markers, ulong[] control, Destructor destructor, uint blockSize, void* offset) {
145     import mir.bitop : cttz;
146     for(uint i = 0; i < markers.length; i++) {
147       ulong toFree = markers[i] ^ control[i];
148       while(toFree != 0) {
149         auto lsbset = cttz(toFree);
150         void* item = offset + blockSize * ((i*64) + (63 - lsbset));
151         destructor(item);
152         toFree = toFree & (toFree-1);
153       }
154     }
155   }
156 }
157 
158 enum MarkResult {
159   WasUnmarked,
160   AlreadyMarked
161 }
162 
163 struct PoolAllocatorIndex {
164   nothrow:
165   void*[] addresses;
166   uint lastFree = 0;
167 
168   private void ensureSpace() {
169     if (addresses.length > lastFree)
170       return;
171     if (addresses.length == 0) {
172       addresses = (cast(void**)WasmAllocator.instance.allocate(32 * (void*).sizeof).ptr)[0..32];
173       return;
174     }
175     size_t n = (void*).sizeof * addresses.length * 2;
176     void*[] biggerList = (cast(void**)WasmAllocator.instance.allocate(n).ptr)[0..addresses.length*2];
177     biggerList[0..lastFree] = addresses[0..lastFree];
178     WasmAllocator.instance.deallocate(addresses);
179     addresses = biggerList;
180   }
181 
182   bool owns(void* ptr) {
183     if (addresses.length == 0)
184       return false;
185     return addresses[0] <= ptr;
186   }
187 
188   MarkResult mark(void* ptr) {
189     if (owns(ptr)) {
190       return findAllocator(ptr).mark(ptr);
191     }
192     return MarkResult.WasUnmarked;
193   }
194 
195   auto findAllocator(void* ptr) {
196     import std.range : assumeSorted;
197     assert(addresses.length > 0);
198     auto lower = addresses[0..lastFree].assumeSorted.lowerBound(ptr);
199     return PoolAllocatorBacking(addresses[lower.length - 1]);
200   }
201 
202   void freeUnmarked() {
203     import std.algorithm : map, each;
204     addresses[0..lastFree].map!(p => PoolAllocatorBacking(p)).each!(p => p.freeUnmarked());
205   }
206 
207   void add(void* start) {
208     import std.range : assumeSorted;
209     ensureSpace();
210 
211     if (lastFree == 0 || addresses[lastFree-1] < start) {
212       addresses[lastFree++] = start;
213       return;
214     }
215     auto lower = addresses[0..lastFree].assumeSorted.lowerBound(start);
216 
217     if (lower.length == addresses.length) {
218       addresses[lastFree++] = start;
219     } else {
220       auto offset = lower.length;
221       addresses[offset+1 .. lastFree+1] = addresses[offset .. lastFree];
222       addresses[offset] = start;
223       lastFree++;
224     }
225   }
226 
227   void remove(void* start) {
228     import std.range : assumeSorted;
229     auto lower = addresses[0..lastFree].assumeSorted.lowerBound(start);
230     auto offset = lower.length;
231     if (offset == lastFree)
232       return;
233     addresses[offset .. lastFree-1] = addresses[offset+1 .. lastFree];
234     lastFree--;
235   }
236 
237   version (SPASM_GC_DEBUG) {
238     static auto getAllocatorStats(void* ptr) {
239       import spasm.ct : tuple;
240       uint blockSize = (cast(uint*)ptr)[0];
241       uint blockCount = (cast(uint*)ptr)[1];
242       uint leadingMarkerUlongs = blockCount.divideRoundUp(64);
243       uint offset = (PoolAllocator.MetaData.sizeof + leadingMarkerUlongs * ulong.sizeof).roundUpToMultipleOf(platformAlignment);
244       void* startOfBitmappedBlock = ptr + offset;
245       ulong[] markers = (cast(ulong*)(ptr + PoolAllocator.MetaData.sizeof))[0..leadingMarkerUlongs];
246       ulong[] control = (cast(ulong*)startOfBitmappedBlock)[0..leadingMarkerUlongs];
247       return tuple!("blockSize", "blockCount", "markers", "control")(blockSize, blockCount, markers, control);
248     }
249 
250     auto getStats() {
251       import std.algorithm : map;
252       return addresses[0..lastFree].map!(PoolAllocatorIndex.getAllocatorStats);
253     }
254   }
255 }
256 
257 static __gshared auto poolAllocatorIndex = PoolAllocatorIndex();
258 
259 struct PoolAllocator {
260   nothrow:
261   static struct MetaData {
262     uint blockSize;
263     uint blocks;
264     Destructor destructor;
265   }
266   import stdx.allocator.common : chooseAtRuntime;
267   alias Block = BitmappedBlock!(chooseAtRuntime, platformAlignment, NullAllocator);
268   alias alignment = platformAlignment;
269   private Block block;
270   void[] memory;
271   this(uint blockSize, size_t capacity, Destructor destructor) {
272     auto pool = initPool(blockSize, capacity);
273     memory = pool.memory;
274     auto metaData = cast(MetaData*)memory.ptr;
275     metaData.blockSize = blockSize;
276     metaData.blocks = pool.blocks;
277     metaData.destructor = destructor;
278     block = Block(cast(ubyte[])pool.memory[pool.metaDataSize..$], blockSize);
279     poolAllocatorIndex.add(memory.ptr);
280   }
281   // TODO: need to figure out how to release this PoolAllocator when AllocatorList kills it
282   // this destructor doesn't work since this object gets moved somewhere
283   // ~this() {
284   //   import spasm.types;
285   //   doLog(-1);
286   //   poolAllocatorIndex.remove(memory.ptr);
287   // }
288   void[] allocate(size_t n) {
289     return block.allocate(n);
290   }
291   bool deallocate(void[] b) {
292     return block.deallocate(b);
293   }
294   Ternary owns(void[] b) const {
295     return block.owns(b);
296   }
297 }
298 
299 unittest {
300   import stdx.allocator.common : chooseAtRuntime;
301   alias Block = BitmappedBlock!(32, platformAlignment, NullAllocator);
302   auto block = Block(new ubyte[128]);
303   void[] mem = block.allocate(16);
304   assert(block.owns(mem) == Ternary.yes);
305   assert(block.deallocate(mem) == true);
306 }
307 
308 unittest {
309   auto allocator = PoolAllocator(32, 128, null);
310   void[] mem = allocator.allocate(16);
311   assert(allocator.owns(mem) == Ternary(true));
312   assert(allocator.deallocate(mem) == true);
313 }
314 
315 auto getGoodCapacity(uint blockSize) {
316   return 8 * 1024;
317 }
318 
319 static struct PoolAllocatorFactory {
320   private uint blockSize;
321   private Destructor destructor;
322   this(uint blockSize, Destructor destructor = null) {
323     this.blockSize = blockSize;
324     this.destructor = destructor;
325   }
326   auto opCall(size_t n) {
327     auto capacity = getGoodCapacity(blockSize);
328     return PoolAllocator(blockSize, capacity, destructor);
329   }
330 }
331 
332 struct PoolAllocatorList(T) {
333   enum blockSize = T.sizeof;
334   static if (hasMember!(T,"__xdtor")) {
335     static void destructor(void* item) nothrow {
336       T* t = cast(T*)item;
337       t.__xdtor();
338     }
339     auto allocator = AllocatorList!(PoolAllocatorFactory, WasmAllocator)(PoolAllocatorFactory(blockSize, &destructor));
340   } else {
341     auto allocator = AllocatorList!(PoolAllocatorFactory, WasmAllocator)(PoolAllocatorFactory(blockSize, null));
342   }
343   alias allocator this;
344 }
345 
346 struct PoolAllocatorList(size_t blockSize) {
347   auto allocator = AllocatorList!(PoolAllocatorFactory, WasmAllocator)(PoolAllocatorFactory(blockSize));
348   alias allocator this;
349 }
350 
351 import std.traits : hasMember;
352 static assert(hasMember!(WasmAllocator, "deallocate"));
353 static assert(hasMember!(PoolAllocatorList!8, "deallocate"));
354 static assert(hasMember!(PoolAllocatorList!16, "empty"));
355 
356 private struct BitVector
357 {
358   ulong[] _rep;
359 
360   @safe pure nothrow @nogc:
361 
362   this(ulong[] data) { _rep = data; }
363 
364   auto markBit(ulong x) {
365     assert(x / 64 <= size_t.max);
366     immutable i = cast(size_t) (x / 64);
367     immutable j = 0x8000_0000_0000_0000UL >> (x % 64);
368     const wasSet = (_rep[i] & j) > 0;
369 
370     _rep[i] |= j;
371     return wasSet;
372   }
373 }