LNXSDK/Kha/Backends/Kinc-HL/hl/src/gc.c

/*
 * Copyright (C)2005-2016 Haxe Foundation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */
#include "hl.h"
#ifdef HL_WIN
#	include <windows.h>
#else
#	include <sys/types.h>
#	include <sys/mman.h>
#endif

#define MZERO(ptr,size)		memset(ptr,0,size)

// GC

#define GC_PAGE_BITS	16
#define GC_PAGE_SIZE	(1 << GC_PAGE_BITS)

#ifndef HL_64
#	define gc_hash(ptr)			((unsigned int)(ptr))
#	define GC_LEVEL0_BITS		8
#	define GC_LEVEL1_BITS		8
#else
#	define GC_LEVEL0_BITS		10
#	define GC_LEVEL1_BITS		10

// we currently discard the higher bits
// we should instead have some special handling for them
// in x86-64 user space grows up to 0x8000-00000000 (16 bits base + 31 bits page id)

#ifdef HL_WIN
#	define gc_hash(ptr)			((int_val)(ptr)&0x0000000FFFFFFFFF)
#else
// Linux gives addresses using the following patterns (X=any,Y=small value - can be 0): 
//		0x0000000YXXX0000
//		0x0007FY0YXXX0000
static int_val gc_hash( void *ptr ) {
	int_val v = (int_val)ptr;
	return (v ^ ((v >> 33) << 28)) & 0x0000000FFFFFFFFF;
}
#endif

#endif

#define GC_MASK_BITS		16
#define GC_GET_LEVEL1(ptr)	hl_gc_page_map[gc_hash(ptr)>>(GC_MASK_BITS+GC_LEVEL1_BITS)]
#define GC_GET_PAGE(ptr)	GC_GET_LEVEL1(ptr)[(gc_hash(ptr)>>GC_MASK_BITS)&GC_LEVEL1_MASK]
#define GC_LEVEL1_MASK		((1 << GC_LEVEL1_BITS) - 1)

#define PAGE_KIND_BITS		2
#define PAGE_KIND_MASK		((1 << PAGE_KIND_BITS) - 1)

#if defined(HL_DEBUG) && !defined(HL_CONSOLE)
#	define GC_DEBUG
#	define GC_MEMCHK
#endif

#define GC_INTERIOR_POINTERS

#define out_of_memory(reason)		hl_fatal("Out of Memory (" reason ")")

typedef struct _gc_pheader gc_pheader;

// page + private total reserved data per page
typedef void (*gc_page_iterator)( gc_pheader *, int );
// block-ptr + size
typedef void (*gc_block_iterator)( void *, int );

//#define GC_EXTERN_API

#ifdef GC_EXTERN_API
typedef void* gc_allocator_page_data;

// Initialize the allocator
void gc_allocator_init();

// Get the block size within the given page. The block validity has already been checked.
int gc_allocator_fast_block_size( gc_pheader *page, void *block );

// Get the block id within the given page, or -1 if it's an invalid ptr. The block is already checked within page bounds
int gc_allocator_get_block_id( gc_pheader *page, void *block );

// Same as get_block_id but handles interior pointers and modify the block value
int gc_allocator_get_block_id_interior( gc_pheader *page, void **block );

// Called before marking starts: should update each page "bmp" with mark_bits
void gc_allocator_before_mark( unsigned char *mark_bits );

// Called when marking ends: should call finalizers, sweep unused blocks and free empty pages
void gc_allocator_after_mark();

// Allocate a block with given size using the specified page kind. 
// Returns NULL if no block could be allocated
// Sets size to really allocated size (could be larger)
// Sets size to -1 if allocation refused (required size is invalid)
void *gc_allocator_alloc( int *size, int page_kind );

// returns the number of pages allocated and private data size (global)
void gc_get_stats( int *page_count, int *private_data);
void gc_iter_pages( gc_page_iterator i );
void gc_iter_live_blocks( gc_pheader *p, gc_block_iterator i );

#else
#	include "allocator.h"
#endif

struct _gc_pheader {
	// const
	unsigned char *base;
	unsigned char *bmp;
	int page_size;
	int page_kind;
	gc_allocator_page_data alloc;
	gc_pheader *next_page;
#ifdef GC_DEBUG
	int page_id;
#endif
};

#ifdef HL_64
#	define INPAGE(ptr,page) ((unsigned char*)(ptr) >= (page)->base && (unsigned char*)(ptr) < (page)->base + (page)->page_size)
#else
#	define INPAGE(ptr,page) true
#endif

#define GC_PROFILE		1
#define GC_DUMP_MEM		2
#define GC_NO_THREADS	4
#define GC_FORCE_MAJOR	8

static int gc_flags = 0;
static gc_pheader *gc_level1_null[1<<GC_LEVEL1_BITS] = {NULL};
static gc_pheader **hl_gc_page_map[1<<GC_LEVEL0_BITS] = {NULL};
static gc_pheader *gc_free_pheaders = NULL;

static gc_pheader *gc_alloc_page( int size, int kind, int block_count );
static void gc_free_page( gc_pheader *page, int block_count );

#ifndef GC_EXTERN_API
#include "allocator.c"
#endif

static hl_threads_info gc_threads;

HL_THREAD_STATIC_VAR hl_thread_info *current_thread;

static struct {
	int64 total_requested;
	int64 total_allocated;
	int64 last_mark;
	int64 last_mark_allocs;
	int64 pages_total_memory;
	int64 allocation_count;
	int pages_count;
	int pages_allocated;
	int pages_blocks;
	int mark_bytes;
	int mark_time;
	int mark_count;
	int alloc_time; // only measured if gc_profile active
} gc_stats = {0};

static struct {
	int64 total_allocated;
	int64 allocation_count;
	int alloc_time;
} last_profile;

#ifdef HL_WIN
#	define TIMESTAMP() ((int)GetTickCount())
#else
#	define TIMESTAMP() 0
#endif

// -------------------------  ROOTS ----------------------------------------------------------

static void ***gc_roots = NULL;
static int gc_roots_count = 0;
static int gc_roots_max = 0;

HL_API hl_thread_info *hl_get_thread() {
	return current_thread;
}

static void gc_save_context(hl_thread_info *t, void *prev_stack ) {
	void *stack_cur = &t;
	setjmp(t->gc_regs);
	// some compilers (such as clang) might push/pop some callee registers in call
	// to gc_save_context (or before) which might hold a gc value !
	// let's capture them immediately in extra per-thread data
	t->stack_cur = &prev_stack;

	// We have no guarantee prev_stack is pointer-aligned
	// All calls are passing a pointer to a bool, which is aligned on 1 byte
	// If pointer is wrongly aligned, the extra_stack_data is misaligned
	// and register pointers save in stack will not be discovered correctly by the GC
	uintptr_t aligned_prev_stack = ((uintptr_t)prev_stack) & ~(sizeof(void*) - 1);
	prev_stack = (void*)aligned_prev_stack;
	int size = (int)((char*)prev_stack - (char*)stack_cur) / sizeof(void*);
	if( size > HL_MAX_EXTRA_STACK ) hl_fatal("GC_SAVE_CONTEXT");
	t->extra_stack_size = size;
	memcpy(t->extra_stack_data, prev_stack, size*sizeof(void*));
}

#ifndef HL_THREADS
#	define gc_global_lock(_)
#else
static void gc_global_lock( bool lock ) {
	hl_thread_info *t = current_thread;
	bool mt = (gc_flags & GC_NO_THREADS) == 0;
	if( !t && gc_threads.count == 0 ) return;
	if( lock ) {
		if( !t )
			hl_fatal("Can't lock GC in unregistered thread");
		if( mt ) gc_save_context(t,&lock);
		t->gc_blocking++;
		if( mt ) hl_mutex_acquire(gc_threads.global_lock);
	} else {
		t->gc_blocking--;
		if( mt ) hl_mutex_release(gc_threads.global_lock);
	}
}
#endif

HL_PRIM void hl_global_lock( bool lock ) {
	if( lock )
		hl_mutex_acquire(gc_threads.exclusive_lock);
	else
		hl_mutex_release(gc_threads.exclusive_lock);
}

HL_PRIM void hl_add_root( void *r ) {
	gc_global_lock(true);
	if( gc_roots_count == gc_roots_max ) {
		int nroots = gc_roots_max ? (gc_roots_max << 1) : 16;
		void ***roots = (void***)malloc(sizeof(void*)*nroots);
		memcpy(roots,gc_roots,sizeof(void*)*gc_roots_count);
		free(gc_roots);
		gc_roots = roots;
		gc_roots_max = nroots;
	}
	gc_roots[gc_roots_count++] = (void**)r;
	gc_global_lock(false);
}

HL_PRIM void hl_remove_root( void *v ) {
	int i;
	gc_global_lock(true);
	for(i=gc_roots_count-1;i>=0;i--)
		if( gc_roots[i] == (void**)v ) {
			gc_roots_count--;
			memmove(gc_roots + i, gc_roots + (i+1), (gc_roots_count - i) * sizeof(void*));
			break;
		}
	gc_global_lock(false);
}

HL_PRIM gc_pheader *hl_gc_get_page( void *v ) {
	gc_pheader *page = GC_GET_PAGE(v);
	if( page && !INPAGE(v,page) )
		page = NULL;
	return page;
}

// -------------------------  THREADS ----------------------------------------------------------

HL_API int hl_thread_id();

HL_API void hl_register_thread( void *stack_top ) {
	if( hl_get_thread() )
		hl_fatal("Thread already registered");

	hl_thread_info *t = (hl_thread_info*)malloc(sizeof(hl_thread_info));
	memset(t, 0, sizeof(hl_thread_info));
	t->thread_id = hl_thread_id();
	#ifdef HL_MAC
	t->mach_thread_id = mach_thread_self();
	t->pthread_id = (pthread_t)hl_thread_current();
	#endif
	t->stack_top = stack_top;
	t->flags = HL_TRACK_MASK << HL_TREAD_TRACK_SHIFT;
	current_thread = t;
	hl_add_root(&t->exc_value);
	hl_add_root(&t->exc_handler);

	gc_global_lock(true);
	hl_thread_info **all = (hl_thread_info**)malloc(sizeof(void*) * (gc_threads.count + 1));
	memcpy(all,gc_threads.threads,sizeof(void*)*gc_threads.count);
	gc_threads.threads = all;
	all[gc_threads.count++] = t;
	gc_global_lock(false);
}

HL_API void hl_unregister_thread() {
	int i;
	hl_thread_info *t = hl_get_thread();
	if( !t )
		hl_fatal("Thread not registered");
	hl_remove_root(&t->exc_value);
	hl_remove_root(&t->exc_handler);
	gc_global_lock(true);
	for(i=0;i<gc_threads.count;i++)
		if( gc_threads.threads[i] == t ) {
			memmove(gc_threads.threads + i, gc_threads.threads + i + 1, sizeof(void*) * (gc_threads.count - i - 1));
			gc_threads.count--;
			break;
		}
	free(t);
	current_thread = NULL;
	// don't use gc_global_lock(false)
	hl_mutex_release(gc_threads.global_lock);
}

HL_API hl_threads_info *hl_gc_threads_info() {
	return &gc_threads;
}

static void gc_stop_world( bool b ) {
#	ifdef HL_THREADS
	if( b ) {
		int i;
		gc_threads.stopping_world = true;
		for(i=0;i<gc_threads.count;i++) {
			hl_thread_info *t = gc_threads.threads[i];
			while( t->gc_blocking == 0 ) {}; // spinwait
		}
	} else {
		// releasing global lock will release all threads
		gc_threads.stopping_world = false;
	}
#	else
	if( b ) gc_save_context(current_thread,&b);
#	endif
}

// -------------------------  ALLOCATOR ----------------------------------------------------------

#ifdef GC_DEBUG
static int PAGE_ID = 0;
#endif

HL_API void hl_gc_dump_memory( const char *filename );
static void gc_major( void );

static void *gc_will_collide( void *p, int size ) {
#	ifdef HL_64
	int i;
	for(i=0;i<size>>GC_MASK_BITS;i++) {
		void *ptr = (unsigned char*)p + (i<<GC_MASK_BITS);
		if( GC_GET_PAGE(ptr) )
			return ptr;
	}
#	endif
	return NULL;
}

static void gc_free_page_memory( void *ptr, int page_size );
static void *gc_alloc_page_memory( int size );

static gc_pheader *gc_alloc_page( int size, int kind, int block_count ) {
	unsigned char *base = (unsigned char*)gc_alloc_page_memory(size);
	if( !base ) {
		int pages = gc_stats.pages_allocated;
		gc_major();
		if( pages != gc_stats.pages_allocated )
			return gc_alloc_page(size, kind, block_count);
		// big block : report stack trace - we should manage to handle it
		if( size >= (8 << 20) ) {
			gc_global_lock(false);
			hl_error("Failed to alloc %d KB",size>>10);
		}
		if( gc_flags & GC_DUMP_MEM ) hl_gc_dump_memory("hlmemory.dump");
		out_of_memory("pages");
	}

	gc_pheader *p = gc_free_pheaders;
	if( !p ) {
		// alloc pages by chunks so we get good memory locality
		int i, count = 100;
		gc_pheader *head = (gc_pheader*)malloc(sizeof(gc_pheader)*count);
		p = head;
		for(i=1;i<count-1;i++) {
			p->next_page = head + i;
			p = p->next_page;
		}
		p->next_page = NULL;
		p = gc_free_pheaders = head;
	}
	gc_free_pheaders = p->next_page;
	memset(p,0,sizeof(gc_pheader));
	p->base = (unsigned char*)base;
	p->page_size = size;

#	ifdef HL_64
	void *ptr = gc_will_collide(p->base,size);
	if( ptr ) {
#		ifdef HL_VCC
		printf("GC Page HASH collide %IX %IX\n",(int_val)GC_GET_PAGE(ptr),(int_val)ptr);
#		else
		printf("GC Page HASH collide %lX %lX\n",(int_val)GC_GET_PAGE(ptr),(int_val)ptr);
#		endif
		return gc_alloc_page(size, kind, block_count);
	}
#endif

#	if defined(GC_DEBUG)
	memset(base,0xDD,size);
	p->page_id = PAGE_ID++;
#	else
	// prevent false positive to access invalid type
	if( kind == MEM_KIND_DYNAMIC ) memset(base, 0, size);
#	endif
	if( ((int_val)base) & ((1<<GC_MASK_BITS) - 1) )
		hl_fatal("Page memory is not correctly aligned");
	p->page_size = size;
	p->page_kind = kind;
	p->bmp = NULL;

	// update stats
	gc_stats.pages_count++;
	gc_stats.pages_allocated++;
	gc_stats.pages_blocks += block_count;
	gc_stats.pages_total_memory += size;
	gc_stats.mark_bytes += (block_count + 7) >> 3;

	// register page in page map
	int i;
	for(i=0;i<size>>GC_MASK_BITS;i++) {
		void *ptr = p->base + (i<<GC_MASK_BITS);
		if( GC_GET_LEVEL1(ptr) == gc_level1_null ) {
			gc_pheader **level = (gc_pheader**)malloc(sizeof(void*) * (1<<GC_LEVEL1_BITS));
			MZERO(level,sizeof(void*) * (1<<GC_LEVEL1_BITS));
			GC_GET_LEVEL1(ptr) = level;
		}
		GC_GET_PAGE(ptr) = p;
	}

	return p;
}

static void gc_free_page( gc_pheader *ph, int block_count ) {
	int i;
	for(i=0;i<ph->page_size>>GC_MASK_BITS;i++) {
		void *ptr = ph->base + (i<<GC_MASK_BITS);
		GC_GET_PAGE(ptr) = NULL;
	}
	gc_stats.pages_count--;
	gc_stats.pages_blocks -= block_count;
	gc_stats.pages_total_memory -= ph->page_size;
	gc_stats.mark_bytes -= (block_count + 7) >> 3;
	gc_free_page_memory(ph->base,ph->page_size);
	ph->next_page = gc_free_pheaders;
	gc_free_pheaders = ph;
}

static void gc_check_mark();

void *hl_gc_alloc_gen( hl_type *t, int size, int flags ) {
	void *ptr;
	int time = 0;
	int allocated = 0;
	if( size == 0 )
		return NULL;
	if( size < 0 )
		hl_error("Invalid allocation size");
	gc_global_lock(true);
	gc_check_mark();
#	ifdef GC_MEMCHK
	size += HL_WSIZE;
#	endif
	if( gc_flags & GC_PROFILE ) time = TIMESTAMP();
	{
		allocated = size;
		gc_stats.allocation_count++;
		gc_stats.total_requested += size;
#		ifdef GC_PRINT_ALLOCS_SIZES
#		define MAX_WORDS 16
		static int SIZE_CATEGORIES[MAX_WORDS] = {0};
		static int LARGE_BLOCKS[33] = {0};
		int wsize = (size + sizeof(void*) - 1) & ~(sizeof(void*)-1);
		if( wsize < MAX_WORDS * sizeof(void*) )
			SIZE_CATEGORIES[wsize/sizeof(void*)]++;
		else {
			int k = 0;
			while( size > (1<<k) && k < 20 ) {
				k++;
			}
			LARGE_BLOCKS[k]++;
		}
		if( (gc_stats.allocation_count & 0xFFFF) == 0 ) {
			int i;
			for(i=0;i<MAX_WORDS;i++)
				if( SIZE_CATEGORIES[i] )
					printf("%d=%.1f ",i*sizeof(void*),(SIZE_CATEGORIES[i] * 100.) / gc_stats.allocation_count);
			for(i=0;i<33;i++)
				if( LARGE_BLOCKS[i] )
					printf("%d=%.2f ",1<<i,(LARGE_BLOCKS[i] * 100.) / gc_stats.allocation_count);
			printf("%d\n",gc_stats.allocation_count);
		}
#		endif
		ptr = gc_allocator_alloc(&allocated,flags & PAGE_KIND_MASK);
		if( ptr == NULL ) {
			if( allocated < 0 ) {
				gc_global_lock(false);
				hl_error("Required memory allocation too big");
			}
			hl_fatal("TODO");
		}
		gc_stats.total_allocated += allocated;
	}
	if( gc_flags & GC_PROFILE ) gc_stats.alloc_time += TIMESTAMP() - time;
#	ifdef GC_DEBUG
	memset(ptr,0xCD,allocated);
#	endif
	if( flags & MEM_ZERO )
		MZERO(ptr,allocated);
	else if( MEM_HAS_PTR(flags) && allocated != size )
		MZERO((char*)ptr+size,allocated-size); // erase possible pointers after data
#	ifdef GC_MEMCHK
	memset((char*)ptr+(allocated - HL_WSIZE),0xEE,HL_WSIZE);
#	endif
	gc_global_lock(false);
	hl_track_call(HL_TRACK_ALLOC, on_alloc(t,size,flags,ptr));
	return ptr;
}

// -------------------------  MARKING ----------------------------------------------------------

static float gc_mark_threshold = 0.2f;
static int mark_size = 0;
static unsigned char *mark_data = NULL;
static void **cur_mark_stack = NULL;
static void **mark_stack_end = NULL;
static int mark_stack_size = 0;

#define GC_PUSH_GEN(ptr,page) \
	if( MEM_HAS_PTR((page)->page_kind) ) { \
		if( mark_stack == mark_stack_end ) mark_stack = hl_gc_mark_grow(mark_stack); \
		*mark_stack++ = ptr; \
	}

HL_PRIM void **hl_gc_mark_grow( void **stack ) {
	int nsize = mark_stack_size ? (((mark_stack_size * 3) >> 1) & ~1) : 256;
	void **nstack = (void**)malloc(sizeof(void**) * nsize);
	void **base_stack = mark_stack_end - mark_stack_size;
	int avail = (int)(stack - base_stack);
	if( nstack == NULL ) {
		out_of_memory("markstack");
		return NULL;
	}
	memcpy(nstack, base_stack, avail * sizeof(void*));
	free(base_stack);
	mark_stack_size = nsize;
	mark_stack_end = nstack + nsize;
	cur_mark_stack = nstack + avail;
	if( avail == 0 )
		*cur_mark_stack++ = 0;
	return cur_mark_stack;
}

#define GC_PRECISE

static void gc_flush_mark() {
	register void **mark_stack = cur_mark_stack;
	while( true ) {
		void **block = (void**)*--mark_stack;
		gc_pheader *page = GC_GET_PAGE(block);
		unsigned int *mark_bits = NULL;
		int pos = 0, nwords;
#		ifdef GC_DEBUG
		vdynamic *ptr = (vdynamic*)block;
		ptr += 0; // prevent unreferenced warning
#		endif
		if( !block ) {
			mark_stack++;
			break;
		}
		int size = gc_allocator_fast_block_size(page, block);
#		ifdef GC_DEBUG
		if( size <= 0 ) hl_fatal("assert");
#		endif
		nwords = size / HL_WSIZE;
#		ifdef GC_PRECISE
		if( page->page_kind == MEM_KIND_DYNAMIC ) {
			hl_type *t = *(hl_type**)block;
#			ifdef GC_DEBUG
#				ifdef HL_64
				if( (int_val)t == 0xDDDDDDDDDDDDDDDD ) continue;
#				else
				if( (int_val)t == 0xDDDDDDDD ) continue;
#				endif
#			endif
			if( t && t->mark_bits && t->kind != HFUN ) {
				mark_bits = t->mark_bits;
				if( t->kind == HENUM ) {
					mark_bits += ((venum*)block)->index;
					block += 2;
					nwords -= 2;
				} else {
					block++;
					pos++;
				}
			}
		}
#		endif
		while( pos < nwords ) {
			void *p;
			if( mark_bits && (mark_bits[pos >> 5] & (1 << (pos&31))) == 0 ) {
				pos++;
				block++;
				continue;
			}
			p = *block++;
			pos++;
			page = GC_GET_PAGE(p);
			if( !page || !INPAGE(p,page) ) continue;
			int bid = gc_allocator_get_block_id(page,p);
			if( bid >= 0 && (page->bmp[bid>>3] & (1<<(bid&7))) == 0 ) {
				page->bmp[bid>>3] |= 1<<(bid&7);
				GC_PUSH_GEN(p,page);
			}
		}
	}
	cur_mark_stack = mark_stack;
}

static void gc_mark_stack( void *start, void *end ) {
	void **mark_stack = cur_mark_stack;
	void **stack_head = (void**)start;
	while( stack_head < (void**)end ) {
		void *p = *stack_head++;
		gc_pheader *page = GC_GET_PAGE(p);
		if( !page || !INPAGE(p,page) ) continue;
#		ifdef GC_INTERIOR_POINTERS
		int bid = gc_allocator_get_block_interior(page, &p);
#		else
		int bid = gc_allocator_get_block_id(page, p);
#		endif
		if( bid >= 0 && (page->bmp[bid>>3] & (1<<(bid&7))) == 0 ) {
			page->bmp[bid>>3] |= 1<<(bid&7);
			GC_PUSH_GEN(p,page);
		}
	}
	cur_mark_stack = mark_stack;
}

static void gc_mark() {
	void **mark_stack = cur_mark_stack;
	int mark_bytes = gc_stats.mark_bytes;
	int i;
	// prepare mark bits
	if( mark_bytes > mark_size ) {
		gc_free_page_memory(mark_data, mark_size);
		if( mark_size == 0 ) mark_size = GC_PAGE_SIZE;
		while( mark_size < mark_bytes )
			mark_size <<= 1;
		mark_data = gc_alloc_page_memory(mark_size);
		if( mark_data == NULL ) out_of_memory("markbits");
	}
	MZERO(mark_data,mark_bytes);
	gc_allocator_before_mark(mark_data);
	// push roots
	for(i=0;i<gc_roots_count;i++) {
		void *p = *gc_roots[i];
		gc_pheader *page;
		if( !p ) continue;
		page = GC_GET_PAGE(p);
		if( !page || !INPAGE(p,page) ) continue; // the value was set to a not gc allocated ptr
		int bid = gc_allocator_get_block_id(page, p);
		if( bid >= 0 && (page->bmp[bid>>3] & (1<<(bid&7))) == 0 ) {
			page->bmp[bid>>3] |= 1<<(bid&7);
			GC_PUSH_GEN(p,page);
		}
	}

	// scan threads stacks & registers
	for(i=0;i<gc_threads.count;i++) {
		hl_thread_info *t = gc_threads.threads[i];
		cur_mark_stack = mark_stack;
		gc_mark_stack(t->stack_cur,t->stack_top);
		gc_mark_stack(&t->gc_regs,(void**)&t->gc_regs + (sizeof(jmp_buf) / sizeof(void*) - 1));
		gc_mark_stack(&t->extra_stack_data,(void**)&t->extra_stack_data + t->extra_stack_size);
		mark_stack = cur_mark_stack;
	}

	cur_mark_stack = mark_stack;
	if( mark_stack ) gc_flush_mark();
	gc_allocator_after_mark();
}

static void gc_major() {
	int time = TIMESTAMP(), dt;
	gc_stats.last_mark = gc_stats.total_allocated;
	gc_stats.last_mark_allocs = gc_stats.allocation_count;
	gc_stop_world(true);
	gc_mark();
	gc_stop_world(false);
	dt = TIMESTAMP() - time;
	gc_stats.mark_count++;
	gc_stats.mark_time += dt;
	if( gc_flags & GC_PROFILE ) {
		printf("GC-PROFILE %d\n\tmark-time %.3g\n\talloc-time %.3g\n\ttotal-mark-time %.3g\n\ttotal-alloc-time %.3g\n\tallocated %d (%dKB)\n",
			gc_stats.mark_count,
			dt/1000.,
			(gc_stats.alloc_time - last_profile.alloc_time)/1000.,
			gc_stats.mark_time/1000.,
			gc_stats.alloc_time/1000.,
			(int)(gc_stats.allocation_count - last_profile.allocation_count),
			(int)((gc_stats.total_allocated - last_profile.total_allocated)>>10)
		);
		last_profile.allocation_count = gc_stats.allocation_count;
		last_profile.alloc_time = gc_stats.alloc_time;
		last_profile.total_allocated = gc_stats.total_allocated;
	}
}

HL_API void hl_gc_major() {
	gc_global_lock(true);
	gc_major();
	gc_global_lock(false);
}

HL_API bool hl_is_gc_ptr( void *ptr ) {
	gc_pheader *page = GC_GET_PAGE(ptr);
	if( !page || !INPAGE(ptr,page) ) return false;
	int bid = gc_allocator_get_block_id(page, ptr);
	if( bid < 0 ) return false;
	//if( page->bmp && page->next_block == page->first_block && (page->bmp[bid>>3]&(1<<(bid&7))) == 0 ) return false;
	return true;
}

HL_API int hl_gc_get_memsize( void *ptr ) {
	gc_pheader *page = GC_GET_PAGE(ptr);
	if( !page || !INPAGE(ptr,page) ) return -1;
	return gc_allocator_fast_block_size(page,ptr);
}


static bool gc_is_active = true;

static void gc_check_mark() {
	int64 m = gc_stats.total_allocated - gc_stats.last_mark;
	int64 b = gc_stats.allocation_count - gc_stats.last_mark_allocs;
	if( (m > gc_stats.pages_total_memory * gc_mark_threshold || b > gc_stats.pages_blocks * gc_mark_threshold || (gc_flags & GC_FORCE_MAJOR)) && gc_is_active )
		gc_major();
}

static void hl_gc_init() {
	int i;
	for(i=0;i<1<<GC_LEVEL0_BITS;i++)
		hl_gc_page_map[i] = gc_level1_null;
	gc_allocator_init();
#	ifndef HL_CONSOLE
	if( getenv("HL_GC_PROFILE") )
		gc_flags |= GC_PROFILE;
	if( getenv("HL_DUMP_MEMORY") )
		gc_flags |= GC_DUMP_MEM;
#	endif
	gc_stats.mark_bytes = 4; // prevent reading out of bmp
	memset(&gc_threads,0,sizeof(gc_threads));
	gc_threads.global_lock = hl_mutex_alloc(false);
	gc_threads.exclusive_lock = hl_mutex_alloc(false);
#	ifdef HL_THREADS
	hl_add_root(&gc_threads.global_lock);
	hl_add_root(&gc_threads.exclusive_lock);
#	endif
}

static void hl_gc_free() {
#	ifdef HL_THREADS
	hl_remove_root(&gc_threads.global_lock);
#	endif
}

// ---- UTILITIES ----------------------

HL_API bool hl_is_blocking() {
	hl_thread_info *t = current_thread;
	// when called from a non GC thread, tells if the main thread is blocking
	if( t == NULL ) {
		if( gc_threads.count == 0 )
			return false;
		t = gc_threads.threads[0];
	}
	return t->gc_blocking > 0;
}

HL_API void hl_blocking( bool b ) {
	hl_thread_info *t = current_thread;
	if( !t )
		return; // allow hl_blocking in non-GC threads
	if( b ) {
#		ifdef HL_THREADS
		if( t->gc_blocking == 0 )
			gc_save_context(t,&b);
#		endif
		t->gc_blocking++;
	} else if( t->gc_blocking == 0 )
		hl_error("Unblocked thread");
	else {
		t->gc_blocking--;
		if( t->gc_blocking == 0 && gc_threads.stopping_world ) {
			gc_global_lock(true);
			gc_global_lock(false);
		}
	}
}

void hl_cache_free();
void hl_cache_init();

void hl_global_init() {
	hl_gc_init();
	hl_cache_init();
}

void hl_global_free() {
	hl_cache_free();
	hl_gc_free();
}

struct hl_alloc_block {
	int size;
	hl_alloc_block *next;
	unsigned char *p;
};

void hl_alloc_init( hl_alloc *a ) {
	a->cur = NULL;
}

void *hl_malloc( hl_alloc *a, int size ) {
	hl_alloc_block *b = a->cur;
	void *p;
	if( !size ) return NULL;
	size += hl_pad_size(size,&hlt_dyn);
	if( b == NULL || b->size <= size ) {
		int alloc = size < 4096-sizeof(hl_alloc_block) ? 4096-sizeof(hl_alloc_block) : size;
		b = (hl_alloc_block *)malloc(sizeof(hl_alloc_block) + alloc);
		if( b == NULL ) out_of_memory("malloc");
		b->p = ((unsigned char*)b) + sizeof(hl_alloc_block);
		b->size = alloc;
		b->next = a->cur;
		a->cur = b;
	}
	p = b->p;
	b->p += size;
	b->size -= size;
	return p;
}

void *hl_zalloc( hl_alloc *a, int size ) {
	void *p = hl_malloc(a,size);
	if( p ) MZERO(p,size);
	return p;
}

void hl_free( hl_alloc *a ) {
	hl_alloc_block *b = a->cur;
	int_val prev = 0;
	int size = 0;
	while( b ) {
		hl_alloc_block *n = b->next;
		size = (int)(b->p + b->size - ((unsigned char*)b));
		prev = (int_val)b;
		free(b);
		b = n;
	}
	// check if our allocator was not part of the last free block
	if( (int_val)a < prev || (int_val)a > prev+size )
		a->cur = NULL;
}

HL_PRIM void *hl_alloc_executable_memory( int size ) {
#ifdef __APPLE__
#  	ifndef MAP_ANONYMOUS
#     		define MAP_ANONYMOUS MAP_ANON
#       endif
#endif
#if defined(HL_WIN) && defined(HL_64)
	static char *jit_address = (char*)0x000076CA9F000000;
	void *ptr;
retry_jit_alloc:
	ptr = VirtualAlloc(jit_address,size,MEM_RESERVE|MEM_COMMIT,PAGE_EXECUTE_READWRITE);
	if( !ptr ) {
		jit_address = (char*)(((int_val)jit_address)>>1); // fix for Win7 - will eventually reach NULL
		goto retry_jit_alloc;
	}
	jit_address += size + ((-size) & (GC_PAGE_SIZE - 1));
	return ptr;
#elif defined(HL_WIN)
	void *ptr = VirtualAlloc(NULL,size,MEM_RESERVE|MEM_COMMIT,PAGE_EXECUTE_READWRITE);
	return ptr;
#elif defined(HL_CONSOLE)
	return NULL;
#else
	void *p;
	p = mmap(NULL,size,PROT_READ|PROT_WRITE|PROT_EXEC,(MAP_PRIVATE|MAP_ANONYMOUS),-1,0);
	return p;
#endif
}

HL_PRIM void hl_free_executable_memory( void *c, int size ) {
#if defined(HL_WIN)
	VirtualFree(c,0,MEM_RELEASE);
#elif !defined(HL_CONSOLE)
	munmap(c, size);
#endif
}

#if defined(HL_CONSOLE)
void *sys_alloc_align( int size, int align );
void sys_free_align( void *ptr, int size );
#elif !defined(HL_WIN)
static void *base_addr = (void*)0x40000000;
typedef struct _pextra pextra;
struct _pextra {
	void *page_ptr;
	void *base_ptr;
	pextra *next;
};
static pextra *extra_pages = NULL;
#define EXTRA_SIZE (GC_PAGE_SIZE + (4<<10))
#endif

static void *gc_alloc_page_memory( int size ) {
#if defined(HL_WIN)
#	if defined(GC_DEBUG) && defined(HL_64)
#		define STATIC_ADDRESS
#	endif
#	ifdef STATIC_ADDRESS
	// force out of 32 bits addresses to check loss of precision
	static char *start_address = (char*)0x100000000;
#	else
	static void *start_address = NULL;
#	endif
	void *ptr = VirtualAlloc(start_address,size,MEM_RESERVE|MEM_COMMIT,PAGE_READWRITE);
#	ifdef STATIC_ADDRESS
	if( ptr == NULL && start_address ) {
		start_address = NULL;
		return gc_alloc_page_memory(size);
	}
	start_address += size + ((-size) & (GC_PAGE_SIZE - 1));
#	endif
	return ptr;
#elif defined(HL_CONSOLE)
	return sys_alloc_align(size, GC_PAGE_SIZE);
#else
	static int recursions = 0;
	int i = 0;
	while( gc_will_collide(base_addr,size) ) {
		base_addr = (char*)base_addr + GC_PAGE_SIZE;
		i++;
		// most likely our hashing creates too many collisions
		if( i >= 1 << (GC_LEVEL0_BITS + GC_LEVEL1_BITS + 2) )
			return NULL;
	}
	void *ptr = mmap(base_addr,size,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0);
	if( ptr == (void*)-1 )
		return NULL;
	if( ((int_val)ptr) & (GC_PAGE_SIZE-1) ) {
		munmap(ptr,size);
		if( recursions >= 5 ) {
			ptr = mmap(base_addr,size+EXTRA_SIZE,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0);
			int offset = (int)((int_val)ptr) & (GC_PAGE_SIZE-1);
			void *aligned = (char*)ptr + (GC_PAGE_SIZE - offset);
			pextra *inf = (pextra*)(offset > (EXTRA_SIZE>>1) ? ((char*)ptr + EXTRA_SIZE - sizeof(pextra)) : (char*)ptr);
			inf->page_ptr = aligned;
			inf->base_ptr = ptr;
			inf->next = extra_pages;
			extra_pages = inf;
			return aligned;
		}
		void *tmp;
		int tmp_size = (int)((int_val)ptr - (int_val)base_addr);
		if( tmp_size > 0 ) {
			base_addr = (void*)((((int_val)ptr) & ~(GC_PAGE_SIZE-1)) + GC_PAGE_SIZE);
			tmp = ptr;
		} else {
			base_addr = (void*)(((int_val)ptr) & ~(GC_PAGE_SIZE-1));
			tmp = NULL;
		}
		if( tmp ) tmp = mmap(tmp,tmp_size,PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0);
		recursions++;
		ptr = gc_alloc_page_memory(size);
		recursions--;
		if( tmp ) munmap(tmp,tmp_size);
		return ptr;
	}
	base_addr = (char*)ptr+size;
	return ptr;
#endif
}

static void gc_free_page_memory( void *ptr, int size ) {
#ifdef HL_WIN
	VirtualFree(ptr, 0, MEM_RELEASE);
#elif defined(HL_CONSOLE)
	sys_free_align(ptr,size);
#else
	pextra *e = extra_pages, *prev = NULL;
	while( e ) {
		if( e->page_ptr == ptr ) {
			if( prev )
				prev->next = e->next;
			else
				extra_pages = e->next;
			munmap(e->base_ptr, size + EXTRA_SIZE);
			return;
		}
		prev = e;
		e = e->next;
	}
	munmap(ptr,size);
#endif
}

vdynamic *hl_alloc_dynamic( hl_type *t ) {
	vdynamic *d = (vdynamic*)hl_gc_alloc_gen(t, sizeof(vdynamic), (hl_is_ptr(t) ? (t->kind == HSTRUCT ? MEM_KIND_RAW : MEM_KIND_DYNAMIC) : MEM_KIND_NOPTR) | MEM_ZERO);
	d->t = t;
	return d;
}

#ifndef HL_64
#	define DYN_PAD	0,
#else
#	define DYN_PAD
#endif

static const vdynamic vdyn_true = { &hlt_bool, DYN_PAD {true} };
static const vdynamic vdyn_false = { &hlt_bool, DYN_PAD {false} };

vdynamic *hl_alloc_dynbool( bool b ) {
	return (vdynamic*)(b ? &vdyn_true : &vdyn_false);
}


vdynamic *hl_alloc_obj( hl_type *t ) {
	vobj *o;
	int size;
	int i;
	hl_runtime_obj *rt = t->obj->rt;
	if( rt == NULL || rt->methods == NULL ) rt = hl_get_obj_proto(t);
	size = rt->size;
	if( size & (HL_WSIZE-1) ) size += HL_WSIZE - (size & (HL_WSIZE-1));
	if( t->kind == HSTRUCT ) {
		o = (vobj*)hl_gc_alloc_gen(t, size, (rt->hasPtr ? MEM_KIND_RAW : MEM_KIND_NOPTR) | MEM_ZERO);
	} else {
		o = (vobj*)hl_gc_alloc_gen(t, size, (rt->hasPtr ? MEM_KIND_DYNAMIC : MEM_KIND_NOPTR) | MEM_ZERO);
		o->t = t;
	}
	for(i=0;i<rt->nbindings;i++) {
		hl_runtime_binding *b = rt->bindings + i;
		*(void**)(((char*)o) + rt->fields_indexes[b->fid]) = b->closure ? hl_alloc_closure_ptr(b->closure,b->ptr,o) : b->ptr;
	}
	return (vdynamic*)o;
}

vdynobj *hl_alloc_dynobj() {
	vdynobj *o = (vdynobj*)hl_gc_alloc_gen(&hlt_dynobj,sizeof(vdynobj),MEM_KIND_DYNAMIC | MEM_ZERO);
	o->t = &hlt_dynobj;
	return o;
}

vvirtual *hl_alloc_virtual( hl_type *t ) {
	vvirtual *v = (vvirtual*)hl_gc_alloc(t, t->virt->dataSize + sizeof(vvirtual) + sizeof(void*) * t->virt->nfields);
	void **fields = (void**)(v + 1);
	char *vdata = (char*)(fields + t->virt->nfields);
	int i;
	v->t = t;
	v->value = NULL;
	v->next = NULL;
	for(i=0;i<t->virt->nfields;i++)
		fields[i] = (char*)v + t->virt->indexes[i];
	MZERO(vdata,t->virt->dataSize);
	return v;
}

HL_API void hl_gc_stats( double *total_allocated, double *allocation_count, double *current_memory ) {
	*total_allocated = (double)gc_stats.total_allocated;
	*allocation_count = (double)gc_stats.allocation_count;
	*current_memory = (double)gc_stats.pages_total_memory;
}

HL_API void hl_gc_enable( bool b ) {
	gc_is_active = b;
}

HL_API int hl_gc_get_flags() {
	return gc_flags;
}

HL_API void hl_gc_set_flags( int f ) {
	gc_flags = f;
}

HL_API void hl_gc_profile( bool b ) {
	if( b )
		gc_flags |= GC_PROFILE;
	else
		gc_flags &= GC_PROFILE;
}

static FILE *fdump;
static void fdump_i( int i ) {
	fwrite(&i,1,4,fdump);
}
static void fdump_p( void *p ) {
	fwrite(&p,1,sizeof(void*),fdump);
}
static void fdump_d( void *p, int size ) {
	fwrite(p,1,size,fdump);
}

static hl_types_dump gc_types_dump = NULL;
HL_API void hl_gc_set_dump_types( hl_types_dump tdump ) {
	gc_types_dump = tdump;
}

static void gc_dump_block( void *block, int size ) {
	fdump_p(block);
	fdump_i(size);
}

static void gc_dump_block_ptr( void *block, int size ) {
	fdump_p(block);
	fdump_i(size);
	if( size >= sizeof(void*) ) fdump_p(*(void**)block);
}

static void gc_dump_page( gc_pheader *p, int private_data ) {
	fdump_p(p->base);
	fdump_i(p->page_kind);
	fdump_i(p->page_size);
	fdump_i(private_data);
	if( p->page_kind & MEM_KIND_NOPTR ) {
		gc_iter_live_blocks(p, gc_dump_block_ptr); // only dump type
		fdump_p(NULL);
	} else {
		gc_iter_live_blocks(p,gc_dump_block);
		fdump_p(NULL);
		fdump_d(p->base, p->page_size);
	}
}

HL_API void hl_gc_dump_memory( const char *filename ) {
	int i;
	gc_global_lock(true);
	gc_stop_world(true);
	gc_mark();
	fdump = fopen(filename,"wb");

	// header
	fdump_d("HMD1",4);
	fdump_i(((sizeof(void*) == 8)?1:0) | ((sizeof(bool) == 4)?2:0));

	// pages
	int page_count, private_data;
	gc_get_stats(&page_count, &private_data);

	// all mallocs
	private_data += sizeof(gc_pheader) * page_count;
	private_data += sizeof(void*) * gc_roots_max;
	private_data += gc_threads.count * (sizeof(void*) + sizeof(hl_thread_info));
	for(i=0;i<1<<GC_LEVEL0_BITS;i++)
		if( hl_gc_page_map[i] != gc_level1_null )
			private_data += sizeof(void*) * (1<<GC_LEVEL1_BITS);

	fdump_i(private_data);
	fdump_i(mark_stack_size); // keep separate
	fdump_i(page_count);
	gc_iter_pages(gc_dump_page);

	// roots
	fdump_i(gc_roots_count);
	for(i=0;i<gc_roots_count;i++)
		fdump_p(*gc_roots[i]);
	// stacks
	fdump_i(gc_threads.count);
	for(i=0;i<gc_threads.count;i++) {
		hl_thread_info *t = gc_threads.threads[i];
		fdump_p(t->stack_top);
		int size = (int)((void**)t->stack_top - (void**)t->stack_cur);
		fdump_i(size);
		fdump_d(t->stack_cur,size*sizeof(void*));
	}
	// types
#	define fdump_t(t)	fdump_i(t.kind); fdump_p(&t);
	fdump_t(hlt_i32);
	fdump_t(hlt_f32);
	fdump_t(hlt_f64);
	fdump_t(hlt_dyn);
	fdump_t(hlt_array);
	fdump_t(hlt_bytes);
	fdump_t(hlt_dynobj);
	fdump_t(hlt_bool);
	fdump_i(-1);
	if( gc_types_dump ) gc_types_dump(fdump_d);
	fclose(fdump);
	fdump = NULL;
	gc_stop_world(false);
	gc_global_lock(false);
}

#ifdef HL_VCC
#	pragma optimize( "", off )
#endif
HL_API vdynamic *hl_debug_call( int mode, vdynamic *v ) {
	return NULL;
}
#ifdef HL_VCC
#	pragma optimize( "", on )
#endif

DEFINE_PRIM(_VOID, gc_major, _NO_ARG);
DEFINE_PRIM(_VOID, gc_enable, _BOOL);
DEFINE_PRIM(_VOID, gc_profile, _BOOL);
DEFINE_PRIM(_VOID, gc_stats, _REF(_F64) _REF(_F64) _REF(_F64));
DEFINE_PRIM(_VOID, gc_dump_memory, _BYTES);
DEFINE_PRIM(_I32, gc_get_flags, _NO_ARG);
DEFINE_PRIM(_VOID, gc_set_flags, _I32);
DEFINE_PRIM(_DYN, debug_call, _I32 _DYN);
DEFINE_PRIM(_VOID, blocking, _BOOL);