diff --git a/common.mk b/common.mk index de1b671..41fb072 100644 --- a/common.mk +++ b/common.mk @@ -683,7 +683,8 @@ gc.$(OBJEXT): {$(VPATH)}gc.c $(RUBY_H_INCLUDES) {$(VPATH)}re.h \ {$(VPATH)}regex.h $(ENCODING_H_INCLUDES) $(VM_CORE_H_INCLUDES) \ {$(VPATH)}gc.h {$(VPATH)}io.h {$(VPATH)}eval_intern.h {$(VPATH)}util.h \ {$(VPATH)}internal.h {$(VPATH)}constant.h \ - {$(VPATH)}thread.h $(PROBES_H_INCLUDES) {$(VPATH)}vm_opts.h {$(VPATH)}debug.h + {$(VPATH)}thread.h $(PROBES_H_INCLUDES) {$(VPATH)}vm_opts.h {$(VPATH)}debug.h \ + {$(VPATH)}pool_alloc.inc.h {$(VPATH)}pool_alloc.h hash.$(OBJEXT): {$(VPATH)}hash.c $(RUBY_H_INCLUDES) {$(VPATH)}util.h \ $(ENCODING_H_INCLUDES) {$(VPATH)}internal.h $(PROBES_H_INCLUDES) {$(VPATH)}vm_opts.h inits.$(OBJEXT): {$(VPATH)}inits.c $(RUBY_H_INCLUDES) \ @@ -747,7 +748,7 @@ signal.$(OBJEXT): {$(VPATH)}signal.c $(RUBY_H_INCLUDES) \ $(VM_CORE_H_INCLUDES) {$(VPATH)}vm_opts.h {$(VPATH)}internal.h {$(VPATH)}ruby_atomic.h {$(VPATH)}eval_intern.h sprintf.$(OBJEXT): {$(VPATH)}sprintf.c $(RUBY_H_INCLUDES) {$(VPATH)}re.h \ {$(VPATH)}regex.h {$(VPATH)}vsnprintf.c $(ENCODING_H_INCLUDES) {$(VPATH)}internal.h -st.$(OBJEXT): {$(VPATH)}st.c $(RUBY_H_INCLUDES) +st.$(OBJEXT): {$(VPATH)}st.c $(RUBY_H_INCLUDES) {$(VPATH)}pool_alloc.h strftime.$(OBJEXT): {$(VPATH)}strftime.c $(RUBY_H_INCLUDES) \ {$(VPATH)}timev.h $(ENCODING_H_INCLUDES) string.$(OBJEXT): {$(VPATH)}string.c $(RUBY_H_INCLUDES) {$(VPATH)}re.h \ diff --git a/gc.c b/gc.c index 9a87bac..776d97d 100644 --- a/gc.c +++ b/gc.c @@ -22,6 +22,7 @@ #include "vm_core.h" #include "internal.h" #include "gc.h" +#include "pool_alloc.h" #include "constant.h" #include "ruby_atomic.h" #include "probes.h" @@ -414,6 +415,23 @@ typedef struct rb_heap_struct { size_t total_slots; /* total slot count (page_length * HEAP_OBJ_LIMIT) */ } rb_heap_t; +#ifdef POOL_ALLOC_API +#define POOL_ALLOC_PART 1 +#include "pool_alloc.inc.h" +#undef POOL_ALLOC_PART + +typedef struct pool_layout_t pool_layout_t; +struct pool_layout_t { + pool_header + p6, /* st_table && st_table_entry */ + p11; /* st_table.bins init size */ +} pool_layout = { + INIT_POOL(void*[6]), + INIT_POOL(void*[11]) +}; +static void pool_finalize_header(pool_header *header); +#endif + typedef struct rb_objspace { struct { size_t limit; @@ -427,6 +445,10 @@ typedef struct rb_objspace { rb_heap_t eden_heap; rb_heap_t tomb_heap; /* heap for zombies and ghosts */ +#ifdef POOL_ALLOC_API + pool_layout_t *pool_headers; +#endif + struct { struct heap_page **sorted; size_t used; @@ -607,7 +629,12 @@ struct heap_page { #define ruby_initial_gc_stress gc_params.gc_stress VALUE *ruby_initial_gc_stress_ptr = &ruby_initial_gc_stress; #else -static rb_objspace_t rb_objspace = {{GC_MALLOC_LIMIT_MIN}}; +static rb_objspace_t rb_objspace = { + {GC_MALLOC_LIMIT_MIN} +#ifdef POOL_ALLOC_API + , &pool_layout +#endif +}; VALUE *ruby_initial_gc_stress_ptr = &rb_objspace.gc_stress; #endif @@ -886,6 +913,10 @@ rb_objspace_alloc(void) malloc_limit = gc_params.malloc_limit_min; +#ifdef POOL_ALLOC_API + objspace->pool_headers = (pool_layout_t*) malloc(sizeof(pool_layout)); + memcpy(objspace->pool_headers, &pool_layout, sizeof(pool_layout)); +#endif return objspace; } #endif @@ -927,6 +958,13 @@ rb_objspace_free(rb_objspace_t *objspace) objspace->eden_heap.pages = NULL; } free_stack_chunks(&objspace->mark_stack); +#ifdef POOL_ALLOC_API + if (objspace->pool_headers) { + pool_finalize_header(&objspace->pool_headers->p6); + pool_finalize_header(&objspace->pool_headers->p11); + free(objspace->pool_headers); + } +#endif free(objspace); } #endif @@ -6434,6 +6472,28 @@ ruby_mimfree(void *ptr) free(mem); } +#ifdef POOL_ALLOC_API +#define POOL_ALLOC_PART 2 +#include "pool_alloc.inc.h" +#undef POOL_ALLOC_PART +#endif + +void +ruby_xpool_free(void *ptr) +{ + pool_free_entry((void**)ptr); +} + +#define DEFINE_POOL_MALLOC(pnts) \ + void * \ + ruby_xpool_malloc_##pnts##p() \ + { \ + return pool_alloc_entry(&rb_objspace.pool_headers->p##pnts); \ + } +DEFINE_POOL_MALLOC(6) +DEFINE_POOL_MALLOC(11) +#undef DEFINE_POOL_MALLOC + #if MALLOC_ALLOCATED_SIZE /* * call-seq: diff --git a/pool_alloc.h b/pool_alloc.h new file mode 100644 index 0000000..957708e --- /dev/null +++ b/pool_alloc.h @@ -0,0 +1,11 @@ +#ifndef POOL_ALLOC_H +#define POOL_ALLOC_H + +#define POOL_ALLOC_API +#ifdef POOL_ALLOC_API +void ruby_xpool_free(void *ptr); +void *ruby_xpool_malloc_6p(); +void *ruby_xpool_malloc_11p(); +#endif + +#endif diff --git a/pool_alloc.inc.h b/pool_alloc.inc.h new file mode 100644 index 0000000..e2ec99a --- /dev/null +++ b/pool_alloc.inc.h @@ -0,0 +1,156 @@ +/* + * this is generic pool allocator + * you should define following macroses: + * ITEM_NAME - unique identifier, which allows to hold functions in a namespace + * ITEM_TYPEDEF(name) - passed to typedef to localize item type + * free_entry - desired name of function for free entry + * alloc_entry - defired name of function for allocate entry + */ + +#if POOL_ALLOC_PART == 1 +#ifdef HEAP_ALIGN_LOG +#define DEFAULT_POOL_SIZE (1 << HEAP_ALIGN_LOG) +#else +#define DEFAULT_POOL_SIZE (sizeof(void*) * 2048) +#endif +typedef unsigned int pool_holder_counter; + +typedef struct pool_entry_list pool_entry_list; +typedef struct pool_holder pool_holder; + +typedef struct pool_header { + pool_holder *first; + pool_holder *_black_magick; + pool_holder_counter size; /* size of entry in sizeof(void*) items */ + pool_holder_counter total; +} pool_header; + +struct pool_holder { + pool_holder_counter free, total; + pool_header *header; + void *freep; + pool_holder *fore, *back; + void *data[1]; +}; +#define POOL_DATA_SIZE(pool_size) (((pool_size) - sizeof(void*) * 6 - offsetof(pool_holder, data)) / sizeof(void*)) +#define POOL_ENTRY_SIZE(item_type) ((sizeof(item_type) - 1) / sizeof(void*) + 1) +#define POOL_HOLDER_COUNT(pool_size, item_type) (POOL_DATA_SIZE(pool_size)/POOL_ENTRY_SIZE(item_type)) +#define INIT_POOL(item_type) {NULL, NULL, POOL_ENTRY_SIZE(item_type), POOL_HOLDER_COUNT(DEFAULT_POOL_SIZE, item_type)} + +#elif POOL_ALLOC_PART == 2 +static pool_holder * +pool_holder_alloc(pool_header *header) +{ + pool_holder *holder; + pool_holder_counter i, size, count; + register void **ptr; + + size_t sz = offsetof(pool_holder, data) + + header->size * header->total * sizeof(void*); +#define objspace (&rb_objspace) + objspace_malloc_prepare(objspace, DEFAULT_POOL_SIZE); + if (header->first != NULL) return header->first; + TRY_WITH_GC(holder = (pool_holder*) aligned_malloc(DEFAULT_POOL_SIZE, sz)); + malloc_increase += DEFAULT_POOL_SIZE; +#if CALC_EXACT_MALLOC_SIZE + objspace->malloc_params.allocated_size += DEFAULT_POOL_SIZE; + objspace->malloc_params.allocations++; +#endif +#undef objspace + + size = header->size; + count = header->total; + holder->free = count; + holder->total = count; + holder->header = header; + holder->fore = NULL; + holder->back = NULL; + holder->freep = &holder->data; + ptr = holder->data; + for(i = count - 1; i; i-- ) { + ptr = *ptr = ptr + size; + } + *ptr = NULL; + header->first = holder; + return holder; +} + +static inline void +pool_holder_unchaing(pool_header *header, pool_holder *holder) +{ + register pool_holder *fore = holder->fore, *back = holder->back; + holder->fore = NULL; + holder->back = NULL; + if (fore != NULL) fore->back = back; + else header->_black_magick = back; + if (back != NULL) back->fore = fore; + else header->first = fore; +} + +static inline pool_holder * +entry_holder(void **entry) +{ + return (pool_holder*)(((uintptr_t)entry) & ~(DEFAULT_POOL_SIZE - 1)); +} + +static inline void +pool_free_entry(void **entry) +{ + pool_holder *holder = entry_holder(entry); + pool_header *header = holder->header; + + if (holder->free++ == 0) { + register pool_holder *first = header->first; + if (first == NULL) { + header->first = holder; + } else { + holder->back = first; + holder->fore = first->fore; + first->fore = holder; + if (holder->fore) + holder->fore->back = holder; + else + header->_black_magick = holder; + } + } else if (holder->free == holder->total && header->first != holder ) { + pool_holder_unchaing(header, holder); + aligned_free(holder); +#if CALC_EXACT_MALLOC_SIZE + rb_objspace.malloc_params.allocated_size -= DEFAULT_POOL_SIZE; + rb_objspace.malloc_params.allocations--; +#endif + return; + } + + *entry = holder->freep; + holder->freep = entry; +} + +static inline void* +pool_alloc_entry(pool_header *header) +{ + pool_holder *holder = header->first; + void **result; + if (holder == NULL) { + holder = pool_holder_alloc(header); + } + + result = holder->freep; + holder->freep = *result; + + if (--holder->free == 0) { + pool_holder_unchaing(header, holder); + } + + return result; +} + +static void +pool_finalize_header(pool_header *header) +{ + if (header->first) { + aligned_free(header->first); + header->first = NULL; + } +} +#endif diff --git a/st.c b/st.c index 6f5e8be..37d0ebe 100644 --- a/st.c +++ b/st.c @@ -7,6 +7,7 @@ #include "st.h" #else #include "ruby/ruby.h" +#include "pool_alloc.h" #endif #include @@ -88,6 +89,39 @@ static void rehash(st_table *); #define do_hash_bin(key,table) (do_hash((key), (table))%(table)->num_bins) /* preparation for possible allocation improvements */ +#ifdef POOL_ALLOC_API +#define st_alloc_entry() (st_table_entry *)ruby_xpool_malloc_6p() +#define st_free_entry(entry) ruby_xpool_free(entry) +#define st_alloc_table() (st_table *)ruby_xpool_malloc_6p() +#define st_dealloc_table(table) ruby_xpool_free(table) +static inline st_table_entry ** +st_alloc_bins(st_index_t size) +{ + st_table_entry **result; + if (size == 11) { + result = (st_table_entry **) ruby_xpool_malloc_11p(); + memset(result, 0, 11 * sizeof(st_table_entry *)); + } + else + result = (st_table_entry **) ruby_xcalloc(size, sizeof(st_table_entry*)); + return result; +} +static inline void +st_free_bins(st_table_entry **bins, st_index_t size) +{ + if (size == 11) + ruby_xpool_free(bins); + else + ruby_xfree(bins); +} +static inline st_table_entry** +st_realloc_bins(st_table_entry **bins, st_index_t newsize, st_index_t oldsize) +{ + st_table_entry **new_bins = st_alloc_bins(newsize); + st_free_bins(bins, oldsize); + return new_bins; +} +#else #define st_alloc_entry() (st_table_entry *)malloc(sizeof(st_table_entry)) #define st_free_entry(entry) free(entry) #define st_alloc_table() (st_table *)malloc(sizeof(st_table)) @@ -101,6 +135,7 @@ st_realloc_bins(st_table_entry **bins, st_index_t newsize, st_index_t oldsize) MEMZERO(bins, st_table_entry*, newsize); return bins; } +#endif /* Shortcut */ #define bins as.big.bins