diff --git a/class.c b/class.c index 8f11f45..ec587a1 100644 --- a/class.c +++ b/class.c @@ -170,6 +170,7 @@ class_alloc(VALUE flags, VALUE klass) RCLASS_REFINED_CLASS(obj) = Qnil; RCLASS_EXT(obj)->allocator = 0; + MEMZERO(&RCLASS_EXT(obj)->cache, struct rb_meth_cache, 1); return (VALUE)obj; } @@ -510,6 +511,7 @@ make_singleton_class(VALUE obj) { VALUE orig_class = RBASIC(obj)->klass; VALUE klass = rb_class_boot(orig_class); + rb_method_cache_copy(orig_class, klass); FL_SET(klass, FL_SINGLETON); RBASIC_SET_CLASS(obj, klass); diff --git a/gc.c b/gc.c index 776d97d..1cf6ead 100644 --- a/gc.c +++ b/gc.c @@ -1588,6 +1588,7 @@ obj_free(rb_objspace_t *objspace, VALUE obj) } rb_class_remove_from_module_subclasses(obj); rb_class_remove_from_super_subclasses(obj); + rb_method_cache_clear(obj); if (RANY(obj)->as.klass.ptr) xfree(RANY(obj)->as.klass.ptr); RANY(obj)->as.klass.ptr = NULL; @@ -1658,6 +1659,7 @@ obj_free(rb_objspace_t *objspace, VALUE obj) } rb_class_remove_from_module_subclasses(obj); rb_class_remove_from_super_subclasses(obj); + rb_method_cache_clear(obj); xfree(RANY(obj)->as.klass.ptr); RANY(obj)->as.klass.ptr = NULL; break; @@ -1827,6 +1829,9 @@ rb_objspace_each_objects(each_obj_callback *callback, void *data) struct os_each_struct { size_t num; VALUE of; +#if METHOD_CACHE_STATS + int with_hidden_klasses; +#endif }; static int @@ -1852,6 +1857,25 @@ internal_object_p(VALUE obj) return 1; } +#if METHOD_CACHE_STATS +static int +hidden_class_p(VALUE obj) +{ + RVALUE *p = (RVALUE *)obj; + + if (p->as.basic.flags) { + switch (BUILTIN_TYPE(p)) { + case T_ICLASS: + return 1; + case T_CLASS: + if (FL_TEST(p, FL_SINGLETON)) + return 1; + } + } + return 0; +} +#endif + int rb_objspace_internal_object_p(VALUE obj) { @@ -1866,7 +1890,11 @@ os_obj_of_i(void *vstart, void *vend, size_t stride, void *data) for (; p != pend; p++) { volatile VALUE v = (VALUE)p; +#if METHOD_CACHE_STATS + if (!internal_object_p(v) || (oes->with_hidden_klasses && hidden_class_p(v))) { +#else if (!internal_object_p(v)) { +#endif if (!oes->of || rb_obj_is_kind_of(v, oes->of)) { rb_yield(v); oes->num++; @@ -1878,12 +1906,19 @@ os_obj_of_i(void *vstart, void *vend, size_t stride, void *data) } static VALUE +#if METHOD_CACHE_STATS +os_obj_of(VALUE of, VALUE whk) +#else os_obj_of(VALUE of) +#endif { struct os_each_struct oes; oes.num = 0; oes.of = of; +#if METHOD_CACHE_STATS + oes.with_hidden_klasses = RTEST(whk); +#endif rb_objspace_each_objects(os_obj_of_i, &oes); return SIZET2NUM(oes.num); } @@ -1924,6 +1959,23 @@ os_obj_of(VALUE of) * */ +#if METHOD_CACHE_STATS +static VALUE +os_each_obj(int argc, VALUE *argv, VALUE os) +{ + VALUE args[2]; + + if (argc == 0) { + args[0] = 0; + args[1] = Qnil; + } + else { + rb_scan_args(argc, argv, "02", &args[0], &args[1]); + } + RETURN_ENUMERATOR(os, 2, args); + return os_obj_of(args[0], args[1]); +} +#else static VALUE os_each_obj(int argc, VALUE *argv, VALUE os) { @@ -1938,6 +1990,7 @@ os_each_obj(int argc, VALUE *argv, VALUE os) RETURN_ENUMERATOR(os, 1, &of); return os_obj_of(of); } +#endif /* * call-seq: diff --git a/internal.h b/internal.h index 1f41d23..15c715c 100644 --- a/internal.h +++ b/internal.h @@ -261,6 +261,58 @@ typedef unsigned long rb_serial_t; #define SERIALT2NUM ULONG2NUM #endif +#if !defined(METHOD_CACHE_STATS) +#define METHOD_CACHE_STATS 0 +#endif + +#if METHOD_CACHE_STATS +struct rb_meth_cache_stats { + size_t sum_capa; + size_t sum_used; + size_t sum_undefs; + size_t alloced; + size_t not_empty; + size_t copies; + size_t resets; + size_t insertions; + size_t created; + size_t destroyed; + size_t copy_alloced; + size_t copy_reset; +}; +extern struct rb_meth_cache_stats rb_meth_cache; +VALUE rb_method_cache_stats(int argc, VALUE* argv, VALUE obj); +#endif + +struct rb_meth_cache_entry { + ID mid; + uintptr_t me; + VALUE defined_class; +}; + +/* valid values for MCACHE_INLINED are 1, 2, 3 and 4 */ +#ifndef MCACHE_INLINED +#define MCACHE_INLINED 3 +#endif +#ifndef MCACHE_RESET_FREES_COPY +#define MCACHE_RESET_FREES_COPY 1 +#endif +struct rb_meth_cache { + rb_serial_t method_state; + rb_serial_t class_serial; + int size, capa; + int is_copy; +#if METHOD_CACHE_STATS + int undefs; + size_t resets; + size_t insertions; +#endif + union { + struct rb_meth_cache_entry *ntries; + struct rb_meth_cache_entry n[MCACHE_INLINED]; + } e; +}; + struct rb_classext_struct { struct st_table *iv_index_tbl; struct st_table *iv_tbl; @@ -277,8 +329,37 @@ struct rb_classext_struct { VALUE origin; VALUE refined_class; rb_alloc_func_t allocator; + struct rb_meth_cache cache; }; +static inline void +rb_method_cache_clear(VALUE klass) +{ + struct rb_classext_struct *ext = RCLASS(klass)->ptr; + if (ext->cache.capa > MCACHE_INLINED && ext->cache.e.ntries) { + xfree(ext->cache.e.ntries); + ext->cache.e.ntries = NULL; +#if METHOD_CACHE_STATS + rb_meth_cache.alloced--; + rb_meth_cache.sum_capa -= ext->cache.capa; + rb_meth_cache.destroyed++; +#endif + ext->cache.capa = 0; + } +#if METHOD_CACHE_STATS + if (ext->cache.is_copy) { + ext->cache.is_copy = 0; + rb_meth_cache.copies--; + } + if (ext->cache.size > 0) { + rb_meth_cache.not_empty--; + rb_meth_cache.sum_used -= ext->cache.size; + rb_meth_cache.sum_undefs -= ext->cache.undefs; + ext->cache.size = 0; + } +#endif +} + struct method_table_wrapper { st_table *tbl; size_t serial; @@ -790,6 +871,7 @@ VALUE rb_extract_keywords(VALUE *orighash); /* vm_method.c */ void Init_eval_method(void); int rb_method_defined_by(VALUE obj, ID mid, VALUE (*cfunc)(ANYARGS)); +void rb_method_cache_copy(VALUE from, VALUE to); /* miniprelude.c, prelude.c */ void Init_prelude(void); diff --git a/vm.c b/vm.c index 212f7b0..185d3f8 100644 --- a/vm.c +++ b/vm.c @@ -2492,6 +2492,9 @@ Init_VM(void) rb_undef_alloc_func(rb_cRubyVM); rb_undef_method(CLASS_OF(rb_cRubyVM), "new"); rb_define_singleton_method(rb_cRubyVM, "stat", vm_stat, -1); +#if METHOD_CACHE_STATS + rb_define_singleton_method(rb_cRubyVM, "method_cache_stats", rb_method_cache_stats, -1); +#endif /* FrozenCore (hidden) */ fcore = rb_class_new(rb_cBasicObject); diff --git a/vm_method.c b/vm_method.c index 2270359..8d85e3f 100644 --- a/vm_method.c +++ b/vm_method.c @@ -2,15 +2,6 @@ * This file is included by vm.c */ -#ifndef GLOBAL_METHOD_CACHE_SIZE -#define GLOBAL_METHOD_CACHE_SIZE 0x800 -#endif -#ifndef GLOBAL_METHOD_CACHE_MASK -#define GLOBAL_METHOD_CACHE_MASK 0x7ff -#endif - -#define GLOBAL_METHOD_CACHE_KEY(c,m) ((((c)>>3)^(m))&GLOBAL_METHOD_CACHE_MASK) -#define GLOBAL_METHOD_CACHE(c,m) (global_method_cache + GLOBAL_METHOD_CACHE_KEY(c,m)) #include "method.h" #define NOEX_NOREDEF 0 @@ -29,15 +20,324 @@ static void rb_vm_check_redefinition_opt_method(const rb_method_entry_t *me, VAL #define singleton_undefined idSingleton_method_undefined #define attached id__attached__ -struct cache_entry { - rb_serial_t method_state; - rb_serial_t class_serial; - ID mid; - rb_method_entry_t* me; - VALUE defined_class; -}; +#if METHOD_CACHE_STATS +struct rb_meth_cache_stats rb_meth_cache; +#endif + +typedef struct rb_meth_cache_entry cache_entry_t; + +#define METHOD_ENTRY(entry) ((rb_method_entry_t*)((entry)->me & ~1)) +#define COLLISION(entry) ((entry).me & 1) +#define HASH(id) (id ^ (id >> 3)) + +static void rb_mcache_resize(struct rb_meth_cache *cache); +static void +rb_mcache_insert(struct rb_meth_cache *cache, ID id, uintptr_t me, VALUE defined_class) +{ + int mask, pos, dlt; + cache_entry_t *ent; +#if METHOD_CACHE_STATS + if (cache->size == 0) { + rb_meth_cache.not_empty++; + } +#endif + if (cache->capa == MCACHE_INLINED) { + if (cache->size < MCACHE_INLINED) { + ent = cache->e.n; + pos = cache->size; + goto found; + } + rb_mcache_resize(cache); + } else if (cache->capa / 4 * 3 <= cache->size) { + rb_mcache_resize(cache); + } + mask = cache->capa - 1; + pos = HASH(id) & mask; + + ent = cache->e.ntries; + if (ent[pos].mid == 0) { + goto found; + } + ent[pos].me |= 1; /* set collision */ + dlt = (id % mask) | 1; + for(;;) { + pos = (pos + dlt) & mask; + if (ent[pos].mid == 0) { + goto found; + } + ent[pos].me |= 1; + } +found: + ent += pos; + ent->defined_class = defined_class; + ent->mid = id; + ent->me = me; + cache->size++; +#if METHOD_CACHE_STATS + cache->insertions++; + cache->undefs += me == 0; + rb_meth_cache.sum_undefs += me == 0; + rb_meth_cache.sum_used++; + rb_meth_cache.insertions++; +#endif +} + +#define MCACHE_MIN_SIZE 8 +#define MCACHE_MIN_SHRINK 16 +#define MCACHE_SHRINK_TRIGGER 16 +#define MCACHE_SHRINK_BOUND 4 +static void +rb_mcache_resize(struct rb_meth_cache *cache) +{ + struct rb_meth_cache tmp; + cache_entry_t *entries; + int i; + + MEMZERO(&tmp, struct rb_meth_cache, 1); + tmp.method_state = cache->method_state; + tmp.class_serial = cache->class_serial; +#if METHOD_CACHE_STATS + tmp.resets = cache->resets; + tmp.insertions = cache->insertions; + rb_meth_cache.sum_undefs -= cache->undefs; + if (cache->size > 0) { + rb_meth_cache.not_empty--; + } +#endif + if (cache->capa == MCACHE_INLINED) { + tmp.capa = MCACHE_MIN_SIZE; + entries = cache->e.n; + } + else { + tmp.capa = cache->capa * 2; + entries = cache->e.ntries; +#if METHOD_CACHE_STATS + rb_meth_cache.sum_capa -= cache->capa; +#endif + } +redo: + tmp.e.ntries = xcalloc(tmp.capa, sizeof(cache_entry_t)); + for(i = 0; i < cache->capa; i++) { + if (entries[i].mid && (entries[i].me & ~1)) { + cache_entry_t *ent = &entries[i]; + rb_mcache_insert(&tmp, ent->mid, ent->me & ~1, ent->defined_class); + } + } + /* deal with lots of cached method_missing */ + if (tmp.size < tmp.capa / MCACHE_SHRINK_TRIGGER && tmp.capa > MCACHE_MIN_SHRINK) { + xfree(tmp.e.ntries); +#if METHOD_CACHE_STATS + rb_meth_cache.sum_used -= tmp.size; + if (tmp.size > 0) { + rb_meth_cache.not_empty--; + } +#endif + while(tmp.size < tmp.capa / MCACHE_SHRINK_BOUND && tmp.capa > MCACHE_MIN_SHRINK) { + tmp.capa /= 2; + } + tmp.size = 0; + goto redo; + } +#if METHOD_CACHE_STATS + rb_meth_cache.sum_used -= cache->size; + rb_meth_cache.sum_capa += tmp.capa; + if (cache->capa == MCACHE_INLINED) { + rb_meth_cache.alloced++; + rb_meth_cache.created++; + } +#endif + if (cache->capa > MCACHE_INLINED) { + xfree(cache->e.ntries); + } + *cache = tmp; +} + +static inline void +rb_mcache_reset(struct rb_meth_cache *cache, rb_serial_t class_serial) +{ +#if METHOD_CACHE_STATS + cache->resets++; + rb_meth_cache.resets++; + rb_meth_cache.sum_used -= cache->size; + rb_meth_cache.sum_undefs -= cache->undefs; + cache->undefs = 0; + if (cache->size > 0) { + rb_meth_cache.not_empty--; + } +#endif + if (cache->is_copy) { +#if METHOD_CACHE_STATS + rb_meth_cache.copies--; +#endif + cache->is_copy = 0; +#if MCACHE_RESET_FREES_COPY + if (cache->capa > MCACHE_INLINED && cache->e.ntries != NULL) { + xfree(cache->e.ntries); + cache->e.ntries = NULL; +#if METHOD_CACHE_STATS + rb_meth_cache.copy_reset++; + rb_meth_cache.sum_capa -= cache->capa; + rb_meth_cache.alloced--; +#endif + cache->capa = 0; + } +#endif + } + cache->method_state = GET_GLOBAL_METHOD_STATE(); + cache->class_serial = class_serial; + cache->size = 0; + if (cache->capa == 0 || cache->capa == MCACHE_INLINED) { + cache->capa = MCACHE_INLINED; + MEMZERO(cache->e.n, cache_entry_t, MCACHE_INLINED); + } + else if (cache->e.ntries != NULL) { + MEMZERO(cache->e.ntries, cache_entry_t, cache->capa); + } +} + +static inline cache_entry_t* +rb_mcache_find(struct rb_meth_cache *cache, ID id) +{ + if (cache->capa == MCACHE_INLINED) { + if (cache->e.n[0].mid == id) return &cache->e.n[0]; +#if MCACHE_INLINED > 1 + if (cache->e.n[1].mid == id) return &cache->e.n[1]; +#endif +#if MCACHE_INLINED > 2 + if (cache->e.n[2].mid == id) return &cache->e.n[2]; +#endif +#if MCACHE_INLINED > 3 + if (cache->e.n[3].mid == id) return &cache->e.n[3]; +#endif +#if MCACHE_INLINED > 4 +#error "Are you serious about such huge MCACHE_INLINED?" +#endif + return NULL; + } + else { + cache_entry_t *ent = cache->e.ntries; + int mask = cache->capa - 1; + int pos = HASH(id) & mask; + int dlt; + if (ent[pos].mid == id) return ent + pos; + if (!COLLISION(ent[pos])) return NULL; + dlt = (id % mask) | 1; + for(;;) { + pos = (pos + dlt) & mask; + if (ent[pos].mid == id) return ent + pos; + if (!COLLISION(ent[pos])) return NULL; + } + } +} + +void +rb_method_cache_copy(VALUE from, VALUE to) +{ + struct rb_meth_cache *from_cache = &RCLASS_EXT(from)->cache, *to_cache = &RCLASS_EXT(to)->cache; + if (!from_cache->size) return; + + if (to_cache->capa > MCACHE_INLINED && to_cache->e.ntries) { + xfree(to_cache->e.ntries); +#if METHOD_CACHE_STATS + rb_meth_cache.alloced--; + rb_meth_cache.sum_capa -= to_cache->capa; + rb_meth_cache.sum_used -= to_cache->size; + rb_meth_cache.sum_undefs -= to_cache->undefs; + } + if (to_cache->is_copy) { + rb_meth_cache.copies--; + } + if (to_cache->size > 0) { + rb_meth_cache.not_empty--; + } + if (from_cache->size > 0) { + rb_meth_cache.not_empty++; +#endif + } + to_cache->capa = from_cache->capa; + to_cache->size = from_cache->size; + to_cache->method_state = from_cache->method_state; + to_cache->class_serial = RCLASS_SERIAL(to); + to_cache->is_copy = 1; + if (from_cache->capa > MCACHE_INLINED) { + to_cache->e.ntries = xcalloc(to_cache->capa, sizeof(cache_entry_t)); + MEMCPY(to_cache->e.ntries, from_cache->e.ntries, cache_entry_t, from_cache->capa); +#if METHOD_CACHE_STATS + rb_meth_cache.alloced++; + rb_meth_cache.sum_capa += to_cache->capa; + rb_meth_cache.copy_alloced++; +#endif + } else { + MEMCPY(to_cache->e.n, from_cache->e.n, cache_entry_t, MCACHE_INLINED); + } +#if METHOD_CACHE_STATS + to_cache->undefs = from_cache->undefs; + rb_meth_cache.sum_used += to_cache->size; + rb_meth_cache.sum_undefs += to_cache->undefs; + rb_meth_cache.copies++; +#endif +} + +#if METHOD_CACHE_STATS +VALUE +rb_method_cache_stats(int argc, VALUE* argv, VALUE obj) +{ + ID used, capa, copy, alloced, resets, insertions, undefs; + ID created, destroyed, copy_alloced, copy_reset, not_empty; + VALUE res = rb_hash_new(); + CONST_ID(used, "used"); + CONST_ID(capa, "capa"); + CONST_ID(copy, "copy"); + CONST_ID(not_empty, "not_empty"); + CONST_ID(alloced, "alloced"); + CONST_ID(resets, "resets"); + CONST_ID(insertions, "insertions"); + CONST_ID(undefs, "undefs"); + CONST_ID(created, "created"); + CONST_ID(destroyed, "destroyed"); + CONST_ID(copy_alloced, "copy_alloced"); + CONST_ID(copy_reset, "copy_reset"); + + rb_check_arity(argc, 0, 1); + if (argc == 1) { + VALUE klass = argv[0]; + struct rb_meth_cache *cache; + int type = BUILTIN_TYPE(klass); + if (type != T_CLASS && type != T_ICLASS) { + Check_Type(klass, T_CLASS); + } + cache = &RCLASS_EXT(klass)->cache; + rb_hash_aset(res, ID2SYM(used), INT2FIX(cache->size)); + rb_hash_aset(res, ID2SYM(undefs), INT2FIX(cache->undefs)); + if (cache->capa <= MCACHE_INLINED) { + rb_hash_aset(res, ID2SYM(capa), INT2FIX(0)); + rb_hash_aset(res, ID2SYM(alloced), INT2FIX(0)); + } else { + rb_hash_aset(res, ID2SYM(capa), INT2FIX(cache->capa)); + rb_hash_aset(res, ID2SYM(alloced), INT2FIX(1)); + } + rb_hash_aset(res, ID2SYM(not_empty), INT2FIX(cache->size > 0)); + rb_hash_aset(res, ID2SYM(copy), INT2FIX(cache->is_copy)); + rb_hash_aset(res, ID2SYM(resets), SIZET2NUM(cache->resets)); + rb_hash_aset(res, ID2SYM(insertions), SIZET2NUM(cache->insertions)); + } else if (argc == 0) { + rb_hash_aset(res, ID2SYM(used), SIZET2NUM(rb_meth_cache.sum_used)); + rb_hash_aset(res, ID2SYM(undefs), SIZET2NUM(rb_meth_cache.sum_undefs)); + rb_hash_aset(res, ID2SYM(capa), SIZET2NUM(rb_meth_cache.sum_capa)); + rb_hash_aset(res, ID2SYM(alloced), SIZET2NUM(rb_meth_cache.alloced)); + rb_hash_aset(res, ID2SYM(not_empty), SIZET2NUM(rb_meth_cache.not_empty)); + rb_hash_aset(res, ID2SYM(copy), SIZET2NUM(rb_meth_cache.copies)); + rb_hash_aset(res, ID2SYM(resets), SIZET2NUM(rb_meth_cache.resets)); + rb_hash_aset(res, ID2SYM(insertions), SIZET2NUM(rb_meth_cache.insertions)); + rb_hash_aset(res, ID2SYM(created), SIZET2NUM(rb_meth_cache.created)); + rb_hash_aset(res, ID2SYM(destroyed), SIZET2NUM(rb_meth_cache.destroyed)); + rb_hash_aset(res, ID2SYM(copy_alloced), SIZET2NUM(rb_meth_cache.copy_alloced)); + rb_hash_aset(res, ID2SYM(copy_reset), SIZET2NUM(rb_meth_cache.copy_reset)); + } + return res; +} +#endif -static struct cache_entry global_method_cache[GLOBAL_METHOD_CACHE_SIZE]; #define ruby_running (GET_VM()->running) /* int ruby_running = 0; */ @@ -578,20 +878,11 @@ rb_method_entry_get_without_cache(VALUE klass, ID id, } if (ruby_running) { - struct cache_entry *ent; - ent = GLOBAL_METHOD_CACHE(klass, id); - ent->class_serial = RCLASS_EXT(klass)->class_serial; - ent->method_state = GET_GLOBAL_METHOD_STATE(); - ent->defined_class = defined_class; - ent->mid = id; - + struct rb_classext_struct *ext = RCLASS_EXT(klass); if (UNDEFINED_METHOD_ENTRY_P(me)) { - ent->me = 0; me = 0; } - else { - ent->me = me; - } + rb_mcache_insert(&ext->cache, id, (uintptr_t)me, defined_class); } if (defined_class_ptr) @@ -617,18 +908,22 @@ rb_method_entry_t * rb_method_entry(VALUE klass, ID id, VALUE *defined_class_ptr) { #if OPT_GLOBAL_METHOD_CACHE - struct cache_entry *ent; - ent = GLOBAL_METHOD_CACHE(klass, id); - if (ent->method_state == GET_GLOBAL_METHOD_STATE() && - ent->class_serial == RCLASS_EXT(klass)->class_serial && - ent->mid == id) { + struct rb_classext_struct *ext = RCLASS_EXT(klass); + if (ext->cache.method_state != GET_GLOBAL_METHOD_STATE() || + ext->cache.class_serial != ext->class_serial) { + rb_mcache_reset(&ext->cache, ext->class_serial); + } else { + cache_entry_t *ent; + ent = rb_mcache_find(&ext->cache, id); + if (ent == NULL) goto not_found; if (defined_class_ptr) *defined_class_ptr = ent->defined_class; #if VM_DEBUG_VERIFY_METHOD_CACHE verify_method_cache(klass, id, ent->defined_class, ent->me); #endif - return ent->me; + return METHOD_ENTRY(ent); } +not_found: #endif return rb_method_entry_get_without_cache(klass, id, defined_class_ptr);