From f224e953c168f246d65530abb57aba2ef1e53da9 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Mon, 28 Sep 2015 16:49:58 -0400 Subject: [PATCH 01/23] Initial commit implementing a per-host circuit breaker state using shared memory and semaphores Shared memory stores the space of int+int+double[], corresponding to successes, array length, and the errors array itself --- .gitignore | 4 +- Rakefile | 8 +- ext/semian_cb_data/extconf.rb | 33 + ext/semian_cb_data/semian_cb_data.c | 900 ++++++++++++++++++++++ lib/semian.rb | 4 + lib/semian/circuit_breaker.rb | 46 +- lib/semian/circuit_breaker_shared_data.rb | 70 ++ semian.gemspec | 2 +- 8 files changed, 1047 insertions(+), 20 deletions(-) create mode 100644 ext/semian_cb_data/extconf.rb create mode 100644 ext/semian_cb_data/semian_cb_data.c create mode 100644 lib/semian/circuit_breaker_shared_data.rb diff --git a/.gitignore b/.gitignore index 91f4f528..5146cc98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ /.bundle/ -/lib/semian/*.so -/lib/semian/*.bundle +/lib/semian*/*.so +/lib/semian*/*.bundle /tmp/* *.gem /html/ diff --git a/Rakefile b/Rakefile index addd57d5..d0e48936 100644 --- a/Rakefile +++ b/Rakefile @@ -27,7 +27,11 @@ if Semian.sysv_semaphores_supported? ext.ext_dir = 'ext/semian' ext.lib_dir = 'lib/semian' end - task build: :compile + Rake::ExtensionTask.new('semian_cb_data', GEMSPEC) do |ext| + ext.ext_dir = 'ext/semian_cb_data' + ext.lib_dir = 'lib/semian_cb_data' + end + task :build => :compile else task :build do end @@ -49,7 +53,7 @@ task test: :build # ========================================================== require 'rdoc/task' RDoc::Task.new do |rdoc| - rdoc.rdoc_files.include("lib/*.rb", "ext/semian/*.c") + rdoc.rdoc_files.include("lib/*.rb", "ext/semian/*.c", "ext/semian_cb_data/*.c") end task default: :test diff --git a/ext/semian_cb_data/extconf.rb b/ext/semian_cb_data/extconf.rb new file mode 100644 index 00000000..9ae0aa2e --- /dev/null +++ b/ext/semian_cb_data/extconf.rb @@ -0,0 +1,33 @@ +$:.unshift File.expand_path("../../../lib", __FILE__) + +require 'semian/platform' + +unless Semian.sysv_semaphores_supported? + File.write "Makefile", < +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +// needed for semctl +union semun { + int val; /* Value for SETVAL */ + struct semid_ds *buf; /* Buffer for IPC_STAT, IPC_SET */ + unsigned short *array; /* Array for GETALL, SETALL */ + struct seminfo *__buf; /* Buffer for IPC_INFO + (Linux-specific) */ +}; + +#if defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL) && defined(HAVE_RUBY_THREAD_H) +// 2.0 +#include +#define WITHOUT_GVL(fn,a,ubf,b) rb_thread_call_without_gvl((fn),(a),(ubf),(b)) +#elif defined(HAVE_RB_THREAD_BLOCKING_REGION) + // 1.9 +typedef VALUE (*my_blocking_fn_t)(void*); +#define WITHOUT_GVL(fn,a,ubf,b) rb_thread_blocking_region((my_blocking_fn_t)(fn),(a),(ubf),(b)) +#endif + +// struct sembuf { // found in sys/sem.h +// unsigned short sem_num; /* semaphore number */ +// short sem_op; /* semaphore operation */ +// short sem_flg; /* operation flags */ +// }; + +typedef struct { + int successes; + int arr_length; + double errors[]; +} shared_cb_data; + +typedef struct { + //semaphore, shared memory data and pointer + key_t key; + size_t arr_max_size; + bool lock_triggered; + int semid; + int shmid; + shared_cb_data *shm_address; +} semian_cb_data; + +static int system_max_semaphore_count; +static const int kCBSemaphoreCount = 1; // # semaphores to be acquired +static const int kCBTicketMax = 1; +static const int kCBInitializeWaitTimeout = 5; /* seconds */ +static const int kCBIndexTicketLock = 0; +static const int kCBInternalTimeout = 5; /* seconds */ + +static struct sembuf decrement; // = { kCBIndexTicketLock, -1, SEM_UNDO}; +static struct sembuf increment; // = { kCBIndexTicketLock, 1, SEM_UNDO}; + +static VALUE eInternal, eSyscall, eTimeout; // Semian errors + +static void semian_cb_data_mark(void *ptr); +static void semian_cb_data_free(void *ptr); +static size_t semian_cb_data_memsize(const void *ptr); +static VALUE semian_cb_data_alloc(VALUE klass); +static VALUE semian_cb_data_init(VALUE self, VALUE name, VALUE size, VALUE permissions); +static VALUE semian_cb_data_clean(VALUE self); +static void set_semaphore_permissions(int sem_id, int permissions); +static void configure_tickets(int sem_id, int tickets, int should_initialize); +static int create_semaphore(int key, int permissions, int *created); +static VALUE semian_cb_data_acquire_semaphore (VALUE self, VALUE permissions); +static VALUE semian_cb_data_delete_semaphore(VALUE self); +static VALUE semian_cb_data_lock(VALUE self); +static VALUE semian_cb_data_unlock(VALUE self); +static void *semian_cb_data_lock_without_gvl(void *self); +static void *semian_cb_data_unlock_without_gvl(void *self); +static VALUE semian_cb_data_acquire_memory(VALUE self, VALUE permissions); +static void semian_cb_data_delete_memory_inner (semian_cb_data *ptr); +static VALUE semian_cb_data_delete_memory (VALUE self); +static VALUE semian_cb_data_get_successes(VALUE self); +static VALUE semian_cb_data_set_successes(VALUE self, VALUE num); +static VALUE semian_cb_data_semid(VALUE self); +static VALUE semian_cb_data_shmid(VALUE self); +static VALUE semian_cb_data_array_at_index(VALUE self, VALUE idx); +static VALUE semian_cb_data_array_set_index(VALUE self, VALUE idx, VALUE val); +static VALUE semian_cb_data_array_length(VALUE self); +static VALUE semian_cb_data_set_push_back(VALUE self, VALUE num); +static VALUE semian_cb_data_set_pop_back(VALUE self); +static VALUE semian_cb_data_set_push_front(VALUE self, VALUE num); +static VALUE semian_cb_data_set_pop_front(VALUE self); + +// needed for TypedData_Make_Struct && TypedData_Get_Struct +static const rb_data_type_t +semian_cb_data_type = { + "semian_cb_data", + { + semian_cb_data_mark, + semian_cb_data_free, + semian_cb_data_memsize + }, + NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY +}; + +/* + * Generate key + */ +static key_t +generate_key(const char *name) +{ + union { + unsigned char str[SHA_DIGEST_LENGTH]; + key_t key; + } digest; + SHA1((const unsigned char *) name, strlen(name), digest.str); + /* TODO: compile-time assertion that sizeof(key_t) > SHA_DIGEST_LENGTH */ + return digest.key; +} + +/* + * Log errors + */ +static void +raise_semian_syscall_error(const char *syscall, int error_num) +{ + rb_raise(eSyscall, "%s failed, errno %d (%s)", syscall, error_num, strerror(error_num)); +} + +/* + * Functions that handle type and memory +*/ +static void +semian_cb_data_mark(void *ptr) +{ + /* noop */ +} + +static void +semian_cb_data_free(void *ptr) +{ + semian_cb_data *data = (semian_cb_data *) ptr; + + + // Under normal circumstances, memory use should be in the order of bytes, + // and shouldn't increase if the same key/id is used + // so there is no need to call this unless certain all other semian processes are stopped + // (also raises concurrency errors: "object allocation during garbage collection phase") + + //semian_cb_data_delete_memory_inner (data); + + xfree(data); +} + +static size_t +semian_cb_data_memsize(const void *ptr) +{ + return sizeof(semian_cb_data); +} + +static VALUE +semian_cb_data_alloc(VALUE klass) +{ + VALUE obj; + semian_cb_data *ptr; + + obj = TypedData_Make_Struct(klass, semian_cb_data, &semian_cb_data_type, ptr); + return obj; +} + + + + + + +/* + * Init function exposed as ._initialize() that is delegated by .initialize() + */ +static VALUE +semian_cb_data_init(VALUE self, VALUE id, VALUE size, VALUE permissions) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + + if (TYPE(id) != T_SYMBOL && TYPE(id) != T_STRING) + rb_raise(rb_eTypeError, "id must be a symbol or string"); + if (TYPE(size) != T_FIXNUM /*|| TYPE(size) != T_BIGNUM*/) + rb_raise(rb_eTypeError, "expected integer for arr_max_size"); + if (TYPE(permissions) != T_FIXNUM) + rb_raise(rb_eTypeError, "expected integer for permissions"); + + if (NUM2SIZET(size) <= 0) + rb_raise(rb_eArgError, "arr_max_size must be larger than 0"); + + const char *id_str = NULL; + if (TYPE(id) == T_SYMBOL) { + id_str = rb_id2name(rb_to_id(id)); + } else if (TYPE(id) == T_STRING) { + id_str = RSTRING_PTR(id); + } + ptr->key = generate_key(id_str); + //rb_warn("converted name %s to key %d", id_str, ptr->key); + + // Guarantee arr_max_size >=1 or error thrown + ptr->arr_max_size = NUM2SIZET(size); + + // id's default to -1 + ptr->semid = -1; + ptr->shmid = -1; + // addresses default to NULL + ptr->shm_address = 0; + ptr->lock_triggered = false; + + semian_cb_data_acquire_semaphore(self, permissions); + semian_cb_data_acquire_memory(self, permissions); + + return self; +} + +static VALUE +semian_cb_data_clean(VALUE self) +{ + semian_cb_data_delete_memory(self); + semian_cb_data_delete_semaphore(self); + return self; +} + + + +/* + * Functions set_semaphore_permissions, configure_tickets, create_semaphore + * are taken from semian.c with extra code removed + */ +static void +set_semaphore_permissions(int sem_id, int permissions) +{ + union semun sem_opts; + struct semid_ds stat_buf; + + sem_opts.buf = &stat_buf; + semctl(sem_id, 0, IPC_STAT, sem_opts); + if ((stat_buf.sem_perm.mode & 0xfff) != permissions) { + stat_buf.sem_perm.mode &= ~0xfff; + stat_buf.sem_perm.mode |= permissions; + semctl(sem_id, 0, IPC_SET, sem_opts); + } +} + + +static void +configure_tickets(int sem_id, int tickets, int should_initialize) +{ + struct timeval start_time, cur_time; + + if (should_initialize) { + if (-1 == semctl(sem_id, 0, SETVAL, kCBTicketMax)) { + rb_warn("semctl: failed to set semaphore with semid %d, position 0 to %d", sem_id, 1); + raise_semian_syscall_error("semctl()", errno); + } else { + //rb_warn("semctl: set semaphore with semid %d, position 0 to %d", sem_id, 1); + } + } else if (tickets > 0) { + // it's possible that we haven't actually initialized the + // semaphore structure yet - wait a bit in that case + int ret; + if (0 == (ret = semctl(sem_id, 0, GETVAL))) { + gettimeofday(&start_time, NULL); + while (0 == (ret = semctl(sem_id, 0, GETVAL))) { // loop while value == 0 + usleep(10000); /* 10ms */ + gettimeofday(&cur_time, NULL); + if ((cur_time.tv_sec - start_time.tv_sec) > kCBInitializeWaitTimeout) { + rb_raise(eInternal, "timeout waiting for semaphore initialization"); + } + } + if (-1 == ret) { + rb_raise(eInternal, "error getting max ticket count, errno: %d (%s)", errno, strerror(errno)); + } + } + + // Rest of the function (originally from semian.c) was removed since it isn't needed + } +} + +static int +create_semaphore(int key, int permissions, int *created) +{ + int semid = 0; + int flags = 0; + + *created = 0; + flags = IPC_EXCL | IPC_CREAT | permissions; + + semid = semget(key, kCBSemaphoreCount, flags); + if (semid >= 0) { + *created = 1; + //rb_warn("semget: received %d semaphore(s) with key %d, semid %d", kCBSemaphoreCount, key, semid); + } else if (semid == -1 && errno == EEXIST) { + flags &= ~IPC_EXCL; + semid = semget(key, kCBSemaphoreCount, flags); + //rb_warn("semget: retrieved existing semaphore with key %d, semid %d", key, semid); + } + return semid; +} + + + + +/* + * Create or acquire previously made semaphore + */ + +static VALUE +semian_cb_data_acquire_semaphore (VALUE self, VALUE permissions) +{ + // Function flow, semaphore creation methods are + // borrowed from semian.c since they have been previously tested + + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + + if (TYPE(permissions) != T_FIXNUM) + rb_raise(rb_eTypeError, "expected integer for permissions"); + + // bool for initializing (configure_tickets) or not + int created = 0; + key_t key = ptr->key; + int semid = create_semaphore(key, FIX2LONG(permissions), &created); + if (-1 == semid) { + raise_semian_syscall_error("semget()", errno); + } + ptr->semid = semid; + + // initialize to 1 and set permissions + configure_tickets(ptr->semid, kCBTicketMax, created); + set_semaphore_permissions(ptr->semid, FIX2LONG(permissions)); + + return self; +} + + +static VALUE +semian_cb_data_delete_semaphore(VALUE self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + if (-1 == ptr->semid) // do nothing if semaphore not acquired + return self; + + if (-1 == semctl(ptr->semid, 0, IPC_RMID)) { + if (EIDRM == errno) { + rb_warn("semctl: failed to delete semaphore set with semid %d: already removed", ptr->semid); + ptr->semid = -1; + } else { + rb_warn("semctl: failed to remove semaphore with semid %d, errno %d (%s)",ptr->semid, errno, strerror(errno)); + } + } else { + //rb_warn("semctl: semaphore set with semid %d deleted", ptr->semid); + ptr->semid = -1; + } + return self; +} + + + + +/* + * semian_cb_data_lock/unlock and associated functions decrement/increment semaphore + */ + +static VALUE +semian_cb_data_lock(VALUE self) +{ + return (VALUE) WITHOUT_GVL(semian_cb_data_lock_without_gvl, (void *)self, RUBY_UBF_IO, NULL); +} + +static void * +semian_cb_data_lock_without_gvl(void *self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct((VALUE)self, semian_cb_data, &semian_cb_data_type, ptr); + if (ptr->lock_triggered) + return (void *)Qtrue; + if (-1 == ptr->semid){ + rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); + return (void *)Qfalse; + } + VALUE retval; + + struct timespec ts = { 0 }; + ts.tv_sec = kCBInternalTimeout; + + if (-1 == semtimedop(ptr->semid,&decrement,1, &ts)) { + rb_raise(eInternal, "error with semop locking, %d: (%s)", errno, strerror(errno)); + retval=Qfalse; + } else + retval=Qtrue; + + ptr->lock_triggered = true; + //rb_warn("semop: lock success"); + return (void *)retval; +} + +static VALUE +semian_cb_data_unlock(VALUE self) +{ + return (VALUE) WITHOUT_GVL(semian_cb_data_unlock_without_gvl, (void *)self, RUBY_UBF_IO, NULL); +} + +static void * +semian_cb_data_unlock_without_gvl(void *self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct((VALUE)self, semian_cb_data, &semian_cb_data_type, ptr); + if (!(ptr->lock_triggered)) + return (void *)Qtrue; + if (-1 == ptr->semid){ + rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); + return (void *)Qfalse; + } + VALUE retval; + + struct timespec ts = { 0 }; + ts.tv_sec = kCBInternalTimeout; + + if (-1 == semtimedop(ptr->semid,&increment,1 , &ts)) { + rb_raise(eInternal, "error with semop unlocking, errno: %d (%s)", errno, strerror(errno)); + retval=Qfalse; + } else + retval=Qtrue; + + ptr->lock_triggered = false; + //rb_warn("semop unlock success"); + return (void *)retval; +} + + + + +/* + Acquire memory by getting shmid, and then attaching it to a memory location, + requires semaphore for locking/unlocking to be setup +*/ +static VALUE +semian_cb_data_acquire_memory(VALUE self, VALUE permissions) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + + if (-1 == ptr->semid){ + rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); + return self; + } + if (TYPE(permissions) != T_FIXNUM) + rb_raise(rb_eTypeError, "expected integer for permissions"); + + if (!semian_cb_data_lock(self)) + return self; + + key_t key = ptr->key; + if (-1 == (ptr->shmid = shmget( key, + 2*sizeof(int) + ptr->arr_max_size * sizeof(double), + IPC_CREAT | IPC_EXCL | FIX2LONG(permissions)))) { + if (errno == EEXIST) + ptr->shmid = shmget(key, ptr->arr_max_size, IPC_CREAT); + } + if (-1 == ptr->shmid) { + rb_raise(eSyscall, "shmget() failed to acquire a memory shmid with key %d, size %zu, errno %d (%s)", key, ptr->arr_max_size, errno, strerror(errno)); + } else { + //rb_warn("shmget: successfully got memory id with key %d, shmid %d, size %zu", key, ptr->shmid, ptr->arr_max_size); + } + + if (0 == ptr->shm_address) { + ptr->shm_address = shmat(ptr->shmid, (void *)0, 0); + if (((void*)-1) == ptr->shm_address) { + rb_raise(eSyscall, "shmat() failed to attach memory with shmid %d, size %zu, errno %d (%s)", ptr->shmid, ptr->arr_max_size, errno, strerror(errno)); + ptr->shm_address = 0; + } else { + //rb_warn("shmat: successfully attached shmid %d to %p", ptr->shmid, ptr->shm_address); + shared_cb_data *data = ptr->shm_address; + data->successes = 0; + data->arr_length = 0; + int i=0; + for (; i< data->arr_length; ++i) + data->errors[i]=0; + + } + } + + semian_cb_data_unlock(self); + return self; +} + +static void +semian_cb_data_delete_memory_inner (semian_cb_data *ptr) +{ + if (0 != ptr->shm_address){ + if (-1 == shmdt(ptr->shm_address)) { + rb_raise(eSyscall,"shmdt: no attached memory at %p, errno %d (%s)", ptr->shm_address, errno, strerror(errno)); + } else { + rb_warn("shmdt: successfully detached memory at %p", ptr->shm_address); + } + ptr->shm_address = 0; + } + + if (-1 != ptr->shmid) { + // Once IPC_RMID is set, no new attaches can be made + if (-1 == shmctl(ptr->shmid, IPC_RMID, 0)) { + if (errno == EINVAL) { + ptr->shmid = -1; + } { + rb_raise(eSyscall,"shmctl: error removing memory with shmid %d, errno %d (%s)", ptr->shmid, errno, strerror(errno)); + } + } else { + ptr->shmid = -1; + } + } +} + + +static VALUE +semian_cb_data_delete_memory (VALUE self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + + if (!semian_cb_data_lock(self)) + return self; + + semian_cb_data_delete_memory_inner(ptr); + + semian_cb_data_unlock(self); + return self; +} + + + + + +/* + * Below are methods for successes, semid, shmid, and array pop, push, peek at front and back + * and clear, length + */ + +static VALUE +semian_cb_data_get_successes(VALUE self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + + // check shared memory for NULL + if (0 == ptr->shm_address) + return Qnil; + + + int successes = ptr->shm_address->successes; + + semian_cb_data_unlock(self); + return INT2NUM(successes); +} + +static VALUE +semian_cb_data_set_successes(VALUE self, VALUE num) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + + if (0 == ptr->shm_address) + return Qnil; + + if (!semian_cb_data_lock(self)) + return Qnil; + + if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT/*|| TYPE(size) != T_BIGNUM*/) + return Qnil; + + ptr->shm_address->successes = NUM2INT(num); + + semian_cb_data_unlock(self); + return num; +} + + +static VALUE +semian_cb_data_semid(VALUE self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + return INT2NUM(ptr->semid); +} +static VALUE +semian_cb_data_shmid(VALUE self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + return INT2NUM(ptr->shmid); +} + +static VALUE +semian_cb_data_array_at_index(VALUE self, VALUE idx) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + + if (0 == ptr->shm_address) + return Qnil; + + if (TYPE(idx) != T_FIXNUM && TYPE(idx) != T_FLOAT/*|| TYPE(size) != T_BIGNUM*/) + return Qnil; + + int index = NUM2INT(idx); + + if (index <0 || index >= ptr->arr_max_size) { + return Qnil; + } + + if (!semian_cb_data_lock(self)) + return Qnil; + shared_cb_data *data = ptr->shm_address; + VALUE retval = index < (data->arr_length) ? DBL2NUM(data->errors[index]) : Qnil; + + semian_cb_data_unlock(self); + return retval; + +} + +static VALUE +semian_cb_data_array_set_index(VALUE self, VALUE idx, VALUE val) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + + // check shared memory for NULL + if (0 == ptr->shm_address) + return Qnil; + + if (TYPE(idx) != T_FIXNUM && TYPE(idx) != T_FLOAT/*|| TYPE(size) != T_BIGNUM*/) + return Qnil; + if (TYPE(val) != T_FIXNUM && TYPE(val) != T_FLOAT/*|| TYPE(size) != T_BIGNUM*/) + return Qnil; + + int index = NUM2INT(idx); + double value = NUM2DBL(val); + + if (index <0 || index >= ptr->arr_max_size) { + return Qnil; + } + + if (!semian_cb_data_lock(self)){ + return Qnil; + } + + ptr->shm_address->errors[index] = value; + ptr->shm_address->arr_length = index+1; + + semian_cb_data_unlock(self); + return val; + +} + +static VALUE +semian_cb_data_array_length(VALUE self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + if (0 == ptr->shm_address) + return Qnil; + if (!semian_cb_data_lock(self)) + return Qnil; + int arr_length =ptr->shm_address->arr_length; + semian_cb_data_unlock(self); + return INT2NUM(arr_length); +} + +static VALUE +semian_cb_data_set_push_back(VALUE self, VALUE num) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + if (0 == ptr->shm_address) + return Qnil; + if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT/*|| TYPE(size) != T_BIGNUM*/) + return Qnil; + + if (!semian_cb_data_lock(self)) + return Qnil; + + shared_cb_data *data = ptr->shm_address; + if (data->arr_length == ptr->arr_max_size) { + int i; + for (i=1; i< ptr->arr_max_size; ++i){ + data->errors[i-1] = data->errors[i]; + } + --(data->arr_length); + } + data->errors[(data->arr_length)] = NUM2DBL(num); + ++(data->arr_length); + semian_cb_data_unlock(self); + return self; +} + +static VALUE +semian_cb_data_set_pop_back(VALUE self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + if (0 == ptr->shm_address) + return Qnil; + + if (!semian_cb_data_lock(self)) + return Qnil; + + VALUE retval; + shared_cb_data *data = ptr->shm_address; + if (0 == data->arr_length) + retval = Qnil; + else { + retval = DBL2NUM(data->errors[data->arr_length-1]); + --(data->arr_length); + } + + semian_cb_data_unlock(self); + return retval; +} + +static VALUE +semian_cb_data_set_pop_front(VALUE self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + if (0 == ptr->shm_address) + return Qnil; + + if (!semian_cb_data_lock(self)) + return Qnil; + + VALUE retval; + shared_cb_data *data = ptr->shm_address; + if (0 >= data->arr_length) + retval = Qnil; + else { + retval = DBL2NUM(data->errors[0]); + int i=0; + for (; iarr_length-1; ++i) + data->errors[i]=data->errors[i+1]; + --(data->arr_length); + } + + semian_cb_data_unlock(self); + return retval; +} + +static VALUE +semian_cb_data_set_push_front(VALUE self, VALUE num) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + if (0 == ptr->shm_address) + return Qnil; + if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT/*|| TYPE(size) != T_BIGNUM*/) + return Qnil; + + if (!semian_cb_data_lock(self)) + return Qnil; + + double val = NUM2DBL(num); + shared_cb_data *data = ptr->shm_address; + + int i=data->arr_length; + for (; i>0; --i) + data->errors[i]=data->errors[i-1]; + + data->errors[0] = val; + ++(data->arr_length); + if (data->arr_length>ptr->arr_max_size) + data->arr_length=ptr->arr_max_size; + + semian_cb_data_unlock(self); + return self; +} + +static VALUE +semian_cb_data_array_clear(VALUE self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + if (0 == ptr->shm_address) + return Qnil; + + if (!semian_cb_data_lock(self)) + return Qnil; + ptr->shm_address->arr_length=0; + + semian_cb_data_unlock(self); + return self; +} + +static VALUE +semian_cb_data_array_first(VALUE self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + if (0 == ptr->shm_address) + return Qnil; + + if (!semian_cb_data_lock(self)) + return Qnil; + + VALUE retval; + if (ptr->shm_address->arr_length >=1 && 1 <= ptr->arr_max_size) + retval = DBL2NUM(ptr->shm_address->errors[0]); + else + retval = Qnil; + + semian_cb_data_unlock(self); + return retval; +} + +static VALUE +semian_cb_data_array_last(VALUE self) +{ + semian_cb_data *ptr; + TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + if (0 == ptr->shm_address) + return Qnil; + + if (!semian_cb_data_lock(self)) + return Qnil; + + VALUE retval; + if (ptr->shm_address->arr_length > 0) + retval = DBL2NUM(ptr->shm_address->errors[ptr->shm_address->arr_length-1]); + else + retval = Qnil; + + semian_cb_data_unlock(self); + return retval; +} + +void +Init_semian_cb_data (void) { + + VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); + + VALUE cCircuitBreakerSharedData = rb_const_get(cSemianModule, rb_intern("CircuitBreakerSharedData")); + + rb_define_alloc_func(cCircuitBreakerSharedData, semian_cb_data_alloc); + rb_define_method(cCircuitBreakerSharedData, "_initialize", semian_cb_data_init, 3); + rb_define_method(cCircuitBreakerSharedData, "_cleanup", semian_cb_data_clean, 0); + //rb_define_method(cCircuitBreakerSharedData, "acquire_semaphore", semian_cb_data_acquire_semaphore, 1); + //rb_define_method(cCircuitBreakerSharedData, "delete_semaphore", semian_cb_data_delete_semaphore, 0); + //rb_define_method(cCircuitBreakerSharedData, "lock", semian_cb_data_lock, 0); + //rb_define_method(cCircuitBreakerSharedData, "unlock", semian_cb_data_unlock, 0); + //rb_define_method(cCircuitBreakerSharedData, "acquire_memory", semian_cb_data_acquire_memory, 1); + //rb_define_method(cCircuitBreakerSharedData, "delete_memory", semian_cb_data_delete_memory, 0); + + rb_define_method(cCircuitBreakerSharedData, "semid", semian_cb_data_semid, 0); + rb_define_method(cCircuitBreakerSharedData, "shmid", semian_cb_data_shmid, 0); + rb_define_method(cCircuitBreakerSharedData, "successes", semian_cb_data_get_successes, 0); + rb_define_method(cCircuitBreakerSharedData, "successes=", semian_cb_data_set_successes, 1); + + rb_define_method(cCircuitBreakerSharedData, "[]", semian_cb_data_array_at_index, 1); + rb_define_method(cCircuitBreakerSharedData, "[]=", semian_cb_data_array_set_index, 2); + rb_define_method(cCircuitBreakerSharedData, "length", semian_cb_data_array_length, 0); + rb_define_method(cCircuitBreakerSharedData, "size", semian_cb_data_array_length, 0); + rb_define_method(cCircuitBreakerSharedData, "count", semian_cb_data_array_length, 0); + rb_define_method(cCircuitBreakerSharedData, "<<", semian_cb_data_set_push_back, 1); + rb_define_method(cCircuitBreakerSharedData, "push", semian_cb_data_set_push_back, 1); + rb_define_method(cCircuitBreakerSharedData, "pop", semian_cb_data_set_pop_back, 0); + rb_define_method(cCircuitBreakerSharedData, "shift", semian_cb_data_set_pop_front, 0); + rb_define_method(cCircuitBreakerSharedData, "unshift", semian_cb_data_set_push_front, 1); + rb_define_method(cCircuitBreakerSharedData, "clear", semian_cb_data_array_clear, 0); + rb_define_method(cCircuitBreakerSharedData, "first", semian_cb_data_array_first, 0); + rb_define_method(cCircuitBreakerSharedData, "last", semian_cb_data_array_last, 0); + + eInternal = rb_const_get(cSemianModule, rb_intern("InternalError")); + eSyscall = rb_const_get(cSemianModule, rb_intern("SyscallError")); + eTimeout = rb_const_get(cSemianModule, rb_intern("TimeoutError")); + + decrement.sem_num = kCBIndexTicketLock; + decrement.sem_op = -1; + decrement.sem_flg = SEM_UNDO; + + increment.sem_num = kCBIndexTicketLock; + increment.sem_op = 1; + increment.sem_flg = SEM_UNDO; + + struct seminfo info_buf; + + if (semctl(0, 0, SEM_INFO, &info_buf) == -1) { + rb_raise(eInternal, "unable to determine maximum semaphore count - semctl() returned %d: %s ", errno, strerror(errno)); + } + system_max_semaphore_count = info_buf.semvmx; + + /* Maximum number of tickets available on this system. */ + rb_const_get(cSemianModule, rb_intern("MAX_TICKETS")); +} diff --git a/lib/semian.rb b/lib/semian.rb index 7c3255da..61c3f0ab 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -119,10 +119,12 @@ def to_s # Returns the registered resource. def register(name, tickets:, permissions: 0660, timeout: 0, error_threshold:, error_timeout:, success_threshold:, exceptions: []) circuit_breaker = CircuitBreaker.new( + name, success_threshold: success_threshold, error_threshold: error_threshold, error_timeout: error_timeout, exceptions: Array(exceptions) + [::Semian::BaseError], + permissions: permissions ) resource = Resource.new(name, tickets: tickets, permissions: permissions, timeout: timeout) resources[name] = ProtectedResource.new(resource, circuit_breaker) @@ -154,8 +156,10 @@ def resources require 'semian/protected_resource' require 'semian/unprotected_resource' require 'semian/platform' +require 'semian/circuit_breaker_shared_data' if Semian.sysv_semaphores_supported? && Semian.semaphores_enabled? require 'semian/semian' + require 'semian_cb_data/semian_cb_data' else Semian::MAX_TICKETS = 0 unless Semian.sysv_semaphores_supported? diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index cc78b6ce..008a644e 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -2,11 +2,17 @@ module Semian class CircuitBreaker attr_reader :state - def initialize(exceptions:, success_threshold:, error_threshold:, error_timeout:) + def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, permissions: 0660) + @name = "#{name}_cb" @success_count_threshold = success_threshold @error_count_threshold = error_threshold @error_timeout = error_timeout @exceptions = exceptions + + @shared_circuit_breaker_data = ::Semian::CircuitBreakerSharedData.new( + @name, + @error_count_threshold, + permissions) reset end @@ -32,8 +38,8 @@ def request_allowed? end def mark_failed(_error) - push_time(@errors, @error_count_threshold, duration: @error_timeout) - + #push_time(@errors, @error_count_threshold, duration: @error_timeout) + push_time(@shared_circuit_breaker_data, @error_count_threshold, duration: @error_timeout) if closed? open if error_threshold_reached? elsif half_open? @@ -43,14 +49,16 @@ def mark_failed(_error) def mark_success return unless half_open? - @successes += 1 + #@successes += 1 + @shared_circuit_breaker_data.successes = @shared_circuit_breaker_data.successes+1 close if success_threshold_reached? end def reset - @errors = [] - @successes = 0 - close + #@errors = [] + @shared_circuit_breaker_data.clear + #@successes = 0 + @shared_circuit_breaker_data.successes=0 end private @@ -62,7 +70,8 @@ def closed? def close log_state_transition(:closed) @state = :closed - @errors = [] + #@errors = [] + @shared_circuit_breaker_data.clear end def open? @@ -81,32 +90,39 @@ def half_open? def half_open log_state_transition(:half_open) @state = :half_open - @successes = 0 + #@successes = 0 + @shared_circuit_breaker_data.successes=0 end def success_threshold_reached? - @successes >= @success_count_threshold + #@successes >= @success_count_threshold + @shared_circuit_breaker_data.successes >= @success_count_threshold end def error_threshold_reached? - @errors.count == @error_count_threshold + #@errors.count == @error_count_threshold + @shared_circuit_breaker_data.count == @error_count_threshold end def error_timeout_expired? - @errors.last && (@errors.last + @error_timeout < Time.now) + #@errors.last && (@errors.last + @error_timeout < Time.now) + time_f = @shared_circuit_breaker_data.last + time_f && (Time.at(time_f) + @error_timeout < Time.now) end def push_time(window, max_size, duration:, time: Time.now) - window.shift while window.first && window.first + duration < time + #window.shift while window.first && window.first + duration < time + window.shift while window.first && Time.at(window.first) + duration < time window.shift if window.size == max_size - window << time + #window << time + window << time.to_f end def log_state_transition(new_state) return if @state.nil? || new_state == @state str = "[#{self.class.name}] State transition from #{@state} to #{new_state}." - str << " success_count=#{@successes} error_count=#{@errors.count}" + str << " success_count=#{@shared_circuit_breaker_data.successes} error_count=#{@shared_circuit_breaker_data.count}" str << " success_count_threshold=#{@success_count_threshold} error_count_threshold=#{@error_count_threshold}" str << " error_timeout=#{@error_timeout} error_last_at=\"#{@error_last_at}\"" Semian.logger.info(str) diff --git a/lib/semian/circuit_breaker_shared_data.rb b/lib/semian/circuit_breaker_shared_data.rb new file mode 100644 index 00000000..e04e0c10 --- /dev/null +++ b/lib/semian/circuit_breaker_shared_data.rb @@ -0,0 +1,70 @@ +module Semian + class CircuitBreakerSharedData #:nodoc: + def initialize(name, arr_max_size, permissions) + _initialize(name, arr_max_size, permissions) if respond_to?(:_initialize) + end + + # For anyone consulting this, the array stores floats. Use Time.at(_float_here_) to convert to time + + def successes + 0 + end + + def successes=(num) + 0 + end + + def semid + 0 + end + + def shmid + 0 + end + + def length + 0 + end + + def size + 0 + end + + def count + 0 + end + + def << (float) + nil + end + + def push(float) + nil + end + + def pop + nil + end + + def shift + nil + end + + def unshift (float) + nil + end + + def clear + nil + end + + def first + nil + end + + def last + nil + end + + end +end diff --git a/semian.gemspec b/semian.gemspec index 1a46c2c8..d72768e8 100644 --- a/semian.gemspec +++ b/semian.gemspec @@ -17,7 +17,7 @@ Gem::Specification.new do |s| s.license = 'MIT' s.files = `git ls-files`.split("\n") - s.extensions = ['ext/semian/extconf.rb'] + s.extensions = ['ext/semian/extconf.rb','ext/semian_cb_data/extconf.rb'] s.add_development_dependency 'rake-compiler', '~> 0.9' s.add_development_dependency 'timecop' s.add_development_dependency 'mysql2' From 711ffa9f84512ed06089b01a279993e21d32f6aa Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Wed, 30 Sep 2015 17:28:23 +0000 Subject: [PATCH 02/23] Refactor to include some suggestions: Remove second Ruby extension Provide fallback Ruby implementation of Sliding Window Sliding window resizes if new size is given and Semian restarts New worker added does not reset shared memory Use integers in ms for time, not floats --- .gitignore | 4 +- Rakefile | 6 +- ext/semian/extconf.rb | 4 +- ext/semian/semian.c | 5 + .../semian_sliding_window.c} | 309 +++++++++--------- ext/semian_cb_data/extconf.rb | 33 -- lib/semian.rb | 3 +- lib/semian/circuit_breaker.rb | 31 +- lib/semian/circuit_breaker_shared_data.rb | 70 ---- lib/semian/sliding_window.rb | 74 +++++ semian.gemspec | 2 +- 11 files changed, 247 insertions(+), 294 deletions(-) rename ext/{semian_cb_data/semian_cb_data.c => semian/semian_sliding_window.c} (77%) delete mode 100644 ext/semian_cb_data/extconf.rb delete mode 100644 lib/semian/circuit_breaker_shared_data.rb create mode 100644 lib/semian/sliding_window.rb diff --git a/.gitignore b/.gitignore index 5146cc98..7631db29 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ /.bundle/ -/lib/semian*/*.so -/lib/semian*/*.bundle +/lib/**/*.so +/lib/**/*.bundle /tmp/* *.gem /html/ diff --git a/Rakefile b/Rakefile index d0e48936..d8642bb6 100644 --- a/Rakefile +++ b/Rakefile @@ -27,10 +27,6 @@ if Semian.sysv_semaphores_supported? ext.ext_dir = 'ext/semian' ext.lib_dir = 'lib/semian' end - Rake::ExtensionTask.new('semian_cb_data', GEMSPEC) do |ext| - ext.ext_dir = 'ext/semian_cb_data' - ext.lib_dir = 'lib/semian_cb_data' - end task :build => :compile else task :build do @@ -53,7 +49,7 @@ task test: :build # ========================================================== require 'rdoc/task' RDoc::Task.new do |rdoc| - rdoc.rdoc_files.include("lib/*.rb", "ext/semian/*.c", "ext/semian_cb_data/*.c") + rdoc.rdoc_files.include("lib/*.rb", "ext/semian/*.c") end task default: :test diff --git a/ext/semian/extconf.rb b/ext/semian/extconf.rb index 597e1ff5..31b68181 100644 --- a/ext/semian/extconf.rb +++ b/ext/semian/extconf.rb @@ -23,8 +23,8 @@ have_func 'rb_thread_blocking_region' have_func 'rb_thread_call_without_gvl' -$CFLAGS = "-D_GNU_SOURCE -Werror -Wall " -if ENV.key?('DEBUG') +$CFLAGS = "-D_GNU_SOURCE -Werror -Wall -std=c99 " +if ENV.has_key?('DEBUG') $CFLAGS << "-O0 -g" else $CFLAGS << "-O3" diff --git a/ext/semian/semian.c b/ext/semian/semian.c index 821872a4..f38637fb 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -447,6 +447,9 @@ semian_resource_id(VALUE self) return LONG2FIX(res->sem_id); } + +void Init_semian_cb_data(); + void Init_semian() { VALUE cSemian, cResource; @@ -504,4 +507,6 @@ void Init_semian() /* Maximum number of tickets available on this system. */ rb_define_const(cSemian, "MAX_TICKETS", INT2FIX(system_max_semaphore_count)); + + Init_semian_cb_data(); } diff --git a/ext/semian_cb_data/semian_cb_data.c b/ext/semian/semian_sliding_window.c similarity index 77% rename from ext/semian_cb_data/semian_cb_data.c rename to ext/semian/semian_sliding_window.c index 81159e67..a7528401 100644 --- a/ext/semian_cb_data/semian_cb_data.c +++ b/ext/semian/semian_sliding_window.c @@ -12,6 +12,8 @@ #include #include +#include + // needed for semctl union semun { int val; /* Value for SETVAL */ @@ -38,22 +40,21 @@ typedef VALUE (*my_blocking_fn_t)(void*); // }; typedef struct { - int successes; - int arr_length; - double errors[]; + int counter; + int window_length; + long window[]; } shared_cb_data; typedef struct { //semaphore, shared memory data and pointer key_t key; - size_t arr_max_size; + size_t max_window_length; bool lock_triggered; int semid; int shmid; shared_cb_data *shm_address; } semian_cb_data; -static int system_max_semaphore_count; static const int kCBSemaphoreCount = 1; // # semaphores to be acquired static const int kCBTicketMax = 1; static const int kCBInitializeWaitTimeout = 5; /* seconds */ @@ -70,7 +71,7 @@ static void semian_cb_data_free(void *ptr); static size_t semian_cb_data_memsize(const void *ptr); static VALUE semian_cb_data_alloc(VALUE klass); static VALUE semian_cb_data_init(VALUE self, VALUE name, VALUE size, VALUE permissions); -static VALUE semian_cb_data_clean(VALUE self); +static VALUE semian_cb_data_destroy(VALUE self); static void set_semaphore_permissions(int sem_id, int permissions); static void configure_tickets(int sem_id, int tickets, int should_initialize); static int create_semaphore(int key, int permissions, int *created); @@ -83,18 +84,19 @@ static void *semian_cb_data_unlock_without_gvl(void *self); static VALUE semian_cb_data_acquire_memory(VALUE self, VALUE permissions); static void semian_cb_data_delete_memory_inner (semian_cb_data *ptr); static VALUE semian_cb_data_delete_memory (VALUE self); -static VALUE semian_cb_data_get_successes(VALUE self); -static VALUE semian_cb_data_set_successes(VALUE self, VALUE num); +static VALUE semian_cb_data_get_counter(VALUE self); +static VALUE semian_cb_data_set_counter(VALUE self, VALUE num); static VALUE semian_cb_data_semid(VALUE self); static VALUE semian_cb_data_shmid(VALUE self); -static VALUE semian_cb_data_array_at_index(VALUE self, VALUE idx); -static VALUE semian_cb_data_array_set_index(VALUE self, VALUE idx, VALUE val); static VALUE semian_cb_data_array_length(VALUE self); static VALUE semian_cb_data_set_push_back(VALUE self, VALUE num); static VALUE semian_cb_data_set_pop_back(VALUE self); static VALUE semian_cb_data_set_push_front(VALUE self, VALUE num); static VALUE semian_cb_data_set_pop_front(VALUE self); +static VALUE semian_cb_data_is_shared(VALUE self); + + // needed for TypedData_Make_Struct && TypedData_Get_Struct static const rb_data_type_t semian_cb_data_type = { @@ -189,12 +191,12 @@ semian_cb_data_init(VALUE self, VALUE id, VALUE size, VALUE permissions) if (TYPE(id) != T_SYMBOL && TYPE(id) != T_STRING) rb_raise(rb_eTypeError, "id must be a symbol or string"); if (TYPE(size) != T_FIXNUM /*|| TYPE(size) != T_BIGNUM*/) - rb_raise(rb_eTypeError, "expected integer for arr_max_size"); + rb_raise(rb_eTypeError, "expected integer for max_window_length"); if (TYPE(permissions) != T_FIXNUM) rb_raise(rb_eTypeError, "expected integer for permissions"); if (NUM2SIZET(size) <= 0) - rb_raise(rb_eArgError, "arr_max_size must be larger than 0"); + rb_raise(rb_eArgError, "max_window_length must be larger than 0"); const char *id_str = NULL; if (TYPE(id) == T_SYMBOL) { @@ -205,8 +207,8 @@ semian_cb_data_init(VALUE self, VALUE id, VALUE size, VALUE permissions) ptr->key = generate_key(id_str); //rb_warn("converted name %s to key %d", id_str, ptr->key); - // Guarantee arr_max_size >=1 or error thrown - ptr->arr_max_size = NUM2SIZET(size); + // Guarantee max_window_length >=1 or error thrown + ptr->max_window_length = NUM2SIZET(size); // id's default to -1 ptr->semid = -1; @@ -222,7 +224,7 @@ semian_cb_data_init(VALUE self, VALUE id, VALUE size, VALUE permissions) } static VALUE -semian_cb_data_clean(VALUE self) +semian_cb_data_destroy(VALUE self) { semian_cb_data_delete_memory(self); semian_cb_data_delete_semaphore(self); @@ -260,8 +262,6 @@ configure_tickets(int sem_id, int tickets, int should_initialize) if (-1 == semctl(sem_id, 0, SETVAL, kCBTicketMax)) { rb_warn("semctl: failed to set semaphore with semid %d, position 0 to %d", sem_id, 1); raise_semian_syscall_error("semctl()", errno); - } else { - //rb_warn("semctl: set semaphore with semid %d, position 0 to %d", sem_id, 1); } } else if (tickets > 0) { // it's possible that we haven't actually initialized the @@ -348,14 +348,16 @@ semian_cb_data_delete_semaphore(VALUE self) semian_cb_data *ptr; TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); if (-1 == ptr->semid) // do nothing if semaphore not acquired - return self; + return Qfalse; if (-1 == semctl(ptr->semid, 0, IPC_RMID)) { if (EIDRM == errno) { rb_warn("semctl: failed to delete semaphore set with semid %d: already removed", ptr->semid); + raise_semian_syscall_error("semctl()", errno); ptr->semid = -1; } else { rb_warn("semctl: failed to remove semaphore with semid %d, errno %d (%s)",ptr->semid, errno, strerror(errno)); + raise_semian_syscall_error("semctl()", errno); } } else { //rb_warn("semctl: semaphore set with semid %d deleted", ptr->semid); @@ -394,7 +396,7 @@ semian_cb_data_lock_without_gvl(void *self) ts.tv_sec = kCBInternalTimeout; if (-1 == semtimedop(ptr->semid,&decrement,1, &ts)) { - rb_raise(eInternal, "error with semop locking, %d: (%s)", errno, strerror(errno)); + rb_raise(eInternal, "error acquiring semaphore lock to mutate circuit breaker structure, %d: (%s)", errno, strerror(errno)); retval=Qfalse; } else retval=Qtrue; @@ -427,7 +429,7 @@ semian_cb_data_unlock_without_gvl(void *self) ts.tv_sec = kCBInternalTimeout; if (-1 == semtimedop(ptr->semid,&increment,1 , &ts)) { - rb_raise(eInternal, "error with semop unlocking, errno: %d (%s)", errno, strerror(errno)); + rb_raise(eInternal, "error unlocking semaphore, %d (%s)", errno, strerror(errno)); retval=Qfalse; } else retval=Qtrue; @@ -458,38 +460,94 @@ semian_cb_data_acquire_memory(VALUE self, VALUE permissions) rb_raise(rb_eTypeError, "expected integer for permissions"); if (!semian_cb_data_lock(self)) - return self; + return Qfalse; + int created = 0; key_t key = ptr->key; - if (-1 == (ptr->shmid = shmget( key, - 2*sizeof(int) + ptr->arr_max_size * sizeof(double), - IPC_CREAT | IPC_EXCL | FIX2LONG(permissions)))) { - if (errno == EEXIST) - ptr->shmid = shmget(key, ptr->arr_max_size, IPC_CREAT); - } + int byte_size = 2*sizeof(int) + ptr->max_window_length * sizeof(long); + int flags = IPC_CREAT | IPC_EXCL | FIX2LONG(permissions); + + if (-1 == (ptr->shmid = shmget( key, byte_size, flags))) { + if (errno == EEXIST) { + ptr->shmid = shmget(key, byte_size, flags & ~IPC_EXCL); + } + } else + created = 1; + + struct shared_cb_data *old_data=NULL; + int old_size = 0; + if (-1 == ptr->shmid) { - rb_raise(eSyscall, "shmget() failed to acquire a memory shmid with key %d, size %zu, errno %d (%s)", key, ptr->arr_max_size, errno, strerror(errno)); - } else { - //rb_warn("shmget: successfully got memory id with key %d, shmid %d, size %zu", key, ptr->shmid, ptr->arr_max_size); + if (errno == EINVAL) { + // EINVAL is either because + // 1. segment with key exists but size given >= size of segment + // 2. segment was requested to be created, but (size > SHMMAX || size < SHMMIN) + + // Unlikely for 2 to occur, but we check by requesting a memory of size 1 byte + // We handle 1 by marking the old memory and shmid as IPC_RMID, not writing to it again, + // copying as much data over to the new memory + + // Changing memory size requires restarting semian with new args for initialization + + int shmid = shmget(key, 1, flags & ~IPC_EXCL); + if (-1 != shmid) { + struct shared_cb_data *data = shmat(shmid, (void *)0, 0); + if ((void *)-1 != data) { + struct shmid_ds shm_info; + if (-1 != shmctl(shmid, IPC_STAT, &shm_info)) { + old_size = shm_info.shm_segsz; + if (byte_size != old_size) { + old_data = malloc(shm_info.shm_segsz); + memcpy(old_data,data,fmin(old_size, byte_size)); + ptr->shmid = shmid; + ptr->shm_address = (shared_cb_data *)data; + semian_cb_data_delete_memory_inner(ptr); + } + + // Flagging for deletion sets a shm's associated key to be 0 so shmget gets a different shmid. + if (-1 != (ptr->shmid = shmget(key, byte_size, flags))) { + created = 1; + } + } + } + } + } + + if (-1 == ptr->shmid && errno == EINVAL) { + if (old_data) + free(old_data); + semian_cb_data_unlock(self); + rb_raise(eSyscall, "shmget() failed to acquire a memory shmid with key %d, size %zu, errno %d (%s)", key, ptr->max_window_length, errno, strerror(errno)); + } } if (0 == ptr->shm_address) { ptr->shm_address = shmat(ptr->shmid, (void *)0, 0); if (((void*)-1) == ptr->shm_address) { - rb_raise(eSyscall, "shmat() failed to attach memory with shmid %d, size %zu, errno %d (%s)", ptr->shmid, ptr->arr_max_size, errno, strerror(errno)); + semian_cb_data_unlock(self); ptr->shm_address = 0; + if (old_data) + free(old_data); + rb_raise(eSyscall, "shmat() failed to attach memory with shmid %d, size %zu, errno %d (%s)", ptr->shmid, ptr->max_window_length, errno, strerror(errno)); } else { - //rb_warn("shmat: successfully attached shmid %d to %p", ptr->shmid, ptr->shm_address); - shared_cb_data *data = ptr->shm_address; - data->successes = 0; - data->arr_length = 0; - int i=0; - for (; i< data->arr_length; ++i) - data->errors[i]=0; - + if (created) { + if (old_data) { + // transfer data over + memcpy(ptr->shm_address,old_data,fmin(old_size, byte_size)); + + ptr->shm_address->window_length = fmin(ptr->max_window_length-1, ptr->shm_address->window_length); + } else { + shared_cb_data *data = ptr->shm_address; + data->counter = 0; + data->window_length = 0; + for (int i=0; i< data->window_length; ++i) + data->window[i]=0; + } + } } } - + if (old_data) + free(old_data); semian_cb_data_unlock(self); return self; } @@ -501,7 +559,6 @@ semian_cb_data_delete_memory_inner (semian_cb_data *ptr) if (-1 == shmdt(ptr->shm_address)) { rb_raise(eSyscall,"shmdt: no attached memory at %p, errno %d (%s)", ptr->shm_address, errno, strerror(errno)); } else { - rb_warn("shmdt: successfully detached memory at %p", ptr->shm_address); } ptr->shm_address = 0; } @@ -512,7 +569,7 @@ semian_cb_data_delete_memory_inner (semian_cb_data *ptr) if (errno == EINVAL) { ptr->shmid = -1; } { - rb_raise(eSyscall,"shmctl: error removing memory with shmid %d, errno %d (%s)", ptr->shmid, errno, strerror(errno)); + rb_raise(eSyscall,"shmctl: error flagging memory for removal with shmid %d, errno %d (%s)", ptr->shmid, errno, strerror(errno)); } } else { ptr->shmid = -1; @@ -541,12 +598,12 @@ semian_cb_data_delete_memory (VALUE self) /* - * Below are methods for successes, semid, shmid, and array pop, push, peek at front and back + * Below are methods for counter, semid, shmid, and array pop, push, peek at front and back * and clear, length */ static VALUE -semian_cb_data_get_successes(VALUE self) +semian_cb_data_get_counter(VALUE self) { semian_cb_data *ptr; TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); @@ -555,15 +612,17 @@ semian_cb_data_get_successes(VALUE self) if (0 == ptr->shm_address) return Qnil; + if (!semian_cb_data_lock(self)) + return Qnil; - int successes = ptr->shm_address->successes; + int counter = ptr->shm_address->counter; semian_cb_data_unlock(self); - return INT2NUM(successes); + return INT2NUM(counter); } static VALUE -semian_cb_data_set_successes(VALUE self, VALUE num) +semian_cb_data_set_counter(VALUE self, VALUE num) { semian_cb_data *ptr; TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); @@ -571,13 +630,13 @@ semian_cb_data_set_successes(VALUE self, VALUE num) if (0 == ptr->shm_address) return Qnil; - if (!semian_cb_data_lock(self)) + if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT) return Qnil; - if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT/*|| TYPE(size) != T_BIGNUM*/) + if (!semian_cb_data_lock(self)) return Qnil; - ptr->shm_address->successes = NUM2INT(num); + ptr->shm_address->counter = NUM2INT(num); semian_cb_data_unlock(self); return num; @@ -599,68 +658,6 @@ semian_cb_data_shmid(VALUE self) return INT2NUM(ptr->shmid); } -static VALUE -semian_cb_data_array_at_index(VALUE self, VALUE idx) -{ - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); - - if (0 == ptr->shm_address) - return Qnil; - - if (TYPE(idx) != T_FIXNUM && TYPE(idx) != T_FLOAT/*|| TYPE(size) != T_BIGNUM*/) - return Qnil; - - int index = NUM2INT(idx); - - if (index <0 || index >= ptr->arr_max_size) { - return Qnil; - } - - if (!semian_cb_data_lock(self)) - return Qnil; - shared_cb_data *data = ptr->shm_address; - VALUE retval = index < (data->arr_length) ? DBL2NUM(data->errors[index]) : Qnil; - - semian_cb_data_unlock(self); - return retval; - -} - -static VALUE -semian_cb_data_array_set_index(VALUE self, VALUE idx, VALUE val) -{ - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); - - // check shared memory for NULL - if (0 == ptr->shm_address) - return Qnil; - - if (TYPE(idx) != T_FIXNUM && TYPE(idx) != T_FLOAT/*|| TYPE(size) != T_BIGNUM*/) - return Qnil; - if (TYPE(val) != T_FIXNUM && TYPE(val) != T_FLOAT/*|| TYPE(size) != T_BIGNUM*/) - return Qnil; - - int index = NUM2INT(idx); - double value = NUM2DBL(val); - - if (index <0 || index >= ptr->arr_max_size) { - return Qnil; - } - - if (!semian_cb_data_lock(self)){ - return Qnil; - } - - ptr->shm_address->errors[index] = value; - ptr->shm_address->arr_length = index+1; - - semian_cb_data_unlock(self); - return val; - -} - static VALUE semian_cb_data_array_length(VALUE self) { @@ -670,9 +667,9 @@ semian_cb_data_array_length(VALUE self) return Qnil; if (!semian_cb_data_lock(self)) return Qnil; - int arr_length =ptr->shm_address->arr_length; + int window_length =ptr->shm_address->window_length; semian_cb_data_unlock(self); - return INT2NUM(arr_length); + return INT2NUM(window_length); } static VALUE @@ -682,22 +679,21 @@ semian_cb_data_set_push_back(VALUE self, VALUE num) TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); if (0 == ptr->shm_address) return Qnil; - if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT/*|| TYPE(size) != T_BIGNUM*/) + if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT && TYPE(num) != T_BIGNUM) return Qnil; if (!semian_cb_data_lock(self)) return Qnil; shared_cb_data *data = ptr->shm_address; - if (data->arr_length == ptr->arr_max_size) { - int i; - for (i=1; i< ptr->arr_max_size; ++i){ - data->errors[i-1] = data->errors[i]; + if (data->window_length == ptr->max_window_length) { + for (int i=1; i< ptr->max_window_length; ++i){ + data->window[i-1] = data->window[i]; } - --(data->arr_length); + --(data->window_length); } - data->errors[(data->arr_length)] = NUM2DBL(num); - ++(data->arr_length); + data->window[(data->window_length)] = NUM2LONG(num); + ++(data->window_length); semian_cb_data_unlock(self); return self; } @@ -715,11 +711,11 @@ semian_cb_data_set_pop_back(VALUE self) VALUE retval; shared_cb_data *data = ptr->shm_address; - if (0 == data->arr_length) + if (0 == data->window_length) retval = Qnil; else { - retval = DBL2NUM(data->errors[data->arr_length-1]); - --(data->arr_length); + retval = LONG2NUM(data->window[data->window_length-1]); + --(data->window_length); } semian_cb_data_unlock(self); @@ -739,14 +735,13 @@ semian_cb_data_set_pop_front(VALUE self) VALUE retval; shared_cb_data *data = ptr->shm_address; - if (0 >= data->arr_length) + if (0 >= data->window_length) retval = Qnil; else { - retval = DBL2NUM(data->errors[0]); - int i=0; - for (; iarr_length-1; ++i) - data->errors[i]=data->errors[i+1]; - --(data->arr_length); + retval = LONG2NUM(data->window[0]); + for (int i=0; iwindow_length-1; ++i) + data->window[i]=data->window[i+1]; + --(data->window_length); } semian_cb_data_unlock(self); @@ -760,23 +755,23 @@ semian_cb_data_set_push_front(VALUE self, VALUE num) TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); if (0 == ptr->shm_address) return Qnil; - if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT/*|| TYPE(size) != T_BIGNUM*/) + if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT && TYPE(num) != T_BIGNUM) return Qnil; if (!semian_cb_data_lock(self)) return Qnil; - double val = NUM2DBL(num); + long val = NUM2LONG(num); shared_cb_data *data = ptr->shm_address; - int i=data->arr_length; + int i=data->window_length; for (; i>0; --i) - data->errors[i]=data->errors[i-1]; + data->window[i]=data->window[i-1]; - data->errors[0] = val; - ++(data->arr_length); - if (data->arr_length>ptr->arr_max_size) - data->arr_length=ptr->arr_max_size; + data->window[0] = val; + ++(data->window_length); + if (data->window_length>ptr->max_window_length) + data->window_length=ptr->max_window_length; semian_cb_data_unlock(self); return self; @@ -792,7 +787,7 @@ semian_cb_data_array_clear(VALUE self) if (!semian_cb_data_lock(self)) return Qnil; - ptr->shm_address->arr_length=0; + ptr->shm_address->window_length=0; semian_cb_data_unlock(self); return self; @@ -810,8 +805,8 @@ semian_cb_data_array_first(VALUE self) return Qnil; VALUE retval; - if (ptr->shm_address->arr_length >=1 && 1 <= ptr->arr_max_size) - retval = DBL2NUM(ptr->shm_address->errors[0]); + if (ptr->shm_address->window_length >=1) + retval = LONG2NUM(ptr->shm_address->window[0]); else retval = Qnil; @@ -831,8 +826,8 @@ semian_cb_data_array_last(VALUE self) return Qnil; VALUE retval; - if (ptr->shm_address->arr_length > 0) - retval = DBL2NUM(ptr->shm_address->errors[ptr->shm_address->arr_length-1]); + if (ptr->shm_address->window_length > 0) + retval = LONG2NUM(ptr->shm_address->window[ptr->shm_address->window_length-1]); else retval = Qnil; @@ -840,16 +835,22 @@ semian_cb_data_array_last(VALUE self) return retval; } +static VALUE +semian_cb_data_is_shared(VALUE self) +{ + return Qtrue; +} + void Init_semian_cb_data (void) { VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); - VALUE cCircuitBreakerSharedData = rb_const_get(cSemianModule, rb_intern("CircuitBreakerSharedData")); + VALUE cCircuitBreakerSharedData = rb_const_get(cSemianModule, rb_intern("SlidingWindow")); rb_define_alloc_func(cCircuitBreakerSharedData, semian_cb_data_alloc); rb_define_method(cCircuitBreakerSharedData, "_initialize", semian_cb_data_init, 3); - rb_define_method(cCircuitBreakerSharedData, "_cleanup", semian_cb_data_clean, 0); + rb_define_method(cCircuitBreakerSharedData, "_destroy", semian_cb_data_destroy, 0); //rb_define_method(cCircuitBreakerSharedData, "acquire_semaphore", semian_cb_data_acquire_semaphore, 1); //rb_define_method(cCircuitBreakerSharedData, "delete_semaphore", semian_cb_data_delete_semaphore, 0); //rb_define_method(cCircuitBreakerSharedData, "lock", semian_cb_data_lock, 0); @@ -859,12 +860,9 @@ Init_semian_cb_data (void) { rb_define_method(cCircuitBreakerSharedData, "semid", semian_cb_data_semid, 0); rb_define_method(cCircuitBreakerSharedData, "shmid", semian_cb_data_shmid, 0); - rb_define_method(cCircuitBreakerSharedData, "successes", semian_cb_data_get_successes, 0); - rb_define_method(cCircuitBreakerSharedData, "successes=", semian_cb_data_set_successes, 1); + rb_define_method(cCircuitBreakerSharedData, "successes", semian_cb_data_get_counter, 0); + rb_define_method(cCircuitBreakerSharedData, "successes=", semian_cb_data_set_counter, 1); - rb_define_method(cCircuitBreakerSharedData, "[]", semian_cb_data_array_at_index, 1); - rb_define_method(cCircuitBreakerSharedData, "[]=", semian_cb_data_array_set_index, 2); - rb_define_method(cCircuitBreakerSharedData, "length", semian_cb_data_array_length, 0); rb_define_method(cCircuitBreakerSharedData, "size", semian_cb_data_array_length, 0); rb_define_method(cCircuitBreakerSharedData, "count", semian_cb_data_array_length, 0); rb_define_method(cCircuitBreakerSharedData, "<<", semian_cb_data_set_push_back, 1); @@ -876,6 +874,8 @@ Init_semian_cb_data (void) { rb_define_method(cCircuitBreakerSharedData, "first", semian_cb_data_array_first, 0); rb_define_method(cCircuitBreakerSharedData, "last", semian_cb_data_array_last, 0); + rb_define_singleton_method(cCircuitBreakerSharedData, "shared?", semian_cb_data_is_shared, 0); + eInternal = rb_const_get(cSemianModule, rb_intern("InternalError")); eSyscall = rb_const_get(cSemianModule, rb_intern("SyscallError")); eTimeout = rb_const_get(cSemianModule, rb_intern("TimeoutError")); @@ -888,13 +888,6 @@ Init_semian_cb_data (void) { increment.sem_op = 1; increment.sem_flg = SEM_UNDO; - struct seminfo info_buf; - - if (semctl(0, 0, SEM_INFO, &info_buf) == -1) { - rb_raise(eInternal, "unable to determine maximum semaphore count - semctl() returned %d: %s ", errno, strerror(errno)); - } - system_max_semaphore_count = info_buf.semvmx; - /* Maximum number of tickets available on this system. */ rb_const_get(cSemianModule, rb_intern("MAX_TICKETS")); } diff --git a/ext/semian_cb_data/extconf.rb b/ext/semian_cb_data/extconf.rb deleted file mode 100644 index 9ae0aa2e..00000000 --- a/ext/semian_cb_data/extconf.rb +++ /dev/null @@ -1,33 +0,0 @@ -$:.unshift File.expand_path("../../../lib", __FILE__) - -require 'semian/platform' - -unless Semian.sysv_semaphores_supported? - File.write "Makefile", <= @success_count_threshold @shared_circuit_breaker_data.successes >= @success_count_threshold end def error_threshold_reached? - #@errors.count == @error_count_threshold - @shared_circuit_breaker_data.count == @error_count_threshold + @shared_circuit_breaker_data.size == @error_count_threshold end def error_timeout_expired? - #@errors.last && (@errors.last + @error_timeout < Time.now) - time_f = @shared_circuit_breaker_data.last - time_f && (Time.at(time_f) + @error_timeout < Time.now) + time_ms = @shared_circuit_breaker_data.last + time_ms && (Time.at(time_ms/1000) + @error_timeout < Time.now) end def push_time(window, max_size, duration:, time: Time.now) - #window.shift while window.first && window.first + duration < time - window.shift while window.first && Time.at(window.first) + duration < time + window.shift while window.first && Time.at(window.first/1000) + duration < time window.shift if window.size == max_size - #window << time - window << time.to_f + window << (time.to_f*1000).to_i end def log_state_transition(new_state) return if @state.nil? || new_state == @state str = "[#{self.class.name}] State transition from #{@state} to #{new_state}." - str << " success_count=#{@shared_circuit_breaker_data.successes} error_count=#{@shared_circuit_breaker_data.count}" + str << " success_count=#{@shared_circuit_breaker_data.successes} error_count=#{@shared_circuit_breaker_data.size}" str << " success_count_threshold=#{@success_count_threshold} error_count_threshold=#{@error_count_threshold}" str << " error_timeout=#{@error_timeout} error_last_at=\"#{@error_last_at}\"" Semian.logger.info(str) diff --git a/lib/semian/circuit_breaker_shared_data.rb b/lib/semian/circuit_breaker_shared_data.rb deleted file mode 100644 index e04e0c10..00000000 --- a/lib/semian/circuit_breaker_shared_data.rb +++ /dev/null @@ -1,70 +0,0 @@ -module Semian - class CircuitBreakerSharedData #:nodoc: - def initialize(name, arr_max_size, permissions) - _initialize(name, arr_max_size, permissions) if respond_to?(:_initialize) - end - - # For anyone consulting this, the array stores floats. Use Time.at(_float_here_) to convert to time - - def successes - 0 - end - - def successes=(num) - 0 - end - - def semid - 0 - end - - def shmid - 0 - end - - def length - 0 - end - - def size - 0 - end - - def count - 0 - end - - def << (float) - nil - end - - def push(float) - nil - end - - def pop - nil - end - - def shift - nil - end - - def unshift (float) - nil - end - - def clear - nil - end - - def first - nil - end - - def last - nil - end - - end -end diff --git a/lib/semian/sliding_window.rb b/lib/semian/sliding_window.rb new file mode 100644 index 00000000..2541fc7d --- /dev/null +++ b/lib/semian/sliding_window.rb @@ -0,0 +1,74 @@ +module Semian + class SlidingWindow #:nodoc: + def initialize(name, max_window_length, permissions) + if respond_to?(:_initialize) + _initialize(name, max_window_length, permissions) + else + @successes = 0 + @max_window_length = max_window_length + @window = [] + end + end + + # For anyone consulting this, the array stores an integer amount of seconds since epoch + # Use Time.at(_time_ms_ / 1000) to convert to time + def self.shared? + false + end + + def successes + @successes + end + + def successes=(num) + @successes=num + end + + def semid + 0 + end + + def shmid + 0 + end + + def size + @window.size + end + + def << (time_ms) + push(time_ms) + end + + def push(time_ms) + @window.shift while @window.size >= @max_window_length + @window << time_ms + end + + def pop + @window.pop + end + + def shift + @window.shift + end + + def unshift (time_ms) + @window.pop while @window.size >= @max_window_length + @window.unshift(time_ms) + end + + def clear + @window = []; + end + + def first + @window.first + end + + def last + @window.last + end + + end +end diff --git a/semian.gemspec b/semian.gemspec index d72768e8..1a46c2c8 100644 --- a/semian.gemspec +++ b/semian.gemspec @@ -17,7 +17,7 @@ Gem::Specification.new do |s| s.license = 'MIT' s.files = `git ls-files`.split("\n") - s.extensions = ['ext/semian/extconf.rb','ext/semian_cb_data/extconf.rb'] + s.extensions = ['ext/semian/extconf.rb'] s.add_development_dependency 'rake-compiler', '~> 0.9' s.add_development_dependency 'timecop' s.add_development_dependency 'mysql2' From cad849712c1e9702af00ba8599427af9cc083a30 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Thu, 1 Oct 2015 15:02:46 -0400 Subject: [PATCH 03/23] Refactor naming, some shared dependencies --- ext/semian/semian.c | 43 +- ext/semian/semian.h | 47 ++ ext/semian/semian_sliding_window.c | 734 ++++++++++++++--------------- lib/semian/sliding_window.rb | 24 +- 4 files changed, 422 insertions(+), 426 deletions(-) create mode 100644 ext/semian/semian.h diff --git a/ext/semian/semian.c b/ext/semian/semian.c index f38637fb..a25a5438 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -1,35 +1,4 @@ -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include - -union semun { - int val; /* Value for SETVAL */ - struct semid_ds *buf; /* Buffer for IPC_STAT, IPC_SET */ - unsigned short *array; /* Array for GETALL, SETALL */ - struct seminfo *__buf; /* Buffer for IPC_INFO - (Linux-specific) */ -}; - -#if defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL) && defined(HAVE_RUBY_THREAD_H) -// 2.0 -#include -#define WITHOUT_GVL(fn,a,ubf,b) rb_thread_call_without_gvl((fn),(a),(ubf),(b)) -#elif defined(HAVE_RB_THREAD_BLOCKING_REGION) - // 1.9 -typedef VALUE (*my_blocking_fn_t)(void*); -#define WITHOUT_GVL(fn,a,ubf,b) rb_thread_blocking_region((my_blocking_fn_t)(fn),(a),(ubf),(b)) -#endif +#include "semian.h" static ID id_timeout; static VALUE eSyscall, eTimeout, eInternal; @@ -48,7 +17,7 @@ typedef struct { char *name; } semian_resource_t; -static key_t +key_t generate_key(const char *name) { union { @@ -67,7 +36,7 @@ ms_to_timespec(long ms, struct timespec *ts) ts->tv_nsec = (ms % 1000) * 1000000; } -static void +void raise_semian_syscall_error(const char *syscall, int error_num) { rb_raise(eSyscall, "%s failed, errno: %d (%s)", syscall, error_num, strerror(error_num)); @@ -115,7 +84,7 @@ semian_resource_alloc(VALUE klass) return obj; } -static void +void set_semaphore_permissions(int sem_id, int permissions) { union semun sem_opts; @@ -448,7 +417,7 @@ semian_resource_id(VALUE self) } -void Init_semian_cb_data(); +void Init_semian_sliding_window(); void Init_semian() { @@ -508,5 +477,5 @@ void Init_semian() /* Maximum number of tickets available on this system. */ rb_define_const(cSemian, "MAX_TICKETS", INT2FIX(system_max_semaphore_count)); - Init_semian_cb_data(); + Init_semian_sliding_window(); } diff --git a/ext/semian/semian.h b/ext/semian/semian.h new file mode 100644 index 00000000..2bad8b2a --- /dev/null +++ b/ext/semian/semian.h @@ -0,0 +1,47 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include + +#include +#include + +#include + +union semun { + int val; /* Value for SETVAL */ + struct semid_ds *buf; /* Buffer for IPC_STAT, IPC_SET */ + unsigned short *array; /* Array for GETALL, SETALL */ + struct seminfo *__buf; /* Buffer for IPC_INFO + (Linux-specific) */ +}; + +#if defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL) && defined(HAVE_RUBY_THREAD_H) +// 2.0 +#include +#define WITHOUT_GVL(fn,a,ubf,b) rb_thread_call_without_gvl((fn),(a),(ubf),(b)) +#elif defined(HAVE_RB_THREAD_BLOCKING_REGION) + // 1.9 +typedef VALUE (*my_blocking_fn_t)(void*); +#define WITHOUT_GVL(fn,a,ubf,b) rb_thread_blocking_region((my_blocking_fn_t)(fn),(a),(ubf),(b)) +#endif + +key_t +generate_key(const char *name); + +void +raise_semian_syscall_error(const char *syscall, int error_num); + + +void +set_semaphore_permissions(int sem_id, int permissions); diff --git a/ext/semian/semian_sliding_window.c b/ext/semian/semian_sliding_window.c index a7528401..2e3e0231 100644 --- a/ext/semian/semian_sliding_window.c +++ b/ext/semian/semian_sliding_window.c @@ -1,37 +1,4 @@ -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -// needed for semctl -union semun { - int val; /* Value for SETVAL */ - struct semid_ds *buf; /* Buffer for IPC_STAT, IPC_SET */ - unsigned short *array; /* Array for GETALL, SETALL */ - struct seminfo *__buf; /* Buffer for IPC_INFO - (Linux-specific) */ -}; - -#if defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL) && defined(HAVE_RUBY_THREAD_H) -// 2.0 -#include -#define WITHOUT_GVL(fn,a,ubf,b) rb_thread_call_without_gvl((fn),(a),(ubf),(b)) -#elif defined(HAVE_RB_THREAD_BLOCKING_REGION) - // 1.9 -typedef VALUE (*my_blocking_fn_t)(void*); -#define WITHOUT_GVL(fn,a,ubf,b) rb_thread_blocking_region((my_blocking_fn_t)(fn),(a),(ubf),(b)) -#endif +#include "semian.h" // struct sembuf { // found in sys/sem.h // unsigned short sem_num; /* semaphore number */ @@ -43,17 +10,18 @@ typedef struct { int counter; int window_length; long window[]; -} shared_cb_data; +} shared_window_data; typedef struct { //semaphore, shared memory data and pointer key_t key; - size_t max_window_length; - bool lock_triggered; + size_t max_window_size; + int lock_triggered; + int permissions; int semid; int shmid; - shared_cb_data *shm_address; -} semian_cb_data; + shared_window_data *shm_address; +} semian_window_data; static const int kCBSemaphoreCount = 1; // # semaphores to be acquired static const int kCBTicketMax = 1; @@ -66,86 +34,61 @@ static struct sembuf increment; // = { kCBIndexTicketLock, 1, SEM_UNDO}; static VALUE eInternal, eSyscall, eTimeout; // Semian errors -static void semian_cb_data_mark(void *ptr); -static void semian_cb_data_free(void *ptr); -static size_t semian_cb_data_memsize(const void *ptr); -static VALUE semian_cb_data_alloc(VALUE klass); -static VALUE semian_cb_data_init(VALUE self, VALUE name, VALUE size, VALUE permissions); -static VALUE semian_cb_data_destroy(VALUE self); -static void set_semaphore_permissions(int sem_id, int permissions); -static void configure_tickets(int sem_id, int tickets, int should_initialize); -static int create_semaphore(int key, int permissions, int *created); -static VALUE semian_cb_data_acquire_semaphore (VALUE self, VALUE permissions); -static VALUE semian_cb_data_delete_semaphore(VALUE self); -static VALUE semian_cb_data_lock(VALUE self); -static VALUE semian_cb_data_unlock(VALUE self); -static void *semian_cb_data_lock_without_gvl(void *self); -static void *semian_cb_data_unlock_without_gvl(void *self); -static VALUE semian_cb_data_acquire_memory(VALUE self, VALUE permissions); -static void semian_cb_data_delete_memory_inner (semian_cb_data *ptr); -static VALUE semian_cb_data_delete_memory (VALUE self); -static VALUE semian_cb_data_get_counter(VALUE self); -static VALUE semian_cb_data_set_counter(VALUE self, VALUE num); -static VALUE semian_cb_data_semid(VALUE self); -static VALUE semian_cb_data_shmid(VALUE self); -static VALUE semian_cb_data_array_length(VALUE self); -static VALUE semian_cb_data_set_push_back(VALUE self, VALUE num); -static VALUE semian_cb_data_set_pop_back(VALUE self); -static VALUE semian_cb_data_set_push_front(VALUE self, VALUE num); -static VALUE semian_cb_data_set_pop_front(VALUE self); - -static VALUE semian_cb_data_is_shared(VALUE self); +static void semian_window_data_mark(void *ptr); +static void semian_window_data_free(void *ptr); +static size_t semian_window_data_memsize(const void *ptr); +static VALUE semian_window_data_alloc(VALUE klass); +static VALUE semian_window_data_init(VALUE self, VALUE name, VALUE size, VALUE permissions); +static VALUE semian_window_data_destroy(VALUE self); +static int create_semaphore_and_initialize(int key, int permissions); +static VALUE semian_window_data_acquire_semaphore (VALUE self, VALUE permissions); +static VALUE semian_window_data_delete_semaphore(VALUE self); +static VALUE semian_window_data_lock(VALUE self); +static VALUE semian_window_data_unlock(VALUE self); +static void *semian_window_data_lock_without_gvl(void *self); +static void *semian_window_data_unlock_without_gvl(void *self); +static VALUE semian_window_data_acquire_memory(VALUE self, VALUE permissions, VALUE should_keep_max_window_size); +static void semian_window_data_delete_memory_inner (semian_window_data *ptr, int should_unlock, VALUE self, void *should_free); +static VALUE semian_window_data_delete_memory (VALUE self); +static void semian_window_data_check_and_resize_if_needed (VALUE self); +static VALUE semian_window_data_get_counter(VALUE self); +static VALUE semian_window_data_set_counter(VALUE self, VALUE num); +static VALUE semian_window_data_semid(VALUE self); +static VALUE semian_window_data_shmid(VALUE self); +static VALUE semian_window_data_array_length(VALUE self); +static VALUE semian_window_data_set_push_back(VALUE self, VALUE num); +static VALUE semian_window_data_set_pop_back(VALUE self); +static VALUE semian_window_data_set_push_front(VALUE self, VALUE num); +static VALUE semian_window_data_set_pop_front(VALUE self); + +static VALUE semian_window_data_is_shared(VALUE self); // needed for TypedData_Make_Struct && TypedData_Get_Struct static const rb_data_type_t -semian_cb_data_type = { - "semian_cb_data", +semian_window_data_type = { + "semian_window_data", { - semian_cb_data_mark, - semian_cb_data_free, - semian_cb_data_memsize + semian_window_data_mark, + semian_window_data_free, + semian_window_data_memsize }, NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY }; -/* - * Generate key - */ -static key_t -generate_key(const char *name) -{ - union { - unsigned char str[SHA_DIGEST_LENGTH]; - key_t key; - } digest; - SHA1((const unsigned char *) name, strlen(name), digest.str); - /* TODO: compile-time assertion that sizeof(key_t) > SHA_DIGEST_LENGTH */ - return digest.key; -} - -/* - * Log errors - */ -static void -raise_semian_syscall_error(const char *syscall, int error_num) -{ - rb_raise(eSyscall, "%s failed, errno %d (%s)", syscall, error_num, strerror(error_num)); -} - /* * Functions that handle type and memory */ static void -semian_cb_data_mark(void *ptr) +semian_window_data_mark(void *ptr) { /* noop */ } static void -semian_cb_data_free(void *ptr) +semian_window_data_free(void *ptr) { - semian_cb_data *data = (semian_cb_data *) ptr; + semian_window_data *data = (semian_window_data *) ptr; // Under normal circumstances, memory use should be in the order of bytes, @@ -153,24 +96,24 @@ semian_cb_data_free(void *ptr) // so there is no need to call this unless certain all other semian processes are stopped // (also raises concurrency errors: "object allocation during garbage collection phase") - //semian_cb_data_delete_memory_inner (data); + //semian_window_data_delete_memory_inner (data); xfree(data); } static size_t -semian_cb_data_memsize(const void *ptr) +semian_window_data_memsize(const void *ptr) { - return sizeof(semian_cb_data); + return sizeof(semian_window_data); } static VALUE -semian_cb_data_alloc(VALUE klass) +semian_window_data_alloc(VALUE klass) { VALUE obj; - semian_cb_data *ptr; + semian_window_data *ptr; - obj = TypedData_Make_Struct(klass, semian_cb_data, &semian_cb_data_type, ptr); + obj = TypedData_Make_Struct(klass, semian_window_data, &semian_window_data_type, ptr); return obj; } @@ -183,20 +126,20 @@ semian_cb_data_alloc(VALUE klass) * Init function exposed as ._initialize() that is delegated by .initialize() */ static VALUE -semian_cb_data_init(VALUE self, VALUE id, VALUE size, VALUE permissions) +semian_window_data_init(VALUE self, VALUE id, VALUE size, VALUE permissions) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (TYPE(id) != T_SYMBOL && TYPE(id) != T_STRING) rb_raise(rb_eTypeError, "id must be a symbol or string"); - if (TYPE(size) != T_FIXNUM /*|| TYPE(size) != T_BIGNUM*/) - rb_raise(rb_eTypeError, "expected integer for max_window_length"); + if (TYPE(size) != T_FIXNUM) + rb_raise(rb_eTypeError, "expected integer for max_window_size"); if (TYPE(permissions) != T_FIXNUM) rb_raise(rb_eTypeError, "expected integer for permissions"); if (NUM2SIZET(size) <= 0) - rb_raise(rb_eArgError, "max_window_length must be larger than 0"); + rb_raise(rb_eArgError, "max_window_size must be larger than 0"); const char *id_str = NULL; if (TYPE(id) == T_SYMBOL) { @@ -207,135 +150,78 @@ semian_cb_data_init(VALUE self, VALUE id, VALUE size, VALUE permissions) ptr->key = generate_key(id_str); //rb_warn("converted name %s to key %d", id_str, ptr->key); - // Guarantee max_window_length >=1 or error thrown - ptr->max_window_length = NUM2SIZET(size); + // Guarantee max_window_size >=1 or error thrown + ptr->max_window_size = NUM2SIZET(size); // id's default to -1 ptr->semid = -1; ptr->shmid = -1; // addresses default to NULL ptr->shm_address = 0; - ptr->lock_triggered = false; + ptr->lock_triggered = 0; + ptr->permissions = FIX2LONG(permissions); - semian_cb_data_acquire_semaphore(self, permissions); - semian_cb_data_acquire_memory(self, permissions); + semian_window_data_acquire_semaphore(self, permissions); + semian_window_data_acquire_memory(self, permissions, Qtrue); return self; } static VALUE -semian_cb_data_destroy(VALUE self) +semian_window_data_destroy(VALUE self) { - semian_cb_data_delete_memory(self); - semian_cb_data_delete_semaphore(self); + semian_window_data_delete_memory(self); + semian_window_data_delete_semaphore(self); return self; } - -/* - * Functions set_semaphore_permissions, configure_tickets, create_semaphore - * are taken from semian.c with extra code removed - */ -static void -set_semaphore_permissions(int sem_id, int permissions) -{ - union semun sem_opts; - struct semid_ds stat_buf; - - sem_opts.buf = &stat_buf; - semctl(sem_id, 0, IPC_STAT, sem_opts); - if ((stat_buf.sem_perm.mode & 0xfff) != permissions) { - stat_buf.sem_perm.mode &= ~0xfff; - stat_buf.sem_perm.mode |= permissions; - semctl(sem_id, 0, IPC_SET, sem_opts); - } -} - - -static void -configure_tickets(int sem_id, int tickets, int should_initialize) -{ - struct timeval start_time, cur_time; - - if (should_initialize) { - if (-1 == semctl(sem_id, 0, SETVAL, kCBTicketMax)) { - rb_warn("semctl: failed to set semaphore with semid %d, position 0 to %d", sem_id, 1); - raise_semian_syscall_error("semctl()", errno); - } - } else if (tickets > 0) { - // it's possible that we haven't actually initialized the - // semaphore structure yet - wait a bit in that case - int ret; - if (0 == (ret = semctl(sem_id, 0, GETVAL))) { - gettimeofday(&start_time, NULL); - while (0 == (ret = semctl(sem_id, 0, GETVAL))) { // loop while value == 0 - usleep(10000); /* 10ms */ - gettimeofday(&cur_time, NULL); - if ((cur_time.tv_sec - start_time.tv_sec) > kCBInitializeWaitTimeout) { - rb_raise(eInternal, "timeout waiting for semaphore initialization"); - } - } - if (-1 == ret) { - rb_raise(eInternal, "error getting max ticket count, errno: %d (%s)", errno, strerror(errno)); - } - } - - // Rest of the function (originally from semian.c) was removed since it isn't needed - } -} - static int -create_semaphore(int key, int permissions, int *created) +create_semaphore_and_initialize(int key, int permissions) { int semid = 0; int flags = 0; - *created = 0; flags = IPC_EXCL | IPC_CREAT | permissions; semid = semget(key, kCBSemaphoreCount, flags); if (semid >= 0) { - *created = 1; - //rb_warn("semget: received %d semaphore(s) with key %d, semid %d", kCBSemaphoreCount, key, semid); + if (-1 == semctl(semid, 0, SETVAL, kCBTicketMax)) { + rb_warn("semctl: failed to set semaphore with semid %d, position 0 to %d", semid, 1); + raise_semian_syscall_error("semctl()", errno); + } } else if (semid == -1 && errno == EEXIST) { flags &= ~IPC_EXCL; semid = semget(key, kCBSemaphoreCount, flags); - //rb_warn("semget: retrieved existing semaphore with key %d, semid %d", key, semid); } return semid; } - - /* * Create or acquire previously made semaphore */ static VALUE -semian_cb_data_acquire_semaphore (VALUE self, VALUE permissions) +semian_window_data_acquire_semaphore (VALUE self, VALUE permissions) { // Function flow, semaphore creation methods are // borrowed from semian.c since they have been previously tested - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (TYPE(permissions) != T_FIXNUM) rb_raise(rb_eTypeError, "expected integer for permissions"); // bool for initializing (configure_tickets) or not - int created = 0; key_t key = ptr->key; - int semid = create_semaphore(key, FIX2LONG(permissions), &created); + int semid = create_semaphore_and_initialize(key, FIX2LONG(permissions)); if (-1 == semid) { raise_semian_syscall_error("semget()", errno); } ptr->semid = semid; - // initialize to 1 and set permissions - configure_tickets(ptr->semid, kCBTicketMax, created); set_semaphore_permissions(ptr->semid, FIX2LONG(permissions)); return self; @@ -343,10 +229,10 @@ semian_cb_data_acquire_semaphore (VALUE self, VALUE permissions) static VALUE -semian_cb_data_delete_semaphore(VALUE self) +semian_window_data_delete_semaphore(VALUE self) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (-1 == ptr->semid) // do nothing if semaphore not acquired return Qfalse; @@ -370,20 +256,20 @@ semian_cb_data_delete_semaphore(VALUE self) /* - * semian_cb_data_lock/unlock and associated functions decrement/increment semaphore + * semian_window_data_lock/unlock and associated functions decrement/increment semaphore */ static VALUE -semian_cb_data_lock(VALUE self) +semian_window_data_lock(VALUE self) { - return (VALUE) WITHOUT_GVL(semian_cb_data_lock_without_gvl, (void *)self, RUBY_UBF_IO, NULL); + return (VALUE) WITHOUT_GVL(semian_window_data_lock_without_gvl, (void *)self, RUBY_UBF_IO, NULL); } static void * -semian_cb_data_lock_without_gvl(void *self) +semian_window_data_lock_without_gvl(void *self) { - semian_cb_data *ptr; - TypedData_Get_Struct((VALUE)self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct((VALUE)self, semian_window_data, &semian_window_data_type, ptr); if (ptr->lock_triggered) return (void *)Qtrue; if (-1 == ptr->semid){ @@ -401,22 +287,21 @@ semian_cb_data_lock_without_gvl(void *self) } else retval=Qtrue; - ptr->lock_triggered = true; - //rb_warn("semop: lock success"); + ptr->lock_triggered = 1; return (void *)retval; } static VALUE -semian_cb_data_unlock(VALUE self) +semian_window_data_unlock(VALUE self) { - return (VALUE) WITHOUT_GVL(semian_cb_data_unlock_without_gvl, (void *)self, RUBY_UBF_IO, NULL); + return (VALUE) WITHOUT_GVL(semian_window_data_unlock_without_gvl, (void *)self, RUBY_UBF_IO, NULL); } static void * -semian_cb_data_unlock_without_gvl(void *self) +semian_window_data_unlock_without_gvl(void *self) { - semian_cb_data *ptr; - TypedData_Get_Struct((VALUE)self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct((VALUE)self, semian_window_data, &semian_window_data_type, ptr); if (!(ptr->lock_triggered)) return (void *)Qtrue; if (-1 == ptr->semid){ @@ -425,32 +310,36 @@ semian_cb_data_unlock_without_gvl(void *self) } VALUE retval; - struct timespec ts = { 0 }; - ts.tv_sec = kCBInternalTimeout; - - if (-1 == semtimedop(ptr->semid,&increment,1 , &ts)) { + if (-1 == semop(ptr->semid,&increment,1)) { rb_raise(eInternal, "error unlocking semaphore, %d (%s)", errno, strerror(errno)); retval=Qfalse; } else retval=Qtrue; - ptr->lock_triggered = false; + ptr->lock_triggered = 0; //rb_warn("semop unlock success"); return (void *)retval; } - - /* Acquire memory by getting shmid, and then attaching it to a memory location, requires semaphore for locking/unlocking to be setup + + Note: should_keep_max_window_size is a bool that decides how ptr->max_window_size + is handled. There may be a discrepancy between the requested memory size and the actual + size of the memory block given. If it is false (0), ptr->max_window_size will be modified + to the actual memory size if there is a difference. This matters when dynamicaly resizing + memory. + Think of should_keep_max_window_size as this worker requesting a size, others resizing, + and !should_keep_max_window_size as another worker requesting a size and this worker + resizing */ static VALUE -semian_cb_data_acquire_memory(VALUE self, VALUE permissions) +semian_window_data_acquire_memory(VALUE self, VALUE permissions, VALUE should_keep_max_window_size) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (-1 == ptr->semid){ rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); @@ -458,86 +347,124 @@ semian_cb_data_acquire_memory(VALUE self, VALUE permissions) } if (TYPE(permissions) != T_FIXNUM) rb_raise(rb_eTypeError, "expected integer for permissions"); + if (TYPE(should_keep_max_window_size) != T_TRUE && + TYPE(should_keep_max_window_size) != T_FALSE) + rb_raise(rb_eTypeError, "expected true or false for should_keep_max_window_size"); + + int should_keep_max_window_size_bool = RTEST(should_keep_max_window_size); - if (!semian_cb_data_lock(self)) + if (!semian_window_data_lock(self)) return Qfalse; + // Below will handle acquiring/creating new memory, and possibly resizing the + // memory or the ptr->max_window_size depending on + // should_keep_max_window_size_bool + int created = 0; key_t key = ptr->key; - int byte_size = 2*sizeof(int) + ptr->max_window_length * sizeof(long); + int requested_byte_size = 2*sizeof(int) + ptr->max_window_size * sizeof(long); int flags = IPC_CREAT | IPC_EXCL | FIX2LONG(permissions); - if (-1 == (ptr->shmid = shmget( key, byte_size, flags))) { + int actual_byte_size = 0; + + // We fill both actual_byte_size and requested_byte_size + // Logic matrix: + // actual=>req | actual=req | actualshmid = shmget( key, requested_byte_size, flags))) { if (errno == EEXIST) { - ptr->shmid = shmget(key, byte_size, flags & ~IPC_EXCL); + ptr->shmid = shmget(key, requested_byte_size, flags & ~IPC_EXCL); // will succeed + if (-1 != ptr->shmid) { + struct shmid_ds shm_info; + if (-1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)){ + actual_byte_size = shm_info.shm_segsz; + } + } else { + failed = 1; + } } - } else + // Else, this could be any numberof errors + // 1. segment with key exists but requested size > current mem size + // 2. segment was requested to be created, but (size > SHMMAX || size < SHMMIN) + // 3. Other error + + // We can only see SHMMAX and SHMMIN through console commands + // Unlikely for 2 to occur, so we check by requesting a memory of size 1 byte + if (-1 != (ptr->shmid = shmget(key, 1, flags & ~IPC_EXCL))) { + struct shmid_ds shm_info; + if (-1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)){ + actual_byte_size = shm_info.shm_segsz; + failed = 0; + } + } else { // Error, exit + failed=2; + } + + } else { created = 1; + actual_byte_size = requested_byte_size; + } - struct shared_cb_data *old_data=NULL; - int old_size = 0; - - if (-1 == ptr->shmid) { - if (errno == EINVAL) { - // EINVAL is either because - // 1. segment with key exists but size given >= size of segment - // 2. segment was requested to be created, but (size > SHMMAX || size < SHMMIN) - - // Unlikely for 2 to occur, but we check by requesting a memory of size 1 byte - // We handle 1 by marking the old memory and shmid as IPC_RMID, not writing to it again, - // copying as much data over to the new memory - - // Changing memory size requires restarting semian with new args for initialization - - int shmid = shmget(key, 1, flags & ~IPC_EXCL); - if (-1 != shmid) { - struct shared_cb_data *data = shmat(shmid, (void *)0, 0); - if ((void *)-1 != data) { - struct shmid_ds shm_info; - if (-1 != shmctl(shmid, IPC_STAT, &shm_info)) { - old_size = shm_info.shm_segsz; - if (byte_size != old_size) { - old_data = malloc(shm_info.shm_segsz); - memcpy(old_data,data,fmin(old_size, byte_size)); - ptr->shmid = shmid; - ptr->shm_address = (shared_cb_data *)data; - semian_cb_data_delete_memory_inner(ptr); - } - - // Flagging for deletion sets a shm's associated key to be 0 so shmget gets a different shmid. - if (-1 != (ptr->shmid = shmget(key, byte_size, flags))) { - created = 1; - } - } + shared_window_data *data_copy=NULL; + if (should_keep_max_window_size_bool && !failed) { // memory resizing may occur + // We flag old mem by IPC_RMID, copy data, and fix it values if it needs fixing + if (actual_byte_size != requested_byte_size) { + shared_window_data *data; + + if ((void *)-1 != (data = shmat(ptr->shmid, (void *)0, 0))) { + data_copy = malloc(actual_byte_size); + memcpy(data_copy,data,actual_byte_size); + ptr->shm_address = data; + semian_window_data_delete_memory_inner(ptr, 1, self, data_copy); + + // Flagging for deletion sets a shm's associated key to be 0 so shmget gets a different shmid. + // If this worker is creating the new memory + if (-1 != (ptr->shmid = shmget(key, requested_byte_size, flags))) { + created = 1; + } else { // failed to get new memory, exit + failed=5; } + } else { // failed to attach, exit + rb_raise(eInternal,"Failed to copy old data, key %d, shmid %d, errno %d (%s)",key, ptr->shmid, errno, strerror(errno)); + failed=6; } } + } else if (!failed){ // ptr->max_window_size may be changed + ptr->max_window_size = (actual_byte_size - 2*sizeof(int))/(sizeof(long)); + } - if (-1 == ptr->shmid && errno == EINVAL) { - if (old_data) - free(old_data); - semian_cb_data_unlock(self); - rb_raise(eSyscall, "shmget() failed to acquire a memory shmid with key %d, size %zu, errno %d (%s)", key, ptr->max_window_length, errno, strerror(errno)); - } + if (failed) { + if (data_copy) + free(data_copy); + semian_window_data_unlock(self); + rb_raise(eSyscall, "shmget() failed to acquire a memory shmid with key %d, size %zu, errno %d (%s)", key, ptr->max_window_size, errno, strerror(errno)); } if (0 == ptr->shm_address) { ptr->shm_address = shmat(ptr->shmid, (void *)0, 0); if (((void*)-1) == ptr->shm_address) { - semian_cb_data_unlock(self); + semian_window_data_unlock(self); ptr->shm_address = 0; - if (old_data) - free(old_data); - rb_raise(eSyscall, "shmat() failed to attach memory with shmid %d, size %zu, errno %d (%s)", ptr->shmid, ptr->max_window_length, errno, strerror(errno)); + if (data_copy) + free(data_copy); + rb_raise(eSyscall, "shmat() failed to attach memory with shmid %d, size %zu, errno %d (%s)", ptr->shmid, ptr->max_window_size, errno, strerror(errno)); } else { if (created) { - if (old_data) { + if (data_copy) { // transfer data over - memcpy(ptr->shm_address,old_data,fmin(old_size, byte_size)); + ptr->shm_address->counter = data_copy->counter; + ptr->shm_address->window_length = fmin(ptr->max_window_size-1, data_copy->window_length); - ptr->shm_address->window_length = fmin(ptr->max_window_length-1, ptr->shm_address->window_length); + // Copy the most recent ptr->shm_address->window_length numbers to new memory + memcpy(&(ptr->shm_address->window), + ((long *)(&(data_copy->window[0])))+data_copy->window_length-ptr->shm_address->window_length, + ptr->shm_address->window_length * sizeof(long)); } else { - shared_cb_data *data = ptr->shm_address; + shared_window_data *data = ptr->shm_address; data->counter = 0; data->window_length = 0; for (int i=0; i< data->window_length; ++i) @@ -546,17 +473,25 @@ semian_cb_data_acquire_memory(VALUE self, VALUE permissions) } } } - if (old_data) - free(old_data); - semian_cb_data_unlock(self); + if (data_copy) + free(data_copy); + semian_window_data_unlock(self); return self; } static void -semian_cb_data_delete_memory_inner (semian_cb_data *ptr) +semian_window_data_delete_memory_inner (semian_window_data *ptr, int should_unlock, VALUE self, void *should_free) { + // This internal function may be called from a variety of contexts + // Sometimes it is under a semaphore lock, sometimes it has extra malloc ptrs + // Arguments handle these conditions + if (0 != ptr->shm_address){ if (-1 == shmdt(ptr->shm_address)) { + if (should_unlock) + semian_window_data_unlock(self); + if (should_free) + free(should_free); rb_raise(eSyscall,"shmdt: no attached memory at %p, errno %d (%s)", ptr->shm_address, errno, strerror(errno)); } else { } @@ -564,11 +499,15 @@ semian_cb_data_delete_memory_inner (semian_cb_data *ptr) } if (-1 != ptr->shmid) { - // Once IPC_RMID is set, no new attaches can be made + // Once IPC_RMID is set, no new shmgets can be made with key, and current values are invalid if (-1 == shmctl(ptr->shmid, IPC_RMID, 0)) { if (errno == EINVAL) { ptr->shmid = -1; - } { + } else { + if (should_unlock) + semian_window_data_unlock(self); + if (should_free) + free(should_free); rb_raise(eSyscall,"shmctl: error flagging memory for removal with shmid %d, errno %d (%s)", ptr->shmid, errno, strerror(errno)); } } else { @@ -579,22 +518,41 @@ semian_cb_data_delete_memory_inner (semian_cb_data *ptr) static VALUE -semian_cb_data_delete_memory (VALUE self) +semian_window_data_delete_memory (VALUE self) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - if (!semian_cb_data_lock(self)) + if (!semian_window_data_lock(self)) return self; - semian_cb_data_delete_memory_inner(ptr); + semian_window_data_delete_memory_inner(ptr, 1, self, NULL); - semian_cb_data_unlock(self); + semian_window_data_unlock(self); return self; } +static void //bool +semian_window_data_check_and_resize_if_needed(VALUE self) { + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); + + if (!semian_window_data_lock(self)) + return; + struct shmid_ds shm_info; + int needs_resize = 0; + if (-1 != ptr->shmid && -1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)) { + needs_resize = shm_info.shm_perm.mode & SHM_DEST; + } + + if (needs_resize) { + semian_window_data_delete_memory_inner(ptr, 1, self, NULL); + semian_window_data_unlock(self); + semian_window_data_acquire_memory(self, LONG2FIX(ptr->permissions), Qfalse); + } +} /* @@ -603,114 +561,127 @@ semian_cb_data_delete_memory (VALUE self) */ static VALUE -semian_cb_data_get_counter(VALUE self) +semian_window_data_get_counter(VALUE self) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); // check shared memory for NULL if (0 == ptr->shm_address) return Qnil; - - if (!semian_cb_data_lock(self)) + semian_window_data_check_and_resize_if_needed(self); + if (!semian_window_data_lock(self)) return Qnil; int counter = ptr->shm_address->counter; - semian_cb_data_unlock(self); + semian_window_data_unlock(self); return INT2NUM(counter); } static VALUE -semian_cb_data_set_counter(VALUE self, VALUE num) +semian_window_data_set_counter(VALUE self, VALUE num) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (0 == ptr->shm_address) return Qnil; if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT) return Qnil; - - if (!semian_cb_data_lock(self)) + semian_window_data_check_and_resize_if_needed(self); + if (!semian_window_data_lock(self)) return Qnil; ptr->shm_address->counter = NUM2INT(num); - semian_cb_data_unlock(self); + semian_window_data_unlock(self); return num; } static VALUE -semian_cb_data_semid(VALUE self) +semian_window_data_semid(VALUE self) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); + semian_window_data_check_and_resize_if_needed(self); return INT2NUM(ptr->semid); } static VALUE -semian_cb_data_shmid(VALUE self) +semian_window_data_shmid(VALUE self) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); + semian_window_data_check_and_resize_if_needed(self); return INT2NUM(ptr->shmid); } static VALUE -semian_cb_data_array_length(VALUE self) +semian_window_data_max_window_size(VALUE self) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); + semian_window_data_check_and_resize_if_needed(self); + int max_window_size =ptr->max_window_size; + return INT2NUM(max_window_size); +} + +static VALUE +semian_window_data_array_length(VALUE self) +{ + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (0 == ptr->shm_address) return Qnil; - if (!semian_cb_data_lock(self)) + semian_window_data_check_and_resize_if_needed(self); + if (!semian_window_data_lock(self)) return Qnil; int window_length =ptr->shm_address->window_length; - semian_cb_data_unlock(self); + semian_window_data_unlock(self); return INT2NUM(window_length); } static VALUE -semian_cb_data_set_push_back(VALUE self, VALUE num) +semian_window_data_set_push_back(VALUE self, VALUE num) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (0 == ptr->shm_address) return Qnil; if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT && TYPE(num) != T_BIGNUM) return Qnil; - - if (!semian_cb_data_lock(self)) + semian_window_data_check_and_resize_if_needed(self); + if (!semian_window_data_lock(self)) return Qnil; - shared_cb_data *data = ptr->shm_address; - if (data->window_length == ptr->max_window_length) { - for (int i=1; i< ptr->max_window_length; ++i){ + shared_window_data *data = ptr->shm_address; + if (data->window_length == ptr->max_window_size) { + for (int i=1; i< ptr->max_window_size; ++i){ data->window[i-1] = data->window[i]; } --(data->window_length); } data->window[(data->window_length)] = NUM2LONG(num); ++(data->window_length); - semian_cb_data_unlock(self); + semian_window_data_unlock(self); return self; } static VALUE -semian_cb_data_set_pop_back(VALUE self) +semian_window_data_set_pop_back(VALUE self) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (0 == ptr->shm_address) return Qnil; - - if (!semian_cb_data_lock(self)) + semian_window_data_check_and_resize_if_needed(self); + if (!semian_window_data_lock(self)) return Qnil; VALUE retval; - shared_cb_data *data = ptr->shm_address; + shared_window_data *data = ptr->shm_address; if (0 == data->window_length) retval = Qnil; else { @@ -718,23 +689,23 @@ semian_cb_data_set_pop_back(VALUE self) --(data->window_length); } - semian_cb_data_unlock(self); + semian_window_data_unlock(self); return retval; } static VALUE -semian_cb_data_set_pop_front(VALUE self) +semian_window_data_set_pop_front(VALUE self) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (0 == ptr->shm_address) return Qnil; - - if (!semian_cb_data_lock(self)) + semian_window_data_check_and_resize_if_needed(self); + if (!semian_window_data_lock(self)) return Qnil; VALUE retval; - shared_cb_data *data = ptr->shm_address; + shared_window_data *data = ptr->shm_address; if (0 >= data->window_length) retval = Qnil; else { @@ -744,25 +715,25 @@ semian_cb_data_set_pop_front(VALUE self) --(data->window_length); } - semian_cb_data_unlock(self); + semian_window_data_unlock(self); return retval; } static VALUE -semian_cb_data_set_push_front(VALUE self, VALUE num) +semian_window_data_set_push_front(VALUE self, VALUE num) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (0 == ptr->shm_address) return Qnil; if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT && TYPE(num) != T_BIGNUM) return Qnil; - - if (!semian_cb_data_lock(self)) + semian_window_data_check_and_resize_if_needed(self); + if (!semian_window_data_lock(self)) return Qnil; long val = NUM2LONG(num); - shared_cb_data *data = ptr->shm_address; + shared_window_data *data = ptr->shm_address; int i=data->window_length; for (; i>0; --i) @@ -770,38 +741,38 @@ semian_cb_data_set_push_front(VALUE self, VALUE num) data->window[0] = val; ++(data->window_length); - if (data->window_length>ptr->max_window_length) - data->window_length=ptr->max_window_length; + if (data->window_length>ptr->max_window_size) + data->window_length=ptr->max_window_size; - semian_cb_data_unlock(self); + semian_window_data_unlock(self); return self; } static VALUE -semian_cb_data_array_clear(VALUE self) +semian_window_data_array_clear(VALUE self) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (0 == ptr->shm_address) return Qnil; - - if (!semian_cb_data_lock(self)) + semian_window_data_check_and_resize_if_needed(self); + if (!semian_window_data_lock(self)) return Qnil; ptr->shm_address->window_length=0; - semian_cb_data_unlock(self); + semian_window_data_unlock(self); return self; } static VALUE -semian_cb_data_array_first(VALUE self) +semian_window_data_array_first(VALUE self) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (0 == ptr->shm_address) return Qnil; - - if (!semian_cb_data_lock(self)) + semian_window_data_check_and_resize_if_needed(self); + if (!semian_window_data_lock(self)) return Qnil; VALUE retval; @@ -810,19 +781,19 @@ semian_cb_data_array_first(VALUE self) else retval = Qnil; - semian_cb_data_unlock(self); + semian_window_data_unlock(self); return retval; } static VALUE -semian_cb_data_array_last(VALUE self) +semian_window_data_array_last(VALUE self) { - semian_cb_data *ptr; - TypedData_Get_Struct(self, semian_cb_data, &semian_cb_data_type, ptr); + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); if (0 == ptr->shm_address) return Qnil; - - if (!semian_cb_data_lock(self)) + semian_window_data_check_and_resize_if_needed(self); + if (!semian_window_data_lock(self)) return Qnil; VALUE retval; @@ -831,50 +802,51 @@ semian_cb_data_array_last(VALUE self) else retval = Qnil; - semian_cb_data_unlock(self); + semian_window_data_unlock(self); return retval; } static VALUE -semian_cb_data_is_shared(VALUE self) +semian_window_data_is_shared(VALUE self) { return Qtrue; } void -Init_semian_cb_data (void) { +Init_semian_sliding_window (void) { VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); VALUE cCircuitBreakerSharedData = rb_const_get(cSemianModule, rb_intern("SlidingWindow")); - rb_define_alloc_func(cCircuitBreakerSharedData, semian_cb_data_alloc); - rb_define_method(cCircuitBreakerSharedData, "_initialize", semian_cb_data_init, 3); - rb_define_method(cCircuitBreakerSharedData, "_destroy", semian_cb_data_destroy, 0); - //rb_define_method(cCircuitBreakerSharedData, "acquire_semaphore", semian_cb_data_acquire_semaphore, 1); - //rb_define_method(cCircuitBreakerSharedData, "delete_semaphore", semian_cb_data_delete_semaphore, 0); - //rb_define_method(cCircuitBreakerSharedData, "lock", semian_cb_data_lock, 0); - //rb_define_method(cCircuitBreakerSharedData, "unlock", semian_cb_data_unlock, 0); - //rb_define_method(cCircuitBreakerSharedData, "acquire_memory", semian_cb_data_acquire_memory, 1); - //rb_define_method(cCircuitBreakerSharedData, "delete_memory", semian_cb_data_delete_memory, 0); - - rb_define_method(cCircuitBreakerSharedData, "semid", semian_cb_data_semid, 0); - rb_define_method(cCircuitBreakerSharedData, "shmid", semian_cb_data_shmid, 0); - rb_define_method(cCircuitBreakerSharedData, "successes", semian_cb_data_get_counter, 0); - rb_define_method(cCircuitBreakerSharedData, "successes=", semian_cb_data_set_counter, 1); - - rb_define_method(cCircuitBreakerSharedData, "size", semian_cb_data_array_length, 0); - rb_define_method(cCircuitBreakerSharedData, "count", semian_cb_data_array_length, 0); - rb_define_method(cCircuitBreakerSharedData, "<<", semian_cb_data_set_push_back, 1); - rb_define_method(cCircuitBreakerSharedData, "push", semian_cb_data_set_push_back, 1); - rb_define_method(cCircuitBreakerSharedData, "pop", semian_cb_data_set_pop_back, 0); - rb_define_method(cCircuitBreakerSharedData, "shift", semian_cb_data_set_pop_front, 0); - rb_define_method(cCircuitBreakerSharedData, "unshift", semian_cb_data_set_push_front, 1); - rb_define_method(cCircuitBreakerSharedData, "clear", semian_cb_data_array_clear, 0); - rb_define_method(cCircuitBreakerSharedData, "first", semian_cb_data_array_first, 0); - rb_define_method(cCircuitBreakerSharedData, "last", semian_cb_data_array_last, 0); - - rb_define_singleton_method(cCircuitBreakerSharedData, "shared?", semian_cb_data_is_shared, 0); + rb_define_alloc_func(cCircuitBreakerSharedData, semian_window_data_alloc); + rb_define_method(cCircuitBreakerSharedData, "_initialize", semian_window_data_init, 3); + rb_define_method(cCircuitBreakerSharedData, "_destroy", semian_window_data_destroy, 0); + //rb_define_method(cCircuitBreakerSharedData, "acquire_semaphore", semian_window_data_acquire_semaphore, 1); + //rb_define_method(cCircuitBreakerSharedData, "delete_semaphore", semian_window_data_delete_semaphore, 0); + //rb_define_method(cCircuitBreakerSharedData, "lock", semian_window_data_lock, 0); + //rb_define_method(cCircuitBreakerSharedData, "unlock", semian_window_data_unlock, 0); + rb_define_method(cCircuitBreakerSharedData, "acquire_memory", semian_window_data_acquire_memory, 2); + rb_define_method(cCircuitBreakerSharedData, "delete_memory", semian_window_data_delete_memory, 0); + rb_define_method(cCircuitBreakerSharedData, "max_window_size", semian_window_data_max_window_size, 0); + + rb_define_method(cCircuitBreakerSharedData, "semid", semian_window_data_semid, 0); + rb_define_method(cCircuitBreakerSharedData, "shmid", semian_window_data_shmid, 0); + rb_define_method(cCircuitBreakerSharedData, "successes", semian_window_data_get_counter, 0); + rb_define_method(cCircuitBreakerSharedData, "successes=", semian_window_data_set_counter, 1); + + rb_define_method(cCircuitBreakerSharedData, "size", semian_window_data_array_length, 0); + rb_define_method(cCircuitBreakerSharedData, "count", semian_window_data_array_length, 0); + rb_define_method(cCircuitBreakerSharedData, "<<", semian_window_data_set_push_back, 1); + rb_define_method(cCircuitBreakerSharedData, "push", semian_window_data_set_push_back, 1); + rb_define_method(cCircuitBreakerSharedData, "pop", semian_window_data_set_pop_back, 0); + rb_define_method(cCircuitBreakerSharedData, "shift", semian_window_data_set_pop_front, 0); + rb_define_method(cCircuitBreakerSharedData, "unshift", semian_window_data_set_push_front, 1); + rb_define_method(cCircuitBreakerSharedData, "clear", semian_window_data_array_clear, 0); + rb_define_method(cCircuitBreakerSharedData, "first", semian_window_data_array_first, 0); + rb_define_method(cCircuitBreakerSharedData, "last", semian_window_data_array_last, 0); + + rb_define_singleton_method(cCircuitBreakerSharedData, "shared?", semian_window_data_is_shared, 0); eInternal = rb_const_get(cSemianModule, rb_intern("InternalError")); eSyscall = rb_const_get(cSemianModule, rb_intern("SyscallError")); diff --git a/lib/semian/sliding_window.rb b/lib/semian/sliding_window.rb index 2541fc7d..b23ea337 100644 --- a/lib/semian/sliding_window.rb +++ b/lib/semian/sliding_window.rb @@ -1,21 +1,30 @@ module Semian class SlidingWindow #:nodoc: - def initialize(name, max_window_length, permissions) + def initialize(name, max_window_size, permissions) if respond_to?(:_initialize) - _initialize(name, max_window_length, permissions) + _initialize(name, max_window_size, permissions) else @successes = 0 - @max_window_length = max_window_length + @max_window_size = max_window_size @window = [] end end # For anyone consulting this, the array stores an integer amount of seconds since epoch # Use Time.at(_time_ms_ / 1000) to convert to time + + # A sliding window is a structure that keeps at most @max_window_size recent timestamps + # in store, like this: if @max_window_size = 4, current time is 10, @window =[5,7,9,10]. + # Another push of (11) at 11 sec would make @window [7,9,10,11], popping off 5. + def self.shared? false end + def max_window_size + @max_window_size + end + def successes @successes end @@ -25,11 +34,11 @@ def successes=(num) end def semid - 0 + -1 end def shmid - 0 + -1 end def size @@ -41,7 +50,7 @@ def << (time_ms) end def push(time_ms) - @window.shift while @window.size >= @max_window_length + @window.shift while @window.size >= @max_window_size @window << time_ms end @@ -54,7 +63,7 @@ def shift end def unshift (time_ms) - @window.pop while @window.size >= @max_window_length + @window.pop while @window.size >= @max_window_size @window.unshift(time_ms) end @@ -69,6 +78,5 @@ def first def last @window.last end - end end From 9faca9a818fb7f842e7826a78c42be361e5c7d45 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Fri, 2 Oct 2015 11:20:05 -0400 Subject: [PATCH 04/23] Refactor: split acquire_memory up into separate functions --- ext/semian/semian_sliding_window.c | 153 +++++++++++++++++------------ 1 file changed, 89 insertions(+), 64 deletions(-) diff --git a/ext/semian/semian_sliding_window.c b/ext/semian/semian_sliding_window.c index 2e3e0231..47434936 100644 --- a/ext/semian/semian_sliding_window.c +++ b/ext/semian/semian_sliding_window.c @@ -322,48 +322,19 @@ semian_window_data_unlock_without_gvl(void *self) } -/* - Acquire memory by getting shmid, and then attaching it to a memory location, - requires semaphore for locking/unlocking to be setup - - Note: should_keep_max_window_size is a bool that decides how ptr->max_window_size - is handled. There may be a discrepancy between the requested memory size and the actual - size of the memory block given. If it is false (0), ptr->max_window_size will be modified - to the actual memory size if there is a difference. This matters when dynamicaly resizing - memory. - Think of should_keep_max_window_size as this worker requesting a size, others resizing, - and !should_keep_max_window_size as another worker requesting a size and this worker - resizing -*/ -static VALUE -semian_window_data_acquire_memory(VALUE self, VALUE permissions, VALUE should_keep_max_window_size) -{ - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - - if (-1 == ptr->semid){ - rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); - return self; - } - if (TYPE(permissions) != T_FIXNUM) - rb_raise(rb_eTypeError, "expected integer for permissions"); - if (TYPE(should_keep_max_window_size) != T_TRUE && - TYPE(should_keep_max_window_size) != T_FALSE) - rb_raise(rb_eTypeError, "expected true or false for should_keep_max_window_size"); - - int should_keep_max_window_size_bool = RTEST(should_keep_max_window_size); - - if (!semian_window_data_lock(self)) - return Qfalse; +static int +create_and_resize_memory(key_t key, int max_window_size, long permissions, int should_keep_max_window_size_bool, + int *created, shared_window_data **data_copy, semian_window_data *ptr, VALUE self) { // Below will handle acquiring/creating new memory, and possibly resizing the // memory or the ptr->max_window_size depending on // should_keep_max_window_size_bool - int created = 0; - key_t key = ptr->key; - int requested_byte_size = 2*sizeof(int) + ptr->max_window_size * sizeof(long); - int flags = IPC_CREAT | IPC_EXCL | FIX2LONG(permissions); + int shmid=-1; + int failed=0; + + int requested_byte_size = 2*sizeof(int) + max_window_size * sizeof(long); + int flags = IPC_CREAT | IPC_EXCL | permissions; int actual_byte_size = 0; @@ -374,75 +345,127 @@ semian_window_data_acquire_memory(VALUE self, VALUE permissions, VALUE should_ke // should_keep_max_window_size_bool | no err, shrink mem | no err | error, expand mem // !should_keep_max_window_size_bool | no err, ++ max size | no err | error, -- max size - int failed = 0; - if (-1 == (ptr->shmid = shmget( key, requested_byte_size, flags))) { + if (-1 == (shmid = shmget( key, requested_byte_size, flags))) { if (errno == EEXIST) { - ptr->shmid = shmget(key, requested_byte_size, flags & ~IPC_EXCL); // will succeed - if (-1 != ptr->shmid) { + if (-1 != (shmid = shmget(key, requested_byte_size, flags & ~IPC_EXCL))) { struct shmid_ds shm_info; - if (-1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)){ + if (-1 != shmctl(shmid, IPC_STAT, &shm_info)){ actual_byte_size = shm_info.shm_segsz; - } + } else + failed = -1; } else { - failed = 1; + failed = -2; } } - // Else, this could be any numberof errors + // Else, this could be any number of errors // 1. segment with key exists but requested size > current mem size // 2. segment was requested to be created, but (size > SHMMAX || size < SHMMIN) // 3. Other error // We can only see SHMMAX and SHMMIN through console commands // Unlikely for 2 to occur, so we check by requesting a memory of size 1 byte - if (-1 != (ptr->shmid = shmget(key, 1, flags & ~IPC_EXCL))) { + if (-1 != (shmid = shmget(key, 1, flags & ~IPC_EXCL))) { struct shmid_ds shm_info; - if (-1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)){ + if (-1 != shmctl(shmid, IPC_STAT, &shm_info)){ actual_byte_size = shm_info.shm_segsz; failed = 0; - } + } else + failed=-3; } else { // Error, exit - failed=2; + failed=-4; } - } else { - created = 1; + *created = 1; actual_byte_size = requested_byte_size; } - shared_window_data *data_copy=NULL; if (should_keep_max_window_size_bool && !failed) { // memory resizing may occur // We flag old mem by IPC_RMID, copy data, and fix it values if it needs fixing + if (actual_byte_size != requested_byte_size) { shared_window_data *data; - if ((void *)-1 != (data = shmat(ptr->shmid, (void *)0, 0))) { - data_copy = malloc(actual_byte_size); - memcpy(data_copy,data,actual_byte_size); + if ((void *)-1 != (data = shmat(shmid, (void *)0, 0))) { + + *data_copy = malloc(actual_byte_size); + memcpy(*data_copy,data,actual_byte_size); + ptr->shmid=shmid; ptr->shm_address = data; - semian_window_data_delete_memory_inner(ptr, 1, self, data_copy); + semian_window_data_delete_memory_inner(ptr, 1, self, *data_copy); // Flagging for deletion sets a shm's associated key to be 0 so shmget gets a different shmid. // If this worker is creating the new memory - if (-1 != (ptr->shmid = shmget(key, requested_byte_size, flags))) { - created = 1; + if (-1 != (shmid = shmget(key, requested_byte_size, flags))) { + *created = 1; } else { // failed to get new memory, exit - failed=5; + failed=-5; } } else { // failed to attach, exit - rb_raise(eInternal,"Failed to copy old data, key %d, shmid %d, errno %d (%s)",key, ptr->shmid, errno, strerror(errno)); - failed=6; + rb_raise(eInternal,"Failed to copy old data, key %d, shmid %d, errno %d (%s)",key, shmid, errno, strerror(errno)); + failed=-6; } + } else{ + failed = -7; } } else if (!failed){ // ptr->max_window_size may be changed ptr->max_window_size = (actual_byte_size - 2*sizeof(int))/(sizeof(long)); + } else + failed=-8; + if (-1 != shmid) { + return shmid; + } else { + return failed; } +} + +/* + Acquire memory by getting shmid, and then attaching it to a memory location, + requires semaphore for locking/unlocking to be setup + + Note: should_keep_max_window_size is a bool that decides how ptr->max_window_size + is handled. There may be a discrepancy between the requested memory size and the actual + size of the memory block given. If it is false (0), ptr->max_window_size will be modified + to the actual memory size if there is a difference. This matters when dynamicaly resizing + memory. + Think of should_keep_max_window_size as this worker requesting a size, others resizing, + and !should_keep_max_window_size as another worker requesting a size and this worker + resizing +*/ +static VALUE +semian_window_data_acquire_memory(VALUE self, VALUE permissions, VALUE should_keep_max_window_size) +{ + semian_window_data *ptr; + TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); + + if (-1 == ptr->semid){ + rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); + return self; + } + if (TYPE(permissions) != T_FIXNUM) + rb_raise(rb_eTypeError, "expected integer for permissions"); + if (TYPE(should_keep_max_window_size) != T_TRUE && + TYPE(should_keep_max_window_size) != T_FALSE) + rb_raise(rb_eTypeError, "expected true or false for should_keep_max_window_size"); + + int should_keep_max_window_size_bool = RTEST(should_keep_max_window_size); + + if (!semian_window_data_lock(self)) + return Qfalse; + + + int created = 0; + key_t key = ptr->key; + shared_window_data *data_copy = NULL; // Will contain contents of old memory, if any - if (failed) { + int shmid = create_and_resize_memory(key, ptr->max_window_size, FIX2LONG(permissions), should_keep_max_window_size_bool, &created, &data_copy, ptr, self); + if (shmid < 0) {// failed if (data_copy) free(data_copy); semian_window_data_unlock(self); - rb_raise(eSyscall, "shmget() failed to acquire a memory shmid with key %d, size %zu, errno %d (%s)", key, ptr->max_window_size, errno, strerror(errno)); - } + rb_raise(eSyscall, "shmget() failed at %d to acquire a memory shmid with key %d, size %zu, errno %d (%s)", shmid, key, ptr->max_window_size, errno, strerror(errno)); + } else + ptr->shmid = shmid; + if (0 == ptr->shm_address) { ptr->shm_address = shmat(ptr->shmid, (void *)0, 0); @@ -551,6 +574,8 @@ semian_window_data_check_and_resize_if_needed(VALUE self) { semian_window_data_delete_memory_inner(ptr, 1, self, NULL); semian_window_data_unlock(self); semian_window_data_acquire_memory(self, LONG2FIX(ptr->permissions), Qfalse); + } else { + semian_window_data_unlock(self); } } From a59e4a99e51126eb8542b9699ca22a487e2c0639 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Tue, 6 Oct 2015 13:44:41 -0400 Subject: [PATCH 05/23] Refactored: added tests, created superclass SharedMemoryObject and inherited subclasses --- ext/semian/semian.c | 5 +- ext/semian/semian.h | 2 + ext/semian/semian_atomic_integer.c | 108 +++ ext/semian/semian_shared_memory_object.c | 629 +++++++++++++++ ext/semian/semian_shared_memory_object.h | 39 + ext/semian/semian_sliding_window.c | 970 +++++------------------ lib/semian.rb | 3 + lib/semian/atomic_enum.rb | 28 + lib/semian/atomic_integer.rb | 26 + lib/semian/circuit_breaker.rb | 74 +- lib/semian/protected_resource.rb | 11 +- lib/semian/shared_memory_object.rb | 46 ++ lib/semian/sliding_window.rb | 46 +- lib/semian/unprotected_resource.rb | 4 + test/atomic_enum_test.rb | 28 + test/atomic_integer_test.rb | 67 ++ test/circuit_breaker_test.rb | 52 ++ test/sliding_window_test.rb | 157 ++++ test/test_helper.rb | 1 + 19 files changed, 1462 insertions(+), 834 deletions(-) create mode 100644 ext/semian/semian_atomic_integer.c create mode 100644 ext/semian/semian_shared_memory_object.c create mode 100644 ext/semian/semian_shared_memory_object.h create mode 100644 lib/semian/atomic_enum.rb create mode 100644 lib/semian/atomic_integer.rb create mode 100644 lib/semian/shared_memory_object.rb create mode 100644 test/atomic_enum_test.rb create mode 100644 test/atomic_integer_test.rb create mode 100644 test/sliding_window_test.rb diff --git a/ext/semian/semian.c b/ext/semian/semian.c index a25a5438..ef888477 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -1,7 +1,6 @@ #include "semian.h" static ID id_timeout; -static VALUE eSyscall, eTimeout, eInternal; static int system_max_semaphore_count; static const int kIndexTickets = 0; @@ -417,7 +416,7 @@ semian_resource_id(VALUE self) } -void Init_semian_sliding_window(); +void Init_semian_shm_object(); void Init_semian() { @@ -477,5 +476,5 @@ void Init_semian() /* Maximum number of tickets available on this system. */ rb_define_const(cSemian, "MAX_TICKETS", INT2FIX(system_max_semaphore_count)); - Init_semian_sliding_window(); + Init_semian_shm_object(); } diff --git a/ext/semian/semian.h b/ext/semian/semian.h index 2bad8b2a..30d49252 100644 --- a/ext/semian/semian.h +++ b/ext/semian/semian.h @@ -36,6 +36,8 @@ typedef VALUE (*my_blocking_fn_t)(void*); #define WITHOUT_GVL(fn,a,ubf,b) rb_thread_blocking_region((my_blocking_fn_t)(fn),(a),(ubf),(b)) #endif +VALUE eSyscall, eTimeout, eInternal; + key_t generate_key(const char *name); diff --git a/ext/semian/semian_atomic_integer.c b/ext/semian/semian_atomic_integer.c new file mode 100644 index 00000000..37052a76 --- /dev/null +++ b/ext/semian/semian_atomic_integer.c @@ -0,0 +1,108 @@ +#include "semian_shared_memory_object.h" + +typedef struct { + int value; +} semian_atomic_int; + +static void semian_atomic_integer_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); +static VALUE semian_atomic_integer_bind_init_fn_wrapper(VALUE self); +static VALUE semian_atomic_integer_get_value(VALUE self); +static VALUE semian_atomic_integer_set_value(VALUE self, VALUE num); +static VALUE semian_atomic_integer_increase_by(VALUE self, VALUE num); + +static void +semian_atomic_integer_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count) +{ + semian_atomic_int *ptr = dest; + semian_atomic_int *old = prev_data; + if (prev_mem_attach_count){ + if (prev_data){ + ptr->value = old->value; + } // else copy nothing, data is same size and no need to copy + } else { + ptr->value=0; + } +} + +static VALUE +semian_atomic_integer_bind_init_fn_wrapper(VALUE self) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + ptr->object_init_fn = &semian_atomic_integer_bind_init_fn; + return self; +} + +static VALUE +semian_atomic_integer_get_value(VALUE self) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + + // check shared memory for NULL + if (0 == ptr->shm_address) + return Qnil; + + semian_shm_object_check_and_resize_if_needed(self); + if (!semian_shm_object_lock(self)) + return Qnil; + + int value = ((semian_atomic_int *)(ptr->shm_address))->value; + semian_shm_object_unlock(self); + return INT2NUM(value); +} + +static VALUE +semian_atomic_integer_set_value(VALUE self, VALUE num) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + + if (0 == ptr->shm_address) + return Qnil; + + if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT) + return Qnil; + semian_shm_object_check_and_resize_if_needed(self); + if (!semian_shm_object_lock(self)) + return Qnil; + + ((semian_atomic_int *)(ptr->shm_address))->value = NUM2INT(num); + + semian_shm_object_unlock(self); + return num; +} + +static VALUE +semian_atomic_integer_increase_by(VALUE self, VALUE num) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + + if (0 == ptr->shm_address) + return Qnil; + + if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT) + return Qnil; + semian_shm_object_check_and_resize_if_needed(self); + if (!semian_shm_object_lock(self)) + return Qnil; + + ((semian_atomic_int *)(ptr->shm_address))->value += NUM2INT(num); + + semian_shm_object_unlock(self); + return self; +} + +void +Init_semian_atomic_integer (void) +{ + // Bind methods to AtomicInteger + VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); + VALUE cAtomicInteger = rb_const_get(cSemianModule, rb_intern("AtomicInteger")); + + rb_define_method(cAtomicInteger, "bind_init_fn", semian_atomic_integer_bind_init_fn_wrapper, 0); + rb_define_method(cAtomicInteger, "value", semian_atomic_integer_get_value, 0); + rb_define_method(cAtomicInteger, "value=", semian_atomic_integer_set_value, 1); + rb_define_method(cAtomicInteger, "increase_by", semian_atomic_integer_increase_by, 1); +} diff --git a/ext/semian/semian_shared_memory_object.c b/ext/semian/semian_shared_memory_object.c new file mode 100644 index 00000000..813e4246 --- /dev/null +++ b/ext/semian/semian_shared_memory_object.c @@ -0,0 +1,629 @@ +#include "semian_shared_memory_object.h" + +const int kSHMSemaphoreCount = 1; // # semaphores to be acquired +const int kSHMTicketMax = 1; +const int kSHMInitializeWaitTimeout = 5; /* seconds */ +const int kSHMIndexTicketLock = 0; +const int kSHMInternalTimeout = 5; /* seconds */ + +static struct sembuf decrement; // = { kSHMIndexTicketLock, -1, SEM_UNDO}; +static struct sembuf increment; // = { kSHMIndexTicketLock, 1, SEM_UNDO}; + +/* + * Functions that handle type and memory +*/ +static void semian_shm_object_mark(void *ptr); +static void semian_shm_object_free(void *ptr); +static size_t semian_shm_object_memsize(const void *ptr); + +static void *semian_shm_object_lock_without_gvl(void *v_ptr); +static void *semian_shm_object_unlock_without_gvl(void *v_ptr); + +const rb_data_type_t +semian_shm_object_type = { + "semian_shm_object", + { + semian_shm_object_mark, + semian_shm_object_free, + semian_shm_object_memsize + }, + NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY +}; + +static void +semian_shm_object_mark(void *ptr) +{ + /* noop */ +} +static void +semian_shm_object_free(void *ptr) +{ + semian_shm_object *data = (semian_shm_object *) ptr; + // Under normal circumstances, memory use should be in the order of bytes, + // and shouldn't increase if the same key/id is used + // so there is no need to call this unless certain all other semian processes are stopped + // (also raises concurrency errors: "object allocation during garbage collection phase") + + xfree(data); +} +static size_t +semian_shm_object_memsize(const void *ptr) +{ + return + + sizeof(semian_shm_object); +} +static VALUE +semian_shm_object_alloc(VALUE klass) +{ + VALUE obj; + semian_shm_object *ptr; + + obj = TypedData_Make_Struct(klass, semian_shm_object, &semian_shm_object_type, ptr); + return obj; +} + +/* + * Implementations + */ + +VALUE +semian_shm_object_is_shared(VALUE klass) +{ + return Qtrue; +} + +VALUE +semian_shm_object_sizeof(VALUE klass, VALUE type) +{ + if (TYPE(type) != T_SYMBOL){ + rb_raise(rb_eTypeError, "id must be a symbol or string"); + } + + if (rb_intern("int") == SYM2ID(type)) + return INT2NUM(sizeof(int)); + else if (rb_intern("long") == SYM2ID(type)) + return INT2NUM(sizeof(long)); + else + return INT2NUM(0); +} + + +VALUE +semian_shm_object_acquire(VALUE self, VALUE name, VALUE byte_size, VALUE permissions) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + + if (TYPE(name) != T_SYMBOL && TYPE(name) != T_STRING) + rb_raise(rb_eTypeError, "id must be a symbol or string"); + if (TYPE(byte_size) != T_FIXNUM) + rb_raise(rb_eTypeError, "expected integer for byte_size"); + if (TYPE(permissions) != T_FIXNUM) + rb_raise(rb_eTypeError, "expected integer for permissions"); + + if (NUM2SIZET(byte_size) <= 0) + rb_raise(rb_eArgError, "byte_size must be larger than 0"); + + const char *id_str = NULL; + if (TYPE(name) == T_SYMBOL) { + id_str = rb_id2name(rb_to_id(name)); + } else if (TYPE(name) == T_STRING) { + id_str = RSTRING_PTR(name); + } + ptr->key = generate_key(id_str); + + // Guarantee byte_size >=1 or error thrown + ptr->byte_size = NUM2SIZET(byte_size); + + // id's default to -1 + ptr->semid = -1; + ptr->shmid = -1; + // addresses default to NULL + ptr->shm_address = 0; + ptr->lock_triggered = 0; + ptr->permissions = FIX2LONG(permissions); + + // Will throw NotImplementedError if not defined in concrete subclasses + // Implement bind_init_fn as a function with type + // static VALUE bind_init_fn(VALUE self) + // that binds ptr->object_init_fn to an init function called after memory is made + rb_funcall(self, rb_intern("bind_init_fn"), 0); + + semian_shm_object_acquire_semaphore(self, permissions); + semian_shm_object_acquire_memory(self, permissions, Qtrue); + + return self; +} + +VALUE +semian_shm_object_destroy(VALUE self) +{ + VALUE result = semian_shm_object_delete_memory(self); + if (!result) + return Qfalse; + result = semian_shm_object_delete_semaphore(self); + return result; +} + + +static int +create_semaphore_and_initialize_and_set_permissions(int key, int permissions) +{ + int semid = 0; + int flags = 0; + + flags = IPC_EXCL | IPC_CREAT | permissions; + + semid = semget(key, kSHMSemaphoreCount, flags); + if (semid >= 0) { + if (-1 == semctl(semid, 0, SETVAL, kSHMTicketMax)) { + rb_warn("semctl: failed to set semaphore with semid %d, position 0 to %d", semid, 1); + raise_semian_syscall_error("semctl()", errno); + } + } else if (semid == -1 && errno == EEXIST) { + flags &= ~IPC_EXCL; + semid = semget(key, kSHMSemaphoreCount, flags); + } + + if (-1 != semid){ + set_semaphore_permissions(semid, permissions); + } + + return semid; +} + + +/* + * Create or acquire previously made semaphore + */ + +VALUE +semian_shm_object_acquire_semaphore (VALUE self, VALUE permissions) +{ + // Function flow, semaphore creation methods are + // borrowed from semian.c since they have been previously tested + + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + + if (TYPE(permissions) != T_FIXNUM) + rb_raise(rb_eTypeError, "expected integer for permissions"); + + key_t key = ptr->key; + int semid = create_semaphore_and_initialize_and_set_permissions(key, FIX2LONG(permissions)); + if (-1 == semid) { + raise_semian_syscall_error("semget()", errno); + } + ptr->semid = semid; + + return self; +} + +VALUE +semian_shm_object_delete_semaphore(VALUE self) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + if (-1 == ptr->semid) // do nothing if semaphore not acquired + return Qfalse; + + if (-1 == semctl(ptr->semid, 0, IPC_RMID)) { + if (EIDRM == errno) { + rb_warn("semctl: failed to delete semaphore set with semid %d: already removed", ptr->semid); + raise_semian_syscall_error("semctl()", errno); + ptr->semid = -1; + } else { + rb_warn("semctl: failed to remove semaphore with semid %d, errno %d (%s)",ptr->semid, errno, strerror(errno)); + raise_semian_syscall_error("semctl()", errno); + } + } else { + ptr->semid = -1; + } + return self; +} + +/* + * semian_shm_object_lock/unlock and associated functions decrement/increment semaphore + */ + +static void * +semian_shm_object_lock_without_gvl(void *v_ptr) +{ + semian_shm_object *ptr = v_ptr; + if (-1 == ptr->semid){ + rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); + return (void *)Qfalse; + } + VALUE retval; + + struct timespec ts = { 0 }; + ts.tv_sec = kSHMInternalTimeout; + if (0 == ptr->lock_triggered) { + if (-1 == semtimedop(ptr->semid,&decrement,1, &ts)) { + rb_raise(eInternal, "error acquiring semaphore lock to mutate circuit breaker structure, %d: (%s)", errno, strerror(errno)); + retval=Qfalse; + } else { + (ptr->lock_triggered)+=1; + retval=Qtrue; + } + } else { + (ptr->lock_triggered)+=1; + retval=Qtrue; + } + return (void *)retval; +} + +VALUE +semian_shm_object_lock(VALUE self) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + return (VALUE) WITHOUT_GVL(semian_shm_object_lock_without_gvl, (void *)ptr, RUBY_UBF_IO, NULL); +} + +static void * +semian_shm_object_unlock_without_gvl(void *v_ptr) +{ + semian_shm_object *ptr = v_ptr; + if (-1 == ptr->semid){ + rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); + return (void *)Qfalse; + } + VALUE retval; + + if (1 == ptr->lock_triggered) { + if (-1 == semop(ptr->semid,&increment,1)) { + rb_raise(eInternal, "error unlocking semaphore, %d (%s)", errno, strerror(errno)); + retval=Qfalse; + } else { + --(ptr->lock_triggered); + retval=Qtrue; + } + } else { + --(ptr->lock_triggered); + retval=Qtrue; + } + return (void *)retval; +} + +VALUE +semian_shm_object_unlock(VALUE self) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + return (VALUE) WITHOUT_GVL(semian_shm_object_unlock_without_gvl, (void *)ptr, RUBY_UBF_IO, NULL); +} + +VALUE +semian_shm_object_unlock_all(VALUE self) +{ + semian_shm_object *ptr; + TypedData_Get_Struct((VALUE)self, semian_shm_object, &semian_shm_object_type, ptr); + while (ptr->lock_triggered > 0) + semian_shm_object_unlock(self); + return self; +} + +static int +create_and_resize_memory(key_t key, int byte_size, long permissions, int should_keep_req_byte_size_bool, + int *created, void **data_copy, size_t *data_copy_byte_size, int *prev_mem_attach_count, semian_shm_object *ptr, VALUE self) { + + // Below will handle acquiring/creating new memory, and possibly resizing the + // memory or the ptr->byte_size depending on + // should_keep_req_byte_size_bool + + int shmid=-1; + int failed=0; + + int requested_byte_size = byte_size; + int flags = IPC_CREAT | IPC_EXCL | permissions; + + int actual_byte_size = 0; + + // We fill both actual_byte_size and requested_byte_size + // Logic matrix: + // actual=>req | actual=req | actual current mem size + // 2. segment was requested to be created, but (size > SHMMAX || size < SHMMIN) + // 3. Other error + + // We can only see SHMMAX and SHMMIN through console commands + // Unlikely for 2 to occur, so we check by requesting a memory of size 1 byte + if (-1 != (shmid = shmget(key, 1, flags & ~IPC_EXCL))) { + struct shmid_ds shm_info; + if (-1 != shmctl(shmid, IPC_STAT, &shm_info)){ + actual_byte_size = shm_info.shm_segsz; + *prev_mem_attach_count = shm_info.shm_nattch; + failed = 0; + } else + failed=-3; + } else { // Error, exit + failed=-4; + } + } else { + *created = 1; + actual_byte_size = requested_byte_size; + } + + *data_copy_byte_size = actual_byte_size; + if (should_keep_req_byte_size_bool && !failed) { // resizing may occur + // we want to keep this worker's data + // We flag old mem by IPC_RMID and copy data + + if (actual_byte_size != requested_byte_size) { + void *data; + + if ((void *)-1 != (data = shmat(shmid, (void *)0, 0))) { + + *data_copy = malloc(actual_byte_size); + memcpy(*data_copy,data,actual_byte_size); + ptr->shmid=shmid; + ptr->shm_address = data; + semian_shm_object_delete_memory_inner(ptr, 1, self, *data_copy); + + // Flagging for deletion sets a shm's associated key to be 0 so shmget gets a different shmid. + // If this worker is creating the new memory + if (-1 != (shmid = shmget(key, requested_byte_size, flags))) { + *created = 1; + } else { // failed to get new memory, exit + failed=-5; + } + } else { // failed to attach, exit + rb_raise(eInternal,"Failed to copy old data, key %d, shmid %d, errno %d (%s)",key, shmid, errno, strerror(errno)); + failed=-6; + } + } else{ + failed = -7; + } + } else if (!failed){ // ptr->byte_size may be changed + // we don't want to keep this worker's data, it is old + ptr->byte_size = actual_byte_size; + } else + failed=-8; + if (-1 != shmid) { + return shmid; + } else { + return failed; + } +} + +/* + Acquire memory by getting shmid, and then attaching it to a memory location, + requires semaphore for locking/unlocking to be setup + + Note: should_keep_req_byte_size is a bool that decides how ptr->byte_size + is handled. There may be a discrepancy between the requested memory size and the actual + size of the memory block given. If it is false (0), ptr->byte_size will be modified + to the actual memory size if there is a difference. This matters when dynamicaly resizing + memory. + Think of should_keep_req_byte_size as this worker requesting a size, others resizing, + and !should_keep_req_byte_size as another worker requesting a size and this worker + resizing +*/ +VALUE +semian_shm_object_acquire_memory(VALUE self, VALUE permissions, VALUE should_keep_req_byte_size) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + + if (-1 == ptr->semid){ + rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); + return self; + } + if (TYPE(permissions) != T_FIXNUM) + rb_raise(rb_eTypeError, "expected integer for permissions"); + if (TYPE(should_keep_req_byte_size) != T_TRUE && + TYPE(should_keep_req_byte_size) != T_FALSE) + rb_raise(rb_eTypeError, "expected true or false for should_keep_req_byte_size"); + + int should_keep_req_byte_size_bool = RTEST(should_keep_req_byte_size); + + if (!semian_shm_object_lock(self)) + return Qfalse; + + + int created = 0; + key_t key = ptr->key; + void *data_copy = NULL; // Will contain contents of old memory, if any + size_t data_copy_byte_size = 0; + int prev_mem_attach_count = 0; + + // Create/acquire memory and manage all resizing cases + int shmid = create_and_resize_memory(key, ptr->byte_size, FIX2LONG(permissions), should_keep_req_byte_size_bool, + &created, &data_copy, &data_copy_byte_size, &prev_mem_attach_count, ptr, self); + if (shmid < 0) {// failed + if (data_copy) + free(data_copy); + semian_shm_object_unlock(self); + rb_raise(eSyscall, "shmget() failed at %d to acquire a memory shmid with key %d, size %zu, errno %d (%s)", shmid, key, ptr->byte_size, errno, strerror(errno)); + } else + ptr->shmid = shmid; + + // Attach memory and call object_init_fn() + if (0 == ptr->shm_address) { + ptr->shm_address = shmat(ptr->shmid, (void *)0, 0); + if (((void*)-1) == ptr->shm_address) { + semian_shm_object_unlock(self); + ptr->shm_address = 0; + if (data_copy) + free(data_copy); + rb_raise(eSyscall, "shmat() failed to attach memory with shmid %d, size %zu, errno %d (%s)", ptr->shmid, ptr->byte_size, errno, strerror(errno)); + } else { + ptr->object_init_fn(ptr->byte_size, ptr->shm_address, data_copy, data_copy_byte_size, prev_mem_attach_count); + } + } + if (data_copy) + free(data_copy); + semian_shm_object_unlock(self); + return self; +} + +void +semian_shm_object_delete_memory_inner (semian_shm_object *ptr, int should_unlock, VALUE self, void *should_free) +{ + // This internal function may be called from a variety of contexts + // Sometimes it is under a semaphore lock, sometimes it has extra malloc ptrs + // Arguments handle these conditions + + if (0 != ptr->shm_address){ + if (-1 == shmdt(ptr->shm_address)) { + if (should_unlock) + semian_shm_object_unlock(self); + if (should_free) + free(should_free); + rb_raise(eSyscall,"shmdt: no attached memory at %p, errno %d (%s)", ptr->shm_address, errno, strerror(errno)); + } else { + } + ptr->shm_address = 0; + } + + if (-1 != ptr->shmid) { + // Once IPC_RMID is set, no new shmgets can be made with key, and current values are invalid + if (-1 == shmctl(ptr->shmid, IPC_RMID, 0)) { + if (errno == EINVAL) { + ptr->shmid = -1; + } else { + if (should_unlock) + semian_shm_object_unlock(self); + if (should_free) + free(should_free); + rb_raise(eSyscall,"shmctl: error flagging memory for removal with shmid %d, errno %d (%s)", ptr->shmid, errno, strerror(errno)); + } + } else { + ptr->shmid = -1; + } + } +} + +VALUE +semian_shm_object_delete_memory (VALUE self) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + + if (!semian_shm_object_lock(self)) + return Qnil; + + semian_shm_object_delete_memory_inner(ptr, 1, self, NULL); + + semian_shm_object_unlock(self); + return self; +} + +VALUE +semian_shm_object_check_and_resize_if_needed(VALUE self) { + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + + if (!semian_shm_object_lock(self)) + return Qnil; + + struct shmid_ds shm_info; + int needs_resize = 0; + if (-1 != ptr->shmid && -1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)) { + needs_resize = shm_info.shm_perm.mode & SHM_DEST; + } + + if (needs_resize) { + semian_shm_object_delete_memory_inner(ptr, 1, self, NULL); + semian_shm_object_unlock(self); + semian_shm_object_acquire_memory(self, LONG2FIX(ptr->permissions), Qfalse); + } else { + semian_shm_object_unlock(self); + } + return self; +} + +static VALUE +semian_shm_object_semid(VALUE self) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + semian_shm_object_check_and_resize_if_needed(self); + return INT2NUM(ptr->semid); +} +static VALUE +semian_shm_object_shmid(VALUE self) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + semian_shm_object_check_and_resize_if_needed(self); + return INT2NUM(ptr->shmid); +} + +static VALUE +semian_shm_object_byte_size(VALUE self) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + semian_shm_object_check_and_resize_if_needed(self); + int byte_size = ptr->byte_size; + return INT2NUM(byte_size); +} + +static VALUE +semian_shm_object_execute_atomically_with_block(VALUE self) +{ + if (!rb_block_given_p()) + rb_raise(rb_eArgError, "Expected block"); + return rb_yield(Qnil); +} + +static VALUE +semian_shm_object_execute_atomically(VALUE self) { + if (!semian_shm_object_lock(self)) + return Qnil; + return rb_ensure(semian_shm_object_execute_atomically_with_block, self, semian_shm_object_unlock_all, self); +} + +void +Init_semian_shm_object (void) { + + VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); + VALUE cSharedMemoryObject = rb_const_get(cSemianModule, rb_intern("SharedMemoryObject")); + + rb_define_alloc_func(cSharedMemoryObject, semian_shm_object_alloc); + rb_define_method(cSharedMemoryObject, "_acquire", semian_shm_object_acquire, 3); + rb_define_method(cSharedMemoryObject, "destroy", semian_shm_object_destroy, 0); + rb_define_method(cSharedMemoryObject, "lock", semian_shm_object_lock, 0); + rb_define_method(cSharedMemoryObject, "unlock", semian_shm_object_unlock, 0); + rb_define_method(cSharedMemoryObject, "byte_size", semian_shm_object_byte_size, 0); + + rb_define_method(cSharedMemoryObject, "semid", semian_shm_object_semid, 0); + rb_define_method(cSharedMemoryObject, "shmid", semian_shm_object_shmid, 0); + rb_define_method(cSharedMemoryObject, "execute_atomically", semian_shm_object_execute_atomically, 0); + + rb_define_singleton_method(cSharedMemoryObject, "shared?", semian_shm_object_is_shared, 0); + rb_define_singleton_method(cSharedMemoryObject, "_sizeof", semian_shm_object_sizeof, 1); + + decrement.sem_num = kSHMIndexTicketLock; + decrement.sem_op = -1; + decrement.sem_flg = SEM_UNDO; + + increment.sem_num = kSHMIndexTicketLock; + increment.sem_op = 1; + increment.sem_flg = SEM_UNDO; + + Init_semian_atomic_integer(); + Init_semian_sliding_window(); +} + diff --git a/ext/semian/semian_shared_memory_object.h b/ext/semian/semian_shared_memory_object.h new file mode 100644 index 00000000..9416cabb --- /dev/null +++ b/ext/semian/semian_shared_memory_object.h @@ -0,0 +1,39 @@ +#include "semian.h" + +typedef struct { + //semaphore, shared memory data and pointer + key_t key; + size_t byte_size; + int lock_triggered; // lock only done from 0 -> 1, unlock only done from 1 -> 0, so we can 'lock' multiple times (such as in nesting functions) without actually locking + int permissions; + int semid; + int shmid; + void (*object_init_fn)(size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); + void *shm_address; +} semian_shm_object; + +extern const rb_data_type_t +semian_shm_object_type; + +/* + * Headers + */ + +VALUE semian_shm_object_is_shared(VALUE klass); +VALUE semian_shm_object_sizeof(VALUE klass, VALUE type); + +VALUE semian_shm_object_acquire(VALUE self, VALUE id, VALUE byte_size, VALUE permissions); +VALUE semian_shm_object_destroy(VALUE self); +VALUE semian_shm_object_acquire_semaphore (VALUE self, VALUE permissions); +VALUE semian_shm_object_delete_semaphore(VALUE self); +VALUE semian_shm_object_lock(VALUE self); +VALUE semian_shm_object_unlock(VALUE self); +VALUE semian_shm_object_unlock_all(VALUE self); +VALUE semian_shm_object_acquire_memory(VALUE self, VALUE permissions, VALUE should_keep_req_byte_size); +void semian_shm_object_delete_memory_inner (semian_shm_object *ptr, int should_unlock, VALUE self, void *should_free); +VALUE semian_shm_object_delete_memory (VALUE self); +VALUE semian_shm_object_check_and_resize_if_needed (VALUE self); + + +void Init_semian_atomic_integer (void); +void Init_semian_sliding_window (void); diff --git a/ext/semian/semian_sliding_window.c b/ext/semian/semian_sliding_window.c index 47434936..62389ef6 100644 --- a/ext/semian/semian_sliding_window.c +++ b/ext/semian/semian_sliding_window.c @@ -1,890 +1,312 @@ -#include "semian.h" - -// struct sembuf { // found in sys/sem.h -// unsigned short sem_num; /* semaphore number */ -// short sem_op; /* semaphore operation */ -// short sem_flg; /* operation flags */ -// }; +#include "semian_shared_memory_object.h" typedef struct { - int counter; - int window_length; + int max_window_size; + int window_size; long window[]; -} shared_window_data; +} semian_sliding_window; + +static void semian_sliding_window_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); +static VALUE semian_sliding_window_bind_init_fn_wrapper(VALUE self); +static VALUE semian_sliding_window_size(VALUE self); +static VALUE semian_sliding_window_max_size(VALUE self); +static VALUE semian_sliding_window_push_back(VALUE self, VALUE num); +static VALUE semian_sliding_window_pop_back(VALUE self); +static VALUE semian_sliding_window_push_front(VALUE self, VALUE num); +static VALUE semian_sliding_window_pop_front(VALUE self); +static VALUE semian_sliding_window_clear(VALUE self); +static VALUE semian_sliding_window_first(VALUE self); +static VALUE semian_sliding_window_last(VALUE self); +static VALUE semian_sliding_window_resize_to(VALUE self, VALUE size); -typedef struct { - //semaphore, shared memory data and pointer - key_t key; - size_t max_window_size; - int lock_triggered; - int permissions; - int semid; - int shmid; - shared_window_data *shm_address; -} semian_window_data; - -static const int kCBSemaphoreCount = 1; // # semaphores to be acquired -static const int kCBTicketMax = 1; -static const int kCBInitializeWaitTimeout = 5; /* seconds */ -static const int kCBIndexTicketLock = 0; -static const int kCBInternalTimeout = 5; /* seconds */ - -static struct sembuf decrement; // = { kCBIndexTicketLock, -1, SEM_UNDO}; -static struct sembuf increment; // = { kCBIndexTicketLock, 1, SEM_UNDO}; - -static VALUE eInternal, eSyscall, eTimeout; // Semian errors - -static void semian_window_data_mark(void *ptr); -static void semian_window_data_free(void *ptr); -static size_t semian_window_data_memsize(const void *ptr); -static VALUE semian_window_data_alloc(VALUE klass); -static VALUE semian_window_data_init(VALUE self, VALUE name, VALUE size, VALUE permissions); -static VALUE semian_window_data_destroy(VALUE self); -static int create_semaphore_and_initialize(int key, int permissions); -static VALUE semian_window_data_acquire_semaphore (VALUE self, VALUE permissions); -static VALUE semian_window_data_delete_semaphore(VALUE self); -static VALUE semian_window_data_lock(VALUE self); -static VALUE semian_window_data_unlock(VALUE self); -static void *semian_window_data_lock_without_gvl(void *self); -static void *semian_window_data_unlock_without_gvl(void *self); -static VALUE semian_window_data_acquire_memory(VALUE self, VALUE permissions, VALUE should_keep_max_window_size); -static void semian_window_data_delete_memory_inner (semian_window_data *ptr, int should_unlock, VALUE self, void *should_free); -static VALUE semian_window_data_delete_memory (VALUE self); -static void semian_window_data_check_and_resize_if_needed (VALUE self); -static VALUE semian_window_data_get_counter(VALUE self); -static VALUE semian_window_data_set_counter(VALUE self, VALUE num); -static VALUE semian_window_data_semid(VALUE self); -static VALUE semian_window_data_shmid(VALUE self); -static VALUE semian_window_data_array_length(VALUE self); -static VALUE semian_window_data_set_push_back(VALUE self, VALUE num); -static VALUE semian_window_data_set_pop_back(VALUE self); -static VALUE semian_window_data_set_push_front(VALUE self, VALUE num); -static VALUE semian_window_data_set_pop_front(VALUE self); - -static VALUE semian_window_data_is_shared(VALUE self); - - -// needed for TypedData_Make_Struct && TypedData_Get_Struct -static const rb_data_type_t -semian_window_data_type = { - "semian_window_data", - { - semian_window_data_mark, - semian_window_data_free, - semian_window_data_memsize - }, - NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY -}; - -/* - * Functions that handle type and memory -*/ static void -semian_window_data_mark(void *ptr) -{ - /* noop */ -} - -static void -semian_window_data_free(void *ptr) -{ - semian_window_data *data = (semian_window_data *) ptr; - - - // Under normal circumstances, memory use should be in the order of bytes, - // and shouldn't increase if the same key/id is used - // so there is no need to call this unless certain all other semian processes are stopped - // (also raises concurrency errors: "object allocation during garbage collection phase") - - //semian_window_data_delete_memory_inner (data); - - xfree(data); -} - -static size_t -semian_window_data_memsize(const void *ptr) -{ - return sizeof(semian_window_data); -} - -static VALUE -semian_window_data_alloc(VALUE klass) -{ - VALUE obj; - semian_window_data *ptr; - - obj = TypedData_Make_Struct(klass, semian_window_data, &semian_window_data_type, ptr); - return obj; -} - - - - - - -/* - * Init function exposed as ._initialize() that is delegated by .initialize() - */ -static VALUE -semian_window_data_init(VALUE self, VALUE id, VALUE size, VALUE permissions) +semian_sliding_window_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count) { - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - - if (TYPE(id) != T_SYMBOL && TYPE(id) != T_STRING) - rb_raise(rb_eTypeError, "id must be a symbol or string"); - if (TYPE(size) != T_FIXNUM) - rb_raise(rb_eTypeError, "expected integer for max_window_size"); - if (TYPE(permissions) != T_FIXNUM) - rb_raise(rb_eTypeError, "expected integer for permissions"); - - if (NUM2SIZET(size) <= 0) - rb_raise(rb_eArgError, "max_window_size must be larger than 0"); - - const char *id_str = NULL; - if (TYPE(id) == T_SYMBOL) { - id_str = rb_id2name(rb_to_id(id)); - } else if (TYPE(id) == T_STRING) { - id_str = RSTRING_PTR(id); - } - ptr->key = generate_key(id_str); - //rb_warn("converted name %s to key %d", id_str, ptr->key); - - // Guarantee max_window_size >=1 or error thrown - ptr->max_window_size = NUM2SIZET(size); - - // id's default to -1 - ptr->semid = -1; - ptr->shmid = -1; - // addresses default to NULL - ptr->shm_address = 0; - ptr->lock_triggered = 0; - ptr->permissions = FIX2LONG(permissions); - - semian_window_data_acquire_semaphore(self, permissions); - semian_window_data_acquire_memory(self, permissions, Qtrue); - - return self; -} - -static VALUE -semian_window_data_destroy(VALUE self) -{ - semian_window_data_delete_memory(self); - semian_window_data_delete_semaphore(self); - return self; -} - - -static int -create_semaphore_and_initialize(int key, int permissions) -{ - int semid = 0; - int flags = 0; - - flags = IPC_EXCL | IPC_CREAT | permissions; - - semid = semget(key, kCBSemaphoreCount, flags); - if (semid >= 0) { - if (-1 == semctl(semid, 0, SETVAL, kCBTicketMax)) { - rb_warn("semctl: failed to set semaphore with semid %d, position 0 to %d", semid, 1); - raise_semian_syscall_error("semctl()", errno); - } - } else if (semid == -1 && errno == EEXIST) { - flags &= ~IPC_EXCL; - semid = semget(key, kCBSemaphoreCount, flags); - } - return semid; -} - - -/* - * Create or acquire previously made semaphore - */ - -static VALUE -semian_window_data_acquire_semaphore (VALUE self, VALUE permissions) -{ - // Function flow, semaphore creation methods are - // borrowed from semian.c since they have been previously tested - - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - - if (TYPE(permissions) != T_FIXNUM) - rb_raise(rb_eTypeError, "expected integer for permissions"); - - // bool for initializing (configure_tickets) or not - key_t key = ptr->key; - int semid = create_semaphore_and_initialize(key, FIX2LONG(permissions)); - if (-1 == semid) { - raise_semian_syscall_error("semget()", errno); - } - ptr->semid = semid; - - set_semaphore_permissions(ptr->semid, FIX2LONG(permissions)); - - return self; -} - - -static VALUE -semian_window_data_delete_semaphore(VALUE self) -{ - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - if (-1 == ptr->semid) // do nothing if semaphore not acquired - return Qfalse; - - if (-1 == semctl(ptr->semid, 0, IPC_RMID)) { - if (EIDRM == errno) { - rb_warn("semctl: failed to delete semaphore set with semid %d: already removed", ptr->semid); - raise_semian_syscall_error("semctl()", errno); - ptr->semid = -1; - } else { - rb_warn("semctl: failed to remove semaphore with semid %d, errno %d (%s)",ptr->semid, errno, strerror(errno)); - raise_semian_syscall_error("semctl()", errno); - } + semian_sliding_window *ptr = dest; + semian_sliding_window *old = prev_data; + + // Logic to initialize: initialize to 0 only if you're the first to attach (else body) + // Otherwise, copy previous data into new data + if (prev_mem_attach_count) { + if (prev_data) { + // transfer data over + ptr->max_window_size = (byte_size-2*sizeof(int))/sizeof(long); + ptr->window_size = fmin(ptr->max_window_size, old->window_size); + + // Copy the most recent ptr->shm_address->window_size numbers to new memory + memcpy(&(ptr->window), + ((long *)(&(old->window[0])))+old->window_size-ptr->window_size, + ptr->window_size * sizeof(long)); + } // else copy nothing, data is same size and no need to copy } else { - //rb_warn("semctl: semaphore set with semid %d deleted", ptr->semid); - ptr->semid = -1; + semian_sliding_window *data = dest; + data->max_window_size = (byte_size-2*sizeof(int))/sizeof(long); + data->window_size = 0; + for (int i=0; i< data->window_size; ++i) + data->window[i]=0; } - return self; } - - - -/* - * semian_window_data_lock/unlock and associated functions decrement/increment semaphore - */ - static VALUE -semian_window_data_lock(VALUE self) +semian_sliding_window_bind_init_fn_wrapper(VALUE self) { - return (VALUE) WITHOUT_GVL(semian_window_data_lock_without_gvl, (void *)self, RUBY_UBF_IO, NULL); -} - -static void * -semian_window_data_lock_without_gvl(void *self) -{ - semian_window_data *ptr; - TypedData_Get_Struct((VALUE)self, semian_window_data, &semian_window_data_type, ptr); - if (ptr->lock_triggered) - return (void *)Qtrue; - if (-1 == ptr->semid){ - rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); - return (void *)Qfalse; - } - VALUE retval; - - struct timespec ts = { 0 }; - ts.tv_sec = kCBInternalTimeout; - - if (-1 == semtimedop(ptr->semid,&decrement,1, &ts)) { - rb_raise(eInternal, "error acquiring semaphore lock to mutate circuit breaker structure, %d: (%s)", errno, strerror(errno)); - retval=Qfalse; - } else - retval=Qtrue; - - ptr->lock_triggered = 1; - return (void *)retval; -} - -static VALUE -semian_window_data_unlock(VALUE self) -{ - return (VALUE) WITHOUT_GVL(semian_window_data_unlock_without_gvl, (void *)self, RUBY_UBF_IO, NULL); -} - -static void * -semian_window_data_unlock_without_gvl(void *self) -{ - semian_window_data *ptr; - TypedData_Get_Struct((VALUE)self, semian_window_data, &semian_window_data_type, ptr); - if (!(ptr->lock_triggered)) - return (void *)Qtrue; - if (-1 == ptr->semid){ - rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); - return (void *)Qfalse; - } - VALUE retval; - - if (-1 == semop(ptr->semid,&increment,1)) { - rb_raise(eInternal, "error unlocking semaphore, %d (%s)", errno, strerror(errno)); - retval=Qfalse; - } else - retval=Qtrue; - - ptr->lock_triggered = 0; - //rb_warn("semop unlock success"); - return (void *)retval; -} - - -static int -create_and_resize_memory(key_t key, int max_window_size, long permissions, int should_keep_max_window_size_bool, - int *created, shared_window_data **data_copy, semian_window_data *ptr, VALUE self) { - - // Below will handle acquiring/creating new memory, and possibly resizing the - // memory or the ptr->max_window_size depending on - // should_keep_max_window_size_bool - - int shmid=-1; - int failed=0; - - int requested_byte_size = 2*sizeof(int) + max_window_size * sizeof(long); - int flags = IPC_CREAT | IPC_EXCL | permissions; - - int actual_byte_size = 0; - - // We fill both actual_byte_size and requested_byte_size - // Logic matrix: - // actual=>req | actual=req | actual current mem size - // 2. segment was requested to be created, but (size > SHMMAX || size < SHMMIN) - // 3. Other error - - // We can only see SHMMAX and SHMMIN through console commands - // Unlikely for 2 to occur, so we check by requesting a memory of size 1 byte - if (-1 != (shmid = shmget(key, 1, flags & ~IPC_EXCL))) { - struct shmid_ds shm_info; - if (-1 != shmctl(shmid, IPC_STAT, &shm_info)){ - actual_byte_size = shm_info.shm_segsz; - failed = 0; - } else - failed=-3; - } else { // Error, exit - failed=-4; - } - } else { - *created = 1; - actual_byte_size = requested_byte_size; - } - - if (should_keep_max_window_size_bool && !failed) { // memory resizing may occur - // We flag old mem by IPC_RMID, copy data, and fix it values if it needs fixing - - if (actual_byte_size != requested_byte_size) { - shared_window_data *data; - - if ((void *)-1 != (data = shmat(shmid, (void *)0, 0))) { - - *data_copy = malloc(actual_byte_size); - memcpy(*data_copy,data,actual_byte_size); - ptr->shmid=shmid; - ptr->shm_address = data; - semian_window_data_delete_memory_inner(ptr, 1, self, *data_copy); - - // Flagging for deletion sets a shm's associated key to be 0 so shmget gets a different shmid. - // If this worker is creating the new memory - if (-1 != (shmid = shmget(key, requested_byte_size, flags))) { - *created = 1; - } else { // failed to get new memory, exit - failed=-5; - } - } else { // failed to attach, exit - rb_raise(eInternal,"Failed to copy old data, key %d, shmid %d, errno %d (%s)",key, shmid, errno, strerror(errno)); - failed=-6; - } - } else{ - failed = -7; - } - } else if (!failed){ // ptr->max_window_size may be changed - ptr->max_window_size = (actual_byte_size - 2*sizeof(int))/(sizeof(long)); - } else - failed=-8; - if (-1 != shmid) { - return shmid; - } else { - return failed; - } -} - -/* - Acquire memory by getting shmid, and then attaching it to a memory location, - requires semaphore for locking/unlocking to be setup - - Note: should_keep_max_window_size is a bool that decides how ptr->max_window_size - is handled. There may be a discrepancy between the requested memory size and the actual - size of the memory block given. If it is false (0), ptr->max_window_size will be modified - to the actual memory size if there is a difference. This matters when dynamicaly resizing - memory. - Think of should_keep_max_window_size as this worker requesting a size, others resizing, - and !should_keep_max_window_size as another worker requesting a size and this worker - resizing -*/ -static VALUE -semian_window_data_acquire_memory(VALUE self, VALUE permissions, VALUE should_keep_max_window_size) -{ - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - - if (-1 == ptr->semid){ - rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); - return self; - } - if (TYPE(permissions) != T_FIXNUM) - rb_raise(rb_eTypeError, "expected integer for permissions"); - if (TYPE(should_keep_max_window_size) != T_TRUE && - TYPE(should_keep_max_window_size) != T_FALSE) - rb_raise(rb_eTypeError, "expected true or false for should_keep_max_window_size"); - - int should_keep_max_window_size_bool = RTEST(should_keep_max_window_size); - - if (!semian_window_data_lock(self)) - return Qfalse; - - - int created = 0; - key_t key = ptr->key; - shared_window_data *data_copy = NULL; // Will contain contents of old memory, if any - - int shmid = create_and_resize_memory(key, ptr->max_window_size, FIX2LONG(permissions), should_keep_max_window_size_bool, &created, &data_copy, ptr, self); - if (shmid < 0) {// failed - if (data_copy) - free(data_copy); - semian_window_data_unlock(self); - rb_raise(eSyscall, "shmget() failed at %d to acquire a memory shmid with key %d, size %zu, errno %d (%s)", shmid, key, ptr->max_window_size, errno, strerror(errno)); - } else - ptr->shmid = shmid; - - - if (0 == ptr->shm_address) { - ptr->shm_address = shmat(ptr->shmid, (void *)0, 0); - if (((void*)-1) == ptr->shm_address) { - semian_window_data_unlock(self); - ptr->shm_address = 0; - if (data_copy) - free(data_copy); - rb_raise(eSyscall, "shmat() failed to attach memory with shmid %d, size %zu, errno %d (%s)", ptr->shmid, ptr->max_window_size, errno, strerror(errno)); - } else { - if (created) { - if (data_copy) { - // transfer data over - ptr->shm_address->counter = data_copy->counter; - ptr->shm_address->window_length = fmin(ptr->max_window_size-1, data_copy->window_length); - - // Copy the most recent ptr->shm_address->window_length numbers to new memory - memcpy(&(ptr->shm_address->window), - ((long *)(&(data_copy->window[0])))+data_copy->window_length-ptr->shm_address->window_length, - ptr->shm_address->window_length * sizeof(long)); - } else { - shared_window_data *data = ptr->shm_address; - data->counter = 0; - data->window_length = 0; - for (int i=0; i< data->window_length; ++i) - data->window[i]=0; - } - } - } - } - if (data_copy) - free(data_copy); - semian_window_data_unlock(self); + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + ptr->object_init_fn = &semian_sliding_window_bind_init_fn; return self; } -static void -semian_window_data_delete_memory_inner (semian_window_data *ptr, int should_unlock, VALUE self, void *should_free) -{ - // This internal function may be called from a variety of contexts - // Sometimes it is under a semaphore lock, sometimes it has extra malloc ptrs - // Arguments handle these conditions - - if (0 != ptr->shm_address){ - if (-1 == shmdt(ptr->shm_address)) { - if (should_unlock) - semian_window_data_unlock(self); - if (should_free) - free(should_free); - rb_raise(eSyscall,"shmdt: no attached memory at %p, errno %d (%s)", ptr->shm_address, errno, strerror(errno)); - } else { - } - ptr->shm_address = 0; - } - - if (-1 != ptr->shmid) { - // Once IPC_RMID is set, no new shmgets can be made with key, and current values are invalid - if (-1 == shmctl(ptr->shmid, IPC_RMID, 0)) { - if (errno == EINVAL) { - ptr->shmid = -1; - } else { - if (should_unlock) - semian_window_data_unlock(self); - if (should_free) - free(should_free); - rb_raise(eSyscall,"shmctl: error flagging memory for removal with shmid %d, errno %d (%s)", ptr->shmid, errno, strerror(errno)); - } - } else { - ptr->shmid = -1; - } - } -} - - static VALUE -semian_window_data_delete_memory (VALUE self) +semian_sliding_window_size(VALUE self) { - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - - if (!semian_window_data_lock(self)) - return self; - - semian_window_data_delete_memory_inner(ptr, 1, self, NULL); - - semian_window_data_unlock(self); - return self; -} - - -static void //bool -semian_window_data_check_and_resize_if_needed(VALUE self) { - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - - if (!semian_window_data_lock(self)) - return; - - struct shmid_ds shm_info; - int needs_resize = 0; - if (-1 != ptr->shmid && -1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)) { - needs_resize = shm_info.shm_perm.mode & SHM_DEST; - } - - if (needs_resize) { - semian_window_data_delete_memory_inner(ptr, 1, self, NULL); - semian_window_data_unlock(self); - semian_window_data_acquire_memory(self, LONG2FIX(ptr->permissions), Qfalse); - } else { - semian_window_data_unlock(self); - } -} - - -/* - * Below are methods for counter, semid, shmid, and array pop, push, peek at front and back - * and clear, length - */ - -static VALUE -semian_window_data_get_counter(VALUE self) -{ - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - - // check shared memory for NULL - if (0 == ptr->shm_address) - return Qnil; - semian_window_data_check_and_resize_if_needed(self); - if (!semian_window_data_lock(self)) - return Qnil; - - int counter = ptr->shm_address->counter; - - semian_window_data_unlock(self); - return INT2NUM(counter); -} - -static VALUE -semian_window_data_set_counter(VALUE self, VALUE num) -{ - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - - if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT) - return Qnil; - semian_window_data_check_and_resize_if_needed(self); - if (!semian_window_data_lock(self)) + semian_shm_object_check_and_resize_if_needed(self); + if (!semian_shm_object_lock(self)) return Qnil; - - ptr->shm_address->counter = NUM2INT(num); - - semian_window_data_unlock(self); - return num; -} - - -static VALUE -semian_window_data_semid(VALUE self) -{ - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - semian_window_data_check_and_resize_if_needed(self); - return INT2NUM(ptr->semid); -} -static VALUE -semian_window_data_shmid(VALUE self) -{ - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - semian_window_data_check_and_resize_if_needed(self); - return INT2NUM(ptr->shmid); + int window_size = ((semian_sliding_window *)(ptr->shm_address))->window_size; + semian_shm_object_unlock(self); + return INT2NUM(window_size); } static VALUE -semian_window_data_max_window_size(VALUE self) +semian_sliding_window_max_size(VALUE self) { - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - semian_window_data_check_and_resize_if_needed(self); - int max_window_size =ptr->max_window_size; - return INT2NUM(max_window_size); -} - -static VALUE -semian_window_data_array_length(VALUE self) -{ - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - semian_window_data_check_and_resize_if_needed(self); - if (!semian_window_data_lock(self)) + semian_shm_object_check_and_resize_if_needed(self); + if (!semian_shm_object_lock(self)) return Qnil; - int window_length =ptr->shm_address->window_length; - semian_window_data_unlock(self); - return INT2NUM(window_length); + int max_length = ((semian_sliding_window *)(ptr->shm_address))->max_window_size; + semian_shm_object_unlock(self); + return INT2NUM(max_length); } static VALUE -semian_window_data_set_push_back(VALUE self, VALUE num) +semian_sliding_window_push_back(VALUE self, VALUE num) { - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT && TYPE(num) != T_BIGNUM) return Qnil; - semian_window_data_check_and_resize_if_needed(self); - if (!semian_window_data_lock(self)) + semian_shm_object_check_and_resize_if_needed(self); + if (!semian_shm_object_lock(self)) return Qnil; - shared_window_data *data = ptr->shm_address; - if (data->window_length == ptr->max_window_size) { - for (int i=1; i< ptr->max_window_size; ++i){ + semian_sliding_window *data = ptr->shm_address; + if (data->window_size == data->max_window_size) { + for (int i=1; i< data->max_window_size; ++i){ data->window[i-1] = data->window[i]; } - --(data->window_length); + --(data->window_size); } - data->window[(data->window_length)] = NUM2LONG(num); - ++(data->window_length); - semian_window_data_unlock(self); + data->window[(data->window_size)] = NUM2LONG(num); + ++(data->window_size); + semian_shm_object_unlock(self); return self; } static VALUE -semian_window_data_set_pop_back(VALUE self) +semian_sliding_window_pop_back(VALUE self) { - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - semian_window_data_check_and_resize_if_needed(self); - if (!semian_window_data_lock(self)) + semian_shm_object_check_and_resize_if_needed(self); + if (!semian_shm_object_lock(self)) return Qnil; VALUE retval; - shared_window_data *data = ptr->shm_address; - if (0 == data->window_length) + semian_sliding_window *data = ptr->shm_address; + if (0 == data->window_size) retval = Qnil; else { - retval = LONG2NUM(data->window[data->window_length-1]); - --(data->window_length); + retval = LONG2NUM(data->window[data->window_size-1]); + --(data->window_size); } - semian_window_data_unlock(self); + semian_shm_object_unlock(self); return retval; } static VALUE -semian_window_data_set_pop_front(VALUE self) +semian_sliding_window_push_front(VALUE self, VALUE num) { - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); - if (0 == ptr->shm_address) - return Qnil; - semian_window_data_check_and_resize_if_needed(self); - if (!semian_window_data_lock(self)) - return Qnil; - - VALUE retval; - shared_window_data *data = ptr->shm_address; - if (0 >= data->window_length) - retval = Qnil; - else { - retval = LONG2NUM(data->window[0]); - for (int i=0; iwindow_length-1; ++i) - data->window[i]=data->window[i+1]; - --(data->window_length); - } - - semian_window_data_unlock(self); - return retval; -} - -static VALUE -semian_window_data_set_push_front(VALUE self, VALUE num) -{ - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT && TYPE(num) != T_BIGNUM) return Qnil; - semian_window_data_check_and_resize_if_needed(self); - if (!semian_window_data_lock(self)) + semian_shm_object_check_and_resize_if_needed(self); + if (!semian_shm_object_lock(self)) return Qnil; long val = NUM2LONG(num); - shared_window_data *data = ptr->shm_address; + semian_sliding_window *data = ptr->shm_address; - int i=data->window_length; + int i=data->window_size; for (; i>0; --i) data->window[i]=data->window[i-1]; data->window[0] = val; - ++(data->window_length); - if (data->window_length>ptr->max_window_size) - data->window_length=ptr->max_window_size; + ++(data->window_size); + if (data->window_size>data->max_window_size) + data->window_size=data->max_window_size; - semian_window_data_unlock(self); + semian_shm_object_unlock(self); return self; } static VALUE -semian_window_data_array_clear(VALUE self) +semian_sliding_window_pop_front(VALUE self) { - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - semian_window_data_check_and_resize_if_needed(self); - if (!semian_window_data_lock(self)) + semian_shm_object_check_and_resize_if_needed(self); + if (!semian_shm_object_lock(self)) return Qnil; - ptr->shm_address->window_length=0; - semian_window_data_unlock(self); + VALUE retval; + semian_sliding_window *data = ptr->shm_address; + if (0 >= data->window_size) + retval = Qnil; + else { + retval = LONG2NUM(data->window[0]); + for (int i=0; iwindow_size-1; ++i) + data->window[i]=data->window[i+1]; + --(data->window_size); + } + + semian_shm_object_unlock(self); + return retval; +} + +static VALUE +semian_sliding_window_clear(VALUE self) +{ + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + if (0 == ptr->shm_address) + return Qnil; + semian_shm_object_check_and_resize_if_needed(self); + if (!semian_shm_object_lock(self)) + return Qnil; + semian_sliding_window *data = ptr->shm_address; + data->window_size=0; + + semian_shm_object_unlock(self); return self; } static VALUE -semian_window_data_array_first(VALUE self) +semian_sliding_window_first(VALUE self) { - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - semian_window_data_check_and_resize_if_needed(self); - if (!semian_window_data_lock(self)) + semian_shm_object_check_and_resize_if_needed(self); + if (!semian_shm_object_lock(self)) return Qnil; VALUE retval; - if (ptr->shm_address->window_length >=1) - retval = LONG2NUM(ptr->shm_address->window[0]); + semian_sliding_window *data = ptr->shm_address; + if (data->window_size >=1) + retval = LONG2NUM(data->window[0]); else retval = Qnil; - semian_window_data_unlock(self); + semian_shm_object_unlock(self); return retval; } static VALUE -semian_window_data_array_last(VALUE self) +semian_sliding_window_last(VALUE self) { - semian_window_data *ptr; - TypedData_Get_Struct(self, semian_window_data, &semian_window_data_type, ptr); + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - semian_window_data_check_and_resize_if_needed(self); - if (!semian_window_data_lock(self)) + semian_shm_object_check_and_resize_if_needed(self); + if (!semian_shm_object_lock(self)) return Qnil; VALUE retval; - if (ptr->shm_address->window_length > 0) - retval = LONG2NUM(ptr->shm_address->window[ptr->shm_address->window_length-1]); + semian_sliding_window *data = ptr->shm_address; + if (data->window_size > 0) + retval = LONG2NUM(data->window[data->window_size-1]); else retval = Qnil; - semian_window_data_unlock(self); + semian_shm_object_unlock(self); return retval; } static VALUE -semian_window_data_is_shared(VALUE self) -{ - return Qtrue; +semian_sliding_window_resize_to(VALUE self, VALUE size) { + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + + if (TYPE(size) != T_FIXNUM && TYPE(size) != T_FLOAT) + return Qnil; + if (NUM2INT(size) <= 0) + rb_raise(rb_eArgError, "size must be larger than 0"); + + if (!semian_shm_object_lock(self)) + return Qnil; + + semian_sliding_window *data_copy = NULL; + size_t byte_size=0; + int prev_mem_attach_count = 0; + if (-1 != ptr->shmid && (void *)-1 != ptr->shm_address) { + data_copy = malloc(ptr->byte_size); + memcpy(data_copy,ptr->shm_address,ptr->byte_size); + byte_size = ptr->byte_size; + struct shmid_ds shm_info; + if (-1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)){ + prev_mem_attach_count = shm_info.shm_nattch; + } + } + + semian_shm_object_delete_memory_inner(ptr, 1, self, NULL); + ptr->byte_size = 2*sizeof(int) + NUM2INT(size) * sizeof(long); + semian_shm_object_unlock(self); + + semian_shm_object_acquire_memory(self, LONG2FIX(ptr->permissions), Qtrue); + if (!semian_shm_object_lock(self)) + return Qnil; + ptr->object_init_fn(ptr->byte_size, ptr->shm_address, data_copy, byte_size, prev_mem_attach_count); + semian_shm_object_unlock(self); + + return self; } void -Init_semian_sliding_window (void) { - +Init_semian_sliding_window (void) +{ VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); - - VALUE cCircuitBreakerSharedData = rb_const_get(cSemianModule, rb_intern("SlidingWindow")); - - rb_define_alloc_func(cCircuitBreakerSharedData, semian_window_data_alloc); - rb_define_method(cCircuitBreakerSharedData, "_initialize", semian_window_data_init, 3); - rb_define_method(cCircuitBreakerSharedData, "_destroy", semian_window_data_destroy, 0); - //rb_define_method(cCircuitBreakerSharedData, "acquire_semaphore", semian_window_data_acquire_semaphore, 1); - //rb_define_method(cCircuitBreakerSharedData, "delete_semaphore", semian_window_data_delete_semaphore, 0); - //rb_define_method(cCircuitBreakerSharedData, "lock", semian_window_data_lock, 0); - //rb_define_method(cCircuitBreakerSharedData, "unlock", semian_window_data_unlock, 0); - rb_define_method(cCircuitBreakerSharedData, "acquire_memory", semian_window_data_acquire_memory, 2); - rb_define_method(cCircuitBreakerSharedData, "delete_memory", semian_window_data_delete_memory, 0); - rb_define_method(cCircuitBreakerSharedData, "max_window_size", semian_window_data_max_window_size, 0); - - rb_define_method(cCircuitBreakerSharedData, "semid", semian_window_data_semid, 0); - rb_define_method(cCircuitBreakerSharedData, "shmid", semian_window_data_shmid, 0); - rb_define_method(cCircuitBreakerSharedData, "successes", semian_window_data_get_counter, 0); - rb_define_method(cCircuitBreakerSharedData, "successes=", semian_window_data_set_counter, 1); - - rb_define_method(cCircuitBreakerSharedData, "size", semian_window_data_array_length, 0); - rb_define_method(cCircuitBreakerSharedData, "count", semian_window_data_array_length, 0); - rb_define_method(cCircuitBreakerSharedData, "<<", semian_window_data_set_push_back, 1); - rb_define_method(cCircuitBreakerSharedData, "push", semian_window_data_set_push_back, 1); - rb_define_method(cCircuitBreakerSharedData, "pop", semian_window_data_set_pop_back, 0); - rb_define_method(cCircuitBreakerSharedData, "shift", semian_window_data_set_pop_front, 0); - rb_define_method(cCircuitBreakerSharedData, "unshift", semian_window_data_set_push_front, 1); - rb_define_method(cCircuitBreakerSharedData, "clear", semian_window_data_array_clear, 0); - rb_define_method(cCircuitBreakerSharedData, "first", semian_window_data_array_first, 0); - rb_define_method(cCircuitBreakerSharedData, "last", semian_window_data_array_last, 0); - - rb_define_singleton_method(cCircuitBreakerSharedData, "shared?", semian_window_data_is_shared, 0); - - eInternal = rb_const_get(cSemianModule, rb_intern("InternalError")); - eSyscall = rb_const_get(cSemianModule, rb_intern("SyscallError")); - eTimeout = rb_const_get(cSemianModule, rb_intern("TimeoutError")); - - decrement.sem_num = kCBIndexTicketLock; - decrement.sem_op = -1; - decrement.sem_flg = SEM_UNDO; - - increment.sem_num = kCBIndexTicketLock; - increment.sem_op = 1; - increment.sem_flg = SEM_UNDO; - - /* Maximum number of tickets available on this system. */ - rb_const_get(cSemianModule, rb_intern("MAX_TICKETS")); + VALUE cSlidingWindow = rb_const_get(cSemianModule, rb_intern("SlidingWindow")); + + rb_define_method(cSlidingWindow, "bind_init_fn", semian_sliding_window_bind_init_fn_wrapper, 0); + rb_define_method(cSlidingWindow, "size", semian_sliding_window_size, 0); + rb_define_method(cSlidingWindow, "max_size", semian_sliding_window_max_size, 0); + rb_define_method(cSlidingWindow, "resize_to", semian_sliding_window_resize_to, 1); + rb_define_method(cSlidingWindow, "<<", semian_sliding_window_push_back, 1); + rb_define_method(cSlidingWindow, "push", semian_sliding_window_push_back, 1); + rb_define_method(cSlidingWindow, "pop", semian_sliding_window_pop_back, 0); + rb_define_method(cSlidingWindow, "shift", semian_sliding_window_pop_front, 0); + rb_define_method(cSlidingWindow, "unshift", semian_sliding_window_push_front, 1); + rb_define_method(cSlidingWindow, "clear", semian_sliding_window_clear, 0); + rb_define_method(cSlidingWindow, "first", semian_sliding_window_first, 0); + rb_define_method(cSlidingWindow, "last", semian_sliding_window_last, 0); } diff --git a/lib/semian.rb b/lib/semian.rb index 76b81d73..793849e0 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -156,7 +156,10 @@ def resources require 'semian/protected_resource' require 'semian/unprotected_resource' require 'semian/platform' +require 'semian/shared_memory_object' require 'semian/sliding_window' +require 'semian/atomic_integer' +require 'semian/atomic_enum' if Semian.sysv_semaphores_supported? && Semian.semaphores_enabled? require 'semian/semian' else diff --git a/lib/semian/atomic_enum.rb b/lib/semian/atomic_enum.rb new file mode 100644 index 00000000..4c44b568 --- /dev/null +++ b/lib/semian/atomic_enum.rb @@ -0,0 +1,28 @@ +module Semian + class AtomicEnum < AtomicInteger + undef :increase_by + + def initialize(name, permissions, symbol_list) + super(name, permissions) + + # Assume symbol_list[0] is mapped to 0 + # Cannot just use #object_id since #object_id for symbols are different for every run + # For now, implement a C-style enum type backed by integers + + @sym_to_num = {} + symbol_list.each.with_index do |sym,idx| + @sym_to_num[sym]=idx + end + @num_to_sym = @sym_to_num.invert + end + + def value + @num_to_sym.fetch super # May raise KeyError if num is not in list (invalid enum) + end + + def value=(sym) + super (@sym_to_num.fetch(sym)) # May raise KeyError if sym is not in list (invalid enum) + end + + end +end diff --git a/lib/semian/atomic_integer.rb b/lib/semian/atomic_integer.rb new file mode 100644 index 00000000..7c91be4f --- /dev/null +++ b/lib/semian/atomic_integer.rb @@ -0,0 +1,26 @@ +module Semian + class AtomicInteger < SharedMemoryObject #:nodoc: + + def initialize(name, permissions) + data_layout = [:int] + if acquire(name, data_layout, permissions) + else + @value=0 + end + end + + def value + @value + end + + def value=(val) + @value=val + end + + def increase_by(val) + @value += val + @value + end + + end +end diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 56556044..81fc1c90 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -1,19 +1,32 @@ module Semian class CircuitBreaker - attr_reader :state def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, permissions: 0660) - @name = "#{name}_circuit_breaker" + @name = "#{name}" @success_count_threshold = success_threshold @error_count_threshold = error_threshold @error_timeout = error_timeout @exceptions = exceptions - @shared_circuit_breaker_data = ::Semian::SlidingWindow.new( - @name, + @sliding_window = ::Semian::SlidingWindow.new( + "#{name}_sliding_window", @error_count_threshold, permissions) - reset + @successes = ::Semian::AtomicInteger.new( + "#{name}_atomic_integer", + permissions) + @state = ::Semian::AtomicEnum.new( + "#{name}_atomic_enum", + permissions, + [:closed, :half_open, :open]) + # We do not need to #reset here since initializing is handled like this: + # (1) if no one is attached to the memory, zero it + # (2) otherwise, respect that data + end + + #Replacement for attr_reader :state + def state + @state.value end def acquire @@ -38,7 +51,7 @@ def request_allowed? end def mark_failed(error) - push_time(@shared_circuit_breaker_data, @error_count_threshold, duration: @error_timeout) + push_time(@sliding_window, @error_count_threshold, duration: @error_timeout) if closed? open if error_threshold_reached? elsif half_open? @@ -48,70 +61,79 @@ def mark_failed(error) def mark_success return unless half_open? - @shared_circuit_breaker_data.successes += 1 + @successes.increase_by 1 close if success_threshold_reached? end def reset - @shared_circuit_breaker_data.clear - @shared_circuit_breaker_data.successes=0 + @sliding_window.clear + @successes.value=0 + close + end + + def destroy + @sliding_window.destroy + @successes.destroy + @state.destroy end private def closed? - state == :closed + @state.value == :closed end def close log_state_transition(:closed) - @state = :closed - @shared_circuit_breaker_data.clear + @state.value = :closed + @sliding_window.clear end def open? - state == :open + @state.value == :open end def open log_state_transition(:open) - @state = :open + @state.value = :open end def half_open? - state == :half_open + @state.value == :half_open end def half_open log_state_transition(:half_open) - @state = :half_open - @shared_circuit_breaker_data.successes=0 + @state.value = :half_open + @successes.value=0 end def success_threshold_reached? - @shared_circuit_breaker_data.successes >= @success_count_threshold + @successes.value >= @success_count_threshold end def error_threshold_reached? - @shared_circuit_breaker_data.size == @error_count_threshold + @sliding_window.size == @error_count_threshold end def error_timeout_expired? - time_ms = @shared_circuit_breaker_data.last + time_ms = @sliding_window.last time_ms && (Time.at(time_ms/1000) + @error_timeout < Time.now) end def push_time(window, max_size, duration:, time: Time.now) - window.shift while window.first && Time.at(window.first/1000) + duration < time - window.shift if window.size == max_size - window << (time.to_f*1000).to_i + @sliding_window.execute_atomically do + window.shift while window.first && Time.at(window.first/1000) + duration < time + window.shift if window.size == max_size + window << (time.to_f*1000).to_i + end end def log_state_transition(new_state) - return if @state.nil? || new_state == @state + return if @state.nil? || new_state == @state.value - str = "[#{self.class.name}] State transition from #{@state} to #{new_state}." - str << " success_count=#{@shared_circuit_breaker_data.successes} error_count=#{@shared_circuit_breaker_data.size}" + str = "[#{self.class.name}] State transition from #{@state.value} to #{new_state}." + str << " success_count=#{@successes.value} error_count=#{@sliding_window.size}" str << " success_count_threshold=#{@success_count_threshold} error_count_threshold=#{@error_count_threshold}" str << " error_timeout=#{@error_timeout} error_last_at=\"#{@error_last_at}\"" Semian.logger.info(str) diff --git a/lib/semian/protected_resource.rb b/lib/semian/protected_resource.rb index 24a0e2f9..661bf70d 100644 --- a/lib/semian/protected_resource.rb +++ b/lib/semian/protected_resource.rb @@ -4,15 +4,20 @@ module Semian class ProtectedResource extend Forwardable - def_delegators :@resource, :destroy, :count, :semid, :tickets, :name - def_delegators :@circuit_breaker, :reset, :mark_failed, :request_allowed? + def_delegators :@resource, :count, :semid, :tickets, :name + def_delegators :@circuit_breaker, :reset, :mark_failed, :mark_success, :request_allowed? def initialize(resource, circuit_breaker) @resource = resource @circuit_breaker = circuit_breaker end - def acquire(timeout: nil, scope: nil, adapter: nil) + def destroy + @resource.destroy + @circuit_breaker.destroy + end + + def acquire(timeout: nil, scope: nil, adapter: nil, &block) @circuit_breaker.acquire do begin @resource.acquire(timeout: timeout) do diff --git a/lib/semian/shared_memory_object.rb b/lib/semian/shared_memory_object.rb new file mode 100644 index 00000000..c762e585 --- /dev/null +++ b/lib/semian/shared_memory_object.rb @@ -0,0 +1,46 @@ +module Semian + class SharedMemoryObject + @@type_size = {} + + def self.shared? + false + end + def self.sizeof(type) + size = (@@type_size[type.to_sym] ||= (respond_to?(:_sizeof) ? _sizeof(type.to_sym) : 0)) + if size <=0 + raise TypeError.new("#{type} is not a valid C type") + end + size + end + + def acquire(name, data_layout, permissions) + byte_size = data_layout.inject(0) { |sum,type| sum+self.class.sizeof(type) } + if respond_to?(:_acquire) and byte_size > 0 + # Will succeed only if #bind_init_fn is defined + # _acquire will call bind_init_fn to bind a function of type + # void (*object_init_fn)(size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size); + # To the C instance object + _acquire(name, byte_size, permissions) + true + else + false + end + end + + def bind_init_fn + # Concrete classes must override this in a subclass in C + raise NotImplementedError + end + + def semid + -1 + end + + def shmid + -1 + end + + + + end +end diff --git a/lib/semian/sliding_window.rb b/lib/semian/sliding_window.rb index b23ea337..51ba42b3 100644 --- a/lib/semian/sliding_window.rb +++ b/lib/semian/sliding_window.rb @@ -1,11 +1,12 @@ module Semian - class SlidingWindow #:nodoc: - def initialize(name, max_window_size, permissions) - if respond_to?(:_initialize) - _initialize(name, max_window_size, permissions) + class SlidingWindow < SharedMemoryObject #:nodoc: + + def initialize(name, max_size, permissions) + data_layout=[:int, :int].concat(Array.new(max_size,:long)) + if acquire(name, data_layout, permissions) + else - @successes = 0 - @max_window_size = max_window_size + @max_size = max_size @window = [] end end @@ -17,28 +18,17 @@ def initialize(name, max_window_size, permissions) # in store, like this: if @max_window_size = 4, current time is 10, @window =[5,7,9,10]. # Another push of (11) at 11 sec would make @window [7,9,10,11], popping off 5. - def self.shared? - false - end - - def max_window_size - @max_window_size - end - - def successes - @successes + def max_size + @max_size end - def successes=(num) - @successes=num - end - - def semid - -1 - end - - def shmid - -1 + def resize_to(size) + if 1 <= size + @max_size=size + @window.shift while @window.size >= @max_size + else + throw ArgumentError.new("size must be larger than 0") + end end def size @@ -50,7 +40,7 @@ def << (time_ms) end def push(time_ms) - @window.shift while @window.size >= @max_window_size + @window.shift while @window.size >= @max_size @window << time_ms end @@ -63,7 +53,7 @@ def shift end def unshift (time_ms) - @window.pop while @window.size >= @max_window_size + @window.pop while @window.size >= @max_size @window.unshift(time_ms) end diff --git a/lib/semian/unprotected_resource.rb b/lib/semian/unprotected_resource.rb index a8c18599..19843357 100644 --- a/lib/semian/unprotected_resource.rb +++ b/lib/semian/unprotected_resource.rb @@ -36,5 +36,9 @@ def request_allowed? def mark_failed(_error) end + + def mark_success + end + end end diff --git a/test/atomic_enum_test.rb b/test/atomic_enum_test.rb new file mode 100644 index 00000000..27bafe44 --- /dev/null +++ b/test/atomic_enum_test.rb @@ -0,0 +1,28 @@ +require 'test_helper' + +class TestAtomicEnum < MiniTest::Unit::TestCase + def setup + + @enum = Semian::AtomicEnum.new("TestAtomicEnum",0660, [:one, :two, :three]) + end + + def test_memory_is_shared + return if !Semian::AtomicEnum.shared? + assert_equal :one,@enum.value + @enum.value= :three + + enum_2 = Semian::AtomicEnum.new("TestAtomicEnum",0660, [:one, :two, :three]) + assert_equal :three,enum_2.value + end + + def test_will_throw_error_when_invalid_symbol_given + assert_raises KeyError do + @enum.value = :four + end + end + + def teardown + @enum.destroy + end + +end diff --git a/test/atomic_integer_test.rb b/test/atomic_integer_test.rb new file mode 100644 index 00000000..0504aa39 --- /dev/null +++ b/test/atomic_integer_test.rb @@ -0,0 +1,67 @@ +require 'test_helper' + +class TestAtomicInteger < MiniTest::Unit::TestCase + def setup + @successes = Semian::AtomicInteger.new("TestAtomicInteger",0660) + @successes.value=0 + end + + def test_memory_is_shared + return if !Semian::AtomicInteger.shared? + successes_2 = Semian::AtomicInteger.new("TestAtomicInteger",0660) + successes_2.value=100 + assert_equal 100, @successes.value + @successes.value=200 + assert_equal 200, successes_2.value + @successes.value = 0 + assert_equal 0, successes_2.value + end + + def test_memory_not_reset_when_at_least_one_worker_using_it + return if !Semian::AtomicInteger.shared? + + @successes.value = 109 + successes_2 = Semian::AtomicInteger.new("TestAtomicInteger",0660) + pid = fork { + successes_3 = Semian::AtomicInteger.new("TestAtomicInteger",0660) + assert_equal 109,successes_3.value + sleep + } + sleep 1 + Process.kill("KILL", pid) + Process.waitall + pid = fork { + successes_3 = Semian::AtomicInteger.new("TestAtomicInteger",0660) + assert_equal 109,successes_3.value + } + Process.waitall + end + + def test_execute_atomically_actually_is_atomic + Timeout::timeout(1) do #assure dont hang + @successes.value=100 + assert_equal 100,@successes.value + end + pids = [] + 5.times { + pids << fork { + successes_2 = Semian::AtomicInteger.new("TestAtomicInteger",0660) + successes_2.execute_atomically { + successes_2.value+=1 + sleep 1 + } + } + } + sleep 1 + pids.each { |pid| Process.kill("KILL", pid) } + assert (@successes.value < 105) + + Process.waitall + end + + + def teardown + @successes.destroy + end + +end diff --git a/test/circuit_breaker_test.rb b/test/circuit_breaker_test.rb index 5fcb904e..1f1e80e3 100644 --- a/test/circuit_breaker_test.rb +++ b/test/circuit_breaker_test.rb @@ -86,6 +86,58 @@ def test_sparse_errors_dont_open_circuit Semian.destroy(:three) end + def test_shared_error_threshold_between_workers_to_open + Semian.destroy(:testing) rescue nil + Semian.register(:testing, tickets: 1, exceptions: [SomeError], error_threshold: 10, error_timeout: 5, success_threshold: 4) + @resource = Semian[:testing] + 10.times { + pid = fork { + @resource.mark_failed SomeError + } + } + Process.waitall + assert_circuit_opened + end + + + def test_shared_success_threshold_between_workers_to_close + test_shared_error_threshold_between_workers_to_open + Timecop.travel(6) + @resource = Semian[:testing] + 5.times { + pid = fork { + @resource.mark_success + } + } + Process.waitall + assert_circuit_closed + end + + def test_shared_fresh_worker_killed_should_not_reset_circuit_breaker_data + # Won't reset if at least one worker is still attached to it. + + Semian.destroy(:testing) rescue nil + Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) + @resource = Semian[:unique_res] + + pid = fork { + Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) + resource_inner = Semian[:unique_res] + open_circuit! resource_inner + sleep + } + sleep 1 + Process.kill("KILL", pid) + Process.waitall + pid = fork { + Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) + resource_inner = Semian[:unique_res] + assert_circuit_opened + } + + Process.waitall + end + private def open_circuit!(resource = @resource) diff --git a/test/sliding_window_test.rb b/test/sliding_window_test.rb new file mode 100644 index 00000000..1f2b56e5 --- /dev/null +++ b/test/sliding_window_test.rb @@ -0,0 +1,157 @@ +require 'test_helper' + +class TestSlidingWindow < MiniTest::Unit::TestCase + def setup + @sliding_window = Semian::SlidingWindow.new("TestSlidingWindow",6,0660) + @sliding_window.clear + end + + def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it + Timeout::timeout(1) do #assure dont hang + @sliding_window<<100 + assert_equal 100,@sliding_window.first + end + + pid = fork { + sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow",6,0660) + sliding_window_2.execute_atomically { + sleep + } + } + + sleep 1 + Process.kill("KILL", pid) + Process.waitall + + Timeout::timeout(1) do #assure dont hang + @sliding_window<<100 + assert_equal 100,@sliding_window.first + end + + end + + def test_sliding_window_memory_is_actually_shared + return if !Semian::SlidingWindow.shared? + + assert_equal 0, @sliding_window.size + sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow",6,0660) + assert_equal 0, sliding_window_2.size + + large_number = (Time.now.to_f*1000).to_i + @sliding_window << large_number + assert_equal large_number, @sliding_window.first + assert_equal large_number, sliding_window_2.first + assert_equal large_number, @sliding_window.last + assert_equal large_number, sliding_window_2.last + sliding_window_2<<6<<4<<3<<2 + assert_equal 2, @sliding_window.last + assert_equal 2, sliding_window_2.last + assert_equal 5, @sliding_window.size + + @sliding_window.clear + assert_equal 0, @sliding_window.size + assert_equal 0, sliding_window_2.size + end + + def test_sliding_window_edge_falloff + + assert_equal 0, @sliding_window.size + + @sliding_window <<1<<2<<3<<4<<5<<6<<7 + assert_equal 6, @sliding_window.size + assert_equal 2, @sliding_window.first + assert_equal 7, @sliding_window.last + + @sliding_window.shift + assert_equal 3, @sliding_window.first + assert_equal 7, @sliding_window.last + @sliding_window.clear + end + + def test_restarting_worker_should_not_reset_queue + return if !Semian::SlidingWindow.shared? + @sliding_window <<10<<20<<30 + sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow", 6, 0660) + assert_equal 3, @sliding_window.size + assert_equal 10, @sliding_window.first + sliding_window_2.pop + assert_equal 20, @sliding_window.last + + sliding_window_3 = Semian::SlidingWindow.new("TestSlidingWindow", 6, 0660) + assert_equal 2, @sliding_window.size + assert_equal 10, @sliding_window.first + assert_equal 20, @sliding_window.last + sliding_window_3.pop + assert_equal 1, @sliding_window.size + assert_equal 10, @sliding_window.last + assert_equal 10, sliding_window_2.last + @sliding_window.clear + end + + def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down + return if !Semian::SlidingWindow.shared? + # Test explicit resizing, and resizing through making new memory associations + + sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow", 4, 0660) + sliding_window_2<<80<<90<<100<<110<<120 + assert_equal 4, @sliding_window.max_size + assert_equal 4, @sliding_window.size + assert_equal 4, sliding_window_2.max_size + assert_equal 4, sliding_window_2.size + assert_equal 90, @sliding_window.first + assert_equal 120, @sliding_window.last + + sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow", 3, 0660) + assert_equal 100, @sliding_window.first + assert_equal 120, @sliding_window.last + assert_equal 100, sliding_window_2.first + assert_equal 120, sliding_window_2.last + + @sliding_window.resize_to 2 + assert_equal 110, @sliding_window.first + assert_equal 120, @sliding_window.last + assert_equal 110, sliding_window_2.first + assert_equal 120, sliding_window_2.last + + sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow", 4, 0660) + assert_equal 4, @sliding_window.max_size + assert_equal 4, sliding_window_2.max_size + assert_equal 2, @sliding_window.size + assert_equal 2, sliding_window_2.size + + @sliding_window.resize_to 6 + assert_equal 6, @sliding_window.max_size + assert_equal 2, @sliding_window.size + assert_equal 110, @sliding_window.first + assert_equal 120, @sliding_window.last + assert_equal 110, sliding_window_2.first + assert_equal 120, sliding_window_2.last + + sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow", 2, 0660) + assert_equal 110, @sliding_window.first + assert_equal 120, @sliding_window.last + assert_equal 110, sliding_window_2.first + assert_equal 120, sliding_window_2.last + + @sliding_window.resize_to 4 + assert_equal 4, @sliding_window.max_size + assert_equal 4, sliding_window_2.max_size + assert_equal 2, @sliding_window.size + assert_equal 2, sliding_window_2.size + + sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow", 6, 0660) + assert_equal 6, @sliding_window.max_size + assert_equal 2, @sliding_window.size + assert_equal 110, @sliding_window.first + assert_equal 120, @sliding_window.last + assert_equal 110, sliding_window_2.first + assert_equal 120, sliding_window_2.last + + sliding_window_2.clear + end + + def teardown + @sliding_window.destroy + end + +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 6209a0ec..3c34f825 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -6,6 +6,7 @@ require 'timecop' require 'tempfile' require 'fileutils' +require 'timeout' require 'helpers/background_helper' From e3bba169bb51a26ef1cb39be032b0a34206a8f6b Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Mon, 2 Nov 2015 18:50:57 +0000 Subject: [PATCH 06/23] Separate out SysV and non-SysV classes, use #increment, fix style for Rubocop, added correctness testing with FakeSysV classes, introduce Semian.extension_loaded --- ext/semian/semian_atomic_integer.c | 2 +- ext/semian/semian_shared_memory_object.c | 8 +- ext/semian/semian_shared_memory_object.h | 2 - ext/semian/semian_sliding_window.c | 2 +- lib/semian.rb | 6 +- lib/semian/atomic_enum.rb | 41 ++-- lib/semian/atomic_integer.rb | 26 +-- lib/semian/circuit_breaker.rb | 35 +-- lib/semian/protected_resource.rb | 5 +- lib/semian/shared_memory_object.rb | 60 ++--- lib/semian/sliding_window.rb | 67 ++---- lib/semian/unprotected_resource.rb | 3 + test/atomic_enum_test.rb | 14 +- test/atomic_integer_test.rb | 129 +++++++---- test/circuit_breaker_test.rb | 46 ++-- test/sliding_window_test.rb | 267 ++++++++++++----------- 16 files changed, 392 insertions(+), 321 deletions(-) diff --git a/ext/semian/semian_atomic_integer.c b/ext/semian/semian_atomic_integer.c index 37052a76..376664dd 100644 --- a/ext/semian/semian_atomic_integer.c +++ b/ext/semian/semian_atomic_integer.c @@ -99,7 +99,7 @@ Init_semian_atomic_integer (void) { // Bind methods to AtomicInteger VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); - VALUE cAtomicInteger = rb_const_get(cSemianModule, rb_intern("AtomicInteger")); + VALUE cAtomicInteger = rb_const_get(cSemianModule, rb_intern("SysVAtomicInteger")); rb_define_method(cAtomicInteger, "bind_init_fn", semian_atomic_integer_bind_init_fn_wrapper, 0); rb_define_method(cAtomicInteger, "value", semian_atomic_integer_get_value, 0); diff --git a/ext/semian/semian_shared_memory_object.c b/ext/semian/semian_shared_memory_object.c index 813e4246..61db6124 100644 --- a/ext/semian/semian_shared_memory_object.c +++ b/ext/semian/semian_shared_memory_object.c @@ -67,11 +67,6 @@ semian_shm_object_alloc(VALUE klass) * Implementations */ -VALUE -semian_shm_object_is_shared(VALUE klass) -{ - return Qtrue; -} VALUE semian_shm_object_sizeof(VALUE klass, VALUE type) @@ -603,7 +598,7 @@ Init_semian_shm_object (void) { rb_define_alloc_func(cSharedMemoryObject, semian_shm_object_alloc); rb_define_method(cSharedMemoryObject, "_acquire", semian_shm_object_acquire, 3); - rb_define_method(cSharedMemoryObject, "destroy", semian_shm_object_destroy, 0); + rb_define_method(cSharedMemoryObject, "_destroy", semian_shm_object_destroy, 0); rb_define_method(cSharedMemoryObject, "lock", semian_shm_object_lock, 0); rb_define_method(cSharedMemoryObject, "unlock", semian_shm_object_unlock, 0); rb_define_method(cSharedMemoryObject, "byte_size", semian_shm_object_byte_size, 0); @@ -612,7 +607,6 @@ Init_semian_shm_object (void) { rb_define_method(cSharedMemoryObject, "shmid", semian_shm_object_shmid, 0); rb_define_method(cSharedMemoryObject, "execute_atomically", semian_shm_object_execute_atomically, 0); - rb_define_singleton_method(cSharedMemoryObject, "shared?", semian_shm_object_is_shared, 0); rb_define_singleton_method(cSharedMemoryObject, "_sizeof", semian_shm_object_sizeof, 1); decrement.sem_num = kSHMIndexTicketLock; diff --git a/ext/semian/semian_shared_memory_object.h b/ext/semian/semian_shared_memory_object.h index 9416cabb..5f26ce5d 100644 --- a/ext/semian/semian_shared_memory_object.h +++ b/ext/semian/semian_shared_memory_object.h @@ -19,7 +19,6 @@ semian_shm_object_type; * Headers */ -VALUE semian_shm_object_is_shared(VALUE klass); VALUE semian_shm_object_sizeof(VALUE klass, VALUE type); VALUE semian_shm_object_acquire(VALUE self, VALUE id, VALUE byte_size, VALUE permissions); @@ -34,6 +33,5 @@ void semian_shm_object_delete_memory_inner (semian_shm_object *ptr, int should_u VALUE semian_shm_object_delete_memory (VALUE self); VALUE semian_shm_object_check_and_resize_if_needed (VALUE self); - void Init_semian_atomic_integer (void); void Init_semian_sliding_window (void); diff --git a/ext/semian/semian_sliding_window.c b/ext/semian/semian_sliding_window.c index 62389ef6..e03b1dbd 100644 --- a/ext/semian/semian_sliding_window.c +++ b/ext/semian/semian_sliding_window.c @@ -295,7 +295,7 @@ void Init_semian_sliding_window (void) { VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); - VALUE cSlidingWindow = rb_const_get(cSemianModule, rb_intern("SlidingWindow")); + VALUE cSlidingWindow = rb_const_get(cSemianModule, rb_intern("SysVSlidingWindow")); rb_define_method(cSlidingWindow, "bind_init_fn", semian_sliding_window_bind_init_fn_wrapper, 0); rb_define_method(cSlidingWindow, "size", semian_sliding_window_size, 0); diff --git a/lib/semian.rb b/lib/semian.rb index 793849e0..f0b88052 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -94,6 +94,7 @@ def to_s end attr_accessor :logger + attr_accessor :extension_loaded self.logger = Logger.new(STDERR) @@ -124,7 +125,7 @@ def register(name, tickets:, permissions: 0660, timeout: 0, error_threshold:, er error_threshold: error_threshold, error_timeout: error_timeout, exceptions: Array(exceptions) + [::Semian::BaseError], - permissions: permissions + permissions: permissions, ) resource = Resource.new(name, tickets: tickets, permissions: permissions, timeout: timeout) resources[name] = ProtectedResource.new(resource, circuit_breaker) @@ -160,7 +161,8 @@ def resources require 'semian/sliding_window' require 'semian/atomic_integer' require 'semian/atomic_enum' -if Semian.sysv_semaphores_supported? && Semian.semaphores_enabled? +Semian.extension_loaded = Semian.sysv_semaphores_supported? && Semian.semaphores_enabled? +if Semian.extension_loaded require 'semian/semian' else Semian::MAX_TICKETS = 0 diff --git a/lib/semian/atomic_enum.rb b/lib/semian/atomic_enum.rb index 4c44b568..fe3931bb 100644 --- a/lib/semian/atomic_enum.rb +++ b/lib/semian/atomic_enum.rb @@ -1,28 +1,41 @@ module Semian - class AtomicEnum < AtomicInteger - undef :increase_by - + module AtomicEnumSharedImplementation def initialize(name, permissions, symbol_list) super(name, permissions) + initialize_lookup(symbol_list) + end + + def value + @num_to_sym.fetch(super) + end + + def value=(sym) + super(@sym_to_num.fetch(sym)) + end + + private + def initialize_lookup(symbol_list) # Assume symbol_list[0] is mapped to 0 - # Cannot just use #object_id since #object_id for symbols are different for every run + # Cannot just use #object_id since #object_id for symbols is different in every run # For now, implement a C-style enum type backed by integers @sym_to_num = {} - symbol_list.each.with_index do |sym,idx| - @sym_to_num[sym]=idx + symbol_list.each.with_index do |sym, idx| + @sym_to_num[sym] = idx end @num_to_sym = @sym_to_num.invert end + end - def value - @num_to_sym.fetch super # May raise KeyError if num is not in list (invalid enum) - end - - def value=(sym) - super (@sym_to_num.fetch(sym)) # May raise KeyError if sym is not in list (invalid enum) - end - + class AtomicEnum < AtomicInteger #:nodoc: + undef :increment_by + undef :increment + include AtomicEnumSharedImplementation + end + class SysVAtomicEnum < SysVAtomicInteger #:nodoc: + undef :increment_by + undef :increment + include AtomicEnumSharedImplementation end end diff --git a/lib/semian/atomic_integer.rb b/lib/semian/atomic_integer.rb index 7c91be4f..fe085d6c 100644 --- a/lib/semian/atomic_integer.rb +++ b/lib/semian/atomic_integer.rb @@ -1,26 +1,24 @@ module Semian class AtomicInteger < SharedMemoryObject #:nodoc: + attr_accessor :value - def initialize(name, permissions) - data_layout = [:int] - if acquire(name, data_layout, permissions) - else - @value=0 - end + def initialize(_name, _permissions) + @value = 0 end - def value - @value + def increment_by(val) + self.value += val end - def value=(val) - @value=val + def increment + increment_by(1) end + end - def increase_by(val) - @value += val - @value + class SysVAtomicInteger < AtomicInteger #:nodoc: + def initialize(name, permissions) + data_layout = [:int] + super unless acquire_memory_object(name, data_layout, permissions) end - end end diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 81fc1c90..734c9889 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -1,30 +1,33 @@ module Semian - class CircuitBreaker - + class CircuitBreaker #:nodoc: def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, permissions: 0660) - @name = "#{name}" + @name = name.to_s @success_count_threshold = success_threshold @error_count_threshold = error_threshold @error_timeout = error_timeout @exceptions = exceptions - @sliding_window = ::Semian::SlidingWindow.new( + @sliding_window = ::Semian::SysVSlidingWindow.new( "#{name}_sliding_window", @error_count_threshold, permissions) - @successes = ::Semian::AtomicInteger.new( + @successes = ::Semian::SysVAtomicInteger.new( "#{name}_atomic_integer", permissions) - @state = ::Semian::AtomicEnum.new( + @state = ::Semian::SysVAtomicEnum.new( "#{name}_atomic_enum", permissions, [:closed, :half_open, :open]) # We do not need to #reset here since initializing is handled like this: + # (0) if data is not shared, then it's zeroed already # (1) if no one is attached to the memory, zero it - # (2) otherwise, respect that data + # (2) otherwise, keep the data + end + + def shared? + @sliding_window.shared? && @successes.shared? && @state.shared? end - #Replacement for attr_reader :state def state @state.value end @@ -50,7 +53,7 @@ def request_allowed? !open? end - def mark_failed(error) + def mark_failed(_error) push_time(@sliding_window, @error_count_threshold, duration: @error_timeout) if closed? open if error_threshold_reached? @@ -61,13 +64,13 @@ def mark_failed(error) def mark_success return unless half_open? - @successes.increase_by 1 + @successes.increment close if success_threshold_reached? end def reset @sliding_window.clear - @successes.value=0 + @successes.value = 0 close end @@ -105,7 +108,7 @@ def half_open? def half_open log_state_transition(:half_open) @state.value = :half_open - @successes.value=0 + @successes.value = 0 end def success_threshold_reached? @@ -118,14 +121,14 @@ def error_threshold_reached? def error_timeout_expired? time_ms = @sliding_window.last - time_ms && (Time.at(time_ms/1000) + @error_timeout < Time.now) + time_ms && (Time.at(time_ms / 1000) + @error_timeout < Time.now) end def push_time(window, max_size, duration:, time: Time.now) - @sliding_window.execute_atomically do - window.shift while window.first && Time.at(window.first/1000) + duration < time + @sliding_window.execute_atomically do # Store an integer amount of milliseconds since epoch + window.shift while window.first && Time.at(window.first / 1000) + duration < time window.shift if window.size == max_size - window << (time.to_f*1000).to_i + window << (time.to_f * 1000).to_i end end diff --git a/lib/semian/protected_resource.rb b/lib/semian/protected_resource.rb index 661bf70d..49736d10 100644 --- a/lib/semian/protected_resource.rb +++ b/lib/semian/protected_resource.rb @@ -4,8 +4,9 @@ module Semian class ProtectedResource extend Forwardable - def_delegators :@resource, :count, :semid, :tickets, :name + def_delegators :@resource, :destroy, :count, :semid, :tickets, :name def_delegators :@circuit_breaker, :reset, :mark_failed, :mark_success, :request_allowed? + def_delegator :@circuit_breaker, :shared?, :circuit_breaker_shared? def initialize(resource, circuit_breaker) @resource = resource @@ -17,7 +18,7 @@ def destroy @circuit_breaker.destroy end - def acquire(timeout: nil, scope: nil, adapter: nil, &block) + def acquire(timeout: nil, scope: nil, adapter: nil) @circuit_breaker.acquire do begin @resource.acquire(timeout: timeout) do diff --git a/lib/semian/shared_memory_object.rb b/lib/semian/shared_memory_object.rb index c762e585..06aeb5a8 100644 --- a/lib/semian/shared_memory_object.rb +++ b/lib/semian/shared_memory_object.rb @@ -1,37 +1,12 @@ module Semian - class SharedMemoryObject - @@type_size = {} - - def self.shared? - false - end + class SharedMemoryObject #:nodoc: + @type_size = {} def self.sizeof(type) - size = (@@type_size[type.to_sym] ||= (respond_to?(:_sizeof) ? _sizeof(type.to_sym) : 0)) - if size <=0 - raise TypeError.new("#{type} is not a valid C type") - end + size = (@type_size[type.to_sym] ||= (respond_to?(:_sizeof) ? _sizeof(type.to_sym) : 0)) + raise TypeError.new("#{type} is not a valid C type") if size <= 0 size end - def acquire(name, data_layout, permissions) - byte_size = data_layout.inject(0) { |sum,type| sum+self.class.sizeof(type) } - if respond_to?(:_acquire) and byte_size > 0 - # Will succeed only if #bind_init_fn is defined - # _acquire will call bind_init_fn to bind a function of type - # void (*object_init_fn)(size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size); - # To the C instance object - _acquire(name, byte_size, permissions) - true - else - false - end - end - - def bind_init_fn - # Concrete classes must override this in a subclass in C - raise NotImplementedError - end - def semid -1 end @@ -40,7 +15,34 @@ def shmid -1 end + def execute_atomically + yield if block_given? + end + + def shared? + @using_shared_memory ||= Semian.extension_loaded && @using_shared_memory + end + def destroy + _destroy if respond_to?(:_destroy) && @using_shared_memory + end + def acquire_memory_object(name, data_layout, permissions) + return @using_shared_memory = false unless Semian.extension_loaded && respond_to?(:_acquire) + + byte_size = data_layout.inject(0) { |sum, type| sum + ::Semian::SharedMemoryObject.sizeof(type) } + raise TypeError.new("Given data layout is 0 bytes: #{data_layout.inspect}") if byte_size <= 0 + # Calls C layer to acquire/create a memory block, calling #bind_init_fn in the process, see below + _acquire(name, byte_size, permissions) + @using_shared_memory = true + end + + def bind_init_fn + # Concrete classes must implement this in a subclass in C to bind a callback function of type + # void (*object_init_fn)(size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size); + # to location ptr->object_init_fn, where ptr is a semian_shm_object* + # It is called when memory needs to be initialized or resized, possibly using previous memory + raise NotImplementedError + end end end diff --git a/lib/semian/sliding_window.rb b/lib/semian/sliding_window.rb index 51ba42b3..beb3d291 100644 --- a/lib/semian/sliding_window.rb +++ b/lib/semian/sliding_window.rb @@ -1,72 +1,51 @@ module Semian class SlidingWindow < SharedMemoryObject #:nodoc: + extend Forwardable - def initialize(name, max_size, permissions) - data_layout=[:int, :int].concat(Array.new(max_size,:long)) - if acquire(name, data_layout, permissions) + def_delegators :@window, :size, :pop, :shift, :first, :last + attr_reader :max_size - else - @max_size = max_size - @window = [] - end + def initialize(_name, max_size, _permissions) + @max_size = max_size + @window = [] end - # For anyone consulting this, the array stores an integer amount of seconds since epoch - # Use Time.at(_time_ms_ / 1000) to convert to time - - # A sliding window is a structure that keeps at most @max_window_size recent timestamps - # in store, like this: if @max_window_size = 4, current time is 10, @window =[5,7,9,10]. - # Another push of (11) at 11 sec would make @window [7,9,10,11], popping off 5. - - def max_size - @max_size - end + # A sliding window is a structure that stores the most @max_size recent timestamps + # like this: if @max_size = 4, current time is 10, @window =[5,7,9,10]. + # Another push of (11) at 11 sec would make @window [7,9,10,11], shifting off 5. def resize_to(size) - if 1 <= size - @max_size=size - @window.shift while @window.size >= @max_size - else - throw ArgumentError.new("size must be larger than 0") - end + throw ArgumentError.new('size must be larger than 0') if size < 1 + @max_size = size + @window.shift while @window.size > @max_size + self end - def size - @window.size - end - - def << (time_ms) + def <<(time_ms) push(time_ms) end def push(time_ms) @window.shift while @window.size >= @max_size @window << time_ms + self end - def pop - @window.pop - end - - def shift - @window.shift - end - - def unshift (time_ms) + def unshift(time_ms) @window.pop while @window.size >= @max_size @window.unshift(time_ms) + self end def clear - @window = []; - end - - def first - @window.first + @window = [] end + end - def last - @window.last + class SysVSlidingWindow < SlidingWindow #:nodoc: + def initialize(name, max_size, permissions) + data_layout = [:int, :int].concat(Array.new(max_size, :long)) + super unless acquire_memory_object(name, data_layout, permissions) end end end diff --git a/lib/semian/unprotected_resource.rb b/lib/semian/unprotected_resource.rb index 19843357..f98926a9 100644 --- a/lib/semian/unprotected_resource.rb +++ b/lib/semian/unprotected_resource.rb @@ -40,5 +40,8 @@ def mark_failed(_error) def mark_success end + def circuit_breaker_shared? + false + end end end diff --git a/test/atomic_enum_test.rb b/test/atomic_enum_test.rb index 27bafe44..7c82ad61 100644 --- a/test/atomic_enum_test.rb +++ b/test/atomic_enum_test.rb @@ -2,17 +2,16 @@ class TestAtomicEnum < MiniTest::Unit::TestCase def setup - - @enum = Semian::AtomicEnum.new("TestAtomicEnum",0660, [:one, :two, :three]) + @enum = Semian::SysVAtomicEnum.new('TestAtomicEnum', 0660, [:one, :two, :three]) end def test_memory_is_shared - return if !Semian::AtomicEnum.shared? - assert_equal :one,@enum.value - @enum.value= :three + return unless @enum.shared? + assert_equal :one, @enum.value + @enum.value = :three - enum_2 = Semian::AtomicEnum.new("TestAtomicEnum",0660, [:one, :two, :three]) - assert_equal :three,enum_2.value + enum_2 = Semian::SysVAtomicEnum.new('TestAtomicEnum', 0660, [:one, :two, :three]) + assert_equal :three, enum_2.value end def test_will_throw_error_when_invalid_symbol_given @@ -24,5 +23,4 @@ def test_will_throw_error_when_invalid_symbol_given def teardown @enum.destroy end - end diff --git a/test/atomic_integer_test.rb b/test/atomic_integer_test.rb index 0504aa39..7b93f5c8 100644 --- a/test/atomic_integer_test.rb +++ b/test/atomic_integer_test.rb @@ -1,67 +1,116 @@ require 'test_helper' class TestAtomicInteger < MiniTest::Unit::TestCase + class FakeSysVAtomicInteger < Semian::AtomicInteger + class << self + attr_accessor :resources + end + self.resources = {} + attr_accessor :name + def self.new(name, permissions) + obj = resources[name] ||= super + obj.name = name + obj + end + + def destroy + self.class.resources.delete(@name) + super + end + + def shared? + true + end + end + def setup - @successes = Semian::AtomicInteger.new("TestAtomicInteger",0660) - @successes.value=0 + @successes = Semian::SysVAtomicInteger.new('TestAtomicInteger', 0660) + @successes.value = 0 + end + + def test_operations + test_proc = proc do |atomic_integer| + atomic_integer.value = 0 + atomic_integer.increment_by(4) + assert_equal(4, atomic_integer.value) + atomic_integer.value = 10 + assert_equal(10, atomic_integer.value) + end + test_proc.call(@successes) + teardown + @successes = Semian::AtomicInteger.new('TestAtomicInteger', 0660) + test_proc.call(@successes) end def test_memory_is_shared - return if !Semian::AtomicInteger.shared? - successes_2 = Semian::AtomicInteger.new("TestAtomicInteger",0660) - successes_2.value=100 - assert_equal 100, @successes.value - @successes.value=200 - assert_equal 200, successes_2.value - @successes.value = 0 - assert_equal 0, successes_2.value + run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness do |klass| + next unless @successes.shared? + successes_2 = klass.new('TestAtomicInteger', 0660) + successes_2.value = 100 + assert_equal 100, @successes.value + @successes.value = 200 + assert_equal 200, successes_2.value + @successes.value = 0 + assert_equal 0, successes_2.value + end end def test_memory_not_reset_when_at_least_one_worker_using_it - return if !Semian::AtomicInteger.shared? + run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness do |klass| + next unless @successes.shared? - @successes.value = 109 - successes_2 = Semian::AtomicInteger.new("TestAtomicInteger",0660) - pid = fork { - successes_3 = Semian::AtomicInteger.new("TestAtomicInteger",0660) - assert_equal 109,successes_3.value - sleep - } - sleep 1 - Process.kill("KILL", pid) - Process.waitall - pid = fork { - successes_3 = Semian::AtomicInteger.new("TestAtomicInteger",0660) - assert_equal 109,successes_3.value - } - Process.waitall + @successes.value = 109 + successes_2 = klass.new('TestAtomicInteger', 0660) + assert_equal @successes.value, successes_2.value + pid = fork do + successes_3 = klass.new('TestAtomicInteger', 0660) + assert_equal 109, successes_3.value + sleep + end + sleep 1 + Process.kill("KILL", pid) + Process.waitall + fork do + successes_3 = klass.new('TestAtomicInteger', 0660) + assert_equal 109, successes_3.value + end + Process.waitall + end end def test_execute_atomically_actually_is_atomic - Timeout::timeout(1) do #assure dont hang - @successes.value=100 - assert_equal 100,@successes.value + Timeout.timeout(1) do # assure dont hang + @successes.value = 100 + assert_equal 100, @successes.value end pids = [] - 5.times { - pids << fork { - successes_2 = Semian::AtomicInteger.new("TestAtomicInteger",0660) - successes_2.execute_atomically { - successes_2.value+=1 + 5.times do + pids << fork do + successes_2 = Semian::SysVAtomicInteger.new('TestAtomicInteger', 0660) + successes_2.execute_atomically do + successes_2.value += 1 sleep 1 - } - } - } + end + end + end sleep 1 - pids.each { |pid| Process.kill("KILL", pid) } - assert (@successes.value < 105) + pids.each { |pid| Process.kill('KILL', pid) } + assert @successes.value < 105 Process.waitall end - def teardown @successes.destroy end + private + + def run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness + yield(Semian::SysVAtomicInteger) + teardown + # Use fake class backed by lookup table by name to make sure results are correct + @successes = FakeSysVAtomicInteger.new('TestAtomicInteger', 0660) + yield(FakeSysVAtomicInteger) + end end diff --git a/test/circuit_breaker_test.rb b/test/circuit_breaker_test.rb index 1f1e80e3..e9f9544e 100644 --- a/test/circuit_breaker_test.rb +++ b/test/circuit_breaker_test.rb @@ -87,53 +87,63 @@ def test_sparse_errors_dont_open_circuit end def test_shared_error_threshold_between_workers_to_open - Semian.destroy(:testing) rescue nil + # only valid if there's persistence + begin + Semian.destroy(:testing) + rescue + nil + end Semian.register(:testing, tickets: 1, exceptions: [SomeError], error_threshold: 10, error_timeout: 5, success_threshold: 4) @resource = Semian[:testing] - 10.times { - pid = fork { + return unless @resource.circuit_breaker_shared? + 10.times do + fork do @resource.mark_failed SomeError - } - } + end + end Process.waitall assert_circuit_opened end - def test_shared_success_threshold_between_workers_to_close + return unless @resource.circuit_breaker_shared? + test_shared_error_threshold_between_workers_to_open Timecop.travel(6) @resource = Semian[:testing] - 5.times { - pid = fork { + 5.times do + fork do @resource.mark_success - } - } + end + end Process.waitall assert_circuit_closed end def test_shared_fresh_worker_killed_should_not_reset_circuit_breaker_data # Won't reset if at least one worker is still attached to it. - - Semian.destroy(:testing) rescue nil + begin + Semian.destroy(:testing) + rescue + nil + end Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) @resource = Semian[:unique_res] + return unless @resource.circuit_breaker_shared? - pid = fork { + pid = fork do Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) resource_inner = Semian[:unique_res] open_circuit! resource_inner sleep - } + end sleep 1 - Process.kill("KILL", pid) + Process.kill('KILL', pid) Process.waitall - pid = fork { + fork do Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) - resource_inner = Semian[:unique_res] assert_circuit_opened - } + end Process.waitall end diff --git a/test/sliding_window_test.rb b/test/sliding_window_test.rb index 1f2b56e5..0300524d 100644 --- a/test/sliding_window_test.rb +++ b/test/sliding_window_test.rb @@ -1,157 +1,178 @@ require 'test_helper' class TestSlidingWindow < MiniTest::Unit::TestCase + class FakeSysVSlidingWindow < Semian::SlidingWindow + class << self + attr_accessor :resources + end + self.resources = {} + attr_accessor :name + def self.new(name, size, permissions) + obj = resources[name] ||= super + obj.name = name + obj.resize_to(size) + obj + end + + def destroy + self.class.resources.delete(@name) + super + end + + def shared? + true + end + end + def setup - @sliding_window = Semian::SlidingWindow.new("TestSlidingWindow",6,0660) + @sliding_window = Semian::SysVSlidingWindow.new('TestSlidingWindow', 6, 0660) @sliding_window.clear end def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it - Timeout::timeout(1) do #assure dont hang - @sliding_window<<100 - assert_equal 100,@sliding_window.first + Timeout.timeout(1) do # assure dont hang + @sliding_window << 100 + assert_equal 100, @sliding_window.first end - pid = fork { - sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow",6,0660) - sliding_window_2.execute_atomically { - sleep - } - } + pid = fork do + sliding_window_2 = Semian::SysVSlidingWindow.new('TestSlidingWindow', 6, 0660) + sliding_window_2.execute_atomically { sleep } + end sleep 1 - Process.kill("KILL", pid) + Process.kill('KILL', pid) Process.waitall - Timeout::timeout(1) do #assure dont hang - @sliding_window<<100 - assert_equal 100,@sliding_window.first + Timeout.timeout(1) do # assure dont hang + @sliding_window << 100 + assert_equal(100, @sliding_window.first) end + end + def test_sliding_window_edge_falloff + test_block = proc do |sliding_window| + assert_equal(0, sliding_window.size) + sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7 + assert_correct_first_and_last_and_size(sliding_window, 2, 7, 6, 6) + sliding_window.shift + assert_correct_first_and_last_and_size(sliding_window, 3, 7, 5, 6) + sliding_window.clear + end + test_block.call(@sliding_window) + teardown + @sliding_window = Semian::SlidingWindow.new('TestSlidingWindow', 6, 0660) + test_block.call(@sliding_window) end def test_sliding_window_memory_is_actually_shared - return if !Semian::SlidingWindow.shared? - - assert_equal 0, @sliding_window.size - sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow",6,0660) - assert_equal 0, sliding_window_2.size - - large_number = (Time.now.to_f*1000).to_i - @sliding_window << large_number - assert_equal large_number, @sliding_window.first - assert_equal large_number, sliding_window_2.first - assert_equal large_number, @sliding_window.last - assert_equal large_number, sliding_window_2.last - sliding_window_2<<6<<4<<3<<2 - assert_equal 2, @sliding_window.last - assert_equal 2, sliding_window_2.last - assert_equal 5, @sliding_window.size + run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness do |klass| + next unless @sliding_window.shared? + assert_equal 0, @sliding_window.size + sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) + assert_equal 0, sliding_window_2.size + + large_number = (Time.now.to_f * 1000).to_i + @sliding_window << large_number + assert_correct_first_and_last_and_size(@sliding_window, large_number, large_number, 1, 6) + assert_correct_first_and_last_and_size(sliding_window_2, large_number, large_number, 1, 6) + sliding_window_2 << 6 << 4 << 3 << 2 + assert_correct_first_and_last_and_size(@sliding_window, large_number, 2, 5, 6) + assert_correct_first_and_last_and_size(sliding_window_2, large_number, 2, 5, 6) + + @sliding_window.clear + assert_correct_first_and_last_and_size(@sliding_window, nil, nil, 0, 6) + assert_correct_first_and_last_and_size(sliding_window_2, nil, nil, 0, 6) + end + end - @sliding_window.clear - assert_equal 0, @sliding_window.size - assert_equal 0, sliding_window_2.size + def test_restarting_worker_should_not_reset_queue + run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness do |klass| + next unless @sliding_window.shared? + @sliding_window << 10 << 20 << 30 + sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) + assert_correct_first_and_last_and_size(@sliding_window, 10, 30, 3, 6) + sliding_window_2.pop + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + + sliding_window_3 = klass.new('TestSlidingWindow', 6, 0660) + assert_correct_first_and_last_and_size(@sliding_window, 10, 20, 2, 6) + sliding_window_3.pop + assert_correct_first_and_last_and_size(@sliding_window, 10, 10, 1, 6) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + @sliding_window.clear + end end - def test_sliding_window_edge_falloff + def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down + run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness do |klass| + next unless @sliding_window.shared? + # Test explicit resizing, and resizing through making new memory associations - assert_equal 0, @sliding_window.size + sliding_window_2 = klass.new('TestSlidingWindow', 4, 0660) + sliding_window_2 << 80 << 90 << 100 << 110 << 120 + assert_correct_first_and_last_and_size(@sliding_window, 90, 120, 4, 4) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - @sliding_window <<1<<2<<3<<4<<5<<6<<7 - assert_equal 6, @sliding_window.size - assert_equal 2, @sliding_window.first - assert_equal 7, @sliding_window.last + sliding_window_2 = klass.new('TestSlidingWindow', 3, 0660) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 100, 120, 3, 3) - @sliding_window.shift - assert_equal 3, @sliding_window.first - assert_equal 7, @sliding_window.last - @sliding_window.clear - end + @sliding_window.resize_to(2) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 2) - def test_restarting_worker_should_not_reset_queue - return if !Semian::SlidingWindow.shared? - @sliding_window <<10<<20<<30 - sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow", 6, 0660) - assert_equal 3, @sliding_window.size - assert_equal 10, @sliding_window.first - sliding_window_2.pop - assert_equal 20, @sliding_window.last - - sliding_window_3 = Semian::SlidingWindow.new("TestSlidingWindow", 6, 0660) - assert_equal 2, @sliding_window.size - assert_equal 10, @sliding_window.first - assert_equal 20, @sliding_window.last - sliding_window_3.pop - assert_equal 1, @sliding_window.size - assert_equal 10, @sliding_window.last - assert_equal 10, sliding_window_2.last - @sliding_window.clear - end + sliding_window_2 = klass.new('TestSlidingWindow', 4, 0660) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 4) - def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down - return if !Semian::SlidingWindow.shared? - # Test explicit resizing, and resizing through making new memory associations - - sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow", 4, 0660) - sliding_window_2<<80<<90<<100<<110<<120 - assert_equal 4, @sliding_window.max_size - assert_equal 4, @sliding_window.size - assert_equal 4, sliding_window_2.max_size - assert_equal 4, sliding_window_2.size - assert_equal 90, @sliding_window.first - assert_equal 120, @sliding_window.last - - sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow", 3, 0660) - assert_equal 100, @sliding_window.first - assert_equal 120, @sliding_window.last - assert_equal 100, sliding_window_2.first - assert_equal 120, sliding_window_2.last - - @sliding_window.resize_to 2 - assert_equal 110, @sliding_window.first - assert_equal 120, @sliding_window.last - assert_equal 110, sliding_window_2.first - assert_equal 120, sliding_window_2.last - - sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow", 4, 0660) - assert_equal 4, @sliding_window.max_size - assert_equal 4, sliding_window_2.max_size - assert_equal 2, @sliding_window.size - assert_equal 2, sliding_window_2.size - - @sliding_window.resize_to 6 - assert_equal 6, @sliding_window.max_size - assert_equal 2, @sliding_window.size - assert_equal 110, @sliding_window.first - assert_equal 120, @sliding_window.last - assert_equal 110, sliding_window_2.first - assert_equal 120, sliding_window_2.last - - sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow", 2, 0660) - assert_equal 110, @sliding_window.first - assert_equal 120, @sliding_window.last - assert_equal 110, sliding_window_2.first - assert_equal 120, sliding_window_2.last - - @sliding_window.resize_to 4 - assert_equal 4, @sliding_window.max_size - assert_equal 4, sliding_window_2.max_size - assert_equal 2, @sliding_window.size - assert_equal 2, sliding_window_2.size - - sliding_window_2 = Semian::SlidingWindow.new("TestSlidingWindow", 6, 0660) - assert_equal 6, @sliding_window.max_size - assert_equal 2, @sliding_window.size - assert_equal 110, @sliding_window.first - assert_equal 120, @sliding_window.last - assert_equal 110, sliding_window_2.first - assert_equal 120, sliding_window_2.last - - sliding_window_2.clear + @sliding_window.resize_to(6) + @sliding_window << 130 + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 110, 130, 3, 6) + + sliding_window_2 = klass.new('TestSlidingWindow', 2, 0660) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 2) + + @sliding_window.resize_to(4) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 4) + + sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 6) + + sliding_window_2.clear + end end def teardown @sliding_window.destroy end + private + + def assert_correct_first_and_last_and_size(sliding_window, first, last, size, max_size) + assert_equal(first, sliding_window.first) + assert_equal(last, sliding_window.last) + assert_equal(size, sliding_window.size) + assert_equal(max_size, sliding_window.max_size) + end + + def assert_sliding_windows_in_sync(sliding_window_1, sliding_window_2) + # it only exposes ends, size, and max_size, so can only check those + assert_equal(sliding_window_1.first, sliding_window_2.first) + assert_equal(sliding_window_1.last, sliding_window_2.last) + assert_equal(sliding_window_1.size, sliding_window_2.size) + assert_equal(sliding_window_1.max_size, sliding_window_2.max_size) + end + + def run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness + yield(Semian::SysVSlidingWindow) + teardown + # Use fake class backed by lookup table by name to make sure results are correct + @sliding_window = FakeSysVSlidingWindow.new('TestSlidingWindow', 6, 0660) + yield(FakeSysVSlidingWindow) + end end From 51762bb052269e0ba42e5fac9196299a93e151b4 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Thu, 5 Nov 2015 16:05:21 +0000 Subject: [PATCH 07/23] Enum implements Forwardable, fix bugs in SharedMemoryOBject implementation (#destroy, #shared), use #run_test_with_x_classes --- ext/semian/semian_atomic_integer.c | 11 +- ext/semian/semian_shared_memory_object.c | 2 +- lib/semian.rb | 11 +- lib/semian/atomic_enum.rb | 37 +++-- lib/semian/atomic_integer.rb | 15 +- lib/semian/circuit_breaker.rb | 29 ++-- lib/semian/protected_resource.rb | 1 - lib/semian/shared_memory_object.rb | 15 +- lib/semian/sliding_window.rb | 9 +- lib/semian/sysv_atomic_enum.rb | 8 + lib/semian/sysv_atomic_integer.rb | 8 + lib/semian/sysv_sliding_window.rb | 8 + lib/semian/unprotected_resource.rb | 4 - test/atomic_enum_test.rb | 39 +++-- test/atomic_integer_test.rb | 139 +++++------------- test/circuit_breaker_test.rb | 4 - test/helpers/class_test_helper.rb | 5 + test/sliding_window_test.rb | 179 +++++------------------ test/sysv_atomic_enum_test.rb | 54 +++++++ test/sysv_atomic_integer_test.rb | 79 ++++++++++ test/sysv_sliding_window_test.rb | 166 +++++++++++++++++++++ test/test_helper.rb | 2 + 22 files changed, 482 insertions(+), 343 deletions(-) create mode 100644 lib/semian/sysv_atomic_enum.rb create mode 100644 lib/semian/sysv_atomic_integer.rb create mode 100644 lib/semian/sysv_sliding_window.rb create mode 100644 test/helpers/class_test_helper.rb create mode 100644 test/sysv_atomic_enum_test.rb create mode 100644 test/sysv_atomic_integer_test.rb create mode 100644 test/sysv_sliding_window_test.rb diff --git a/ext/semian/semian_atomic_integer.c b/ext/semian/semian_atomic_integer.c index 376664dd..6bbb70c7 100644 --- a/ext/semian/semian_atomic_integer.c +++ b/ext/semian/semian_atomic_integer.c @@ -8,7 +8,7 @@ static void semian_atomic_integer_bind_init_fn (size_t byte_size, void *dest, vo static VALUE semian_atomic_integer_bind_init_fn_wrapper(VALUE self); static VALUE semian_atomic_integer_get_value(VALUE self); static VALUE semian_atomic_integer_set_value(VALUE self, VALUE num); -static VALUE semian_atomic_integer_increase_by(VALUE self, VALUE num); +static VALUE semian_atomic_integer_increment(int argc, VALUE *argv, VALUE self); static void semian_atomic_integer_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count) @@ -74,8 +74,13 @@ semian_atomic_integer_set_value(VALUE self, VALUE num) } static VALUE -semian_atomic_integer_increase_by(VALUE self, VALUE num) +semian_atomic_integer_increment(int argc, VALUE *argv, VALUE self) { + VALUE num; + rb_scan_args(argc, argv, "01", &num); + if (num == Qnil) + num = INT2NUM(1); + semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); @@ -104,5 +109,5 @@ Init_semian_atomic_integer (void) rb_define_method(cAtomicInteger, "bind_init_fn", semian_atomic_integer_bind_init_fn_wrapper, 0); rb_define_method(cAtomicInteger, "value", semian_atomic_integer_get_value, 0); rb_define_method(cAtomicInteger, "value=", semian_atomic_integer_set_value, 1); - rb_define_method(cAtomicInteger, "increase_by", semian_atomic_integer_increase_by, 1); + rb_define_method(cAtomicInteger, "increment", semian_atomic_integer_increment, -1); } diff --git a/ext/semian/semian_shared_memory_object.c b/ext/semian/semian_shared_memory_object.c index 61db6124..e605e066 100644 --- a/ext/semian/semian_shared_memory_object.c +++ b/ext/semian/semian_shared_memory_object.c @@ -605,7 +605,7 @@ Init_semian_shm_object (void) { rb_define_method(cSharedMemoryObject, "semid", semian_shm_object_semid, 0); rb_define_method(cSharedMemoryObject, "shmid", semian_shm_object_shmid, 0); - rb_define_method(cSharedMemoryObject, "execute_atomically", semian_shm_object_execute_atomically, 0); + rb_define_method(cSharedMemoryObject, "_execute_atomically", semian_shm_object_execute_atomically, 0); rb_define_singleton_method(cSharedMemoryObject, "_sizeof", semian_shm_object_sizeof, 1); diff --git a/lib/semian.rb b/lib/semian.rb index f0b88052..85cfa9ff 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -78,7 +78,7 @@ module Semian OpenCircuitError = Class.new(BaseError) def semaphores_enabled? - !ENV['SEMIAN_SEMAPHORES_DISABLED'] + !ENV['SEMIAN_SEMAPHORES_DISABLED'] && Semian.sysv_semaphores_supported? end module AdapterError @@ -94,7 +94,6 @@ def to_s end attr_accessor :logger - attr_accessor :extension_loaded self.logger = Logger.new(STDERR) @@ -161,8 +160,10 @@ def resources require 'semian/sliding_window' require 'semian/atomic_integer' require 'semian/atomic_enum' -Semian.extension_loaded = Semian.sysv_semaphores_supported? && Semian.semaphores_enabled? -if Semian.extension_loaded +require 'semian/sysv_sliding_window' +require 'semian/sysv_atomic_integer' +require 'semian/sysv_atomic_enum' +if Semian.semaphores_enabled? require 'semian/semian' else Semian::MAX_TICKETS = 0 @@ -170,7 +171,7 @@ def resources Semian.logger.info("Semian sysv semaphores are not supported on #{RUBY_PLATFORM} - all operations will no-op") end - unless Semian.semaphores_enabled? + if ENV['SEMIAN_SEMAPHORES_DISABLED'] Semian.logger.info("Semian semaphores are disabled, is this what you really want? - all operations will no-op") end end diff --git a/lib/semian/atomic_enum.rb b/lib/semian/atomic_enum.rb index fe3931bb..564157ba 100644 --- a/lib/semian/atomic_enum.rb +++ b/lib/semian/atomic_enum.rb @@ -1,16 +1,29 @@ +require 'forwardable' + module Semian - module AtomicEnumSharedImplementation + class AtomicEnum < SharedMemoryObject #:nodoc: + extend Forwardable + + def_delegators :@integer, :semid, :shmid, :execute_atomically, :transaction, + :shared?, :destroy, :acquire_memory_object, :bind_init_fn + private :acquire_memory_object, :bind_init_fn + def initialize(name, permissions, symbol_list) - super(name, permissions) + @integer = Semian::AtomicInteger.new(name, permissions) initialize_lookup(symbol_list) end + def increment(val = 1) + @integer.value = (@integer.value + val) % @sym_to_num.size + value + end + def value - @num_to_sym.fetch(super) + @num_to_sym.fetch(@integer.value) end def value=(sym) - super(@sym_to_num.fetch(sym)) + @integer.value = @sym_to_num.fetch(sym) end private @@ -20,22 +33,8 @@ def initialize_lookup(symbol_list) # Cannot just use #object_id since #object_id for symbols is different in every run # For now, implement a C-style enum type backed by integers - @sym_to_num = {} - symbol_list.each.with_index do |sym, idx| - @sym_to_num[sym] = idx - end + @sym_to_num = Hash[symbol_list.each_with_index.to_a] @num_to_sym = @sym_to_num.invert end end - - class AtomicEnum < AtomicInteger #:nodoc: - undef :increment_by - undef :increment - include AtomicEnumSharedImplementation - end - class SysVAtomicEnum < SysVAtomicInteger #:nodoc: - undef :increment_by - undef :increment - include AtomicEnumSharedImplementation - end end diff --git a/lib/semian/atomic_integer.rb b/lib/semian/atomic_integer.rb index fe085d6c..8ae041a6 100644 --- a/lib/semian/atomic_integer.rb +++ b/lib/semian/atomic_integer.rb @@ -6,19 +6,8 @@ def initialize(_name, _permissions) @value = 0 end - def increment_by(val) - self.value += val - end - - def increment - increment_by(1) - end - end - - class SysVAtomicInteger < AtomicInteger #:nodoc: - def initialize(name, permissions) - data_layout = [:int] - super unless acquire_memory_object(name, data_layout, permissions) + def increment(val = 1) + @value += val end end end diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 734c9889..73df2fb3 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -7,7 +7,7 @@ def initialize(name, exceptions:, success_threshold:, error_threshold:, error_ti @error_timeout = error_timeout @exceptions = exceptions - @sliding_window = ::Semian::SysVSlidingWindow.new( + @errors = ::Semian::SysVSlidingWindow.new( "#{name}_sliding_window", @error_count_threshold, permissions) @@ -24,14 +24,6 @@ def initialize(name, exceptions:, success_threshold:, error_threshold:, error_ti # (2) otherwise, keep the data end - def shared? - @sliding_window.shared? && @successes.shared? && @state.shared? - end - - def state - @state.value - end - def acquire raise OpenCircuitError unless request_allowed? @@ -54,7 +46,7 @@ def request_allowed? end def mark_failed(_error) - push_time(@sliding_window, @error_count_threshold, duration: @error_timeout) + push_time(@errors, duration: @error_timeout) if closed? open if error_threshold_reached? elsif half_open? @@ -69,13 +61,13 @@ def mark_success end def reset - @sliding_window.clear + @errors.clear @successes.value = 0 close end def destroy - @sliding_window.destroy + @errors.destroy @successes.destroy @state.destroy end @@ -89,7 +81,7 @@ def closed? def close log_state_transition(:closed) @state.value = :closed - @sliding_window.clear + @errors.clear end def open? @@ -116,18 +108,17 @@ def success_threshold_reached? end def error_threshold_reached? - @sliding_window.size == @error_count_threshold + @errors.size == @error_count_threshold end def error_timeout_expired? - time_ms = @sliding_window.last + time_ms = @errors.last time_ms && (Time.at(time_ms / 1000) + @error_timeout < Time.now) end - def push_time(window, max_size, duration:, time: Time.now) - @sliding_window.execute_atomically do # Store an integer amount of milliseconds since epoch + def push_time(window, duration:, time: Time.now) + @errors.execute_atomically do # Store an integer amount of milliseconds since epoch window.shift while window.first && Time.at(window.first / 1000) + duration < time - window.shift if window.size == max_size window << (time.to_f * 1000).to_i end end @@ -136,7 +127,7 @@ def log_state_transition(new_state) return if @state.nil? || new_state == @state.value str = "[#{self.class.name}] State transition from #{@state.value} to #{new_state}." - str << " success_count=#{@successes.value} error_count=#{@sliding_window.size}" + str << " success_count=#{@successes.value} error_count=#{@errors.size}" str << " success_count_threshold=#{@success_count_threshold} error_count_threshold=#{@error_count_threshold}" str << " error_timeout=#{@error_timeout} error_last_at=\"#{@error_last_at}\"" Semian.logger.info(str) diff --git a/lib/semian/protected_resource.rb b/lib/semian/protected_resource.rb index 49736d10..71ac8a6f 100644 --- a/lib/semian/protected_resource.rb +++ b/lib/semian/protected_resource.rb @@ -6,7 +6,6 @@ class ProtectedResource def_delegators :@resource, :destroy, :count, :semid, :tickets, :name def_delegators :@circuit_breaker, :reset, :mark_failed, :mark_success, :request_allowed? - def_delegator :@circuit_breaker, :shared?, :circuit_breaker_shared? def initialize(resource, circuit_breaker) @resource = resource diff --git a/lib/semian/shared_memory_object.rb b/lib/semian/shared_memory_object.rb index 06aeb5a8..2fe02731 100644 --- a/lib/semian/shared_memory_object.rb +++ b/lib/semian/shared_memory_object.rb @@ -15,20 +15,25 @@ def shmid -1 end - def execute_atomically + def execute_atomically(&proc) + return _execute_atomically(&proc) if respond_to?(:_execute_atomically) && @using_shared_memory yield if block_given? end - def shared? - @using_shared_memory ||= Semian.extension_loaded && @using_shared_memory - end + alias_method :transaction, :execute_atomically def destroy _destroy if respond_to?(:_destroy) && @using_shared_memory end + def shared? + @using_shared_memory ||= Semian.semaphores_enabled? && @using_shared_memory + end + + private + def acquire_memory_object(name, data_layout, permissions) - return @using_shared_memory = false unless Semian.extension_loaded && respond_to?(:_acquire) + return @using_shared_memory = false unless Semian.semaphores_enabled? && respond_to?(:_acquire) byte_size = data_layout.inject(0) { |sum, type| sum + ::Semian::SharedMemoryObject.sizeof(type) } raise TypeError.new("Given data layout is 0 bytes: #{data_layout.inspect}") if byte_size <= 0 diff --git a/lib/semian/sliding_window.rb b/lib/semian/sliding_window.rb index beb3d291..d8a5a023 100644 --- a/lib/semian/sliding_window.rb +++ b/lib/semian/sliding_window.rb @@ -15,7 +15,7 @@ def initialize(_name, max_size, _permissions) # Another push of (11) at 11 sec would make @window [7,9,10,11], shifting off 5. def resize_to(size) - throw ArgumentError.new('size must be larger than 0') if size < 1 + raise ArgumentError.new('size must be larger than 0') if size < 1 @max_size = size @window.shift while @window.size > @max_size self @@ -41,11 +41,4 @@ def clear @window = [] end end - - class SysVSlidingWindow < SlidingWindow #:nodoc: - def initialize(name, max_size, permissions) - data_layout = [:int, :int].concat(Array.new(max_size, :long)) - super unless acquire_memory_object(name, data_layout, permissions) - end - end end diff --git a/lib/semian/sysv_atomic_enum.rb b/lib/semian/sysv_atomic_enum.rb new file mode 100644 index 00000000..3d5bd88e --- /dev/null +++ b/lib/semian/sysv_atomic_enum.rb @@ -0,0 +1,8 @@ +module Semian + class SysVAtomicEnum < AtomicEnum #:nodoc: + def initialize(name, permissions, symbol_list) + @integer = Semian::SysVAtomicInteger.new(name, permissions) + initialize_lookup(symbol_list) + end + end +end diff --git a/lib/semian/sysv_atomic_integer.rb b/lib/semian/sysv_atomic_integer.rb new file mode 100644 index 00000000..9d3224b6 --- /dev/null +++ b/lib/semian/sysv_atomic_integer.rb @@ -0,0 +1,8 @@ +module Semian + class SysVAtomicInteger < AtomicInteger #:nodoc: + def initialize(name, permissions) + data_layout = [:int] + super unless acquire_memory_object(name, data_layout, permissions) + end + end +end diff --git a/lib/semian/sysv_sliding_window.rb b/lib/semian/sysv_sliding_window.rb new file mode 100644 index 00000000..93b053db --- /dev/null +++ b/lib/semian/sysv_sliding_window.rb @@ -0,0 +1,8 @@ +module Semian + class SysVSlidingWindow < SlidingWindow #:nodoc: + def initialize(name, max_size, permissions) + data_layout = [:int, :int].concat(Array.new(max_size, :long)) + super unless acquire_memory_object(name, data_layout, permissions) + end + end +end diff --git a/lib/semian/unprotected_resource.rb b/lib/semian/unprotected_resource.rb index f98926a9..3faa1bea 100644 --- a/lib/semian/unprotected_resource.rb +++ b/lib/semian/unprotected_resource.rb @@ -39,9 +39,5 @@ def mark_failed(_error) def mark_success end - - def circuit_breaker_shared? - false - end end end diff --git a/test/atomic_enum_test.rb b/test/atomic_enum_test.rb index 7c82ad61..d5362297 100644 --- a/test/atomic_enum_test.rb +++ b/test/atomic_enum_test.rb @@ -1,26 +1,35 @@ require 'test_helper' class TestAtomicEnum < MiniTest::Unit::TestCase - def setup - @enum = Semian::SysVAtomicEnum.new('TestAtomicEnum', 0660, [:one, :two, :three]) - end - - def test_memory_is_shared - return unless @enum.shared? - assert_equal :one, @enum.value - @enum.value = :three - - enum_2 = Semian::SysVAtomicEnum.new('TestAtomicEnum', 0660, [:one, :two, :three]) - assert_equal :three, enum_2.value + def test_functionality + run_test_with_atomic_enum_classes do + @enum.value = :two + assert_equal :two, @enum.value + end end def test_will_throw_error_when_invalid_symbol_given - assert_raises KeyError do - @enum.value = :four + run_test_with_atomic_enum_classes do + assert_raises KeyError do + @enum.value = :four + end end end - def teardown - @enum.destroy + private + + def atomic_enum_classes + @classes ||= [::Semian::AtomicEnum] + end + + def run_test_with_atomic_enum_classes(klasses = atomic_enum_classes) + klasses.each do |klass| + begin + @enum = klass.new('TestAtomicEnum', 0660, [:one, :two, :three]) + yield(klass) + ensure + @enum.destroy + end + end end end diff --git a/test/atomic_integer_test.rb b/test/atomic_integer_test.rb index 7b93f5c8..d5ab3bbe 100644 --- a/test/atomic_integer_test.rb +++ b/test/atomic_integer_test.rb @@ -1,116 +1,49 @@ require 'test_helper' class TestAtomicInteger < MiniTest::Unit::TestCase - class FakeSysVAtomicInteger < Semian::AtomicInteger - class << self - attr_accessor :resources - end - self.resources = {} - attr_accessor :name - def self.new(name, permissions) - obj = resources[name] ||= super - obj.name = name - obj - end - - def destroy - self.class.resources.delete(@name) - super - end - - def shared? - true - end - end - - def setup - @successes = Semian::SysVAtomicInteger.new('TestAtomicInteger', 0660) - @successes.value = 0 - end - - def test_operations - test_proc = proc do |atomic_integer| - atomic_integer.value = 0 - atomic_integer.increment_by(4) - assert_equal(4, atomic_integer.value) - atomic_integer.value = 10 - assert_equal(10, atomic_integer.value) - end - test_proc.call(@successes) - teardown - @successes = Semian::AtomicInteger.new('TestAtomicInteger', 0660) - test_proc.call(@successes) - end - - def test_memory_is_shared - run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness do |klass| - next unless @successes.shared? - successes_2 = klass.new('TestAtomicInteger', 0660) - successes_2.value = 100 - assert_equal 100, @successes.value - @successes.value = 200 - assert_equal 200, successes_2.value - @successes.value = 0 - assert_equal 0, successes_2.value + def test_access_value + run_test_with_atomic_integer_classes do + @integer.value = 0 + assert_equal(0, @integer.value) + @integer.value = 99 + assert_equal(99, @integer.value) + time_now = (Time.now).to_i + @integer.value = time_now + assert_equal(time_now, @integer.value) + @integer.value = 6 + assert_equal(6, @integer.value) + @integer.value = 6 + assert_equal(6, @integer.value) + end + end + + def test_increment + run_test_with_atomic_integer_classes do + @integer.value = 0 + @integer.increment(4) + assert_equal(4, @integer.value) + @integer.increment + assert_equal(5, @integer.value) + @integer.increment(-2) + assert_equal(3, @integer.value) end end - def test_memory_not_reset_when_at_least_one_worker_using_it - run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness do |klass| - next unless @successes.shared? + private - @successes.value = 109 - successes_2 = klass.new('TestAtomicInteger', 0660) - assert_equal @successes.value, successes_2.value - pid = fork do - successes_3 = klass.new('TestAtomicInteger', 0660) - assert_equal 109, successes_3.value - sleep - end - sleep 1 - Process.kill("KILL", pid) - Process.waitall - fork do - successes_3 = klass.new('TestAtomicInteger', 0660) - assert_equal 109, successes_3.value - end - Process.waitall - end + def atomic_integer_classes + @classes ||= [::Semian::AtomicInteger] end - def test_execute_atomically_actually_is_atomic - Timeout.timeout(1) do # assure dont hang - @successes.value = 100 - assert_equal 100, @successes.value - end - pids = [] - 5.times do - pids << fork do - successes_2 = Semian::SysVAtomicInteger.new('TestAtomicInteger', 0660) - successes_2.execute_atomically do - successes_2.value += 1 - sleep 1 - end + def run_test_with_atomic_integer_classes(klasses = atomic_integer_classes) + klasses.each do |klass| + begin + @integer = klass.new('TestAtomicInteger', 0660) + @integer.value = 0 + yield(klass) + ensure + @integer.destroy end end - sleep 1 - pids.each { |pid| Process.kill('KILL', pid) } - assert @successes.value < 105 - - Process.waitall - end - - def teardown - @successes.destroy - end - - private - - def run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness - yield(Semian::SysVAtomicInteger) - teardown - # Use fake class backed by lookup table by name to make sure results are correct - @successes = FakeSysVAtomicInteger.new('TestAtomicInteger', 0660) - yield(FakeSysVAtomicInteger) end end diff --git a/test/circuit_breaker_test.rb b/test/circuit_breaker_test.rb index e9f9544e..305c31b0 100644 --- a/test/circuit_breaker_test.rb +++ b/test/circuit_breaker_test.rb @@ -95,7 +95,6 @@ def test_shared_error_threshold_between_workers_to_open end Semian.register(:testing, tickets: 1, exceptions: [SomeError], error_threshold: 10, error_timeout: 5, success_threshold: 4) @resource = Semian[:testing] - return unless @resource.circuit_breaker_shared? 10.times do fork do @resource.mark_failed SomeError @@ -106,8 +105,6 @@ def test_shared_error_threshold_between_workers_to_open end def test_shared_success_threshold_between_workers_to_close - return unless @resource.circuit_breaker_shared? - test_shared_error_threshold_between_workers_to_open Timecop.travel(6) @resource = Semian[:testing] @@ -129,7 +126,6 @@ def test_shared_fresh_worker_killed_should_not_reset_circuit_breaker_data end Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) @resource = Semian[:unique_res] - return unless @resource.circuit_breaker_shared? pid = fork do Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) diff --git a/test/helpers/class_test_helper.rb b/test/helpers/class_test_helper.rb new file mode 100644 index 00000000..0981e520 --- /dev/null +++ b/test/helpers/class_test_helper.rb @@ -0,0 +1,5 @@ +module ClassTestHelper + def retrieve_descendants(klass) + ObjectSpace.each_object(klass.singleton_class).to_a + end +end diff --git a/test/sliding_window_test.rb b/test/sliding_window_test.rb index 0300524d..ae765702 100644 --- a/test/sliding_window_test.rb +++ b/test/sliding_window_test.rb @@ -1,158 +1,59 @@ require 'test_helper' class TestSlidingWindow < MiniTest::Unit::TestCase - class FakeSysVSlidingWindow < Semian::SlidingWindow - class << self - attr_accessor :resources - end - self.resources = {} - attr_accessor :name - def self.new(name, size, permissions) - obj = resources[name] ||= super - obj.name = name - obj.resize_to(size) - obj - end - - def destroy - self.class.resources.delete(@name) - super - end - - def shared? - true + def test_slidinv_window_push + run_test_with_sliding_window_classes do + assert_equal(0, @sliding_window.size) + @sliding_window << 1 + assert_correct_first_and_last_and_size(@sliding_window, 1, 1, 1, 6) + @sliding_window << 5 + assert_correct_first_and_last_and_size(@sliding_window, 1, 5, 2, 6) end end - def setup - @sliding_window = Semian::SysVSlidingWindow.new('TestSlidingWindow', 6, 0660) - @sliding_window.clear - end - - def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it - Timeout.timeout(1) do # assure dont hang - @sliding_window << 100 - assert_equal 100, @sliding_window.first - end - - pid = fork do - sliding_window_2 = Semian::SysVSlidingWindow.new('TestSlidingWindow', 6, 0660) - sliding_window_2.execute_atomically { sleep } - end - - sleep 1 - Process.kill('KILL', pid) - Process.waitall - - Timeout.timeout(1) do # assure dont hang - @sliding_window << 100 - assert_equal(100, @sliding_window.first) + def test_sliding_window_resize + run_test_with_sliding_window_classes do + assert_equal(0, @sliding_window.size) + @sliding_window << 1 << 2 << 3 << 4 << 5 << 6 + assert_correct_first_and_last_and_size(@sliding_window, 1, 6, 6, 6) + @sliding_window.resize_to 6 + assert_correct_first_and_last_and_size(@sliding_window, 1, 6, 6, 6) + @sliding_window.resize_to 5 + assert_correct_first_and_last_and_size(@sliding_window, 2, 6, 5, 5) + @sliding_window.resize_to 6 + assert_correct_first_and_last_and_size(@sliding_window, 2, 6, 5, 6) end end def test_sliding_window_edge_falloff - test_block = proc do |sliding_window| - assert_equal(0, sliding_window.size) - sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7 - assert_correct_first_and_last_and_size(sliding_window, 2, 7, 6, 6) - sliding_window.shift - assert_correct_first_and_last_and_size(sliding_window, 3, 7, 5, 6) - sliding_window.clear - end - test_block.call(@sliding_window) - teardown - @sliding_window = Semian::SlidingWindow.new('TestSlidingWindow', 6, 0660) - test_block.call(@sliding_window) - end - - def test_sliding_window_memory_is_actually_shared - run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness do |klass| - next unless @sliding_window.shared? - assert_equal 0, @sliding_window.size - sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) - assert_equal 0, sliding_window_2.size - - large_number = (Time.now.to_f * 1000).to_i - @sliding_window << large_number - assert_correct_first_and_last_and_size(@sliding_window, large_number, large_number, 1, 6) - assert_correct_first_and_last_and_size(sliding_window_2, large_number, large_number, 1, 6) - sliding_window_2 << 6 << 4 << 3 << 2 - assert_correct_first_and_last_and_size(@sliding_window, large_number, 2, 5, 6) - assert_correct_first_and_last_and_size(sliding_window_2, large_number, 2, 5, 6) - + run_test_with_sliding_window_classes do + assert_equal(0, @sliding_window.size) + @sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7 + assert_correct_first_and_last_and_size(@sliding_window, 2, 7, 6, 6) + @sliding_window.shift + assert_correct_first_and_last_and_size(@sliding_window, 3, 7, 5, 6) @sliding_window.clear - assert_correct_first_and_last_and_size(@sliding_window, nil, nil, 0, 6) - assert_correct_first_and_last_and_size(sliding_window_2, nil, nil, 0, 6) end end - def test_restarting_worker_should_not_reset_queue - run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness do |klass| - next unless @sliding_window.shared? - @sliding_window << 10 << 20 << 30 - sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) - assert_correct_first_and_last_and_size(@sliding_window, 10, 30, 3, 6) - sliding_window_2.pop - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + private - sliding_window_3 = klass.new('TestSlidingWindow', 6, 0660) - assert_correct_first_and_last_and_size(@sliding_window, 10, 20, 2, 6) - sliding_window_3.pop - assert_correct_first_and_last_and_size(@sliding_window, 10, 10, 1, 6) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - @sliding_window.clear - end + def sliding_window_classes + @classes ||= [::Semian::SlidingWindow] end - def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down - run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness do |klass| - next unless @sliding_window.shared? - # Test explicit resizing, and resizing through making new memory associations - - sliding_window_2 = klass.new('TestSlidingWindow', 4, 0660) - sliding_window_2 << 80 << 90 << 100 << 110 << 120 - assert_correct_first_and_last_and_size(@sliding_window, 90, 120, 4, 4) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - - sliding_window_2 = klass.new('TestSlidingWindow', 3, 0660) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 100, 120, 3, 3) - - @sliding_window.resize_to(2) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 2) - - sliding_window_2 = klass.new('TestSlidingWindow', 4, 0660) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 4) - - @sliding_window.resize_to(6) - @sliding_window << 130 - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 110, 130, 3, 6) - - sliding_window_2 = klass.new('TestSlidingWindow', 2, 0660) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 2) - - @sliding_window.resize_to(4) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 4) - - sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 6) - - sliding_window_2.clear + def run_test_with_sliding_window_classes(klasses = sliding_window_classes) + klasses.each do |klass| + begin + @sliding_window = klass.new('TestSlidingWindow', 6, 0660) + @sliding_window.clear + yield(klass) + ensure + @sliding_window.destroy + end end end - def teardown - @sliding_window.destroy - end - - private - def assert_correct_first_and_last_and_size(sliding_window, first, last, size, max_size) assert_equal(first, sliding_window.first) assert_equal(last, sliding_window.last) @@ -167,12 +68,4 @@ def assert_sliding_windows_in_sync(sliding_window_1, sliding_window_2) assert_equal(sliding_window_1.size, sliding_window_2.size) assert_equal(sliding_window_1.max_size, sliding_window_2.max_size) end - - def run_test_once_with_sysv_and_once_without_sysv_to_assert_correctness - yield(Semian::SysVSlidingWindow) - teardown - # Use fake class backed by lookup table by name to make sure results are correct - @sliding_window = FakeSysVSlidingWindow.new('TestSlidingWindow', 6, 0660) - yield(FakeSysVSlidingWindow) - end end diff --git a/test/sysv_atomic_enum_test.rb b/test/sysv_atomic_enum_test.rb new file mode 100644 index 00000000..a1e7560b --- /dev/null +++ b/test/sysv_atomic_enum_test.rb @@ -0,0 +1,54 @@ +require 'test_helper' + +class TestSysVAtomicEnum < MiniTest::Unit::TestCase + # Emulate sharedness to test correctness against real SysVAtomicEnum class + class FakeSysVAtomicEnum < Semian::AtomicEnum + class << self + attr_accessor :resources + end + self.resources = {} + attr_accessor :name + def self.new(name, _permissions, _symbol_list) + obj = resources[name] ||= super + obj.name = name + obj + end + + def destroy + self.class.resources.delete(@name) + super + end + + def shared? + true + end + end + + def test_memory_is_shared + run_test_with_atomic_enum_classes do |klass| + return unless @enum.shared? + assert_equal :one, @enum.value + @enum.value = :three + + enum_2 = klass.new('TestAtomicEnum', 0660, [:one, :two, :three]) + assert_equal :three, enum_2.value + end + end + + private + + def atomic_enum_classes + @classes ||= [::Semian::SysVAtomicEnum, FakeSysVAtomicEnum] + end + + def run_test_with_atomic_enum_classes(klasses = atomic_enum_classes) + klasses.each do |klass| + begin + @enum = klass.new('TestAtomicEnum', 0660, [:one, :two, :three]) + yield(klass) + ensure + @enum.destroy + end + end + end +end diff --git a/test/sysv_atomic_integer_test.rb b/test/sysv_atomic_integer_test.rb new file mode 100644 index 00000000..40e95488 --- /dev/null +++ b/test/sysv_atomic_integer_test.rb @@ -0,0 +1,79 @@ +require 'test_helper' + +class TestSysVAtomicInteger < MiniTest::Unit::TestCase + # Emulate sharedness to test correctness against real SysVAtomicInteger class + class FakeSysVAtomicInteger < Semian::AtomicInteger + class << self + attr_accessor :resources + end + self.resources = {} + attr_accessor :name + def self.new(name, _permissions) + obj = resources[name] ||= super + obj.name = name + obj + end + + def destroy + self.class.resources.delete(@name) + super + end + + def shared? + true + end + end + + def test_memory_is_shared + run_test_with_atomic_integer_classes do |klass| + return unless @integer.shared? + integer_2 = klass.new('TestAtomicInteger', 0660) + integer_2.value = 100 + assert_equal 100, @integer.value + @integer.value = 200 + assert_equal 200, integer_2.value + @integer.value = 0 + assert_equal 0, integer_2.value + end + end + + def test_memory_not_reset_when_at_least_one_worker_using_it + run_test_with_atomic_integer_classes do |klass| + return unless @integer.shared? + @integer.value = 109 + integer_2 = klass.new('TestAtomicInteger', 0660) + assert_equal @integer.value, integer_2.value + pid = fork do + integer_3 = klass.new('TestAtomicInteger', 0660) + assert_equal 109, integer_3.value + sleep + end + sleep 1 + Process.kill("KILL", pid) + Process.waitall + fork do + integer_3 = klass.new('TestAtomicInteger', 0660) + assert_equal 109, integer_3.value + end + Process.waitall + end + end + + private + + def atomic_integer_classes + @classes ||= [::Semian::SysVAtomicInteger, FakeSysVAtomicInteger] + end + + def run_test_with_atomic_integer_classes(klasses = atomic_integer_classes) + klasses.each do |klass| + begin + @integer = klass.new('TestAtomicInteger', 0660) + @integer.value = 0 + yield(klass) + ensure + @integer.destroy + end + end + end +end diff --git a/test/sysv_sliding_window_test.rb b/test/sysv_sliding_window_test.rb new file mode 100644 index 00000000..b49ee895 --- /dev/null +++ b/test/sysv_sliding_window_test.rb @@ -0,0 +1,166 @@ +require 'test_helper' + +class TestSlidingWindow < MiniTest::Unit::TestCase + # Emulate sharedness to test correctness against real SysVSlidingWindow class + class FakeSysVSlidingWindow < Semian::SlidingWindow + class << self + attr_accessor :resources + end + self.resources = {} + attr_accessor :name + def self.new(name, size, permissions) + obj = resources[name] ||= super + obj.name = name + obj.resize_to(size) + obj + end + + def destroy + self.class.resources.delete(@name) + super + end + + def shared? + true + end + end + + def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it + run_test_with_sliding_window_classes do |klass| + return unless @sliding_window.shared? + Timeout.timeout(1) do # assure dont hang + @sliding_window << 100 + assert_equal 100, @sliding_window.first + end + + pid = fork do + sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) + sliding_window_2.execute_atomically { sleep } + end + + sleep 1 + Process.kill('KILL', pid) + Process.waitall + + Timeout.timeout(1) do # assure dont hang + @sliding_window << 100 + assert_equal(100, @sliding_window.first) + end + end + end + + def test_sliding_window_memory_is_actually_shared + run_test_with_sliding_window_classes do |klass| + return unless @sliding_window.shared? + assert_equal 0, @sliding_window.size + sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) + assert_equal 0, sliding_window_2.size + + large_number = (Time.now.to_f * 1000).to_i + @sliding_window << large_number + assert_correct_first_and_last_and_size(@sliding_window, large_number, large_number, 1, 6) + assert_correct_first_and_last_and_size(sliding_window_2, large_number, large_number, 1, 6) + sliding_window_2 << 6 << 4 << 3 << 2 + assert_correct_first_and_last_and_size(@sliding_window, large_number, 2, 5, 6) + assert_correct_first_and_last_and_size(sliding_window_2, large_number, 2, 5, 6) + + @sliding_window.clear + assert_correct_first_and_last_and_size(@sliding_window, nil, nil, 0, 6) + assert_correct_first_and_last_and_size(sliding_window_2, nil, nil, 0, 6) + end + end + + def test_restarting_worker_should_not_reset_queue + run_test_with_sliding_window_classes do |klass| + return unless @sliding_window.shared? + @sliding_window << 10 << 20 << 30 + sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) + assert_correct_first_and_last_and_size(@sliding_window, 10, 30, 3, 6) + sliding_window_2.pop + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + + sliding_window_3 = klass.new('TestSlidingWindow', 6, 0660) + assert_correct_first_and_last_and_size(@sliding_window, 10, 20, 2, 6) + sliding_window_3.pop + assert_correct_first_and_last_and_size(@sliding_window, 10, 10, 1, 6) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + @sliding_window.clear + end + end + + def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down + run_test_with_sliding_window_classes do |klass| + return unless @sliding_window.shared? + # Test explicit resizing, and resizing through making new memory associations + + sliding_window_2 = klass.new('TestSlidingWindow', 4, 0660) + sliding_window_2 << 80 << 90 << 100 << 110 << 120 + assert_correct_first_and_last_and_size(@sliding_window, 90, 120, 4, 4) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + + sliding_window_2 = klass.new('TestSlidingWindow', 3, 0660) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 100, 120, 3, 3) + + @sliding_window.resize_to(2) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 2) + + sliding_window_2 = klass.new('TestSlidingWindow', 4, 0660) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 4) + + @sliding_window.resize_to(6) + @sliding_window << 130 + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 110, 130, 3, 6) + + sliding_window_2 = klass.new('TestSlidingWindow', 2, 0660) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 2) + + @sliding_window.resize_to(4) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 4) + + sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 6) + + sliding_window_2.clear + end + end + + private + + def shared_sliding_window_classes + @classes ||= [::Semian::SysVSlidingWindow, FakeSysVSlidingWindow] + end + + def run_test_with_sliding_window_classes(klasses = shared_sliding_window_classes) + klasses.each do |klass| + begin + @sliding_window = klass.new('TestSlidingWindow', 6, 0660) + @sliding_window.clear + yield(klass) + ensure + @sliding_window.destroy + end + end + end + + def assert_correct_first_and_last_and_size(sliding_window, first, last, size, max_size) + assert_equal(first, sliding_window.first) + assert_equal(last, sliding_window.last) + assert_equal(size, sliding_window.size) + assert_equal(max_size, sliding_window.max_size) + end + + def assert_sliding_windows_in_sync(sliding_window_1, sliding_window_2) + # it only exposes ends, size, and max_size, so can only check those + assert_equal(sliding_window_1.first, sliding_window_2.first) + assert_equal(sliding_window_1.last, sliding_window_2.last) + assert_equal(sliding_window_1.size, sliding_window_2.size) + assert_equal(sliding_window_1.max_size, sliding_window_2.max_size) + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 3c34f825..89d8ac6b 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -9,6 +9,7 @@ require 'timeout' require 'helpers/background_helper' +require 'helpers/class_test_helper' Semian.logger = Logger.new(nil) Toxiproxy.populate([ @@ -26,4 +27,5 @@ class MiniTest::Unit::TestCase include BackgroundHelper + include ClassTestHelper end From b1bb06ccd8fc533606e33c00ff7cb6d63e00220a Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Thu, 5 Nov 2015 20:34:26 +0000 Subject: [PATCH 08/23] Made #shared private, removed uses of it --- lib/semian/atomic_enum.rb | 2 +- lib/semian/shared_memory_object.rb | 4 ++-- test/sysv_atomic_enum_test.rb | 1 - test/sysv_atomic_integer_test.rb | 2 -- test/sysv_sliding_window_test.rb | 4 ---- 5 files changed, 3 insertions(+), 10 deletions(-) diff --git a/lib/semian/atomic_enum.rb b/lib/semian/atomic_enum.rb index 564157ba..f08ec5af 100644 --- a/lib/semian/atomic_enum.rb +++ b/lib/semian/atomic_enum.rb @@ -6,7 +6,7 @@ class AtomicEnum < SharedMemoryObject #:nodoc: def_delegators :@integer, :semid, :shmid, :execute_atomically, :transaction, :shared?, :destroy, :acquire_memory_object, :bind_init_fn - private :acquire_memory_object, :bind_init_fn + private :shared?, :acquire_memory_object, :bind_init_fn def initialize(name, permissions, symbol_list) @integer = Semian::AtomicInteger.new(name, permissions) diff --git a/lib/semian/shared_memory_object.rb b/lib/semian/shared_memory_object.rb index 2fe02731..6140c17e 100644 --- a/lib/semian/shared_memory_object.rb +++ b/lib/semian/shared_memory_object.rb @@ -26,12 +26,12 @@ def destroy _destroy if respond_to?(:_destroy) && @using_shared_memory end + private + def shared? @using_shared_memory ||= Semian.semaphores_enabled? && @using_shared_memory end - private - def acquire_memory_object(name, data_layout, permissions) return @using_shared_memory = false unless Semian.semaphores_enabled? && respond_to?(:_acquire) diff --git a/test/sysv_atomic_enum_test.rb b/test/sysv_atomic_enum_test.rb index a1e7560b..45696ca8 100644 --- a/test/sysv_atomic_enum_test.rb +++ b/test/sysv_atomic_enum_test.rb @@ -26,7 +26,6 @@ def shared? def test_memory_is_shared run_test_with_atomic_enum_classes do |klass| - return unless @enum.shared? assert_equal :one, @enum.value @enum.value = :three diff --git a/test/sysv_atomic_integer_test.rb b/test/sysv_atomic_integer_test.rb index 40e95488..4d08e91f 100644 --- a/test/sysv_atomic_integer_test.rb +++ b/test/sysv_atomic_integer_test.rb @@ -26,7 +26,6 @@ def shared? def test_memory_is_shared run_test_with_atomic_integer_classes do |klass| - return unless @integer.shared? integer_2 = klass.new('TestAtomicInteger', 0660) integer_2.value = 100 assert_equal 100, @integer.value @@ -39,7 +38,6 @@ def test_memory_is_shared def test_memory_not_reset_when_at_least_one_worker_using_it run_test_with_atomic_integer_classes do |klass| - return unless @integer.shared? @integer.value = 109 integer_2 = klass.new('TestAtomicInteger', 0660) assert_equal @integer.value, integer_2.value diff --git a/test/sysv_sliding_window_test.rb b/test/sysv_sliding_window_test.rb index b49ee895..5cbed2ae 100644 --- a/test/sysv_sliding_window_test.rb +++ b/test/sysv_sliding_window_test.rb @@ -27,7 +27,6 @@ def shared? def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it run_test_with_sliding_window_classes do |klass| - return unless @sliding_window.shared? Timeout.timeout(1) do # assure dont hang @sliding_window << 100 assert_equal 100, @sliding_window.first @@ -51,7 +50,6 @@ def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it def test_sliding_window_memory_is_actually_shared run_test_with_sliding_window_classes do |klass| - return unless @sliding_window.shared? assert_equal 0, @sliding_window.size sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) assert_equal 0, sliding_window_2.size @@ -72,7 +70,6 @@ def test_sliding_window_memory_is_actually_shared def test_restarting_worker_should_not_reset_queue run_test_with_sliding_window_classes do |klass| - return unless @sliding_window.shared? @sliding_window << 10 << 20 << 30 sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) assert_correct_first_and_last_and_size(@sliding_window, 10, 30, 3, 6) @@ -90,7 +87,6 @@ def test_restarting_worker_should_not_reset_queue def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down run_test_with_sliding_window_classes do |klass| - return unless @sliding_window.shared? # Test explicit resizing, and resizing through making new memory associations sliding_window_2 = klass.new('TestSlidingWindow', 4, 0660) From 56c103a00ea218847cea3fc227ccc7c96223e983 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Mon, 9 Nov 2015 20:16:00 +0000 Subject: [PATCH 09/23] Use keyword parameters for init, test by importing modules, initializer using options, mutex for non-sysV --- lib/semian.rb | 18 +++ lib/semian/atomic_enum.rb | 4 +- lib/semian/atomic_integer.rb | 13 +- lib/semian/circuit_breaker.rb | 21 ++- lib/semian/sliding_window.rb | 19 ++- lib/semian/sysv_atomic_enum.rb | 4 +- lib/semian/sysv_atomic_integer.rb | 2 +- lib/semian/sysv_sliding_window.rb | 2 +- test/atomic_enum_test.rb | 57 +++++--- test/atomic_integer_test.rb | 37 ++--- test/helpers/class_test_helper.rb | 5 - test/sliding_window_test.rb | 70 +++++---- test/sysv_atomic_enum_test.rb | 41 +++--- test/sysv_atomic_integer_test.rb | 76 +++++----- test/sysv_sliding_window_test.rb | 235 +++++++++++++++--------------- test/test_helper.rb | 2 - 16 files changed, 310 insertions(+), 296 deletions(-) delete mode 100644 test/helpers/class_test_helper.rb diff --git a/lib/semian.rb b/lib/semian.rb index 85cfa9ff..5a8a819b 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -149,6 +149,24 @@ def destroy(name) def resources @resources ||= {} end + + module ReentrantMutex + attr_reader :monitor + + def self.included(base) + def base.surround_with_mutex(*names) + names.each do |name| + m = instance_method(name) + define_method(name) do |*args, &block| + @monitor ||= Monitor.new + @monitor.synchronize do + m.bind(self).call(*args, &block) + end + end + end + end + end + end end require 'semian/resource' diff --git a/lib/semian/atomic_enum.rb b/lib/semian/atomic_enum.rb index f08ec5af..48f2fde1 100644 --- a/lib/semian/atomic_enum.rb +++ b/lib/semian/atomic_enum.rb @@ -8,8 +8,8 @@ class AtomicEnum < SharedMemoryObject #:nodoc: :shared?, :destroy, :acquire_memory_object, :bind_init_fn private :shared?, :acquire_memory_object, :bind_init_fn - def initialize(name, permissions, symbol_list) - @integer = Semian::AtomicInteger.new(name, permissions) + def initialize(symbol_list, **options) + @integer = Semian::AtomicInteger.new initialize_lookup(symbol_list) end diff --git a/lib/semian/atomic_integer.rb b/lib/semian/atomic_integer.rb index 8ae041a6..50a058a6 100644 --- a/lib/semian/atomic_integer.rb +++ b/lib/semian/atomic_integer.rb @@ -1,13 +1,24 @@ module Semian class AtomicInteger < SharedMemoryObject #:nodoc: + include ::Semian::ReentrantMutex attr_accessor :value - def initialize(_name, _permissions) + def initialize(**options) @value = 0 end def increment(val = 1) @value += val end + + def destroy + if shared? + super + else + @value = 0 + end + end + + surround_with_mutex :value, :value=, :increment end end diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 73df2fb3..9820f506 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -1,23 +1,20 @@ module Semian class CircuitBreaker #:nodoc: - def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, permissions: 0660) + def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, permissions:) @name = name.to_s @success_count_threshold = success_threshold @error_count_threshold = error_threshold @error_timeout = error_timeout @exceptions = exceptions - @errors = ::Semian::SysVSlidingWindow.new( - "#{name}_sliding_window", - @error_count_threshold, - permissions) - @successes = ::Semian::SysVAtomicInteger.new( - "#{name}_atomic_integer", - permissions) - @state = ::Semian::SysVAtomicEnum.new( - "#{name}_atomic_enum", - permissions, - [:closed, :half_open, :open]) + @errors = ::Semian::SysVSlidingWindow.new(@error_count_threshold, + name: "#{name}_sliding_window", + permissions: permissions) + @successes = ::Semian::SysVAtomicInteger.new(name: "#{name}_atomic_integer", + permissions: permissions) + @state = ::Semian::SysVAtomicEnum.new([:closed, :half_open, :open], + name: "#{name}_atomic_enum", + permissions: permissions) # We do not need to #reset here since initializing is handled like this: # (0) if data is not shared, then it's zeroed already # (1) if no one is attached to the memory, zero it diff --git a/lib/semian/sliding_window.rb b/lib/semian/sliding_window.rb index d8a5a023..cfee4bc7 100644 --- a/lib/semian/sliding_window.rb +++ b/lib/semian/sliding_window.rb @@ -1,11 +1,12 @@ module Semian class SlidingWindow < SharedMemoryObject #:nodoc: extend Forwardable + include ::Semian::ReentrantMutex def_delegators :@window, :size, :pop, :shift, :first, :last attr_reader :max_size - def initialize(_name, max_size, _permissions) + def initialize(max_size, **options) @max_size = max_size @window = [] end @@ -31,14 +32,20 @@ def push(time_ms) self end - def unshift(time_ms) - @window.pop while @window.size >= @max_size - @window.unshift(time_ms) + def clear + @window = [] self end - def clear - @window = [] + def destroy + if shared? + super + else + clear + end end + + surround_with_mutex :size, :pop, :shift, :first, :last, :max_size, + :resize_to, :<<, :push, :clear end end diff --git a/lib/semian/sysv_atomic_enum.rb b/lib/semian/sysv_atomic_enum.rb index 3d5bd88e..ed5a3ad5 100644 --- a/lib/semian/sysv_atomic_enum.rb +++ b/lib/semian/sysv_atomic_enum.rb @@ -1,7 +1,7 @@ module Semian class SysVAtomicEnum < AtomicEnum #:nodoc: - def initialize(name, permissions, symbol_list) - @integer = Semian::SysVAtomicInteger.new(name, permissions) + def initialize(symbol_list, name:, permissions:) + @integer = Semian::SysVAtomicInteger.new(name: name, permissions: permissions) initialize_lookup(symbol_list) end end diff --git a/lib/semian/sysv_atomic_integer.rb b/lib/semian/sysv_atomic_integer.rb index 9d3224b6..98c2e00f 100644 --- a/lib/semian/sysv_atomic_integer.rb +++ b/lib/semian/sysv_atomic_integer.rb @@ -1,6 +1,6 @@ module Semian class SysVAtomicInteger < AtomicInteger #:nodoc: - def initialize(name, permissions) + def initialize(name:, permissions:) data_layout = [:int] super unless acquire_memory_object(name, data_layout, permissions) end diff --git a/lib/semian/sysv_sliding_window.rb b/lib/semian/sysv_sliding_window.rb index 93b053db..527ca8e7 100644 --- a/lib/semian/sysv_sliding_window.rb +++ b/lib/semian/sysv_sliding_window.rb @@ -1,6 +1,6 @@ module Semian class SysVSlidingWindow < SlidingWindow #:nodoc: - def initialize(name, max_size, permissions) + def initialize(max_size, name:, permissions:) data_layout = [:int, :int].concat(Array.new(max_size, :long)) super unless acquire_memory_object(name, data_layout, permissions) end diff --git a/test/atomic_enum_test.rb b/test/atomic_enum_test.rb index d5362297..0c7009f9 100644 --- a/test/atomic_enum_test.rb +++ b/test/atomic_enum_test.rb @@ -1,35 +1,50 @@ require 'test_helper' class TestAtomicEnum < MiniTest::Unit::TestCase - def test_functionality - run_test_with_atomic_enum_classes do + CLASS = ::Semian::AtomicEnum + + def setup + @enum = CLASS.new([:one, :two, :three], + name: 'TestAtomicEnum', + permissions: 0660) + end + + def teardown + @enum.destroy + end + + module AtomicEnumTestCases + def test_assigning + old = @enum.value + @enum.value = @enum.value + assert_equal old, @enum.value @enum.value = :two assert_equal :two, @enum.value end - end - def test_will_throw_error_when_invalid_symbol_given - run_test_with_atomic_enum_classes do + def test_iterate_enum + @enum.value = :one + @enum.increment + assert_equal :two, @enum.value + @enum.increment + assert_equal :three, @enum.value + @enum.increment + assert_equal :one, @enum.value + @enum.increment(2) + assert_equal :three, @enum.value + @enum.increment(4) + assert_equal :one, @enum.value + @enum.increment(0) + assert_equal :one, @enum.value + end + + def test_will_throw_error_when_invalid_symbol_given assert_raises KeyError do @enum.value = :four end end end - private - - def atomic_enum_classes - @classes ||= [::Semian::AtomicEnum] - end - - def run_test_with_atomic_enum_classes(klasses = atomic_enum_classes) - klasses.each do |klass| - begin - @enum = klass.new('TestAtomicEnum', 0660, [:one, :two, :three]) - yield(klass) - ensure - @enum.destroy - end - end - end + include AtomicEnumTestCases end + diff --git a/test/atomic_integer_test.rb b/test/atomic_integer_test.rb index d5ab3bbe..5cb0c950 100644 --- a/test/atomic_integer_test.rb +++ b/test/atomic_integer_test.rb @@ -1,8 +1,19 @@ require 'test_helper' class TestAtomicInteger < MiniTest::Unit::TestCase - def test_access_value - run_test_with_atomic_integer_classes do + CLASS = ::Semian::AtomicInteger + + def setup + @integer = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) + @integer.value = 0 + end + + def teardown + @integer.destroy + end + + module AtomicIntegerTestCases + def test_access_value @integer.value = 0 assert_equal(0, @integer.value) @integer.value = 99 @@ -15,10 +26,8 @@ def test_access_value @integer.value = 6 assert_equal(6, @integer.value) end - end - def test_increment - run_test_with_atomic_integer_classes do + def test_increment @integer.value = 0 @integer.increment(4) assert_equal(4, @integer.value) @@ -29,21 +38,5 @@ def test_increment end end - private - - def atomic_integer_classes - @classes ||= [::Semian::AtomicInteger] - end - - def run_test_with_atomic_integer_classes(klasses = atomic_integer_classes) - klasses.each do |klass| - begin - @integer = klass.new('TestAtomicInteger', 0660) - @integer.value = 0 - yield(klass) - ensure - @integer.destroy - end - end - end + include AtomicIntegerTestCases end diff --git a/test/helpers/class_test_helper.rb b/test/helpers/class_test_helper.rb deleted file mode 100644 index 0981e520..00000000 --- a/test/helpers/class_test_helper.rb +++ /dev/null @@ -1,5 +0,0 @@ -module ClassTestHelper - def retrieve_descendants(klass) - ObjectSpace.each_object(klass.singleton_class).to_a - end -end diff --git a/test/sliding_window_test.rb b/test/sliding_window_test.rb index ae765702..4630a8b3 100644 --- a/test/sliding_window_test.rb +++ b/test/sliding_window_test.rb @@ -1,18 +1,29 @@ require 'test_helper' class TestSlidingWindow < MiniTest::Unit::TestCase - def test_slidinv_window_push - run_test_with_sliding_window_classes do + CLASS = ::Semian::SlidingWindow + + def setup + @sliding_window = CLASS.new(6, + name: 'TestSlidingWindow', + permissions: 0660) + @sliding_window.clear + end + + def teardown + @sliding_window.destroy + end + + module SlidingWindowTestCases + def test_sliding_window_push assert_equal(0, @sliding_window.size) @sliding_window << 1 assert_correct_first_and_last_and_size(@sliding_window, 1, 1, 1, 6) @sliding_window << 5 assert_correct_first_and_last_and_size(@sliding_window, 1, 5, 2, 6) end - end - def test_sliding_window_resize - run_test_with_sliding_window_classes do + def test_sliding_window_resize assert_equal(0, @sliding_window.size) @sliding_window << 1 << 2 << 3 << 4 << 5 << 6 assert_correct_first_and_last_and_size(@sliding_window, 1, 6, 6, 6) @@ -23,49 +34,36 @@ def test_sliding_window_resize @sliding_window.resize_to 6 assert_correct_first_and_last_and_size(@sliding_window, 2, 6, 5, 6) end - end - def test_sliding_window_edge_falloff - run_test_with_sliding_window_classes do + def test_sliding_window_edge_falloff assert_equal(0, @sliding_window.size) @sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7 assert_correct_first_and_last_and_size(@sliding_window, 2, 7, 6, 6) @sliding_window.shift assert_correct_first_and_last_and_size(@sliding_window, 3, 7, 5, 6) - @sliding_window.clear end end - private - - def sliding_window_classes - @classes ||= [::Semian::SlidingWindow] - end + module SlidingWindowUtilityMethods + def assert_correct_first_and_last_and_size(sliding_window, first, last, size, max_size) + assert_equal(first, sliding_window.first) + assert_equal(last, sliding_window.last) + assert_equal(size, sliding_window.size) + assert_equal(max_size, sliding_window.max_size) + end - def run_test_with_sliding_window_classes(klasses = sliding_window_classes) - klasses.each do |klass| - begin - @sliding_window = klass.new('TestSlidingWindow', 6, 0660) - @sliding_window.clear - yield(klass) - ensure - @sliding_window.destroy - end + def assert_sliding_windows_in_sync(sliding_window_1, sliding_window_2) + # it only exposes ends, size, and max_size, so can only check those + assert_equal(sliding_window_1.first, sliding_window_2.first) + assert_equal(sliding_window_1.last, sliding_window_2.last) + assert_equal(sliding_window_1.size, sliding_window_2.size) + assert_equal(sliding_window_1.max_size, sliding_window_2.max_size) end end - def assert_correct_first_and_last_and_size(sliding_window, first, last, size, max_size) - assert_equal(first, sliding_window.first) - assert_equal(last, sliding_window.last) - assert_equal(size, sliding_window.size) - assert_equal(max_size, sliding_window.max_size) - end + include SlidingWindowTestCases - def assert_sliding_windows_in_sync(sliding_window_1, sliding_window_2) - # it only exposes ends, size, and max_size, so can only check those - assert_equal(sliding_window_1.first, sliding_window_2.first) - assert_equal(sliding_window_1.last, sliding_window_2.last) - assert_equal(sliding_window_1.size, sliding_window_2.size) - assert_equal(sliding_window_1.max_size, sliding_window_2.max_size) - end + private + + include SlidingWindowUtilityMethods end diff --git a/test/sysv_atomic_enum_test.rb b/test/sysv_atomic_enum_test.rb index 45696ca8..8468f2ce 100644 --- a/test/sysv_atomic_enum_test.rb +++ b/test/sysv_atomic_enum_test.rb @@ -8,7 +8,7 @@ class << self end self.resources = {} attr_accessor :name - def self.new(name, _permissions, _symbol_list) + def self.new(symbol_list, name:, permissions:) obj = resources[name] ||= super obj.name = name obj @@ -16,7 +16,6 @@ def self.new(name, _permissions, _symbol_list) def destroy self.class.resources.delete(@name) - super end def shared? @@ -24,30 +23,28 @@ def shared? end end - def test_memory_is_shared - run_test_with_atomic_enum_classes do |klass| - assert_equal :one, @enum.value - @enum.value = :three + CLASS = ::Semian::SysVAtomicEnum - enum_2 = klass.new('TestAtomicEnum', 0660, [:one, :two, :three]) - assert_equal :three, enum_2.value - end + def setup + @enum = CLASS.new([:one, :two, :three], + name: 'TestAtomicEnum', + permissions: 0660) end - private - - def atomic_enum_classes - @classes ||= [::Semian::SysVAtomicEnum, FakeSysVAtomicEnum] + def teardown + @enum.destroy end - def run_test_with_atomic_enum_classes(klasses = atomic_enum_classes) - klasses.each do |klass| - begin - @enum = klass.new('TestAtomicEnum', 0660, [:one, :two, :three]) - yield(klass) - ensure - @enum.destroy - end - end + include TestAtomicEnum::AtomicEnumTestCases + + def test_memory_is_shared + assert_equal :one, @enum.value + @enum.value = :three + + enum_2 = CLASS.new([:one, :two, :three], + name: 'TestAtomicEnum', + permissions: 0660) + assert_equal :three, enum_2.value end + end diff --git a/test/sysv_atomic_integer_test.rb b/test/sysv_atomic_integer_test.rb index 4d08e91f..2362db5c 100644 --- a/test/sysv_atomic_integer_test.rb +++ b/test/sysv_atomic_integer_test.rb @@ -8,7 +8,7 @@ class << self end self.resources = {} attr_accessor :name - def self.new(name, _permissions) + def self.new(name:, permissions:) obj = resources[name] ||= super obj.name = name obj @@ -16,7 +16,6 @@ def self.new(name, _permissions) def destroy self.class.resources.delete(@name) - super end def shared? @@ -24,54 +23,45 @@ def shared? end end - def test_memory_is_shared - run_test_with_atomic_integer_classes do |klass| - integer_2 = klass.new('TestAtomicInteger', 0660) - integer_2.value = 100 - assert_equal 100, @integer.value - @integer.value = 200 - assert_equal 200, integer_2.value - @integer.value = 0 - assert_equal 0, integer_2.value - end + CLASS = ::Semian::SysVAtomicInteger + + def setup + @integer = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) + @integer.value = 0 end - def test_memory_not_reset_when_at_least_one_worker_using_it - run_test_with_atomic_integer_classes do |klass| - @integer.value = 109 - integer_2 = klass.new('TestAtomicInteger', 0660) - assert_equal @integer.value, integer_2.value - pid = fork do - integer_3 = klass.new('TestAtomicInteger', 0660) - assert_equal 109, integer_3.value - sleep - end - sleep 1 - Process.kill("KILL", pid) - Process.waitall - fork do - integer_3 = klass.new('TestAtomicInteger', 0660) - assert_equal 109, integer_3.value - end - Process.waitall - end + def teardown + @integer.destroy end - private + include TestAtomicInteger::AtomicIntegerTestCases - def atomic_integer_classes - @classes ||= [::Semian::SysVAtomicInteger, FakeSysVAtomicInteger] + def test_memory_is_shared + integer_2 = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) + integer_2.value = 100 + assert_equal 100, @integer.value + @integer.value = 200 + assert_equal 200, integer_2.value + @integer.value = 0 + assert_equal 0, integer_2.value end - def run_test_with_atomic_integer_classes(klasses = atomic_integer_classes) - klasses.each do |klass| - begin - @integer = klass.new('TestAtomicInteger', 0660) - @integer.value = 0 - yield(klass) - ensure - @integer.destroy - end + def test_memory_not_reset_when_at_least_one_worker_using_it + @integer.value = 109 + integer_2 = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) + assert_equal @integer.value, integer_2.value + pid = fork do + integer_3 = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) + assert_equal 109, integer_3.value + sleep + end + sleep 1 + Process.kill("KILL", pid) + Process.waitall + fork do + integer_3 = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) + assert_equal 109, integer_3.value end + Process.waitall end end diff --git a/test/sysv_sliding_window_test.rb b/test/sysv_sliding_window_test.rb index 5cbed2ae..a1a7749d 100644 --- a/test/sysv_sliding_window_test.rb +++ b/test/sysv_sliding_window_test.rb @@ -1,6 +1,6 @@ require 'test_helper' -class TestSlidingWindow < MiniTest::Unit::TestCase +class TestSysVSlidingWindow < MiniTest::Unit::TestCase # Emulate sharedness to test correctness against real SysVSlidingWindow class class FakeSysVSlidingWindow < Semian::SlidingWindow class << self @@ -8,16 +8,15 @@ class << self end self.resources = {} attr_accessor :name - def self.new(name, size, permissions) + def self.new(max_size, name:, permissions:) obj = resources[name] ||= super obj.name = name - obj.resize_to(size) + obj.resize_to(max_size) obj end def destroy self.class.resources.delete(@name) - super end def shared? @@ -25,138 +24,134 @@ def shared? end end - def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it - run_test_with_sliding_window_classes do |klass| - Timeout.timeout(1) do # assure dont hang - @sliding_window << 100 - assert_equal 100, @sliding_window.first - end - - pid = fork do - sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) - sliding_window_2.execute_atomically { sleep } - end - - sleep 1 - Process.kill('KILL', pid) - Process.waitall - - Timeout.timeout(1) do # assure dont hang - @sliding_window << 100 - assert_equal(100, @sliding_window.first) - end - end - end + CLASS = ::Semian::SysVSlidingWindow - def test_sliding_window_memory_is_actually_shared - run_test_with_sliding_window_classes do |klass| - assert_equal 0, @sliding_window.size - sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) - assert_equal 0, sliding_window_2.size - - large_number = (Time.now.to_f * 1000).to_i - @sliding_window << large_number - assert_correct_first_and_last_and_size(@sliding_window, large_number, large_number, 1, 6) - assert_correct_first_and_last_and_size(sliding_window_2, large_number, large_number, 1, 6) - sliding_window_2 << 6 << 4 << 3 << 2 - assert_correct_first_and_last_and_size(@sliding_window, large_number, 2, 5, 6) - assert_correct_first_and_last_and_size(sliding_window_2, large_number, 2, 5, 6) - - @sliding_window.clear - assert_correct_first_and_last_and_size(@sliding_window, nil, nil, 0, 6) - assert_correct_first_and_last_and_size(sliding_window_2, nil, nil, 0, 6) - end + def setup + @sliding_window = CLASS.new(6, + name: 'TestSlidingWindow', + permissions: 0660) + @sliding_window.clear end - def test_restarting_worker_should_not_reset_queue - run_test_with_sliding_window_classes do |klass| - @sliding_window << 10 << 20 << 30 - sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) - assert_correct_first_and_last_and_size(@sliding_window, 10, 30, 3, 6) - sliding_window_2.pop - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - - sliding_window_3 = klass.new('TestSlidingWindow', 6, 0660) - assert_correct_first_and_last_and_size(@sliding_window, 10, 20, 2, 6) - sliding_window_3.pop - assert_correct_first_and_last_and_size(@sliding_window, 10, 10, 1, 6) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - @sliding_window.clear - end + def teardown + @sliding_window.destroy end - def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down - run_test_with_sliding_window_classes do |klass| - # Test explicit resizing, and resizing through making new memory associations - - sliding_window_2 = klass.new('TestSlidingWindow', 4, 0660) - sliding_window_2 << 80 << 90 << 100 << 110 << 120 - assert_correct_first_and_last_and_size(@sliding_window, 90, 120, 4, 4) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - - sliding_window_2 = klass.new('TestSlidingWindow', 3, 0660) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 100, 120, 3, 3) - - @sliding_window.resize_to(2) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 2) - - sliding_window_2 = klass.new('TestSlidingWindow', 4, 0660) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 4) - - @sliding_window.resize_to(6) - @sliding_window << 130 - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 110, 130, 3, 6) + include TestSlidingWindow::SlidingWindowTestCases - sliding_window_2 = klass.new('TestSlidingWindow', 2, 0660) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 2) + def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it + Timeout.timeout(1) do # assure dont hang + @sliding_window << 100 + assert_equal 100, @sliding_window.first + end - @sliding_window.resize_to(4) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 4) + pid = fork do + sliding_window_2 = CLASS.new(6, + name: 'TestSlidingWindow', + permissions: 0660) + sliding_window_2.execute_atomically { sleep } + end - sliding_window_2 = klass.new('TestSlidingWindow', 6, 0660) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 6) + sleep 1 + Process.kill('KILL', pid) + Process.waitall - sliding_window_2.clear + Timeout.timeout(1) do # assure dont hang + @sliding_window << 100 + assert_equal(100, @sliding_window.first) end end - private - - def shared_sliding_window_classes - @classes ||= [::Semian::SysVSlidingWindow, FakeSysVSlidingWindow] + def test_sliding_window_memory_is_actually_shared + assert_equal 0, @sliding_window.size + sliding_window_2 = CLASS.new(6, + name: 'TestSlidingWindow', + permissions: 0660) + assert_equal 0, sliding_window_2.size + + large_number = (Time.now.to_f * 1000).to_i + @sliding_window << large_number + assert_correct_first_and_last_and_size(@sliding_window, large_number, large_number, 1, 6) + assert_correct_first_and_last_and_size(sliding_window_2, large_number, large_number, 1, 6) + sliding_window_2 << 6 << 4 << 3 << 2 + assert_correct_first_and_last_and_size(@sliding_window, large_number, 2, 5, 6) + assert_correct_first_and_last_and_size(sliding_window_2, large_number, 2, 5, 6) + + @sliding_window.clear + assert_correct_first_and_last_and_size(@sliding_window, nil, nil, 0, 6) + assert_correct_first_and_last_and_size(sliding_window_2, nil, nil, 0, 6) end - def run_test_with_sliding_window_classes(klasses = shared_sliding_window_classes) - klasses.each do |klass| - begin - @sliding_window = klass.new('TestSlidingWindow', 6, 0660) - @sliding_window.clear - yield(klass) - ensure - @sliding_window.destroy - end - end + def test_restarting_worker_should_not_reset_queue + @sliding_window << 10 << 20 << 30 + sliding_window_2 = CLASS.new(6, + name: 'TestSlidingWindow', + permissions: 0660) + assert_correct_first_and_last_and_size(@sliding_window, 10, 30, 3, 6) + sliding_window_2.pop + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + + sliding_window_3 = CLASS.new(6, + name: 'TestSlidingWindow', + permissions: 0660) + assert_correct_first_and_last_and_size(@sliding_window, 10, 20, 2, 6) + sliding_window_3.pop + assert_correct_first_and_last_and_size(@sliding_window, 10, 10, 1, 6) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + @sliding_window.clear end - def assert_correct_first_and_last_and_size(sliding_window, first, last, size, max_size) - assert_equal(first, sliding_window.first) - assert_equal(last, sliding_window.last) - assert_equal(size, sliding_window.size) - assert_equal(max_size, sliding_window.max_size) + def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down + # Test explicit resizing, and resizing through making new memory associations + + sliding_window_2 = CLASS.new(4, + name: 'TestSlidingWindow', + permissions: 0660) + sliding_window_2 << 80 << 90 << 100 << 110 << 120 + assert_correct_first_and_last_and_size(@sliding_window, 90, 120, 4, 4) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + + sliding_window_2 = CLASS.new(3, + name: 'TestSlidingWindow', + permissions: 0660) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 100, 120, 3, 3) + + @sliding_window.resize_to(2) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 2) + + sliding_window_2 = CLASS.new(4, + name: 'TestSlidingWindow', + permissions: 0660) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 4) + + @sliding_window.resize_to(6) + @sliding_window << 130 + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 110, 130, 3, 6) + + sliding_window_2 = CLASS.new(2, + name: 'TestSlidingWindow', + permissions: 0660) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 2) + + @sliding_window.resize_to(4) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 4) + + sliding_window_2 = CLASS.new(6, + name: 'TestSlidingWindow', + permissions: 0660) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) + assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 6) + + sliding_window_2.clear end - def assert_sliding_windows_in_sync(sliding_window_1, sliding_window_2) - # it only exposes ends, size, and max_size, so can only check those - assert_equal(sliding_window_1.first, sliding_window_2.first) - assert_equal(sliding_window_1.last, sliding_window_2.last) - assert_equal(sliding_window_1.size, sliding_window_2.size) - assert_equal(sliding_window_1.max_size, sliding_window_2.max_size) - end + private + + include TestSlidingWindow::SlidingWindowUtilityMethods end diff --git a/test/test_helper.rb b/test/test_helper.rb index 89d8ac6b..3c34f825 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -9,7 +9,6 @@ require 'timeout' require 'helpers/background_helper' -require 'helpers/class_test_helper' Semian.logger = Logger.new(nil) Toxiproxy.populate([ @@ -27,5 +26,4 @@ class MiniTest::Unit::TestCase include BackgroundHelper - include ClassTestHelper end From 1b178f5e29d401651a68b3293832e5a815a608d6 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Tue, 10 Nov 2015 16:24:08 +0000 Subject: [PATCH 10/23] Rebased, small changes had to be done --- Rakefile | 2 +- ext/semian/extconf.rb | 2 +- lib/semian.rb | 15 +++++++++++---- lib/semian/atomic_enum.rb | 2 +- lib/semian/atomic_integer.rb | 3 +-- lib/semian/shared_memory_object.rb | 9 +++++++-- lib/semian/sliding_window.rb | 3 +-- lib/semian/sysv_atomic_enum.rb | 2 +- lib/semian/sysv_atomic_integer.rb | 2 +- lib/semian/sysv_sliding_window.rb | 2 +- test/atomic_enum_test.rb | 5 +---- test/atomic_integer_test.rb | 2 +- test/sliding_window_test.rb | 4 +--- test/sysv_atomic_enum_test.rb | 5 ++--- 14 files changed, 31 insertions(+), 27 deletions(-) diff --git a/Rakefile b/Rakefile index d8642bb6..addd57d5 100644 --- a/Rakefile +++ b/Rakefile @@ -27,7 +27,7 @@ if Semian.sysv_semaphores_supported? ext.ext_dir = 'ext/semian' ext.lib_dir = 'lib/semian' end - task :build => :compile + task build: :compile else task :build do end diff --git a/ext/semian/extconf.rb b/ext/semian/extconf.rb index 31b68181..55f85424 100644 --- a/ext/semian/extconf.rb +++ b/ext/semian/extconf.rb @@ -24,7 +24,7 @@ have_func 'rb_thread_call_without_gvl' $CFLAGS = "-D_GNU_SOURCE -Werror -Wall -std=c99 " -if ENV.has_key?('DEBUG') +if ENV.key?('DEBUG') $CFLAGS << "-O0 -g" else $CFLAGS << "-O3" diff --git a/lib/semian.rb b/lib/semian.rb index 5a8a819b..d9ae85d8 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -153,14 +153,21 @@ def resources module ReentrantMutex attr_reader :monitor + def call_with_mutex + @monitor ||= Monitor.new + @monitor.synchronize do + yield if block_given? + end + end + def self.included(base) def base.surround_with_mutex(*names) names.each do |name| - m = instance_method(name) + new_name = "#{name}_inner".freeze + alias_method new_name, name define_method(name) do |*args, &block| - @monitor ||= Monitor.new - @monitor.synchronize do - m.bind(self).call(*args, &block) + call_with_mutex do + method(new_name).call(*args, &block) end end end diff --git a/lib/semian/atomic_enum.rb b/lib/semian/atomic_enum.rb index 48f2fde1..77335225 100644 --- a/lib/semian/atomic_enum.rb +++ b/lib/semian/atomic_enum.rb @@ -8,7 +8,7 @@ class AtomicEnum < SharedMemoryObject #:nodoc: :shared?, :destroy, :acquire_memory_object, :bind_init_fn private :shared?, :acquire_memory_object, :bind_init_fn - def initialize(symbol_list, **options) + def initialize(symbol_list) @integer = Semian::AtomicInteger.new initialize_lookup(symbol_list) end diff --git a/lib/semian/atomic_integer.rb b/lib/semian/atomic_integer.rb index 50a058a6..77234669 100644 --- a/lib/semian/atomic_integer.rb +++ b/lib/semian/atomic_integer.rb @@ -1,9 +1,8 @@ module Semian class AtomicInteger < SharedMemoryObject #:nodoc: - include ::Semian::ReentrantMutex attr_accessor :value - def initialize(**options) + def initialize @value = 0 end diff --git a/lib/semian/shared_memory_object.rb b/lib/semian/shared_memory_object.rb index 6140c17e..660cc1c0 100644 --- a/lib/semian/shared_memory_object.rb +++ b/lib/semian/shared_memory_object.rb @@ -1,5 +1,7 @@ module Semian class SharedMemoryObject #:nodoc: + include ReentrantMutex + @type_size = {} def self.sizeof(type) size = (@type_size[type.to_sym] ||= (respond_to?(:_sizeof) ? _sizeof(type.to_sym) : 0)) @@ -16,8 +18,11 @@ def shmid end def execute_atomically(&proc) - return _execute_atomically(&proc) if respond_to?(:_execute_atomically) && @using_shared_memory - yield if block_given? + if respond_to?(:_execute_atomically) && @using_shared_memory + return _execute_atomically(&proc) + else + call_with_mutex(&proc) + end end alias_method :transaction, :execute_atomically diff --git a/lib/semian/sliding_window.rb b/lib/semian/sliding_window.rb index cfee4bc7..ddf3b17e 100644 --- a/lib/semian/sliding_window.rb +++ b/lib/semian/sliding_window.rb @@ -1,12 +1,11 @@ module Semian class SlidingWindow < SharedMemoryObject #:nodoc: extend Forwardable - include ::Semian::ReentrantMutex def_delegators :@window, :size, :pop, :shift, :first, :last attr_reader :max_size - def initialize(max_size, **options) + def initialize(max_size) @max_size = max_size @window = [] end diff --git a/lib/semian/sysv_atomic_enum.rb b/lib/semian/sysv_atomic_enum.rb index ed5a3ad5..15b3b3db 100644 --- a/lib/semian/sysv_atomic_enum.rb +++ b/lib/semian/sysv_atomic_enum.rb @@ -1,6 +1,6 @@ module Semian class SysVAtomicEnum < AtomicEnum #:nodoc: - def initialize(symbol_list, name:, permissions:) + def initialize(symbol_list, name:, permissions:) @integer = Semian::SysVAtomicInteger.new(name: name, permissions: permissions) initialize_lookup(symbol_list) end diff --git a/lib/semian/sysv_atomic_integer.rb b/lib/semian/sysv_atomic_integer.rb index 98c2e00f..6dbe9caf 100644 --- a/lib/semian/sysv_atomic_integer.rb +++ b/lib/semian/sysv_atomic_integer.rb @@ -2,7 +2,7 @@ module Semian class SysVAtomicInteger < AtomicInteger #:nodoc: def initialize(name:, permissions:) data_layout = [:int] - super unless acquire_memory_object(name, data_layout, permissions) + super() unless acquire_memory_object(name, data_layout, permissions) end end end diff --git a/lib/semian/sysv_sliding_window.rb b/lib/semian/sysv_sliding_window.rb index 527ca8e7..3978f679 100644 --- a/lib/semian/sysv_sliding_window.rb +++ b/lib/semian/sysv_sliding_window.rb @@ -2,7 +2,7 @@ module Semian class SysVSlidingWindow < SlidingWindow #:nodoc: def initialize(max_size, name:, permissions:) data_layout = [:int, :int].concat(Array.new(max_size, :long)) - super unless acquire_memory_object(name, data_layout, permissions) + super(max_size) unless acquire_memory_object(name, data_layout, permissions) end end end diff --git a/test/atomic_enum_test.rb b/test/atomic_enum_test.rb index 0c7009f9..bcc81173 100644 --- a/test/atomic_enum_test.rb +++ b/test/atomic_enum_test.rb @@ -4,9 +4,7 @@ class TestAtomicEnum < MiniTest::Unit::TestCase CLASS = ::Semian::AtomicEnum def setup - @enum = CLASS.new([:one, :two, :three], - name: 'TestAtomicEnum', - permissions: 0660) + @enum = CLASS.new([:one, :two, :three]) end def teardown @@ -47,4 +45,3 @@ def test_will_throw_error_when_invalid_symbol_given include AtomicEnumTestCases end - diff --git a/test/atomic_integer_test.rb b/test/atomic_integer_test.rb index 5cb0c950..99682173 100644 --- a/test/atomic_integer_test.rb +++ b/test/atomic_integer_test.rb @@ -4,7 +4,7 @@ class TestAtomicInteger < MiniTest::Unit::TestCase CLASS = ::Semian::AtomicInteger def setup - @integer = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) + @integer = CLASS.new @integer.value = 0 end diff --git a/test/sliding_window_test.rb b/test/sliding_window_test.rb index 4630a8b3..bb366caf 100644 --- a/test/sliding_window_test.rb +++ b/test/sliding_window_test.rb @@ -4,9 +4,7 @@ class TestSlidingWindow < MiniTest::Unit::TestCase CLASS = ::Semian::SlidingWindow def setup - @sliding_window = CLASS.new(6, - name: 'TestSlidingWindow', - permissions: 0660) + @sliding_window = CLASS.new(6) @sliding_window.clear end diff --git a/test/sysv_atomic_enum_test.rb b/test/sysv_atomic_enum_test.rb index 8468f2ce..f4a5ea2a 100644 --- a/test/sysv_atomic_enum_test.rb +++ b/test/sysv_atomic_enum_test.rb @@ -42,9 +42,8 @@ def test_memory_is_shared @enum.value = :three enum_2 = CLASS.new([:one, :two, :three], - name: 'TestAtomicEnum', - permissions: 0660) + name: 'TestAtomicEnum', + permissions: 0660) assert_equal :three, enum_2.value end - end From 7103df0c8362b1c410d4a2814e18937eff553bda Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Tue, 10 Nov 2015 20:19:23 +0000 Subject: [PATCH 11/23] Redistribute under Simple and SysV namespace --- ext/semian/semian_atomic_integer.c | 3 +- ext/semian/semian_sliding_window.c | 3 +- lib/semian.rb | 17 +++--- lib/semian/atomic_enum.rb | 40 -------------- lib/semian/atomic_integer.rb | 23 -------- lib/semian/circuit_breaker.rb | 18 +++---- lib/semian/simple_enum.rb | 42 +++++++++++++++ lib/semian/simple_integer.rb | 25 +++++++++ lib/semian/simple_sliding_window.rb | 52 +++++++++++++++++++ lib/semian/sliding_window.rb | 50 ------------------ lib/semian/sysv_atomic_enum.rb | 8 --- lib/semian/sysv_atomic_integer.rb | 8 --- lib/semian/sysv_enum.rb | 10 ++++ lib/semian/sysv_integer.rb | 10 ++++ lib/semian/sysv_sliding_window.rb | 10 ++-- ...tomic_enum_test.rb => simple_enum_test.rb} | 8 +-- ...integer_test.rb => simple_integer_test.rb} | 8 +-- ..._test.rb => simple_sliding_window_test.rb} | 4 +- ..._atomic_enum_test.rb => sysv_enum_test.rb} | 8 +-- ...atomic_integer_test.rb => sysv_integer.rb} | 8 +-- test/sysv_sliding_window_test.rb | 8 +-- 21 files changed, 189 insertions(+), 174 deletions(-) delete mode 100644 lib/semian/atomic_enum.rb delete mode 100644 lib/semian/atomic_integer.rb create mode 100644 lib/semian/simple_enum.rb create mode 100644 lib/semian/simple_integer.rb create mode 100644 lib/semian/simple_sliding_window.rb delete mode 100644 lib/semian/sliding_window.rb delete mode 100644 lib/semian/sysv_atomic_enum.rb delete mode 100644 lib/semian/sysv_atomic_integer.rb create mode 100644 lib/semian/sysv_enum.rb create mode 100644 lib/semian/sysv_integer.rb rename test/{atomic_enum_test.rb => simple_enum_test.rb} (86%) rename test/{atomic_integer_test.rb => simple_integer_test.rb} (84%) rename test/{sliding_window_test.rb => simple_sliding_window_test.rb} (95%) rename test/{sysv_atomic_enum_test.rb => sysv_enum_test.rb} (83%) rename test/{sysv_atomic_integer_test.rb => sysv_integer.rb} (88%) diff --git a/ext/semian/semian_atomic_integer.c b/ext/semian/semian_atomic_integer.c index 6bbb70c7..e65fd462 100644 --- a/ext/semian/semian_atomic_integer.c +++ b/ext/semian/semian_atomic_integer.c @@ -104,7 +104,8 @@ Init_semian_atomic_integer (void) { // Bind methods to AtomicInteger VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); - VALUE cAtomicInteger = rb_const_get(cSemianModule, rb_intern("SysVAtomicInteger")); + VALUE cSysVModule = rb_const_get(cSemianModule, rb_intern("SysV")); + VALUE cAtomicInteger = rb_const_get(cSysVModule, rb_intern("Integer")); rb_define_method(cAtomicInteger, "bind_init_fn", semian_atomic_integer_bind_init_fn_wrapper, 0); rb_define_method(cAtomicInteger, "value", semian_atomic_integer_get_value, 0); diff --git a/ext/semian/semian_sliding_window.c b/ext/semian/semian_sliding_window.c index e03b1dbd..7eb7a28d 100644 --- a/ext/semian/semian_sliding_window.c +++ b/ext/semian/semian_sliding_window.c @@ -295,7 +295,8 @@ void Init_semian_sliding_window (void) { VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); - VALUE cSlidingWindow = rb_const_get(cSemianModule, rb_intern("SysVSlidingWindow")); + VALUE cSysVModule = rb_const_get(cSemianModule, rb_intern("SysV")); + VALUE cSlidingWindow = rb_const_get(cSysVModule, rb_intern("SlidingWindow")); rb_define_method(cSlidingWindow, "bind_init_fn", semian_sliding_window_bind_init_fn_wrapper, 0); rb_define_method(cSlidingWindow, "size", semian_sliding_window_size, 0); diff --git a/lib/semian.rb b/lib/semian.rb index d9ae85d8..f1881630 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -125,6 +125,7 @@ def register(name, tickets:, permissions: 0660, timeout: 0, error_threshold:, er error_timeout: error_timeout, exceptions: Array(exceptions) + [::Semian::BaseError], permissions: permissions, + type_namespace: ::Semian::SysV, ) resource = Resource.new(name, tickets: tickets, permissions: permissions, timeout: timeout) resources[name] = ProtectedResource.new(resource, circuit_breaker) @@ -151,11 +152,11 @@ def resources end module ReentrantMutex - attr_reader :monitor + attr_reader :mutex def call_with_mutex - @monitor ||= Monitor.new - @monitor.synchronize do + @mutex ||= Monitor.new + @mutex.synchronize do yield if block_given? end end @@ -182,12 +183,12 @@ def base.surround_with_mutex(*names) require 'semian/unprotected_resource' require 'semian/platform' require 'semian/shared_memory_object' -require 'semian/sliding_window' -require 'semian/atomic_integer' -require 'semian/atomic_enum' +require 'semian/simple_sliding_window' +require 'semian/simple_integer' +require 'semian/simple_enum' require 'semian/sysv_sliding_window' -require 'semian/sysv_atomic_integer' -require 'semian/sysv_atomic_enum' +require 'semian/sysv_integer' +require 'semian/sysv_enum' if Semian.semaphores_enabled? require 'semian/semian' else diff --git a/lib/semian/atomic_enum.rb b/lib/semian/atomic_enum.rb deleted file mode 100644 index 77335225..00000000 --- a/lib/semian/atomic_enum.rb +++ /dev/null @@ -1,40 +0,0 @@ -require 'forwardable' - -module Semian - class AtomicEnum < SharedMemoryObject #:nodoc: - extend Forwardable - - def_delegators :@integer, :semid, :shmid, :execute_atomically, :transaction, - :shared?, :destroy, :acquire_memory_object, :bind_init_fn - private :shared?, :acquire_memory_object, :bind_init_fn - - def initialize(symbol_list) - @integer = Semian::AtomicInteger.new - initialize_lookup(symbol_list) - end - - def increment(val = 1) - @integer.value = (@integer.value + val) % @sym_to_num.size - value - end - - def value - @num_to_sym.fetch(@integer.value) - end - - def value=(sym) - @integer.value = @sym_to_num.fetch(sym) - end - - private - - def initialize_lookup(symbol_list) - # Assume symbol_list[0] is mapped to 0 - # Cannot just use #object_id since #object_id for symbols is different in every run - # For now, implement a C-style enum type backed by integers - - @sym_to_num = Hash[symbol_list.each_with_index.to_a] - @num_to_sym = @sym_to_num.invert - end - end -end diff --git a/lib/semian/atomic_integer.rb b/lib/semian/atomic_integer.rb deleted file mode 100644 index 77234669..00000000 --- a/lib/semian/atomic_integer.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Semian - class AtomicInteger < SharedMemoryObject #:nodoc: - attr_accessor :value - - def initialize - @value = 0 - end - - def increment(val = 1) - @value += val - end - - def destroy - if shared? - super - else - @value = 0 - end - end - - surround_with_mutex :value, :value=, :increment - end -end diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 9820f506..c18e783d 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -1,20 +1,20 @@ module Semian class CircuitBreaker #:nodoc: - def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, permissions:) + def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, permissions:, type_namespace:) @name = name.to_s @success_count_threshold = success_threshold @error_count_threshold = error_threshold @error_timeout = error_timeout @exceptions = exceptions - @errors = ::Semian::SysVSlidingWindow.new(@error_count_threshold, - name: "#{name}_sliding_window", - permissions: permissions) - @successes = ::Semian::SysVAtomicInteger.new(name: "#{name}_atomic_integer", - permissions: permissions) - @state = ::Semian::SysVAtomicEnum.new([:closed, :half_open, :open], - name: "#{name}_atomic_enum", - permissions: permissions) + @errors = type_namespace::SlidingWindow.new(@error_count_threshold, + name: "#{name}_sysv_sliding_window", + permissions: permissions) + @successes = type_namespace::Integer.new(name: "#{name}_sysv_integer", + permissions: permissions) + @state = type_namespace::Enum.new([:closed, :half_open, :open], + name: "#{name}_sysv_enum", + permissions: permissions) # We do not need to #reset here since initializing is handled like this: # (0) if data is not shared, then it's zeroed already # (1) if no one is attached to the memory, zero it diff --git a/lib/semian/simple_enum.rb b/lib/semian/simple_enum.rb new file mode 100644 index 00000000..97193f57 --- /dev/null +++ b/lib/semian/simple_enum.rb @@ -0,0 +1,42 @@ +require 'forwardable' + +module Semian + module Simple + class Enum < SharedMemoryObject #:nodoc: + extend Forwardable + + def_delegators :@integer, :semid, :shmid, :execute_atomically, :transaction, + :shared?, :destroy, :acquire_memory_object, :bind_init_fn + private :shared?, :acquire_memory_object, :bind_init_fn + + def initialize(symbol_list) + @integer = Semian::Simple::Integer.new + initialize_lookup(symbol_list) + end + + def increment(val = 1) + @integer.value = (@integer.value + val) % @sym_to_num.size + value + end + + def value + @num_to_sym.fetch(@integer.value) + end + + def value=(sym) + @integer.value = @sym_to_num.fetch(sym) + end + + private + + def initialize_lookup(symbol_list) + # Assume symbol_list[0] is mapped to 0 + # Cannot just use #object_id since #object_id for symbols is different in every run + # For now, implement a C-style enum type backed by integers + + @sym_to_num = Hash[symbol_list.each_with_index.to_a] + @num_to_sym = @sym_to_num.invert + end + end + end +end diff --git a/lib/semian/simple_integer.rb b/lib/semian/simple_integer.rb new file mode 100644 index 00000000..6d6fc99d --- /dev/null +++ b/lib/semian/simple_integer.rb @@ -0,0 +1,25 @@ +module Semian + module Simple + class Integer < SharedMemoryObject #:nodoc: + attr_accessor :value + + def initialize + @value = 0 + end + + def increment(val = 1) + @value += val + end + + def destroy + if shared? + super + else + @value = 0 + end + end + + surround_with_mutex :value, :value=, :increment + end + end +end diff --git a/lib/semian/simple_sliding_window.rb b/lib/semian/simple_sliding_window.rb new file mode 100644 index 00000000..344d3966 --- /dev/null +++ b/lib/semian/simple_sliding_window.rb @@ -0,0 +1,52 @@ +module Semian + module Simple + class SlidingWindow < SharedMemoryObject #:nodoc: + extend Forwardable + + def_delegators :@window, :size, :pop, :shift, :first, :last + attr_reader :max_size + + def initialize(max_size) + @max_size = max_size + @window = [] + end + + # A sliding window is a structure that stores the most @max_size recent timestamps + # like this: if @max_size = 4, current time is 10, @window =[5,7,9,10]. + # Another push of (11) at 11 sec would make @window [7,9,10,11], shifting off 5. + + def resize_to(size) + raise ArgumentError.new('size must be larger than 0') if size < 1 + @max_size = size + @window.shift while @window.size > @max_size + self + end + + def <<(time_ms) + push(time_ms) + end + + def push(time_ms) + @window.shift while @window.size >= @max_size + @window << time_ms + self + end + + def clear + @window = [] + self + end + + def destroy + if shared? + super + else + clear + end + end + + surround_with_mutex :size, :pop, :shift, :first, :last, :max_size, + :resize_to, :<<, :push, :clear + end + end +end diff --git a/lib/semian/sliding_window.rb b/lib/semian/sliding_window.rb deleted file mode 100644 index ddf3b17e..00000000 --- a/lib/semian/sliding_window.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Semian - class SlidingWindow < SharedMemoryObject #:nodoc: - extend Forwardable - - def_delegators :@window, :size, :pop, :shift, :first, :last - attr_reader :max_size - - def initialize(max_size) - @max_size = max_size - @window = [] - end - - # A sliding window is a structure that stores the most @max_size recent timestamps - # like this: if @max_size = 4, current time is 10, @window =[5,7,9,10]. - # Another push of (11) at 11 sec would make @window [7,9,10,11], shifting off 5. - - def resize_to(size) - raise ArgumentError.new('size must be larger than 0') if size < 1 - @max_size = size - @window.shift while @window.size > @max_size - self - end - - def <<(time_ms) - push(time_ms) - end - - def push(time_ms) - @window.shift while @window.size >= @max_size - @window << time_ms - self - end - - def clear - @window = [] - self - end - - def destroy - if shared? - super - else - clear - end - end - - surround_with_mutex :size, :pop, :shift, :first, :last, :max_size, - :resize_to, :<<, :push, :clear - end -end diff --git a/lib/semian/sysv_atomic_enum.rb b/lib/semian/sysv_atomic_enum.rb deleted file mode 100644 index 15b3b3db..00000000 --- a/lib/semian/sysv_atomic_enum.rb +++ /dev/null @@ -1,8 +0,0 @@ -module Semian - class SysVAtomicEnum < AtomicEnum #:nodoc: - def initialize(symbol_list, name:, permissions:) - @integer = Semian::SysVAtomicInteger.new(name: name, permissions: permissions) - initialize_lookup(symbol_list) - end - end -end diff --git a/lib/semian/sysv_atomic_integer.rb b/lib/semian/sysv_atomic_integer.rb deleted file mode 100644 index 6dbe9caf..00000000 --- a/lib/semian/sysv_atomic_integer.rb +++ /dev/null @@ -1,8 +0,0 @@ -module Semian - class SysVAtomicInteger < AtomicInteger #:nodoc: - def initialize(name:, permissions:) - data_layout = [:int] - super() unless acquire_memory_object(name, data_layout, permissions) - end - end -end diff --git a/lib/semian/sysv_enum.rb b/lib/semian/sysv_enum.rb new file mode 100644 index 00000000..0d6b912d --- /dev/null +++ b/lib/semian/sysv_enum.rb @@ -0,0 +1,10 @@ +module Semian + module SysV + class Enum < Semian::Simple::Enum #:nodoc: + def initialize(symbol_list, name:, permissions:) + @integer = Semian::SysV::Integer.new(name: name, permissions: permissions) + initialize_lookup(symbol_list) + end + end + end +end diff --git a/lib/semian/sysv_integer.rb b/lib/semian/sysv_integer.rb new file mode 100644 index 00000000..6f91bc2b --- /dev/null +++ b/lib/semian/sysv_integer.rb @@ -0,0 +1,10 @@ +module Semian + module SysV + class Integer < Semian::Simple::Integer #:nodoc: + def initialize(name:, permissions:) + data_layout = [:int] + super() unless acquire_memory_object(name, data_layout, permissions) + end + end + end +end diff --git a/lib/semian/sysv_sliding_window.rb b/lib/semian/sysv_sliding_window.rb index 3978f679..86c36c48 100644 --- a/lib/semian/sysv_sliding_window.rb +++ b/lib/semian/sysv_sliding_window.rb @@ -1,8 +1,10 @@ module Semian - class SysVSlidingWindow < SlidingWindow #:nodoc: - def initialize(max_size, name:, permissions:) - data_layout = [:int, :int].concat(Array.new(max_size, :long)) - super(max_size) unless acquire_memory_object(name, data_layout, permissions) + module SysV + class SlidingWindow < Semian::Simple::SlidingWindow #:nodoc: + def initialize(max_size, name:, permissions:) + data_layout = [:int, :int].concat(Array.new(max_size, :long)) + super(max_size) unless acquire_memory_object(name, data_layout, permissions) + end end end end diff --git a/test/atomic_enum_test.rb b/test/simple_enum_test.rb similarity index 86% rename from test/atomic_enum_test.rb rename to test/simple_enum_test.rb index bcc81173..d843350b 100644 --- a/test/atomic_enum_test.rb +++ b/test/simple_enum_test.rb @@ -1,7 +1,7 @@ require 'test_helper' -class TestAtomicEnum < MiniTest::Unit::TestCase - CLASS = ::Semian::AtomicEnum +class TestSimpleEnum < MiniTest::Unit::TestCase + CLASS = ::Semian::Simple::Enum def setup @enum = CLASS.new([:one, :two, :three]) @@ -11,7 +11,7 @@ def teardown @enum.destroy end - module AtomicEnumTestCases + module EnumTestCases def test_assigning old = @enum.value @enum.value = @enum.value @@ -43,5 +43,5 @@ def test_will_throw_error_when_invalid_symbol_given end end - include AtomicEnumTestCases + include EnumTestCases end diff --git a/test/atomic_integer_test.rb b/test/simple_integer_test.rb similarity index 84% rename from test/atomic_integer_test.rb rename to test/simple_integer_test.rb index 99682173..a571ede3 100644 --- a/test/atomic_integer_test.rb +++ b/test/simple_integer_test.rb @@ -1,7 +1,7 @@ require 'test_helper' -class TestAtomicInteger < MiniTest::Unit::TestCase - CLASS = ::Semian::AtomicInteger +class TestSimpleInteger < MiniTest::Unit::TestCase + CLASS = ::Semian::Simple::Integer def setup @integer = CLASS.new @@ -12,7 +12,7 @@ def teardown @integer.destroy end - module AtomicIntegerTestCases + module IntegerTestCases def test_access_value @integer.value = 0 assert_equal(0, @integer.value) @@ -38,5 +38,5 @@ def test_increment end end - include AtomicIntegerTestCases + include IntegerTestCases end diff --git a/test/sliding_window_test.rb b/test/simple_sliding_window_test.rb similarity index 95% rename from test/sliding_window_test.rb rename to test/simple_sliding_window_test.rb index bb366caf..7a135aa1 100644 --- a/test/sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -1,7 +1,7 @@ require 'test_helper' -class TestSlidingWindow < MiniTest::Unit::TestCase - CLASS = ::Semian::SlidingWindow +class TestSimpleSlidingWindow < MiniTest::Unit::TestCase + CLASS = ::Semian::Simple::SlidingWindow def setup @sliding_window = CLASS.new(6) diff --git a/test/sysv_atomic_enum_test.rb b/test/sysv_enum_test.rb similarity index 83% rename from test/sysv_atomic_enum_test.rb rename to test/sysv_enum_test.rb index f4a5ea2a..8fa90a7d 100644 --- a/test/sysv_atomic_enum_test.rb +++ b/test/sysv_enum_test.rb @@ -1,8 +1,8 @@ require 'test_helper' -class TestSysVAtomicEnum < MiniTest::Unit::TestCase +class TestSysVEnum < MiniTest::Unit::TestCase # Emulate sharedness to test correctness against real SysVAtomicEnum class - class FakeSysVAtomicEnum < Semian::AtomicEnum + class FakeSysVAtomicEnum < Semian::Simple::Enum class << self attr_accessor :resources end @@ -23,7 +23,7 @@ def shared? end end - CLASS = ::Semian::SysVAtomicEnum + CLASS = ::Semian::SysV::Enum def setup @enum = CLASS.new([:one, :two, :three], @@ -35,7 +35,7 @@ def teardown @enum.destroy end - include TestAtomicEnum::AtomicEnumTestCases + include TestSimpleEnum::EnumTestCases def test_memory_is_shared assert_equal :one, @enum.value diff --git a/test/sysv_atomic_integer_test.rb b/test/sysv_integer.rb similarity index 88% rename from test/sysv_atomic_integer_test.rb rename to test/sysv_integer.rb index 2362db5c..a36bc653 100644 --- a/test/sysv_atomic_integer_test.rb +++ b/test/sysv_integer.rb @@ -1,8 +1,8 @@ require 'test_helper' -class TestSysVAtomicInteger < MiniTest::Unit::TestCase +class TestSysVInteger < MiniTest::Unit::TestCase # Emulate sharedness to test correctness against real SysVAtomicInteger class - class FakeSysVAtomicInteger < Semian::AtomicInteger + class FakeSysVAtomicInteger < Semian::Simple::Integer class << self attr_accessor :resources end @@ -23,7 +23,7 @@ def shared? end end - CLASS = ::Semian::SysVAtomicInteger + CLASS = ::Semian::SysV::Integer def setup @integer = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) @@ -34,7 +34,7 @@ def teardown @integer.destroy end - include TestAtomicInteger::AtomicIntegerTestCases + include TestSimpleInteger::IntegerTestCases def test_memory_is_shared integer_2 = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) diff --git a/test/sysv_sliding_window_test.rb b/test/sysv_sliding_window_test.rb index a1a7749d..22d1c631 100644 --- a/test/sysv_sliding_window_test.rb +++ b/test/sysv_sliding_window_test.rb @@ -2,7 +2,7 @@ class TestSysVSlidingWindow < MiniTest::Unit::TestCase # Emulate sharedness to test correctness against real SysVSlidingWindow class - class FakeSysVSlidingWindow < Semian::SlidingWindow + class FakeSysVSlidingWindow < Semian::Simple::SlidingWindow class << self attr_accessor :resources end @@ -24,7 +24,7 @@ def shared? end end - CLASS = ::Semian::SysVSlidingWindow + CLASS = ::Semian::SysV::SlidingWindow def setup @sliding_window = CLASS.new(6, @@ -37,7 +37,7 @@ def teardown @sliding_window.destroy end - include TestSlidingWindow::SlidingWindowTestCases + include TestSimpleSlidingWindow::SlidingWindowTestCases def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it Timeout.timeout(1) do # assure dont hang @@ -153,5 +153,5 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down private - include TestSlidingWindow::SlidingWindowUtilityMethods + include TestSimpleSlidingWindow::SlidingWindowUtilityMethods end From 7ab2f3616e367824f26eed8b40de5970cdbf102e Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Wed, 11 Nov 2015 16:56:09 +0000 Subject: [PATCH 12/23] Change API a bit --- lib/semian.rb | 2 +- lib/semian/circuit_breaker.rb | 16 +++----- lib/semian/simple_enum.rb | 2 +- lib/semian/simple_integer.rb | 4 ++ lib/semian/simple_sliding_window.rb | 8 ++-- lib/semian/sysv_enum.rb | 2 +- lib/semian/sysv_sliding_window.rb | 2 +- test/simple_enum_test.rb | 2 +- test/simple_sliding_window_test.rb | 40 +++++++++--------- test/sysv_enum_test.rb | 6 +-- .../{sysv_integer.rb => sysv_integer_test.rb} | 0 test/sysv_sliding_window_test.rb | 41 ++++++++++++++----- 12 files changed, 70 insertions(+), 55 deletions(-) rename test/{sysv_integer.rb => sysv_integer_test.rb} (100%) diff --git a/lib/semian.rb b/lib/semian.rb index f1881630..90f8a507 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -125,7 +125,7 @@ def register(name, tickets:, permissions: 0660, timeout: 0, error_threshold:, er error_timeout: error_timeout, exceptions: Array(exceptions) + [::Semian::BaseError], permissions: permissions, - type_namespace: ::Semian::SysV, + implementation: ::Semian::SysV, ) resource = Resource.new(name, tickets: tickets, permissions: permissions, timeout: timeout) resources[name] = ProtectedResource.new(resource, circuit_breaker) diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index c18e783d..27174a96 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -1,24 +1,20 @@ module Semian class CircuitBreaker #:nodoc: - def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, permissions:, type_namespace:) + def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, permissions:, implementation:) @name = name.to_s @success_count_threshold = success_threshold @error_count_threshold = error_threshold @error_timeout = error_timeout @exceptions = exceptions - @errors = type_namespace::SlidingWindow.new(@error_count_threshold, + @errors = implementation::SlidingWindow.new(max_size: @error_count_threshold, name: "#{name}_sysv_sliding_window", permissions: permissions) - @successes = type_namespace::Integer.new(name: "#{name}_sysv_integer", + @successes = implementation::Integer.new(name: "#{name}_sysv_integer", permissions: permissions) - @state = type_namespace::Enum.new([:closed, :half_open, :open], + @state = implementation::Enum.new(symbol_list: [:closed, :half_open, :open], name: "#{name}_sysv_enum", permissions: permissions) - # We do not need to #reset here since initializing is handled like this: - # (0) if data is not shared, then it's zeroed already - # (1) if no one is attached to the memory, zero it - # (2) otherwise, keep the data end def acquire @@ -59,7 +55,7 @@ def mark_success def reset @errors.clear - @successes.value = 0 + @successes.reset close end @@ -115,7 +111,7 @@ def error_timeout_expired? def push_time(window, duration:, time: Time.now) @errors.execute_atomically do # Store an integer amount of milliseconds since epoch - window.shift while window.first && Time.at(window.first / 1000) + duration < time + window.shift while window.first && window.first / 1000 + duration < time.to_i window << (time.to_f * 1000).to_i end end diff --git a/lib/semian/simple_enum.rb b/lib/semian/simple_enum.rb index 97193f57..7f6ab740 100644 --- a/lib/semian/simple_enum.rb +++ b/lib/semian/simple_enum.rb @@ -9,7 +9,7 @@ class Enum < SharedMemoryObject #:nodoc: :shared?, :destroy, :acquire_memory_object, :bind_init_fn private :shared?, :acquire_memory_object, :bind_init_fn - def initialize(symbol_list) + def initialize(symbol_list:) @integer = Semian::Simple::Integer.new initialize_lookup(symbol_list) end diff --git a/lib/semian/simple_integer.rb b/lib/semian/simple_integer.rb index 6d6fc99d..8778681a 100644 --- a/lib/semian/simple_integer.rb +++ b/lib/semian/simple_integer.rb @@ -11,6 +11,10 @@ def increment(val = 1) @value += val end + def reset + @value = 0 + end + def destroy if shared? super diff --git a/lib/semian/simple_sliding_window.rb b/lib/semian/simple_sliding_window.rb index 344d3966..0efe8137 100644 --- a/lib/semian/simple_sliding_window.rb +++ b/lib/semian/simple_sliding_window.rb @@ -6,7 +6,7 @@ class SlidingWindow < SharedMemoryObject #:nodoc: def_delegators :@window, :size, :pop, :shift, :first, :last attr_reader :max_size - def initialize(max_size) + def initialize(max_size:) @max_size = max_size @window = [] end @@ -22,16 +22,14 @@ def resize_to(size) self end - def <<(time_ms) - push(time_ms) - end - def push(time_ms) @window.shift while @window.size >= @max_size @window << time_ms self end + alias_method :<<, :push + def clear @window = [] self diff --git a/lib/semian/sysv_enum.rb b/lib/semian/sysv_enum.rb index 0d6b912d..6122eb65 100644 --- a/lib/semian/sysv_enum.rb +++ b/lib/semian/sysv_enum.rb @@ -1,7 +1,7 @@ module Semian module SysV class Enum < Semian::Simple::Enum #:nodoc: - def initialize(symbol_list, name:, permissions:) + def initialize(symbol_list:, name:, permissions:) @integer = Semian::SysV::Integer.new(name: name, permissions: permissions) initialize_lookup(symbol_list) end diff --git a/lib/semian/sysv_sliding_window.rb b/lib/semian/sysv_sliding_window.rb index 86c36c48..4a9659ac 100644 --- a/lib/semian/sysv_sliding_window.rb +++ b/lib/semian/sysv_sliding_window.rb @@ -1,7 +1,7 @@ module Semian module SysV class SlidingWindow < Semian::Simple::SlidingWindow #:nodoc: - def initialize(max_size, name:, permissions:) + def initialize(max_size:, name:, permissions:) data_layout = [:int, :int].concat(Array.new(max_size, :long)) super(max_size) unless acquire_memory_object(name, data_layout, permissions) end diff --git a/test/simple_enum_test.rb b/test/simple_enum_test.rb index d843350b..6846a6dc 100644 --- a/test/simple_enum_test.rb +++ b/test/simple_enum_test.rb @@ -4,7 +4,7 @@ class TestSimpleEnum < MiniTest::Unit::TestCase CLASS = ::Semian::Simple::Enum def setup - @enum = CLASS.new([:one, :two, :three]) + @enum = CLASS.new(symbol_list: [:one, :two, :three]) end def teardown diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index 7a135aa1..0e673e36 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -4,7 +4,7 @@ class TestSimpleSlidingWindow < MiniTest::Unit::TestCase CLASS = ::Semian::Simple::SlidingWindow def setup - @sliding_window = CLASS.new(6) + @sliding_window = CLASS.new(max_size: 6) @sliding_window.clear end @@ -16,47 +16,45 @@ module SlidingWindowTestCases def test_sliding_window_push assert_equal(0, @sliding_window.size) @sliding_window << 1 - assert_correct_first_and_last_and_size(@sliding_window, 1, 1, 1, 6) + assert_sliding_window(@sliding_window, [1], 6) @sliding_window << 5 - assert_correct_first_and_last_and_size(@sliding_window, 1, 5, 2, 6) + assert_sliding_window(@sliding_window, [1, 5], 6) end def test_sliding_window_resize assert_equal(0, @sliding_window.size) @sliding_window << 1 << 2 << 3 << 4 << 5 << 6 - assert_correct_first_and_last_and_size(@sliding_window, 1, 6, 6, 6) + assert_sliding_window(@sliding_window, [1, 2, 3, 4, 5, 6], 6) @sliding_window.resize_to 6 - assert_correct_first_and_last_and_size(@sliding_window, 1, 6, 6, 6) + assert_sliding_window(@sliding_window, [1, 2, 3, 4, 5, 6], 6) @sliding_window.resize_to 5 - assert_correct_first_and_last_and_size(@sliding_window, 2, 6, 5, 5) + assert_sliding_window(@sliding_window, [2, 3, 4, 5, 6], 5) @sliding_window.resize_to 6 - assert_correct_first_and_last_and_size(@sliding_window, 2, 6, 5, 6) + assert_sliding_window(@sliding_window, [2, 3, 4, 5, 6], 6) end def test_sliding_window_edge_falloff assert_equal(0, @sliding_window.size) @sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7 - assert_correct_first_and_last_and_size(@sliding_window, 2, 7, 6, 6) + assert_sliding_window(@sliding_window, [2, 3, 4, 5, 6, 7], 6) @sliding_window.shift - assert_correct_first_and_last_and_size(@sliding_window, 3, 7, 5, 6) + assert_sliding_window(@sliding_window, [3, 4, 5, 6, 7], 6) + end + + def resize_to_less_than_1_raises + assert_raises ArgumentError do + @sliding_window.resize_to 0 + end end end module SlidingWindowUtilityMethods - def assert_correct_first_and_last_and_size(sliding_window, first, last, size, max_size) - assert_equal(first, sliding_window.first) - assert_equal(last, sliding_window.last) - assert_equal(size, sliding_window.size) + def assert_sliding_window(sliding_window, array, max_size) + # Get private member, the sliding_window doesn't expose the entire array + data = sliding_window.instance_variable_get("@window") + assert_equal(array, data) assert_equal(max_size, sliding_window.max_size) end - - def assert_sliding_windows_in_sync(sliding_window_1, sliding_window_2) - # it only exposes ends, size, and max_size, so can only check those - assert_equal(sliding_window_1.first, sliding_window_2.first) - assert_equal(sliding_window_1.last, sliding_window_2.last) - assert_equal(sliding_window_1.size, sliding_window_2.size) - assert_equal(sliding_window_1.max_size, sliding_window_2.max_size) - end end include SlidingWindowTestCases diff --git a/test/sysv_enum_test.rb b/test/sysv_enum_test.rb index 8fa90a7d..ecf64784 100644 --- a/test/sysv_enum_test.rb +++ b/test/sysv_enum_test.rb @@ -8,7 +8,7 @@ class << self end self.resources = {} attr_accessor :name - def self.new(symbol_list, name:, permissions:) + def self.new(symbol_list:, name:, permissions:) obj = resources[name] ||= super obj.name = name obj @@ -26,7 +26,7 @@ def shared? CLASS = ::Semian::SysV::Enum def setup - @enum = CLASS.new([:one, :two, :three], + @enum = CLASS.new(symbol_list: [:one, :two, :three], name: 'TestAtomicEnum', permissions: 0660) end @@ -41,7 +41,7 @@ def test_memory_is_shared assert_equal :one, @enum.value @enum.value = :three - enum_2 = CLASS.new([:one, :two, :three], + enum_2 = CLASS.new(symbol_list: [:one, :two, :three], name: 'TestAtomicEnum', permissions: 0660) assert_equal :three, enum_2.value diff --git a/test/sysv_integer.rb b/test/sysv_integer_test.rb similarity index 100% rename from test/sysv_integer.rb rename to test/sysv_integer_test.rb diff --git a/test/sysv_sliding_window_test.rb b/test/sysv_sliding_window_test.rb index 22d1c631..888f6d99 100644 --- a/test/sysv_sliding_window_test.rb +++ b/test/sysv_sliding_window_test.rb @@ -8,7 +8,7 @@ class << self end self.resources = {} attr_accessor :name - def self.new(max_size, name:, permissions:) + def self.new(max_size:, name:, permissions:) obj = resources[name] ||= super obj.name = name obj.resize_to(max_size) @@ -27,7 +27,7 @@ def shared? CLASS = ::Semian::SysV::SlidingWindow def setup - @sliding_window = CLASS.new(6, + @sliding_window = CLASS.new(max_size: 6, name: 'TestSlidingWindow', permissions: 0660) @sliding_window.clear @@ -46,7 +46,7 @@ def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it end pid = fork do - sliding_window_2 = CLASS.new(6, + sliding_window_2 = CLASS.new(max_size: 6, name: 'TestSlidingWindow', permissions: 0660) sliding_window_2.execute_atomically { sleep } @@ -64,7 +64,7 @@ def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it def test_sliding_window_memory_is_actually_shared assert_equal 0, @sliding_window.size - sliding_window_2 = CLASS.new(6, + sliding_window_2 = CLASS.new(max_size: 6, name: 'TestSlidingWindow', permissions: 0660) assert_equal 0, sliding_window_2.size @@ -84,14 +84,14 @@ def test_sliding_window_memory_is_actually_shared def test_restarting_worker_should_not_reset_queue @sliding_window << 10 << 20 << 30 - sliding_window_2 = CLASS.new(6, + sliding_window_2 = CLASS.new(max_size: 6, name: 'TestSlidingWindow', permissions: 0660) assert_correct_first_and_last_and_size(@sliding_window, 10, 30, 3, 6) sliding_window_2.pop assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - sliding_window_3 = CLASS.new(6, + sliding_window_3 = CLASS.new(max_size: 6, name: 'TestSlidingWindow', permissions: 0660) assert_correct_first_and_last_and_size(@sliding_window, 10, 20, 2, 6) @@ -104,14 +104,14 @@ def test_restarting_worker_should_not_reset_queue def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down # Test explicit resizing, and resizing through making new memory associations - sliding_window_2 = CLASS.new(4, + sliding_window_2 = CLASS.new(max_size: 4, name: 'TestSlidingWindow', permissions: 0660) sliding_window_2 << 80 << 90 << 100 << 110 << 120 assert_correct_first_and_last_and_size(@sliding_window, 90, 120, 4, 4) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - sliding_window_2 = CLASS.new(3, + sliding_window_2 = CLASS.new(max_size: 3, name: 'TestSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) @@ -121,7 +121,7 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 2) - sliding_window_2 = CLASS.new(4, + sliding_window_2 = CLASS.new(max_size: 4, name: 'TestSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) @@ -132,7 +132,7 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) assert_correct_first_and_last_and_size(@sliding_window, 110, 130, 3, 6) - sliding_window_2 = CLASS.new(2, + sliding_window_2 = CLASS.new(max_size: 2, name: 'TestSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) @@ -142,7 +142,7 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 4) - sliding_window_2 = CLASS.new(6, + sliding_window_2 = CLASS.new(max_size: 6, name: 'TestSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) @@ -153,5 +153,24 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down private + def assert_sliding_window(sliding_window, array, max_size) + assert_correct_first_and_last_and_size(sliding_window, array.first, array.last, array.size, max_size) + end + + def assert_correct_first_and_last_and_size(sliding_window, first, last, size, max_size) + assert_equal(first, sliding_window.first) + assert_equal(last, sliding_window.last) + assert_equal(size, sliding_window.size) + assert_equal(max_size, sliding_window.max_size) + end + + def assert_sliding_windows_in_sync(sliding_window_1, sliding_window_2) + # it only exposes ends, size, and max_size, so can only check those + assert_equal(sliding_window_1.first, sliding_window_2.first) + assert_equal(sliding_window_1.last, sliding_window_2.last) + assert_equal(sliding_window_1.size, sliding_window_2.size) + assert_equal(sliding_window_1.max_size, sliding_window_2.max_size) + end + include TestSimpleSlidingWindow::SlidingWindowUtilityMethods end From 2e9cd7ac29721c31ab00e56601765727aea68322 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Thu, 12 Nov 2015 16:32:39 +0000 Subject: [PATCH 13/23] Change to State from Enum, updated to match merged PR, removed Mutex code --- lib/semian.rb | 29 ++------------- lib/semian/circuit_breaker.rb | 28 ++++++--------- lib/semian/shared_memory_object.rb | 2 -- lib/semian/simple_enum.rb | 42 ---------------------- lib/semian/simple_integer.rb | 2 -- lib/semian/simple_sliding_window.rb | 3 -- lib/semian/simple_state.rb | 45 +++++++++++++++++++++++ lib/semian/sysv_enum.rb | 10 ------ lib/semian/sysv_state.rb | 56 +++++++++++++++++++++++++++++ test/circuit_breaker_test.rb | 2 +- test/simple_enum_test.rb | 47 ------------------------ test/simple_state_test.rb | 45 +++++++++++++++++++++++ test/sysv_enum_test.rb | 49 ------------------------- test/sysv_integer_test.rb | 32 +++-------------- test/sysv_sliding_window_test.rb | 43 ++++++---------------- test/sysv_state_test.rb | 44 +++++++++++++++++++++++ 16 files changed, 218 insertions(+), 261 deletions(-) delete mode 100644 lib/semian/simple_enum.rb create mode 100644 lib/semian/simple_state.rb delete mode 100644 lib/semian/sysv_enum.rb create mode 100644 lib/semian/sysv_state.rb delete mode 100644 test/simple_enum_test.rb create mode 100644 test/simple_state_test.rb delete mode 100644 test/sysv_enum_test.rb create mode 100644 test/sysv_state_test.rb diff --git a/lib/semian.rb b/lib/semian.rb index 90f8a507..c16ff802 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -150,31 +150,6 @@ def destroy(name) def resources @resources ||= {} end - - module ReentrantMutex - attr_reader :mutex - - def call_with_mutex - @mutex ||= Monitor.new - @mutex.synchronize do - yield if block_given? - end - end - - def self.included(base) - def base.surround_with_mutex(*names) - names.each do |name| - new_name = "#{name}_inner".freeze - alias_method new_name, name - define_method(name) do |*args, &block| - call_with_mutex do - method(new_name).call(*args, &block) - end - end - end - end - end - end end require 'semian/resource' @@ -185,10 +160,10 @@ def base.surround_with_mutex(*names) require 'semian/shared_memory_object' require 'semian/simple_sliding_window' require 'semian/simple_integer' -require 'semian/simple_enum' +require 'semian/simple_state' require 'semian/sysv_sliding_window' require 'semian/sysv_integer' -require 'semian/sysv_enum' +require 'semian/sysv_state' if Semian.semaphores_enabled? require 'semian/semian' else diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 27174a96..b6567314 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -1,5 +1,7 @@ module Semian class CircuitBreaker #:nodoc: + extend Forwardable + def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, permissions:, implementation:) @name = name.to_s @success_count_threshold = success_threshold @@ -12,9 +14,8 @@ def initialize(name, exceptions:, success_threshold:, error_threshold:, error_ti permissions: permissions) @successes = implementation::Integer.new(name: "#{name}_sysv_integer", permissions: permissions) - @state = implementation::Enum.new(symbol_list: [:closed, :half_open, :open], - name: "#{name}_sysv_enum", - permissions: permissions) + @state = implementation::State.new(name: "#{name}_sysv_state", + permissions: permissions) end def acquire @@ -67,33 +68,24 @@ def destroy private - def closed? - @state.value == :closed - end + def_delegators :@state, :closed?, :open?, :half_open? + private :closed?, :open?, :half_open? def close log_state_transition(:closed) - @state.value = :closed + @state.close @errors.clear end - def open? - @state.value == :open - end - def open log_state_transition(:open) - @state.value = :open - end - - def half_open? - @state.value == :half_open + @state.open end def half_open log_state_transition(:half_open) - @state.value = :half_open - @successes.value = 0 + @state.half_open + @successes.reset end def success_threshold_reached? diff --git a/lib/semian/shared_memory_object.rb b/lib/semian/shared_memory_object.rb index 660cc1c0..bad3433f 100644 --- a/lib/semian/shared_memory_object.rb +++ b/lib/semian/shared_memory_object.rb @@ -1,7 +1,5 @@ module Semian class SharedMemoryObject #:nodoc: - include ReentrantMutex - @type_size = {} def self.sizeof(type) size = (@type_size[type.to_sym] ||= (respond_to?(:_sizeof) ? _sizeof(type.to_sym) : 0)) diff --git a/lib/semian/simple_enum.rb b/lib/semian/simple_enum.rb deleted file mode 100644 index 7f6ab740..00000000 --- a/lib/semian/simple_enum.rb +++ /dev/null @@ -1,42 +0,0 @@ -require 'forwardable' - -module Semian - module Simple - class Enum < SharedMemoryObject #:nodoc: - extend Forwardable - - def_delegators :@integer, :semid, :shmid, :execute_atomically, :transaction, - :shared?, :destroy, :acquire_memory_object, :bind_init_fn - private :shared?, :acquire_memory_object, :bind_init_fn - - def initialize(symbol_list:) - @integer = Semian::Simple::Integer.new - initialize_lookup(symbol_list) - end - - def increment(val = 1) - @integer.value = (@integer.value + val) % @sym_to_num.size - value - end - - def value - @num_to_sym.fetch(@integer.value) - end - - def value=(sym) - @integer.value = @sym_to_num.fetch(sym) - end - - private - - def initialize_lookup(symbol_list) - # Assume symbol_list[0] is mapped to 0 - # Cannot just use #object_id since #object_id for symbols is different in every run - # For now, implement a C-style enum type backed by integers - - @sym_to_num = Hash[symbol_list.each_with_index.to_a] - @num_to_sym = @sym_to_num.invert - end - end - end -end diff --git a/lib/semian/simple_integer.rb b/lib/semian/simple_integer.rb index 8778681a..ea27d6eb 100644 --- a/lib/semian/simple_integer.rb +++ b/lib/semian/simple_integer.rb @@ -22,8 +22,6 @@ def destroy @value = 0 end end - - surround_with_mutex :value, :value=, :increment end end end diff --git a/lib/semian/simple_sliding_window.rb b/lib/semian/simple_sliding_window.rb index 0efe8137..e551c8f6 100644 --- a/lib/semian/simple_sliding_window.rb +++ b/lib/semian/simple_sliding_window.rb @@ -42,9 +42,6 @@ def destroy clear end end - - surround_with_mutex :size, :pop, :shift, :first, :last, :max_size, - :resize_to, :<<, :push, :clear end end end diff --git a/lib/semian/simple_state.rb b/lib/semian/simple_state.rb new file mode 100644 index 00000000..59d6774a --- /dev/null +++ b/lib/semian/simple_state.rb @@ -0,0 +1,45 @@ +require 'forwardable' + +module Semian + module Simple + class State < SharedMemoryObject #:nodoc: + def initialize + reset + end + + attr_reader :value + + def open? + value == :open + end + + def closed? + value == :closed + end + + def half_open? + value == :half_open + end + + def open + @value = :open + end + + def close + @value = :closed + end + + def half_open + @value = :half_open + end + + def reset + close + end + + def destroy + reset + end + end + end +end diff --git a/lib/semian/sysv_enum.rb b/lib/semian/sysv_enum.rb deleted file mode 100644 index 6122eb65..00000000 --- a/lib/semian/sysv_enum.rb +++ /dev/null @@ -1,10 +0,0 @@ -module Semian - module SysV - class Enum < Semian::Simple::Enum #:nodoc: - def initialize(symbol_list:, name:, permissions:) - @integer = Semian::SysV::Integer.new(name: name, permissions: permissions) - initialize_lookup(symbol_list) - end - end - end -end diff --git a/lib/semian/sysv_state.rb b/lib/semian/sysv_state.rb new file mode 100644 index 00000000..2d0f4568 --- /dev/null +++ b/lib/semian/sysv_state.rb @@ -0,0 +1,56 @@ +module Semian + module SysV + class State < Semian::Simple::State #:nodoc: + extend Forwardable + + def_delegators :@integer, :semid, :shmid, :execute_atomically, :transaction, + :shared?, :acquire_memory_object, :bind_init_fn + private :shared?, :acquire_memory_object, :bind_init_fn + + def initialize(name:, permissions:) + @integer = Semian::SysV::Integer.new(name: name, permissions: permissions) + initialize_lookup([:closed, :open, :half_open]) + end + + def open + self.value = :open + end + + def close + self.value = :closed + end + + def half_open + self.value = :half_open + end + + def reset + close + end + + def destroy + reset + @integer.destroy + end + + def value + @num_to_sym.fetch(@integer.value) { raise ArgumentError } + end + + private + + def value=(sym) + @integer.value = @sym_to_num.fetch(sym) { raise ArgumentError } + end + + def initialize_lookup(symbol_list) + # Assume symbol_list[0] is mapped to 0 + # Cannot just use #object_id since #object_id for symbols is different in every run + # For now, implement a C-style enum type backed by integers + + @sym_to_num = Hash[symbol_list.each_with_index.to_a] + @num_to_sym = @sym_to_num.invert + end + end + end +end diff --git a/test/circuit_breaker_test.rb b/test/circuit_breaker_test.rb index 305c31b0..e79964eb 100644 --- a/test/circuit_breaker_test.rb +++ b/test/circuit_breaker_test.rb @@ -138,7 +138,7 @@ def test_shared_fresh_worker_killed_should_not_reset_circuit_breaker_data Process.waitall fork do Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) - assert_circuit_opened + assert_circuit_opened Semian[:unique_res] end Process.waitall diff --git a/test/simple_enum_test.rb b/test/simple_enum_test.rb deleted file mode 100644 index 6846a6dc..00000000 --- a/test/simple_enum_test.rb +++ /dev/null @@ -1,47 +0,0 @@ -require 'test_helper' - -class TestSimpleEnum < MiniTest::Unit::TestCase - CLASS = ::Semian::Simple::Enum - - def setup - @enum = CLASS.new(symbol_list: [:one, :two, :three]) - end - - def teardown - @enum.destroy - end - - module EnumTestCases - def test_assigning - old = @enum.value - @enum.value = @enum.value - assert_equal old, @enum.value - @enum.value = :two - assert_equal :two, @enum.value - end - - def test_iterate_enum - @enum.value = :one - @enum.increment - assert_equal :two, @enum.value - @enum.increment - assert_equal :three, @enum.value - @enum.increment - assert_equal :one, @enum.value - @enum.increment(2) - assert_equal :three, @enum.value - @enum.increment(4) - assert_equal :one, @enum.value - @enum.increment(0) - assert_equal :one, @enum.value - end - - def test_will_throw_error_when_invalid_symbol_given - assert_raises KeyError do - @enum.value = :four - end - end - end - - include EnumTestCases -end diff --git a/test/simple_state_test.rb b/test/simple_state_test.rb new file mode 100644 index 00000000..edab7c22 --- /dev/null +++ b/test/simple_state_test.rb @@ -0,0 +1,45 @@ +require 'test_helper' + +class TestSimpleState < MiniTest::Unit::TestCase + CLASS = ::Semian::Simple::State + + def setup + @state = CLASS.new + end + + def teardown + @state.destroy + end + + module StateTestCases + def test_start_closed? + assert @state.closed? + end + + def test_open + @state.open + assert @state.open? + assert_equal @state.value, :open + end + + def test_close + @state.close + assert @state.closed? + assert_equal @state.value, :closed + end + + def test_half_open + @state.half_open + assert @state.half_open? + assert_equal @state.value, :half_open + end + + def test_reset + @state.reset + assert @state.closed? + assert_equal @state.value, :closed + end + end + + include StateTestCases +end diff --git a/test/sysv_enum_test.rb b/test/sysv_enum_test.rb deleted file mode 100644 index ecf64784..00000000 --- a/test/sysv_enum_test.rb +++ /dev/null @@ -1,49 +0,0 @@ -require 'test_helper' - -class TestSysVEnum < MiniTest::Unit::TestCase - # Emulate sharedness to test correctness against real SysVAtomicEnum class - class FakeSysVAtomicEnum < Semian::Simple::Enum - class << self - attr_accessor :resources - end - self.resources = {} - attr_accessor :name - def self.new(symbol_list:, name:, permissions:) - obj = resources[name] ||= super - obj.name = name - obj - end - - def destroy - self.class.resources.delete(@name) - end - - def shared? - true - end - end - - CLASS = ::Semian::SysV::Enum - - def setup - @enum = CLASS.new(symbol_list: [:one, :two, :three], - name: 'TestAtomicEnum', - permissions: 0660) - end - - def teardown - @enum.destroy - end - - include TestSimpleEnum::EnumTestCases - - def test_memory_is_shared - assert_equal :one, @enum.value - @enum.value = :three - - enum_2 = CLASS.new(symbol_list: [:one, :two, :three], - name: 'TestAtomicEnum', - permissions: 0660) - assert_equal :three, enum_2.value - end -end diff --git a/test/sysv_integer_test.rb b/test/sysv_integer_test.rb index a36bc653..c445193d 100644 --- a/test/sysv_integer_test.rb +++ b/test/sysv_integer_test.rb @@ -1,32 +1,10 @@ require 'test_helper' class TestSysVInteger < MiniTest::Unit::TestCase - # Emulate sharedness to test correctness against real SysVAtomicInteger class - class FakeSysVAtomicInteger < Semian::Simple::Integer - class << self - attr_accessor :resources - end - self.resources = {} - attr_accessor :name - def self.new(name:, permissions:) - obj = resources[name] ||= super - obj.name = name - obj - end - - def destroy - self.class.resources.delete(@name) - end - - def shared? - true - end - end - CLASS = ::Semian::SysV::Integer def setup - @integer = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) + @integer = CLASS.new(name: 'TestSysVInteger', permissions: 0660) @integer.value = 0 end @@ -37,7 +15,7 @@ def teardown include TestSimpleInteger::IntegerTestCases def test_memory_is_shared - integer_2 = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) + integer_2 = CLASS.new(name: 'TestSysVInteger', permissions: 0660) integer_2.value = 100 assert_equal 100, @integer.value @integer.value = 200 @@ -48,10 +26,10 @@ def test_memory_is_shared def test_memory_not_reset_when_at_least_one_worker_using_it @integer.value = 109 - integer_2 = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) + integer_2 = CLASS.new(name: 'TestSysVInteger', permissions: 0660) assert_equal @integer.value, integer_2.value pid = fork do - integer_3 = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) + integer_3 = CLASS.new(name: 'TestSysVInteger', permissions: 0660) assert_equal 109, integer_3.value sleep end @@ -59,7 +37,7 @@ def test_memory_not_reset_when_at_least_one_worker_using_it Process.kill("KILL", pid) Process.waitall fork do - integer_3 = CLASS.new(name: 'TestAtomicInteger', permissions: 0660) + integer_3 = CLASS.new(name: 'TestSysVInteger', permissions: 0660) assert_equal 109, integer_3.value end Process.waitall diff --git a/test/sysv_sliding_window_test.rb b/test/sysv_sliding_window_test.rb index 888f6d99..3ee7be64 100644 --- a/test/sysv_sliding_window_test.rb +++ b/test/sysv_sliding_window_test.rb @@ -1,34 +1,11 @@ require 'test_helper' class TestSysVSlidingWindow < MiniTest::Unit::TestCase - # Emulate sharedness to test correctness against real SysVSlidingWindow class - class FakeSysVSlidingWindow < Semian::Simple::SlidingWindow - class << self - attr_accessor :resources - end - self.resources = {} - attr_accessor :name - def self.new(max_size:, name:, permissions:) - obj = resources[name] ||= super - obj.name = name - obj.resize_to(max_size) - obj - end - - def destroy - self.class.resources.delete(@name) - end - - def shared? - true - end - end - CLASS = ::Semian::SysV::SlidingWindow def setup @sliding_window = CLASS.new(max_size: 6, - name: 'TestSlidingWindow', + name: 'TestSysVSlidingWindow', permissions: 0660) @sliding_window.clear end @@ -47,7 +24,7 @@ def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it pid = fork do sliding_window_2 = CLASS.new(max_size: 6, - name: 'TestSlidingWindow', + name: 'TestSysVSlidingWindow', permissions: 0660) sliding_window_2.execute_atomically { sleep } end @@ -65,7 +42,7 @@ def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it def test_sliding_window_memory_is_actually_shared assert_equal 0, @sliding_window.size sliding_window_2 = CLASS.new(max_size: 6, - name: 'TestSlidingWindow', + name: 'TestSysVSlidingWindow', permissions: 0660) assert_equal 0, sliding_window_2.size @@ -85,14 +62,14 @@ def test_sliding_window_memory_is_actually_shared def test_restarting_worker_should_not_reset_queue @sliding_window << 10 << 20 << 30 sliding_window_2 = CLASS.new(max_size: 6, - name: 'TestSlidingWindow', + name: 'TestSysVSlidingWindow', permissions: 0660) assert_correct_first_and_last_and_size(@sliding_window, 10, 30, 3, 6) sliding_window_2.pop assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) sliding_window_3 = CLASS.new(max_size: 6, - name: 'TestSlidingWindow', + name: 'TestSysVSlidingWindow', permissions: 0660) assert_correct_first_and_last_and_size(@sliding_window, 10, 20, 2, 6) sliding_window_3.pop @@ -105,14 +82,14 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down # Test explicit resizing, and resizing through making new memory associations sliding_window_2 = CLASS.new(max_size: 4, - name: 'TestSlidingWindow', + name: 'TestSysVSlidingWindow', permissions: 0660) sliding_window_2 << 80 << 90 << 100 << 110 << 120 assert_correct_first_and_last_and_size(@sliding_window, 90, 120, 4, 4) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) sliding_window_2 = CLASS.new(max_size: 3, - name: 'TestSlidingWindow', + name: 'TestSysVSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) assert_correct_first_and_last_and_size(@sliding_window, 100, 120, 3, 3) @@ -122,7 +99,7 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 2) sliding_window_2 = CLASS.new(max_size: 4, - name: 'TestSlidingWindow', + name: 'TestSysVSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 4) @@ -133,7 +110,7 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down assert_correct_first_and_last_and_size(@sliding_window, 110, 130, 3, 6) sliding_window_2 = CLASS.new(max_size: 2, - name: 'TestSlidingWindow', + name: 'TestSysVSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 2) @@ -143,7 +120,7 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 4) sliding_window_2 = CLASS.new(max_size: 6, - name: 'TestSlidingWindow', + name: 'TestSysVSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 6) diff --git a/test/sysv_state_test.rb b/test/sysv_state_test.rb new file mode 100644 index 00000000..a5b77e3b --- /dev/null +++ b/test/sysv_state_test.rb @@ -0,0 +1,44 @@ +require 'test_helper' + +class TestSysVState < MiniTest::Unit::TestCase + CLASS = ::Semian::SysV::State + + def setup + @state = CLASS.new(name: 'TestSysVState', + permissions: 0660) + end + + def teardown + @state.destroy + end + + include TestSimpleState::StateTestCases + + def test_memory_is_shared + assert_equal :closed, @state.value + @state.open + + state_2 = CLASS.new(name: 'TestSysVState', + permissions: 0660) + assert_equal :open, state_2.value + assert state_2.open? + end + + def test_will_throw_error_when_invalid_symbol_given + # May occur if underlying integer gets into bad state + integer = @state.instance_eval "@integer" + integer.value = 100 + assert_raises ArgumentError do + @state.value + end + assert_raises ArgumentError do + @state.open? + end + assert_raises ArgumentError do + @state.half_open? + end + assert_raises ArgumentError do + @state.closed? + end + end +end From 4896c3c36afc39dddbe7a54c523ebe3f244825e9 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Thu, 12 Nov 2015 18:21:13 +0000 Subject: [PATCH 14/23] Rename SharedMemoryObject to SysVSharedMemory and made it a mixin module, remove "Atomic" naming, change execute_atomically to synchronize --- ...mian_atomic_integer.c => semian_integer.c} | 51 ++++++++++--------- ext/semian/semian_shared_memory_object.c | 37 ++++++++------ ext/semian/semian_shared_memory_object.h | 3 +- ext/semian/semian_sliding_window.c | 3 ++ lib/semian.rb | 2 +- lib/semian/circuit_breaker.rb | 2 +- lib/semian/simple_integer.rb | 8 +-- lib/semian/simple_sliding_window.rb | 20 ++++---- lib/semian/simple_state.rb | 4 +- lib/semian/sysv_integer.rb | 2 + ...memory_object.rb => sysv_shared_memory.rb} | 22 ++++---- lib/semian/sysv_sliding_window.rb | 4 +- lib/semian/sysv_state.rb | 5 +- test/sysv_sliding_window_test.rb | 2 +- 14 files changed, 91 insertions(+), 74 deletions(-) rename ext/semian/{semian_atomic_integer.c => semian_integer.c} (52%) rename lib/semian/{shared_memory_object.rb => sysv_shared_memory.rb} (75%) diff --git a/ext/semian/semian_atomic_integer.c b/ext/semian/semian_integer.c similarity index 52% rename from ext/semian/semian_atomic_integer.c rename to ext/semian/semian_integer.c index e65fd462..a94b5377 100644 --- a/ext/semian/semian_atomic_integer.c +++ b/ext/semian/semian_integer.c @@ -2,19 +2,19 @@ typedef struct { int value; -} semian_atomic_int; +} semian_int; -static void semian_atomic_integer_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); -static VALUE semian_atomic_integer_bind_init_fn_wrapper(VALUE self); -static VALUE semian_atomic_integer_get_value(VALUE self); -static VALUE semian_atomic_integer_set_value(VALUE self, VALUE num); -static VALUE semian_atomic_integer_increment(int argc, VALUE *argv, VALUE self); +static void semian_integer_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); +static VALUE semian_integer_bind_init_fn_wrapper(VALUE self); +static VALUE semian_integer_get_value(VALUE self); +static VALUE semian_integer_set_value(VALUE self, VALUE num); +static VALUE semian_integer_increment(int argc, VALUE *argv, VALUE self); static void -semian_atomic_integer_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count) +semian_integer_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count) { - semian_atomic_int *ptr = dest; - semian_atomic_int *old = prev_data; + semian_int *ptr = dest; + semian_int *old = prev_data; if (prev_mem_attach_count){ if (prev_data){ ptr->value = old->value; @@ -25,16 +25,16 @@ semian_atomic_integer_bind_init_fn (size_t byte_size, void *dest, void *prev_dat } static VALUE -semian_atomic_integer_bind_init_fn_wrapper(VALUE self) +semian_integer_bind_init_fn_wrapper(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - ptr->object_init_fn = &semian_atomic_integer_bind_init_fn; + ptr->object_init_fn = &semian_integer_bind_init_fn; return self; } static VALUE -semian_atomic_integer_get_value(VALUE self) +semian_integer_get_value(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); @@ -47,13 +47,13 @@ semian_atomic_integer_get_value(VALUE self) if (!semian_shm_object_lock(self)) return Qnil; - int value = ((semian_atomic_int *)(ptr->shm_address))->value; + int value = ((semian_int *)(ptr->shm_address))->value; semian_shm_object_unlock(self); return INT2NUM(value); } static VALUE -semian_atomic_integer_set_value(VALUE self, VALUE num) +semian_integer_set_value(VALUE self, VALUE num) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); @@ -67,14 +67,14 @@ semian_atomic_integer_set_value(VALUE self, VALUE num) if (!semian_shm_object_lock(self)) return Qnil; - ((semian_atomic_int *)(ptr->shm_address))->value = NUM2INT(num); + ((semian_int *)(ptr->shm_address))->value = NUM2INT(num); semian_shm_object_unlock(self); return num; } static VALUE -semian_atomic_integer_increment(int argc, VALUE *argv, VALUE self) +semian_integer_increment(int argc, VALUE *argv, VALUE self) { VALUE num; rb_scan_args(argc, argv, "01", &num); @@ -93,22 +93,25 @@ semian_atomic_integer_increment(int argc, VALUE *argv, VALUE self) if (!semian_shm_object_lock(self)) return Qnil; - ((semian_atomic_int *)(ptr->shm_address))->value += NUM2INT(num); + ((semian_int *)(ptr->shm_address))->value += NUM2INT(num); semian_shm_object_unlock(self); return self; } void -Init_semian_atomic_integer (void) +Init_semian_integer (void) { - // Bind methods to AtomicInteger + // Bind methods to Integer VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); + VALUE cSysVSharedMemory = rb_const_get(cSemianModule, rb_intern("SysVSharedMemory")); VALUE cSysVModule = rb_const_get(cSemianModule, rb_intern("SysV")); - VALUE cAtomicInteger = rb_const_get(cSysVModule, rb_intern("Integer")); + VALUE cInteger = rb_const_get(cSysVModule, rb_intern("Integer")); - rb_define_method(cAtomicInteger, "bind_init_fn", semian_atomic_integer_bind_init_fn_wrapper, 0); - rb_define_method(cAtomicInteger, "value", semian_atomic_integer_get_value, 0); - rb_define_method(cAtomicInteger, "value=", semian_atomic_integer_set_value, 1); - rb_define_method(cAtomicInteger, "increment", semian_atomic_integer_increment, -1); + semian_shm_object_replace_alloc(cSysVSharedMemory, cInteger); + + rb_define_method(cInteger, "bind_init_fn", semian_integer_bind_init_fn_wrapper, 0); + rb_define_method(cInteger, "value", semian_integer_get_value, 0); + rb_define_method(cInteger, "value=", semian_integer_set_value, 1); + rb_define_method(cInteger, "increment", semian_integer_increment, -1); } diff --git a/ext/semian/semian_shared_memory_object.c b/ext/semian/semian_shared_memory_object.c index e605e066..f98850b1 100644 --- a/ext/semian/semian_shared_memory_object.c +++ b/ext/semian/semian_shared_memory_object.c @@ -67,6 +67,12 @@ semian_shm_object_alloc(VALUE klass) * Implementations */ +VALUE +semian_shm_object_replace_alloc(VALUE klass, VALUE target) +{ + rb_define_alloc_func(target, semian_shm_object_alloc); + return target; +} VALUE semian_shm_object_sizeof(VALUE klass, VALUE type) @@ -576,7 +582,7 @@ semian_shm_object_byte_size(VALUE self) } static VALUE -semian_shm_object_execute_atomically_with_block(VALUE self) +semian_shm_object_synchronize_with_block(VALUE self) { if (!rb_block_given_p()) rb_raise(rb_eArgError, "Expected block"); @@ -584,30 +590,31 @@ semian_shm_object_execute_atomically_with_block(VALUE self) } static VALUE -semian_shm_object_execute_atomically(VALUE self) { +semian_shm_object_synchronize(VALUE self) { if (!semian_shm_object_lock(self)) return Qnil; - return rb_ensure(semian_shm_object_execute_atomically_with_block, self, semian_shm_object_unlock_all, self); + return rb_ensure(semian_shm_object_synchronize_with_block, self, semian_shm_object_unlock_all, self); } void Init_semian_shm_object (void) { VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); - VALUE cSharedMemoryObject = rb_const_get(cSemianModule, rb_intern("SharedMemoryObject")); + VALUE cSysVSharedMemory = rb_const_get(cSemianModule, rb_intern("SysVSharedMemory")); - rb_define_alloc_func(cSharedMemoryObject, semian_shm_object_alloc); - rb_define_method(cSharedMemoryObject, "_acquire", semian_shm_object_acquire, 3); - rb_define_method(cSharedMemoryObject, "_destroy", semian_shm_object_destroy, 0); - rb_define_method(cSharedMemoryObject, "lock", semian_shm_object_lock, 0); - rb_define_method(cSharedMemoryObject, "unlock", semian_shm_object_unlock, 0); - rb_define_method(cSharedMemoryObject, "byte_size", semian_shm_object_byte_size, 0); + //rb_define_alloc_func(cSysVSharedMemory, semian_shm_object_alloc); + rb_define_method(cSysVSharedMemory, "_acquire", semian_shm_object_acquire, 3); + rb_define_method(cSysVSharedMemory, "_destroy", semian_shm_object_destroy, 0); + rb_define_method(cSysVSharedMemory, "lock", semian_shm_object_lock, 0); + rb_define_method(cSysVSharedMemory, "unlock", semian_shm_object_unlock, 0); + rb_define_method(cSysVSharedMemory, "byte_size", semian_shm_object_byte_size, 0); - rb_define_method(cSharedMemoryObject, "semid", semian_shm_object_semid, 0); - rb_define_method(cSharedMemoryObject, "shmid", semian_shm_object_shmid, 0); - rb_define_method(cSharedMemoryObject, "_execute_atomically", semian_shm_object_execute_atomically, 0); + rb_define_method(cSysVSharedMemory, "semid", semian_shm_object_semid, 0); + rb_define_method(cSysVSharedMemory, "shmid", semian_shm_object_shmid, 0); + rb_define_method(cSysVSharedMemory, "_synchronize", semian_shm_object_synchronize, 0); - rb_define_singleton_method(cSharedMemoryObject, "_sizeof", semian_shm_object_sizeof, 1); + rb_define_singleton_method(cSysVSharedMemory, "_sizeof", semian_shm_object_sizeof, 1); + rb_define_singleton_method(cSysVSharedMemory, "replace_alloc", semian_shm_object_replace_alloc, 1); decrement.sem_num = kSHMIndexTicketLock; decrement.sem_op = -1; @@ -617,7 +624,7 @@ Init_semian_shm_object (void) { increment.sem_op = 1; increment.sem_flg = SEM_UNDO; - Init_semian_atomic_integer(); + Init_semian_integer(); Init_semian_sliding_window(); } diff --git a/ext/semian/semian_shared_memory_object.h b/ext/semian/semian_shared_memory_object.h index 5f26ce5d..6281c7c7 100644 --- a/ext/semian/semian_shared_memory_object.h +++ b/ext/semian/semian_shared_memory_object.h @@ -20,6 +20,7 @@ semian_shm_object_type; */ VALUE semian_shm_object_sizeof(VALUE klass, VALUE type); +VALUE semian_shm_object_replace_alloc(VALUE klass, VALUE target); VALUE semian_shm_object_acquire(VALUE self, VALUE id, VALUE byte_size, VALUE permissions); VALUE semian_shm_object_destroy(VALUE self); @@ -33,5 +34,5 @@ void semian_shm_object_delete_memory_inner (semian_shm_object *ptr, int should_u VALUE semian_shm_object_delete_memory (VALUE self); VALUE semian_shm_object_check_and_resize_if_needed (VALUE self); -void Init_semian_atomic_integer (void); +void Init_semian_integer (void); void Init_semian_sliding_window (void); diff --git a/ext/semian/semian_sliding_window.c b/ext/semian/semian_sliding_window.c index 7eb7a28d..81a06dbf 100644 --- a/ext/semian/semian_sliding_window.c +++ b/ext/semian/semian_sliding_window.c @@ -296,8 +296,11 @@ Init_semian_sliding_window (void) { VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); VALUE cSysVModule = rb_const_get(cSemianModule, rb_intern("SysV")); + VALUE cSysVSharedMemory = rb_const_get(cSemianModule, rb_intern("SysVSharedMemory")); VALUE cSlidingWindow = rb_const_get(cSysVModule, rb_intern("SlidingWindow")); + semian_shm_object_replace_alloc(cSysVSharedMemory, cSlidingWindow); + rb_define_method(cSlidingWindow, "bind_init_fn", semian_sliding_window_bind_init_fn_wrapper, 0); rb_define_method(cSlidingWindow, "size", semian_sliding_window_size, 0); rb_define_method(cSlidingWindow, "max_size", semian_sliding_window_max_size, 0); diff --git a/lib/semian.rb b/lib/semian.rb index c16ff802..aa5efde3 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -157,10 +157,10 @@ def resources require 'semian/protected_resource' require 'semian/unprotected_resource' require 'semian/platform' -require 'semian/shared_memory_object' require 'semian/simple_sliding_window' require 'semian/simple_integer' require 'semian/simple_state' +require 'semian/sysv_shared_memory' require 'semian/sysv_sliding_window' require 'semian/sysv_integer' require 'semian/sysv_state' diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index b6567314..48073445 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -102,7 +102,7 @@ def error_timeout_expired? end def push_time(window, duration:, time: Time.now) - @errors.execute_atomically do # Store an integer amount of milliseconds since epoch + @errors.synchronize do # Store an integer amount of milliseconds since epoch window.shift while window.first && window.first / 1000 + duration < time.to_i window << (time.to_f * 1000).to_i end diff --git a/lib/semian/simple_integer.rb b/lib/semian/simple_integer.rb index ea27d6eb..44d17a25 100644 --- a/lib/semian/simple_integer.rb +++ b/lib/semian/simple_integer.rb @@ -1,6 +1,6 @@ module Semian module Simple - class Integer < SharedMemoryObject #:nodoc: + class Integer #:nodoc: attr_accessor :value def initialize @@ -16,11 +16,7 @@ def reset end def destroy - if shared? - super - else - @value = 0 - end + reset end end end diff --git a/lib/semian/simple_sliding_window.rb b/lib/semian/simple_sliding_window.rb index e551c8f6..0da81970 100644 --- a/lib/semian/simple_sliding_window.rb +++ b/lib/semian/simple_sliding_window.rb @@ -1,20 +1,22 @@ +require 'forwardable' + module Semian module Simple - class SlidingWindow < SharedMemoryObject #:nodoc: + class SlidingWindow #:nodoc: extend Forwardable def_delegators :@window, :size, :pop, :shift, :first, :last attr_reader :max_size + # A sliding window is a structure that stores the most @max_size recent timestamps + # like this: if @max_size = 4, current time is 10, @window =[5,7,9,10]. + # Another push of (11) at 11 sec would make @window [7,9,10,11], shifting off 5. + def initialize(max_size:) @max_size = max_size @window = [] end - # A sliding window is a structure that stores the most @max_size recent timestamps - # like this: if @max_size = 4, current time is 10, @window =[5,7,9,10]. - # Another push of (11) at 11 sec would make @window [7,9,10,11], shifting off 5. - def resize_to(size) raise ArgumentError.new('size must be larger than 0') if size < 1 @max_size = size @@ -31,16 +33,12 @@ def push(time_ms) alias_method :<<, :push def clear - @window = [] + @window.clear self end def destroy - if shared? - super - else - clear - end + clear end end end diff --git a/lib/semian/simple_state.rb b/lib/semian/simple_state.rb index 59d6774a..d97f4960 100644 --- a/lib/semian/simple_state.rb +++ b/lib/semian/simple_state.rb @@ -1,8 +1,6 @@ -require 'forwardable' - module Semian module Simple - class State < SharedMemoryObject #:nodoc: + class State #:nodoc: def initialize reset end diff --git a/lib/semian/sysv_integer.rb b/lib/semian/sysv_integer.rb index 6f91bc2b..72554cc7 100644 --- a/lib/semian/sysv_integer.rb +++ b/lib/semian/sysv_integer.rb @@ -1,6 +1,8 @@ module Semian module SysV class Integer < Semian::Simple::Integer #:nodoc: + include SysVSharedMemory + def initialize(name:, permissions:) data_layout = [:int] super() unless acquire_memory_object(name, data_layout, permissions) diff --git a/lib/semian/shared_memory_object.rb b/lib/semian/sysv_shared_memory.rb similarity index 75% rename from lib/semian/shared_memory_object.rb rename to lib/semian/sysv_shared_memory.rb index bad3433f..07b896ae 100644 --- a/lib/semian/shared_memory_object.rb +++ b/lib/semian/sysv_shared_memory.rb @@ -1,5 +1,5 @@ module Semian - class SharedMemoryObject #:nodoc: + module SysVSharedMemory #:nodoc: @type_size = {} def self.sizeof(type) size = (@type_size[type.to_sym] ||= (respond_to?(:_sizeof) ? _sizeof(type.to_sym) : 0)) @@ -15,30 +15,34 @@ def shmid -1 end - def execute_atomically(&proc) - if respond_to?(:_execute_atomically) && @using_shared_memory - return _execute_atomically(&proc) + def synchronize(&proc) + if respond_to?(:_synchronize) && @using_shared_memory + return _synchronize(&proc) else - call_with_mutex(&proc) + yield if block_given? end end - alias_method :transaction, :execute_atomically + alias_method :transaction, :synchronize def destroy - _destroy if respond_to?(:_destroy) && @using_shared_memory + if respond_to?(:_destroy) && @using_shared_memory + _destroy + else + super + end end private def shared? - @using_shared_memory ||= Semian.semaphores_enabled? && @using_shared_memory + @using_shared_memory end def acquire_memory_object(name, data_layout, permissions) return @using_shared_memory = false unless Semian.semaphores_enabled? && respond_to?(:_acquire) - byte_size = data_layout.inject(0) { |sum, type| sum + ::Semian::SharedMemoryObject.sizeof(type) } + byte_size = data_layout.inject(0) { |sum, type| sum + ::Semian::SysVSharedMemory.sizeof(type) } raise TypeError.new("Given data layout is 0 bytes: #{data_layout.inspect}") if byte_size <= 0 # Calls C layer to acquire/create a memory block, calling #bind_init_fn in the process, see below _acquire(name, byte_size, permissions) diff --git a/lib/semian/sysv_sliding_window.rb b/lib/semian/sysv_sliding_window.rb index 4a9659ac..85089553 100644 --- a/lib/semian/sysv_sliding_window.rb +++ b/lib/semian/sysv_sliding_window.rb @@ -1,9 +1,11 @@ module Semian module SysV class SlidingWindow < Semian::Simple::SlidingWindow #:nodoc: + include SysVSharedMemory + def initialize(max_size:, name:, permissions:) data_layout = [:int, :int].concat(Array.new(max_size, :long)) - super(max_size) unless acquire_memory_object(name, data_layout, permissions) + super(max_size: max_size) unless acquire_memory_object(name, data_layout, permissions) end end end diff --git a/lib/semian/sysv_state.rb b/lib/semian/sysv_state.rb index 2d0f4568..0cd267c0 100644 --- a/lib/semian/sysv_state.rb +++ b/lib/semian/sysv_state.rb @@ -1,9 +1,12 @@ +require 'forwardable' + module Semian module SysV class State < Semian::Simple::State #:nodoc: + include SysVSharedMemory extend Forwardable - def_delegators :@integer, :semid, :shmid, :execute_atomically, :transaction, + def_delegators :@integer, :semid, :shmid, :synchronize, :transaction, :shared?, :acquire_memory_object, :bind_init_fn private :shared?, :acquire_memory_object, :bind_init_fn diff --git a/test/sysv_sliding_window_test.rb b/test/sysv_sliding_window_test.rb index 3ee7be64..0c4f29fa 100644 --- a/test/sysv_sliding_window_test.rb +++ b/test/sysv_sliding_window_test.rb @@ -26,7 +26,7 @@ def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it sliding_window_2 = CLASS.new(max_size: 6, name: 'TestSysVSlidingWindow', permissions: 0660) - sliding_window_2.execute_atomically { sleep } + sliding_window_2.synchronize { sleep } end sleep 1 From 1aa5cf4c513b3a9aeed7057f99a5b13c97087f50 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Thu, 12 Nov 2015 19:23:36 +0000 Subject: [PATCH 15/23] Small Changes --- ext/semian/semian_integer.c | 7 +++++++ lib/semian/circuit_breaker.rb | 5 +++-- lib/semian/simple_integer.rb | 4 ++-- lib/semian/simple_sliding_window.rb | 6 +++--- lib/semian/simple_state.rb | 2 +- test/simple_integer_test.rb | 13 ++++++++++--- 6 files changed, 26 insertions(+), 11 deletions(-) diff --git a/ext/semian/semian_integer.c b/ext/semian/semian_integer.c index a94b5377..91fd8d21 100644 --- a/ext/semian/semian_integer.c +++ b/ext/semian/semian_integer.c @@ -73,6 +73,12 @@ semian_integer_set_value(VALUE self, VALUE num) return num; } +static VALUE +semian_integer_reset(VALUE self) +{ + return semian_integer_set_value(self, INT2NUM(0)); +} + static VALUE semian_integer_increment(int argc, VALUE *argv, VALUE self) { @@ -113,5 +119,6 @@ Init_semian_integer (void) rb_define_method(cInteger, "bind_init_fn", semian_integer_bind_init_fn_wrapper, 0); rb_define_method(cInteger, "value", semian_integer_get_value, 0); rb_define_method(cInteger, "value=", semian_integer_set_value, 1); + rb_define_method(cInteger, "reset", semian_integer_reset, 0); rb_define_method(cInteger, "increment", semian_integer_increment, -1); } diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 48073445..be74ec95 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -3,7 +3,7 @@ class CircuitBreaker #:nodoc: extend Forwardable def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, permissions:, implementation:) - @name = name.to_s + @name = name.to_sym @success_count_threshold = success_threshold @error_count_threshold = error_threshold @error_timeout = error_timeout @@ -102,7 +102,8 @@ def error_timeout_expired? end def push_time(window, duration:, time: Time.now) - @errors.synchronize do # Store an integer amount of milliseconds since epoch + # The sliding window stores the integer amount of milliseconds since epoch as a timestamp + @errors.synchronize do window.shift while window.first && window.first / 1000 + duration < time.to_i window << (time.to_f * 1000).to_i end diff --git a/lib/semian/simple_integer.rb b/lib/semian/simple_integer.rb index 44d17a25..d2814aec 100644 --- a/lib/semian/simple_integer.rb +++ b/lib/semian/simple_integer.rb @@ -3,8 +3,8 @@ module Simple class Integer #:nodoc: attr_accessor :value - def initialize - @value = 0 + def initialize(**_) + reset end def increment(val = 1) diff --git a/lib/semian/simple_sliding_window.rb b/lib/semian/simple_sliding_window.rb index 0da81970..e3de9978 100644 --- a/lib/semian/simple_sliding_window.rb +++ b/lib/semian/simple_sliding_window.rb @@ -12,7 +12,7 @@ class SlidingWindow #:nodoc: # like this: if @max_size = 4, current time is 10, @window =[5,7,9,10]. # Another push of (11) at 11 sec would make @window [7,9,10,11], shifting off 5. - def initialize(max_size:) + def initialize(max_size:, **_) @max_size = max_size @window = [] end @@ -24,9 +24,9 @@ def resize_to(size) self end - def push(time_ms) + def push(value) @window.shift while @window.size >= @max_size - @window << time_ms + @window << value self end diff --git a/lib/semian/simple_state.rb b/lib/semian/simple_state.rb index d97f4960..e8880d48 100644 --- a/lib/semian/simple_state.rb +++ b/lib/semian/simple_state.rb @@ -1,7 +1,7 @@ module Semian module Simple class State #:nodoc: - def initialize + def initialize(**_) reset end diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index a571ede3..195051d9 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -5,7 +5,6 @@ class TestSimpleInteger < MiniTest::Unit::TestCase def setup @integer = CLASS.new - @integer.value = 0 end def teardown @@ -14,7 +13,6 @@ def teardown module IntegerTestCases def test_access_value - @integer.value = 0 assert_equal(0, @integer.value) @integer.value = 99 assert_equal(99, @integer.value) @@ -28,7 +26,6 @@ def test_access_value end def test_increment - @integer.value = 0 @integer.increment(4) assert_equal(4, @integer.value) @integer.increment @@ -36,6 +33,16 @@ def test_increment @integer.increment(-2) assert_equal(3, @integer.value) end + + def test_reset_on_init + assert_equal(0, @integer.value) + end + + def test_reset + @integer.increment(5) + @integer.reset + assert_equal(0, @integer.value) + end end include IntegerTestCases From 6e49d3d6c1f04c18dd5f067d7e6f32827284716e Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Fri, 13 Nov 2015 21:54:15 +0000 Subject: [PATCH 16/23] Unified use of #synchronize, removed potentially unsafe lock and unlock functions, trimmed C code, introduced a define_method_with_synchronize --- ext/semian/semian_integer.c | 21 +-- ext/semian/semian_shared_memory_object.c | 199 +++++++++++------------ ext/semian/semian_shared_memory_object.h | 14 +- ext/semian/semian_sliding_window.c | 71 ++------ lib/semian/sysv_shared_memory.rb | 19 ++- test/sysv_integer_test.rb | 12 +- test/sysv_sliding_window_test.rb | 57 ++++--- test/sysv_state_test.rb | 1 + 8 files changed, 178 insertions(+), 216 deletions(-) diff --git a/ext/semian/semian_integer.c b/ext/semian/semian_integer.c index 91fd8d21..d09f6a27 100644 --- a/ext/semian/semian_integer.c +++ b/ext/semian/semian_integer.c @@ -43,12 +43,7 @@ semian_integer_get_value(VALUE self) if (0 == ptr->shm_address) return Qnil; - semian_shm_object_check_and_resize_if_needed(self); - if (!semian_shm_object_lock(self)) - return Qnil; - int value = ((semian_int *)(ptr->shm_address))->value; - semian_shm_object_unlock(self); return INT2NUM(value); } @@ -63,13 +58,9 @@ semian_integer_set_value(VALUE self, VALUE num) if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT) return Qnil; - semian_shm_object_check_and_resize_if_needed(self); - if (!semian_shm_object_lock(self)) - return Qnil; ((semian_int *)(ptr->shm_address))->value = NUM2INT(num); - semian_shm_object_unlock(self); return num; } @@ -95,13 +86,9 @@ semian_integer_increment(int argc, VALUE *argv, VALUE self) if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT) return Qnil; - semian_shm_object_check_and_resize_if_needed(self); - if (!semian_shm_object_lock(self)) - return Qnil; ((semian_int *)(ptr->shm_address))->value += NUM2INT(num); - semian_shm_object_unlock(self); return self; } @@ -117,8 +104,8 @@ Init_semian_integer (void) semian_shm_object_replace_alloc(cSysVSharedMemory, cInteger); rb_define_method(cInteger, "bind_init_fn", semian_integer_bind_init_fn_wrapper, 0); - rb_define_method(cInteger, "value", semian_integer_get_value, 0); - rb_define_method(cInteger, "value=", semian_integer_set_value, 1); - rb_define_method(cInteger, "reset", semian_integer_reset, 0); - rb_define_method(cInteger, "increment", semian_integer_increment, -1); + define_method_with_synchronize(cInteger, "value", semian_integer_get_value, 0); + define_method_with_synchronize(cInteger, "value=", semian_integer_set_value, 1); + define_method_with_synchronize(cInteger, "reset", semian_integer_reset, 0); + define_method_with_synchronize(cInteger, "increment", semian_integer_increment, -1); } diff --git a/ext/semian/semian_shared_memory_object.c b/ext/semian/semian_shared_memory_object.c index f98850b1..e687e06b 100644 --- a/ext/semian/semian_shared_memory_object.c +++ b/ext/semian/semian_shared_memory_object.c @@ -49,9 +49,7 @@ semian_shm_object_free(void *ptr) static size_t semian_shm_object_memsize(const void *ptr) { - return - - sizeof(semian_shm_object); + return sizeof(semian_shm_object); } static VALUE semian_shm_object_alloc(VALUE klass) @@ -85,6 +83,7 @@ semian_shm_object_sizeof(VALUE klass, VALUE type) return INT2NUM(sizeof(int)); else if (rb_intern("long") == SYM2ID(type)) return INT2NUM(sizeof(long)); + // Can definitely add more else return INT2NUM(0); } @@ -122,7 +121,7 @@ semian_shm_object_acquire(VALUE self, VALUE name, VALUE byte_size, VALUE permiss ptr->shmid = -1; // addresses default to NULL ptr->shm_address = 0; - ptr->lock_triggered = 0; + ptr->lock_count = 0; ptr->permissions = FIX2LONG(permissions); // Will throw NotImplementedError if not defined in concrete subclasses @@ -131,8 +130,8 @@ semian_shm_object_acquire(VALUE self, VALUE name, VALUE byte_size, VALUE permiss // that binds ptr->object_init_fn to an init function called after memory is made rb_funcall(self, rb_intern("bind_init_fn"), 0); - semian_shm_object_acquire_semaphore(self, permissions); - semian_shm_object_acquire_memory(self, permissions, Qtrue); + semian_shm_object_acquire_semaphore(self); + semian_shm_object_synchronize(self); return self; } @@ -147,6 +146,9 @@ semian_shm_object_destroy(VALUE self) return result; } +/* + * Create or acquire previously made semaphore + */ static int create_semaphore_and_initialize_and_set_permissions(int key, int permissions) @@ -175,12 +177,8 @@ create_semaphore_and_initialize_and_set_permissions(int key, int permissions) } -/* - * Create or acquire previously made semaphore - */ - VALUE -semian_shm_object_acquire_semaphore (VALUE self, VALUE permissions) +semian_shm_object_acquire_semaphore (VALUE self) { // Function flow, semaphore creation methods are // borrowed from semian.c since they have been previously tested @@ -188,11 +186,8 @@ semian_shm_object_acquire_semaphore (VALUE self, VALUE permissions) semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - if (TYPE(permissions) != T_FIXNUM) - rb_raise(rb_eTypeError, "expected integer for permissions"); - key_t key = ptr->key; - int semid = create_semaphore_and_initialize_and_set_permissions(key, FIX2LONG(permissions)); + int semid = create_semaphore_and_initialize_and_set_permissions(key, ptr->permissions); if (-1 == semid) { raise_semian_syscall_error("semget()", errno); } @@ -225,7 +220,8 @@ semian_shm_object_delete_semaphore(VALUE self) } /* - * semian_shm_object_lock/unlock and associated functions decrement/increment semaphore + * lock & unlock functions, should be called like + * (VALUE) WITHOUT_GVL(semian_shm_object_unlock_without_gvl, (void *)ptr, RUBY_UBF_IO, NULL) */ static void * @@ -240,29 +236,21 @@ semian_shm_object_lock_without_gvl(void *v_ptr) struct timespec ts = { 0 }; ts.tv_sec = kSHMInternalTimeout; - if (0 == ptr->lock_triggered) { + if (0 == ptr->lock_count) { if (-1 == semtimedop(ptr->semid,&decrement,1, &ts)) { rb_raise(eInternal, "error acquiring semaphore lock to mutate circuit breaker structure, %d: (%s)", errno, strerror(errno)); retval=Qfalse; } else { - (ptr->lock_triggered)+=1; + (ptr->lock_count)+=1; retval=Qtrue; } } else { - (ptr->lock_triggered)+=1; + (ptr->lock_count)+=1; retval=Qtrue; } return (void *)retval; } -VALUE -semian_shm_object_lock(VALUE self) -{ - semian_shm_object *ptr; - TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - return (VALUE) WITHOUT_GVL(semian_shm_object_lock_without_gvl, (void *)ptr, RUBY_UBF_IO, NULL); -} - static void * semian_shm_object_unlock_without_gvl(void *v_ptr) { @@ -273,42 +261,76 @@ semian_shm_object_unlock_without_gvl(void *v_ptr) } VALUE retval; - if (1 == ptr->lock_triggered) { + if (1 == ptr->lock_count) { if (-1 == semop(ptr->semid,&increment,1)) { rb_raise(eInternal, "error unlocking semaphore, %d (%s)", errno, strerror(errno)); retval=Qfalse; } else { - --(ptr->lock_triggered); + --(ptr->lock_count); retval=Qtrue; } } else { - --(ptr->lock_triggered); + --(ptr->lock_count); retval=Qtrue; } return (void *)retval; } -VALUE -semian_shm_object_unlock(VALUE self) +/* + * Wrap the lock-unlock functionality in ensures + */ + +typedef struct { + int pre_block_lock_count_state; + semian_shm_object *ptr; +} lock_status; + +static VALUE +semian_shm_object_synchronize_with_block(VALUE self) +{ + semian_shm_object_check_and_resize_if_needed(self); + + if (!rb_block_given_p()) + return Qnil; + return rb_yield(Qnil); +} + +static VALUE +semian_shm_object_synchronize_restore_lock_status(VALUE v_status) { + lock_status *status = (lock_status *) v_status; + while (status->ptr->lock_count > status->pre_block_lock_count_state) + return (VALUE) WITHOUT_GVL(semian_shm_object_unlock_without_gvl, (void *)(status->ptr), RUBY_UBF_IO, NULL); + while (status->ptr->lock_count < status->pre_block_lock_count_state) + return (VALUE) WITHOUT_GVL(semian_shm_object_lock_without_gvl, (void *)(status->ptr), RUBY_UBF_IO, NULL); + return Qnil; +} + +VALUE +semian_shm_object_synchronize(VALUE self) { // receives a block semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - return (VALUE) WITHOUT_GVL(semian_shm_object_unlock_without_gvl, (void *)ptr, RUBY_UBF_IO, NULL); + lock_status status = { ptr->lock_count, ptr }; + if (!(WITHOUT_GVL(semian_shm_object_lock_without_gvl, (void *)ptr, RUBY_UBF_IO, NULL))) + return Qnil; + return rb_ensure(semian_shm_object_synchronize_with_block, self, semian_shm_object_synchronize_restore_lock_status, (VALUE)&status); } -VALUE -semian_shm_object_unlock_all(VALUE self) +void +define_method_with_synchronize(VALUE klass, const char *name, VALUE (*func)(ANYARGS), int argc) { - semian_shm_object *ptr; - TypedData_Get_Struct((VALUE)self, semian_shm_object, &semian_shm_object_type, ptr); - while (ptr->lock_triggered > 0) - semian_shm_object_unlock(self); - return self; + rb_define_method(klass, name, func, argc); + rb_funcall(klass, rb_intern("do_with_sync"), 1, rb_str_new2(name)); } +/* + * Memory functions + */ + + static int create_and_resize_memory(key_t key, int byte_size, long permissions, int should_keep_req_byte_size_bool, - int *created, void **data_copy, size_t *data_copy_byte_size, int *prev_mem_attach_count, semian_shm_object *ptr, VALUE self) { + void **data_copy, size_t *data_copy_byte_size, int *prev_mem_attach_count, semian_shm_object *ptr, VALUE self) { // Below will handle acquiring/creating new memory, and possibly resizing the // memory or the ptr->byte_size depending on @@ -361,7 +383,6 @@ create_and_resize_memory(key_t key, int byte_size, long permissions, int should_ failed=-4; } } else { - *created = 1; actual_byte_size = requested_byte_size; } @@ -375,16 +396,18 @@ create_and_resize_memory(key_t key, int byte_size, long permissions, int should_ if ((void *)-1 != (data = shmat(shmid, (void *)0, 0))) { - *data_copy = malloc(actual_byte_size); - memcpy(*data_copy,data,actual_byte_size); + char copy_data[actual_byte_size]; + memcpy(©_data, data, actual_byte_size); ptr->shmid=shmid; ptr->shm_address = data; - semian_shm_object_delete_memory_inner(ptr, 1, self, *data_copy); + semian_shm_object_delete_memory_inner(self); + + *data_copy = malloc(actual_byte_size); + memcpy(*data_copy, ©_data, actual_byte_size); // Flagging for deletion sets a shm's associated key to be 0 so shmget gets a different shmid. // If this worker is creating the new memory if (-1 != (shmid = shmget(key, requested_byte_size, flags))) { - *created = 1; } else { // failed to get new memory, exit failed=-5; } @@ -421,7 +444,7 @@ create_and_resize_memory(key_t key, int byte_size, long permissions, int should_ resizing */ VALUE -semian_shm_object_acquire_memory(VALUE self, VALUE permissions, VALUE should_keep_req_byte_size) +semian_shm_object_acquire_memory(VALUE self, VALUE should_keep_req_byte_size) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); @@ -430,31 +453,23 @@ semian_shm_object_acquire_memory(VALUE self, VALUE permissions, VALUE should_kee rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); return self; } - if (TYPE(permissions) != T_FIXNUM) - rb_raise(rb_eTypeError, "expected integer for permissions"); if (TYPE(should_keep_req_byte_size) != T_TRUE && TYPE(should_keep_req_byte_size) != T_FALSE) rb_raise(rb_eTypeError, "expected true or false for should_keep_req_byte_size"); int should_keep_req_byte_size_bool = RTEST(should_keep_req_byte_size); - if (!semian_shm_object_lock(self)) - return Qfalse; - - - int created = 0; key_t key = ptr->key; void *data_copy = NULL; // Will contain contents of old memory, if any size_t data_copy_byte_size = 0; int prev_mem_attach_count = 0; // Create/acquire memory and manage all resizing cases - int shmid = create_and_resize_memory(key, ptr->byte_size, FIX2LONG(permissions), should_keep_req_byte_size_bool, - &created, &data_copy, &data_copy_byte_size, &prev_mem_attach_count, ptr, self); + int shmid = create_and_resize_memory(key, ptr->byte_size, ptr->permissions, should_keep_req_byte_size_bool, + &data_copy, &data_copy_byte_size, &prev_mem_attach_count, ptr, self); if (shmid < 0) {// failed if (data_copy) free(data_copy); - semian_shm_object_unlock(self); rb_raise(eSyscall, "shmget() failed at %d to acquire a memory shmid with key %d, size %zu, errno %d (%s)", shmid, key, ptr->byte_size, errno, strerror(errno)); } else ptr->shmid = shmid; @@ -463,7 +478,6 @@ semian_shm_object_acquire_memory(VALUE self, VALUE permissions, VALUE should_kee if (0 == ptr->shm_address) { ptr->shm_address = shmat(ptr->shmid, (void *)0, 0); if (((void*)-1) == ptr->shm_address) { - semian_shm_object_unlock(self); ptr->shm_address = 0; if (data_copy) free(data_copy); @@ -474,23 +488,21 @@ semian_shm_object_acquire_memory(VALUE self, VALUE permissions, VALUE should_kee } if (data_copy) free(data_copy); - semian_shm_object_unlock(self); return self; } -void -semian_shm_object_delete_memory_inner (semian_shm_object *ptr, int should_unlock, VALUE self, void *should_free) +VALUE +semian_shm_object_delete_memory_inner (VALUE self) { // This internal function may be called from a variety of contexts // Sometimes it is under a semaphore lock, sometimes it has extra malloc ptrs // Arguments handle these conditions + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + if (0 != ptr->shm_address){ if (-1 == shmdt(ptr->shm_address)) { - if (should_unlock) - semian_shm_object_unlock(self); - if (should_free) - free(should_free); rb_raise(eSyscall,"shmdt: no attached memory at %p, errno %d (%s)", ptr->shm_address, errno, strerror(errno)); } else { } @@ -503,16 +515,13 @@ semian_shm_object_delete_memory_inner (semian_shm_object *ptr, int should_unlock if (errno == EINVAL) { ptr->shmid = -1; } else { - if (should_unlock) - semian_shm_object_unlock(self); - if (should_free) - free(should_free); rb_raise(eSyscall,"shmctl: error flagging memory for removal with shmid %d, errno %d (%s)", ptr->shmid, errno, strerror(errno)); } } else { ptr->shmid = -1; } } + return Qnil; } VALUE @@ -520,14 +529,10 @@ semian_shm_object_delete_memory (VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - - if (!semian_shm_object_lock(self)) + lock_status status = { ptr->lock_count, ptr }; + if (!(WITHOUT_GVL(semian_shm_object_lock_without_gvl, (void *)ptr, RUBY_UBF_IO, NULL))) return Qnil; - - semian_shm_object_delete_memory_inner(ptr, 1, self, NULL); - - semian_shm_object_unlock(self); - return self; + return rb_ensure(semian_shm_object_delete_memory_inner, self, semian_shm_object_synchronize_restore_lock_status, (VALUE)&status); } VALUE @@ -535,21 +540,19 @@ semian_shm_object_check_and_resize_if_needed(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - if (!semian_shm_object_lock(self)) - return Qnil; - struct shmid_ds shm_info; int needs_resize = 0; if (-1 != ptr->shmid && -1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)) { needs_resize = shm_info.shm_perm.mode & SHM_DEST; } - + VALUE priority = Qfalse; + if (-1 == ptr->shmid && 0 == ptr->shm_address){ + needs_resize = 1; + priority = Qtrue; + } if (needs_resize) { - semian_shm_object_delete_memory_inner(ptr, 1, self, NULL); - semian_shm_object_unlock(self); - semian_shm_object_acquire_memory(self, LONG2FIX(ptr->permissions), Qfalse); - } else { - semian_shm_object_unlock(self); + semian_shm_object_delete_memory_inner(self); + semian_shm_object_acquire_memory(self, priority); } return self; } @@ -559,7 +562,7 @@ semian_shm_object_semid(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - semian_shm_object_check_and_resize_if_needed(self); + semian_shm_object_synchronize(self); return INT2NUM(ptr->semid); } static VALUE @@ -567,7 +570,7 @@ semian_shm_object_shmid(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - semian_shm_object_check_and_resize_if_needed(self); + semian_shm_object_synchronize(self); return INT2NUM(ptr->shmid); } @@ -576,24 +579,8 @@ semian_shm_object_byte_size(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - semian_shm_object_check_and_resize_if_needed(self); - int byte_size = ptr->byte_size; - return INT2NUM(byte_size); -} - -static VALUE -semian_shm_object_synchronize_with_block(VALUE self) -{ - if (!rb_block_given_p()) - rb_raise(rb_eArgError, "Expected block"); - return rb_yield(Qnil); -} - -static VALUE -semian_shm_object_synchronize(VALUE self) { - if (!semian_shm_object_lock(self)) - return Qnil; - return rb_ensure(semian_shm_object_synchronize_with_block, self, semian_shm_object_unlock_all, self); + semian_shm_object_synchronize(self); + return INT2NUM(ptr->byte_size); } void @@ -605,8 +592,6 @@ Init_semian_shm_object (void) { //rb_define_alloc_func(cSysVSharedMemory, semian_shm_object_alloc); rb_define_method(cSysVSharedMemory, "_acquire", semian_shm_object_acquire, 3); rb_define_method(cSysVSharedMemory, "_destroy", semian_shm_object_destroy, 0); - rb_define_method(cSysVSharedMemory, "lock", semian_shm_object_lock, 0); - rb_define_method(cSysVSharedMemory, "unlock", semian_shm_object_unlock, 0); rb_define_method(cSysVSharedMemory, "byte_size", semian_shm_object_byte_size, 0); rb_define_method(cSysVSharedMemory, "semid", semian_shm_object_semid, 0); diff --git a/ext/semian/semian_shared_memory_object.h b/ext/semian/semian_shared_memory_object.h index 6281c7c7..8470c725 100644 --- a/ext/semian/semian_shared_memory_object.h +++ b/ext/semian/semian_shared_memory_object.h @@ -4,7 +4,7 @@ typedef struct { //semaphore, shared memory data and pointer key_t key; size_t byte_size; - int lock_triggered; // lock only done from 0 -> 1, unlock only done from 1 -> 0, so we can 'lock' multiple times (such as in nesting functions) without actually locking + int lock_count; // lock only done from 0 -> 1, unlock only done from 1 -> 0, so we can 'lock' multiple times (such as in nesting functions) without actually locking int permissions; int semid; int shmid; @@ -24,15 +24,15 @@ VALUE semian_shm_object_replace_alloc(VALUE klass, VALUE target); VALUE semian_shm_object_acquire(VALUE self, VALUE id, VALUE byte_size, VALUE permissions); VALUE semian_shm_object_destroy(VALUE self); -VALUE semian_shm_object_acquire_semaphore (VALUE self, VALUE permissions); +VALUE semian_shm_object_acquire_semaphore (VALUE self); VALUE semian_shm_object_delete_semaphore(VALUE self); -VALUE semian_shm_object_lock(VALUE self); -VALUE semian_shm_object_unlock(VALUE self); -VALUE semian_shm_object_unlock_all(VALUE self); -VALUE semian_shm_object_acquire_memory(VALUE self, VALUE permissions, VALUE should_keep_req_byte_size); -void semian_shm_object_delete_memory_inner (semian_shm_object *ptr, int should_unlock, VALUE self, void *should_free); +VALUE semian_shm_object_acquire_memory(VALUE self, VALUE should_keep_req_byte_size); +VALUE semian_shm_object_delete_memory_inner (VALUE self); VALUE semian_shm_object_delete_memory (VALUE self); VALUE semian_shm_object_check_and_resize_if_needed (VALUE self); +VALUE semian_shm_object_synchronize(VALUE self); +void define_method_with_synchronize(VALUE klass, const char *name, VALUE (*func)(ANYARGS), int argc); + void Init_semian_integer (void); void Init_semian_sliding_window (void); diff --git a/ext/semian/semian_sliding_window.c b/ext/semian/semian_sliding_window.c index 81a06dbf..84081ba2 100644 --- a/ext/semian/semian_sliding_window.c +++ b/ext/semian/semian_sliding_window.c @@ -63,11 +63,7 @@ semian_sliding_window_size(VALUE self) TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - semian_shm_object_check_and_resize_if_needed(self); - if (!semian_shm_object_lock(self)) - return Qnil; int window_size = ((semian_sliding_window *)(ptr->shm_address))->window_size; - semian_shm_object_unlock(self); return INT2NUM(window_size); } @@ -78,11 +74,7 @@ semian_sliding_window_max_size(VALUE self) TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - semian_shm_object_check_and_resize_if_needed(self); - if (!semian_shm_object_lock(self)) - return Qnil; int max_length = ((semian_sliding_window *)(ptr->shm_address))->max_window_size; - semian_shm_object_unlock(self); return INT2NUM(max_length); } @@ -95,9 +87,6 @@ semian_sliding_window_push_back(VALUE self, VALUE num) return Qnil; if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT && TYPE(num) != T_BIGNUM) return Qnil; - semian_shm_object_check_and_resize_if_needed(self); - if (!semian_shm_object_lock(self)) - return Qnil; semian_sliding_window *data = ptr->shm_address; if (data->window_size == data->max_window_size) { @@ -108,7 +97,6 @@ semian_sliding_window_push_back(VALUE self, VALUE num) } data->window[(data->window_size)] = NUM2LONG(num); ++(data->window_size); - semian_shm_object_unlock(self); return self; } @@ -119,9 +107,6 @@ semian_sliding_window_pop_back(VALUE self) TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - semian_shm_object_check_and_resize_if_needed(self); - if (!semian_shm_object_lock(self)) - return Qnil; VALUE retval; semian_sliding_window *data = ptr->shm_address; @@ -131,8 +116,6 @@ semian_sliding_window_pop_back(VALUE self) retval = LONG2NUM(data->window[data->window_size-1]); --(data->window_size); } - - semian_shm_object_unlock(self); return retval; } @@ -145,9 +128,6 @@ semian_sliding_window_push_front(VALUE self, VALUE num) return Qnil; if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT && TYPE(num) != T_BIGNUM) return Qnil; - semian_shm_object_check_and_resize_if_needed(self); - if (!semian_shm_object_lock(self)) - return Qnil; long val = NUM2LONG(num); semian_sliding_window *data = ptr->shm_address; @@ -161,7 +141,6 @@ semian_sliding_window_push_front(VALUE self, VALUE num) if (data->window_size>data->max_window_size) data->window_size=data->max_window_size; - semian_shm_object_unlock(self); return self; } @@ -172,9 +151,6 @@ semian_sliding_window_pop_front(VALUE self) TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - semian_shm_object_check_and_resize_if_needed(self); - if (!semian_shm_object_lock(self)) - return Qnil; VALUE retval; semian_sliding_window *data = ptr->shm_address; @@ -187,7 +163,6 @@ semian_sliding_window_pop_front(VALUE self) --(data->window_size); } - semian_shm_object_unlock(self); return retval; } @@ -198,13 +173,9 @@ semian_sliding_window_clear(VALUE self) TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - semian_shm_object_check_and_resize_if_needed(self); - if (!semian_shm_object_lock(self)) - return Qnil; semian_sliding_window *data = ptr->shm_address; data->window_size=0; - semian_shm_object_unlock(self); return self; } @@ -215,9 +186,6 @@ semian_sliding_window_first(VALUE self) TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - semian_shm_object_check_and_resize_if_needed(self); - if (!semian_shm_object_lock(self)) - return Qnil; VALUE retval; semian_sliding_window *data = ptr->shm_address; @@ -226,7 +194,6 @@ semian_sliding_window_first(VALUE self) else retval = Qnil; - semian_shm_object_unlock(self); return retval; } @@ -237,9 +204,6 @@ semian_sliding_window_last(VALUE self) TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (0 == ptr->shm_address) return Qnil; - semian_shm_object_check_and_resize_if_needed(self); - if (!semian_shm_object_lock(self)) - return Qnil; VALUE retval; semian_sliding_window *data = ptr->shm_address; @@ -248,7 +212,6 @@ semian_sliding_window_last(VALUE self) else retval = Qnil; - semian_shm_object_unlock(self); return retval; } @@ -262,9 +225,6 @@ semian_sliding_window_resize_to(VALUE self, VALUE size) { if (NUM2INT(size) <= 0) rb_raise(rb_eArgError, "size must be larger than 0"); - if (!semian_shm_object_lock(self)) - return Qnil; - semian_sliding_window *data_copy = NULL; size_t byte_size=0; int prev_mem_attach_count = 0; @@ -278,15 +238,12 @@ semian_sliding_window_resize_to(VALUE self, VALUE size) { } } - semian_shm_object_delete_memory_inner(ptr, 1, self, NULL); + semian_shm_object_delete_memory_inner(self); ptr->byte_size = 2*sizeof(int) + NUM2INT(size) * sizeof(long); - semian_shm_object_unlock(self); - semian_shm_object_acquire_memory(self, LONG2FIX(ptr->permissions), Qtrue); - if (!semian_shm_object_lock(self)) - return Qnil; + semian_shm_object_acquire_memory(self, Qtrue); + ptr->object_init_fn(ptr->byte_size, ptr->shm_address, data_copy, byte_size, prev_mem_attach_count); - semian_shm_object_unlock(self); return self; } @@ -302,15 +259,15 @@ Init_semian_sliding_window (void) semian_shm_object_replace_alloc(cSysVSharedMemory, cSlidingWindow); rb_define_method(cSlidingWindow, "bind_init_fn", semian_sliding_window_bind_init_fn_wrapper, 0); - rb_define_method(cSlidingWindow, "size", semian_sliding_window_size, 0); - rb_define_method(cSlidingWindow, "max_size", semian_sliding_window_max_size, 0); - rb_define_method(cSlidingWindow, "resize_to", semian_sliding_window_resize_to, 1); - rb_define_method(cSlidingWindow, "<<", semian_sliding_window_push_back, 1); - rb_define_method(cSlidingWindow, "push", semian_sliding_window_push_back, 1); - rb_define_method(cSlidingWindow, "pop", semian_sliding_window_pop_back, 0); - rb_define_method(cSlidingWindow, "shift", semian_sliding_window_pop_front, 0); - rb_define_method(cSlidingWindow, "unshift", semian_sliding_window_push_front, 1); - rb_define_method(cSlidingWindow, "clear", semian_sliding_window_clear, 0); - rb_define_method(cSlidingWindow, "first", semian_sliding_window_first, 0); - rb_define_method(cSlidingWindow, "last", semian_sliding_window_last, 0); + define_method_with_synchronize(cSlidingWindow, "size", semian_sliding_window_size, 0); + define_method_with_synchronize(cSlidingWindow, "max_size", semian_sliding_window_max_size, 0); + define_method_with_synchronize(cSlidingWindow, "resize_to", semian_sliding_window_resize_to, 1); + define_method_with_synchronize(cSlidingWindow, "<<", semian_sliding_window_push_back, 1); + define_method_with_synchronize(cSlidingWindow, "push", semian_sliding_window_push_back, 1); + define_method_with_synchronize(cSlidingWindow, "pop", semian_sliding_window_pop_back, 0); + define_method_with_synchronize(cSlidingWindow, "shift", semian_sliding_window_pop_front, 0); + define_method_with_synchronize(cSlidingWindow, "unshift", semian_sliding_window_push_front, 1); + define_method_with_synchronize(cSlidingWindow, "clear", semian_sliding_window_clear, 0); + define_method_with_synchronize(cSlidingWindow, "first", semian_sliding_window_first, 0); + define_method_with_synchronize(cSlidingWindow, "last", semian_sliding_window_last, 0); } diff --git a/lib/semian/sysv_shared_memory.rb b/lib/semian/sysv_shared_memory.rb index 07b896ae..18aefa2a 100644 --- a/lib/semian/sysv_shared_memory.rb +++ b/lib/semian/sysv_shared_memory.rb @@ -7,6 +7,21 @@ def self.sizeof(type) size end + def self.included(base) + def base.do_with_sync(*names) + names.each do |name| + new_name = "#{name}_inner".freeze.to_sym + alias_method new_name, name + private new_name + define_method(name) do |*args, &block| + synchronize do + method(new_name).call(*args, &block) + end + end + end + end + end + def semid -1 end @@ -15,9 +30,9 @@ def shmid -1 end - def synchronize(&proc) + def synchronize(&block) if respond_to?(:_synchronize) && @using_shared_memory - return _synchronize(&proc) + _synchronize(&block) else yield if block_given? end diff --git a/test/sysv_integer_test.rb b/test/sysv_integer_test.rb index c445193d..3404f0fc 100644 --- a/test/sysv_integer_test.rb +++ b/test/sysv_integer_test.rb @@ -5,7 +5,7 @@ class TestSysVInteger < MiniTest::Unit::TestCase def setup @integer = CLASS.new(name: 'TestSysVInteger', permissions: 0660) - @integer.value = 0 + @integer.reset end def teardown @@ -42,4 +42,14 @@ def test_memory_not_reset_when_at_least_one_worker_using_it end Process.waitall end + + def test_memory_reset_when_no_workers_using_it + fork do + integer = CLASS.new(name: 'TestSysVInteger_2', permissions: 0660) + integer.value = 109 + end + Process.waitall + @integer = CLASS.new(name: 'TestSysVInteger_2', permissions: 0660) + assert_equal 0, @integer.value + end end diff --git a/test/sysv_sliding_window_test.rb b/test/sysv_sliding_window_test.rb index 0c4f29fa..c8d9eff1 100644 --- a/test/sysv_sliding_window_test.rb +++ b/test/sysv_sliding_window_test.rb @@ -48,15 +48,15 @@ def test_sliding_window_memory_is_actually_shared large_number = (Time.now.to_f * 1000).to_i @sliding_window << large_number - assert_correct_first_and_last_and_size(@sliding_window, large_number, large_number, 1, 6) - assert_correct_first_and_last_and_size(sliding_window_2, large_number, large_number, 1, 6) + assert_sliding_window(@sliding_window, [large_number], 6) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) sliding_window_2 << 6 << 4 << 3 << 2 - assert_correct_first_and_last_and_size(@sliding_window, large_number, 2, 5, 6) - assert_correct_first_and_last_and_size(sliding_window_2, large_number, 2, 5, 6) + assert_sliding_window(@sliding_window, [large_number, 6, 4, 3, 2], 6) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) @sliding_window.clear - assert_correct_first_and_last_and_size(@sliding_window, nil, nil, 0, 6) - assert_correct_first_and_last_and_size(sliding_window_2, nil, nil, 0, 6) + assert_sliding_window(@sliding_window, [], 6) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) end def test_restarting_worker_should_not_reset_queue @@ -64,68 +64,75 @@ def test_restarting_worker_should_not_reset_queue sliding_window_2 = CLASS.new(max_size: 6, name: 'TestSysVSlidingWindow', permissions: 0660) - assert_correct_first_and_last_and_size(@sliding_window, 10, 30, 3, 6) + assert_sliding_window(sliding_window_2, [10, 20, 30], 6) sliding_window_2.pop + assert_sliding_window(sliding_window_2, [10, 20], 6) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) sliding_window_3 = CLASS.new(max_size: 6, name: 'TestSysVSlidingWindow', permissions: 0660) - assert_correct_first_and_last_and_size(@sliding_window, 10, 20, 2, 6) + assert_sliding_window(sliding_window_3, [10, 20], 6) sliding_window_3.pop - assert_correct_first_and_last_and_size(@sliding_window, 10, 10, 1, 6) + assert_sliding_window(@sliding_window, [10], 6) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - @sliding_window.clear end def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down # Test explicit resizing, and resizing through making new memory associations + # B resize down through init sliding_window_2 = CLASS.new(max_size: 4, name: 'TestSysVSlidingWindow', permissions: 0660) sliding_window_2 << 80 << 90 << 100 << 110 << 120 - assert_correct_first_and_last_and_size(@sliding_window, 90, 120, 4, 4) - assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - - sliding_window_2 = CLASS.new(max_size: 3, - name: 'TestSysVSlidingWindow', - permissions: 0660) + assert_sliding_window(sliding_window_2, [90, 100, 110, 120], 4) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 100, 120, 3, 3) + # A explicit resize down, @sliding_window.resize_to(2) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 2) + assert_sliding_window(@sliding_window, [110, 120], 2) + # B resize up through init sliding_window_2 = CLASS.new(max_size: 4, name: 'TestSysVSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 110, 120, 2, 4) + assert_sliding_window(@sliding_window, [110, 120], 4) + # A explicit resize up @sliding_window.resize_to(6) @sliding_window << 130 assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 110, 130, 3, 6) + assert_sliding_window(@sliding_window, [110, 120, 130], 6) + # B resize down through init sliding_window_2 = CLASS.new(max_size: 2, name: 'TestSysVSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 2) + assert_sliding_window(@sliding_window, [120, 130], 2) + # A explicit resize up @sliding_window.resize_to(4) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 4) + assert_sliding_window(@sliding_window, [120, 130], 4) + # B resize up through init sliding_window_2 = CLASS.new(max_size: 6, name: 'TestSysVSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - assert_correct_first_and_last_and_size(@sliding_window, 120, 130, 2, 6) - - sliding_window_2.clear + assert_sliding_window(@sliding_window, [120, 130], 6) + + # B resize, but no final size change + sliding_window_2 << 140 << 150 << 160 << 170 + sliding_window_2.resize_to(4) + sliding_window_2 << 180 + sliding_window_2.resize_to(6) + assert_sliding_window(@sliding_window, [150, 160, 170, 180], 6) + assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) end private diff --git a/test/sysv_state_test.rb b/test/sysv_state_test.rb index a5b77e3b..768602a2 100644 --- a/test/sysv_state_test.rb +++ b/test/sysv_state_test.rb @@ -6,6 +6,7 @@ class TestSysVState < MiniTest::Unit::TestCase def setup @state = CLASS.new(name: 'TestSysVState', permissions: 0660) + @state.reset end def teardown From 882da9d927c91425233f85d31c32742223df83b7 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Tue, 17 Nov 2015 16:32:16 +0000 Subject: [PATCH 17/23] Changed naming to initialize_memory and bind_initialize_memory_callback --- ext/semian/semian_integer.c | 12 ++++++------ ext/semian/semian_shared_memory_object.c | 22 +++++++++------------- ext/semian/semian_shared_memory_object.h | 2 +- ext/semian/semian_sliding_window.c | 14 +++++++------- lib/semian/simple_integer.rb | 2 +- lib/semian/simple_sliding_window.rb | 2 +- lib/semian/simple_state.rb | 2 +- lib/semian/sysv_shared_memory.rb | 8 ++++---- lib/semian/sysv_sliding_window.rb | 2 +- lib/semian/sysv_state.rb | 10 +++++----- 10 files changed, 36 insertions(+), 40 deletions(-) diff --git a/ext/semian/semian_integer.c b/ext/semian/semian_integer.c index d09f6a27..6b74be08 100644 --- a/ext/semian/semian_integer.c +++ b/ext/semian/semian_integer.c @@ -4,14 +4,14 @@ typedef struct { int value; } semian_int; -static void semian_integer_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); -static VALUE semian_integer_bind_init_fn_wrapper(VALUE self); +static void semian_integer_initialize_memory (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); +static VALUE semian_integer_bind_initialize_memory_callback(VALUE self); static VALUE semian_integer_get_value(VALUE self); static VALUE semian_integer_set_value(VALUE self, VALUE num); static VALUE semian_integer_increment(int argc, VALUE *argv, VALUE self); static void -semian_integer_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count) +semian_integer_initialize_memory (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count) { semian_int *ptr = dest; semian_int *old = prev_data; @@ -25,11 +25,11 @@ semian_integer_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size } static VALUE -semian_integer_bind_init_fn_wrapper(VALUE self) +semian_integer_bind_initialize_memory_callback(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - ptr->object_init_fn = &semian_integer_bind_init_fn; + ptr->initialize_memory = &semian_integer_initialize_memory; return self; } @@ -103,7 +103,7 @@ Init_semian_integer (void) semian_shm_object_replace_alloc(cSysVSharedMemory, cInteger); - rb_define_method(cInteger, "bind_init_fn", semian_integer_bind_init_fn_wrapper, 0); + rb_define_private_method(cInteger, "bind_initialize_memory_callback", semian_integer_bind_initialize_memory_callback, 0); define_method_with_synchronize(cInteger, "value", semian_integer_get_value, 0); define_method_with_synchronize(cInteger, "value=", semian_integer_set_value, 1); define_method_with_synchronize(cInteger, "reset", semian_integer_reset, 0); diff --git a/ext/semian/semian_shared_memory_object.c b/ext/semian/semian_shared_memory_object.c index e687e06b..67ab2583 100644 --- a/ext/semian/semian_shared_memory_object.c +++ b/ext/semian/semian_shared_memory_object.c @@ -39,11 +39,9 @@ static void semian_shm_object_free(void *ptr) { semian_shm_object *data = (semian_shm_object *) ptr; - // Under normal circumstances, memory use should be in the order of bytes, - // and shouldn't increase if the same key/id is used - // so there is no need to call this unless certain all other semian processes are stopped - // (also raises concurrency errors: "object allocation during garbage collection phase") - + // Under normal circumstances, memory use should be in the order of bytes, and shouldn't + // increase if the same key/id is used, so there is no need to delete the shared memory + // (also raises a concurrency-related bug: "object allocation during garbage collection phase") xfree(data); } static size_t @@ -56,7 +54,6 @@ semian_shm_object_alloc(VALUE klass) { VALUE obj; semian_shm_object *ptr; - obj = TypedData_Make_Struct(klass, semian_shm_object, &semian_shm_object_type, ptr); return obj; } @@ -125,11 +122,10 @@ semian_shm_object_acquire(VALUE self, VALUE name, VALUE byte_size, VALUE permiss ptr->permissions = FIX2LONG(permissions); // Will throw NotImplementedError if not defined in concrete subclasses - // Implement bind_init_fn as a function with type - // static VALUE bind_init_fn(VALUE self) - // that binds ptr->object_init_fn to an init function called after memory is made - rb_funcall(self, rb_intern("bind_init_fn"), 0); - + // Implement bind_initialize_memory_callback as a function with type + // static VALUE bind_initialize_memory_callback(VALUE self) + // that a callback to ptr->initialize_memory, called when memory needs to be initialized + rb_funcall(self, rb_intern("bind_initialize_memory_callback"), 0); semian_shm_object_acquire_semaphore(self); semian_shm_object_synchronize(self); @@ -474,7 +470,7 @@ semian_shm_object_acquire_memory(VALUE self, VALUE should_keep_req_byte_size) } else ptr->shmid = shmid; - // Attach memory and call object_init_fn() + // Attach memory and call initialize_memory() if (0 == ptr->shm_address) { ptr->shm_address = shmat(ptr->shmid, (void *)0, 0); if (((void*)-1) == ptr->shm_address) { @@ -483,7 +479,7 @@ semian_shm_object_acquire_memory(VALUE self, VALUE should_keep_req_byte_size) free(data_copy); rb_raise(eSyscall, "shmat() failed to attach memory with shmid %d, size %zu, errno %d (%s)", ptr->shmid, ptr->byte_size, errno, strerror(errno)); } else { - ptr->object_init_fn(ptr->byte_size, ptr->shm_address, data_copy, data_copy_byte_size, prev_mem_attach_count); + ptr->initialize_memory(ptr->byte_size, ptr->shm_address, data_copy, data_copy_byte_size, prev_mem_attach_count); } } if (data_copy) diff --git a/ext/semian/semian_shared_memory_object.h b/ext/semian/semian_shared_memory_object.h index 8470c725..8631b0a8 100644 --- a/ext/semian/semian_shared_memory_object.h +++ b/ext/semian/semian_shared_memory_object.h @@ -8,7 +8,7 @@ typedef struct { int permissions; int semid; int shmid; - void (*object_init_fn)(size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); + void (*initialize_memory)(size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); void *shm_address; } semian_shm_object; diff --git a/ext/semian/semian_sliding_window.c b/ext/semian/semian_sliding_window.c index 84081ba2..813e5f56 100644 --- a/ext/semian/semian_sliding_window.c +++ b/ext/semian/semian_sliding_window.c @@ -6,8 +6,8 @@ typedef struct { long window[]; } semian_sliding_window; -static void semian_sliding_window_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); -static VALUE semian_sliding_window_bind_init_fn_wrapper(VALUE self); +static void semian_sliding_window_initialize_memory (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); +static VALUE semian_sliding_window_bind_initialize_memory_callback(VALUE self); static VALUE semian_sliding_window_size(VALUE self); static VALUE semian_sliding_window_max_size(VALUE self); static VALUE semian_sliding_window_push_back(VALUE self, VALUE num); @@ -20,7 +20,7 @@ static VALUE semian_sliding_window_last(VALUE self); static VALUE semian_sliding_window_resize_to(VALUE self, VALUE size); static void -semian_sliding_window_bind_init_fn (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count) +semian_sliding_window_initialize_memory (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count) { semian_sliding_window *ptr = dest; semian_sliding_window *old = prev_data; @@ -48,11 +48,11 @@ semian_sliding_window_bind_init_fn (size_t byte_size, void *dest, void *prev_dat } static VALUE -semian_sliding_window_bind_init_fn_wrapper(VALUE self) +semian_sliding_window_bind_initialize_memory_callback(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - ptr->object_init_fn = &semian_sliding_window_bind_init_fn; + ptr->initialize_memory = &semian_sliding_window_initialize_memory; return self; } @@ -243,7 +243,7 @@ semian_sliding_window_resize_to(VALUE self, VALUE size) { semian_shm_object_acquire_memory(self, Qtrue); - ptr->object_init_fn(ptr->byte_size, ptr->shm_address, data_copy, byte_size, prev_mem_attach_count); + ptr->initialize_memory(ptr->byte_size, ptr->shm_address, data_copy, byte_size, prev_mem_attach_count); return self; } @@ -258,7 +258,7 @@ Init_semian_sliding_window (void) semian_shm_object_replace_alloc(cSysVSharedMemory, cSlidingWindow); - rb_define_method(cSlidingWindow, "bind_init_fn", semian_sliding_window_bind_init_fn_wrapper, 0); + rb_define_private_method(cSlidingWindow, "bind_initialize_memory_callback", semian_sliding_window_bind_initialize_memory_callback, 0); define_method_with_synchronize(cSlidingWindow, "size", semian_sliding_window_size, 0); define_method_with_synchronize(cSlidingWindow, "max_size", semian_sliding_window_max_size, 0); define_method_with_synchronize(cSlidingWindow, "resize_to", semian_sliding_window_resize_to, 1); diff --git a/lib/semian/simple_integer.rb b/lib/semian/simple_integer.rb index d2814aec..cb4403e9 100644 --- a/lib/semian/simple_integer.rb +++ b/lib/semian/simple_integer.rb @@ -3,7 +3,7 @@ module Simple class Integer #:nodoc: attr_accessor :value - def initialize(**_) + def initialize(**) reset end diff --git a/lib/semian/simple_sliding_window.rb b/lib/semian/simple_sliding_window.rb index e3de9978..bfd57e54 100644 --- a/lib/semian/simple_sliding_window.rb +++ b/lib/semian/simple_sliding_window.rb @@ -12,7 +12,7 @@ class SlidingWindow #:nodoc: # like this: if @max_size = 4, current time is 10, @window =[5,7,9,10]. # Another push of (11) at 11 sec would make @window [7,9,10,11], shifting off 5. - def initialize(max_size:, **_) + def initialize(max_size:, **) @max_size = max_size @window = [] end diff --git a/lib/semian/simple_state.rb b/lib/semian/simple_state.rb index e8880d48..cefea9dd 100644 --- a/lib/semian/simple_state.rb +++ b/lib/semian/simple_state.rb @@ -1,7 +1,7 @@ module Semian module Simple class State #:nodoc: - def initialize(**_) + def initialize(**) reset end diff --git a/lib/semian/sysv_shared_memory.rb b/lib/semian/sysv_shared_memory.rb index 18aefa2a..8f3b9283 100644 --- a/lib/semian/sysv_shared_memory.rb +++ b/lib/semian/sysv_shared_memory.rb @@ -59,15 +59,15 @@ def acquire_memory_object(name, data_layout, permissions) byte_size = data_layout.inject(0) { |sum, type| sum + ::Semian::SysVSharedMemory.sizeof(type) } raise TypeError.new("Given data layout is 0 bytes: #{data_layout.inspect}") if byte_size <= 0 - # Calls C layer to acquire/create a memory block, calling #bind_init_fn in the process, see below + # Calls C layer to acquire/create a memory block, calling #initialize_memory in the process, see below _acquire(name, byte_size, permissions) @using_shared_memory = true end - def bind_init_fn + def bind_initialize_memory_callback # Concrete classes must implement this in a subclass in C to bind a callback function of type - # void (*object_init_fn)(size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size); - # to location ptr->object_init_fn, where ptr is a semian_shm_object* + # void (*initialize_memory)(size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size); + # to location ptr->initialize_memory, where ptr is a semian_shm_object* # It is called when memory needs to be initialized or resized, possibly using previous memory raise NotImplementedError end diff --git a/lib/semian/sysv_sliding_window.rb b/lib/semian/sysv_sliding_window.rb index 85089553..4c54c022 100644 --- a/lib/semian/sysv_sliding_window.rb +++ b/lib/semian/sysv_sliding_window.rb @@ -4,7 +4,7 @@ class SlidingWindow < Semian::Simple::SlidingWindow #:nodoc: include SysVSharedMemory def initialize(max_size:, name:, permissions:) - data_layout = [:int, :int].concat(Array.new(max_size, :long)) + data_layout = [:int, :int] + [:long] * max_size super(max_size: max_size) unless acquire_memory_object(name, data_layout, permissions) end end diff --git a/lib/semian/sysv_state.rb b/lib/semian/sysv_state.rb index 0cd267c0..9286e668 100644 --- a/lib/semian/sysv_state.rb +++ b/lib/semian/sysv_state.rb @@ -7,12 +7,12 @@ class State < Semian::Simple::State #:nodoc: extend Forwardable def_delegators :@integer, :semid, :shmid, :synchronize, :transaction, - :shared?, :acquire_memory_object, :bind_init_fn - private :shared?, :acquire_memory_object, :bind_init_fn + :shared?, :acquire_memory_object, :initialize_memory + private :shared?, :acquire_memory_object, :initialize_memory def initialize(name:, permissions:) @integer = Semian::SysV::Integer.new(name: name, permissions: permissions) - initialize_lookup([:closed, :open, :half_open]) + initialize_lookup end def open @@ -46,12 +46,12 @@ def value=(sym) @integer.value = @sym_to_num.fetch(sym) { raise ArgumentError } end - def initialize_lookup(symbol_list) + def initialize_lookup # Assume symbol_list[0] is mapped to 0 # Cannot just use #object_id since #object_id for symbols is different in every run # For now, implement a C-style enum type backed by integers - @sym_to_num = Hash[symbol_list.each_with_index.to_a] + @sym_to_num = {closed: 0, open: 1, half_open: 2} @num_to_sym = @sym_to_num.invert end end From 3e99639883f6b8e41bcdddd3120e0f71aa556ca4 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Tue, 17 Nov 2015 20:27:36 +0000 Subject: [PATCH 18/23] Nitpick --- lib/semian/circuit_breaker.rb | 6 +++--- lib/semian/simple_state.rb | 9 +++++---- lib/semian/sysv_state.rb | 18 +----------------- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index be74ec95..cb0c8581 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -10,11 +10,11 @@ def initialize(name, exceptions:, success_threshold:, error_threshold:, error_ti @exceptions = exceptions @errors = implementation::SlidingWindow.new(max_size: @error_count_threshold, - name: "#{name}_sysv_sliding_window", + name: "#{name}_sliding_window", permissions: permissions) - @successes = implementation::Integer.new(name: "#{name}_sysv_integer", + @successes = implementation::Integer.new(name: "#{name}_integer", permissions: permissions) - @state = implementation::State.new(name: "#{name}_sysv_state", + @state = implementation::State.new(name: "#{name}_state", permissions: permissions) end diff --git a/lib/semian/simple_state.rb b/lib/semian/simple_state.rb index cefea9dd..eff76a98 100644 --- a/lib/semian/simple_state.rb +++ b/lib/semian/simple_state.rb @@ -5,7 +5,8 @@ def initialize(**) reset end - attr_reader :value + attr_accessor :value + private :value= def open? value == :open @@ -20,15 +21,15 @@ def half_open? end def open - @value = :open + self.value = :open end def close - @value = :closed + self.value = :closed end def half_open - @value = :half_open + self.value = :half_open end def reset diff --git a/lib/semian/sysv_state.rb b/lib/semian/sysv_state.rb index 9286e668..0312a426 100644 --- a/lib/semian/sysv_state.rb +++ b/lib/semian/sysv_state.rb @@ -15,24 +15,8 @@ def initialize(name:, permissions:) initialize_lookup end - def open - self.value = :open - end - - def close - self.value = :closed - end - - def half_open - self.value = :half_open - end - - def reset - close - end - def destroy - reset + super @integer.destroy end From 2f09204f8edfabdcd64df4a2d03a6ee23ae47a01 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Tue, 17 Nov 2015 22:07:42 +0000 Subject: [PATCH 19/23] Cleanup lock, unlock, delete memory code, rename to cleanup_memory --- ext/semian/semian_shared_memory_object.c | 171 ++++++++--------------- ext/semian/semian_shared_memory_object.h | 4 +- ext/semian/semian_sliding_window.c | 6 +- 3 files changed, 67 insertions(+), 114 deletions(-) diff --git a/ext/semian/semian_shared_memory_object.c b/ext/semian/semian_shared_memory_object.c index 67ab2583..f66ccc68 100644 --- a/ext/semian/semian_shared_memory_object.c +++ b/ext/semian/semian_shared_memory_object.c @@ -38,7 +38,7 @@ semian_shm_object_mark(void *ptr) static void semian_shm_object_free(void *ptr) { - semian_shm_object *data = (semian_shm_object *) ptr; + semian_shm_object *data = (semian_shm_object *)ptr; // Under normal circumstances, memory use should be in the order of bytes, and shouldn't // increase if the same key/id is used, so there is no need to delete the shared memory // (also raises a concurrency-related bug: "object allocation during garbage collection phase") @@ -109,16 +109,11 @@ semian_shm_object_acquire(VALUE self, VALUE name, VALUE byte_size, VALUE permiss id_str = RSTRING_PTR(name); } ptr->key = generate_key(id_str); - - // Guarantee byte_size >=1 or error thrown - ptr->byte_size = NUM2SIZET(byte_size); - - // id's default to -1 - ptr->semid = -1; + ptr->byte_size = NUM2SIZET(byte_size); // byte_size >=1 or error would have been raised earlier + ptr->semid = -1; // id's default to -1 ptr->shmid = -1; - // addresses default to NULL - ptr->shm_address = 0; - ptr->lock_count = 0; + ptr->shm_address = 0; // address defaults to NULL + ptr->lock_count = 0; // Emulates recursive mutex, 0->1 locks, 1->0 unlocks, rest noops ptr->permissions = FIX2LONG(permissions); // Will throw NotImplementedError if not defined in concrete subclasses @@ -135,7 +130,7 @@ semian_shm_object_acquire(VALUE self, VALUE name, VALUE byte_size, VALUE permiss VALUE semian_shm_object_destroy(VALUE self) { - VALUE result = semian_shm_object_delete_memory(self); + VALUE result = semian_shm_object_cleanup_memory(self); if (!result) return Qfalse; result = semian_shm_object_delete_semaphore(self); @@ -166,7 +161,7 @@ create_semaphore_and_initialize_and_set_permissions(int key, int permissions) } if (-1 != semid){ - set_semaphore_permissions(semid, permissions); + set_semaphore_permissions(semid, permissions); // Borrowed from semian.c } return semid; @@ -176,9 +171,6 @@ create_semaphore_and_initialize_and_set_permissions(int key, int permissions) VALUE semian_shm_object_acquire_semaphore (VALUE self) { - // Function flow, semaphore creation methods are - // borrowed from semian.c since they have been previously tested - semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); @@ -188,7 +180,6 @@ semian_shm_object_acquire_semaphore (VALUE self) raise_semian_syscall_error("semget()", errno); } ptr->semid = semid; - return self; } @@ -197,16 +188,16 @@ semian_shm_object_delete_semaphore(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - if (-1 == ptr->semid) // do nothing if semaphore not acquired + if (-1 == ptr->semid){ // do nothing if semaphore not acquired return Qfalse; - + } if (-1 == semctl(ptr->semid, 0, IPC_RMID)) { if (EIDRM == errno) { rb_warn("semctl: failed to delete semaphore set with semid %d: already removed", ptr->semid); raise_semian_syscall_error("semctl()", errno); ptr->semid = -1; } else { - rb_warn("semctl: failed to remove semaphore with semid %d, errno %d (%s)",ptr->semid, errno, strerror(errno)); + rb_warn("semctl: failed to remove semaphore with semid %d, errno %d (%s)", ptr->semid, errno, strerror(errno)); raise_semian_syscall_error("semctl()", errno); } } else { @@ -224,27 +215,17 @@ static void * semian_shm_object_lock_without_gvl(void *v_ptr) { semian_shm_object *ptr = v_ptr; - if (-1 == ptr->semid){ + if (-1 == ptr->semid) { rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); - return (void *)Qfalse; } - VALUE retval; - struct timespec ts = { 0 }; ts.tv_sec = kSHMInternalTimeout; - if (0 == ptr->lock_count) { - if (-1 == semtimedop(ptr->semid,&decrement,1, &ts)) { - rb_raise(eInternal, "error acquiring semaphore lock to mutate circuit breaker structure, %d: (%s)", errno, strerror(errno)); - retval=Qfalse; - } else { - (ptr->lock_count)+=1; - retval=Qtrue; - } + if (0 != ptr->lock_count || -1 != semtimedop(ptr->semid, &decrement, 1, &ts)) { + ptr->lock_count += 1; } else { - (ptr->lock_count)+=1; - retval=Qtrue; + rb_raise(eInternal, "error acquiring semaphore lock to mutate circuit breaker structure, %d: (%s)", errno, strerror(errno)); } - return (void *)retval; + return (void *)Qtrue; } static void * @@ -253,30 +234,20 @@ semian_shm_object_unlock_without_gvl(void *v_ptr) semian_shm_object *ptr = v_ptr; if (-1 == ptr->semid){ rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); - return (void *)Qfalse; } - VALUE retval; - - if (1 == ptr->lock_count) { - if (-1 == semop(ptr->semid,&increment,1)) { - rb_raise(eInternal, "error unlocking semaphore, %d (%s)", errno, strerror(errno)); - retval=Qfalse; - } else { - --(ptr->lock_count); - retval=Qtrue; - } + if (1 != ptr->lock_count || -1 != semop(ptr->semid, &increment, 1)) { + ptr->lock_count -= 1; } else { - --(ptr->lock_count); - retval=Qtrue; + rb_raise(eInternal, "error unlocking semaphore, %d (%s)", errno, strerror(errno)); } - return (void *)retval; + return (void *)Qtrue; } /* * Wrap the lock-unlock functionality in ensures */ -typedef struct { +typedef struct { // Workaround rb_ensure only allows one argument for each callback function int pre_block_lock_count_state; semian_shm_object *ptr; } lock_status; @@ -285,7 +256,6 @@ static VALUE semian_shm_object_synchronize_with_block(VALUE self) { semian_shm_object_check_and_resize_if_needed(self); - if (!rb_block_given_p()) return Qnil; return rb_yield(Qnil); @@ -323,19 +293,22 @@ define_method_with_synchronize(VALUE klass, const char *name, VALUE (*func)(ANYA * Memory functions */ - static int -create_and_resize_memory(key_t key, int byte_size, long permissions, int should_keep_req_byte_size_bool, - void **data_copy, size_t *data_copy_byte_size, int *prev_mem_attach_count, semian_shm_object *ptr, VALUE self) { +create_and_resize_memory(key_t key, int should_keep_req_byte_size_bool, + void **data_copy, size_t *data_copy_byte_size, int *prev_mem_attach_count, VALUE self) { // Below will handle acquiring/creating new memory, and possibly resizing the // memory or the ptr->byte_size depending on // should_keep_req_byte_size_bool + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + int shmid=-1; int failed=0; - int requested_byte_size = byte_size; + int requested_byte_size = ptr->byte_size; + long permissions = ptr->permissions; int flags = IPC_CREAT | IPC_EXCL | permissions; int actual_byte_size = 0; @@ -396,7 +369,7 @@ create_and_resize_memory(key_t key, int byte_size, long permissions, int should_ memcpy(©_data, data, actual_byte_size); ptr->shmid=shmid; ptr->shm_address = data; - semian_shm_object_delete_memory_inner(self); + semian_shm_object_cleanup_memory(self); *data_copy = malloc(actual_byte_size); memcpy(*data_copy, ©_data, actual_byte_size); @@ -445,14 +418,6 @@ semian_shm_object_acquire_memory(VALUE self, VALUE should_keep_req_byte_size) semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - if (-1 == ptr->semid){ - rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); - return self; - } - if (TYPE(should_keep_req_byte_size) != T_TRUE && - TYPE(should_keep_req_byte_size) != T_FALSE) - rb_raise(rb_eTypeError, "expected true or false for should_keep_req_byte_size"); - int should_keep_req_byte_size_bool = RTEST(should_keep_req_byte_size); key_t key = ptr->key; @@ -461,8 +426,8 @@ semian_shm_object_acquire_memory(VALUE self, VALUE should_keep_req_byte_size) int prev_mem_attach_count = 0; // Create/acquire memory and manage all resizing cases - int shmid = create_and_resize_memory(key, ptr->byte_size, ptr->permissions, should_keep_req_byte_size_bool, - &data_copy, &data_copy_byte_size, &prev_mem_attach_count, ptr, self); + int shmid = create_and_resize_memory(key, should_keep_req_byte_size_bool, &data_copy, + &data_copy_byte_size, &prev_mem_attach_count, self); if (shmid < 0) {// failed if (data_copy) free(data_copy); @@ -488,7 +453,29 @@ semian_shm_object_acquire_memory(VALUE self, VALUE should_keep_req_byte_size) } VALUE -semian_shm_object_delete_memory_inner (VALUE self) +semian_shm_object_check_and_resize_if_needed(VALUE self) { + semian_shm_object *ptr; + TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + + struct shmid_ds shm_info; + int needs_resize = 0; + if (-1 != ptr->shmid && -1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)) { + needs_resize = shm_info.shm_perm.mode & SHM_DEST; + } + VALUE priority = Qfalse; + if (-1 == ptr->shmid && 0 == ptr->shm_address){ + needs_resize = 1; + priority = Qtrue; + } + if (needs_resize) { + semian_shm_object_cleanup_memory(self); + semian_shm_object_acquire_memory(self, priority); + } + return self; +} + +VALUE +semian_shm_object_cleanup_memory_inner(VALUE self) { // This internal function may be called from a variety of contexts // Sometimes it is under a semaphore lock, sometimes it has extra malloc ptrs @@ -497,60 +484,28 @@ semian_shm_object_delete_memory_inner (VALUE self) semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - if (0 != ptr->shm_address){ - if (-1 == shmdt(ptr->shm_address)) { - rb_raise(eSyscall,"shmdt: no attached memory at %p, errno %d (%s)", ptr->shm_address, errno, strerror(errno)); - } else { - } - ptr->shm_address = 0; + if (0 != ptr->shm_address && -1 == shmdt(ptr->shm_address)) { + rb_raise(eSyscall,"shmdt: no attached memory at %p, errno %d (%s)", ptr->shm_address, errno, strerror(errno)); } + ptr->shm_address = 0; - if (-1 != ptr->shmid) { - // Once IPC_RMID is set, no new shmgets can be made with key, and current values are invalid - if (-1 == shmctl(ptr->shmid, IPC_RMID, 0)) { - if (errno == EINVAL) { - ptr->shmid = -1; - } else { - rb_raise(eSyscall,"shmctl: error flagging memory for removal with shmid %d, errno %d (%s)", ptr->shmid, errno, strerror(errno)); - } - } else { - ptr->shmid = -1; - } + if (-1 != ptr->shmid && -1 == shmctl(ptr->shmid, IPC_RMID, 0)) { + if (errno != EINVAL) + rb_raise(eSyscall,"shmctl: error flagging memory for removal with shmid %d, errno %d (%s)", ptr->shmid, errno, strerror(errno)); } + ptr->shmid = -1; return Qnil; } VALUE -semian_shm_object_delete_memory (VALUE self) +semian_shm_object_cleanup_memory(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); lock_status status = { ptr->lock_count, ptr }; if (!(WITHOUT_GVL(semian_shm_object_lock_without_gvl, (void *)ptr, RUBY_UBF_IO, NULL))) return Qnil; - return rb_ensure(semian_shm_object_delete_memory_inner, self, semian_shm_object_synchronize_restore_lock_status, (VALUE)&status); -} - -VALUE -semian_shm_object_check_and_resize_if_needed(VALUE self) { - semian_shm_object *ptr; - TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - - struct shmid_ds shm_info; - int needs_resize = 0; - if (-1 != ptr->shmid && -1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)) { - needs_resize = shm_info.shm_perm.mode & SHM_DEST; - } - VALUE priority = Qfalse; - if (-1 == ptr->shmid && 0 == ptr->shm_address){ - needs_resize = 1; - priority = Qtrue; - } - if (needs_resize) { - semian_shm_object_delete_memory_inner(self); - semian_shm_object_acquire_memory(self, priority); - } - return self; + return rb_ensure(semian_shm_object_cleanup_memory_inner, self, semian_shm_object_synchronize_restore_lock_status, (VALUE)&status); } static VALUE @@ -585,7 +540,6 @@ Init_semian_shm_object (void) { VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); VALUE cSysVSharedMemory = rb_const_get(cSemianModule, rb_intern("SysVSharedMemory")); - //rb_define_alloc_func(cSysVSharedMemory, semian_shm_object_alloc); rb_define_method(cSysVSharedMemory, "_acquire", semian_shm_object_acquire, 3); rb_define_method(cSysVSharedMemory, "_destroy", semian_shm_object_destroy, 0); rb_define_method(cSysVSharedMemory, "byte_size", semian_shm_object_byte_size, 0); @@ -608,4 +562,3 @@ Init_semian_shm_object (void) { Init_semian_integer(); Init_semian_sliding_window(); } - diff --git a/ext/semian/semian_shared_memory_object.h b/ext/semian/semian_shared_memory_object.h index 8631b0a8..65f62f1d 100644 --- a/ext/semian/semian_shared_memory_object.h +++ b/ext/semian/semian_shared_memory_object.h @@ -26,9 +26,7 @@ VALUE semian_shm_object_acquire(VALUE self, VALUE id, VALUE byte_size, VALUE per VALUE semian_shm_object_destroy(VALUE self); VALUE semian_shm_object_acquire_semaphore (VALUE self); VALUE semian_shm_object_delete_semaphore(VALUE self); -VALUE semian_shm_object_acquire_memory(VALUE self, VALUE should_keep_req_byte_size); -VALUE semian_shm_object_delete_memory_inner (VALUE self); -VALUE semian_shm_object_delete_memory (VALUE self); +VALUE semian_shm_object_cleanup_memory (VALUE self); VALUE semian_shm_object_check_and_resize_if_needed (VALUE self); VALUE semian_shm_object_synchronize(VALUE self); diff --git a/ext/semian/semian_sliding_window.c b/ext/semian/semian_sliding_window.c index 813e5f56..cfa3e247 100644 --- a/ext/semian/semian_sliding_window.c +++ b/ext/semian/semian_sliding_window.c @@ -238,10 +238,12 @@ semian_sliding_window_resize_to(VALUE self, VALUE size) { } } - semian_shm_object_delete_memory_inner(self); + semian_shm_object_cleanup_memory(self); ptr->byte_size = 2*sizeof(int) + NUM2INT(size) * sizeof(long); + ptr->shmid = -1; + ptr->shm_address = 0; - semian_shm_object_acquire_memory(self, Qtrue); + semian_shm_object_check_and_resize_if_needed(self); ptr->initialize_memory(ptr->byte_size, ptr->shm_address, data_copy, byte_size, prev_mem_attach_count); From 5e55f6e98f80c297a7f022d4691c975940554386 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Thu, 19 Nov 2015 20:06:20 +0000 Subject: [PATCH 20/23] Refactored resizing and initializing code, reduced size from 300~ lines to 100 --- ext/semian/semian_integer.c | 10 +- ext/semian/semian_shared_memory_object.c | 242 +++++++---------------- ext/semian/semian_shared_memory_object.h | 4 +- ext/semian/semian_sliding_window.c | 76 +++---- test/circuit_breaker_test.rb | 19 +- test/sysv_integer_test.rb | 18 +- test/sysv_sliding_window_test.rb | 13 +- 7 files changed, 129 insertions(+), 253 deletions(-) diff --git a/ext/semian/semian_integer.c b/ext/semian/semian_integer.c index 6b74be08..7cf404fd 100644 --- a/ext/semian/semian_integer.c +++ b/ext/semian/semian_integer.c @@ -4,21 +4,19 @@ typedef struct { int value; } semian_int; -static void semian_integer_initialize_memory (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); +static void semian_integer_initialize_memory (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size); static VALUE semian_integer_bind_initialize_memory_callback(VALUE self); static VALUE semian_integer_get_value(VALUE self); static VALUE semian_integer_set_value(VALUE self, VALUE num); static VALUE semian_integer_increment(int argc, VALUE *argv, VALUE self); static void -semian_integer_initialize_memory (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count) +semian_integer_initialize_memory (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size) { semian_int *ptr = dest; semian_int *old = prev_data; - if (prev_mem_attach_count){ - if (prev_data){ - ptr->value = old->value; - } // else copy nothing, data is same size and no need to copy + if (prev_data){ + ptr->value = old->value; } else { ptr->value=0; } diff --git a/ext/semian/semian_shared_memory_object.c b/ext/semian/semian_shared_memory_object.c index f66ccc68..d397f6c1 100644 --- a/ext/semian/semian_shared_memory_object.c +++ b/ext/semian/semian_shared_memory_object.c @@ -255,7 +255,7 @@ typedef struct { // Workaround rb_ensure only allows one argument for each callb static VALUE semian_shm_object_synchronize_with_block(VALUE self) { - semian_shm_object_check_and_resize_if_needed(self); + semian_shm_object_synchronize_memory_and_size(self, Qfalse); if (!rb_block_given_p()) return Qnil; return rb_yield(Qnil); @@ -265,10 +265,17 @@ static VALUE semian_shm_object_synchronize_restore_lock_status(VALUE v_status) { lock_status *status = (lock_status *) v_status; - while (status->ptr->lock_count > status->pre_block_lock_count_state) + int tries = 0; + int allowed_tries = 5; + while (++tries < allowed_tries && status->ptr->lock_count > status->pre_block_lock_count_state) return (VALUE) WITHOUT_GVL(semian_shm_object_unlock_without_gvl, (void *)(status->ptr), RUBY_UBF_IO, NULL); - while (status->ptr->lock_count < status->pre_block_lock_count_state) + if (tries >= allowed_tries) + rb_raise(eSyscall, "Failed to restore lock status after %d tries", allowed_tries); + tries = 0; + while (++tries < allowed_tries && status->ptr->lock_count < status->pre_block_lock_count_state) return (VALUE) WITHOUT_GVL(semian_shm_object_lock_without_gvl, (void *)(status->ptr), RUBY_UBF_IO, NULL); + if (tries >= allowed_tries) + rb_raise(eSyscall, "Failed to restore lock status after %d tries", allowed_tries); return Qnil; } @@ -293,194 +300,81 @@ define_method_with_synchronize(VALUE klass, const char *name, VALUE (*func)(ANYA * Memory functions */ -static int -create_and_resize_memory(key_t key, int should_keep_req_byte_size_bool, - void **data_copy, size_t *data_copy_byte_size, int *prev_mem_attach_count, VALUE self) { - - // Below will handle acquiring/creating new memory, and possibly resizing the - // memory or the ptr->byte_size depending on - // should_keep_req_byte_size_bool - +VALUE +semian_shm_object_synchronize_memory_and_size(VALUE self, VALUE is_master_obj) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - int shmid=-1; - int failed=0; - - int requested_byte_size = ptr->byte_size; - long permissions = ptr->permissions; - int flags = IPC_CREAT | IPC_EXCL | permissions; - - int actual_byte_size = 0; - - // We fill both actual_byte_size and requested_byte_size - // Logic matrix: - // actual=>req | actual=req | actual current mem size - // 2. segment was requested to be created, but (size > SHMMAX || size < SHMMIN) - // 3. Other error - - // We can only see SHMMAX and SHMMIN through console commands - // Unlikely for 2 to occur, so we check by requesting a memory of size 1 byte - if (-1 != (shmid = shmget(key, 1, flags & ~IPC_EXCL))) { - struct shmid_ds shm_info; - if (-1 != shmctl(shmid, IPC_STAT, &shm_info)){ - actual_byte_size = shm_info.shm_segsz; - *prev_mem_attach_count = shm_info.shm_nattch; - failed = 0; - } else - failed=-3; - } else { // Error, exit - failed=-4; - } - } else { - actual_byte_size = requested_byte_size; + struct shmid_ds shm_info = { }; + const int SHMMIN = 1; + key_t key = ptr->key; + + int is_master = RTEST(is_master_obj); + is_master |= (-1 == ptr->shmid) && (0 == ptr->shm_address); + + int shmid_out_of_sync = 0; + shmid_out_of_sync |= (-1 == ptr->shmid) && (0 == ptr->shm_address); // If not attached at all + if ((-1 != ptr->shmid) && (-1 != shmctl(ptr->shmid, IPC_STAT, &shm_info))) { + // If current attached memory is marked for deletion + shmid_out_of_sync |= shm_info.shm_perm.mode & SHM_DEST && + ptr->shmid != shmget(key, SHMMIN, IPC_CREAT | ptr->permissions); } - *data_copy_byte_size = actual_byte_size; - if (should_keep_req_byte_size_bool && !failed) { // resizing may occur - // we want to keep this worker's data - // We flag old mem by IPC_RMID and copy data - - if (actual_byte_size != requested_byte_size) { - void *data; - - if ((void *)-1 != (data = shmat(shmid, (void *)0, 0))) { - - char copy_data[actual_byte_size]; - memcpy(©_data, data, actual_byte_size); - ptr->shmid=shmid; - ptr->shm_address = data; - semian_shm_object_cleanup_memory(self); - - *data_copy = malloc(actual_byte_size); - memcpy(*data_copy, ©_data, actual_byte_size); - - // Flagging for deletion sets a shm's associated key to be 0 so shmget gets a different shmid. - // If this worker is creating the new memory - if (-1 != (shmid = shmget(key, requested_byte_size, flags))) { - } else { // failed to get new memory, exit - failed=-5; - } - } else { // failed to attach, exit - rb_raise(eInternal,"Failed to copy old data, key %d, shmid %d, errno %d (%s)",key, shmid, errno, strerror(errno)); - failed=-6; + size_t requested_byte_size = ptr->byte_size; + int first_sync = (-1 == ptr->shmid) && (shmid_out_of_sync); + + if (shmid_out_of_sync) { + semian_shm_object_cleanup_memory(self); + if ((-1 == (ptr->shmid = shmget(key, SHMMIN, ptr->permissions)))) { + if ((-1 == (ptr->shmid = shmget(key, ptr->byte_size, IPC_CREAT | IPC_EXCL | ptr->permissions)))) { + rb_raise(eSyscall, "shmget failed to create or attach current memory with key %d shmid %d errno %d (%s)", key, ptr->shmid, errno, strerror(errno)); } - } else{ - failed = -7; } - } else if (!failed){ // ptr->byte_size may be changed - // we don't want to keep this worker's data, it is old - ptr->byte_size = actual_byte_size; - } else - failed=-8; - if (-1 != shmid) { - return shmid; - } else { - return failed; + if ((void *)-1 == (ptr->shm_address = shmat(ptr->shmid, NULL, 0))) { + ptr->shm_address = NULL; + rb_raise(eSyscall, "shmat failed to mount current memory with key %d shmid %d errno %d (%s)", key, ptr->shmid, errno, strerror(errno)); + } } -} -/* - Acquire memory by getting shmid, and then attaching it to a memory location, - requires semaphore for locking/unlocking to be setup - - Note: should_keep_req_byte_size is a bool that decides how ptr->byte_size - is handled. There may be a discrepancy between the requested memory size and the actual - size of the memory block given. If it is false (0), ptr->byte_size will be modified - to the actual memory size if there is a difference. This matters when dynamicaly resizing - memory. - Think of should_keep_req_byte_size as this worker requesting a size, others resizing, - and !should_keep_req_byte_size as another worker requesting a size and this worker - resizing -*/ -VALUE -semian_shm_object_acquire_memory(VALUE self, VALUE should_keep_req_byte_size) -{ - semian_shm_object *ptr; - TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + if (-1 == shmctl(ptr->shmid, IPC_STAT, &shm_info)){ + rb_raise(eSyscall, "shmctl failed to inspect current memory with key %d shmid %d errno %d (%s)", key, ptr->shmid, errno, strerror(errno)); + } + ptr->byte_size = shm_info.shm_segsz; - int should_keep_req_byte_size_bool = RTEST(should_keep_req_byte_size); + int old_mem_attach_count = shm_info.shm_nattch; - key_t key = ptr->key; - void *data_copy = NULL; // Will contain contents of old memory, if any - size_t data_copy_byte_size = 0; - int prev_mem_attach_count = 0; - - // Create/acquire memory and manage all resizing cases - int shmid = create_and_resize_memory(key, should_keep_req_byte_size_bool, &data_copy, - &data_copy_byte_size, &prev_mem_attach_count, self); - if (shmid < 0) {// failed - if (data_copy) - free(data_copy); - rb_raise(eSyscall, "shmget() failed at %d to acquire a memory shmid with key %d, size %zu, errno %d (%s)", shmid, key, ptr->byte_size, errno, strerror(errno)); - } else - ptr->shmid = shmid; - - // Attach memory and call initialize_memory() - if (0 == ptr->shm_address) { - ptr->shm_address = shmat(ptr->shmid, (void *)0, 0); - if (((void*)-1) == ptr->shm_address) { - ptr->shm_address = 0; - if (data_copy) - free(data_copy); - rb_raise(eSyscall, "shmat() failed to attach memory with shmid %d, size %zu, errno %d (%s)", ptr->shmid, ptr->byte_size, errno, strerror(errno)); + if (is_master) { + if (ptr->byte_size == requested_byte_size && first_sync && 1 == old_mem_attach_count) { + ptr->initialize_memory(ptr->byte_size, ptr->shm_address, NULL, 0); } else { - ptr->initialize_memory(ptr->byte_size, ptr->shm_address, data_copy, data_copy_byte_size, prev_mem_attach_count); - } - } - if (data_copy) - free(data_copy); - return self; -} - -VALUE -semian_shm_object_check_and_resize_if_needed(VALUE self) { - semian_shm_object *ptr; - TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + void *old_shm_address = ptr->shm_address; + size_t old_byte_size = ptr->byte_size; + void *old_memory_content = NULL; + + char old_memory_content_tmp[old_byte_size]; + memcpy(old_memory_content_tmp, old_shm_address, old_byte_size); + semian_shm_object_cleanup_memory(self); + old_memory_content = malloc(old_byte_size); + memcpy(old_memory_content, old_memory_content_tmp, old_byte_size); + + if (-1 == (ptr->shmid = shmget(key, requested_byte_size, IPC_CREAT | IPC_EXCL | ptr->permissions))) { + rb_raise(eSyscall, "shmget failed to create new resized memory with key %d shmid %d errno %d (%s)", key, ptr->shmid, errno, strerror(errno)); + } + if ((void *)-1 == (ptr->shm_address = shmat(ptr->shmid, NULL, 0))) { + rb_raise(eSyscall, "shmat failed to mount new resized memory with key %d shmid %d errno %d (%s)", key, ptr->shmid, errno, strerror(errno)); + } + ptr->byte_size = requested_byte_size; - struct shmid_ds shm_info; - int needs_resize = 0; - if (-1 != ptr->shmid && -1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)) { - needs_resize = shm_info.shm_perm.mode & SHM_DEST; - } - VALUE priority = Qfalse; - if (-1 == ptr->shmid && 0 == ptr->shm_address){ - needs_resize = 1; - priority = Qtrue; - } - if (needs_resize) { - semian_shm_object_cleanup_memory(self); - semian_shm_object_acquire_memory(self, priority); + ptr->initialize_memory(ptr->byte_size, ptr->shm_address, old_memory_content, old_byte_size); + free(old_memory_content); + } } return self; } -VALUE +static VALUE semian_shm_object_cleanup_memory_inner(VALUE self) { - // This internal function may be called from a variety of contexts - // Sometimes it is under a semaphore lock, sometimes it has extra malloc ptrs - // Arguments handle these conditions - semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); @@ -542,11 +436,11 @@ Init_semian_shm_object (void) { rb_define_method(cSysVSharedMemory, "_acquire", semian_shm_object_acquire, 3); rb_define_method(cSysVSharedMemory, "_destroy", semian_shm_object_destroy, 0); - rb_define_method(cSysVSharedMemory, "byte_size", semian_shm_object_byte_size, 0); + rb_define_method(cSysVSharedMemory, "_synchronize", semian_shm_object_synchronize, 0); rb_define_method(cSysVSharedMemory, "semid", semian_shm_object_semid, 0); rb_define_method(cSysVSharedMemory, "shmid", semian_shm_object_shmid, 0); - rb_define_method(cSysVSharedMemory, "_synchronize", semian_shm_object_synchronize, 0); + rb_define_method(cSysVSharedMemory, "byte_size", semian_shm_object_byte_size, 0); rb_define_singleton_method(cSysVSharedMemory, "_sizeof", semian_shm_object_sizeof, 1); rb_define_singleton_method(cSysVSharedMemory, "replace_alloc", semian_shm_object_replace_alloc, 1); diff --git a/ext/semian/semian_shared_memory_object.h b/ext/semian/semian_shared_memory_object.h index 65f62f1d..7639a9a6 100644 --- a/ext/semian/semian_shared_memory_object.h +++ b/ext/semian/semian_shared_memory_object.h @@ -8,7 +8,7 @@ typedef struct { int permissions; int semid; int shmid; - void (*initialize_memory)(size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); + void (*initialize_memory)(size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size); void *shm_address; } semian_shm_object; @@ -27,7 +27,7 @@ VALUE semian_shm_object_destroy(VALUE self); VALUE semian_shm_object_acquire_semaphore (VALUE self); VALUE semian_shm_object_delete_semaphore(VALUE self); VALUE semian_shm_object_cleanup_memory (VALUE self); -VALUE semian_shm_object_check_and_resize_if_needed (VALUE self); +VALUE semian_shm_object_synchronize_memory_and_size (VALUE self, VALUE is_master); VALUE semian_shm_object_synchronize(VALUE self); void define_method_with_synchronize(VALUE klass, const char *name, VALUE (*func)(ANYARGS), int argc); diff --git a/ext/semian/semian_sliding_window.c b/ext/semian/semian_sliding_window.c index cfa3e247..33350bfc 100644 --- a/ext/semian/semian_sliding_window.c +++ b/ext/semian/semian_sliding_window.c @@ -6,7 +6,7 @@ typedef struct { long window[]; } semian_sliding_window; -static void semian_sliding_window_initialize_memory (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count); +static void semian_sliding_window_initialize_memory (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size); static VALUE semian_sliding_window_bind_initialize_memory_callback(VALUE self); static VALUE semian_sliding_window_size(VALUE self); static VALUE semian_sliding_window_max_size(VALUE self); @@ -20,30 +20,28 @@ static VALUE semian_sliding_window_last(VALUE self); static VALUE semian_sliding_window_resize_to(VALUE self, VALUE size); static void -semian_sliding_window_initialize_memory (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size, int prev_mem_attach_count) +semian_sliding_window_initialize_memory (size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size) { semian_sliding_window *ptr = dest; semian_sliding_window *old = prev_data; // Logic to initialize: initialize to 0 only if you're the first to attach (else body) // Otherwise, copy previous data into new data - if (prev_mem_attach_count) { - if (prev_data) { - // transfer data over - ptr->max_window_size = (byte_size-2*sizeof(int))/sizeof(long); - ptr->window_size = fmin(ptr->max_window_size, old->window_size); - - // Copy the most recent ptr->shm_address->window_size numbers to new memory - memcpy(&(ptr->window), - ((long *)(&(old->window[0])))+old->window_size-ptr->window_size, - ptr->window_size * sizeof(long)); - } // else copy nothing, data is same size and no need to copy + if (prev_data) { + // transfer data over + ptr->max_window_size = (byte_size - 2 * sizeof(int)) / sizeof(long); + ptr->window_size = fmin(ptr->max_window_size, old->window_size); + + // Copy the most recent ptr->shm_address->window_size numbers to new memory + memcpy(&(ptr->window), + ((long *)(&(old->window[0]))) + old->window_size - ptr->window_size, + ptr->window_size * sizeof(long)); } else { semian_sliding_window *data = dest; - data->max_window_size = (byte_size-2*sizeof(int))/sizeof(long); + data->max_window_size = (byte_size - 2 * sizeof(int)) / sizeof(long); data->window_size = 0; - for (int i=0; i< data->window_size; ++i) - data->window[i]=0; + for (int i = 0; i < data->window_size; ++i) + data->window[i] = 0; } } @@ -90,8 +88,8 @@ semian_sliding_window_push_back(VALUE self, VALUE num) semian_sliding_window *data = ptr->shm_address; if (data->window_size == data->max_window_size) { - for (int i=1; i< data->max_window_size; ++i){ - data->window[i-1] = data->window[i]; + for (int i = 1; i < data->max_window_size; ++i){ + data->window[i - 1] = data->window[i]; } --(data->window_size); } @@ -113,7 +111,7 @@ semian_sliding_window_pop_back(VALUE self) if (0 == data->window_size) retval = Qnil; else { - retval = LONG2NUM(data->window[data->window_size-1]); + retval = LONG2NUM(data->window[data->window_size - 1]); --(data->window_size); } return retval; @@ -132,13 +130,12 @@ semian_sliding_window_push_front(VALUE self, VALUE num) long val = NUM2LONG(num); semian_sliding_window *data = ptr->shm_address; - int i=data->window_size; - for (; i>0; --i) - data->window[i]=data->window[i-1]; + for (int i=data->window_size; i > 0; --i) + data->window[i] = data->window[i - 1]; data->window[0] = val; ++(data->window_size); - if (data->window_size>data->max_window_size) + if (data->window_size > data->max_window_size) data->window_size=data->max_window_size; return self; @@ -158,8 +155,8 @@ semian_sliding_window_pop_front(VALUE self) retval = Qnil; else { retval = LONG2NUM(data->window[0]); - for (int i=0; iwindow_size-1; ++i) - data->window[i]=data->window[i+1]; + for (int i = 0; i < data->window_size - 1; ++i) + data->window[i] = data->window[i + 1]; --(data->window_size); } @@ -174,7 +171,7 @@ semian_sliding_window_clear(VALUE self) if (0 == ptr->shm_address) return Qnil; semian_sliding_window *data = ptr->shm_address; - data->window_size=0; + data->window_size = 0; return self; } @@ -189,7 +186,7 @@ semian_sliding_window_first(VALUE self) VALUE retval; semian_sliding_window *data = ptr->shm_address; - if (data->window_size >=1) + if (data->window_size >= 1) retval = LONG2NUM(data->window[0]); else retval = Qnil; @@ -208,7 +205,7 @@ semian_sliding_window_last(VALUE self) VALUE retval; semian_sliding_window *data = ptr->shm_address; if (data->window_size > 0) - retval = LONG2NUM(data->window[data->window_size-1]); + retval = LONG2NUM(data->window[data->window_size - 1]); else retval = Qnil; @@ -225,27 +222,8 @@ semian_sliding_window_resize_to(VALUE self, VALUE size) { if (NUM2INT(size) <= 0) rb_raise(rb_eArgError, "size must be larger than 0"); - semian_sliding_window *data_copy = NULL; - size_t byte_size=0; - int prev_mem_attach_count = 0; - if (-1 != ptr->shmid && (void *)-1 != ptr->shm_address) { - data_copy = malloc(ptr->byte_size); - memcpy(data_copy,ptr->shm_address,ptr->byte_size); - byte_size = ptr->byte_size; - struct shmid_ds shm_info; - if (-1 != shmctl(ptr->shmid, IPC_STAT, &shm_info)){ - prev_mem_attach_count = shm_info.shm_nattch; - } - } - - semian_shm_object_cleanup_memory(self); - ptr->byte_size = 2*sizeof(int) + NUM2INT(size) * sizeof(long); - ptr->shmid = -1; - ptr->shm_address = 0; - - semian_shm_object_check_and_resize_if_needed(self); - - ptr->initialize_memory(ptr->byte_size, ptr->shm_address, data_copy, byte_size, prev_mem_attach_count); + ptr->byte_size = 2 * sizeof(int) + NUM2INT(size) * sizeof(long); + semian_shm_object_synchronize_memory_and_size(self, Qtrue); return self; } diff --git a/test/circuit_breaker_test.rb b/test/circuit_breaker_test.rb index e79964eb..37abf78d 100644 --- a/test/circuit_breaker_test.rb +++ b/test/circuit_breaker_test.rb @@ -124,24 +124,23 @@ def test_shared_fresh_worker_killed_should_not_reset_circuit_breaker_data rescue nil end - Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) - @resource = Semian[:unique_res] + reader, writer = IO.pipe pid = fork do + reader.close Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) resource_inner = Semian[:unique_res] + resource_inner.reset open_circuit! resource_inner + writer.puts "Done" + writer.close sleep end - sleep 1 - Process.kill('KILL', pid) - Process.waitall - fork do - Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) - assert_circuit_opened Semian[:unique_res] - end - Process.waitall + reader.gets + Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) + Process.kill(9, pid) + assert_circuit_opened Semian[:unique_res] end private diff --git a/test/sysv_integer_test.rb b/test/sysv_integer_test.rb index 3404f0fc..45149e54 100644 --- a/test/sysv_integer_test.rb +++ b/test/sysv_integer_test.rb @@ -28,19 +28,21 @@ def test_memory_not_reset_when_at_least_one_worker_using_it @integer.value = 109 integer_2 = CLASS.new(name: 'TestSysVInteger', permissions: 0660) assert_equal @integer.value, integer_2.value + + reader, writer = IO.pipe pid = fork do + reader.close integer_3 = CLASS.new(name: 'TestSysVInteger', permissions: 0660) assert_equal 109, integer_3.value + integer_3.value = 110 + writer.puts "Done" + writer.close sleep end - sleep 1 - Process.kill("KILL", pid) - Process.waitall - fork do - integer_3 = CLASS.new(name: 'TestSysVInteger', permissions: 0660) - assert_equal 109, integer_3.value - end - Process.waitall + + reader.gets + Process.kill(9, pid) + assert_equal 110, integer_2.value end def test_memory_reset_when_no_workers_using_it diff --git a/test/sysv_sliding_window_test.rb b/test/sysv_sliding_window_test.rb index c8d9eff1..d18f600f 100644 --- a/test/sysv_sliding_window_test.rb +++ b/test/sysv_sliding_window_test.rb @@ -22,16 +22,21 @@ def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it assert_equal 100, @sliding_window.first end + reader, writer = IO.pipe pid = fork do + reader.close sliding_window_2 = CLASS.new(max_size: 6, name: 'TestSysVSlidingWindow', permissions: 0660) - sliding_window_2.synchronize { sleep } + sliding_window_2.synchronize do + writer.puts "Done" + writer.close + sleep + end end - sleep 1 - Process.kill('KILL', pid) - Process.waitall + reader.gets + Process.kill(9, pid) Timeout.timeout(1) do # assure dont hang @sliding_window << 100 From a35d0d79ac6500f7156556fb3bce995bf7e46ff1 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Fri, 20 Nov 2015 20:52:19 +0000 Subject: [PATCH 21/23] Made fallback boot-time instead of runtime, remove unneeded functions --- ext/semian/semian_shared_memory_object.c | 47 ++++++++++-------------- lib/semian/sysv_shared_memory.rb | 39 ++++---------------- lib/semian/sysv_state.rb | 28 ++++---------- test/circuit_breaker_test.rb | 1 + test/resource_test.rb | 28 +++++++++----- test/simple_integer_test.rb | 4 +- test/simple_sliding_window_test.rb | 4 +- test/simple_state_test.rb | 4 +- test/sysv_integer_test.rb | 14 +++---- test/sysv_sliding_window_test.rb | 21 ++++++----- test/sysv_state_test.rb | 6 +-- 11 files changed, 81 insertions(+), 115 deletions(-) diff --git a/ext/semian/semian_shared_memory_object.c b/ext/semian/semian_shared_memory_object.c index d397f6c1..65f20d93 100644 --- a/ext/semian/semian_shared_memory_object.c +++ b/ext/semian/semian_shared_memory_object.c @@ -82,25 +82,29 @@ semian_shm_object_sizeof(VALUE klass, VALUE type) return INT2NUM(sizeof(long)); // Can definitely add more else - return INT2NUM(0); + rb_raise(rb_eTypeError, "%s is not a valid C type", rb_id2name(SYM2ID(type))); } VALUE -semian_shm_object_acquire(VALUE self, VALUE name, VALUE byte_size, VALUE permissions) +semian_shm_object_acquire(VALUE self, VALUE name, VALUE data_layout, VALUE permissions) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (TYPE(name) != T_SYMBOL && TYPE(name) != T_STRING) rb_raise(rb_eTypeError, "id must be a symbol or string"); - if (TYPE(byte_size) != T_FIXNUM) - rb_raise(rb_eTypeError, "expected integer for byte_size"); + if (TYPE(data_layout) != T_ARRAY) + rb_raise(rb_eTypeError, "expected array for data_layout"); if (TYPE(permissions) != T_FIXNUM) rb_raise(rb_eTypeError, "expected integer for permissions"); - if (NUM2SIZET(byte_size) <= 0) - rb_raise(rb_eArgError, "byte_size must be larger than 0"); + int byte_size = 0; + for (int i = 0; i < RARRAY_LEN(data_layout); ++i) + byte_size += NUM2INT(semian_shm_object_sizeof(rb_cObject, RARRAY_PTR(data_layout)[i])); + + if (byte_size <= 0) + rb_raise(rb_eArgError, "total size must be larger than 0"); const char *id_str = NULL; if (TYPE(name) == T_SYMBOL) { @@ -109,7 +113,7 @@ semian_shm_object_acquire(VALUE self, VALUE name, VALUE byte_size, VALUE permiss id_str = RSTRING_PTR(name); } ptr->key = generate_key(id_str); - ptr->byte_size = NUM2SIZET(byte_size); // byte_size >=1 or error would have been raised earlier + ptr->byte_size = byte_size; // byte_size >=1 or error would have been raised earlier ptr->semid = -1; // id's default to -1 ptr->shmid = -1; ptr->shm_address = 0; // address defaults to NULL @@ -124,7 +128,9 @@ semian_shm_object_acquire(VALUE self, VALUE name, VALUE byte_size, VALUE permiss semian_shm_object_acquire_semaphore(self); semian_shm_object_synchronize(self); - return self; + + + return Qtrue; } VALUE @@ -349,13 +355,10 @@ semian_shm_object_synchronize_memory_and_size(VALUE self, VALUE is_master_obj) { } else { void *old_shm_address = ptr->shm_address; size_t old_byte_size = ptr->byte_size; - void *old_memory_content = NULL; + unsigned char old_memory_content[old_byte_size]; - char old_memory_content_tmp[old_byte_size]; - memcpy(old_memory_content_tmp, old_shm_address, old_byte_size); + memcpy(old_memory_content, old_shm_address, old_byte_size); semian_shm_object_cleanup_memory(self); - old_memory_content = malloc(old_byte_size); - memcpy(old_memory_content, old_memory_content_tmp, old_byte_size); if (-1 == (ptr->shmid = shmget(key, requested_byte_size, IPC_CREAT | IPC_EXCL | ptr->permissions))) { rb_raise(eSyscall, "shmget failed to create new resized memory with key %d shmid %d errno %d (%s)", key, ptr->shmid, errno, strerror(errno)); @@ -366,7 +369,6 @@ semian_shm_object_synchronize_memory_and_size(VALUE self, VALUE is_master_obj) { ptr->byte_size = requested_byte_size; ptr->initialize_memory(ptr->byte_size, ptr->shm_address, old_memory_content, old_byte_size); - free(old_memory_content); } } return self; @@ -419,30 +421,19 @@ semian_shm_object_shmid(VALUE self) return INT2NUM(ptr->shmid); } -static VALUE -semian_shm_object_byte_size(VALUE self) -{ - semian_shm_object *ptr; - TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - semian_shm_object_synchronize(self); - return INT2NUM(ptr->byte_size); -} - void Init_semian_shm_object (void) { VALUE cSemianModule = rb_const_get(rb_cObject, rb_intern("Semian")); VALUE cSysVSharedMemory = rb_const_get(cSemianModule, rb_intern("SysVSharedMemory")); - rb_define_method(cSysVSharedMemory, "_acquire", semian_shm_object_acquire, 3); - rb_define_method(cSysVSharedMemory, "_destroy", semian_shm_object_destroy, 0); - rb_define_method(cSysVSharedMemory, "_synchronize", semian_shm_object_synchronize, 0); + rb_define_method(cSysVSharedMemory, "acquire_memory_object", semian_shm_object_acquire, 3); + rb_define_method(cSysVSharedMemory, "destroy", semian_shm_object_destroy, 0); + rb_define_method(cSysVSharedMemory, "synchronize", semian_shm_object_synchronize, 0); rb_define_method(cSysVSharedMemory, "semid", semian_shm_object_semid, 0); rb_define_method(cSysVSharedMemory, "shmid", semian_shm_object_shmid, 0); - rb_define_method(cSysVSharedMemory, "byte_size", semian_shm_object_byte_size, 0); - rb_define_singleton_method(cSysVSharedMemory, "_sizeof", semian_shm_object_sizeof, 1); rb_define_singleton_method(cSysVSharedMemory, "replace_alloc", semian_shm_object_replace_alloc, 1); decrement.sem_num = kSHMIndexTicketLock; diff --git a/lib/semian/sysv_shared_memory.rb b/lib/semian/sysv_shared_memory.rb index 8f3b9283..58abe7ee 100644 --- a/lib/semian/sysv_shared_memory.rb +++ b/lib/semian/sysv_shared_memory.rb @@ -1,12 +1,5 @@ module Semian module SysVSharedMemory #:nodoc: - @type_size = {} - def self.sizeof(type) - size = (@type_size[type.to_sym] ||= (respond_to?(:_sizeof) ? _sizeof(type.to_sym) : 0)) - raise TypeError.new("#{type} is not a valid C type") if size <= 0 - size - end - def self.included(base) def base.do_with_sync(*names) names.each do |name| @@ -30,38 +23,20 @@ def shmid -1 end - def synchronize(&block) - if respond_to?(:_synchronize) && @using_shared_memory - _synchronize(&block) - else - yield if block_given? - end + def synchronize + yield if block_given? end - alias_method :transaction, :synchronize - def destroy - if respond_to?(:_destroy) && @using_shared_memory - _destroy - else - super - end + super end private - def shared? - @using_shared_memory - end - - def acquire_memory_object(name, data_layout, permissions) - return @using_shared_memory = false unless Semian.semaphores_enabled? && respond_to?(:_acquire) - - byte_size = data_layout.inject(0) { |sum, type| sum + ::Semian::SysVSharedMemory.sizeof(type) } - raise TypeError.new("Given data layout is 0 bytes: #{data_layout.inspect}") if byte_size <= 0 - # Calls C layer to acquire/create a memory block, calling #initialize_memory in the process, see below - _acquire(name, byte_size, permissions) - @using_shared_memory = true + def acquire_memory_object(*) + # Concrete classes must call this method before accessing shared memory + # If SysV is enabled, a C method overrides this stub and returns true if acquiring succeeds + false end def bind_initialize_memory_callback diff --git a/lib/semian/sysv_state.rb b/lib/semian/sysv_state.rb index 0312a426..fc338a9a 100644 --- a/lib/semian/sysv_state.rb +++ b/lib/semian/sysv_state.rb @@ -6,37 +6,25 @@ class State < Semian::Simple::State #:nodoc: include SysVSharedMemory extend Forwardable - def_delegators :@integer, :semid, :shmid, :synchronize, :transaction, - :shared?, :acquire_memory_object, :initialize_memory - private :shared?, :acquire_memory_object, :initialize_memory + SYM_TO_NUM = {closed: 0, open: 1, half_open: 2}.freeze + NUM_TO_SYM = SYM_TO_NUM.invert.freeze + + def_delegators :@integer, :semid, :shmid, :synchronize, :acquire_memory_object, + :bind_initialize_memory_callback, :destroy + private :acquire_memory_object, :bind_initialize_memory_callback def initialize(name:, permissions:) @integer = Semian::SysV::Integer.new(name: name, permissions: permissions) - initialize_lookup - end - - def destroy - super - @integer.destroy end def value - @num_to_sym.fetch(@integer.value) { raise ArgumentError } + NUM_TO_SYM.fetch(@integer.value) { raise ArgumentError } end private def value=(sym) - @integer.value = @sym_to_num.fetch(sym) { raise ArgumentError } - end - - def initialize_lookup - # Assume symbol_list[0] is mapped to 0 - # Cannot just use #object_id since #object_id for symbols is different in every run - # For now, implement a C-style enum type backed by integers - - @sym_to_num = {closed: 0, open: 1, half_open: 2} - @num_to_sym = @sym_to_num.invert + @integer.value = SYM_TO_NUM.fetch(sym) { raise ArgumentError } end end end diff --git a/test/circuit_breaker_test.rb b/test/circuit_breaker_test.rb index 37abf78d..441a2e88 100644 --- a/test/circuit_breaker_test.rb +++ b/test/circuit_breaker_test.rb @@ -140,6 +140,7 @@ def test_shared_fresh_worker_killed_should_not_reset_circuit_breaker_data reader.gets Semian.register(:unique_res, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) Process.kill(9, pid) + Process.waitall assert_circuit_opened Semian[:unique_res] end diff --git a/test/resource_test.rb b/test/resource_test.rb index fa48c606..f9fd2cf0 100644 --- a/test/resource_test.rb +++ b/test/resource_test.rb @@ -108,19 +108,29 @@ def test_acquire_timeout_override end def test_acquire_with_fork - resource = create_resource :testing, tickets: 2, timeout: 0.5 - - resource.acquire do - fork do - resource.acquire do - assert_raises Semian::TimeoutError do - resource.acquire {} - end + pids = [] + + 2.times do + reader, writer = IO.pipe + pids << fork do + create_resource(:testing, tickets: 2, timeout: 0.5).acquire do + reader.close + writer.puts "Acquired" + writer.close + sleep end end + reader.gets + end + + assert_raises Semian::TimeoutError do + create_resource(:testing, tickets: 2, timeout: 0.5).acquire {} + end - Process.wait + pids.each do |pid| + Process.kill(9, pid) end + Process.waitall end def test_acquire_releases_on_kill diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index 195051d9..5e52c194 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -1,10 +1,10 @@ require 'test_helper' class TestSimpleInteger < MiniTest::Unit::TestCase - CLASS = ::Semian::Simple::Integer + KLASS = ::Semian::Simple::Integer def setup - @integer = CLASS.new + @integer = KLASS.new end def teardown diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index 0e673e36..11487180 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -1,10 +1,10 @@ require 'test_helper' class TestSimpleSlidingWindow < MiniTest::Unit::TestCase - CLASS = ::Semian::Simple::SlidingWindow + KLASS = ::Semian::Simple::SlidingWindow def setup - @sliding_window = CLASS.new(max_size: 6) + @sliding_window = KLASS.new(max_size: 6) @sliding_window.clear end diff --git a/test/simple_state_test.rb b/test/simple_state_test.rb index edab7c22..f05d86a2 100644 --- a/test/simple_state_test.rb +++ b/test/simple_state_test.rb @@ -1,10 +1,10 @@ require 'test_helper' class TestSimpleState < MiniTest::Unit::TestCase - CLASS = ::Semian::Simple::State + KLASS = ::Semian::Simple::State def setup - @state = CLASS.new + @state = KLASS.new end def teardown diff --git a/test/sysv_integer_test.rb b/test/sysv_integer_test.rb index 45149e54..1d140cb4 100644 --- a/test/sysv_integer_test.rb +++ b/test/sysv_integer_test.rb @@ -1,10 +1,10 @@ require 'test_helper' class TestSysVInteger < MiniTest::Unit::TestCase - CLASS = ::Semian::SysV::Integer + KLASS = ::Semian::SysV::Integer def setup - @integer = CLASS.new(name: 'TestSysVInteger', permissions: 0660) + @integer = KLASS.new(name: 'TestSysVInteger', permissions: 0660) @integer.reset end @@ -15,7 +15,7 @@ def teardown include TestSimpleInteger::IntegerTestCases def test_memory_is_shared - integer_2 = CLASS.new(name: 'TestSysVInteger', permissions: 0660) + integer_2 = KLASS.new(name: 'TestSysVInteger', permissions: 0660) integer_2.value = 100 assert_equal 100, @integer.value @integer.value = 200 @@ -26,13 +26,13 @@ def test_memory_is_shared def test_memory_not_reset_when_at_least_one_worker_using_it @integer.value = 109 - integer_2 = CLASS.new(name: 'TestSysVInteger', permissions: 0660) + integer_2 = KLASS.new(name: 'TestSysVInteger', permissions: 0660) assert_equal @integer.value, integer_2.value reader, writer = IO.pipe pid = fork do reader.close - integer_3 = CLASS.new(name: 'TestSysVInteger', permissions: 0660) + integer_3 = KLASS.new(name: 'TestSysVInteger', permissions: 0660) assert_equal 109, integer_3.value integer_3.value = 110 writer.puts "Done" @@ -47,11 +47,11 @@ def test_memory_not_reset_when_at_least_one_worker_using_it def test_memory_reset_when_no_workers_using_it fork do - integer = CLASS.new(name: 'TestSysVInteger_2', permissions: 0660) + integer = KLASS.new(name: 'TestSysVInteger_2', permissions: 0660) integer.value = 109 end Process.waitall - @integer = CLASS.new(name: 'TestSysVInteger_2', permissions: 0660) + @integer = KLASS.new(name: 'TestSysVInteger_2', permissions: 0660) assert_equal 0, @integer.value end end diff --git a/test/sysv_sliding_window_test.rb b/test/sysv_sliding_window_test.rb index d18f600f..aca43679 100644 --- a/test/sysv_sliding_window_test.rb +++ b/test/sysv_sliding_window_test.rb @@ -1,10 +1,10 @@ require 'test_helper' class TestSysVSlidingWindow < MiniTest::Unit::TestCase - CLASS = ::Semian::SysV::SlidingWindow + KLASS = ::Semian::SysV::SlidingWindow def setup - @sliding_window = CLASS.new(max_size: 6, + @sliding_window = KLASS.new(max_size: 6, name: 'TestSysVSlidingWindow', permissions: 0660) @sliding_window.clear @@ -25,7 +25,7 @@ def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it reader, writer = IO.pipe pid = fork do reader.close - sliding_window_2 = CLASS.new(max_size: 6, + sliding_window_2 = KLASS.new(max_size: 6, name: 'TestSysVSlidingWindow', permissions: 0660) sliding_window_2.synchronize do @@ -37,6 +37,7 @@ def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it reader.gets Process.kill(9, pid) + Process.waitall Timeout.timeout(1) do # assure dont hang @sliding_window << 100 @@ -46,7 +47,7 @@ def test_forcefully_killing_worker_holding_on_to_semaphore_releases_it def test_sliding_window_memory_is_actually_shared assert_equal 0, @sliding_window.size - sliding_window_2 = CLASS.new(max_size: 6, + sliding_window_2 = KLASS.new(max_size: 6, name: 'TestSysVSlidingWindow', permissions: 0660) assert_equal 0, sliding_window_2.size @@ -66,7 +67,7 @@ def test_sliding_window_memory_is_actually_shared def test_restarting_worker_should_not_reset_queue @sliding_window << 10 << 20 << 30 - sliding_window_2 = CLASS.new(max_size: 6, + sliding_window_2 = KLASS.new(max_size: 6, name: 'TestSysVSlidingWindow', permissions: 0660) assert_sliding_window(sliding_window_2, [10, 20, 30], 6) @@ -74,7 +75,7 @@ def test_restarting_worker_should_not_reset_queue assert_sliding_window(sliding_window_2, [10, 20], 6) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) - sliding_window_3 = CLASS.new(max_size: 6, + sliding_window_3 = KLASS.new(max_size: 6, name: 'TestSysVSlidingWindow', permissions: 0660) assert_sliding_window(sliding_window_3, [10, 20], 6) @@ -87,7 +88,7 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down # Test explicit resizing, and resizing through making new memory associations # B resize down through init - sliding_window_2 = CLASS.new(max_size: 4, + sliding_window_2 = KLASS.new(max_size: 4, name: 'TestSysVSlidingWindow', permissions: 0660) sliding_window_2 << 80 << 90 << 100 << 110 << 120 @@ -100,7 +101,7 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down assert_sliding_window(@sliding_window, [110, 120], 2) # B resize up through init - sliding_window_2 = CLASS.new(max_size: 4, + sliding_window_2 = KLASS.new(max_size: 4, name: 'TestSysVSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) @@ -113,7 +114,7 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down assert_sliding_window(@sliding_window, [110, 120, 130], 6) # B resize down through init - sliding_window_2 = CLASS.new(max_size: 2, + sliding_window_2 = KLASS.new(max_size: 2, name: 'TestSysVSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) @@ -125,7 +126,7 @@ def test_other_workers_automatically_switching_to_new_memory_resizing_up_or_down assert_sliding_window(@sliding_window, [120, 130], 4) # B resize up through init - sliding_window_2 = CLASS.new(max_size: 6, + sliding_window_2 = KLASS.new(max_size: 6, name: 'TestSysVSlidingWindow', permissions: 0660) assert_sliding_windows_in_sync(@sliding_window, sliding_window_2) diff --git a/test/sysv_state_test.rb b/test/sysv_state_test.rb index 768602a2..11219626 100644 --- a/test/sysv_state_test.rb +++ b/test/sysv_state_test.rb @@ -1,10 +1,10 @@ require 'test_helper' class TestSysVState < MiniTest::Unit::TestCase - CLASS = ::Semian::SysV::State + KLASS = ::Semian::SysV::State def setup - @state = CLASS.new(name: 'TestSysVState', + @state = KLASS.new(name: 'TestSysVState', permissions: 0660) @state.reset end @@ -19,7 +19,7 @@ def test_memory_is_shared assert_equal :closed, @state.value @state.open - state_2 = CLASS.new(name: 'TestSysVState', + state_2 = KLASS.new(name: 'TestSysVState', permissions: 0660) assert_equal :open, state_2.value assert state_2.open? From 2bbd717e07c83665bc4174ebf0c83d54eab5af7e Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Thu, 26 Nov 2015 19:55:29 +0000 Subject: [PATCH 22/23] Small changes to code --- ext/semian/semian.h | 1 - ext/semian/semian_integer.c | 11 --- ext/semian/semian_shared_memory_object.c | 104 ++++++++++------------- ext/semian/semian_sliding_window.c | 7 +- lib/semian.rb | 10 +-- lib/semian/sysv_shared_memory.rb | 23 ++--- 6 files changed, 65 insertions(+), 91 deletions(-) diff --git a/ext/semian/semian.h b/ext/semian/semian.h index 30d49252..d6eca09e 100644 --- a/ext/semian/semian.h +++ b/ext/semian/semian.h @@ -44,6 +44,5 @@ generate_key(const char *name); void raise_semian_syscall_error(const char *syscall, int error_num); - void set_semaphore_permissions(int sem_id, int permissions); diff --git a/ext/semian/semian_integer.c b/ext/semian/semian_integer.c index 7cf404fd..f3b2514d 100644 --- a/ext/semian/semian_integer.c +++ b/ext/semian/semian_integer.c @@ -36,11 +36,8 @@ semian_integer_get_value(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - - // check shared memory for NULL if (0 == ptr->shm_address) return Qnil; - int value = ((semian_int *)(ptr->shm_address))->value; return INT2NUM(value); } @@ -50,15 +47,11 @@ semian_integer_set_value(VALUE self, VALUE num) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - if (0 == ptr->shm_address) return Qnil; - if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT) return Qnil; - ((semian_int *)(ptr->shm_address))->value = NUM2INT(num); - return num; } @@ -78,15 +71,11 @@ semian_integer_increment(int argc, VALUE *argv, VALUE self) semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - if (0 == ptr->shm_address) return Qnil; - if (TYPE(num) != T_FIXNUM && TYPE(num) != T_FLOAT) return Qnil; - ((semian_int *)(ptr->shm_address))->value += NUM2INT(num); - return self; } diff --git a/ext/semian/semian_shared_memory_object.c b/ext/semian/semian_shared_memory_object.c index 65f20d93..379d961b 100644 --- a/ext/semian/semian_shared_memory_object.c +++ b/ext/semian/semian_shared_memory_object.c @@ -1,10 +1,11 @@ #include "semian_shared_memory_object.h" -const int kSHMSemaphoreCount = 1; // # semaphores to be acquired +const int kSHMSemaphoreCount = 1; // semaphores to be acquired const int kSHMTicketMax = 1; const int kSHMInitializeWaitTimeout = 5; /* seconds */ const int kSHMIndexTicketLock = 0; const int kSHMInternalTimeout = 5; /* seconds */ +const int kSHMRestoreLockStateRetryCount = 5; // perform semtimedop 5 times max static struct sembuf decrement; // = { kSHMIndexTicketLock, -1, SEM_UNDO}; static struct sembuf increment; // = { kSHMIndexTicketLock, 1, SEM_UNDO}; @@ -16,9 +17,6 @@ static void semian_shm_object_mark(void *ptr); static void semian_shm_object_free(void *ptr); static size_t semian_shm_object_memsize(const void *ptr); -static void *semian_shm_object_lock_without_gvl(void *v_ptr); -static void *semian_shm_object_unlock_without_gvl(void *v_ptr); - const rb_data_type_t semian_shm_object_type = { "semian_shm_object", @@ -69,23 +67,6 @@ semian_shm_object_replace_alloc(VALUE klass, VALUE target) return target; } -VALUE -semian_shm_object_sizeof(VALUE klass, VALUE type) -{ - if (TYPE(type) != T_SYMBOL){ - rb_raise(rb_eTypeError, "id must be a symbol or string"); - } - - if (rb_intern("int") == SYM2ID(type)) - return INT2NUM(sizeof(int)); - else if (rb_intern("long") == SYM2ID(type)) - return INT2NUM(sizeof(long)); - // Can definitely add more - else - rb_raise(rb_eTypeError, "%s is not a valid C type", rb_id2name(SYM2ID(type))); -} - - VALUE semian_shm_object_acquire(VALUE self, VALUE name, VALUE data_layout, VALUE permissions) { @@ -100,8 +81,19 @@ semian_shm_object_acquire(VALUE self, VALUE name, VALUE data_layout, VALUE permi rb_raise(rb_eTypeError, "expected integer for permissions"); int byte_size = 0; - for (int i = 0; i < RARRAY_LEN(data_layout); ++i) - byte_size += NUM2INT(semian_shm_object_sizeof(rb_cObject, RARRAY_PTR(data_layout)[i])); + for (int i = 0; i < RARRAY_LEN(data_layout); ++i) { + VALUE type_symbol = RARRAY_PTR(data_layout)[i]; + if (TYPE(type_symbol) != T_SYMBOL) + rb_raise(rb_eTypeError, "id must be a symbol or string"); + + if (rb_intern("int") == SYM2ID(type_symbol)) + byte_size += sizeof(int); + else if (rb_intern("long") == SYM2ID(type_symbol)) + byte_size += sizeof(long); + // Can definitely add more + else + rb_raise(rb_eTypeError, "%s is not a valid C type", rb_id2name(SYM2ID(type_symbol))); + } if (byte_size <= 0) rb_raise(rb_eArgError, "total size must be larger than 0"); @@ -119,17 +111,18 @@ semian_shm_object_acquire(VALUE self, VALUE name, VALUE data_layout, VALUE permi ptr->shm_address = 0; // address defaults to NULL ptr->lock_count = 0; // Emulates recursive mutex, 0->1 locks, 1->0 unlocks, rest noops ptr->permissions = FIX2LONG(permissions); + ptr->initialize_memory = NULL; - // Will throw NotImplementedError if not defined in concrete subclasses - // Implement bind_initialize_memory_callback as a function with type - // static VALUE bind_initialize_memory_callback(VALUE self) - // that a callback to ptr->initialize_memory, called when memory needs to be initialized + // Concrete classes must implement this in a subclass in C to bind a callback function of type + // void (*initialize_memory)(size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size); + // to location ptr->initialize_memory, where ptr is a semian_shm_object* + // It is called when memory needs to be initialized or resized, possibly using previous memory rb_funcall(self, rb_intern("bind_initialize_memory_callback"), 0); + if (NULL == ptr->initialize_memory) + rb_raise(rb_eNotImpError, "callback was not bound to ptr->initialize_memory"); semian_shm_object_acquire_semaphore(self); semian_shm_object_synchronize(self); - - return Qtrue; } @@ -180,12 +173,9 @@ semian_shm_object_acquire_semaphore (VALUE self) semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - key_t key = ptr->key; - int semid = create_semaphore_and_initialize_and_set_permissions(key, ptr->permissions); - if (-1 == semid) { + if (-1 == (ptr->semid = create_semaphore_and_initialize_and_set_permissions(ptr->key, ptr->permissions))) { raise_semian_syscall_error("semget()", errno); } - ptr->semid = semid; return self; } @@ -241,7 +231,7 @@ semian_shm_object_unlock_without_gvl(void *v_ptr) if (-1 == ptr->semid){ rb_raise(eInternal, "semid not set, errno %d: (%s)", errno, strerror(errno)); } - if (1 != ptr->lock_count || -1 != semop(ptr->semid, &increment, 1)) { + if (1 != ptr->lock_count || -1 != semop(ptr->semid, &increment, 1)) { // No need for semtimedop ptr->lock_count -= 1; } else { rb_raise(eInternal, "error unlocking semaphore, %d (%s)", errno, strerror(errno)); @@ -272,16 +262,15 @@ semian_shm_object_synchronize_restore_lock_status(VALUE v_status) { lock_status *status = (lock_status *) v_status; int tries = 0; - int allowed_tries = 5; - while (++tries < allowed_tries && status->ptr->lock_count > status->pre_block_lock_count_state) + while (++tries < kSHMRestoreLockStateRetryCount && status->ptr->lock_count > status->pre_block_lock_count_state) return (VALUE) WITHOUT_GVL(semian_shm_object_unlock_without_gvl, (void *)(status->ptr), RUBY_UBF_IO, NULL); - if (tries >= allowed_tries) - rb_raise(eSyscall, "Failed to restore lock status after %d tries", allowed_tries); + if (tries >= kSHMRestoreLockStateRetryCount) + rb_raise(eSyscall, "Failed to restore lock status after %d tries", kSHMRestoreLockStateRetryCount); tries = 0; - while (++tries < allowed_tries && status->ptr->lock_count < status->pre_block_lock_count_state) + while (++tries < kSHMRestoreLockStateRetryCount && status->ptr->lock_count < status->pre_block_lock_count_state) return (VALUE) WITHOUT_GVL(semian_shm_object_lock_without_gvl, (void *)(status->ptr), RUBY_UBF_IO, NULL); - if (tries >= allowed_tries) - rb_raise(eSyscall, "Failed to restore lock status after %d tries", allowed_tries); + if (tries >= kSHMRestoreLockStateRetryCount) + rb_raise(eSyscall, "Failed to restore lock status after %d tries", kSHMRestoreLockStateRetryCount); return Qnil; } @@ -290,8 +279,7 @@ semian_shm_object_synchronize(VALUE self) { // receives a block semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); lock_status status = { ptr->lock_count, ptr }; - if (!(WITHOUT_GVL(semian_shm_object_lock_without_gvl, (void *)ptr, RUBY_UBF_IO, NULL))) - return Qnil; + WITHOUT_GVL(semian_shm_object_lock_without_gvl, (void *)ptr, RUBY_UBF_IO, NULL); return rb_ensure(semian_shm_object_synchronize_with_block, self, semian_shm_object_synchronize_restore_lock_status, (VALUE)&status); } @@ -312,29 +300,28 @@ semian_shm_object_synchronize_memory_and_size(VALUE self, VALUE is_master_obj) { TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); struct shmid_ds shm_info = { }; - const int SHMMIN = 1; + const int SHMMIN = 1; // minimum size of shared memory on linux key_t key = ptr->key; - int is_master = RTEST(is_master_obj); + int is_master = RTEST(is_master_obj); // Controls whether synchronization is master or slave (both fast-forward, only master resizes/initializes) is_master |= (-1 == ptr->shmid) && (0 == ptr->shm_address); int shmid_out_of_sync = 0; shmid_out_of_sync |= (-1 == ptr->shmid) && (0 == ptr->shm_address); // If not attached at all if ((-1 != ptr->shmid) && (-1 != shmctl(ptr->shmid, IPC_STAT, &shm_info))) { - // If current attached memory is marked for deletion - shmid_out_of_sync |= shm_info.shm_perm.mode & SHM_DEST && - ptr->shmid != shmget(key, SHMMIN, IPC_CREAT | ptr->permissions); + shmid_out_of_sync |= shm_info.shm_perm.mode & SHM_DEST && // If current attached memory is marked for deletion + ptr->shmid != shmget(key, SHMMIN, IPC_CREAT | ptr->permissions); // If shmid not in sync } size_t requested_byte_size = ptr->byte_size; int first_sync = (-1 == ptr->shmid) && (shmid_out_of_sync); - if (shmid_out_of_sync) { + if (shmid_out_of_sync) { // We need to fast-forward to the current state and memory attachment semian_shm_object_cleanup_memory(self); if ((-1 == (ptr->shmid = shmget(key, SHMMIN, ptr->permissions)))) { if ((-1 == (ptr->shmid = shmget(key, ptr->byte_size, IPC_CREAT | IPC_EXCL | ptr->permissions)))) { rb_raise(eSyscall, "shmget failed to create or attach current memory with key %d shmid %d errno %d (%s)", key, ptr->shmid, errno, strerror(errno)); - } + } // If we can neither create a new memory block nor get the current one with a key, something's wrong } if ((void *)-1 == (ptr->shm_address = shmat(ptr->shmid, NULL, 0))) { ptr->shm_address = NULL; @@ -351,14 +338,13 @@ semian_shm_object_synchronize_memory_and_size(VALUE self, VALUE is_master_obj) { if (is_master) { if (ptr->byte_size == requested_byte_size && first_sync && 1 == old_mem_attach_count) { - ptr->initialize_memory(ptr->byte_size, ptr->shm_address, NULL, 0); - } else { + ptr->initialize_memory(ptr->byte_size, ptr->shm_address, NULL, 0); // We clear the memory if worker is first to attach + } else if (ptr->byte_size != requested_byte_size) { void *old_shm_address = ptr->shm_address; size_t old_byte_size = ptr->byte_size; - unsigned char old_memory_content[old_byte_size]; - + unsigned char old_memory_content[old_byte_size]; // It is unsafe to use malloc here to store a copy of the memory memcpy(old_memory_content, old_shm_address, old_byte_size); - semian_shm_object_cleanup_memory(self); + semian_shm_object_cleanup_memory(self); // This may fail if (-1 == (ptr->shmid = shmget(key, requested_byte_size, IPC_CREAT | IPC_EXCL | ptr->permissions))) { rb_raise(eSyscall, "shmget failed to create new resized memory with key %d shmid %d errno %d (%s)", key, ptr->shmid, errno, strerror(errno)); @@ -399,8 +385,7 @@ semian_shm_object_cleanup_memory(VALUE self) semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); lock_status status = { ptr->lock_count, ptr }; - if (!(WITHOUT_GVL(semian_shm_object_lock_without_gvl, (void *)ptr, RUBY_UBF_IO, NULL))) - return Qnil; + WITHOUT_GVL(semian_shm_object_lock_without_gvl, (void *)ptr, RUBY_UBF_IO, NULL); return rb_ensure(semian_shm_object_cleanup_memory_inner, self, semian_shm_object_synchronize_restore_lock_status, (VALUE)&status); } @@ -409,6 +394,8 @@ semian_shm_object_semid(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); + if (-1 == ptr->semid) + return -1; semian_shm_object_synchronize(self); return INT2NUM(ptr->semid); } @@ -417,7 +404,6 @@ semian_shm_object_shmid(VALUE self) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); - semian_shm_object_synchronize(self); return INT2NUM(ptr->shmid); } @@ -432,7 +418,7 @@ Init_semian_shm_object (void) { rb_define_method(cSysVSharedMemory, "synchronize", semian_shm_object_synchronize, 0); rb_define_method(cSysVSharedMemory, "semid", semian_shm_object_semid, 0); - rb_define_method(cSysVSharedMemory, "shmid", semian_shm_object_shmid, 0); + define_method_with_synchronize(cSysVSharedMemory, "shmid", semian_shm_object_shmid, 0); rb_define_singleton_method(cSysVSharedMemory, "replace_alloc", semian_shm_object_replace_alloc, 1); diff --git a/ext/semian/semian_sliding_window.c b/ext/semian/semian_sliding_window.c index 33350bfc..60637927 100644 --- a/ext/semian/semian_sliding_window.c +++ b/ext/semian/semian_sliding_window.c @@ -25,17 +25,14 @@ semian_sliding_window_initialize_memory (size_t byte_size, void *dest, void *pre semian_sliding_window *ptr = dest; semian_sliding_window *old = prev_data; - // Logic to initialize: initialize to 0 only if you're the first to attach (else body) - // Otherwise, copy previous data into new data if (prev_data) { - // transfer data over ptr->max_window_size = (byte_size - 2 * sizeof(int)) / sizeof(long); ptr->window_size = fmin(ptr->max_window_size, old->window_size); // Copy the most recent ptr->shm_address->window_size numbers to new memory memcpy(&(ptr->window), - ((long *)(&(old->window[0]))) + old->window_size - ptr->window_size, - ptr->window_size * sizeof(long)); + ((long *)(&(old->window[0]))) + old->window_size - ptr->window_size, + ptr->window_size * sizeof(long)); } else { semian_sliding_window *data = dest; data->max_window_size = (byte_size - 2 * sizeof(int)) / sizeof(long); diff --git a/lib/semian.rb b/lib/semian.rb index aa5efde3..a8865131 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -125,7 +125,7 @@ def register(name, tickets:, permissions: 0660, timeout: 0, error_threshold:, er error_timeout: error_timeout, exceptions: Array(exceptions) + [::Semian::BaseError], permissions: permissions, - implementation: ::Semian::SysV, + implementation: Semian.semaphores_enabled? ? ::Semian::SysV : ::Semian::Simple, ) resource = Resource.new(name, tickets: tickets, permissions: permissions, timeout: timeout) resources[name] = ProtectedResource.new(resource, circuit_breaker) @@ -160,11 +160,11 @@ def resources require 'semian/simple_sliding_window' require 'semian/simple_integer' require 'semian/simple_state' -require 'semian/sysv_shared_memory' -require 'semian/sysv_sliding_window' -require 'semian/sysv_integer' -require 'semian/sysv_state' if Semian.semaphores_enabled? + require 'semian/sysv_shared_memory' + require 'semian/sysv_sliding_window' + require 'semian/sysv_integer' + require 'semian/sysv_state' require 'semian/semian' else Semian::MAX_TICKETS = 0 diff --git a/lib/semian/sysv_shared_memory.rb b/lib/semian/sysv_shared_memory.rb index 58abe7ee..dbbbbae1 100644 --- a/lib/semian/sysv_shared_memory.rb +++ b/lib/semian/sysv_shared_memory.rb @@ -1,9 +1,12 @@ module Semian module SysVSharedMemory #:nodoc: - def self.included(base) - def base.do_with_sync(*names) + module SysVSynchronizeHelper + # This is a helper method for wrapping a method in :synchronize + # Its usage is to be called from C: where rb_define_method() is originally + # used, define_method_with_synchronize() is used instead, which calls this + def do_with_sync(*names) names.each do |name| - new_name = "#{name}_inner".freeze.to_sym + new_name = "#{name}_inner" alias_method new_name, name private new_name define_method(name) do |*args, &block| @@ -15,6 +18,12 @@ def base.do_with_sync(*names) end end + extend SysVSynchronizeHelper + + def self.included(base) + base.extend(SysVSynchronizeHelper) + end + def semid -1 end @@ -34,16 +43,10 @@ def destroy private def acquire_memory_object(*) - # Concrete classes must call this method before accessing shared memory - # If SysV is enabled, a C method overrides this stub and returns true if acquiring succeeds - false + raise NotImplementedError end def bind_initialize_memory_callback - # Concrete classes must implement this in a subclass in C to bind a callback function of type - # void (*initialize_memory)(size_t byte_size, void *dest, void *prev_data, size_t prev_data_byte_size); - # to location ptr->initialize_memory, where ptr is a semian_shm_object* - # It is called when memory needs to be initialized or resized, possibly using previous memory raise NotImplementedError end end From d6ce4aac728931584cd54487da0bf0fcb777c350 Mon Sep 17 00:00:00 2001 From: Kye Wei Date: Thu, 3 Dec 2015 15:57:28 +0000 Subject: [PATCH 23/23] Abstract away passing in symbols for data_layout, remove unless --- ext/semian/semian_integer.c | 7 +++++++ ext/semian/semian_shared_memory_object.c | 25 +++++------------------- ext/semian/semian_sliding_window.c | 7 +++++++ lib/semian/sysv_integer.rb | 3 +-- lib/semian/sysv_sliding_window.rb | 3 +-- 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/ext/semian/semian_integer.c b/ext/semian/semian_integer.c index f3b2514d..d9d0ce9b 100644 --- a/ext/semian/semian_integer.c +++ b/ext/semian/semian_integer.c @@ -79,6 +79,12 @@ semian_integer_increment(int argc, VALUE *argv, VALUE self) return self; } +static VALUE +semian_integer_calculate_byte_size(VALUE klass) +{ + return SIZET2NUM(sizeof(int)); +} + void Init_semian_integer (void) { @@ -95,4 +101,5 @@ Init_semian_integer (void) define_method_with_synchronize(cInteger, "value=", semian_integer_set_value, 1); define_method_with_synchronize(cInteger, "reset", semian_integer_reset, 0); define_method_with_synchronize(cInteger, "increment", semian_integer_increment, -1); + rb_define_method(cInteger, "calculate_byte_size", semian_integer_calculate_byte_size, 0); } diff --git a/ext/semian/semian_shared_memory_object.c b/ext/semian/semian_shared_memory_object.c index 379d961b..f67b8ab1 100644 --- a/ext/semian/semian_shared_memory_object.c +++ b/ext/semian/semian_shared_memory_object.c @@ -68,34 +68,19 @@ semian_shm_object_replace_alloc(VALUE klass, VALUE target) } VALUE -semian_shm_object_acquire(VALUE self, VALUE name, VALUE data_layout, VALUE permissions) +semian_shm_object_acquire(VALUE self, VALUE name, VALUE byte_size, VALUE permissions) { semian_shm_object *ptr; TypedData_Get_Struct(self, semian_shm_object, &semian_shm_object_type, ptr); if (TYPE(name) != T_SYMBOL && TYPE(name) != T_STRING) rb_raise(rb_eTypeError, "id must be a symbol or string"); - if (TYPE(data_layout) != T_ARRAY) - rb_raise(rb_eTypeError, "expected array for data_layout"); + if (TYPE(byte_size) != T_FIXNUM) + rb_raise(rb_eTypeError, "expected integer for byte_size"); if (TYPE(permissions) != T_FIXNUM) rb_raise(rb_eTypeError, "expected integer for permissions"); - int byte_size = 0; - for (int i = 0; i < RARRAY_LEN(data_layout); ++i) { - VALUE type_symbol = RARRAY_PTR(data_layout)[i]; - if (TYPE(type_symbol) != T_SYMBOL) - rb_raise(rb_eTypeError, "id must be a symbol or string"); - - if (rb_intern("int") == SYM2ID(type_symbol)) - byte_size += sizeof(int); - else if (rb_intern("long") == SYM2ID(type_symbol)) - byte_size += sizeof(long); - // Can definitely add more - else - rb_raise(rb_eTypeError, "%s is not a valid C type", rb_id2name(SYM2ID(type_symbol))); - } - - if (byte_size <= 0) + if (NUM2SIZET(byte_size) <= 0) rb_raise(rb_eArgError, "total size must be larger than 0"); const char *id_str = NULL; @@ -105,7 +90,7 @@ semian_shm_object_acquire(VALUE self, VALUE name, VALUE data_layout, VALUE permi id_str = RSTRING_PTR(name); } ptr->key = generate_key(id_str); - ptr->byte_size = byte_size; // byte_size >=1 or error would have been raised earlier + ptr->byte_size = NUM2SIZET(byte_size); // byte_size >=1 or error would have been raised earlier ptr->semid = -1; // id's default to -1 ptr->shmid = -1; ptr->shm_address = 0; // address defaults to NULL diff --git a/ext/semian/semian_sliding_window.c b/ext/semian/semian_sliding_window.c index 60637927..2162d86e 100644 --- a/ext/semian/semian_sliding_window.c +++ b/ext/semian/semian_sliding_window.c @@ -225,6 +225,12 @@ semian_sliding_window_resize_to(VALUE self, VALUE size) { return self; } +static VALUE +semian_sliding_window_calculate_byte_size(VALUE klass, VALUE size) +{ + return INT2NUM(2 * sizeof(int) + NUM2INT(size) * sizeof(long)); +} + void Init_semian_sliding_window (void) { @@ -247,4 +253,5 @@ Init_semian_sliding_window (void) define_method_with_synchronize(cSlidingWindow, "clear", semian_sliding_window_clear, 0); define_method_with_synchronize(cSlidingWindow, "first", semian_sliding_window_first, 0); define_method_with_synchronize(cSlidingWindow, "last", semian_sliding_window_last, 0); + rb_define_method(cSlidingWindow, "calculate_byte_size", semian_sliding_window_calculate_byte_size, 1); } diff --git a/lib/semian/sysv_integer.rb b/lib/semian/sysv_integer.rb index 72554cc7..15a4b7eb 100644 --- a/lib/semian/sysv_integer.rb +++ b/lib/semian/sysv_integer.rb @@ -4,8 +4,7 @@ class Integer < Semian::Simple::Integer #:nodoc: include SysVSharedMemory def initialize(name:, permissions:) - data_layout = [:int] - super() unless acquire_memory_object(name, data_layout, permissions) + acquire_memory_object(name, calculate_byte_size, permissions) end end end diff --git a/lib/semian/sysv_sliding_window.rb b/lib/semian/sysv_sliding_window.rb index 4c54c022..b5cc1187 100644 --- a/lib/semian/sysv_sliding_window.rb +++ b/lib/semian/sysv_sliding_window.rb @@ -4,8 +4,7 @@ class SlidingWindow < Semian::Simple::SlidingWindow #:nodoc: include SysVSharedMemory def initialize(max_size:, name:, permissions:) - data_layout = [:int, :int] + [:long] * max_size - super(max_size: max_size) unless acquire_memory_object(name, data_layout, permissions) + acquire_memory_object(name, calculate_byte_size(max_size), permissions) end end end