/* * The MIT License (MIT) * * Copyright © 2015-2016 Franklin "Snaipe" Mathieu * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ #define CRITERION_LOGGING_COLORS #include "config.h" #include #include #include #include NN_H #include "criterion/internal/test.h" #include "criterion/options.h" #include "criterion/internal/ordered-set.h" #include "criterion/internal/preprocess.h" #include "criterion/redirect.h" #include "protocol/protocol.h" #include "protocol/connect.h" #include "protocol/messages.h" #include "compat/alloc.h" #include "compat/time.h" #include "compat/posix.h" #include "compat/processor.h" #include "compat/kill.h" #include "csptr/smalloc.h" #include "string/diff.h" #include "string/extglobmatch.h" #include "string/i18n.h" #include "io/event.h" #include "io/output.h" #include "log/logging.h" #include "abort.h" #include "client.h" #include "common.h" #include "err.h" #include "report.h" #include "runner.h" #include "runner_coroutine.h" #include "stats.h" #ifdef __GNUC__ # include #else # define ENABLE_VALGRIND_ERRORS # define RUNNING_ON_VALGRIND 0 #endif int cri_is_runner; typedef const char *const msg_t; #ifdef ENABLE_NLS static msg_t msg_valgrind_jobs = N_("%1$sWarning! Criterion has detected " "that it is running under valgrind, but the number of jobs have been " "explicitely set. Reports might appear confusing!%2$s\n"); #else static msg_t msg_valgrind_jobs = "%sWarning! Criterion has detected " "that it is running under valgrind, but the number of jobs have been " "explicitely set. Reports might appear confusing!%s\n"; #endif static int cmp_suite(void *a, void *b) { struct criterion_suite *s1 = a, *s2 = b; return strcmp(s1->name, s2->name); } static int cmp_test(void *a, void *b) { struct criterion_test *s1 = a, *s2 = b; return strcmp(s1->name, s2->name); } static void dtor_suite_set(void *ptr, CR_UNUSED void *meta) { struct criterion_suite_set *s = ptr; sfree(s->tests); } static void dtor_test_set(void *ptr, CR_UNUSED void *meta) { struct criterion_test_set *t = ptr; sfree(t->suites); } CR_API void criterion_register_test(struct criterion_test_set *set, struct criterion_test *test) { struct criterion_suite_set css = { .suite = { .name = test->category }, }; struct criterion_suite_set *s = insert_ordered_set(set->suites, &css, sizeof (css)); if (!s->tests) s->tests = new_ordered_set(cmp_test, NULL); insert_ordered_set(s->tests, test, sizeof (*test)); ++set->tests; } struct criterion_test_set *criterion_init(void) { struct criterion_ordered_set *suites = new_ordered_set(cmp_suite, dtor_suite_set); struct cri_section *sections = NULL; if (!cri_sections_getaddr("cr_sts", §ions)) { for (struct cri_section *s = sections; s->addr; ++s) { void *start = s->addr; void *end = (char *) start + s->length; FOREACH_SUITE_SEC(s, start, end) { if (!*s || !*(*s)->name) continue; struct criterion_suite_set css = { .suite = **s, }; insert_ordered_set(suites, &css, sizeof (css)); } } } free(sections); struct criterion_test_set *set = smalloc( .size = sizeof (struct criterion_test_set), .dtor = dtor_test_set); *set = (struct criterion_test_set) { suites, 0, }; sections = NULL; if (!cri_sections_getaddr("cr_tst", §ions)) { for (struct cri_section *s = sections; s->addr; ++s) { void *start = s->addr; void *end = (char *) start + s->length; FOREACH_TEST_SEC(test, start, end) { if (!*test) continue; if (!*(*test)->category || !*(*test)->name) continue; criterion_register_test(set, *test); } } } free(sections); return set; } #define push_event(...) \ do { \ stat_push_event(ctx->stats, \ ctx->suite_stats, \ ctx->test_stats, \ &(struct event) { \ .kind = CR_VA_HEAD(__VA_ARGS__), \ CR_VA_TAIL(__VA_ARGS__) \ }); \ report(CR_VA_HEAD(__VA_ARGS__), ctx->test_stats); \ } while (0) void disable_unmatching(struct criterion_test_set *set) { if (!compile_pattern(criterion_options.pattern)) exit(3); FOREACH_SET(struct criterion_suite_set *s, set->suites) { if ((s->suite.data && s->suite.data->disabled) || !s->tests) continue; FOREACH_SET(struct criterion_test *test, s->tests) { int ret = match(test->data->identifier_); if (ret == 0) test->data->disabled = true; } } free_pattern(); } CR_API struct criterion_test_set *criterion_initialize(void) { /* make sure we don't re-enter from a test worker. See #247. */ if (getenv("BXFI_MAP")) { cr_panic("Re-entering criterion from a test worker. This is a " "catastrophic bug, please report it on the issue tracker.\n" "Bailing out to avoid fork-bombing the system."); } init_i18n(); cri_diff_init(); #ifndef ENABLE_VALGRIND_ERRORS VALGRIND_DISABLE_ERROR_REPORTING; #endif if (RUNNING_ON_VALGRIND) { criterion_options.no_early_exit = 1; criterion_options.jobs = 1; } criterion_register_output_provider("tap", tap_report); criterion_register_output_provider("xml", xml_report); criterion_register_output_provider("json", json_report); setup_parent_job(); return criterion_init(); } CR_API void criterion_finalize(struct criterion_test_set *set) { sfree(set); #ifndef ENABLE_VALGRIND_ERRORS VALGRIND_ENABLE_ERROR_REPORTING; #endif cri_diff_fini(); criterion_free_output(); } static struct client_ctx *spawn_next_client(struct server_ctx *sctx, ccrContext *ctx) { struct client_ctx new_ctx; bxf_instance *instance = cri_run_next_test(NULL, NULL, NULL, &new_ctx, ctx); if (!instance) return NULL; return add_client_from_worker(sctx, &new_ctx, instance); } static void run_tests_async(struct criterion_test_set *set, struct criterion_global_stats *stats, const char *url, int socket) { ccrContext ctx = 0; size_t nb_workers = DEF(criterion_options.jobs, get_processor_count()); size_t active_workers = 0; int has_msg = 0; struct server_ctx sctx; init_server_context(&sctx, stats); sctx.socket = socket; /* initialization of coroutine */ cri_run_next_test(set, stats, url, NULL, &ctx); for (size_t i = 0; i < nb_workers; ++i) { struct client_ctx *cctx = spawn_next_client(&sctx, &ctx); if (!cctx) break; ++active_workers; } if (!active_workers && !criterion_options.wait_for_clients) goto cleanup; criterion_protocol_msg msg = criterion_protocol_msg_init_zero; while ((has_msg = read_message(socket, &msg)) == 1) { struct client_ctx *cctx = process_client_message(&sctx, &msg); /* drop invalid messages */ if (!cctx) continue; if (cctx->dead) { if ((cctx->tstats->test_status == CR_STATUS_FAILED) && criterion_options.fail_fast) cr_terminate(cctx->gstats); if (cctx->kind == WORKER) { remove_client_by_pid(&sctx, cctx->instance->pid); cctx = spawn_next_client(&sctx, &ctx); if (cctx == NULL) --active_workers; } } if (!active_workers && !criterion_options.wait_for_clients) break; free_message(&msg); } cleanup: if (has_msg) free_message(&msg); destroy_server_context(&sctx); ccrAbort(ctx); } static int criterion_run_all_tests_impl(struct criterion_test_set *set) { cri_report_init(); if (RUNNING_ON_VALGRIND) { if (criterion_options.jobs != 1) criterion_pimportant(CRITERION_PREFIX_DASHES, _(msg_valgrind_jobs), CR_FG_BOLD, CR_RESET); } char url[sizeof ("ipc://" NN_SOCKET_PATH "criterion_.sock") + 21]; snprintf(url, sizeof (url), "ipc://" NN_SOCKET_PATH "criterion_%llu.sock", get_process_id()); int sock = cri_proto_bind(url); if (sock < 0) cr_panic("Could not initialize the message server: %s.", strerror(errno)); g_client_socket = cri_proto_connect(url); if (g_client_socket < 0) cr_panic("Could not initialize the message client: %s.", strerror(errno)); cri_alloc_init(); report(PRE_ALL, set); log(pre_all, set); struct criterion_global_stats *stats = stats_init(); run_tests_async(set, stats, url, sock); report(POST_ALL, stats); if (criterion_options.logging_threshold == CRITERION_LOG_LEVEL_QUIET) { cri_restore_outputs(); } process_all_output(stats); log(post_all, stats); cri_alloc_term(); cri_report_term(); cri_proto_close(g_client_socket); cri_proto_close(sock); int ok = stats->tests_failed == 0 && stats->errors == 0; if (!criterion_options.ignore_warnings) ok = ok && stats->warnings == 0; sfree(stats); return ok; } CR_API int criterion_run_all_tests(struct criterion_test_set *set) { /* make sure we don't re-enter from a test worker. See #247. */ if (getenv("BXFI_MAP")) { cr_panic("Re-entering criterion from a test worker. This is a " "catastrophic bug, please report it on the issue tracker.\n" "Bailing out to avoid fork-bombing the system."); } #ifndef ENABLE_VALGRIND_ERRORS VALGRIND_DISABLE_ERROR_REPORTING; #endif cri_is_runner = 1; if (criterion_options.pattern) disable_unmatching(set); if (criterion_options.debug) { criterion_options.jobs = 1; criterion_options.crash = true; criterion_options.logging_threshold = 1; } if (criterion_options.logging_threshold == CRITERION_LOG_LEVEL_QUIET) { cri_silence_outputs(); } int res = criterion_run_all_tests_impl(set); #ifndef ENABLE_VALGRIND_ERRORS VALGRIND_ENABLE_ERROR_REPORTING; #endif return criterion_options.always_succeed || res; }