/*
* The MIT License (MIT)
*
* Copyright © 2015-2016 Franklin "Snaipe" Mathieu
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#define CRITERION_LOGGING_COLORS
#include "config.h"
#include
#include
#include
#include NN_H
#include "criterion/internal/test.h"
#include "criterion/options.h"
#include "criterion/internal/ordered-set.h"
#include "criterion/internal/preprocess.h"
#include "criterion/redirect.h"
#include "protocol/protocol.h"
#include "protocol/connect.h"
#include "protocol/messages.h"
#include "compat/alloc.h"
#include "compat/time.h"
#include "compat/posix.h"
#include "compat/processor.h"
#include "compat/kill.h"
#include "csptr/smalloc.h"
#include "string/diff.h"
#include "string/extglobmatch.h"
#include "string/i18n.h"
#include "io/event.h"
#include "io/output.h"
#include "log/logging.h"
#include "abort.h"
#include "client.h"
#include "common.h"
#include "err.h"
#include "report.h"
#include "runner.h"
#include "runner_coroutine.h"
#include "stats.h"
#ifdef __GNUC__
# include
#else
# define ENABLE_VALGRIND_ERRORS
# define RUNNING_ON_VALGRIND 0
#endif
int cri_is_runner;
typedef const char *const msg_t;
#ifdef ENABLE_NLS
static msg_t msg_valgrind_jobs = N_("%1$sWarning! Criterion has detected "
"that it is running under valgrind, but the number of jobs have been "
"explicitely set. Reports might appear confusing!%2$s\n");
#else
static msg_t msg_valgrind_jobs = "%sWarning! Criterion has detected "
"that it is running under valgrind, but the number of jobs have been "
"explicitely set. Reports might appear confusing!%s\n";
#endif
static int cmp_suite(void *a, void *b)
{
struct criterion_suite *s1 = a, *s2 = b;
return strcmp(s1->name, s2->name);
}
static int cmp_test(void *a, void *b)
{
struct criterion_test *s1 = a, *s2 = b;
return strcmp(s1->name, s2->name);
}
static void dtor_suite_set(void *ptr, CR_UNUSED void *meta)
{
struct criterion_suite_set *s = ptr;
sfree(s->tests);
}
static void dtor_test_set(void *ptr, CR_UNUSED void *meta)
{
struct criterion_test_set *t = ptr;
sfree(t->suites);
}
CR_API void criterion_register_test(struct criterion_test_set *set,
struct criterion_test *test)
{
struct criterion_suite_set css = {
.suite = { .name = test->category },
};
struct criterion_suite_set *s = insert_ordered_set(set->suites, &css, sizeof (css));
if (!s->tests)
s->tests = new_ordered_set(cmp_test, NULL);
insert_ordered_set(s->tests, test, sizeof (*test));
++set->tests;
}
struct criterion_test_set *criterion_init(void)
{
struct criterion_ordered_set *suites = new_ordered_set(cmp_suite, dtor_suite_set);
struct cri_section *sections = NULL;
if (!cri_sections_getaddr("cr_sts", §ions)) {
for (struct cri_section *s = sections; s->addr; ++s) {
void *start = s->addr;
void *end = (char *) start + s->length;
FOREACH_SUITE_SEC(s, start, end) {
if (!*s || !*(*s)->name)
continue;
struct criterion_suite_set css = {
.suite = **s,
};
insert_ordered_set(suites, &css, sizeof (css));
}
}
}
free(sections);
struct criterion_test_set *set = smalloc(
.size = sizeof (struct criterion_test_set),
.dtor = dtor_test_set);
*set = (struct criterion_test_set) {
suites,
0,
};
sections = NULL;
if (!cri_sections_getaddr("cr_tst", §ions)) {
for (struct cri_section *s = sections; s->addr; ++s) {
void *start = s->addr;
void *end = (char *) start + s->length;
FOREACH_TEST_SEC(test, start, end) {
if (!*test)
continue;
if (!*(*test)->category || !*(*test)->name)
continue;
criterion_register_test(set, *test);
}
}
}
free(sections);
return set;
}
#define push_event(...) \
do { \
stat_push_event(ctx->stats, \
ctx->suite_stats, \
ctx->test_stats, \
&(struct event) { \
.kind = CR_VA_HEAD(__VA_ARGS__), \
CR_VA_TAIL(__VA_ARGS__) \
}); \
report(CR_VA_HEAD(__VA_ARGS__), ctx->test_stats); \
} while (0)
void disable_unmatching(struct criterion_test_set *set)
{
if (!compile_pattern(criterion_options.pattern))
exit(3);
FOREACH_SET(struct criterion_suite_set *s, set->suites) {
if ((s->suite.data && s->suite.data->disabled) || !s->tests)
continue;
FOREACH_SET(struct criterion_test *test, s->tests) {
int ret = match(test->data->identifier_);
if (ret == 0)
test->data->disabled = true;
}
}
free_pattern();
}
CR_API struct criterion_test_set *criterion_initialize(void)
{
/* make sure we don't re-enter from a test worker. See #247. */
if (getenv("BXFI_MAP")) {
cr_panic("Re-entering criterion from a test worker. This is a "
"catastrophic bug, please report it on the issue tracker.\n"
"Bailing out to avoid fork-bombing the system.");
}
init_i18n();
cri_diff_init();
#ifndef ENABLE_VALGRIND_ERRORS
VALGRIND_DISABLE_ERROR_REPORTING;
#endif
if (RUNNING_ON_VALGRIND) {
criterion_options.no_early_exit = 1;
criterion_options.jobs = 1;
}
criterion_register_output_provider("tap", tap_report);
criterion_register_output_provider("xml", xml_report);
criterion_register_output_provider("json", json_report);
setup_parent_job();
return criterion_init();
}
CR_API void criterion_finalize(struct criterion_test_set *set)
{
sfree(set);
#ifndef ENABLE_VALGRIND_ERRORS
VALGRIND_ENABLE_ERROR_REPORTING;
#endif
cri_diff_fini();
criterion_free_output();
}
static struct client_ctx *spawn_next_client(struct server_ctx *sctx, ccrContext *ctx)
{
struct client_ctx new_ctx;
bxf_instance *instance = cri_run_next_test(NULL, NULL, NULL, &new_ctx, ctx);
if (!instance)
return NULL;
return add_client_from_worker(sctx, &new_ctx, instance);
}
static void run_tests_async(struct criterion_test_set *set,
struct criterion_global_stats *stats,
const char *url,
int socket)
{
ccrContext ctx = 0;
size_t nb_workers = DEF(criterion_options.jobs, get_processor_count());
size_t active_workers = 0;
int has_msg = 0;
struct server_ctx sctx;
init_server_context(&sctx, stats);
sctx.socket = socket;
/* initialization of coroutine */
cri_run_next_test(set, stats, url, NULL, &ctx);
for (size_t i = 0; i < nb_workers; ++i) {
struct client_ctx *cctx = spawn_next_client(&sctx, &ctx);
if (!cctx)
break;
++active_workers;
}
if (!active_workers && !criterion_options.wait_for_clients)
goto cleanup;
criterion_protocol_msg msg = criterion_protocol_msg_init_zero;
while ((has_msg = read_message(socket, &msg)) == 1) {
struct client_ctx *cctx = process_client_message(&sctx, &msg);
/* drop invalid messages */
if (!cctx)
continue;
if (cctx->dead) {
if ((cctx->tstats->test_status == CR_STATUS_FAILED) && criterion_options.fail_fast)
cr_terminate(cctx->gstats);
if (cctx->kind == WORKER) {
remove_client_by_pid(&sctx, cctx->instance->pid);
cctx = spawn_next_client(&sctx, &ctx);
if (cctx == NULL)
--active_workers;
}
}
if (!active_workers && !criterion_options.wait_for_clients)
break;
free_message(&msg);
}
cleanup:
if (has_msg)
free_message(&msg);
destroy_server_context(&sctx);
ccrAbort(ctx);
}
static int criterion_run_all_tests_impl(struct criterion_test_set *set)
{
cri_report_init();
if (RUNNING_ON_VALGRIND) {
if (criterion_options.jobs != 1)
criterion_pimportant(CRITERION_PREFIX_DASHES,
_(msg_valgrind_jobs), CR_FG_BOLD, CR_RESET);
}
char url[sizeof ("ipc://" NN_SOCKET_PATH "criterion_.sock") + 21];
snprintf(url, sizeof (url), "ipc://" NN_SOCKET_PATH "criterion_%llu.sock", get_process_id());
int sock = cri_proto_bind(url);
if (sock < 0)
cr_panic("Could not initialize the message server: %s.", strerror(errno));
g_client_socket = cri_proto_connect(url);
if (g_client_socket < 0)
cr_panic("Could not initialize the message client: %s.", strerror(errno));
cri_alloc_init();
report(PRE_ALL, set);
log(pre_all, set);
struct criterion_global_stats *stats = stats_init();
run_tests_async(set, stats, url, sock);
report(POST_ALL, stats);
if (criterion_options.logging_threshold == CRITERION_LOG_LEVEL_QUIET) {
cri_restore_outputs();
}
process_all_output(stats);
log(post_all, stats);
cri_alloc_term();
cri_report_term();
cri_proto_close(g_client_socket);
cri_proto_close(sock);
int ok = stats->tests_failed == 0 && stats->errors == 0;
if (!criterion_options.ignore_warnings)
ok = ok && stats->warnings == 0;
sfree(stats);
return ok;
}
CR_API int criterion_run_all_tests(struct criterion_test_set *set)
{
/* make sure we don't re-enter from a test worker. See #247. */
if (getenv("BXFI_MAP")) {
cr_panic("Re-entering criterion from a test worker. This is a "
"catastrophic bug, please report it on the issue tracker.\n"
"Bailing out to avoid fork-bombing the system.");
}
#ifndef ENABLE_VALGRIND_ERRORS
VALGRIND_DISABLE_ERROR_REPORTING;
#endif
cri_is_runner = 1;
if (criterion_options.pattern)
disable_unmatching(set);
if (criterion_options.debug) {
criterion_options.jobs = 1;
criterion_options.crash = true;
criterion_options.logging_threshold = 1;
}
if (criterion_options.logging_threshold == CRITERION_LOG_LEVEL_QUIET) {
cri_silence_outputs();
}
int res = criterion_run_all_tests_impl(set);
#ifndef ENABLE_VALGRIND_ERRORS
VALGRIND_ENABLE_ERROR_REPORTING;
#endif
return criterion_options.always_succeed || res;
}