@article{Falessi2018,
abstract = {{\textcopyright} 2017 Springer Science+Business Media New York[Context] Controlled experiments are an important empirical method to generate and validate theories. Many software engineering experiments are conducted with students. It is often claimed that the use of students as participants in experiments comes at the cost of low external validity while using professionals does not. [Objective] We believe a deeper understanding is needed on the external validity of software engineering experiments conducted with students or with professionals. We aim to gain insight about the pros and cons of using students and professionals in experiments. [Method] We performed an unconventional, focus group approach and a follow-up survey. First, during a session at ISERN 2014, 65 empirical researchers, including the seven authors, argued and discussed the use of students in experiments with an open mind. Afterwards, we revisited the topic and elicited experts' opinions to foster discussions. Then we derived 14 statements and asked the ISERN attendees excluding the authors, to provide their level of agreement with the statements. Finally, we analyzed the researchers' opinions and used the findings to further discuss the statements. [Results] Our survey results showed that, in general, the respondents disagreed with us about the drawbacks of professionals. We, on the contrary, strongly believe that no population (students, professionals, or others) can be deemed better than another in absolute terms. [Conclusion] Using students as participants remains a valid simplification of reality needed in laboratory contexts. It is an effective way to advance software engineering theories and technologies but, like any other aspect of study settings, should be carefully considered during the design, execution, interpretation, and reporting of an experiment. The key is to understand which developer population portion is being represented by the participants in an experiment. Thus, a proposal for describing experimental participants is put forward.},
author = {Falessi, Davide and Juristo, Natalia and Wohlin, Claes and Turhan, Burak and M{\"{u}}nch, J{\"{u}}rgen and Jedlitschka, Andreas and Oivo, Markku},
doi = {10.1007/s10664-017-9523-3},
issn = {1382-3256},
journal = {Empirical Software Engineering},
keywords = {Experimentation,Generalization,Participants in experiments,Subjects of experiments,Threats to validity},
month = {feb},
number = {1},
pages = {452--489},
publisher = {Empirical Software Engineering},
title = {{Empirical software engineering experts on the use of students and professionals in experiments}},
volume = {23},
year = {2018}
}
@article{Host2000,
abstract = {In many studies in software engineering students are used instead of professional software developers, although the objective is to draw conclusions valid for professional software developers. This paper presents a study where the difference between the two groups is evaluated. People from the two groups have individually carried out a non-trivial software engineering judgement task involving the assessment of howten different factors affect the lead-time of software development projects. It is found that the differences are only minor, and it is concluded that software engineering students may be used instead of professional software developers under certain conditions. These conditions are identified and described based on generally accepted criteria for validity evaluation of empirical studies.},
author = {Host, Martin and Regnell, Bj{\"{o}}rn and Wohlin, Claes},
doi = {10.1023/A:1026586415054},
issn = {1382-3256},
journal = {Empirical Software Engineering},
number = {3},
pages = {201--214},
title = {{Using students as subjects - a comparative study of students and professionals in lead-time impact assessment}},
volume = {5},
year = {2000}
}
@incollection{Jedlitschka2008,
address = {London},
author = {Jedlitschka, Andreas and Ciolkowski, Marcus and Pfahl, Dietmar},
booktitle = {Guide to Advanced Empirical Software Engineering},
doi = {10.1007/978-1-84800-044-5_8},
pages = {201--228},
publisher = {Springer London},
title = {{Reporting Experiments in Software Engineering}},
year = {2008}
}
@inproceedings{Salman2015,
abstract = {Background: Most of the experiments in software engineering (SE) employ students as subjects. This raises concerns about the realism of the results acquired through students and adaptability of the results to software industry. Aim: We compare students and professionals to understand how well students represent professionals as experimental subjects in SE research. Method: The comparison was made in the context of two test-driven development experiments conducted with students in an academic setting and with professionals in a software organization. We measured the code quality of several tasks implemented by both subject groups and checked whether students and professionals perform similarly in terms of code quality metrics. Results: Except for minor differences, neither of the subject groups is better than the other. Professionals produce larger, yet less complex, methods when they use their traditional development approach, whereas both subject groups perform similarly when they apply a new approach for the first time. Conclusion: Given a carefully scoped experiment on a development approach that is new to both students and professionals, similar performances are observed. Further investigation is necessary to analyze the effects of subject demographics and level of experience on the results of SE experiments.},
author = {Salman, Iflaah and Misirli, Ayse Tosun and Juristo, Natalia},
booktitle = {2015 IEEE/ACM 37th IEEE International Conference on Software Engineering},
doi = {10.1109/ICSE.2015.82},
isbn = {978-1-4799-1934-5},
issn = {02705257},
keywords = {Code quality,Empirical study,Experimentation,Test-driven development},
month = {may},
pages = {666--676},
publisher = {IEEE},
title = {{Are Students Representatives of Professionals in Software Engineering Experiments?}},
volume = {1},
year = {2015}
}
@book{Wohlin2012,
abstract = {Empirical software engineering research can be organized in several ways, including experiments, cases studies, and surveys. Experiments sample over the variables, trying to represent all possible cases; cases studies sample from the variables, representing only the typical cases(s). Every case study or experiment should have a hypothesis to express the desired result. The experimental design is especially important because it identifies key variables and their relationships. The design uses balancing, blocking, and local control to help minimize error. Analysis techniques depend on the design, the distribution of the data, and the type of investigation being carried out. Different techniques allow us to look at variable interaction and to look at combinations of effects. Using a technique similar to a board game, we can determine when we have enough evidence to demonstrate clear relationships among variables. {\textcopyright} 1997 Academic Press Inc.},
address = {Berlin, Heidelberg},
author = {Wohlin, Claes and Runeson, Per and H{\"{o}}st, Martin and Ohlsson, Magnus C. and Regnell, Bj{\"{o}}rn and Wessl{\'{e}}n, Anders},
booktitle = {Experimentation in Software Engineering},
doi = {10.1007/978-3-642-29044-2},
isbn = {978-3-642-29043-5},
pages = {1--236},
publisher = {Springer Berlin Heidelberg},
title = {{Experimentation in Software Engineering}},
volume = {9783642290},
year = {2012}
}