@article{Garousi2019,
abstract = {Context: A Multivocal Literature Review (MLR) is a form of a Systematic Literature Review (SLR) which includes the grey literature (e.g., blog posts, videos and white papers) in addition to the published (formal) literature (e.g., journal and conference papers). MLRs are useful for both researchers and practitioners since they provide summaries both the state-of-the art and –practice in a given area. MLRs are popular in other fields and have recently started to appear in software engineering (SE). As more MLR studies are conducted and reported, it is important to have a set of guidelines to ensure high quality of MLR processes and their results. Objective: There are several guidelines to conduct SLR studies in SE. However, several phases of MLRs differ from those of traditional SLRs, for instance with respect to the search process and source quality assessment. Therefore, SLR guidelines are only partially useful for conducting MLR studies. Our goal in this paper is to present guidelines on how to conduct MLR studies in SE. Method: To develop the MLR guidelines, we benefit from several inputs: (1) existing SLR guidelines in SE, (2), a literature survey of MLR guidelines and experience papers in other fields, and (3) our own experiences in conducting several MLRs in SE. We took the popular SLR guidelines of Kitchenham and Charters as the baseline and extended/adopted them to conduct MLR studies in SE. All derived guidelines are discussed in the context of an already-published MLR in SE as the running example. Results: The resulting guidelines cover all phases of conducting and reporting MLRs in SE from the planning phase, over conducting the review to the final reporting of the review. In particular, we believe that incorporating and adopting a vast set of experience-based recommendations from MLR guidelines and experience papers in other fields have enabled us to propose a set of guidelines with solid foundations. Conclusion: Having been developed on the basis of several types of experience and evidence, the provided MLR guidelines will support researchers to effectively and efficiently conduct new MLRs in any area of SE. The authors recommend the researchers to utilize these guidelines in their MLR studies and then share their lessons learned and experiences.},
author = {Garousi, Vahid and Felderer, Michael and M{\"{a}}ntyl{\"{a}}, Mika V.},
doi = {10.1016/j.infsof.2018.09.006},
issn = {09505849},
journal = {Information and Software Technology},
keywords = {Evidence-based software engineering,Grey literature,Guidelines,Literature study,Multivocal literature review,Systematic literature review,Systematic mapping study},
month = {feb},
number = {September 2018},
pages = {101--121},
publisher = {Elsevier B.V.},
title = {{Guidelines for including grey literature and conducting multivocal literature reviews in software engineering}},
volume = {106},
year = {2019}
}
@inproceedings{Garousi2016,
abstract = {Systematic Literature Reviews (SLR) may not provide insight into the "state of the practice" in SE, as they do not typically include the "grey" (non-published) literature. A Multivocal Literature Review (MLR) is a form of a SLR which includes grey literature in addition to the published (formal) literature. Only a few MLRs have been published in SE so far. We aim at raising the awareness for MLRs in SE by addressing two research questions (RQs): (1) What types of knowledge are missed when a SLR does not include the multivocal literature in a SE field? and (2) What do we, as a community, gain when we include the multivocal literature and conduct MLRs? To answer these RQs, we sample a few example SLRs and MLRs and identify the missing and the gained knowledge due to excluding or including the grey literature. We find that (1) grey literature can give substantial benefits in certain areas of SE, and that (2) the inclusion of grey literature brings forward certain challenges as evidence in them is often experience and opinion based. Given these conflicting viewpoints, the authors are planning to prepare systematic guidelines for performing MLRs in SE.},
address = {New York, New York, USA},
author = {Garousi, Vahid and Felderer, Michael and M{\"{a}}ntyl{\"{a}}, Mika V.},
booktitle = {Proceedings of the 20th International Conference on Evaluation and Assessment in Software Engineering - EASE '16},
doi = {10.1145/2915970.2916008},
isbn = {9781450336918},
keywords = {Empirical software engineering,Grey literature,MLR,Multivocal Literature Reviews,Research methodology,SLR,Systematic literature reviews},
pages = {1--6},
publisher = {ACM Press},
title = {{The need for multivocal literature reviews in software engineering}},
volume = {01-03-June},
year = {2016}
}
@incollection{Garousi2020,
abstract = {Researchers generally place the most trust in peer-reviewed, published information, such as journals and conference papers. By contrast, software engineering (SE) practitioners typically do not have the time, access or expertise to review and benefit from such publications. As a result, practitioners are more likely to turn to other sources of information that they trust, e.g., trade magazines, online blog-posts, survey results or technical reports, collectively referred to as Grey Literature (GL). Furthermore, practitioners also share their ideas and experiences as GL, which can serve as a valuable data source for research. While GL itself is not a new topic in SE, using, benefitting and synthesizing knowledge from the GL in SE is a contemporary topic in empirical SE research and we are seeing that researchers are increasingly benefitting from the knowledge available within GL. The goal of this chapter is to provide an overview to GL in SE, together with insights on how SE researchers can effectively use and benefit from the knowledge and evidence available in the vast amount of GL.},
address = {Cham},
author = {Garousi, Vahid and Felderer, Michael and M{\"{a}}ntyl{\"{a}}, Mika V. and Rainer, Austen},
booktitle = {Contemporary Empirical Methods in Software Engineering},
doi = {10.1007/978-3-030-32489-6_14},
isbn = {9783030324896},
pages = {385--413},
publisher = {Springer International Publishing},
title = {{Benefitting from the Grey Literature in Software Engineering Research}},
year = {2020}
}
@inproceedings{Neto2019,
abstract = {Background: In recent years, studies involving Grey Literature (GL) have been growing and attracting the attention of researchers in software engineering (SE). One of the sources of GL refers to content produced by professionals based on their practical experiences? Recent researches in the SE states that GL can complement areas of research that are not yet clearly defined in the scientific literature. In this context, the Multivocal Literature Review (MLR), a form of Systematic Literature Review (SLR) with the inclusion of GL, emerges. Goal: Provide preliminary work about the current research involving MLR studies? First, we investigate the motivation of the researchers to include GL in review studies; and second, we examine how GL was included in the studies. Method: A tertiary study was conducted to search MLR studies published between 2009 to April of 2019. Results: The main motivations for including GL in review studies are: lack of academic research on the topic, emerging research on this topic, and complementary evidence in the GL? Internet articles and white papers were the main sources of GL data used. Conclusions: The conducting of MLR studies is still in its early stages; we have identified only 12 secondary studies. The MLR studies were conducted using guidelines for performing SLRs. What we consider to be a threat to the validity of these studies, since guidelines to conduct SLR studies do not provide recommendations for quality analysis and synthesis of primary studies, including GL.},
author = {Neto, Geraldo Torres G. and Santos, Wylliams B. and Endo, Patricia Takako and Fagundes, Roberta A.A.},
booktitle = {2019 ACM/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM)},
doi = {10.1109/ESEM.2019.8870142},
isbn = {978-1-7281-2968-6},
month = {sep},
pages = {1--6},
publisher = {IEEE},
title = {{Multivocal literature reviews in software engineering: Preliminary findings from a tertiary study}},
volume = {2019-Septe},
year = {2019}
}
@article{Rainer2019,
abstract = {Background: Software engineering research has a growing interest in grey literature (GL). Aim: To improve the identification of relevant and rigorous GL. Method: We develop and demonstrate heuristics to find more relevant and rigorous GL. The heuristics generate stratified samples of search and post–search datasets using a formally structured set of search keywords. Conclusion: The heuristics require further evaluation. We are developing a tool to implement the heuristics.},
author = {Rainer, Austen and Williams, Ashley},
doi = {10.1016/j.infsof.2018.10.007},
issn = {09505849},
journal = {Information and Software Technology},
keywords = {Grey literature review,Quality criteria,Reasoning,Search engines},
month = {feb},
pages = {231--233},
publisher = {Elsevier B.V.},
title = {{Heuristics for improving the rigour and relevance of grey literature searches for software engineering research}},
volume = {106},
year = {2019}
}