<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="generator" content="pandoc">
<meta name="author" content="Fill In Your Name" />
<title>Statistical Power</title>
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">
<link rel="stylesheet" href="power-slides_files/reveal.js-3.3.0.1/css/reveal.css"/>
<style type="text/css">
pre > code.sourceCode { white-space: pre; position: relative; }
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
pre > code.sourceCode > span:empty { height: 1.2em; }
.sourceCode { overflow: visible; }
code.sourceCode > span { color: inherit; text-decoration: inherit; }
div.sourceCode { margin: 1em 0; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
pre > code.sourceCode { white-space: pre-wrap; }
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
}
pre.numberSource code
{ counter-reset: source-line 0; }
pre.numberSource code > span
{ position: relative; left: -4em; counter-increment: source-line; }
pre.numberSource code > span > a:first-child::before
{ content: counter(source-line);
position: relative; left: -1em; text-align: right; vertical-align: baseline;
border: none; display: inline-block;
-webkit-touch-callout: none; -webkit-user-select: none;
-khtml-user-select: none; -moz-user-select: none;
-ms-user-select: none; user-select: none;
padding: 0 4px; width: 4em;
color: #aaaaaa;
}
pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; }
div.sourceCode
{ }
@media screen {
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
}
code span.al { color: #ff0000; font-weight: bold; } /* Alert */
code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */
code span.at { color: #7d9029; } /* Attribute */
code span.bn { color: #40a070; } /* BaseN */
code span.bu { } /* BuiltIn */
code span.cf { color: #007020; font-weight: bold; } /* ControlFlow */
code span.ch { color: #4070a0; } /* Char */
code span.cn { color: #880000; } /* Constant */
code span.co { color: #60a0b0; font-style: italic; } /* Comment */
code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */
code span.do { color: #ba2121; font-style: italic; } /* Documentation */
code span.dt { color: #902000; } /* DataType */
code span.dv { color: #40a070; } /* DecVal */
code span.er { color: #ff0000; font-weight: bold; } /* Error */
code span.ex { } /* Extension */
code span.fl { color: #40a070; } /* Float */
code span.fu { color: #06287e; } /* Function */
code span.im { } /* Import */
code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */
code span.kw { color: #007020; font-weight: bold; } /* Keyword */
code span.op { color: #666666; } /* Operator */
code span.ot { color: #007020; } /* Other */
code span.pp { color: #bc7a00; } /* Preprocessor */
code span.sc { color: #4070a0; } /* SpecialChar */
code span.ss { color: #bb6688; } /* SpecialString */
code span.st { color: #4070a0; } /* String */
code span.va { color: #19177c; } /* Variable */
code span.vs { color: #4070a0; } /* VerbatimString */
code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */
</style>
<link rel="stylesheet" href="power-slides_files/reveal.js-3.3.0.1/css/theme/simple.css" id="theme">
<!-- some tweaks to reveal css -->
<style type="text/css">
.reveal h1 { font-size: 2.0em; }
.reveal h2 { font-size: 1.5em; }
.reveal h3 { font-size: 1.25em; }
.reveal h4 { font-size: 1em; }
.reveal .slides>section,
.reveal .slides>section>section {
padding: 0px 0px;
}
.reveal table {
border-width: 1px;
border-spacing: 2px;
border-style: dotted;
border-color: gray;
border-collapse: collapse;
font-size: 0.7em;
}
.reveal table th {
border-width: 1px;
padding-left: 10px;
padding-right: 25px;
font-weight: bold;
border-style: dotted;
border-color: gray;
}
.reveal table td {
border-width: 1px;
padding-left: 10px;
padding-right: 25px;
border-style: dotted;
border-color: gray;
}
</style>
<style type="text/css">code{white-space: pre;}</style>
<!-- Printing and PDF exports -->
<script id="paper-css" type="application/dynamic-css">
/* Default Print Stylesheet Template
by Rob Glazebrook of CSSnewbie.com
Last Updated: June 4, 2008
Feel free (nay, compelled) to edit, append, and
manipulate this file as you see fit. */
@media print {
/* SECTION 1: Set default width, margin, float, and
background. This prevents elements from extending
beyond the edge of the printed page, and prevents
unnecessary background images from printing */
html {
background: #fff;
width: auto;
height: auto;
overflow: visible;
}
body {
background: #fff;
font-size: 20pt;
width: auto;
height: auto;
border: 0;
margin: 0 5%;
padding: 0;
overflow: visible;
float: none !important;
}
/* SECTION 2: Remove any elements not needed in print.
This would include navigation, ads, sidebars, etc. */
.nestedarrow,
.controls,
.fork-reveal,
.share-reveal,
.state-background,
.reveal .progress,
.reveal .backgrounds {
display: none !important;
}
/* SECTION 3: Set body font face, size, and color.
Consider using a serif font for readability. */
body, p, td, li, div {
font-size: 20pt!important;
font-family: Georgia, "Times New Roman", Times, serif !important;
color: #000;
}
/* SECTION 4: Set heading font face, sizes, and color.
Differentiate your headings from your body text.
Perhaps use a large sans-serif for distinction. */
h1,h2,h3,h4,h5,h6 {
color: #000!important;
height: auto;
line-height: normal;
font-family: Georgia, "Times New Roman", Times, serif !important;
text-shadow: 0 0 0 #000 !important;
text-align: left;
letter-spacing: normal;
}
/* Need to reduce the size of the fonts for printing */
h1 { font-size: 28pt !important; }
h2 { font-size: 24pt !important; }
h3 { font-size: 22pt !important; }
h4 { font-size: 22pt !important; font-variant: small-caps; }
h5 { font-size: 21pt !important; }
h6 { font-size: 20pt !important; font-style: italic; }
/* SECTION 5: Make hyperlinks more usable.
Ensure links are underlined, and consider appending
the URL to the end of the link for usability. */
a:link,
a:visited {
color: #000 !important;
font-weight: bold;
text-decoration: underline;
}
/*
.reveal a:link:after,
.reveal a:visited:after {
content: " (" attr(href) ") ";
color: #222 !important;
font-size: 90%;
}
*/
/* SECTION 6: more reveal.js specific additions by @skypanther */
ul, ol, div, p {
visibility: visible;
position: static;
width: auto;
height: auto;
display: block;
overflow: visible;
margin: 0;
text-align: left !important;
}
.reveal pre,
.reveal table {
margin-left: 0;
margin-right: 0;
}
.reveal pre code {
padding: 20px;
border: 1px solid #ddd;
}
.reveal blockquote {
margin: 20px 0;
}
.reveal .slides {
position: static !important;
width: auto !important;
height: auto !important;
left: 0 !important;
top: 0 !important;
margin-left: 0 !important;
margin-top: 0 !important;
padding: 0 !important;
zoom: 1 !important;
overflow: visible !important;
display: block !important;
text-align: left !important;
-webkit-perspective: none;
-moz-perspective: none;
-ms-perspective: none;
perspective: none;
-webkit-perspective-origin: 50% 50%;
-moz-perspective-origin: 50% 50%;
-ms-perspective-origin: 50% 50%;
perspective-origin: 50% 50%;
}
.reveal .slides section {
visibility: visible !important;
position: static !important;
width: auto !important;
height: auto !important;
display: block !important;
overflow: visible !important;
left: 0 !important;
top: 0 !important;
margin-left: 0 !important;
margin-top: 0 !important;
padding: 60px 20px !important;
z-index: auto !important;
opacity: 1 !important;
page-break-after: always !important;
-webkit-transform-style: flat !important;
-moz-transform-style: flat !important;
-ms-transform-style: flat !important;
transform-style: flat !important;
-webkit-transform: none !important;
-moz-transform: none !important;
-ms-transform: none !important;
transform: none !important;
-webkit-transition: none !important;
-moz-transition: none !important;
-ms-transition: none !important;
transition: none !important;
}
.reveal .slides section.stack {
padding: 0 !important;
}
.reveal section:last-of-type {
page-break-after: avoid !important;
}
.reveal section .fragment {
opacity: 1 !important;
visibility: visible !important;
-webkit-transform: none !important;
-moz-transform: none !important;
-ms-transform: none !important;
transform: none !important;
}
.reveal section img {
display: block;
margin: 15px 0px;
background: rgba(255,255,255,1);
border: 1px solid #666;
box-shadow: none;
}
.reveal section small {
font-size: 0.8em;
}
}
</script>
<script id="pdf-css" type="application/dynamic-css">
/**
* This stylesheet is used to print reveal.js
* presentations to PDF.
*
* https://github.com/hakimel/reveal.js#pdf-export
*/
* {
-webkit-print-color-adjust: exact;
}
body {
margin: 0 auto !important;
border: 0;
padding: 0;
float: none !important;
overflow: visible;
}
html {
width: 100%;
height: 100%;
overflow: visible;
}
/* Remove any elements not needed in print. */
.nestedarrow,
.reveal .controls,
.reveal .progress,
.reveal .playback,
.reveal.overview,
.fork-reveal,
.share-reveal,
.state-background {
display: none !important;
}
h1, h2, h3, h4, h5, h6 {
text-shadow: 0 0 0 #000 !important;
}
.reveal pre code {
overflow: hidden !important;
font-family: Courier, 'Courier New', monospace !important;
}
ul, ol, div, p {
visibility: visible;
position: static;
width: auto;
height: auto;
display: block;
overflow: visible;
margin: auto;
}
.reveal {
width: auto !important;
height: auto !important;
overflow: hidden !important;
}
.reveal .slides {
position: static;
width: 100%;
height: auto;
left: auto;
top: auto;
margin: 0 !important;
padding: 0 !important;
overflow: visible;
display: block;
-webkit-perspective: none;
-moz-perspective: none;
-ms-perspective: none;
perspective: none;
-webkit-perspective-origin: 50% 50%; /* there isn't a none/auto value but 50-50 is the default */
-moz-perspective-origin: 50% 50%;
-ms-perspective-origin: 50% 50%;
perspective-origin: 50% 50%;
}
.reveal .slides section {
page-break-after: always !important;
visibility: visible !important;
position: relative !important;
display: block !important;
position: relative !important;
margin: 0 !important;
padding: 0 !important;
box-sizing: border-box !important;
min-height: 1px;
opacity: 1 !important;
-webkit-transform-style: flat !important;
-moz-transform-style: flat !important;
-ms-transform-style: flat !important;
transform-style: flat !important;
-webkit-transform: none !important;
-moz-transform: none !important;
-ms-transform: none !important;
transform: none !important;
}
.reveal section.stack {
margin: 0 !important;
padding: 0 !important;
page-break-after: avoid !important;
height: auto !important;
min-height: auto !important;
}
.reveal img {
box-shadow: none;
}
.reveal .roll {
overflow: visible;
line-height: 1em;
}
/* Slide backgrounds are placed inside of their slide when exporting to PDF */
.reveal section .slide-background {
display: block !important;
position: absolute;
top: 0;
left: 0;
width: 100%;
z-index: -1;
}
/* All elements should be above the slide-background */
.reveal section>* {
position: relative;
z-index: 1;
}
/* Display slide speaker notes when 'showNotes' is enabled */
.reveal .speaker-notes-pdf {
display: block;
width: 100%;
max-height: none;
left: auto;
top: auto;
z-index: 100;
}
/* Display slide numbers when 'slideNumber' is enabled */
.reveal .slide-number-pdf {
display: block;
position: absolute;
font-size: 14px;
}
</script>
<script>
var style = document.createElement( 'style' );
style.type = 'text/css';
var style_script_id = window.location.search.match( /print-pdf/gi ) ? 'pdf-css' : 'paper-css';
var style_script = document.getElementById(style_script_id).text;
style.innerHTML = style_script;
document.getElementsByTagName('head')[0].appendChild(style);
</script>
<script src="power-slides_files/header-attrs-2.10/header-attrs.js"></script>
<link href="power-slides_files/font-awesome-5.1.0/css/all.css" rel="stylesheet" />
<link href="power-slides_files/font-awesome-5.1.0/css/v4-shims.css" rel="stylesheet" />
</head>
<body>
<div class="reveal">
<div class="slides">
<section>
<h1 class="title">Statistical Power</h1>
<h2 class="author">Fill In Your Name</h2>
<h3 class="date">01 March, 2022</h3>
</section>
<section>
<section id="what-is-power" class="title-slide slide level1">
<h1>What is power?</h1>
</section>
<section id="what-is-power-1" class="slide level2">
<h2>What is power?</h2>
<ul>
<li><p>We want to separate signal from noise.</p></li>
<li><p>Power = probability of rejecting null hypothesis, given true
effect <span class="math inline">\(\ne\)</span> 0.</p></li>
<li><p>In other words, it is the ability to detect an effect given that
it exists.</p></li>
<li><p>Formally: (1 - Type II) error rate.</p></li>
<li><p>Thus, power <span class="math inline">\(\in\)</span> (0,
1).</p></li>
<li><p>Standard thresholds: 0.8 or 0.9.</p></li>
</ul>
</section>
<section id="starting-point-for-power-analysis" class="slide level2">
<h2>Starting point for power analysis</h2>
<ul>
<li><p>Power analysis is something we do <em>before</em> we run a
study.</p>
<ul>
<li><p>Helps you figure out the sample you need to detect a given effect
size.</p></li>
<li><p>Or helps you figure out a minimal detectable difference given a
set sample size.</p></li>
<li><p>May help you decide whether to run a study.</p></li>
</ul></li>
<li><p>It is hard to learn from an under-powered null finding.</p>
<ul>
<li>Was there an effect, but we were unable to detect it? or was there
no effect? We can’t say.</li>
</ul></li>
</ul>
</section>
<section id="power" class="slide level2">
<h2>Power</h2>
<ul>
<li><p>Say there truly is a treatment effect and you run your experiment
many times. How often will you get a statistically significant
result?</p></li>
<li><p>Some guesswork to answer this question.</p>
<ul>
<li><p>How big is your treatment effect?</p></li>
<li><p>How many units are treated, measured?</p></li>
<li><p>How much noise is there in the measurement of your
outcome?</p></li>
</ul></li>
</ul>
</section>
<section id="approaches-to-power-calculation" class="slide level2">
<h2>Approaches to power calculation</h2>
<ul>
<li><p>Analytical calculations of power</p></li>
<li><p>Simulation</p></li>
</ul>
</section>
<section id="power-calculation-tools" class="slide level2">
<h2>Power calculation tools</h2>
<ul>
<li><p>Interactive</p>
<ul>
<li><p><a href="https://egap.shinyapps.io/power-app/">EGAP Power
Calculator</a></p></li>
<li><p><a
href="https://rpsychologist.com/d3/NHST/">rpsychologist</a></p></li>
</ul></li>
<li><p>R Packages</p>
<ul>
<li><p><a
href="https://cran.r-project.org/web/packages/pwr/index.html">pwr</a></p></li>
<li><p><a
href="https://cran.r-project.org/web/packages/DeclareDesign/index.html">DeclareDesign</a>,
see also <a href="https://declaredesign.org/"
class="uri">https://declaredesign.org/</a></p></li>
</ul></li>
</ul>
</section></section>
<section>
<section id="analytical-calculations-of-power"
class="title-slide slide level1">
<h1>Analytical calculations of power</h1>
</section>
<section id="analytical-calculations-of-power-1" class="slide level2">
<h2>Analytical calculations of power</h2>
<ul>
<li><p>Formula: <span class="math display">\[\begin{align*}
\text{Power} &= \Phi\left(\frac{|\tau| \sqrt{N}}{2\sigma}-
\Phi^{-1}(1- \frac{\alpha}{2})\right)
\end{align*}\]</span></p></li>
<li><p>Components:</p>
<ul>
<li><span class="math inline">\(\phi\)</span>: standard normal CDF is
monotonically increasing</li>
<li><span class="math inline">\(\tau\)</span>: the effect size</li>
<li><span class="math inline">\(N\)</span>: the sample size</li>
<li><span class="math inline">\(\sigma\)</span>: the standard deviation
of the outcome</li>
<li><span class="math inline">\(\alpha\)</span>: the significance level
(typically 0.05)</li>
</ul></li>
</ul>
</section>
<section id="example-analytical-calculations-of-power"
class="slide level2">
<h2>Example: Analytical calculations of power</h2>
<div class="sourceCode" id="cb1"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Power for a study with 80 obserations and effect</span></span>
<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="co"># size of 0.25</span></span>
<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="fu">library</span>(pwr)</span>
<span id="cb1-4"><a href="#cb1-4" aria-hidden="true" tabindex="-1"></a><span class="fu">pwr.t.test</span>(</span>
<span id="cb1-5"><a href="#cb1-5" aria-hidden="true" tabindex="-1"></a> <span class="at">n =</span> <span class="dv">40</span>, <span class="at">d =</span> <span class="fl">0.25</span>, <span class="at">sig.level =</span> <span class="fl">0.05</span>,</span>
<span id="cb1-6"><a href="#cb1-6" aria-hidden="true" tabindex="-1"></a> <span class="at">power =</span> <span class="cn">NULL</span>, <span class="at">type =</span> <span class="fu">c</span>(</span>
<span id="cb1-7"><a href="#cb1-7" aria-hidden="true" tabindex="-1"></a> <span class="st">"two.sample"</span>,</span>
<span id="cb1-8"><a href="#cb1-8" aria-hidden="true" tabindex="-1"></a> <span class="st">"one.sample"</span>, <span class="st">"paired"</span></span>
<span id="cb1-9"><a href="#cb1-9" aria-hidden="true" tabindex="-1"></a> )</span>
<span id="cb1-10"><a href="#cb1-10" aria-hidden="true" tabindex="-1"></a>)</span></code></pre></div>
<pre><code>
Two-sample t test power calculation
n = 40
d = 0.25
sig.level = 0.05
power = 0.1972
alternative = two.sided
NOTE: n is number in *each* group</code></pre>
</section>
<section id="limitations-to-analytical-power-calculations"
class="slide level2">
<h2>Limitations to analytical power calculations</h2>
<ul>
<li><p>Only derived for some test statistics (differences of
means)</p></li>
<li><p>Makes specific assumptions about the data-generating
process</p></li>
<li><p>Incompatible with more complex designs</p></li>
</ul>
</section></section>
<section>
<section id="simulation-based-power-calculation"
class="title-slide slide level1">
<h1>Simulation-based power calculation</h1>
</section>
<section id="simulation-based-power-calculation-1" class="slide level2">
<h2>Simulation-based power calculation</h2>
<ul>
<li><p>Create dataset and simulate research design.</p></li>
<li><p>Assumptions are necessary for simulation studies, but you make
your own.</p></li>
<li><p>For the DeclareDesign approach, see <a
href="https://declaredesign.org/"
class="uri">https://declaredesign.org/</a></p></li>
</ul>
</section>
<section id="steps" class="slide level2">
<h2>Steps</h2>
<ul>
<li><p>Define the sample and the potential outcomes function.</p></li>
<li><p>Define the treatment assignment procedure.</p></li>
<li><p>Create data.</p></li>
<li><p>Assign treatment, then estimate the effect.</p></li>
<li><p>Do this many times.</p></li>
</ul>
</section>
<section id="examples" class="slide level2">
<h2>Examples</h2>
<ul>
<li><p>Complete randomization</p></li>
<li><p>With covariates</p></li>
<li><p>With cluster randomization</p></li>
</ul>
</section>
<section id="example-simulation-based-power-for-complete-randomization"
class="slide level2">
<h2>Example: Simulation-based power for complete randomization</h2>
<div class="sourceCode" id="cb3"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># install.packages("randomizr")</span></span>
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="fu">library</span>(randomizr)</span>
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="fu">library</span>(estimatr)</span>
<span id="cb3-4"><a href="#cb3-4" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb3-5"><a href="#cb3-5" aria-hidden="true" tabindex="-1"></a><span class="do">## Y0 is fixed in most field experiments.</span></span>
<span id="cb3-6"><a href="#cb3-6" aria-hidden="true" tabindex="-1"></a><span class="do">## So we only generate it once:</span></span>
<span id="cb3-7"><a href="#cb3-7" aria-hidden="true" tabindex="-1"></a>make_Y0 <span class="ot"><-</span> <span class="cf">function</span>(N) {</span>
<span id="cb3-8"><a href="#cb3-8" aria-hidden="true" tabindex="-1"></a> <span class="fu">rnorm</span>(<span class="at">n =</span> N)</span>
<span id="cb3-9"><a href="#cb3-9" aria-hidden="true" tabindex="-1"></a>}</span>
<span id="cb3-10"><a href="#cb3-10" aria-hidden="true" tabindex="-1"></a>repeat_experiment_and_test <span class="ot"><-</span> <span class="cf">function</span>(N, Y0, tau) {</span>
<span id="cb3-11"><a href="#cb3-11" aria-hidden="true" tabindex="-1"></a> Y1 <span class="ot"><-</span> Y0 <span class="sc">+</span> tau</span>
<span id="cb3-12"><a href="#cb3-12" aria-hidden="true" tabindex="-1"></a> Z <span class="ot"><-</span> <span class="fu">complete_ra</span>(<span class="at">N =</span> N)</span>
<span id="cb3-13"><a href="#cb3-13" aria-hidden="true" tabindex="-1"></a> Yobs <span class="ot"><-</span> Z <span class="sc">*</span> Y1 <span class="sc">+</span> (<span class="dv">1</span> <span class="sc">-</span> Z) <span class="sc">*</span> Y0</span>
<span id="cb3-14"><a href="#cb3-14" aria-hidden="true" tabindex="-1"></a> estimator <span class="ot"><-</span> <span class="fu">lm_robust</span>(Yobs <span class="sc">~</span> Z)</span>
<span id="cb3-15"><a href="#cb3-15" aria-hidden="true" tabindex="-1"></a> pval <span class="ot"><-</span> estimator<span class="sc">$</span>p.value[<span class="dv">2</span>]</span>
<span id="cb3-16"><a href="#cb3-16" aria-hidden="true" tabindex="-1"></a> <span class="fu">return</span>(pval)</span>
<span id="cb3-17"><a href="#cb3-17" aria-hidden="true" tabindex="-1"></a>}</span></code></pre></div>
</section>
<section
id="example-simulation-based-power-for-complete-randomization-1"
class="slide level2">
<h2>Example: Simulation-based power for complete randomization</h2>
<div class="sourceCode" id="cb4"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>power_sim <span class="ot"><-</span> <span class="cf">function</span>(N, tau, sims) {</span>
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a> Y0 <span class="ot"><-</span> <span class="fu">make_Y0</span>(N)</span>
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a> pvals <span class="ot"><-</span> <span class="fu">replicate</span>(</span>
<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a> <span class="at">n =</span> sims,</span>
<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a> <span class="fu">repeat_experiment_and_test</span>(<span class="at">N =</span> N, <span class="at">Y0 =</span> Y0, <span class="at">tau =</span> tau)</span>
<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a> )</span>
<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a> pow <span class="ot"><-</span> <span class="fu">sum</span>(pvals <span class="sc"><</span> .<span class="dv">05</span>) <span class="sc">/</span> sims</span>
<span id="cb4-8"><a href="#cb4-8" aria-hidden="true" tabindex="-1"></a> <span class="fu">return</span>(pow)</span>
<span id="cb4-9"><a href="#cb4-9" aria-hidden="true" tabindex="-1"></a>}</span>
<span id="cb4-10"><a href="#cb4-10" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb4-11"><a href="#cb4-11" aria-hidden="true" tabindex="-1"></a><span class="fu">set.seed</span>(<span class="dv">12345</span>)</span>
<span id="cb4-12"><a href="#cb4-12" aria-hidden="true" tabindex="-1"></a><span class="fu">power_sim</span>(<span class="at">N =</span> <span class="dv">80</span>, <span class="at">tau =</span> .<span class="dv">25</span>, <span class="at">sims =</span> <span class="dv">100</span>)</span></code></pre></div>
<pre><code>[1] 0.15</code></pre>
<div class="sourceCode" id="cb6"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="fu">power_sim</span>(<span class="at">N =</span> <span class="dv">80</span>, <span class="at">tau =</span> .<span class="dv">25</span>, <span class="at">sims =</span> <span class="dv">100</span>)</span></code></pre></div>
<pre><code>[1] 0.21</code></pre>
</section>
<section id="example-using-declaredesign"
class="slide level2 allowframebreaks">
<h2 class="allowframebreaks">Example: Using DeclareDesign</h2>
<div class="sourceCode" id="cb8"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="fu">library</span>(DeclareDesign)</span>
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a><span class="fu">library</span>(tidyverse)</span>
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a>P0 <span class="ot"><-</span> <span class="fu">declare_population</span>(N, <span class="at">u0 =</span> <span class="fu">rnorm</span>(N))</span>
<span id="cb8-4"><a href="#cb8-4" aria-hidden="true" tabindex="-1"></a><span class="co"># declare Y(Z=1) and Y(Z=0)</span></span>
<span id="cb8-5"><a href="#cb8-5" aria-hidden="true" tabindex="-1"></a>O0 <span class="ot"><-</span> <span class="fu">declare_potential_outcomes</span>(<span class="at">Y_Z_0 =</span> <span class="dv">5</span> <span class="sc">+</span> u0, <span class="at">Y_Z_1 =</span> Y_Z_0 <span class="sc">+</span> tau)</span>
<span id="cb8-6"><a href="#cb8-6" aria-hidden="true" tabindex="-1"></a><span class="co"># design is to assign m units to treatment</span></span>
<span id="cb8-7"><a href="#cb8-7" aria-hidden="true" tabindex="-1"></a>A0 <span class="ot"><-</span> <span class="fu">declare_assignment</span>(<span class="at">Z =</span> <span class="fu">conduct_ra</span>(<span class="at">N =</span> N, <span class="at">m =</span> <span class="fu">round</span>(N <span class="sc">/</span> <span class="dv">2</span>)))</span>
<span id="cb8-8"><a href="#cb8-8" aria-hidden="true" tabindex="-1"></a><span class="co"># estimand is the average difference between Y(Z=1) and Y(Z=0)</span></span>
<span id="cb8-9"><a href="#cb8-9" aria-hidden="true" tabindex="-1"></a>estimand_ate <span class="ot"><-</span> <span class="fu">declare_inquiry</span>(<span class="at">ATE =</span> <span class="fu">mean</span>(Y_Z_1 <span class="sc">-</span> Y_Z_0))</span>
<span id="cb8-10"><a href="#cb8-10" aria-hidden="true" tabindex="-1"></a>R0 <span class="ot"><-</span> <span class="fu">declare_reveal</span>(Y, Z)</span>
<span id="cb8-11"><a href="#cb8-11" aria-hidden="true" tabindex="-1"></a>design0_base <span class="ot"><-</span> P0 <span class="sc">+</span> A0 <span class="sc">+</span> O0 <span class="sc">+</span> R0</span>
<span id="cb8-12"><a href="#cb8-12" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb8-13"><a href="#cb8-13" aria-hidden="true" tabindex="-1"></a><span class="do">## For example:</span></span>
<span id="cb8-14"><a href="#cb8-14" aria-hidden="true" tabindex="-1"></a>design0_N100_tau25 <span class="ot"><-</span> <span class="fu">redesign</span>(design0_base, <span class="at">N =</span> <span class="dv">100</span>, <span class="at">tau =</span> .<span class="dv">25</span>)</span>
<span id="cb8-15"><a href="#cb8-15" aria-hidden="true" tabindex="-1"></a>dat0_N100_tau25 <span class="ot"><-</span> <span class="fu">draw_data</span>(design0_N100_tau25)</span>
<span id="cb8-16"><a href="#cb8-16" aria-hidden="true" tabindex="-1"></a><span class="fu">head</span>(dat0_N100_tau25)</span></code></pre></div>
<pre><code> ID u0 Z Y_Z_0 Y_Z_1 Y
1 001 -0.2060 0 4.794 5.044 4.794
2 002 -0.5875 0 4.413 4.663 4.413
3 003 -0.2908 1 4.709 4.959 4.959
4 004 -2.5649 0 2.435 2.685 2.435
5 005 -1.8967 0 3.103 3.353 3.103
6 006 -1.6401 1 3.360 3.610 3.610</code></pre>
<div class="sourceCode" id="cb10"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="fu">with</span>(dat0_N100_tau25, <span class="fu">mean</span>(Y_Z_1 <span class="sc">-</span> Y_Z_0)) <span class="co"># true ATE</span></span></code></pre></div>
<pre><code>[1] 0.25</code></pre>
<div class="sourceCode" id="cb12"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="fu">with</span>(dat0_N100_tau25, <span class="fu">mean</span>(Y[Z <span class="sc">==</span> <span class="dv">1</span>]) <span class="sc">-</span> <span class="fu">mean</span>(Y[Z <span class="sc">==</span> <span class="dv">0</span>])) <span class="co"># estimate</span></span></code></pre></div>
<pre><code>[1] 0.5569</code></pre>
<div class="sourceCode" id="cb14"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb14-1"><a href="#cb14-1" aria-hidden="true" tabindex="-1"></a><span class="fu">lm_robust</span>(Y <span class="sc">~</span> Z, <span class="at">data =</span> dat0_N100_tau25)<span class="sc">$</span>coef <span class="co"># estimate</span></span></code></pre></div>
<pre><code>(Intercept) Z
4.8458 0.5569 </code></pre>
<div class="sourceCode" id="cb16"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb16-1"><a href="#cb16-1" aria-hidden="true" tabindex="-1"></a>E0 <span class="ot"><-</span> <span class="fu">declare_estimator</span>(Y <span class="sc">~</span> Z,</span>
<span id="cb16-2"><a href="#cb16-2" aria-hidden="true" tabindex="-1"></a> <span class="at">model =</span> lm_robust, <span class="at">label =</span> <span class="st">"t test 1"</span>,</span>
<span id="cb16-3"><a href="#cb16-3" aria-hidden="true" tabindex="-1"></a> <span class="at">inquiry =</span> <span class="st">"ATE"</span></span>
<span id="cb16-4"><a href="#cb16-4" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb16-5"><a href="#cb16-5" aria-hidden="true" tabindex="-1"></a>t_test <span class="ot"><-</span> <span class="cf">function</span>(data) {</span>
<span id="cb16-6"><a href="#cb16-6" aria-hidden="true" tabindex="-1"></a> test <span class="ot"><-</span> <span class="fu">with</span>(data, <span class="fu">t.test</span>(<span class="at">x =</span> Y[Z <span class="sc">==</span> <span class="dv">1</span>], <span class="at">y =</span> Y[Z <span class="sc">==</span> <span class="dv">0</span>]))</span>
<span id="cb16-7"><a href="#cb16-7" aria-hidden="true" tabindex="-1"></a> <span class="fu">data.frame</span>(<span class="at">statistic =</span> test<span class="sc">$</span>statistic, <span class="at">p.value =</span> test<span class="sc">$</span>p.value)</span>
<span id="cb16-8"><a href="#cb16-8" aria-hidden="true" tabindex="-1"></a>}</span>
<span id="cb16-9"><a href="#cb16-9" aria-hidden="true" tabindex="-1"></a>T0 <span class="ot"><-</span> <span class="fu">declare_test</span>(<span class="at">handler =</span> <span class="fu">label_test</span>(t_test), <span class="at">label =</span> <span class="st">"t test 2"</span>)</span>
<span id="cb16-10"><a href="#cb16-10" aria-hidden="true" tabindex="-1"></a>design0_plus_tests <span class="ot"><-</span> design0_base <span class="sc">+</span> E0 <span class="sc">+</span> T0</span>
<span id="cb16-11"><a href="#cb16-11" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb16-12"><a href="#cb16-12" aria-hidden="true" tabindex="-1"></a>design0_N100_tau25_plus <span class="ot"><-</span> <span class="fu">redesign</span>(design0_plus_tests, <span class="at">N =</span> <span class="dv">100</span>, <span class="at">tau =</span> .<span class="dv">25</span>)</span>
<span id="cb16-13"><a href="#cb16-13" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb16-14"><a href="#cb16-14" aria-hidden="true" tabindex="-1"></a><span class="do">## Only repeat the random assignment, not the creation of Y0. Ignore warning</span></span>
<span id="cb16-15"><a href="#cb16-15" aria-hidden="true" tabindex="-1"></a><span class="fu">names</span>(design0_N100_tau25_plus)</span></code></pre></div>
<pre><code>[1] "P0" "A0" "O0" "R0" "t test 1" "t test 2"</code></pre>
<div class="sourceCode" id="cb18"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb18-1"><a href="#cb18-1" aria-hidden="true" tabindex="-1"></a>design0_N100_tau25_sims <span class="ot"><-</span> <span class="fu">simulate_design</span>(design0_N100_tau25_plus,</span>
<span id="cb18-2"><a href="#cb18-2" aria-hidden="true" tabindex="-1"></a> <span class="at">sims =</span> <span class="fu">c</span>(<span class="dv">1</span>, <span class="dv">100</span>, <span class="dv">1</span>, <span class="dv">1</span>, <span class="dv">1</span>, <span class="dv">1</span>)</span>
<span id="cb18-3"><a href="#cb18-3" aria-hidden="true" tabindex="-1"></a>) <span class="co"># only repeat the random assignment</span></span></code></pre></div>
<pre><code>Warning: We recommend you choose a higher number of simulations than 1 for the top level of simulation.</code></pre>
<div class="sourceCode" id="cb20"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb20-1"><a href="#cb20-1" aria-hidden="true" tabindex="-1"></a><span class="co"># design0_N100_tau25_sims has 200 rows (2 tests * 100 random assignments)</span></span>
<span id="cb20-2"><a href="#cb20-2" aria-hidden="true" tabindex="-1"></a><span class="co"># just look at the first 6 rows</span></span>
<span id="cb20-3"><a href="#cb20-3" aria-hidden="true" tabindex="-1"></a><span class="fu">head</span>(design0_N100_tau25_sims)</span></code></pre></div>
<pre><code> design N tau sim_ID estimator term estimate std.error statistic p.value conf.low conf.high df outcome inquiry
1 design0_N100_tau25_plus 100 0.25 1 t test 1 Z 0.1108 0.2150 0.5153 0.60752 -0.3158 0.5374 98 Y ATE
2 design0_N100_tau25_plus 100 0.25 1 t test 2 <NA> NA NA 0.5153 0.60754 NA NA NA <NA> <NA>
3 design0_N100_tau25_plus 100 0.25 2 t test 1 Z 0.2458 0.2154 1.1411 0.25661 -0.1817 0.6733 98 Y ATE
4 design0_N100_tau25_plus 100 0.25 2 t test 2 <NA> NA NA 1.1411 0.25662 NA NA NA <NA> <NA>
5 design0_N100_tau25_plus 100 0.25 3 t test 1 Z 0.5463 0.2133 2.5608 0.01197 0.1229 0.9697 98 Y ATE
6 design0_N100_tau25_plus 100 0.25 3 t test 2 <NA> NA NA 2.5608 0.01203 NA NA NA <NA> <NA>
step_1_draw step_2_draw
1 1 1
2 1 1
3 1 2
4 1 2
5 1 3
6 1 3</code></pre>
<div class="sourceCode" id="cb22"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb22-1"><a href="#cb22-1" aria-hidden="true" tabindex="-1"></a><span class="co"># for each estimator, power = proportion of simulations with p.value < 0.5</span></span>
<span id="cb22-2"><a href="#cb22-2" aria-hidden="true" tabindex="-1"></a>design0_N100_tau25_sims <span class="sc">%>%</span></span>
<span id="cb22-3"><a href="#cb22-3" aria-hidden="true" tabindex="-1"></a> <span class="fu">group_by</span>(estimator) <span class="sc">%>%</span></span>
<span id="cb22-4"><a href="#cb22-4" aria-hidden="true" tabindex="-1"></a> <span class="fu">summarize</span>(<span class="at">pow =</span> <span class="fu">mean</span>(p.value <span class="sc"><</span> .<span class="dv">05</span>), <span class="at">.groups =</span> <span class="st">"drop"</span>)</span></code></pre></div>
<pre><code># A tibble: 2 × 2
estimator pow
<chr> <dbl>
1 t test 1 0.2
2 t test 2 0.2</code></pre>
</section></section>
<section>
<section id="power-with-covariate-adjustment"
class="title-slide slide level1">
<h1>Power with covariate adjustment</h1>
</section>
<section id="covariate-adjustment-and-power" class="slide level2">
<h2>Covariate adjustment and power</h2>
<ul>
<li><p>Covariate adjustment can improve power because it mops up
variation in the outcome variable.</p>
<ul>
<li><p>If prognostic, covariate adjustment can reduce variance
dramatically. Lower variance means higher power.</p></li>
<li><p>If non-prognostic, power gains are minimal.</p></li>
</ul></li>
<li><p>All covariates must be pre-treatment. Do not drop observations on
account of missingness.</p>
<ul>
<li>See the module on <a
href="threats-to-internal-validity-of-randomized-experiments.html">threats
to internal validity</a> and the <a
href="https://egap.org/resource/10-things-to-know-about-covariate-adjustment/">10
things to know about covariate adjustment</a>.</li>
</ul></li>
<li><p>Freedman’s bias as n of observations decreases and K covariates
increases.</p></li>
</ul>
<!-- ## Covariate adjustment: best practices -->
<!-- - All covariates must be pre-treatment -->
<!-- - Never adjust for post-treatment variables -->
<!-- - In practice, if all controls are pre-treatment, you can add whatever controls you want -->
<!-- - But there is a limit to the number -->
<!-- - Also see -->
<!-- - Missingness in pre-treatment covariates -->
<!-- - Do not drop observations on account of pre-treatment missingness -->
<!-- - Impute mean/median for pretreatment variable -->
<!-- - Include missingness indicator and impute some value in the missing variable -->
</section>
<section id="blocking" class="slide level2">
<h2>Blocking</h2>
<ul>
<li><p>Blocking: randomly assign treatment within blocks</p>
<ul>
<li><p>“Ex-ante” covariate adjustment</p></li>
<li><p>Higher precision/efficiency implies more power</p></li>
<li><p>Reduce “conditional bias”: association between treatment
assignment and potential outcomes</p></li>
<li><p>Benefits of blocking over covariate adjustment clearest in small
experiments</p></li>
</ul></li>
</ul>
</section>
<section id="example-simulation-based-power-with-a-covariate"
class="slide level2 allowframebreaks">
<h2 class="allowframebreaks">Example: Simulation-based power with a
covariate</h2>
<div class="sourceCode" id="cb24"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb24-1"><a href="#cb24-1" aria-hidden="true" tabindex="-1"></a><span class="do">## Y0 is fixed in most field experiments. So we only generate it once</span></span>
<span id="cb24-2"><a href="#cb24-2" aria-hidden="true" tabindex="-1"></a>make_Y0_cov <span class="ot"><-</span> <span class="cf">function</span>(N) {</span>
<span id="cb24-3"><a href="#cb24-3" aria-hidden="true" tabindex="-1"></a> u0 <span class="ot"><-</span> <span class="fu">rnorm</span>(<span class="at">n =</span> N)</span>
<span id="cb24-4"><a href="#cb24-4" aria-hidden="true" tabindex="-1"></a> x <span class="ot"><-</span> <span class="fu">rpois</span>(<span class="at">n =</span> N, <span class="at">lambda =</span> <span class="dv">2</span>)</span>
<span id="cb24-5"><a href="#cb24-5" aria-hidden="true" tabindex="-1"></a> Y0 <span class="ot"><-</span> .<span class="dv">5</span> <span class="sc">*</span> <span class="fu">sd</span>(u0) <span class="sc">*</span> x <span class="sc">+</span> u0</span>
<span id="cb24-6"><a href="#cb24-6" aria-hidden="true" tabindex="-1"></a> <span class="fu">return</span>(<span class="fu">data.frame</span>(<span class="at">Y0 =</span> Y0, <span class="at">x =</span> x))</span>
<span id="cb24-7"><a href="#cb24-7" aria-hidden="true" tabindex="-1"></a>}</span>
<span id="cb24-8"><a href="#cb24-8" aria-hidden="true" tabindex="-1"></a><span class="do">## X is moderarely predictive of Y0.</span></span>
<span id="cb24-9"><a href="#cb24-9" aria-hidden="true" tabindex="-1"></a>test_dat <span class="ot"><-</span> <span class="fu">make_Y0_cov</span>(<span class="dv">100</span>)</span>
<span id="cb24-10"><a href="#cb24-10" aria-hidden="true" tabindex="-1"></a>test_lm <span class="ot"><-</span> <span class="fu">lm_robust</span>(Y0 <span class="sc">~</span> x, <span class="at">data =</span> test_dat)</span>
<span id="cb24-11"><a href="#cb24-11" aria-hidden="true" tabindex="-1"></a><span class="fu">summary</span>(test_lm)</span></code></pre></div>
<pre><code>
Call:
lm_robust(formula = Y0 ~ x, data = test_dat)
Standard error type: HC2
Coefficients:
Estimate Std. Error t value Pr(>|t|) CI Lower CI Upper DF
(Intercept) 0.11 0.1880 0.585 0.559753653 -0.263 0.483 98
x 0.44 0.0814 5.413 0.000000441 0.279 0.602 98
Multiple R-squared: 0.231 , Adjusted R-squared: 0.223
F-statistic: 29.3 on 1 and 98 DF, p-value: 0.000000441</code></pre>
<div class="sourceCode" id="cb26"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb26-1"><a href="#cb26-1" aria-hidden="true" tabindex="-1"></a><span class="do">## now set up the simulation</span></span>
<span id="cb26-2"><a href="#cb26-2" aria-hidden="true" tabindex="-1"></a>repeat_experiment_and_test_cov <span class="ot"><-</span> <span class="cf">function</span>(N, tau, Y0, x) {</span>
<span id="cb26-3"><a href="#cb26-3" aria-hidden="true" tabindex="-1"></a> Y1 <span class="ot"><-</span> Y0 <span class="sc">+</span> tau</span>
<span id="cb26-4"><a href="#cb26-4" aria-hidden="true" tabindex="-1"></a> Z <span class="ot"><-</span> <span class="fu">complete_ra</span>(<span class="at">N =</span> N)</span>
<span id="cb26-5"><a href="#cb26-5" aria-hidden="true" tabindex="-1"></a> Yobs <span class="ot"><-</span> Z <span class="sc">*</span> Y1 <span class="sc">+</span> (<span class="dv">1</span> <span class="sc">-</span> Z) <span class="sc">*</span> Y0</span>
<span id="cb26-6"><a href="#cb26-6" aria-hidden="true" tabindex="-1"></a> estimator <span class="ot"><-</span> <span class="fu">lm_robust</span>(Yobs <span class="sc">~</span> Z <span class="sc">+</span> x, <span class="at">data =</span> <span class="fu">data.frame</span>(Y0, Z, x))</span>
<span id="cb26-7"><a href="#cb26-7" aria-hidden="true" tabindex="-1"></a> pval <span class="ot"><-</span> estimator<span class="sc">$</span>p.value[<span class="dv">2</span>]</span>
<span id="cb26-8"><a href="#cb26-8" aria-hidden="true" tabindex="-1"></a> <span class="fu">return</span>(pval)</span>
<span id="cb26-9"><a href="#cb26-9" aria-hidden="true" tabindex="-1"></a>}</span>
<span id="cb26-10"><a href="#cb26-10" aria-hidden="true" tabindex="-1"></a><span class="do">## create the data once, randomly assign treatment sims times</span></span>
<span id="cb26-11"><a href="#cb26-11" aria-hidden="true" tabindex="-1"></a><span class="do">## report what proportion return p-value < 0.05</span></span>
<span id="cb26-12"><a href="#cb26-12" aria-hidden="true" tabindex="-1"></a>power_sim_cov <span class="ot"><-</span> <span class="cf">function</span>(N, tau, sims) {</span>
<span id="cb26-13"><a href="#cb26-13" aria-hidden="true" tabindex="-1"></a> dat <span class="ot"><-</span> <span class="fu">make_Y0_cov</span>(N)</span>
<span id="cb26-14"><a href="#cb26-14" aria-hidden="true" tabindex="-1"></a> pvals <span class="ot"><-</span> <span class="fu">replicate</span>(<span class="at">n =</span> sims, <span class="fu">repeat_experiment_and_test_cov</span>(</span>
<span id="cb26-15"><a href="#cb26-15" aria-hidden="true" tabindex="-1"></a> <span class="at">N =</span> N,</span>
<span id="cb26-16"><a href="#cb26-16" aria-hidden="true" tabindex="-1"></a> <span class="at">tau =</span> tau, <span class="at">Y0 =</span> dat<span class="sc">$</span>Y0, <span class="at">x =</span> dat<span class="sc">$</span>x</span>
<span id="cb26-17"><a href="#cb26-17" aria-hidden="true" tabindex="-1"></a> ))</span>
<span id="cb26-18"><a href="#cb26-18" aria-hidden="true" tabindex="-1"></a> pow <span class="ot"><-</span> <span class="fu">sum</span>(pvals <span class="sc"><</span> .<span class="dv">05</span>) <span class="sc">/</span> sims</span>
<span id="cb26-19"><a href="#cb26-19" aria-hidden="true" tabindex="-1"></a> <span class="fu">return</span>(pow)</span>
<span id="cb26-20"><a href="#cb26-20" aria-hidden="true" tabindex="-1"></a>}</span></code></pre></div>
<div class="sourceCode" id="cb27"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb27-1"><a href="#cb27-1" aria-hidden="true" tabindex="-1"></a><span class="fu">set.seed</span>(<span class="dv">12345</span>)</span>
<span id="cb27-2"><a href="#cb27-2" aria-hidden="true" tabindex="-1"></a><span class="fu">power_sim_cov</span>(<span class="at">N =</span> <span class="dv">80</span>, <span class="at">tau =</span> .<span class="dv">25</span>, <span class="at">sims =</span> <span class="dv">100</span>)</span></code></pre></div>
<pre><code>[1] 0.13</code></pre>
<div class="sourceCode" id="cb29"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb29-1"><a href="#cb29-1" aria-hidden="true" tabindex="-1"></a><span class="fu">power_sim_cov</span>(<span class="at">N =</span> <span class="dv">80</span>, <span class="at">tau =</span> .<span class="dv">25</span>, <span class="at">sims =</span> <span class="dv">100</span>)</span></code></pre></div>
<pre><code>[1] 0.19</code></pre>
</section></section>
<section>
<section id="power-for-cluster-randomization"
class="title-slide slide level1">
<h1>Power for cluster randomization</h1>
</section>
<section id="power-and-clustered-designs" class="slide level2">
<h2>Power and clustered designs</h2>
<ul>
<li><p>Recall the <a href="randomization.html">randomization
module</a>.</p></li>
<li><p>Given a fixed <span class="math inline">\(N\)</span>, a clustered
design is weakly less powered than a non-clustered design.</p>
<ul>
<li>The difference is often substantial.</li>
</ul></li>
<li><p>We have to estimate variance correctly:</p>
<ul>
<li>Clustering standard errors (the usual)</li>
<li>Randomization inference</li>
</ul></li>
<li><p>To increase power:</p>
<ul>
<li>Better to increase number of clusters than number of units per
cluster.</li>
<li>How much clusters reduce power depends critically on the
intra-cluster correlation (the ratio of variance within clusters to
total variance).</li>
</ul></li>
</ul>
</section>
<section id="a-note-on-clustering-in-observational-research"
class="slide level2">
<h2>A note on clustering in observational research</h2>
<ul>
<li><p>Often overlooked, leading to (possibly) wildly understated
uncertainty.</p>
<ul>
<li><p>Frequentist inference based on ratio <span
class="math inline">\(\hat{\beta}/\hat{se}\)</span></p></li>
<li><p>If we underestimate <span
class="math inline">\(\hat{se}\)</span>, we are much more likely to
reject <span class="math inline">\(H_0\)</span>. (Type-I error rate is
too high.)</p></li>
</ul></li>
<li><p>Many observational designs much less powered than we think they
are.</p></li>
</ul>
</section>
<section id="example-simulation-based-power-for-cluster-randomization"
class="slide level2 allowframebreaks">
<h2 class="allowframebreaks">Example: Simulation-based power for cluster
randomization</h2>
<div class="sourceCode" id="cb31"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb31-1"><a href="#cb31-1" aria-hidden="true" tabindex="-1"></a><span class="do">## Y0 is fixed in most field experiments. So we only generate it once</span></span>
<span id="cb31-2"><a href="#cb31-2" aria-hidden="true" tabindex="-1"></a>make_Y0_clus <span class="ot"><-</span> <span class="cf">function</span>(n_indivs, n_clus) {</span>
<span id="cb31-3"><a href="#cb31-3" aria-hidden="true" tabindex="-1"></a> <span class="co"># n_indivs is number of people per cluster</span></span>
<span id="cb31-4"><a href="#cb31-4" aria-hidden="true" tabindex="-1"></a> <span class="co"># n_clus is number of clusters</span></span>
<span id="cb31-5"><a href="#cb31-5" aria-hidden="true" tabindex="-1"></a> clus_id <span class="ot"><-</span> <span class="fu">gl</span>(n_clus, n_indivs)</span>
<span id="cb31-6"><a href="#cb31-6" aria-hidden="true" tabindex="-1"></a> N <span class="ot"><-</span> n_clus <span class="sc">*</span> n_indivs</span>
<span id="cb31-7"><a href="#cb31-7" aria-hidden="true" tabindex="-1"></a> u0 <span class="ot"><-</span> fabricatr<span class="sc">::</span><span class="fu">draw_normal_icc</span>(<span class="at">N =</span> N, <span class="at">clusters =</span> clus_id, <span class="at">ICC =</span> .<span class="dv">1</span>)</span>
<span id="cb31-8"><a href="#cb31-8" aria-hidden="true" tabindex="-1"></a> Y0 <span class="ot"><-</span> u0</span>
<span id="cb31-9"><a href="#cb31-9" aria-hidden="true" tabindex="-1"></a> <span class="fu">return</span>(<span class="fu">data.frame</span>(<span class="at">Y0 =</span> Y0, <span class="at">clus_id =</span> clus_id))</span>
<span id="cb31-10"><a href="#cb31-10" aria-hidden="true" tabindex="-1"></a>}</span>
<span id="cb31-11"><a href="#cb31-11" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb31-12"><a href="#cb31-12" aria-hidden="true" tabindex="-1"></a>test_dat <span class="ot"><-</span> <span class="fu">make_Y0_clus</span>(<span class="at">n_indivs =</span> <span class="dv">10</span>, <span class="at">n_clus =</span> <span class="dv">100</span>)</span>
<span id="cb31-13"><a href="#cb31-13" aria-hidden="true" tabindex="-1"></a><span class="co"># confirm that this produces data with 10 in each of 100 clusters</span></span>
<span id="cb31-14"><a href="#cb31-14" aria-hidden="true" tabindex="-1"></a><span class="fu">table</span>(test_dat<span class="sc">$</span>clus_id)</span></code></pre></div>
<pre><code>
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10
34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10
100
10 </code></pre>
<div class="sourceCode" id="cb33"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb33-1"><a href="#cb33-1" aria-hidden="true" tabindex="-1"></a><span class="co"># confirm ICC</span></span>
<span id="cb33-2"><a href="#cb33-2" aria-hidden="true" tabindex="-1"></a>ICC<span class="sc">::</span><span class="fu">ICCbare</span>(<span class="at">y =</span> Y0, <span class="at">x =</span> clus_id, <span class="at">data =</span> test_dat)</span></code></pre></div>
<pre><code>[1] 0.09655</code></pre>
<div class="sourceCode" id="cb35"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb35-1"><a href="#cb35-1" aria-hidden="true" tabindex="-1"></a>repeat_experiment_and_test_clus <span class="ot"><-</span> <span class="cf">function</span>(N, tau, Y0, clus_id) {</span>
<span id="cb35-2"><a href="#cb35-2" aria-hidden="true" tabindex="-1"></a> Y1 <span class="ot"><-</span> Y0 <span class="sc">+</span> tau</span>
<span id="cb35-3"><a href="#cb35-3" aria-hidden="true" tabindex="-1"></a> <span class="co"># here we randomize Z at the cluster level</span></span>
<span id="cb35-4"><a href="#cb35-4" aria-hidden="true" tabindex="-1"></a> Z <span class="ot"><-</span> <span class="fu">cluster_ra</span>(<span class="at">clusters =</span> clus_id)</span>
<span id="cb35-5"><a href="#cb35-5" aria-hidden="true" tabindex="-1"></a> Yobs <span class="ot"><-</span> Z <span class="sc">*</span> Y1 <span class="sc">+</span> (<span class="dv">1</span> <span class="sc">-</span> Z) <span class="sc">*</span> Y0</span>
<span id="cb35-6"><a href="#cb35-6" aria-hidden="true" tabindex="-1"></a> estimator <span class="ot"><-</span> <span class="fu">lm_robust</span>(Yobs <span class="sc">~</span> Z,</span>
<span id="cb35-7"><a href="#cb35-7" aria-hidden="true" tabindex="-1"></a> <span class="at">clusters =</span> clus_id,</span>
<span id="cb35-8"><a href="#cb35-8" aria-hidden="true" tabindex="-1"></a> <span class="at">data =</span> <span class="fu">data.frame</span>(Y0, Z, clus_id), <span class="at">se_type =</span> <span class="st">"CR2"</span></span>
<span id="cb35-9"><a href="#cb35-9" aria-hidden="true" tabindex="-1"></a> )</span>
<span id="cb35-10"><a href="#cb35-10" aria-hidden="true" tabindex="-1"></a> pval <span class="ot"><-</span> estimator<span class="sc">$</span>p.value[<span class="dv">2</span>]</span>
<span id="cb35-11"><a href="#cb35-11" aria-hidden="true" tabindex="-1"></a> <span class="fu">return</span>(pval)</span>
<span id="cb35-12"><a href="#cb35-12" aria-hidden="true" tabindex="-1"></a>}</span>
<span id="cb35-13"><a href="#cb35-13" aria-hidden="true" tabindex="-1"></a>power_sim_clus <span class="ot"><-</span> <span class="cf">function</span>(n_indivs, n_clus, tau, sims) {</span>
<span id="cb35-14"><a href="#cb35-14" aria-hidden="true" tabindex="-1"></a> dat <span class="ot"><-</span> <span class="fu">make_Y0_clus</span>(n_indivs, n_clus)</span>
<span id="cb35-15"><a href="#cb35-15" aria-hidden="true" tabindex="-1"></a> N <span class="ot"><-</span> n_indivs <span class="sc">*</span> n_clus</span>
<span id="cb35-16"><a href="#cb35-16" aria-hidden="true" tabindex="-1"></a> <span class="co"># randomize treatment sims times</span></span>
<span id="cb35-17"><a href="#cb35-17" aria-hidden="true" tabindex="-1"></a> pvals <span class="ot"><-</span> <span class="fu">replicate</span>(</span>
<span id="cb35-18"><a href="#cb35-18" aria-hidden="true" tabindex="-1"></a> <span class="at">n =</span> sims,</span>
<span id="cb35-19"><a href="#cb35-19" aria-hidden="true" tabindex="-1"></a> <span class="fu">repeat_experiment_and_test_clus</span>(</span>
<span id="cb35-20"><a href="#cb35-20" aria-hidden="true" tabindex="-1"></a> <span class="at">N =</span> N, <span class="at">tau =</span> tau,</span>
<span id="cb35-21"><a href="#cb35-21" aria-hidden="true" tabindex="-1"></a> <span class="at">Y0 =</span> dat<span class="sc">$</span>Y0, <span class="at">clus_id =</span> dat<span class="sc">$</span>clus_id</span>
<span id="cb35-22"><a href="#cb35-22" aria-hidden="true" tabindex="-1"></a> )</span>
<span id="cb35-23"><a href="#cb35-23" aria-hidden="true" tabindex="-1"></a> )</span>
<span id="cb35-24"><a href="#cb35-24" aria-hidden="true" tabindex="-1"></a> pow <span class="ot"><-</span> <span class="fu">sum</span>(pvals <span class="sc"><</span> .<span class="dv">05</span>) <span class="sc">/</span> sims</span>
<span id="cb35-25"><a href="#cb35-25" aria-hidden="true" tabindex="-1"></a> <span class="fu">return</span>(pow)</span>
<span id="cb35-26"><a href="#cb35-26" aria-hidden="true" tabindex="-1"></a>}</span></code></pre></div>
<div class="sourceCode" id="cb36"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb36-1"><a href="#cb36-1" aria-hidden="true" tabindex="-1"></a><span class="fu">set.seed</span>(<span class="dv">12345</span>)</span>
<span id="cb36-2"><a href="#cb36-2" aria-hidden="true" tabindex="-1"></a><span class="fu">power_sim_clus</span>(<span class="at">n_indivs =</span> <span class="dv">8</span>, <span class="at">n_clus =</span> <span class="dv">100</span>, <span class="at">tau =</span> .<span class="dv">25</span>, <span class="at">sims =</span> <span class="dv">100</span>)</span></code></pre></div>
<pre><code>[1] 0.66</code></pre>
<div class="sourceCode" id="cb38"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb38-1"><a href="#cb38-1" aria-hidden="true" tabindex="-1"></a><span class="fu">power_sim_clus</span>(<span class="at">n_indivs =</span> <span class="dv">8</span>, <span class="at">n_clus =</span> <span class="dv">100</span>, <span class="at">tau =</span> .<span class="dv">25</span>, <span class="at">sims =</span> <span class="dv">100</span>)</span></code></pre></div>
<pre><code>[1] 0.68</code></pre>
</section>
<section
id="example-simulation-based-power-for-cluster-randomization-declaredesign"
class="slide level2 allowframebreaks">
<h2 class="allowframebreaks">Example: Simulation-based power for cluster
randomization (DeclareDesign)</h2>
<div class="sourceCode" id="cb40"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb40-1"><a href="#cb40-1" aria-hidden="true" tabindex="-1"></a>P1 <span class="ot"><-</span> <span class="fu">declare_population</span>(</span>
<span id="cb40-2"><a href="#cb40-2" aria-hidden="true" tabindex="-1"></a> <span class="at">N =</span> n_clus <span class="sc">*</span> n_indivs,</span>
<span id="cb40-3"><a href="#cb40-3" aria-hidden="true" tabindex="-1"></a> <span class="at">clusters =</span> <span class="fu">gl</span>(n_clus, n_indivs),</span>
<span id="cb40-4"><a href="#cb40-4" aria-hidden="true" tabindex="-1"></a> <span class="at">u0 =</span> <span class="fu">draw_normal_icc</span>(<span class="at">N =</span> N, <span class="at">clusters =</span> clusters, <span class="at">ICC =</span> .<span class="dv">2</span>)</span>
<span id="cb40-5"><a href="#cb40-5" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb40-6"><a href="#cb40-6" aria-hidden="true" tabindex="-1"></a>O1 <span class="ot"><-</span> <span class="fu">declare_potential_outcomes</span>(<span class="at">Y_Z_0 =</span> <span class="dv">5</span> <span class="sc">+</span> u0, <span class="at">Y_Z_1 =</span> Y_Z_0 <span class="sc">+</span> tau)</span>
<span id="cb40-7"><a href="#cb40-7" aria-hidden="true" tabindex="-1"></a>A1 <span class="ot"><-</span> <span class="fu">declare_assignment</span>(<span class="at">Z =</span> <span class="fu">conduct_ra</span>(<span class="at">N =</span> N, <span class="at">clusters =</span> clusters))</span>
<span id="cb40-8"><a href="#cb40-8" aria-hidden="true" tabindex="-1"></a>estimand_ate <span class="ot"><-</span> <span class="fu">declare_inquiry</span>(<span class="at">ATE =</span> <span class="fu">mean</span>(Y_Z_1 <span class="sc">-</span> Y_Z_0))</span>
<span id="cb40-9"><a href="#cb40-9" aria-hidden="true" tabindex="-1"></a>R1 <span class="ot"><-</span> <span class="fu">declare_reveal</span>(Y, Z)</span>
<span id="cb40-10"><a href="#cb40-10" aria-hidden="true" tabindex="-1"></a>design1_base <span class="ot"><-</span> P1 <span class="sc">+</span> A1 <span class="sc">+</span> O1 <span class="sc">+</span> R1 <span class="sc">+</span> estimand_ate</span>
<span id="cb40-11"><a href="#cb40-11" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb40-12"><a href="#cb40-12" aria-hidden="true" tabindex="-1"></a><span class="do">## For example:</span></span>
<span id="cb40-13"><a href="#cb40-13" aria-hidden="true" tabindex="-1"></a>design1_test <span class="ot"><-</span> <span class="fu">redesign</span>(design1_base, <span class="at">n_clus =</span> <span class="dv">10</span>, <span class="at">n_indivs =</span> <span class="dv">100</span>, <span class="at">tau =</span> .<span class="dv">25</span>)</span>
<span id="cb40-14"><a href="#cb40-14" aria-hidden="true" tabindex="-1"></a>test_d1 <span class="ot"><-</span> <span class="fu">draw_data</span>(design1_test)</span>
<span id="cb40-15"><a href="#cb40-15" aria-hidden="true" tabindex="-1"></a><span class="co"># confirm all individuals in a cluster have the same treatment assignment</span></span>
<span id="cb40-16"><a href="#cb40-16" aria-hidden="true" tabindex="-1"></a><span class="fu">with</span>(test_d1, <span class="fu">table</span>(Z, clusters))</span></code></pre></div>
<pre><code> clusters
Z 1 2 3 4 5 6 7 8 9 10
0 100 0 100 100 100 0 0 100 0 0
1 0 100 0 0 0 100 100 0 100 100</code></pre>
<div class="sourceCode" id="cb42"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb42-1"><a href="#cb42-1" aria-hidden="true" tabindex="-1"></a><span class="co"># three estimators, differ in se_type:</span></span>
<span id="cb42-2"><a href="#cb42-2" aria-hidden="true" tabindex="-1"></a>E1a <span class="ot"><-</span> <span class="fu">declare_estimator</span>(Y <span class="sc">~</span> Z,</span>
<span id="cb42-3"><a href="#cb42-3" aria-hidden="true" tabindex="-1"></a> <span class="at">model =</span> lm_robust, <span class="at">clusters =</span> clusters,</span>
<span id="cb42-4"><a href="#cb42-4" aria-hidden="true" tabindex="-1"></a> <span class="at">se_type =</span> <span class="st">"CR2"</span>, <span class="at">label =</span> <span class="st">"CR2 cluster t test"</span>,</span>
<span id="cb42-5"><a href="#cb42-5" aria-hidden="true" tabindex="-1"></a> <span class="at">inquiry =</span> <span class="st">"ATE"</span></span>
<span id="cb42-6"><a href="#cb42-6" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb42-7"><a href="#cb42-7" aria-hidden="true" tabindex="-1"></a>E1b <span class="ot"><-</span> <span class="fu">declare_estimator</span>(Y <span class="sc">~</span> Z,</span>
<span id="cb42-8"><a href="#cb42-8" aria-hidden="true" tabindex="-1"></a> <span class="at">model =</span> lm_robust, <span class="at">clusters =</span> clusters,</span>
<span id="cb42-9"><a href="#cb42-9" aria-hidden="true" tabindex="-1"></a> <span class="at">se_type =</span> <span class="st">"CR0"</span>, <span class="at">label =</span> <span class="st">"CR0 cluster t test"</span>,</span>
<span id="cb42-10"><a href="#cb42-10" aria-hidden="true" tabindex="-1"></a> <span class="at">inquiry =</span> <span class="st">"ATE"</span></span>
<span id="cb42-11"><a href="#cb42-11" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb42-12"><a href="#cb42-12" aria-hidden="true" tabindex="-1"></a>E1c <span class="ot"><-</span> <span class="fu">declare_estimator</span>(Y <span class="sc">~</span> Z,</span>
<span id="cb42-13"><a href="#cb42-13" aria-hidden="true" tabindex="-1"></a> <span class="at">model =</span> lm_robust, <span class="at">clusters =</span> clusters,</span>
<span id="cb42-14"><a href="#cb42-14" aria-hidden="true" tabindex="-1"></a> <span class="at">se_type =</span> <span class="st">"stata"</span>, <span class="at">label =</span> <span class="st">"stata RCSE t test"</span>,</span>
<span id="cb42-15"><a href="#cb42-15" aria-hidden="true" tabindex="-1"></a> <span class="at">inquiry =</span> <span class="st">"ATE"</span></span>
<span id="cb42-16"><a href="#cb42-16" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb42-17"><a href="#cb42-17" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb42-18"><a href="#cb42-18" aria-hidden="true" tabindex="-1"></a>design1_plus <span class="ot"><-</span> design1_base <span class="sc">+</span> E1a <span class="sc">+</span> E1b <span class="sc">+</span> E1c</span>
<span id="cb42-19"><a href="#cb42-19" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb42-20"><a href="#cb42-20" aria-hidden="true" tabindex="-1"></a>design1_plus_tosim <span class="ot"><-</span> <span class="fu">redesign</span>(design1_plus, <span class="at">n_clus =</span> <span class="dv">10</span>, <span class="at">n_indivs =</span> <span class="dv">100</span>, <span class="at">tau =</span> .<span class="dv">25</span>)</span></code></pre></div>
<div class="sourceCode" id="cb43"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb43-1"><a href="#cb43-1" aria-hidden="true" tabindex="-1"></a><span class="do">## Only repeat the random assignment, not the creation of Y0. Ignore warning</span></span>
<span id="cb43-2"><a href="#cb43-2" aria-hidden="true" tabindex="-1"></a><span class="do">## We would want more simulations in practice.</span></span>
<span id="cb43-3"><a href="#cb43-3" aria-hidden="true" tabindex="-1"></a><span class="fu">set.seed</span>(<span class="dv">12355</span>)</span>
<span id="cb43-4"><a href="#cb43-4" aria-hidden="true" tabindex="-1"></a>design1_sims <span class="ot"><-</span> <span class="fu">simulate_design</span>(design1_plus_tosim,</span>
<span id="cb43-5"><a href="#cb43-5" aria-hidden="true" tabindex="-1"></a> <span class="at">sims =</span> <span class="fu">c</span>(<span class="dv">1</span>, <span class="dv">1000</span>, <span class="fu">rep</span>(<span class="dv">1</span>, <span class="fu">length</span>(design1_plus_tosim) <span class="sc">-</span> <span class="dv">2</span>))</span>
<span id="cb43-6"><a href="#cb43-6" aria-hidden="true" tabindex="-1"></a>)</span></code></pre></div>
<pre><code>Warning: We recommend you choose a higher number of simulations than 1 for the top level of simulation.</code></pre>
<div class="sourceCode" id="cb45"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb45-1"><a href="#cb45-1" aria-hidden="true" tabindex="-1"></a>design1_sims <span class="sc">%>%</span></span>
<span id="cb45-2"><a href="#cb45-2" aria-hidden="true" tabindex="-1"></a> <span class="fu">group_by</span>(estimator) <span class="sc">%>%</span></span>
<span id="cb45-3"><a href="#cb45-3" aria-hidden="true" tabindex="-1"></a> <span class="fu">summarize</span>(</span>
<span id="cb45-4"><a href="#cb45-4" aria-hidden="true" tabindex="-1"></a> <span class="at">pow =</span> <span class="fu">mean</span>(p.value <span class="sc"><</span> .<span class="dv">05</span>),</span>
<span id="cb45-5"><a href="#cb45-5" aria-hidden="true" tabindex="-1"></a> <span class="at">coverage =</span> <span class="fu">mean</span>(estimand <span class="sc"><=</span> conf.high <span class="sc">&</span> estimand <span class="sc">>=</span> conf.low),</span>
<span id="cb45-6"><a href="#cb45-6" aria-hidden="true" tabindex="-1"></a> <span class="at">.groups =</span> <span class="st">"drop"</span></span>
<span id="cb45-7"><a href="#cb45-7" aria-hidden="true" tabindex="-1"></a> )</span></code></pre></div>
<pre><code># A tibble: 3 × 3
estimator pow coverage
<chr> <dbl> <dbl>
1 CR0 cluster t test 0.155 0.911
2 CR2 cluster t test 0.105 0.936
3 stata RCSE t test 0.131 0.918</code></pre>
<div class="sourceCode" id="cb47"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb47-1"><a href="#cb47-1" aria-hidden="true" tabindex="-1"></a><span class="fu">library</span>(DesignLibrary)</span>
<span id="cb47-2"><a href="#cb47-2" aria-hidden="true" tabindex="-1"></a><span class="do">## This may be simpler than the above:</span></span>
<span id="cb47-3"><a href="#cb47-3" aria-hidden="true" tabindex="-1"></a>d1 <span class="ot"><-</span> <span class="fu">block_cluster_two_arm_designer</span>(</span>
<span id="cb47-4"><a href="#cb47-4" aria-hidden="true" tabindex="-1"></a> <span class="at">N_blocks =</span> <span class="dv">1</span>,</span>
<span id="cb47-5"><a href="#cb47-5" aria-hidden="true" tabindex="-1"></a> <span class="at">N_clusters_in_block =</span> <span class="dv">10</span>,</span>
<span id="cb47-6"><a href="#cb47-6" aria-hidden="true" tabindex="-1"></a> <span class="at">N_i_in_cluster =</span> <span class="dv">100</span>,</span>
<span id="cb47-7"><a href="#cb47-7" aria-hidden="true" tabindex="-1"></a> <span class="at">sd_block =</span> <span class="dv">0</span>,</span>
<span id="cb47-8"><a href="#cb47-8" aria-hidden="true" tabindex="-1"></a> <span class="at">sd_cluster =</span> .<span class="dv">3</span>,</span>
<span id="cb47-9"><a href="#cb47-9" aria-hidden="true" tabindex="-1"></a> <span class="at">ate =</span> .<span class="dv">25</span></span>
<span id="cb47-10"><a href="#cb47-10" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb47-11"><a href="#cb47-11" aria-hidden="true" tabindex="-1"></a>d1_plus <span class="ot"><-</span> d1 <span class="sc">+</span> E1b <span class="sc">+</span> E1c</span>
<span id="cb47-12"><a href="#cb47-12" aria-hidden="true" tabindex="-1"></a>d1_sims <span class="ot"><-</span> <span class="fu">simulate_design</span>(d1_plus, <span class="at">sims =</span> <span class="fu">c</span>(<span class="dv">1</span>, <span class="dv">1</span>, <span class="dv">1000</span>, <span class="dv">1</span>, <span class="dv">1</span>, <span class="dv">1</span>, <span class="dv">1</span>, <span class="dv">1</span>))</span></code></pre></div>
<div class="sourceCode" id="cb48"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb48-1"><a href="#cb48-1" aria-hidden="true" tabindex="-1"></a>d1_sims <span class="sc">%>%</span></span>
<span id="cb48-2"><a href="#cb48-2" aria-hidden="true" tabindex="-1"></a> <span class="fu">group_by</span>(estimator) <span class="sc">%>%</span></span>
<span id="cb48-3"><a href="#cb48-3" aria-hidden="true" tabindex="-1"></a> <span class="fu">summarize</span>(</span>
<span id="cb48-4"><a href="#cb48-4" aria-hidden="true" tabindex="-1"></a> <span class="at">pow =</span> <span class="fu">mean</span>(p.value <span class="sc"><</span> .<span class="dv">05</span>),</span>
<span id="cb48-5"><a href="#cb48-5" aria-hidden="true" tabindex="-1"></a> <span class="at">coverage =</span> <span class="fu">mean</span>(estimand <span class="sc"><=</span> conf.high <span class="sc">&</span> estimand <span class="sc">>=</span> conf.low),</span>
<span id="cb48-6"><a href="#cb48-6" aria-hidden="true" tabindex="-1"></a> <span class="at">.groups =</span> <span class="st">"drop"</span></span>
<span id="cb48-7"><a href="#cb48-7" aria-hidden="true" tabindex="-1"></a> )</span></code></pre></div>
<pre><code># A tibble: 3 × 3
estimator pow coverage
<chr> <dbl> <dbl>
1 CR0 cluster t test 0.209 0.914
2 estimator 0.143 0.941
3 stata RCSE t test 0.194 0.925</code></pre>
</section></section>
<section>
<section id="comparative-statics" class="title-slide slide level1">
<h1>Comparative statics</h1>
</section>
<section id="comparative-statics-1" class="slide level2">
<h2>Comparative Statics</h2>
<ul>
<li>Power is:
<ul>
<li>Increasing in <span class="math inline">\(N\)</span></li>
<li>Increasing in <span class="math inline">\(|\tau|\)</span></li>
<li>Decreasing in <span class="math inline">\(\sigma\)</span></li>
</ul></li>
</ul>
</section>
<section id="power-by-sample-size"
class="slide level2 allowframebreaks">
<h2 class="allowframebreaks">Power by sample size</h2>
<div class="sourceCode" id="cb50"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb50-1"><a href="#cb50-1" aria-hidden="true" tabindex="-1"></a>some_ns <span class="ot"><-</span> <span class="fu">seq</span>(<span class="dv">10</span>, <span class="dv">800</span>, <span class="at">by =</span> <span class="dv">10</span>)</span>
<span id="cb50-2"><a href="#cb50-2" aria-hidden="true" tabindex="-1"></a>pow_by_n <span class="ot"><-</span> <span class="fu">sapply</span>(some_ns, <span class="cf">function</span>(then) {</span>
<span id="cb50-3"><a href="#cb50-3" aria-hidden="true" tabindex="-1"></a> <span class="fu">pwr.t.test</span>(<span class="at">n =</span> then, <span class="at">d =</span> <span class="fl">0.25</span>, <span class="at">sig.level =</span> <span class="fl">0.05</span>)<span class="sc">$</span>power</span>
<span id="cb50-4"><a href="#cb50-4" aria-hidden="true" tabindex="-1"></a>})</span>
<span id="cb50-5"><a href="#cb50-5" aria-hidden="true" tabindex="-1"></a><span class="fu">plot</span>(some_ns, pow_by_n,</span>
<span id="cb50-6"><a href="#cb50-6" aria-hidden="true" tabindex="-1"></a> <span class="at">xlab =</span> <span class="st">"Sample Size"</span>,</span>
<span id="cb50-7"><a href="#cb50-7" aria-hidden="true" tabindex="-1"></a> <span class="at">ylab =</span> <span class="st">"Power"</span></span>
<span id="cb50-8"><a href="#cb50-8" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb50-9"><a href="#cb50-9" aria-hidden="true" tabindex="-1"></a><span class="fu">abline</span>(<span class="at">h =</span> .<span class="dv">8</span>)</span></code></pre></div>
<p><img src="figs/figpowplot_by_n-1.png" width=".8\textwidth" /></p>
<div class="sourceCode" id="cb51"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb51-1"><a href="#cb51-1" aria-hidden="true" tabindex="-1"></a><span class="do">## See https://cran.r-project.org/web/packages/pwr/vignettes/pwr-vignette.html</span></span>
<span id="cb51-2"><a href="#cb51-2" aria-hidden="true" tabindex="-1"></a><span class="do">## for fancier plots</span></span>
<span id="cb51-3"><a href="#cb51-3" aria-hidden="true" tabindex="-1"></a><span class="do">## ptest <- pwr.t.test(n = NULL, d = 0.25, sig.level = 0.05, power = .8)</span></span>
<span id="cb51-4"><a href="#cb51-4" aria-hidden="true" tabindex="-1"></a><span class="do">## plot(ptest)</span></span></code></pre></div>
</section>
<section id="power-by-treatment-effect-size"
class="slide level2 allowframebreaks">
<h2 class="allowframebreaks">Power by treatment effect size</h2>
<div class="sourceCode" id="cb52"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb52-1"><a href="#cb52-1" aria-hidden="true" tabindex="-1"></a>some_taus <span class="ot"><-</span> <span class="fu">seq</span>(<span class="dv">0</span>, <span class="dv">1</span>, <span class="at">by =</span> .<span class="dv">05</span>)</span>
<span id="cb52-2"><a href="#cb52-2" aria-hidden="true" tabindex="-1"></a>pow_by_tau <span class="ot"><-</span> <span class="fu">sapply</span>(some_taus, <span class="cf">function</span>(thetau) {</span>
<span id="cb52-3"><a href="#cb52-3" aria-hidden="true" tabindex="-1"></a> <span class="fu">pwr.t.test</span>(<span class="at">n =</span> <span class="dv">200</span>, <span class="at">d =</span> thetau, <span class="at">sig.level =</span> <span class="fl">0.05</span>)<span class="sc">$</span>power</span>
<span id="cb52-4"><a href="#cb52-4" aria-hidden="true" tabindex="-1"></a>})</span>
<span id="cb52-5"><a href="#cb52-5" aria-hidden="true" tabindex="-1"></a><span class="fu">plot</span>(some_taus, pow_by_tau,</span>
<span id="cb52-6"><a href="#cb52-6" aria-hidden="true" tabindex="-1"></a> <span class="at">xlab =</span> <span class="st">"Average Treatment Effect (Standardized)"</span>,</span>
<span id="cb52-7"><a href="#cb52-7" aria-hidden="true" tabindex="-1"></a> <span class="at">ylab =</span> <span class="st">"Power"</span></span>
<span id="cb52-8"><a href="#cb52-8" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb52-9"><a href="#cb52-9" aria-hidden="true" tabindex="-1"></a><span class="fu">abline</span>(<span class="at">h =</span> .<span class="dv">8</span>)</span></code></pre></div>
<p><img src="figs/figpowplot_by_tau-1.png" width=".8\textwidth" /></p>
</section>
<section id="egap-power-calculator" class="slide level2">
<h2>EGAP Power Calculator</h2>
<ul>
<li><p>Try the calculator at: <a
href="https://egap.shinyapps.io/power-app/"
class="uri">https://egap.shinyapps.io/power-app/</a></p></li>
<li><p>For cluster randomization designs, try adjusting:</p>
<ul>
<li>Number of clusters</li>
<li>Number of units per clusters</li>
<li>Intra-cluster correlation</li>
<li>Treatment effect</li>
</ul></li>
</ul>
</section>
<section id="comments" class="slide level2">
<h2>Comments</h2>
<ul>
<li><p>Know your outcome variable.</p></li>
<li><p>What effects can you realistically expect from your
treatment?</p></li>
<li><p>What is the plausible range of variation of the outcome
variable?</p>
<ul>
<li>A design with limited possible movement in the outcome variable may
not be well-powered.</li>
</ul></li>
</ul>
</section>
<section id="conclusion-how-to-improve-your-power" class="slide level2">
<h2>Conclusion: How to improve your power</h2>
<ol type="1">
<li><p>Increase the <span class="math inline">\(N\)</span></p>
<ul>
<li>If clustered, increase the number of clusters if at all
possible</li>
</ul></li>
<li><p>Strengthen the treatment</p></li>
<li><p>Improve precision</p>
<ul>
<li><p>Covariate adjustment</p></li>
<li><p>Blocking</p></li>
</ul></li>
<li><p>Better measurement of the outcome variable</p></li>
</ol>
</section></section>
</div>
</div>
<script src="power-slides_files/reveal.js-3.3.0.1/lib/js/head.min.js"></script>
<script src="power-slides_files/reveal.js-3.3.0.1/js/reveal.js"></script>
<script>
// Full list of configuration options available at:
// https://github.com/hakimel/reveal.js#configuration
Reveal.initialize({
// Display controls in the bottom right corner
controls: true,
// Display a presentation progress bar
progress: true,
// Display the page number of the current slide
slideNumber: true,
// Push each slide change to the browser history
history: true,
// Enable keyboard shortcuts for navigation
keyboard: true,
// Enable the slide overview mode
overview: true,
// Vertical centering of slides
center: false,
// Enables touch navigation on devices with touch input
touch: true,
// Turns fragments on and off globally
fragments: true,
// Flags if we should show a help overlay when the questionmark
// key is pressed
help: true,
// Number of milliseconds between automatically proceeding to the
// next slide, disabled when set to 0, this value can be overwritten
// by using a data-autoslide attribute on your slides
autoSlide: 0,
// Stop auto-sliding after user input
autoSlideStoppable: true,
// Opens links in an iframe preview overlay
previewLinks: true,
// Transition style
transition: 'fade', // none/fade/slide/convex/concave/zoom
// Transition speed
transitionSpeed: 'default', // default/fast/slow
// Transition style for full page slide backgrounds
backgroundTransition: 'default', // none/fade/slide/convex/concave/zoom
// Number of slides away from the current that are visible
viewDistance: 3,
chalkboard: {
toggleNotesButton: false,
theme: 'whiteboard',
},
keyboard: {
67: function() { RevealChalkboard.toggleNotesCanvas() }, // toggle notes canvas when 'c' is pressed
66: function() { RevealChalkboard.toggleChalkboard() }, // toggle chalkboard when 'b' is pressed
46: function() { RevealChalkboard.clear() }, // clear chalkboard when 'DEL' is pressed
8: function() { RevealChalkboard.reset() }, // reset chalkboard data on current slide when 'BACKSPACE' is pressed
68: function() { RevealChalkboard.download() }, // downlad recorded chalkboard drawing when 'd' is pressed
},
// Optional reveal.js plugins
dependencies: [
{ src: 'power-slides_files/reveal.js-3.3.0.1/plugin/notes/notes.js', async: true },
{ src: 'power-slides_files/reveal.js-3.3.0.1/plugin/search/search.js', async: true },
{ src: 'power-slides_files/reveal.js-3.3.0.1/plugin/chalkboard/chalkboard.js', async: true },
]
});
</script>
<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
(function () {
var script = document.createElement("script");
script.type = "text/javascript";
script.src = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
document.getElementsByTagName("head")[0].appendChild(script);
})();
</script>
<script>
(function() {
if (window.jQuery) {
Reveal.addEventListener( 'slidechanged', function(event) {
window.jQuery(event.previousSlide).trigger('hidden');
window.jQuery(event.currentSlide).trigger('shown');
});
}
})();
</script>
</body>
</html>