Errors, Power, Likelihood Ratios, and Multiple Comparisons
// ---------------------------------------------------------------------------
// ldk — a tiny, dependency-free plotting + statistics library for these decks.
//
// Everything runs in the browser as plain JavaScript building SVG nodes, so a
// slider move costs a fraction of a millisecond instead of a round trip
// through a WebAssembly R runtime.
// ---------------------------------------------------------------------------
ldk = {
const NS = "http://www.w3.org/2000/svg";
let uid = 0;
// Deck palette, matching ldkslides.css
const C = {
green: "#74b83d",
blue: "#4072c2",
pink: "#ED017D",
orange: "#d18b2a",
grey: "#6E6E6E",
ink: "#222222",
axis: "#9aa0a6",
faint: "#efefef"
};
function el(tag, attrs, text) {
const e = document.createElementNS(NS, tag);
if (attrs) for (const k in attrs) if (attrs[k] != null) e.setAttribute(k, attrs[k]);
if (text != null)
// SVG collapses runs of whitespace, which would eat the wide gaps that
// separate the readout columns; non-breaking spaces survive. (xml:space
// is a namespaced attribute and does not survive setAttribute here.)
e.textContent = tag === "text"
? String(text).replace(/ {2,}/g, m => " ".repeat(m.length))
: text;
return e;
}
function alpha(hex, a) {
const n = parseInt(hex.slice(1), 16);
return `rgba(${(n >> 16) & 255},${(n >> 8) & 255},${n & 255},${a})`;
}
// ------------------------------------------------------------------ numerics
const LGC = [676.5203681218851, -1259.1392167224028, 771.32342877765313,
-176.61502916214059, 12.507343278686905, -0.13857109526572012,
9.9843695780195716e-6, 1.5056327351493116e-7];
function lgamma(z) {
if (z < 0.5) return Math.log(Math.PI / Math.sin(Math.PI * z)) - lgamma(1 - z);
z -= 1;
let x = 0.99999999999980993;
for (let i = 0; i < 8; i++) x += LGC[i] / (z + i + 1);
const t = z + 7.5;
return 0.5 * Math.log(2 * Math.PI) + (z + 0.5) * Math.log(t) - t + Math.log(x);
}
const lchoose = (n, k) => lgamma(n + 1) - lgamma(k + 1) - lgamma(n - k + 1);
// Chebyshev erfc (Numerical Recipes); plenty of accuracy for plotting.
const ERFC_COF = [
-1.3026537197817094, 6.4196979235649026e-1, 1.9476473204185836e-2,
-9.561514786808631e-3, -9.46595344482036e-4, 3.66839497852761e-4,
4.2523324806907e-5, -2.0278578112534e-5, -1.624290004647e-6,
1.303655835580e-6, 1.5626441722e-8, -8.5238095915e-8,
6.529054439e-9, 5.059343495e-9, -9.91364156e-10, -2.27365122e-10,
9.6467911e-11, 2.394038e-12, -6.886027e-12, 8.94487e-13,
3.13092e-13, -1.12708e-13, 3.81e-16, 7.106e-15];
function erfc(x) {
const z = Math.abs(x), t = 2 / (2 + z), ty = 4 * t - 2;
let d = 0, dd = 0;
for (let j = ERFC_COF.length - 1; j > 0; j--) {
const tmp = d;
d = ty * d - dd + ERFC_COF[j];
dd = tmp;
}
const ans = t * Math.exp(-z * z + 0.5 * (ERFC_COF[0] + ty * d) - dd);
return x >= 0 ? ans : 2 - ans;
}
const dnorm = (x, m = 0, s = 1) => {
const z = (x - m) / s;
return Math.exp(-0.5 * z * z) / (s * Math.sqrt(2 * Math.PI));
};
const pnorm = (x, m = 0, s = 1) => 0.5 * erfc(-(x - m) / (s * Math.SQRT2));
// Acklam's inverse normal CDF (relative error < 1.2e-9)
const QN_A = [-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02,
1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00];
const QN_B = [-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02,
6.680131188771972e+01, -1.328068155288572e+01];
const QN_C = [-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00,
-2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00];
const QN_D = [7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00,
3.754408661907416e+00];
function qnorm(p, m = 0, s = 1) {
if (p <= 0) return -Infinity;
if (p >= 1) return Infinity;
const lo = 0.02425, hi = 1 - lo;
let x, q, r;
if (p < lo) {
q = Math.sqrt(-2 * Math.log(p));
x = (((((QN_C[0] * q + QN_C[1]) * q + QN_C[2]) * q + QN_C[3]) * q + QN_C[4]) * q + QN_C[5]) /
((((QN_D[0] * q + QN_D[1]) * q + QN_D[2]) * q + QN_D[3]) * q + 1);
} else if (p <= hi) {
q = p - 0.5; r = q * q;
x = (((((QN_A[0] * r + QN_A[1]) * r + QN_A[2]) * r + QN_A[3]) * r + QN_A[4]) * r + QN_A[5]) * q /
(((((QN_B[0] * r + QN_B[1]) * r + QN_B[2]) * r + QN_B[3]) * r + QN_B[4]) * r + 1);
} else {
q = Math.sqrt(-2 * Math.log(1 - p));
x = -(((((QN_C[0] * q + QN_C[1]) * q + QN_C[2]) * q + QN_C[3]) * q + QN_C[4]) * q + QN_C[5]) /
((((QN_D[0] * q + QN_D[1]) * q + QN_D[2]) * q + QN_D[3]) * q + 1);
}
return m + s * x;
}
function dbinom(k, n, p) {
if (k < 0 || k > n) return 0;
if (p <= 0) return k === 0 ? 1 : 0;
if (p >= 1) return k === n ? 1 : 0;
return Math.exp(lchoose(n, k) + k * Math.log(p) + (n - k) * Math.log1p(-p));
}
const dpois = (k, lam) => k < 0 ? 0 : Math.exp(-lam + k * Math.log(lam) - lgamma(k + 1));
const dexp = (x, rate) => x < 0 ? 0 : rate * Math.exp(-rate * x);
const pexp = (x, rate) => x <= 0 ? 0 : 1 - Math.exp(-rate * x);
const dunif = (x, a, b) => (x >= a && x <= b) ? 1 / (b - a) : 0;
const punif = (x, a, b) => x <= a ? 0 : x >= b ? 1 : (x - a) / (b - a);
const dlnorm = (x, ml = 0, sl = 1) => x <= 0 ? 0 : dnorm(Math.log(x), ml, sl) / x;
const dchisq1 = (x) => x <= 0 ? 0 : dnorm(Math.sqrt(x)) / Math.sqrt(x);
// ---- regularized lower incomplete gamma P(a,x): series + continued fraction
function gammaP(a, x) {
if (!(x >= 0) || !(a > 0)) return NaN;
if (x === 0) return 0;
const pref = Math.exp(-x + a * Math.log(x) - lgamma(a));
if (x < a + 1) { // series
let ap = a, sum = 1 / a, del = sum;
for (let i = 0; i < 500; i++) {
ap++; del *= x / ap; sum += del;
if (Math.abs(del) < Math.abs(sum) * 1e-16) break;
}
return sum * pref;
}
const TINY = 1e-300; // continued fraction for Q = 1 - P
let b = x + 1 - a, c = 1 / TINY, d = 1 / b, h = d;
for (let i = 1; i <= 500; i++) {
const an = -i * (i - a);
b += 2;
d = an * d + b; if (Math.abs(d) < TINY) d = TINY;
c = b + an / c; if (Math.abs(c) < TINY) c = TINY;
d = 1 / d;
const del = d * c;
h *= del;
if (Math.abs(del - 1) < 1e-16) break;
}
return 1 - pref * h;
}
// ---- regularized incomplete beta I_x(a,b)
function betacf(a, b, x) {
const TINY = 1e-300, qab = a + b, qap = a + 1, qam = a - 1;
let c = 1, d = 1 - qab * x / qap;
if (Math.abs(d) < TINY) d = TINY;
d = 1 / d;
let h = d;
for (let m = 1; m <= 400; m++) {
const m2 = 2 * m;
let aa = m * (b - m) * x / ((qam + m2) * (a + m2));
d = 1 + aa * d; if (Math.abs(d) < TINY) d = TINY;
c = 1 + aa / c; if (Math.abs(c) < TINY) c = TINY;
d = 1 / d; h *= d * c;
aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2));
d = 1 + aa * d; if (Math.abs(d) < TINY) d = TINY;
c = 1 + aa / c; if (Math.abs(c) < TINY) c = TINY;
d = 1 / d;
const del = d * c;
h *= del;
if (Math.abs(del - 1) < 3e-16) break;
}
return h;
}
function betai(a, b, x) {
if (!(x > 0)) return 0;
if (x >= 1) return 1;
const bt = Math.exp(lgamma(a + b) - lgamma(a) - lgamma(b) +
a * Math.log(x) + b * Math.log1p(-x));
return x < (a + 1) / (a + b + 2)
? bt * betacf(a, b, x) / a
: 1 - bt * betacf(b, a, 1 - x) / b;
}
// ---- chi-squared and Student t
const dchisq = (x, k) => x <= 0 ? 0 :
Math.exp((k / 2 - 1) * Math.log(x) - x / 2 - lgamma(k / 2) - (k / 2) * Math.LN2);
const pchisq = (x, k) => x <= 0 ? 0 : gammaP(k / 2, x / 2);
const dt = (x, nu) => Math.exp(lgamma((nu + 1) / 2) - lgamma(nu / 2)
- 0.5 * Math.log(nu * Math.PI) - (nu + 1) / 2 * Math.log1p(x * x / nu));
const pt = (x, nu) => {
const half = 0.5 * betai(nu / 2, 0.5, nu / (nu + x * x));
return x > 0 ? 1 - half : half;
};
// quantiles by bisection — a handful of calls per redraw, so speed is moot
function invert(cdf, p, lo, hi) {
for (let i = 0; i < 90; i++) {
const mid = (lo + hi) / 2;
if (cdf(mid) < p) lo = mid; else hi = mid;
}
return (lo + hi) / 2;
}
const qchisq = (p, k) => invert(x => pchisq(x, k), p, 0, Math.max(60, k * 25));
const qt = (p, nu) => invert(x => pt(x, nu), p, -1e4, 1e4);
// Cumulative sums of a pmf enumerated over 0..n — cheaper and more accurate
// here than a general incomplete-beta / incomplete-gamma implementation.
function cdfFromPmf(pmf) {
const out = new Array(pmf.length);
let s = 0;
for (let i = 0; i < pmf.length; i++) { s += pmf[i]; out[i] = Math.min(s, 1); }
return out;
}
// ----------------------------------------------------------------- sequences
function seq(a, b, n) {
const out = new Array(n);
for (let i = 0; i < n; i++) out[i] = a + (b - a) * i / (n - 1);
return out;
}
const cumsum = (v) => { let s = 0; return v.map(x => (s += x)); };
const sum = (v) => v.reduce((a, b) => a + b, 0);
const mean = (v) => sum(v) / v.length;
function sd(v) {
const m = mean(v);
return Math.sqrt(v.reduce((a, x) => a + (x - m) * (x - m), 0) / (v.length - 1));
}
function quantileSorted(s, p) {
const h = (s.length - 1) * p, i = Math.floor(h);
const j = Math.min(i + 1, s.length - 1);
return s[i] + (h - i) * (s[j] - s[i]);
}
function hist(v, lo, hi, nb) {
const cnt = new Float64Array(nb), w = (hi - lo) / nb;
for (let i = 0; i < v.length; i++) {
const k = Math.floor((v[i] - lo) / w);
if (k >= 0 && k < nb) cnt[k]++;
else if (v[i] === hi) cnt[nb - 1]++;
}
const dens = new Array(nb);
for (let i = 0; i < nb; i++) dens[i] = cnt[i] / (v.length * w);
return { lo, hi, w, nb, dens, count: cnt };
}
// --------------------------------------------------------------------- rngs
// mulberry32: small, fast, seedable — the demos stay reproducible.
function rng(seed) {
let a = seed >>> 0;
const u = () => {
a |= 0; a = (a + 0x6D2B79F5) | 0;
let t = Math.imul(a ^ (a >>> 15), 1 | a);
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
let spare = null;
u.normal = () => { // Marsaglia polar
if (spare !== null) { const v = spare; spare = null; return v; }
let x, y, s;
do { x = 2 * u() - 1; y = 2 * u() - 1; s = x * x + y * y; } while (s >= 1 || s === 0);
const f = Math.sqrt(-2 * Math.log(s) / s);
spare = y * f;
return x * f;
};
u.exp = () => -Math.log(1 - u());
u.cauchy = () => Math.tan(Math.PI * (u() - 0.5));
return u;
}
// Weighted sampling without replacement (Efraimidis–Spirakis exponential race)
function sampleWeighted(n, w, u) {
const N = w.length;
const keys = new Array(N);
for (let i = 0; i < N; i++) keys[i] = w[i] > 0 ? -Math.log(1 - u()) / w[i] : Infinity;
const idx = Array.from({ length: N }, (_, i) => i);
idx.sort((a, b) => keys[a] - keys[b]);
return idx.slice(0, n);
}
// ---------------------------------------------------------------- formatting
const fmt = (v, d = 2) => (Number.isFinite(v) ? v.toFixed(d) : "—");
const signed = (v, d = 2) => (Number.isFinite(v) ? (v >= 0 ? "+" : "") + v.toFixed(d) : "—");
function smart(v) {
if (!Number.isFinite(v)) return "—";
const a = Math.abs(v);
if (a >= 1e5 || (a > 0 && a < 1e-3)) return v.toExponential(1);
return v.toFixed(a >= 100 ? 0 : a >= 10 ? 1 : 2);
}
function nice(lo, hi, n) {
const span = (hi - lo) || Math.abs(hi) || 1;
const raw = span / n;
const mag = Math.pow(10, Math.floor(Math.log10(raw)));
const norm = raw / mag;
const step = mag * (norm < 1.5 ? 1 : norm < 3 ? 2 : norm < 7 ? 5 : 10);
const out = [];
for (let v = Math.ceil(lo / step - 1e-9) * step; v <= hi + 1e-9 * span; v += step)
out.push(Math.abs(v) < step * 1e-9 ? 0 : v);
return { ticks: out, step };
}
function logTicks(lo, hi) {
const decades = Math.log10(hi / lo);
const mults = decades > 2.2 ? [1] : decades > 1.1 ? [1, 3] : [1, 2, 5];
const out = [];
for (let e = Math.floor(Math.log10(lo)); e <= Math.ceil(Math.log10(hi)); e++)
for (const k of mults) {
const v = k * Math.pow(10, e);
if (v >= lo * 0.9999 && v <= hi * 1.0001) out.push(v);
}
return out;
}
const decimalsFor = (step) =>
Math.max(0, Math.min(6, -Math.floor(Math.log10(Math.abs(step)) + 1e-9)));
// -------------------------------------------------------------------- charts
function chart(W, H, extra) {
const svg = el("svg", Object.assign({
viewBox: `0 0 ${W} ${H}`,
style: `width:100%;height:auto;display:block;font-family:inherit;color:${C.ink}`
}, extra || {}));
svg.__W = W;
svg.__H = H;
return svg;
}
function panel(svg, o) {
const L = o.left ?? 0, T = o.top ?? 0;
const W = o.w ?? svg.__W, H = o.h ?? svg.__H;
const m = Object.assign({ l: 74, r: 22, t: 52, b: 54 }, o.margin || {});
const fs = o.fontSize ?? 15;
const iw = W - m.l - m.r, ih = H - m.t - m.b;
const px0 = L + m.l, py0 = T + m.t;
let [xa, xb] = o.xdom, [ya, yb] = o.ydom;
if (!(xb > xa)) xb = xa + 1;
if (!(yb > ya)) yb = ya + 1;
const lg = o.xlog ? Math.log : (v => v);
const xA = lg(xa), xB = lg(xb);
const x = v => px0 + (lg(v) - xA) / (xB - xA) * iw;
const y = v => py0 + ih - (v - ya) / (yb - ya) * ih;
const gAxis = el("g");
const id = `ldkclip${++uid}`;
const defs = el("defs");
const cp = el("clipPath", { id });
cp.append(el("rect", { x: px0 - 3, y: py0 - 8, width: iw + 6, height: ih + 11 }));
defs.append(cp);
const gc = el("g", { "clip-path": `url(#${id})` });
const gt = el("g");
svg.append(defs, gAxis, gc, gt);
// ticks
let xt, xfmt, yt, yfmt;
if (o.xticks) {
xt = o.xticks;
xfmt = o.xfmt || (v => String(v));
} else if (o.xlog) {
xt = logTicks(xa, xb);
xfmt = o.xfmt || (v => v >= 1 ? String(Math.round(v)) : String(v));
} else {
const n = nice(xa, xb, o.nx ?? 6);
xt = n.ticks;
xfmt = o.xfmt || (v => v.toFixed(decimalsFor(n.step)));
}
if (o.yticks) {
yt = o.yticks;
yfmt = o.yfmt || (v => String(v));
} else {
const n = nice(ya, yb, o.ny ?? 5);
yt = n.ticks;
yfmt = o.yfmt || (v => v.toFixed(decimalsFor(n.step)));
}
if (o.axes === false) { xt = []; yt = []; } // bare drawing surface
if (o.axes !== false) {
gAxis.append(el("line", { x1: px0, y1: py0 + ih, x2: px0 + iw, y2: py0 + ih,
stroke: C.axis, "stroke-width": 1.3 }));
gAxis.append(el("line", { x1: px0, y1: py0, x2: px0, y2: py0 + ih,
stroke: C.axis, "stroke-width": 1.3 }));
}
for (const t of xt) {
const px = x(t);
if (px < px0 - 0.5 || px > px0 + iw + 0.5) continue;
gAxis.append(el("line", { x1: px, y1: py0 + ih, x2: px, y2: py0 + ih + 5,
stroke: C.axis, "stroke-width": 1.3 }));
gAxis.append(el("text", { x: px, y: py0 + ih + 6 + fs, "text-anchor": "middle",
"font-size": fs, fill: C.grey }, xfmt(t)));
}
for (const t of yt) {
const py = y(t);
if (py < py0 - 0.5 || py > py0 + ih + 0.5) continue;
gAxis.append(el("line", { x1: px0 - 5, y1: py, x2: px0, y2: py,
stroke: C.axis, "stroke-width": 1.3 }));
gAxis.append(el("text", { x: px0 - 10, y: py + fs * 0.35, "text-anchor": "end",
"font-size": fs, fill: C.grey }, yfmt(t)));
}
if (o.xlab)
gAxis.append(el("text", { x: px0 + iw / 2, y: T + H - 8, "text-anchor": "middle",
"font-size": fs + 2, fill: C.ink }, o.xlab));
if (o.ylab) {
const ty = py0 + ih / 2, tx = L + 20;
gAxis.append(el("text", { x: tx, y: ty, "text-anchor": "middle", "font-size": fs + 2,
fill: C.ink, transform: `rotate(-90 ${tx} ${ty})` }, o.ylab));
}
const P = {
x, y, svg, gAxis, gc, gt, fs,
xdom: [xa, xb], ydom: [ya, yb],
area: { x0: px0, y0: py0, w: iw, h: ih },
add(node, top) { (top ? gt : gc).append(node); return node; },
path(pts, a) {
let d = "";
for (let i = 0; i < pts.length; i++)
d += (i ? "L" : "M") + x(pts[i][0]).toFixed(2) + "," + y(pts[i][1]).toFixed(2);
return P.add(el("path", Object.assign(
{ d, fill: "none", "stroke-linejoin": "round", "stroke-linecap": "round" }, a)));
},
fill(pts, base, a) {
let d = "";
for (let i = 0; i < pts.length; i++)
d += (i ? "L" : "M") + x(pts[i][0]).toFixed(2) + "," + y(pts[i][1]).toFixed(2);
d += "L" + x(pts[pts.length - 1][0]).toFixed(2) + "," + y(base).toFixed(2);
d += "L" + x(pts[0][0]).toFixed(2) + "," + y(base).toFixed(2) + "Z";
return P.add(el("path", Object.assign({ d, stroke: "none" }, a)));
},
seg(x1, y1, x2, y2, a) {
return P.add(el("line", Object.assign(
{ x1: x(x1), y1: y(y1), x2: x(x2), y2: y(y2) }, a)));
},
vline(v, a) { return P.seg(v, ya, v, yb, a); },
hline(v, a) { return P.seg(xa, v, xb, v, a); },
bar(x1, x2, y1, y2, a) {
const X1 = x(x1), X2 = x(x2), Y1 = y(y1), Y2 = y(y2);
return P.add(el("rect", Object.assign({
x: Math.min(X1, X2), y: Math.min(Y1, Y2),
width: Math.max(Math.abs(X2 - X1), 0.5),
height: Math.max(Math.abs(Y2 - Y1), 0)
}, a)));
},
dot(vx, vy, r, a) {
return P.add(el("circle", Object.assign({ cx: x(vx), cy: y(vy), r }, a)));
},
// mtext-style annotation above the plot region
header(s, a = {}) {
const adj = a.adj ?? 0;
const py = py0 - 14 - (a.line ?? 0) * (fs + 7);
return P.add(el("text", {
x: adj === 1 ? px0 + iw : adj === 0.5 ? px0 + iw / 2 : px0,
y: py,
"text-anchor": adj === 1 ? "end" : adj === 0.5 ? "middle" : "start",
"font-size": a.size ?? fs + 2,
"font-weight": a.weight ?? "bold",
fill: a.fill ?? C.ink
}, s), true);
},
legend(items, opt = {}) {
const size = opt.size ?? fs;
const pad = opt.pad ?? 8;
const rowH = size + 8, swW = 30;
const g = el("g");
let maxw = 0;
items.forEach((it, i) => {
const ty = i * rowH + size;
if (it.swatch)
g.append(el("rect", { x: 0, y: ty - size * 0.82, width: swW,
height: size * 0.85, fill: it.swatch, stroke: "none" }));
else
g.append(el("line", { x1: 0, y1: ty - size * 0.35, x2: swW, y2: ty - size * 0.35,
stroke: it.color, "stroke-width": it.lw ?? 3.5,
"stroke-dasharray": it.dash ?? null,
"stroke-linecap": "round" }));
g.append(el("text", { x: swW + 9, y: ty, "font-size": size,
fill: opt.fill ?? C.ink }, it.label));
maxw = Math.max(maxw, swW + 9 + it.label.length * size * 0.54);
});
// opt-in backing panel, for legends that must sit over the data
if (opt.bg) {
const q = 8;
g.insertBefore(el("rect", {
x: -q, y: -q + 3, width: maxw + 2 * q,
height: items.length * rowH + 2 * q - 6,
fill: "white", "fill-opacity": opt.bg === true ? 0.85 : opt.bg, rx: 5
}), g.firstChild);
}
const corner = opt.corner ?? "topright";
const gx = corner.includes("right") ? px0 + iw - maxw - pad : px0 + pad;
const gy = corner.includes("top") ? py0 + pad
: py0 + ih - items.length * rowH - pad;
g.setAttribute("transform", `translate(${gx},${gy})`);
gt.append(g);
return g;
}
};
return P;
}
// Convenience: one chart with exactly one panel filling it.
function single(W, H, o) {
const svg = chart(W, H);
return { svg, p: panel(svg, o) };
}
return {
C, el, alpha, chart, panel, single,
lgamma, lchoose, erfc, dnorm, pnorm, qnorm, dbinom, dpois, dexp, pexp,
dunif, punif, dlnorm, dchisq1, cdfFromPmf,
gammaP, betai, dchisq, pchisq, qchisq, dt, pt, qt, invert,
seq, cumsum, sum, mean, sd, quantileSorted, hist, rng, sampleWeighted,
fmt, signed, smart, nice, logTicks
};
}Definition — Statistical Hypothesis Test
A statistical hypothesis test is a procedure for deciding between two competing claims about a population parameter \(\theta\).
Null Hypothesis
\[H_0: \theta \in \Theta_0\] The hypothesis to be tested — the status quo assumption.
Alternative Hypothesis
\[H_1: \theta \in \Theta_1\] The research hypothesis — what we want to detect.
Mathematical Requirements
Simple hypothesis — specifies a single parameter value \[H_0: \theta = \theta_0\]
Composite hypothesis — specifies a range of values \[H_0: \theta \geq \theta_0\]
Two-sided
\[\begin{aligned} H_0: \theta &= \theta_0\\ H_1: \theta &\neq \theta_0 \end{aligned}\] Deviations in either direction.
Right-tailed
\[\begin{aligned} H_0: \theta &\leq \theta_0\\ H_1: \theta &> \theta_0 \end{aligned}\] Increases above \(\theta_0\).
Left-tailed
\[\begin{aligned} H_0: \theta &\geq \theta_0\\ H_1: \theta &< \theta_0 \end{aligned}\] Decreases below \(\theta_0\).
The alternative decides where the rejection region goes; \(\alpha\) decides how big it is.
The curve is the distribution of the test statistic under \(H_0\). The shaded area is \(\alpha\) — the probability of rejecting a true null.
{
const C = ldk.C, a = rr_alpha;
let crit, regions, rule;
if (rr_side === "two") {
const c = ldk.qnorm(1 - a / 2);
crit = [-c, c];
regions = [[-5, -c], [c, 5]];
rule = `reject H₀ when |Z| > ${ldk.fmt(c, 3)}`;
} else if (rr_side === "right") {
const c = ldk.qnorm(1 - a);
crit = [c];
regions = [[c, 5]];
rule = `reject H₀ when Z > ${ldk.fmt(c, 3)}`;
} else {
const c = ldk.qnorm(a);
crit = [c];
regions = [[-5, c]];
rule = `reject H₀ when Z < ${ldk.fmt(c, 3)}`;
}
const { svg, p } = ldk.single(880, 520, {
xdom: [-4.4, 4.4], ydom: [0, 0.48],
xlab: "test statistic Z", ylab: "density under H₀", margin: { t: 74 }
});
const grid = ldk.seq(-4.4, 4.4, 700);
const curve = grid.map(z => [z, ldk.dnorm(z)]);
// the do-not-reject region, then the rejection tails on top
p.fill(curve, 0, { fill: ldk.alpha(C.green, 0.13) });
for (const [lo, hi] of regions) {
const seg = grid.filter(z => z >= lo && z <= hi).map(z => [z, ldk.dnorm(z)]);
if (seg.length > 1) p.fill(seg, 0, { fill: ldk.alpha(C.pink, 0.55) });
}
p.path(curve, { stroke: C.blue, "stroke-width": 3.5 });
for (const c of crit) {
p.seg(c, 0, c, ldk.dnorm(c) + 0.055, { stroke: C.pink, "stroke-width": 2.5 });
p.add(ldk.el("text", { x: p.x(c), y: p.y(ldk.dnorm(c) + 0.06),
"text-anchor": "middle", "font-size": 17,
"font-weight": "bold", fill: C.pink }, ldk.fmt(c, 2)), true);
}
p.legend([
{ label: `rejection region — area α = ${ldk.fmt(a, 3)}`, swatch: ldk.alpha(C.pink, 0.55) },
{ label: `do not reject — area ${ldk.fmt(1 - a, 3)}`, swatch: ldk.alpha(C.green, 0.13) }
], { size: 15 });
p.header(rule, { line: 1, size: 19, fill: C.pink });
p.header("critical values are just quantiles of the null distribution",
{ size: 16, fill: C.grey });
return svg;
}Definition — Test Statistic
A test statistic \(T(X)\) is a function of the sample data used to decide between \(H_0\) and \(H_1\).
Decision Rule
A test is defined by its critical region (rejection region) \(C\): \[\text{Reject } H_0 \text{ if } T(X) \in C\]
Critical Value
The boundary value \(c\) such that \[P(T(X) > c \mid H_0) = \alpha\]
Significance Level
\[\alpha = P(\text{Reject } H_0 \mid H_0 \text{ true})\] Common choices: 0.01, 0.05, 0.10.
| Decision | \(H_0\) true | \(H_0\) false |
|---|---|---|
| Reject \(H_0\) | Type I error — probability \(\alpha\) | Correct decision — power \(1-\beta\) |
| Fail to reject \(H_0\) | Correct decision — probability \(1-\alpha\) | Type II error — probability \(\beta\) |
Error Definitions
\[\alpha = P(\text{Reject } H_0 \mid H_0 \text{ true})\] \[\beta = P(\text{Fail to reject } H_0 \mid H_1 \text{ true})\]
The Trade-off
One threshold splits both curves at once: you cannot shrink one shaded area without growing the other.
viewof er_side = Inputs.select(
new Map([["Right-tailed", "right"], ["Two-sided", "two"]]),
{label: "Alternative", value: "right"}
)
viewof er_delta = Inputs.range([0, 1.5], {step: 0.05, value: 0.5,
label: "Effect size δ = (μ₁−μ₀)/σ"})
viewof er_n = Inputs.range([1, 100], {step: 1, value: 16, label: "Sample size n"})
viewof er_alpha = Inputs.range([0.001, 0.2], {step: 0.001, value: 0.05,
label: "Significance α"})Both curves are the sampling distribution of \(\bar{X}\), under \(H_0\) and under \(H_1\). Raising \(n\) narrows both — the only move that shrinks \(\alpha\) and \(\beta\) together.
{
const C = ldk.C;
const n = er_n, d = er_delta, a = er_alpha;
const se = 1 / Math.sqrt(n); // sigma = 1 throughout
// critical value(s) on the x-bar scale
const twoSided = er_side === "two";
const zc = ldk.qnorm(1 - (twoSided ? a / 2 : a));
const cHi = zc * se, cLo = -zc * se;
// beta = P(do not reject | H1), computed on the standardised scale
const lam = d / se; // = sqrt(n) * delta
const beta = twoSided
? ldk.pnorm(zc - lam) - ldk.pnorm(-zc - lam)
: ldk.pnorm(zc - lam);
const power = 1 - beta;
const XLO = -1.15, XHI = 2.15; // fixed in sigma units
const ymax = ldk.dnorm(0) / se * 1.22; // depends on n only
const { svg, p } = ldk.single(880, 520, {
xdom: [XLO, XHI], ydom: [0, ymax],
xlab: "sample mean x̄ (in units of σ)", ylab: "density",
margin: { t: 74 }
});
const grid = ldk.seq(XLO, XHI, 800);
const f0 = grid.map(x => [x, ldk.dnorm(x, 0, se)]);
const f1 = grid.map(x => [x, ldk.dnorm(x, d, se)]);
// beta: the H1 mass that falls inside the do-not-reject region
const keep1 = grid.filter(x => twoSided ? (x > cLo && x < cHi) : x < cHi)
.map(x => [x, ldk.dnorm(x, d, se)]);
if (keep1.length > 1) p.fill(keep1, 0, { fill: ldk.alpha(C.orange, 0.55) });
// alpha: the H0 mass in the rejection region
for (const [lo, hi] of twoSided ? [[XLO, cLo], [cHi, XHI]] : [[cHi, XHI]]) {
const seg = grid.filter(x => x >= lo && x <= hi).map(x => [x, ldk.dnorm(x, 0, se)]);
if (seg.length > 1) p.fill(seg, 0, { fill: ldk.alpha(C.pink, 0.6) });
}
p.path(f0, { stroke: C.green, "stroke-width": 3.5 });
p.path(f1, { stroke: C.blue, "stroke-width": 3.5 });
for (const c of twoSided ? [cLo, cHi] : [cHi])
p.seg(c, 0, c, ymax * 0.94, { stroke: C.ink, "stroke-width": 2.5,
"stroke-dasharray": "7,5" });
p.legend([
{ label: "H₀: μ = μ₀", color: C.green, lw: 3.5 },
{ label: `H₁: μ = μ₀ + ${ldk.fmt(d, 2)}σ`, color: C.blue, lw: 3.5 },
{ label: `Type I error α = ${ldk.fmt(a, 3)}`, swatch: ldk.alpha(C.pink, 0.6) },
{ label: `Type II error β = ${ldk.fmt(beta, 3)}`, swatch: ldk.alpha(C.orange, 0.55) }
], { size: 15 });
p.header(`n = ${n} threshold at x̄ = ${ldk.fmt(cHi, 3)}` +
(twoSided ? ` and ${ldk.fmt(cLo, 3)}` : ""), { line: 1, size: 17, fill: C.grey });
p.header(`power = 1 − β = ${ldk.fmt(power, 3)}`, { size: 20, fill: C.blue });
return svg;
}Definition — Power
The power of a test is the probability of correctly rejecting \(H_0\) when it is false: \[\text{Power} = 1 - \beta = P(\text{Reject } H_0 \mid H_1 \text{ true})\]
Power Function
\[\pi(\theta) = P_\theta(\text{Reject } H_0)\] How power varies with the true parameter value.
Properties
Factors
Each curve is one sample size. Where they cross \(\theta_0\), every curve is pinned at \(\alpha\).
viewof pw_side = Inputs.select(
new Map([["Two-sided", "two"], ["Right-tailed", "right"], ["Left-tailed", "left"]]),
{label: "Alternative", value: "two"}
)
viewof pw_alpha = Inputs.range([0.001, 0.2], {step: 0.001, value: 0.05,
label: "Significance α"})
viewof pw_delta = Inputs.range([-1.5, 1.5], {step: 0.05, value: 0.5,
label: "Read off at δ ="})\(\delta = (\theta - \theta_0)/\sigma\) is the true effect in standard deviations. The legend reports \(\pi(\delta)\) for each \(n\) at the marked value.
{
const C = ldk.C, a = pw_alpha;
const NS = [5, 10, 20, 50, 100];
const zc = ldk.qnorm(1 - (pw_side === "two" ? a / 2 : a));
const pow = (d, n) => {
const lam = Math.sqrt(n) * d;
if (pw_side === "two") return 1 - ldk.pnorm(zc - lam) + ldk.pnorm(-zc - lam);
if (pw_side === "right") return 1 - ldk.pnorm(zc - lam);
return ldk.pnorm(-zc - lam);
};
const { svg, p } = ldk.single(880, 520, {
xdom: [-1.5, 1.5], ydom: [0, 1.02],
xlab: "true effect size δ = (θ − θ₀)/σ", ylab: "power π(δ)",
margin: { t: 74 }
});
p.hline(0.8, { stroke: ldk.alpha(C.green, 0.7), "stroke-width": 2,
"stroke-dasharray": "7,6" });
p.hline(a, { stroke: ldk.alpha(C.pink, 0.8), "stroke-width": 2,
"stroke-dasharray": "7,6" });
p.vline(0, { stroke: C.grey, "stroke-width": 1.5 });
p.vline(pw_delta, { stroke: C.ink, "stroke-width": 2, "stroke-dasharray": "3,5" });
const grid = ldk.seq(-1.5, 1.5, 500);
const stops = [C.blue, C.green, C.orange].map(h => {
const v = parseInt(h.slice(1), 16);
return [(v >> 16) & 255, (v >> 8) & 255, v & 255];
});
const ramp = t => {
const s = Math.min(t * 2, 1.999), i = Math.floor(s), f = s - i;
const [u, v] = [stops[i], stops[i + 1]];
return `rgb(${u.map((c, k) => Math.round(c + (v[k] - c) * f)).join(",")})`;
};
const items = [];
NS.forEach((n, j) => {
const col = ramp(j / (NS.length - 1));
p.path(grid.map(d => [d, pow(d, n)]), { stroke: col, "stroke-width": 3 });
p.dot(pw_delta, pow(pw_delta, n), 5, { fill: col });
items.push({ label: `n = ${String(n).padStart(3)} → π = ${ldk.fmt(pow(pw_delta, n), 3)}`,
color: col, lw: 3 });
});
p.legend(items, { size: 15, corner: "topleft", bg: true });
p.add(ldk.el("text", { x: p.x(1.47), y: p.y(0.8) - 7, "text-anchor": "end",
"font-size": 15, fill: C.green }, "power = 0.80"), true);
p.add(ldk.el("text", { x: p.x(1.47), y: p.y(a) - 7, "text-anchor": "end",
"font-size": 15, fill: C.pink }, `α = ${ldk.fmt(a, 3)}`), true);
p.header(`at δ = 0 every curve equals α = ${ldk.fmt(a, 3)}` +
" — a test cannot beat its own size on the null",
{ line: 1, size: 17, fill: C.grey });
p.header(`reading off at δ = ${ldk.fmt(pw_delta, 2)}`, { size: 19, fill: C.ink });
return svg;
}Definition — Likelihood Ratio Statistic
\[\Lambda(x) = \frac{\sup_{\theta \in \Theta_0} L(\theta; x)}{\sup_{\theta \in \Theta} L(\theta; x)}\]
LRT Decision Rule
Reject \(H_0\) if \(\Lambda(x) < c\), with \(c\) chosen so that \[P(\Lambda(X) < c \mid H_0) = \alpha\]
Theorem — Wilks
Under regularity conditions, as \(n \to \infty\), \[-2\log\Lambda(X) \xrightarrow{\ d\ } \chi^2_k\] where \(k = \dim(\Theta) - \dim(\Theta_0)\).
Why it matters
\(-2\log\Lambda\) is exactly twice the drop in log-likelihood from its peak down to the best point allowed by \(H_0\).
viewof lr_model = Inputs.select(
new Map([["Normal mean, σ = 1 H₀: μ = 0", "norm"],
["Exponential rate H₀: λ = 1", "exp"],
["Bernoulli H₀: p = 0.5", "bern"]]),
{label: "Model", value: "norm"}
)
viewof lr_n = Inputs.range([2, 200], {step: 1, value: 20, label: "Sample size n"})
viewof lr_stat = Inputs.range([0, 1], {step: 0.005, value: 0.62,
label: "Observed statistic"})Left: the log-likelihood, shifted so its peak is 0. Right: the \(\chi^2_1\) reference. For the normal mean the \(\chi^2_1\) law is exact; otherwise it is the large-\(n\) approximation.
{
const C = ldk.C, n = lr_n, t = lr_stat;
// each model is summarised by one sufficient statistic, so the whole
// log-likelihood is available in closed form
const M = {
norm: { dom: [-1.6, 1.6], th0: 0, stat: -1.5 + 3 * t,
ll: (mu, s) => -n * (s - mu) * (s - mu) / 2,
hat: s => s, xlab: "μ", statLab: "x̄" },
exp: { dom: [0.25, 3.2], th0: 1, stat: 0.4 + 2.1 * t,
ll: (lam, s) => n * (Math.log(lam) - lam * s),
hat: s => 1 / s, xlab: "λ", statLab: "x̄" },
bern: { dom: [0.02, 0.98], th0: 0.5, stat: 0.05 + 0.9 * t,
ll: (p, s) => n * (s * Math.log(p) + (1 - s) * Math.log1p(-p)),
hat: s => s, xlab: "p", statLab: "p̂" }
}[lr_model];
const s = M.stat, hat = M.hat(s);
const W = 2 * (M.ll(hat, s) - M.ll(M.th0, s)); // -2 log Lambda
const pval = 1 - ldk.pchisq(W, 1);
const svg = ldk.chart(920, 500);
// ---- left: the log-likelihood and the drop that H0 costs
const gx = ldk.seq(M.dom[0], M.dom[1], 600);
const rel = gx.map(v => [v, M.ll(v, s) - M.ll(hat, s)]);
const ylo = -Math.max(6, W * 0.85);
const pl = ldk.panel(svg, {
left: 0, w: 460, xdom: M.dom, ydom: [ylo, 0.10 * -ylo],
xlab: M.xlab, ylab: "log-likelihood (peak = 0)",
margin: { l: 72, r: 14, t: 72, b: 52 }, nx: 5
});
pl.path(rel, { stroke: C.blue, "stroke-width": 3.5 });
pl.seg(M.th0, ylo, M.th0, -W / 2, { stroke: C.pink, "stroke-width": 2,
"stroke-dasharray": "4,5" });
pl.seg(M.dom[0], -W / 2, M.th0, -W / 2, { stroke: C.pink, "stroke-width": 2,
"stroke-dasharray": "4,5" });
pl.seg(hat, -W / 2, hat, 0, { stroke: C.orange, "stroke-width": 3 });
pl.dot(hat, 0, 6, { fill: C.orange });
pl.dot(M.th0, -W / 2, 6, { fill: C.pink });
pl.add(ldk.el("text", { x: pl.x(hat) + 9, y: pl.y(-W / 4), "font-size": 16,
"font-weight": "bold", fill: C.orange },
`drop = ${ldk.fmt(W / 2, 2)}`), true);
pl.header(`${M.statLab} = ${ldk.fmt(s, 3)}, θ̂ = ${ldk.fmt(hat, 3)}, θ₀ = ${M.th0}`,
{ line: 1, size: 16, fill: C.grey });
pl.header("log-likelihood", { size: 19 });
// ---- right: the chi-squared reference and the p-value tail
const xhi = Math.max(9, W * 1.4);
const gz = ldk.seq(1e-4, xhi, 700);
const pr = ldk.panel(svg, {
left: 460, w: 460, xdom: [0, xhi], ydom: [0, 1.05],
xlab: "−2 log Λ", ylab: "χ²₁ density",
margin: { l: 72, r: 14, t: 72, b: 52 }, nx: 5
});
const tail = gz.filter(v => v >= W).map(v => [v, ldk.dchisq(v, 1)]);
if (tail.length > 1) pr.fill(tail, 0, { fill: ldk.alpha(C.pink, 0.5) });
pr.path(gz.map(v => [v, ldk.dchisq(v, 1)]), { stroke: C.blue, "stroke-width": 3.5 });
pr.seg(W, 0, W, 0.95, { stroke: C.orange, "stroke-width": 3 });
pr.add(ldk.el("text", { x: pr.x(W), y: pr.y(0.97), "text-anchor": "middle",
"font-size": 16, "font-weight": "bold", fill: C.orange },
`W = ${ldk.fmt(W, 2)}`), true);
pr.header(`p = P(χ²₁ > W) = ${pval < 1e-4 ? pval.toExponential(1) : ldk.fmt(pval, 4)}`,
{ line: 1, size: 17, fill: C.pink });
pr.header(lr_model === "norm" ? "reference: χ²₁ (exact here)"
: "reference: χ²₁ (large-n approximation)",
{ size: 19 });
return svg;
}Definition — p-value
The p-value is the probability of observing a test statistic as extreme or more extreme than the observed value, assuming \(H_0\) is true: \[p\text{-value} = P\big(T(X) \geq T(x_{\text{obs}}) \mid H_0\big)\]
Correct Interpretation
A p-value is NOT
A large \(p\)-value is not evidence that \(H_0\) is true — only that the data do not contradict it.
Under \(H_0\) the p-value is exactly uniform. Every property people expect of it follows from that one fact.
Exact density of the two-sided p-value for a \(z\)-test. At \(\delta = 0\) it is flat at 1 — so \(P(p < \alpha) = \alpha\), which is what “level \(\alpha\)” means. Raising \(\delta\) or \(n\) piles the mass up against 0.
{
const C = ldk.C, a = pv_alpha;
const lam = Math.sqrt(pv_n) * pv_delta; // non-centrality
// p = 2(1 - Phi(|Z|)) with Z ~ N(lam, 1). Writing q = Phi^{-1}(1 - p/2),
// the change of variables gives the density below; at lam = 0 it is 1.
const fP = p => {
const q = ldk.qnorm(1 - p / 2);
return (ldk.dnorm(q - lam) + ldk.dnorm(q + lam)) / (2 * ldk.dnorm(q));
};
const zc = ldk.qnorm(1 - a / 2);
const power = 1 - ldk.pnorm(zc - lam) + ldk.pnorm(-zc - lam);
const grid = ldk.seq(0.0008, 1, 900);
const curve = grid.map(p => [p, fP(p)]);
const peak = Math.max(...curve.map(d => d[1]));
const ymax = Math.min(Math.max(peak * 1.12, 1.45), 14);
const { svg, p } = ldk.single(880, 520, {
xdom: [0, 1], ydom: [0, ymax],
xlab: "p-value", ylab: "density", margin: { t: 74 }
});
const sig = curve.filter(d => d[0] <= a);
if (sig.length > 1) p.fill(sig, 0, { fill: ldk.alpha(C.pink, 0.5) });
p.fill(curve, 0, { fill: ldk.alpha(C.blue, 0.13) });
p.path(curve, { stroke: C.blue, "stroke-width": 3.5 });
p.hline(1, { stroke: C.grey, "stroke-width": 2, "stroke-dasharray": "9,6" });
p.seg(a, 0, a, ymax, { stroke: C.pink, "stroke-width": 2.5 });
p.legend([
{ label: `density of p (δ = ${ldk.fmt(pv_delta, 2)}, n = ${pv_n})`, color: C.blue, lw: 3.5 },
{ label: "uniform — the H₀ law", color: C.grey, lw: 2, dash: "9,6" },
{ label: `P(p < α) = ${ldk.fmt(power, 3)}`, swatch: ldk.alpha(C.pink, 0.5) }
], { size: 15 });
p.header(lam === 0
? "δ = 0: the density is flat, so P(p < α) = α exactly"
: `shaded area = power = ${ldk.fmt(power, 3)}`,
{ line: 1, size: 18, fill: lam === 0 ? C.green : C.pink });
p.header(peak > ymax
? `peak ${ldk.fmt(peak, 1)} runs off the top — the mass piles up at 0`
: "a p-value is a random variable; H₀ fixes its distribution",
{ size: 16, fill: C.grey });
return svg;
}Problem Setup
Test \(H_0: \mu = \mu_0\) against \(H_1: \mu \neq \mu_0\) for \(X_1, \ldots, X_n \sim N(\mu, \sigma^2)\).
Case 1 — Known Variance
\[Z = \frac{\bar{X} - \mu_0}{\sigma/\sqrt{n}} \sim N(0,1) \text{ under } H_0\]
Reject if \(|Z| > z_{\alpha/2}\); \(\quad p = 2P(Z > |z_{\text{obs}}|)\)
Case 2 — Unknown Variance
\[T = \frac{\bar{X} - \mu_0}{S/\sqrt{n}} \sim t_{n-1} \text{ under } H_0\]
with \(S^2 = \frac{1}{n-1}\sum_i (X_i - \bar{X})^2\).
Reject if \(|T| > t_{n-1,\alpha/2}\); \(\quad p = 2P(t_{n-1} > |t_{\text{obs}}|)\)
Estimating \(\sigma\) costs you something. The price is paid in the tails.
Using \(z_{\alpha/2}\) when \(\sigma\) is estimated does not give you level \(\alpha\) — it gives you the larger error rate in the readout. The gap closes as \(n \to \infty\).
{
const C = ldk.C, nu = tz_n - 1, a = tz_alpha;
const tc = ldk.qt(1 - a / 2, nu);
const zc = ldk.qnorm(1 - a / 2);
const realErr = 2 * (1 - ldk.pt(zc, nu)); // true size of the naive z-rule
const XL = Math.max(5, tc * 1.15);
const { svg, p } = ldk.single(880, 520, {
xdom: [-XL, XL], ydom: [0, 0.46],
xlab: "test statistic", ylab: "density", margin: { t: 74 }
});
const grid = ldk.seq(-XL, XL, 800);
p.fill(grid.map(x => [x, ldk.dt(x, nu)]), 0, { fill: ldk.alpha(C.blue, 0.13) });
p.path(grid.map(x => [x, ldk.dnorm(x)]),
{ stroke: C.pink, "stroke-width": 2.5, "stroke-dasharray": "9,6" });
p.path(grid.map(x => [x, ldk.dt(x, nu)]), { stroke: C.blue, "stroke-width": 3.5 });
for (const c of [-tc, tc])
p.seg(c, 0, c, 0.40, { stroke: C.blue, "stroke-width": 2.5 });
for (const c of [-zc, zc])
p.seg(c, 0, c, 0.40, { stroke: C.pink, "stroke-width": 2.5,
"stroke-dasharray": "5,4" });
p.legend([
{ label: `t with ν = ${nu}`, color: C.blue, lw: 3.5 },
{ label: "N(0,1)", color: C.pink, lw: 2.5, dash: "9,6" }
], { size: 15 });
p.header(`t critical ±${ldk.fmt(tc, 3)} z critical ±${ldk.fmt(zc, 3)}`,
{ line: 1, size: 17, fill: C.grey });
p.header(`using z here would reject a true H₀ ${ldk.fmt(100 * realErr, 1)}% of the time,` +
` not ${ldk.fmt(100 * a, 1)}%`,
{ size: 17, fill: realErr > a * 1.1 ? C.pink : C.green });
return svg;
}Problem Setup
Test \(H_0: \sigma^2 = \sigma_0^2\) against \(H_1: \sigma^2 \neq \sigma_0^2\) for \(X_1, \ldots, X_n \sim N(\mu, \sigma^2)\).
Chi-Square Test
\[\chi^2 = \frac{(n-1)S^2}{\sigma_0^2} \sim \chi^2_{n-1} \text{ under } H_0\]
Reject if \(\chi^2 < \chi^2_{n-1,1-\alpha/2}\) or \(\chi^2 > \chi^2_{n-1,\alpha/2}\).
Health Warning
Note the null distribution is skewed, so the two critical values are not symmetric about \(n-1\) — unlike the \(z\) and \(t\) tests.
The Problem
Running \(m\) independent tests at level \(\alpha\), the chance of at least one Type I error is \[P(\text{at least one Type I error}) = 1 - (1-\alpha)^m\] With \(\alpha = 0.05\) and \(m = 20\): \(P \approx 0.64\).
Family-wise Error Rate
\[\text{FWER} = P(\text{reject at least one true } H_0)\] Bonferroni: test each at \(\alpha/m\), which guarantees \(\text{FWER} \leq \alpha\).
False Discovery Rate
\[\text{FDR} = E\left[\frac{\#\text{ false discoveries}}{\#\text{ total discoveries}}\right]\] Less conservative than FWER control — Benjamini–Hochberg.
Every square is one test. The nulls are genuinely true — every red square is a mistake the procedure made.
viewof mt_m = Inputs.range([10, 200], {step: 10, value: 100, label: "Number of tests m"})
viewof mt_k = Inputs.range([0, 40], {step: 1, value: 10, label: "True effects"})
viewof mt_alpha = Inputs.range([0.001, 0.2], {step: 0.001, value: 0.05, label: "Level α"})
viewof mt_corr = Inputs.select(
new Map([["None", "none"], ["Bonferroni", "bonf"], ["Benjamini–Hochberg", "bh"]]),
{label: "Correction", value: "none"}
)
viewof mt_go = Inputs.button("New draw"){
const C = ldk.C;
const m = mt_m, k = Math.min(mt_k, m), a = mt_alpha;
const LAM = 3.2; // effect size of the non-null tests
const u = ldk.rng(4021 + mt_go);
const tests = [];
for (let i = 0; i < m; i++) {
const isEffect = i < k;
const z = (isEffect ? LAM : 0) + u.normal();
tests.push({ isEffect, p: 2 * (1 - ldk.pnorm(Math.abs(z))) });
}
// decision rule
let cut;
if (mt_corr === "bonf") cut = a / m;
else if (mt_corr === "bh") {
const sorted = tests.map(t => t.p).sort((x, y) => x - y);
cut = 0;
for (let j = sorted.length; j >= 1; j--)
if (sorted[j - 1] <= j * a / m) { cut = sorted[j - 1]; break; }
} else cut = a;
for (const t of tests) t.rej = t.p <= cut;
const TP = tests.filter(t => t.rej && t.isEffect).length;
const FP = tests.filter(t => t.rej && !t.isEffect).length;
const FN = tests.filter(t => !t.rej && t.isEffect).length;
const disc = TP + FP;
const fdr = disc ? FP / disc : 0;
const svg = ldk.chart(920, 500);
// ---- left: how the family-wise error rate grows with m
const pf = ldk.panel(svg, {
left: 0, w: 380, xdom: [1, 200], ydom: [0, 1.02],
xlab: "number of tests m", ylab: "P(at least one false positive)",
margin: { l: 70, r: 14, t: 72, b: 52 }, nx: 4
});
const ms = ldk.seq(1, 200, 300);
pf.path(ms.map(v => [v, 1 - Math.pow(1 - a, v)]),
{ stroke: C.pink, "stroke-width": 3.5 });
pf.path(ms.map(v => [v, 1 - Math.pow(1 - a / v, v)]),
{ stroke: C.green, "stroke-width": 3, "stroke-dasharray": "8,5" });
pf.hline(a, { stroke: C.grey, "stroke-width": 1.5, "stroke-dasharray": "4,5" });
pf.dot(m, 1 - Math.pow(1 - a, m), 6, { fill: C.pink });
// labelled at the curve ends: any in-plot legend box collides with one of them
pf.add(ldk.el("text", { x: pf.x(196), y: pf.y(1) + 22, "text-anchor": "end",
"font-size": 15, "font-weight": "bold", fill: C.pink },
"no correction"), true);
pf.add(ldk.el("text", { x: pf.x(196), y: pf.y(a) - 10, "text-anchor": "end",
"font-size": 15, "font-weight": "bold", fill: C.green },
"Bonferroni"), true);
pf.header(`at m = ${m}: ${ldk.fmt(1 - Math.pow(1 - a, m), 3)}`,
{ line: 1, size: 16, fill: C.pink });
pf.header("family-wise error rate", { size: 18 });
// ---- right: one realisation, one square per test
const GW = 520 - 20 - 16, GH = 500 - 72 - 78; // inner size of the grid panel
const cols = Math.max(1, Math.round(Math.sqrt(m * GW / GH))); // near-square cells
const rows = Math.ceil(m / cols);
const pg = ldk.panel(svg, {
left: 400, w: 520, xdom: [0, cols], ydom: [0, rows], axes: false,
margin: { l: 20, r: 16, t: 72, b: 78 }
});
const FILL = {
tp: C.green, fp: C.pink,
fn: ldk.alpha(C.orange, 0.45), tn: "#e6e6e6"
};
for (let i = 0; i < m; i++) {
const t = tests[i];
const kind = t.rej ? (t.isEffect ? "tp" : "fp") : (t.isEffect ? "fn" : "tn");
const cx = i % cols, cy = Math.floor(i / cols);
pg.bar(cx + 0.08, cx + 0.92, rows - cy - 0.92, rows - cy - 0.08,
{ fill: FILL[kind], stroke: "white", "stroke-width": 1 });
}
// key laid out horizontally underneath, clear of the squares
let lx = pg.area.x0, ly = pg.area.y0 + pg.area.h + 30;
for (const [lab, col] of [[`true positive ${TP}`, FILL.tp],
[`false positive ${FP}`, FILL.fp],
[`missed ${FN}`, FILL.fn],
[`correct ${m - k - FP}`, FILL.tn]]) {
svg.append(ldk.el("rect", { x: lx, y: ly - 12, width: 18, height: 13,
fill: col, stroke: "none" }));
svg.append(ldk.el("text", { x: lx + 23, y: ly, "font-size": 14, fill: C.ink }, lab));
lx += 23 + lab.length * 7.3 + 16;
}
pg.header(`${disc} discoveries, ${FP} of them false` +
(disc ? ` — realised FDR ${ldk.fmt(fdr, 3)}` : ""),
{ line: 1, size: 16, fill: FP ? C.pink : C.green });
pg.header(`m = ${m} tests, ${k} with a real effect`, { size: 18 });
return svg;
}The point
There is no correction that is simply “best”. FWER and FDR answer different questions, and you must decide which error you can live with before looking at the data.
The Test Toolkit
Critical Reminder
Statistical significance \(\neq\) practical significance.
Best practice
Choose \(\alpha\) and the sample size before seeing the data; report effect sizes alongside p-values; account for every test you ran, not just the ones that worked.