% Options for packages loaded elsewhere
% Options for packages loaded elsewhere
\PassOptionsToPackage{unicode}{hyperref}
\PassOptionsToPackage{hyphens}{url}
\PassOptionsToPackage{dvipsnames,svgnames,x11names}{xcolor}
%
\documentclass[
  letterpaper,
  DIV=11,
  numbers=noendperiod]{scrartcl}
\usepackage{xcolor}
\usepackage{amsmath,amssymb}
\setcounter{secnumdepth}{-\maxdimen} % remove section numbering
\usepackage{iftex}
\ifPDFTeX
  \usepackage[T1]{fontenc}
  \usepackage[utf8]{inputenc}
  \usepackage{textcomp} % provide euro and other symbols
\else % if luatex or xetex
  \usepackage{unicode-math} % this also loads fontspec
  \defaultfontfeatures{Scale=MatchLowercase}
  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
\fi
\usepackage{lmodern}
\ifPDFTeX\else
  % xetex/luatex font selection
\fi
% Use upquote if available, for straight quotes in verbatim environments
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
\IfFileExists{microtype.sty}{% use microtype if available
  \usepackage[]{microtype}
  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
}{}
\makeatletter
\@ifundefined{KOMAClassName}{% if non-KOMA class
  \IfFileExists{parskip.sty}{%
    \usepackage{parskip}
  }{% else
    \setlength{\parindent}{0pt}
    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
}{% if KOMA class
  \KOMAoptions{parskip=half}}
\makeatother
% Make \paragraph and \subparagraph free-standing
\makeatletter
\ifx\paragraph\undefined\else
  \let\oldparagraph\paragraph
  \renewcommand{\paragraph}{
    \@ifstar
      \xxxParagraphStar
      \xxxParagraphNoStar
  }
  \newcommand{\xxxParagraphStar}[1]{\oldparagraph*{#1}\mbox{}}
  \newcommand{\xxxParagraphNoStar}[1]{\oldparagraph{#1}\mbox{}}
\fi
\ifx\subparagraph\undefined\else
  \let\oldsubparagraph\subparagraph
  \renewcommand{\subparagraph}{
    \@ifstar
      \xxxSubParagraphStar
      \xxxSubParagraphNoStar
  }
  \newcommand{\xxxSubParagraphStar}[1]{\oldsubparagraph*{#1}\mbox{}}
  \newcommand{\xxxSubParagraphNoStar}[1]{\oldsubparagraph{#1}\mbox{}}
\fi
\makeatother

\usepackage{color}
\usepackage{fancyvrb}
\newcommand{\VerbBar}{|}
\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
% Add ',fontsize=\small' for more characters per line
\usepackage{framed}
\definecolor{shadecolor}{RGB}{241,243,245}
\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{#1}}
\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.40,0.45,0.13}{#1}}
\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\BuiltInTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{#1}}
\newcommand{\CharTok}[1]{\textcolor[rgb]{0.13,0.47,0.30}{#1}}
\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{#1}}
\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{\textit{#1}}}
\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{\textbf{#1}}}
\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{\textit{#1}}}
\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\ExtensionTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{#1}}
\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.28,0.35,0.67}{#1}}
\newcommand{\ImportTok}[1]{\textcolor[rgb]{0.00,0.46,0.62}{#1}}
\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{#1}}
\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{\textbf{#1}}}
\newcommand{\NormalTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{#1}}
\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{#1}}
\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{#1}}
\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\RegionMarkerTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{#1}}
\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{#1}}
\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.13,0.47,0.30}{#1}}
\newcommand{\StringTok}[1]{\textcolor[rgb]{0.13,0.47,0.30}{#1}}
\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.07,0.07,0.07}{#1}}
\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.13,0.47,0.30}{#1}}
\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{\textit{#1}}}

\usepackage{longtable,booktabs,array}
\newcounter{none} % for unnumbered tables
\usepackage{calc} % for calculating minipage widths
% Correct order of tables after \paragraph or \subparagraph
\usepackage{etoolbox}
\makeatletter
\patchcmd\longtable{\par}{\if@noskipsec\mbox{}\fi\par}{}{}
\makeatother
% Allow footnotes in longtable head/foot
\IfFileExists{footnotehyper.sty}{\usepackage{footnotehyper}}{\usepackage{footnote}}
\makesavenoteenv{longtable}
\usepackage{graphicx}
\makeatletter
\newsavebox\pandoc@box
\newcommand*\pandocbounded[1]{% scales image to fit in text height/width
  \sbox\pandoc@box{#1}%
  \Gscale@div\@tempa{\textheight}{\dimexpr\ht\pandoc@box+\dp\pandoc@box\relax}%
  \Gscale@div\@tempb{\linewidth}{\wd\pandoc@box}%
  \ifdim\@tempb\p@<\@tempa\p@\let\@tempa\@tempb\fi% select the smaller of both
  \ifdim\@tempa\p@<\p@\scalebox{\@tempa}{\usebox\pandoc@box}%
  \else\usebox{\pandoc@box}%
  \fi%
}
% Set default figure placement to htbp
\def\fps@figure{htbp}
\makeatother





\setlength{\emergencystretch}{3em} % prevent overfull lines

\providecommand{\tightlist}{%
  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}



 


\KOMAoption{captions}{tableheading}
\makeatletter
\@ifpackageloaded{caption}{}{\usepackage{caption}}
\AtBeginDocument{%
\ifdefined\contentsname
  \renewcommand*\contentsname{Table of contents}
\else
  \newcommand\contentsname{Table of contents}
\fi
\ifdefined\listfigurename
  \renewcommand*\listfigurename{List of Figures}
\else
  \newcommand\listfigurename{List of Figures}
\fi
\ifdefined\listtablename
  \renewcommand*\listtablename{List of Tables}
\else
  \newcommand\listtablename{List of Tables}
\fi
\ifdefined\figurename
  \renewcommand*\figurename{Figure}
\else
  \newcommand\figurename{Figure}
\fi
\ifdefined\tablename
  \renewcommand*\tablename{Table}
\else
  \newcommand\tablename{Table}
\fi
}
\@ifpackageloaded{float}{}{\usepackage{float}}
\floatstyle{ruled}
\@ifundefined{c@chapter}{\newfloat{codelisting}{h}{lop}}{\newfloat{codelisting}{h}{lop}[chapter]}
\floatname{codelisting}{Listing}
\newcommand*\listoflistings{\listof{codelisting}{List of Listings}}
\makeatother
\makeatletter
\makeatother
\makeatletter
\@ifpackageloaded{caption}{}{\usepackage{caption}}
\@ifpackageloaded{subcaption}{}{\usepackage{subcaption}}
\makeatother
\usepackage{bookmark}
\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
\urlstyle{same}
\hypersetup{
  pdftitle={Lab: Framing},
  colorlinks=true,
  linkcolor={blue},
  filecolor={Maroon},
  citecolor={Blue},
  urlcolor={Blue},
  pdfcreator={LaTeX via pandoc}}


\title{Lab: Framing}
\usepackage{etoolbox}
\makeatletter
\providecommand{\subtitle}[1]{% add subtitle to \maketitle
  \apptocmd{\@title}{\par {\large #1 \par}}{}{}
}
\makeatother
\subtitle{Building an Experiment and Statistical Tests}
\author{}
\date{}
\begin{document}
\maketitle


\subsection{Introduction}\label{introduction}

The goal of this exercise is to get a feel for running and designing
real experiments and how important the framing of a question can be to
the outcome of the study. We will design and conduct our own versions of
a well-known psychology experiment that investigates exactly this.
Finally, we implement statistical tests learned in class to get a better
understanding of how they work in the real world, and considerations
towards statistical power.

\subsection{The Experiment}\label{the-experiment}

For this lab, we will be working with the Tversky and Kahneman
(1981)\footnote{Tversky, A., \& Kahneman, D. (1981). \emph{The framing
  of decisions and the psychology of choice}. Science, 211(4481),
  453-458.} paper that studied the effects of framing on participant
response and risk aversion. Read through the paper, understanding the
effect that they are trying to elicit and the setting.

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\item
  Identify and list the four components of the experimental design they
  used, i.e.

  \begin{itemize}
  \item
    The Nature of the Treatments.
  \item
    Choice of the Experimental Units.
  \item
    Manner of Assigning Units to Treatments
  \item
    The Nature of the Response
  \end{itemize}
\end{enumerate}

\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{1}
\tightlist
\item
  Think of another question that elicits this same ``framing'' effect.
  Use a similar approach, but change the setting. Keep in mind how
  language and specific words can influence this. Do you think your
  question would elicit a smaller or larger effect size?
\end{enumerate}

\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break

\subsection{Conducting Your Own
Experiment}\label{conducting-your-own-experiment}

Use your new question to conduct your own framing experiment in class.
Go around asking your two versions of the question to all others
present, and collect data on their responses. Keep the following
questions in mind, and write very brief answers explaining your choice
in each.

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{2}
\tightlist
\item
  How are you going to ask the question? Consider options such as Google
  Forms, e-mails, writing on paper slips, or verbally asking. How do you
  think the mode of the question affects the response?
\end{enumerate}

\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{3}
\tightlist
\item
  Are there other factors that may be relevant? Including a time-limit
  on answering? Using percentages instead of numbers? Adding a default
  such as ``Most people choose A''? Including a ``How confident are you
  in your response'' score? Consider other ways of capturing more
  information or eliciting the effect.
\end{enumerate}

\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{4}
\tightlist
\item
  How are you planning to assign units to treatments? Tossing a coin?
  Using a random number generator? Do you experiment on pairs instead,
  one of each being in the treatment group and the other control?
\end{enumerate}

\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break
\hfill\break

\subsection{Data Analysis}\label{data-analysis}

We will analyze both the original dataset as well as our own collected
data. Ensure that you tabulate your data into a \texttt{.csv} file. This
can be done by entering your data into Google Sheets or Microsoft Excel
before saving as \texttt{.csv}.

We will be using data collected from the Many Labs study (2020), which
implemented the same question as Tversky and Kahneman. The experiment
was the same in most regards, and they expanded their sample to 6,344
participants recruited from 36 different sources including university
subject pools, Amazon Mechanical Turk, Project Implicit, and other
sources.

The dataset is stored in a \texttt{.csv} file. As earlier, use the
\texttt{readr} package, which is included inside the \texttt{tidyverse}.
If you haven't installed the tidyverse before, you can do so by running
\texttt{install.packages("tidyverse")} once.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load tidyverse (includes readr for CSVs)}
\FunctionTok{library}\NormalTok{(tidyverse)}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load the data into R}
\NormalTok{framing }\OtherTok{\textless{}{-}} \FunctionTok{read\_csv}\NormalTok{(}\StringTok{"https://stat158.berkeley.edu/spring{-}2026/data/framing/framing.csv"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load your collected data into R as well}

\CommentTok{\#my\_data \textless{}{-} }
\end{Highlighting}
\end{Shaded}

\subsection{Statistical tests}\label{statistical-tests}

\subsection{Non-parametric Methods: Randomization
Tests}\label{non-parametric-methods-randomization-tests}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{5}
\tightlist
\item
  We shall first implement this for the larger ManyLabs dataset
  provided. In order to conduct a randomization test, we will be working
  under the Potential Outcomes framework.
\end{enumerate}

\textbf{Q:} What assumptions are required by the permutation test? What
is random in this setup?

\textbf{A:}\\
\strut \\
\strut \\
\strut \\

Modify the dataset to reflect two columns, \texttt{Y\_1} and
\texttt{Y\_0}, representing the potential outcomes. Note that this will
have missing values.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# convert data to potential outcomes}
\end{Highlighting}
\end{Shaded}

As in class, we will be testing the null hypothesis that (some people
call it the \textbf{Fisher Sharp Null Hypothesis})
\[Y_i(1) - Y_i(0) = 0 \text{ for any }i.\]

Under the \textbf{Fisher Sharp Null Hypothesis}, impute the missing
potential outcomes.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Fill in the missing values }
\end{Highlighting}
\end{Shaded}

We now implement the randomization test from class. The following
function evaluates the distribution of the test statistic under the null
by repeatedly randomizing who receives treatment.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Randomization distribution}

\NormalTok{rand\_stats }\OtherTok{\textless{}{-}} \ControlFlowTok{function}\NormalTok{(schedule, d\_i, }\AttributeTok{stat =} \StringTok{"diff in means"}\NormalTok{, }\AttributeTok{reps =} \DecValTok{1000}\NormalTok{) \{}

  \CommentTok{\# replicate the schedule reps times and stack them on top of one another}
\NormalTok{  randomized\_exp\_df }\OtherTok{\textless{}{-}} \FunctionTok{map\_dfr}\NormalTok{(}\DecValTok{1}\SpecialCharTok{:}\NormalTok{reps, }\SpecialCharTok{\textasciitilde{}}\NormalTok{ schedule, }\AttributeTok{.id =} \StringTok{"experiment"}\NormalTok{) }\SpecialCharTok{|\textgreater{}}
    \FunctionTok{mutate}\NormalTok{(}\AttributeTok{experiment =} \FunctionTok{factor}\NormalTok{(experiment, }\AttributeTok{levels =} \FunctionTok{as.character}\NormalTok{(}\DecValTok{1}\SpecialCharTok{:}\NormalTok{reps), }\AttributeTok{ordered =} \ConstantTok{TRUE}\NormalTok{),}
           \AttributeTok{d\_i =} \FunctionTok{c}\NormalTok{(}\FunctionTok{replicate}\NormalTok{(}\AttributeTok{n =}\NormalTok{ reps, }\FunctionTok{sample}\NormalTok{(d\_i))),  }\CommentTok{\# create random assignments}
           \AttributeTok{y\_i =}\NormalTok{ Y\_1 }\SpecialCharTok{*}\NormalTok{ d\_i }\SpecialCharTok{+}\NormalTok{  Y\_0 }\SpecialCharTok{*}\NormalTok{ (}\DecValTok{1} \SpecialCharTok{{-}}\NormalTok{ d\_i)) }\SpecialCharTok{|\textgreater{}}    \CommentTok{\# find observed responses}
    \FunctionTok{arrange}\NormalTok{(experiment)}

  \CommentTok{\# calculate test statistic for every random assignment}
  \ControlFlowTok{if}\NormalTok{ (stat }\SpecialCharTok{==} \StringTok{"diff in means"}\NormalTok{) \{}
\NormalTok{    stats }\OtherTok{\textless{}{-}}\NormalTok{ randomized\_exp\_df }\SpecialCharTok{|\textgreater{}}
      \FunctionTok{group\_by}\NormalTok{(experiment, d\_i) }\SpecialCharTok{|\textgreater{}}
      \FunctionTok{summarize}\NormalTok{(}\AttributeTok{ybar =} \FunctionTok{mean}\NormalTok{(y\_i),     }\CommentTok{\# average within each group within each experiment}
                \AttributeTok{.groups =} \StringTok{"drop\_last"}\NormalTok{) }\SpecialCharTok{|\textgreater{}} 
      \FunctionTok{summarize}\NormalTok{(}\AttributeTok{ATE\_hat =} \FunctionTok{diff}\NormalTok{(ybar),}
                \AttributeTok{.groups =} \StringTok{"drop"}\NormalTok{) }\SpecialCharTok{|\textgreater{}}   \CommentTok{\# take the difference between the two groups mean }
      \FunctionTok{pull}\NormalTok{()}
\NormalTok{  \} }\ControlFlowTok{else}\NormalTok{ \{}
    \FunctionTok{stop}\NormalTok{(}\StringTok{"Statistic not implemented"}\NormalTok{)}
\NormalTok{  \}}

  \FunctionTok{return}\NormalTok{(stats)}
\NormalTok{\}}
\end{Highlighting}
\end{Shaded}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{6}
\tightlist
\item
  Fill in your own version of \texttt{schedule} and \texttt{d\_i} based
  on the dataset we are working with. Proceed to then calculate a
  p-value. See the
  \href{https://stat158.berkeley.edu/spring-2026/05-testing/code.html}{Stat
  158 testing code page}.
\end{enumerate}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Randomization Test }

\CommentTok{\# Fill these in}
\CommentTok{\# my\_null\_schedule \textless{}{-} }
  
\CommentTok{\# my\_d\_i \textless{}{-}}
  
\CommentTok{\# my\_ATE\_obs \textless{}{-}}
  
\CommentTok{\# calculate the test statistic distribution  }
\NormalTok{stats }\OtherTok{\textless{}{-}} \FunctionTok{rand\_stats}\NormalTok{(}\AttributeTok{schedule =}\NormalTok{ my\_null\_sched, }\AttributeTok{d\_i =}\NormalTok{ my\_d\_i)}

\CommentTok{\# calculate p{-}value}
\FunctionTok{mean}\NormalTok{(}\FunctionTok{abs}\NormalTok{(stats) }\SpecialCharTok{\textgreater{}}\NormalTok{ my\_ATE\_obs)}
\end{Highlighting}
\end{Shaded}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{7}
\tightlist
\item
  Repeat these steps for your own collected dataset. Convert it to a
  filled-in potential outcomes style table, generate the randomization
  distribution and calculate the p-value.
\end{enumerate}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Repeat on your own data }
\end{Highlighting}
\end{Shaded}

\subsection{Model-based Inference: The
Z-Test}\label{model-based-inference-the-z-test}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{8}
\tightlist
\item
  Before we conduct our z-test, we should be explicit about our
  hypothesis and assumptions. Answer the following questions in brief,
  but complete sentences.
\end{enumerate}

\textbf{Q:} What is the null hypothesis of our z-test in the context of
our specific problem? Is it different from that of the permutation test?
\textbf{A:}\\
\strut \\
\strut \\
\strut \\

\textbf{Q:} What assumptions are required by the z-test?\\
\textbf{A:}\\
\strut \\
\strut \\
\strut \\

Note that though we are working with binary data, the proportion of 1's
can often be well approximated by a normal distribution, particularly
with larger sample sizes. Thus a z-test is still valid in such settings.

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{9}
\tightlist
\item
  Fortunately, \texttt{R} has a built-in function, called
  \texttt{prop.test}, that conducts a two-sample z-test on such binary
  data. Run the test on the Many Labs data in the chunk below and report
  the p-value. Be sure to add \texttt{correct\ =\ FALSE.} in the
  arguments.
\end{enumerate}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# conduct a two{-}sample z{-}test}
\CommentTok{\# ex: prop.test((p1, p2), (n1, n2), correct = FALSE)}
\end{Highlighting}
\end{Shaded}

Run this on your own collected data as well.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# conduct a two{-}sample z{-}test}
\end{Highlighting}
\end{Shaded}

\textbf{Q:} What can you conclude from the above two test results?\\
\textbf{A:}\\
\strut \\
\strut \\
\strut \\

\textbf{Q:} What do you think this tells you about the power of the
z-test and the normal approximation?\\
\textbf{A:}\\
\strut \\
\strut \\
\strut \\




\end{document}
