planout-abtest/1.0-planout-abtest-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Notebook demonstrates end to end session with planout\n",
    "# experiment library including experimental specification,\n",
    "# simulation of decisions, reading of logs and analysis of data.\n",
    "\n",
    "# The simulated experiment is the proverbial 'button colour'\n",
    "# A/B test.\n",
    "\n",
    "# References:\n",
    "\n",
    "# https://facebook.github.io/planout/\n",
    "# Bakshy et al 2014\n",
    "\n",
    "# Extensions\n",
    "# - Log outcomes.\n",
    "# - Multiple parameters.\n",
    "# - Multiple units (userid, contentid).\n",
    "# - Exposure over time.\n",
    "\n",
    "import math\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import bernoulli, norm\n",
    "from planout.experiment import SimpleExperiment\n",
    "from planout.ops.random import UniformChoice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define experiment\n",
    "\n",
    "class ButtonExperiment(SimpleExperiment):\n",
    "  def assign(self, params, userid):\n",
    "    params.button_color = UniformChoice(choices=['A', 'B'], unit=userid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Simulate exposures and click throughs\n",
    "\n",
    "num_users = 1000\n",
    "\n",
    "# Simulated click through rates\n",
    "ctr = {'A': 0.5, 'B': 0.6}\n",
    "\n",
    "for userid in range(0, num_users):\n",
    "    exp = ButtonExperiment(userid=userid)\n",
    "\n",
    "    # Log exposure to VotingExperiment.log\n",
    "    call_to_action_color = exp.get('button_color')\n",
    "    \n",
    "    # Log an action\n",
    "    simulated_decision = bernoulli.rvs(ctr[call_to_action_color])\n",
    "    if simulated_decision:\n",
    "        exp.log_event('button_click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "event                  button_click  exposure\n",
      "params                                       \n",
      "{'button_color': 'A'}           267       519\n",
      "{'button_color': 'B'}           300       481\n"
     ]
    }
   ],
   "source": [
    "# Now perform summary statistics for each unique combination of parameters\n",
    "\n",
    "# params = A, B\n",
    "# num_exposure, num_click\n",
    "\n",
    "# Read planout logs (rows of json data)\n",
    "data = []\n",
    "with open(exp.name+'.log') as file:\n",
    "    for line in file:\n",
    "       data.append(json.loads(line))\n",
    "\n",
    "# Extract event and params data. Convert params to string.\n",
    "event = []\n",
    "params = []\n",
    "for line in data:\n",
    "    event.append(line['event'])\n",
    "    params.append(str(line['params']))\n",
    "                  \n",
    "# Insert data into a data frame\n",
    "df = pd.DataFrame.from_items([('params', params), ('event', event) ])\n",
    "\n",
    "# Calculate cross table to get summary statistics\n",
    "crosstab = pd.crosstab(df.params, df.event)\n",
    "print(crosstab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'button_color': 'A'} success rate =  0.514450867052\n",
      "{'button_color': 'B'} success rate =  0.623700623701\n",
      "Z statistic =  3.48370376386\n",
      "Cumulative area under normal distribution = 0.99975273674\n"
     ]
    }
   ],
   "source": [
    "# Discussion of Z statistic here:\n",
    "# https://making.lyst.com/2014/05/10/bayesian-ab-testing/\n",
    "\n",
    "def zstat(crosstab):\n",
    "    '''Calculate the Z statistic for an A/B test given\n",
    "    the summary statistics in a crosstab (exposures and events)'''\n",
    "    \n",
    "    crosstab_values = crosstab.copy().values\n",
    "    nrow = crosstab_values.shape[0]\n",
    "\n",
    "    success_rate = []\n",
    "    exposure = []\n",
    "    total_events = 0\n",
    "    total_exposures = 0\n",
    "    for row in range(0, nrow):\n",
    "        events = crosstab_values[row, 0]\n",
    "        exposures = crosstab_values[row, 1]\n",
    "        \n",
    "        rate = events/exposures\n",
    "        print(crosstab.axes[0][row], 'success rate = ', rate)\n",
    "        \n",
    "        success_rate.append(rate)\n",
    "        exposure.append(exposures)\n",
    "        \n",
    "        total_exposures += exposures\n",
    "        total_events += events\n",
    "    \n",
    "    combined_success_rate = total_events/total_exposures\n",
    "    \n",
    "    return (success_rate[1] - success_rate[0])/ \\\n",
    "            math.sqrt(combined_success_rate*(1.-combined_success_rate)* \\\n",
    "            (1.0/exposure[0] + 1.0/exposure[1]))\n",
    "\n",
    "    \n",
    "abtest_zstat = zstat(crosstab)\n",
    "print('Z statistic = ', abtest_zstat)\n",
    "print('Cumulative area under normal distribution =',norm.cdf(abtest_zstat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}