between_the_lines/user_study_analysis/section5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import json\n",
    "import copy\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "import pickle\n",
    "import logging\n",
    "import math\n",
    "import matplotlib\n",
    "# plot calibration of Copilot confidences with XGBoost predictions\n",
    "from re import S\n",
    "from scipy.stats.stats import pearsonr  \n",
    "matplotlib.rcParams['pdf.fonttype'] = 42\n",
    "matplotlib.rcParams['ps.fonttype'] = 42\n",
    "plt.rc('text', usetex=True)\n",
    "plt.rc('font', family='sans-serif')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "with open('../data/data_labeled_study.pkl', 'rb') as f:\n",
    "    logs_by_user_session_labeled = pickle.load(f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_survey = pd.read_csv('../data/copilot_post_survey.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Participants Details (Section 5.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create bar chart for each question\n",
    "from tkinter import font\n",
    "\n",
    "for i in range(0, len(df_survey.columns)):\n",
    "    # create a new figure\n",
    "    plt.figure(figsize=(7, 7))\n",
    "    # get the column name\n",
    "    col_name = df_survey.columns[i]\n",
    "    # replace Copilot in the column name with CodeRec\n",
    "    #col_name = col_name.replace('Copilot', 'CodeRec')\n",
    "    #col_name = col_name.replace('copilot', 'CodeRec')\n",
    "    if col_name not in ['Which best describes your programming experience?', 'How proficient are you with Python?', 'How often do you use Copilot outside of today’s session?']:\n",
    "        continue\n",
    "    # get the column values\n",
    "    col_values = df_survey[col_name].value_counts()\n",
    "    # get the column values as a list\n",
    "    col_values_list = col_values.tolist()\n",
    "    # get the column index as a list\n",
    "    col_index_list = col_values.index.tolist()\n",
    "    # get the column index as a list\n",
    "    col_index_list = [str(i) for i in col_index_list]\n",
    "    # create horizontal bar chart with colors from bad to good but dont sort\n",
    "\n",
    "    #plt.barh(col_index_list, col_values_list, color=['#ff0000', '#ff4000', '#ff8000', '#ffbf00', '#ffff00', '#bfff00', '#80ff00', '#40ff00', '#00ff00'])    \n",
    "    plt.barh(col_index_list, col_values_list, height = 0.5)\n",
    "    # set the title\n",
    "    plt.title(col_name, fontsize = 20)\n",
    "    # set the x label\n",
    "    # set the y label\n",
    "    plt.xlabel('Number of Participants', fontsize = 20)\n",
    "    plt.yticks(fontsize=20)\n",
    "# x ticks make x-large\n",
    "    plt.xticks(fontsize=20)\n",
    "    # save the figure\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Average Statistics (Section 5.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_states = []\n",
    "all_actions = []\n",
    "all_times = []\n",
    "all_states_continuous = []\n",
    "all_states_list = []\n",
    "all_times_list = []\n",
    "\n",
    "all_times_continuous = []\n",
    "for i in range(len(logs_by_user_session_labeled)):\n",
    "    states = logs_by_user_session_labeled[i].LabeledState.to_numpy()\n",
    "    actions = logs_by_user_session_labeled[i].StateName.to_numpy()\n",
    "    times = logs_by_user_session_labeled[i].TimeSpentInState.to_numpy()\n",
    "    # clean up states\n",
    "    for j in range(len(states)):\n",
    "        if states[j] == 'Edditing Last Suggestion (X)':\n",
    "            states[j] = 'Editing Last Suggestion (X)'\n",
    "        if states[j] == 'IDK':\n",
    "            states[j] = 'IDK (I)'\n",
    "        if states[j] == 'Looking up documentation' or states[j] == 'Looking up documentation (N)':\n",
    "            states[j] = 'Looking up Documentation (N)'\n",
    "        if states[j] == 'h':\n",
    "            states[j] = 'Thinking About New Code To Write (F)'\n",
    "        if states[j] == 'Thinking about new code to write (F)':\n",
    "            states[j] = 'Thinking About New Code To Write (F)'\n",
    "\n",
    "    # get indices where state is IDK\n",
    "    idk_indices = [i for i in range(len(states)) if states[i] == 'IDK (I)']\n",
    "    states = [states[i] for i in range(len(states)) if i not in idk_indices]\n",
    "    actions = [actions[i] for i in range(len(actions)) if i not in idk_indices]\n",
    "    times = [times[i] for i in range(len(times)) if i not in idk_indices]\n",
    "\n",
    "    states_continuous = []\n",
    "    times_continuous = []\n",
    "    # merge states with same name\n",
    "    last_state = states[0]\n",
    "    times_cumalitive = 0\n",
    "    for i in range(len(states)):\n",
    "        if states[i] != last_state:\n",
    "            states_continuous.append(last_state)\n",
    "            last_state = states[i]\n",
    "            times_continuous.append(times_cumalitive)\n",
    "            times_cumalitive = times[i]\n",
    "        else:\n",
    "            times_cumalitive += times[i]\n",
    "    states_continuous.append(last_state)\n",
    "    times_continuous.append(times_cumalitive)\n",
    "    \n",
    "    all_states_list.extend(states)\n",
    "    all_times_list.extend(times)\n",
    "    all_states_continuous.extend(states_continuous)\n",
    "    all_times_continuous.extend(times_continuous)\n",
    "    all_states.append(states)\n",
    "    all_actions.append(actions)\n",
    "    all_times.append(times)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get average lenght of session\n",
    "session_lengths = [ np.sum(times)/60 for times in all_times]\n",
    "# avrage session length\n",
    "np.mean(session_lengths)\n",
    "# standard deviation of session length\n",
    "np.std(session_lengths)\n",
    "print(f'Average session length: {np.mean(session_lengths):.2f} +- {np.std(session_lengths):.2f}')\n",
    "print(f' max lenght of session: {np.max(session_lengths):.2f} min lenght of session: {np.min(session_lengths):.2f}')\n",
    "\n",
    "# number of events per session\n",
    "session_counts_events = [len(times) for times in all_times]\n",
    "print (f'Average number of events per session: {np.mean(session_counts_events):.2f} +- {np.std(session_counts_events):.2f}')\n",
    "print(f'total number of events: {np.sum(session_counts_events)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# unroll all actions into one list\n",
    "all_actions_unrolled = []\n",
    "for i in range(len(all_actions)):\n",
    "    all_actions_unrolled.extend(all_actions[i])\n",
    "# count how many shown events\n",
    "\n",
    "# histogram of actions frequency\n",
    "# counterr\n",
    "import collections\n",
    "counter = collections.Counter(all_actions_unrolled)\n",
    "# normalize counter\n",
    "counter_normalized = {k: v/sum(counter.values()) for k, v in counter.items()}\n",
    "print(counter)\n",
    "print(f' Accept Rate is {counter[\"Accepted\"]/ counter[\"Shown\"]}')\n",
    "all_actions_binary = [] \n",
    "# for each list in all_actions, keep only Accepted and Rejected and convert to binary\n",
    "for i in range(len(all_actions)):\n",
    "    all_actions_binary.append([1 if action == 'Accepted' else 0 for action in all_actions[i] if action == 'Accepted' or action == 'Rejected'])\n",
    "all_actions_binary = np.array(all_actions_binary)\n",
    "# get mean for each list\n",
    "all_actions_binary_mean = [np.mean(all_actions_binary[i]) for i in range(len(all_actions_binary))]\n",
    "# get standard error\n",
    "print(np.mean(all_actions_binary_mean))\n",
    "print(np.std(all_actions_binary_mean))\n",
    "print(np.median(all_actions_binary_mean))\n",
    "print(np.min(all_actions_binary_mean))\n",
    "print(np.max(all_actions_binary_mean))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use of Custom State (Section 5.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_states_clean = []\n",
    "all_states_continuous = []\n",
    "for i in range(len(logs_by_user_session_labeled)):\n",
    "    states = logs_by_user_session_labeled[i].LabeledState.to_numpy()\n",
    "    # clean up states\n",
    "    for j in range(len(states)):\n",
    "        if states[j] == 'Edditing Last Suggestion (X)':\n",
    "            states[j] = 'Editing Last Suggestion (X)'\n",
    "        if states[j] == 'IDK':\n",
    "            states[j] = 'IDK (I)'\n",
    "        if states[j] == 'Looking up documentation' or states[j] == 'Looking up documentation (N)':\n",
    "            states[j] = 'Looking up Documentation (N)'\n",
    "        if states[j] == 'h':\n",
    "            states[j] = 'Thinking About New Code To Write (F)'\n",
    "        if states[j] == 'Thinking about new code to write (F)':\n",
    "            states[j] = 'Thinking About New Code To Write (F)'\n",
    "\n",
    "\n",
    "    # get indices where state is IDK\n",
    "    all_states_clean.append(states)\n",
    "    all_states_continuous.extend(states)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(21):\n",
    "    counter = collections.Counter(all_states_clean[i])\n",
    "    print(counter['IDK (I)'])\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compare raw and cleaned states\n",
    "import collections\n",
    "counter = collections.Counter(all_states_continuous)\n",
    "print(counter)\n",
    "# get the total for counter\n",
    "total = sum(counter.values())\n",
    "print(total)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('hussein': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "e710af216e809473b1f2bcdee939ed3d8fc69fba0a18fe13b179176696cffea0"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}