progress.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.5 64-bit ('dcc': conda)",
   "metadata": {
    "interpreter": {
     "hash": "d660374ac31277b9ea7ee26abd64205cafc644f5ebc9b3efcbdb7eb83107acd0"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from plot_results import fetch_run_data\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import wandb\n",
    "from functools import lru_cache\n",
    "import matplotlib.dates as mdates\n",
    "from datetime import datetime \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "@lru_cache(maxsize=None)\n",
    "def fetch_run_data(descriptor: str, metrics):\n",
    "    if isinstance(metrics, str):\n",
    "        metrics = [metrics]\n",
    "    else:\n",
    "        metrics = list(metrics)\n",
    "    api = wandb.Api()\n",
    "    runs = api.runs(\"cswinter/deep-codecraft-vs\", {\"config.descriptor\": descriptor})\n",
    "    \n",
    "    curves = []\n",
    "    for run in runs:\n",
    "        step = []\n",
    "        value = []\n",
    "        vals = run.history(keys=metrics, samples=100, pandas=False)\n",
    "        for entry in vals:\n",
    "            if metrics[0] in entry:\n",
    "                step.append(entry['_step'] * 1e-6)\n",
    "                meanvalue = np.array([entry[metric] for metric in metrics]).mean()\n",
    "                value.append(meanvalue)\n",
    "        curves.append((np.array(step), np.array(value)))\n",
    "    return curves, runs[0].summary[\"_timestamp\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "runs = [\n",
    "    \"154506-agents15-hpsetstandard-steps150e6\",\n",
    "    \"24e131-agents15-hpsetstandard-steps150e6\",\n",
    "    \"613056-agents15-hpsetstandard-steps150e6\",\n",
    "    \"87c1ab-hpsetstandard\",\n",
    "    \"8af81d-hpsetstandard-num_self_play30-num_vs_aggro_replicator1-num_vs_destroyer2-num_vs_replicator1\",\n",
    "    \"d33903-batches_per_update32-batches_per_update_schedule-hpsetstandard-lr0.001-lr_schedulecosine-steps150e6\",\n",
    "    \"49b7fa-entropy_bonus0.02-entropy_bonus_schedulelin 20e6:0.005,60e6:0.0-hpsetstandard\",\n",
    "    \"49b7fa-feat_dist_to_wallTrue-hpsetstandard\",\n",
    "    \"b9bab7-hpsetstandard-max_hardness150\",\n",
    "    \"46e0b2-hpsetstandard-spatial_attnFalse\",\n",
    "    \"2d9e29-hpsetstandard\",\n",
    "    \"30ed5b-hpsetstandard-max_hardness175\",\n",
    "    \"fc244e-hpsetstandard-spatial_attnTrue-spatial_attn_lr_multiplier10.0\",\n",
    "    \"0a5940-hpsetstandard-item_item_attn_layers1-item_item_spatial_attnTrue-item_item_spatial_attn_vfFalse-max_grad_norm200\",\n",
    "    \"0a5940-hpsetstandard-mothership_damage_scale4.0-mothership_damage_scale_schedulelin 50e6:1.0,150:0.0\",\n",
    "    \"83a3af-hpsetstandard-mothership_damage_scale4.0-mothership_damage_scale_schedulelin 50e6:0.0\",\n",
    "    \"667ac7-hpsetstandard\",\n",
    "    \"80a87d-entropy_bonus0.15-entropy_bonus_schedulelin 15e6:0.07,60e6:0.0-hpsetstandard\",\n",
    "    \"80a87d-entropy_bonus0.2-entropy_bonus_schedulelin 15e6:0.1,60e6:0.0-final_lr5e-05-hpsetstandard-lr0.0005-vf_coef1.0\",\n",
    "    \"c0b3b4-hpsetstandard-partial_score0\",\n",
    "    \"9fc3de-hpsetstandard\",\n",
    "    \"9fc3de-adr_hstepsize0.001-hpsetstandard-linear_hardnessFalse\",\n",
    "    \"ac84c0-gamma0.9997-hpsetstandard\",\n",
    "    \"a1210b-gamma_schedulecos 1.0-hpsetstandard\",\n",
    "    \"b9f907-adr_average_cost_target1-hpsetstandard\",\n",
    "    \"5fb181-hpsetstandard\",\n",
    "    \"5fb181-hpsetstandard-steps150e6\",\n",
    "    \"3c69a5-adr_average_cost_target0.5-adr_avg_cost_schedulelin 80e6:1.0-hpsetstandard\",\n",
    "    \"35b3a7-hpsetstandard-nearby_mapFalse-steps150e6\",\n",
    "    \"152ec3-hpsetstandard-nearby_mapFalse-steps125e6\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(19, 10))\n",
    "cmap = plt.get_cmap('viridis')\n",
    "\n",
    "t0 = 1593959023.8568478\n",
    "tn = 1607756232\n",
    "ts = []\n",
    "for ri, run in enumerate(runs):\n",
    "    #print(f\"Fetching {run}\")\n",
    "    curves, date = fetch_run_data(run, \"eval_mean_score\")\n",
    "    samples = []\n",
    "    values = []\n",
    "    for curve in curves:\n",
    "        ax.plot(curve[0], curve[1], color=cmap((date-t0)/(tn-t0)), marker='o')\n",
    "        for i, value in enumerate(curve[1]):\n",
    "            if len(values) <= i:\n",
    "                samples.append(curve[0][i])\n",
    "                values.append([value])\n",
    "            else:\n",
    "                values[i].append(value)\n",
    "    #values = np.array([np.array(vals).mean() for vals in values])\n",
    "    #ax.plot(samples, values, color=cmap((date-t0)/(tn-t0)), marker='o')\n",
    "    #ts.append(mdates.date2num(datetime.fromtimestamp(date)))\n",
    "\n",
    "from matplotlib.cm import ScalarMappable\n",
    "from matplotlib.colors import Normalize\n",
    "loc = mdates.AutoDateLocator()\n",
    "def dateformatter(x, pos=None):\n",
    "    return datetime.fromtimestamp(x*(tn-t0)+t0).strftime('%Y-%m-%d')\n",
    "fig.colorbar(ScalarMappable(cmap=cmap), ticks=loc, format=dateformatter)\n",
    "\n",
    "ax.set_yticks([-1.0, -0.5, 0, 0.5, 1])\n",
    "ax.set_xlim(0, 200)\n",
    "ax.grid()\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(20, 15))\n",
    "cmap = plt.get_cmap('viridis')\n",
    "\n",
    "t0 = 1593959023.8568478\n",
    "tn = 1607756232\n",
    "ts = []\n",
    "for ri, run in enumerate(runs):\n",
    "    #print(f\"Fetching {run}\")\n",
    "    curves, date = fetch_run_data(run, \"eval_mean_score\")\n",
    "    samples = []\n",
    "    values = []\n",
    "    for curve in curves:\n",
    "        for i, value in enumerate(curve[1]):\n",
    "            if len(values) <= i:\n",
    "                samples.append(curve[0][i])\n",
    "                values.append([value])\n",
    "            else:\n",
    "                values[i].append(value)\n",
    "    values = np.array([np.array(vals).mean() for vals in values])\n",
    "    ax.plot(samples, values)#, color=cmap((date-t0)/(tn-t0)))\n",
    "    #ts.append(mdates.date2num(datetime.fromtimestamp(date)))\n",
    "\n",
    "#from matplotlib.cm import ScalarMappable\n",
    "#from matplotlib.colors import Normalize\n",
    "#loc = mdates.AutoDateLocator()\n",
    "#fig.colorbar(ScalarMappable(norm=Normalize(t0, tn), cmap=cmap))#, ticks=loc, format=mdates.AutoDateFormatter(loc))\n",
    "\n",
    "ax.set(xlabel='million samples', ylim=(-1, 1))\n",
    "ax.set_yticks([-1.0, -0.5, 0, 0.5, 1])\n",
    "ax.set_xlim(0, 200e6)\n",
    "#ax.set_xticks([0, 25, 50, 75, 100, 125])\n",
    "ax.legend(loc='upper left')\n",
    "ax.grid()\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "api = wandb.Api()\n",
    "runs = api.runs(\"cswinter/deep-codecraft-vs\", {\"config.descriptor\": runs[0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fetch_run_data(runs[-1], 'eval_mean_score')[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(runs[0])\n",
    "{metric: values for metric, values in runs[0].summary.items() if metric.startswith('eval')}"
   ]
  }
 ]
}