automated-issue-ref.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f64573a2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed  10 milestones\n",
      "processed  20 milestones\n",
      "processed  30 milestones\n",
      "processed  40 milestones\n",
      "processed  50 milestones\n",
      "processed  60 milestones\n",
      "processed  70 milestones\n",
      "processed  80 milestones\n",
      "processed  90 milestones\n",
      "processed  100 milestones\n",
      "processed  110 milestones\n",
      "processed  120 milestones\n",
      "processed  130 milestones\n",
      "processed  140 milestones\n",
      "processed  150 milestones\n",
      "processed  160 milestones\n",
      "processed  170 milestones\n",
      "processed  180 milestones\n",
      "processed  190 milestones\n",
      "processed  200 milestones\n",
      "processed  210 milestones\n",
      "processed  220 milestones\n",
      "processed  230 milestones\n",
      "processed  240 milestones\n",
      "processed  250 milestones\n",
      "processed  260 milestones\n",
      "processed  270 milestones\n",
      "processed  280 milestones\n",
      "processed  290 milestones\n",
      "processed  300 milestones\n",
      "processed  310 milestones\n",
      "processed  320 milestones\n",
      "processed  330 milestones\n",
      "processed  340 milestones\n",
      "processed  350 milestones\n",
      "processed  360 milestones\n",
      "processed  370 milestones\n",
      "processed  380 milestones\n",
      "processed  390 milestones\n",
      "processed  400 milestones\n",
      "processed  410 milestones\n",
      "processed  420 milestones\n",
      "processed  430 milestones\n",
      "processed  440 milestones\n",
      "processed  450 milestones\n",
      "processed  460 milestones\n",
      "processed  470 milestones\n",
      "processed  480 milestones\n",
      "processed  490 milestones\n",
      "processed  500 milestones\n",
      "processed  510 milestones\n",
      "processed  520 milestones\n",
      "processed  530 milestones\n",
      "processed  540 milestones\n",
      "processed  550 milestones\n",
      "processed  560 milestones\n",
      "processed  570 milestones\n",
      "processed  580 milestones\n",
      "processed  590 milestones\n",
      "processed  600 milestones\n",
      "processed  610 milestones\n",
      "processed  620 milestones\n",
      "processed  630 milestones\n",
      "processed  640 milestones\n",
      "processed  650 milestones\n",
      "processed  660 milestones\n",
      "processed  670 milestones\n",
      "processed  680 milestones\n",
      "processed  690 milestones\n",
      "processed  700 milestones\n",
      "processed  710 milestones\n",
      "processed  720 milestones\n",
      "processed  730 milestones\n",
      "processed  740 milestones\n",
      "processed  750 milestones\n",
      "processed  760 milestones\n",
      "processed  770 milestones\n",
      "processed  780 milestones\n",
      "processed  790 milestones\n",
      "processed  800 milestones\n",
      "processed  810 milestones\n",
      "processed  820 milestones\n",
      "processed  830 milestones\n",
      "processed  840 milestones\n",
      "processed  850 milestones\n",
      "processed  860 milestones\n",
      "processed  870 milestones\n",
      "processed  880 milestones\n",
      "processed  890 milestones\n",
      "processed  900 milestones\n",
      "processed  910 milestones\n",
      "processed  920 milestones\n",
      "processed  930 milestones\n",
      "processed  940 milestones\n",
      "processed  950 milestones\n",
      "processed  960 milestones\n",
      "processed  970 milestones\n",
      "processed  980 milestones\n",
      "processed  990 milestones\n",
      "processed  1000 milestones\n",
      "processed  1010 milestones\n",
      "processed  1020 milestones\n",
      "processed  1030 milestones\n",
      "processed  1040 milestones\n",
      "processed  1050 milestones\n",
      "processed  1060 milestones\n",
      "processed  1070 milestones\n",
      "processed  1080 milestones\n",
      "processed  1090 milestones\n",
      "processed  1100 milestones\n",
      "processed  1110 milestones\n",
      "processed  1120 milestones\n",
      "processed  1130 milestones\n",
      "processed  1140 milestones\n",
      "processed  1150 milestones\n",
      "processed  1160 milestones\n",
      "processed  1170 milestones\n",
      "processed  1180 milestones\n",
      "processed  1190 milestones\n",
      "processed  1200 milestones\n",
      "processed  1210 milestones\n",
      "processed  1220 milestones\n",
      "processed  1230 milestones\n",
      "processed  1240 milestones\n",
      "processed  1250 milestones\n",
      "processed  1260 milestones\n",
      "processed  1270 milestones\n",
      "processed  1280 milestones\n",
      "processed  1290 milestones\n",
      "processed  1300 milestones\n",
      "processed  1310 milestones\n",
      "processed  1320 milestones\n",
      "processed  1330 milestones\n",
      "processed  1340 milestones\n",
      "processed  1350 milestones\n",
      "processed  1360 milestones\n",
      "processed  1370 milestones\n",
      "processed  1380 milestones\n",
      "processed  1390 milestones\n",
      "processed  1400 milestones\n",
      "processed  1410 milestones\n",
      "processed  1420 milestones\n",
      "processed  1430 milestones\n",
      "processed  1440 milestones\n",
      "processed  1450 milestones\n",
      "processed  1460 milestones\n",
      "processed  1470 milestones\n",
      "processed  1480 milestones\n",
      "processed  1490 milestones\n",
      "processed  1500 milestones\n",
      "processed  1510 milestones\n",
      "processed  1520 milestones\n",
      "processed  1530 milestones\n",
      "processed  1540 milestones\n",
      "processed  1550 milestones\n",
      "processed  1560 milestones\n",
      "processed  1570 milestones\n",
      "processed  1580 milestones\n",
      "processed  1590 milestones\n",
      "processed  1600 milestones\n",
      "processed  1610 milestones\n",
      "processed  1620 milestones\n",
      "processed  1630 milestones\n",
      "processed  1640 milestones\n",
      "processed  1650 milestones\n",
      "processed  1660 milestones\n",
      "processed  1670 milestones\n",
      "processed  1680 milestones\n",
      "processed  1690 milestones\n",
      "processed  1700 milestones\n",
      "processed  1710 milestones\n",
      "processed  1720 milestones\n",
      "processed  1730 milestones\n",
      "processed  1740 milestones\n",
      "processed  1750 milestones\n",
      "processed  1760 milestones\n",
      "processed  1770 milestones\n",
      "processed  1780 milestones\n",
      "processed  1790 milestones\n",
      "processed  1800 milestones\n",
      "processed  1810 milestones\n",
      "processed  1820 milestones\n",
      "processed  1830 milestones\n",
      "processed  1840 milestones\n",
      "processed  1850 milestones\n"
     ]
    }
   ],
   "source": [
    "# automated ref adding for lineage-note.txt, written by Xu Zou\n",
    "\n",
    "\n",
    "import bs4\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib import request\n",
    "import os\n",
    "import json\n",
    "# Use AngieHinrich's better query code for milestones\n",
    "\n",
    "# Build milestone_list from API query that lists all milestones in a repo.\n",
    "milestone_list = []\n",
    "milestone_already=[]\n",
    "milestone_title_dict={}\n",
    "listd=os.listdir()\n",
    "\n",
    "reassign_milestone=True\n",
    "if not(\"autoref\" in listd):\n",
    "    os.mkdir(\"autoref\")\n",
    "lista=os.listdir(\"autoref\")\n",
    "\n",
    "if not(reassign_milestone):\n",
    "    if (\"milestone.txt\" in lista):\n",
    "        w=open(\"autoref/milestone.txt\",'r')\n",
    "        milestones=w.readlines()\n",
    "        for line in milestones:\n",
    "            linsp=line.split()\n",
    "            if len(linsp)>2:\n",
    "                issue=linsp[0]\n",
    "                name=linsp[1]\n",
    "                milestone_list.append([linsp[0],linsp[1],linsp[2]])\n",
    "                if linsp[1]!='withdrawn':\n",
    "                    milestone_title_dict[linsp[1]]=[linsp[2],linsp[0]]\n",
    "                milestone_already.append(issue)\n",
    "        w.close()\n",
    "\n",
    "# refresh token\n",
    "\n",
    "w=open(\"autoref/milestone.txt\",'w')\n",
    "for item in milestone_list:\n",
    "    print(item[0],item[1],item[2],file=w)\n",
    "item_count=0\n",
    "for repo in ['cov-lineages/pango-designation', 'sars-cov-2-variants/lineage-proposals']:\n",
    "    if repo == 'sars-cov-2-variants/lineage-proposals':\n",
    "        prefix = 'sars#'\n",
    "    else:\n",
    "        prefix = ''\n",
    "    # Request pages starting with 1, until we get an empty result list.\n",
    "    page = 1\n",
    "    while 1:\n",
    "        url = f'https://api.github.com/repos/{repo}/milestones?per_page=100&page={page}'\n",
    "        res_text = request.urlopen(url).read().decode('utf-8')\n",
    "        res_list = json.loads(res_text)\n",
    "        \n",
    "        if len(res_list) == 0:\n",
    "            break\n",
    "        # Each element of this page's result list includes the milestone number and title.  Parse out lineage names from title.\n",
    "        \n",
    "        for milestone_info in res_list:\n",
    "            #print(milestone_info)\n",
    "            \n",
    "            milestone = milestone_info['number']\n",
    "            milestone=prefix+str(milestone)\n",
    "            num_issues=milestone_info['open_issues']+milestone_info['closed_issues']\n",
    "            if (milestone not in milestone_already) and (num_issues>0):\n",
    "                \n",
    "                try:\n",
    "                    title = milestone_info['title']\n",
    "                    title_words = title.replace('. ','.').replace(',', ' ').split()\n",
    "                    title = title.replace(' ', '+')\n",
    "                    \n",
    "                    for item in title_words:\n",
    "                        milestone_list.append([milestone, item, title])\n",
    "                        if item!='withdrawn':\n",
    "                            # only add milestones to those without already assigned\n",
    "                            if not(item in milestone_title_dict):\n",
    "                                milestone_title_dict[item]=[title,milestone]\n",
    "                            # update sars-cov-2 to main branch\n",
    "                            if (\"sars\" in milestone_title_dict[item][1] and not('sars' in milestone)):\n",
    "                                milestone_title_dict[item]=[title,milestone]\n",
    "                        item_count+=1\n",
    "                        print(milestone, item,title,file=w)\n",
    "                        if item_count%10==0:\n",
    "                            print(\"processed \",item_count,\"milestones\")\n",
    "                except Exception as e:\n",
    "                    print(milestone, e)\n",
    "        page += 1\n",
    "w.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "88a367db",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"NL.4.1+NL.4.1.1\"\n",
      "sars#2342\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "                \n",
    "def read_milestone_issues(name):\n",
    "    # or sars-cov-2 proj if it starts to add milestones\n",
    "    \n",
    "    # first, deal with multiple names\n",
    "    #print(name)\n",
    "    real_milestone=milestone_title_dict[name]\n",
    "    di='cov-lineages/pango-designation'\n",
    "    if 'sars' in real_milestone[1]:\n",
    "        di='sars-cov-2-variants/lineage-proposals'\n",
    "    actual_milestone='\"'+real_milestone[0]+'\"'\n",
    "    url=\"https://github.com/\"+di+\"/issues?q=milestone:\"+actual_milestone\n",
    "    print(url)\n",
    "    res=request.urlopen(url)\n",
    "    rp=res.read()\n",
    "    rp=rp.decode('utf-8')\n",
    "    html_soup=BeautifulSoup(rp,\"html\")\n",
    "    #print(html_soup)\n",
    "    #html=html_soup.text\n",
    "    target_attrs = {\n",
    "        'data-target': 'react-app.embeddedData',\n",
    "        'type': 'application/json'\n",
    "    }\n",
    "\n",
    "    # Find the script tag using the specified attributes\n",
    "    target_script_tag = html_soup.find('script', attrs=target_attrs)\n",
    "\n",
    "    #print(\"preloadedQueries:\",target_script_tag.string)\n",
    "    data=json.loads(target_script_tag.string)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    try:\n",
    "        payload=data['payload']['preloadedQueries'][0]['result']['data']['repository']['search']['edges'][0]['node']\n",
    "        \n",
    "        outid=str(payload['number'])\n",
    "        if 'sars' in real_milestone[1]:\n",
    "            outid='sars#'+outid\n",
    "        return outid\n",
    "    except Exception as e:\n",
    "        print(name,e)\n",
    "        return '0'\n",
    "\n",
    "w=read_milestone_issues(\"NL.4.1.1\")\n",
    "print(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f5b15585",
   "metadata": {},
   "outputs": [],
   "source": [
    "def examine_line(line):\n",
    "    # deal with issues/ situation\n",
    "    if 'issues/' in line:\n",
    "        linsp=line.split('issues/')\n",
    "        rel=linsp[0]+'issues/'\n",
    "        pos=0\n",
    "        while (linsp[1][pos]>='0' and linsp[1][pos]<='9' and pos<len(linsp[1])):\n",
    "            pos+=1\n",
    "        rel=rel+linsp[1][:pos]+'\\n'\n",
    "        return rel\n",
    "    if '#' in line:\n",
    "        return line\n",
    "    return ''\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "a56544a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"NL.4.1+NL.4.1.1\"\n",
      "['NL.4.1.1', 'sars-cov-2-variants/lineage-proposals#2342']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"JN.1.59.1+NK.1+NL.1+XEJ.2+KP.4.2.6+NL.5.2\"\n",
      "['NL.5.2', 'sars-cov-2-variants/lineage-proposals#1771']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"NL.9\"\n",
      "['NL.9', 'sars-cov-2-variants/lineage-proposals#2429']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"LP.8.1+LP.8.2+LP.8.1.1+NY.1+NY.3+NY.6+NY.7+NY.8+LP.8.1.2+LP.8.1.6+LP.8.1.9+NW.1+LP.8.1.3+LP.8.1.6+LP.8.2.1\"\n",
      "['NY.7', 'sars-cov-2-variants/lineage-proposals#2199']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"KP.2.6.4\"\n",
      "['KP.2.6.4', 'sars-cov-2-variants/lineage-proposals#2339']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"KP.2.25.2+KP.2.28\"\n",
      "['KP.2.28', 'sars-cov-2-variants/lineage-proposals#1768']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"MC.1.7\"\n",
      "['MC.1.7', 'sars-cov-2-variants/lineage-proposals#2195']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"MC.6.1\"\n",
      "['MC.6.1', 'sars-cov-2-variants/lineage-proposals#2147']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"MC.9.1\"\n",
      "['MC.9.1', 'sars-cov-2-variants/lineage-proposals#2064']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"MC.32.1\"\n",
      "['MC.32.1', 'sars-cov-2-variants/lineage-proposals#2340']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"MC.38\"\n",
      "['MC.38', 'sars-cov-2-variants/lineage-proposals#2414']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"MC.40.1\"\n",
      "['MC.40.1', 'sars-cov-2-variants/lineage-proposals#2394']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"MR.2\"\n",
      "['MR.2', 'sars-cov-2-variants/lineage-proposals#2089']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"KP.3.3.9\"\n",
      "['KP.3.3.9', 'sars-cov-2-variants/lineage-proposals#2252']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"NT.2\"\n",
      "['NT.2', 'sars-cov-2-variants/lineage-proposals#2426']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"LF.7.1.4\"\n",
      "['LF.7.1.4', 'sars-cov-2-variants/lineage-proposals#2211']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"LF.7.1.5\"\n",
      "['LF.7.1.5', 'sars-cov-2-variants/lineage-proposals#2329']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"LF.7.3.2\"\n",
      "['LF.7.3.2', 'sars-cov-2-variants/lineage-proposals#2371']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"LF.7.3.3\"\n",
      "['LF.7.3.3', 'sars-cov-2-variants/lineage-proposals#2386']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"LF.7.7.1\"\n",
      "['LF.7.7.1', 'sars-cov-2-variants/lineage-proposals#2215']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"LF.7.7.2\"\n",
      "['LF.7.7.2', 'sars-cov-2-variants/lineage-proposals#2349']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"LF.7.8\"\n",
      "['LF.7.8', 'sars-cov-2-variants/lineage-proposals#2392']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"LF.7.9\"\n",
      "['LF.7.9', 'sars-cov-2-variants/lineage-proposals#2473']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"ND.1.1.3\"\n",
      "['ND.1.1.3', 'sars-cov-2-variants/lineage-proposals#2326']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"LF.7.6.2+LS.2\"\n",
      "['LS.2', 'sars-cov-2-variants/lineage-proposals#2346']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"MD.1.1.2\"\n",
      "['MD.1.1.2', 'sars-cov-2-variants/lineage-proposals#2399']\n",
      "https://github.com/cov-lineages/pango-designation/issues?q=milestone:\"JH.1+JH.2\"\n",
      "['JH.2', '#2068']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"NB.1.6\"\n",
      "['NB.1.6', 'sars-cov-2-variants/lineage-proposals#2363']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"NB.1.7\"\n",
      "['NB.1.7', 'sars-cov-2-variants/lineage-proposals#2418']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"NB.1.8+NB.1.8.1\"\n",
      "['NB.1.8', 'sars-cov-2-variants/lineage-proposals#2431']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"NB.1.8+NB.1.8.1\"\n",
      "['NB.1.8.1', 'sars-cov-2-variants/lineage-proposals#2431']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"XDY.2+XDY.4\"\n",
      "['XDY.4', 'sars-cov-2-variants/lineage-proposals#1784']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"XEC.3+XEC.5+XEC.6+XEC.7+XEC.8.1+XEC.10+XEC.11+XEC.11.1+XEC.13+XEC.16+XEC.17+XEC.20.1+XEC.21+XEC.22+XEC.24.1+XEK.4+XEK.6\"\n",
      "['XEC.8.1', 'sars-cov-2-variants/lineage-proposals#2088']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"XEC.3+XEC.5+XEC.6+XEC.7+XEC.8.1+XEC.10+XEC.11+XEC.11.1+XEC.13+XEC.16+XEC.17+XEC.20.1+XEC.21+XEC.22+XEC.24.1+XEK.4+XEK.6\"\n",
      "['XEC.11.1', 'sars-cov-2-variants/lineage-proposals#2088']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"XEC.3+XEC.5+XEC.6+XEC.7+XEC.8.1+XEC.10+XEC.11+XEC.11.1+XEC.13+XEC.16+XEC.17+XEC.20.1+XEC.21+XEC.22+XEC.24.1+XEK.4+XEK.6\"\n",
      "['XEC.20.1', 'sars-cov-2-variants/lineage-proposals#2088']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"XEC.3+XEC.5+XEC.6+XEC.7+XEC.8.1+XEC.10+XEC.11+XEC.11.1+XEC.13+XEC.16+XEC.17+XEC.20.1+XEC.21+XEC.22+XEC.24.1+XEK.4+XEK.6\"\n",
      "['XEC.22', 'sars-cov-2-variants/lineage-proposals#2088']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"XEC.3+XEC.5+XEC.6+XEC.7+XEC.8.1+XEC.10+XEC.11+XEC.11.1+XEC.13+XEC.16+XEC.17+XEC.20.1+XEC.21+XEC.22+XEC.24.1+XEK.4+XEK.6\"\n",
      "['XEC.24.1', 'sars-cov-2-variants/lineage-proposals#2088']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"XEC.25.1\"\n",
      "['XEC.25.1', 'sars-cov-2-variants/lineage-proposals#2485']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"XEF.1\"\n",
      "['XEF.1', 'sars-cov-2-variants/lineage-proposals#2130']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"XEF.2\"\n",
      "['XEF.2', 'sars-cov-2-variants/lineage-proposals#2131']\n",
      "https://github.com/sars-cov-2-variants/lineage-proposals/issues?q=milestone:\"XEF.3\"\n",
      "['XEF.3', 'sars-cov-2-variants/lineage-proposals#2305']\n"
     ]
    }
   ],
   "source": [
    "# read lineage_notes.txt\n",
    "\n",
    "\n",
    "def milestone2issue(num):\n",
    "    return 0\n",
    "w=open(\"lineage_notes.txt\",'r',encoding='utf-8')\n",
    "lineage_notes=w.readlines()\n",
    "w.close()\n",
    "milestone_tobecheck=[]\n",
    "\n",
    "\n",
    "changed_items=0\n",
    "missing_milestones=[]\n",
    "for num in range(1,len(lineage_notes)):\n",
    "    line=lineage_notes[num]\n",
    "    linsp=line.split()\n",
    "    if (len(linsp)>=2):\n",
    "        title=linsp[0]\n",
    "        # for all lineages without an issue label but with a milestone\n",
    "        if not('#' in line or 'issues/' in line):\n",
    "            if title in milestone_title_dict:\n",
    "                # this time we need to deal with the milestone \n",
    "                #print(title)\n",
    "                issue_num=read_milestone_issues(title)\n",
    "                if issue_num!='0':\n",
    "                    if 'sars' in issue_num:\n",
    "                        issue_num=issue_num.replace('sars','sars-cov-2-variants/lineage-proposals')\n",
    "                    else:\n",
    "                        issue_num='#'+issue_num\n",
    "                        \n",
    "                    missing_milestones.append([title,issue_num])\n",
    "                    print(missing_milestones[-1])\n",
    "        if ('lineage-proposals#' in line):\n",
    "            if title in milestone_title_dict:\n",
    "                if not('sars' in milestone_title_dict[title][1]):\n",
    "                    issue_num=read_milestone_issues(title)\n",
    "                    if issue_num!='0':\n",
    "                        issue_num='#'+issue_num\n",
    "\n",
    "                        missing_milestones.append([title,issue_num])\n",
    "                        print(missing_milestones[-1])\n",
    "                    \n",
    "#rewrite lineage_notes.txt\n",
    "w=open(\"autoref/missing_refs.txt\",'w')\n",
    "for item in missing_milestones:\n",
    "    print(item[0],item[1],file=w)\n",
    "w.close()\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "1c9369e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add  1  references from cov-lineages/pango-designation.\n",
      "Upgrade  0  references from cov-lineages/pango-designation.\n",
      "Add  40  references from sars-cov-2-variants/lineage-proposals.\n"
     ]
    }
   ],
   "source": [
    "#final rewrite of lineage_notes.txt\n",
    "w=open(\"autoref/missing_refs.txt\",'r',encoding='utf-8')\n",
    "data=w.readlines()\n",
    "w.close()\n",
    "missing_refs={}\n",
    "for line in data:\n",
    "    linsp=line.split()\n",
    "    missing_refs[linsp[0]]=linsp[1].replace('\\n','')\n",
    "\n",
    "w=open(\"lineage_notes.txt\",'r',encoding='utf-8')\n",
    "data=w.readlines()\n",
    "w.close()\n",
    "normal_count=0\n",
    "sub_count=0\n",
    "upgrade_count=0\n",
    "for i in range(len(data)):\n",
    "    linsp=data[i].split()\n",
    "    \n",
    "    if (len(linsp)>0) and (linsp[0] in missing_refs):\n",
    "        if \"，from\" in data[i]:\n",
    "            data[i]=data[i].split(\", from\")[0]\n",
    "            upgrade_count+=1\n",
    "        data[i]=data[i].replace('\\n','')+\", from \"+missing_refs[linsp[0]]+'\\n'\n",
    "        if 'sars' in missing_refs[linsp[0]]:\n",
    "            sub_count+=1\n",
    "        else:\n",
    "            normal_count+=1\n",
    "\n",
    "print(\"Add \",normal_count-upgrade_count,\" references from cov-lineages/pango-designation.\" )\n",
    "print(\"Upgrade \",upgrade_count,\" references from cov-lineages/pango-designation.\" )\n",
    "print(\"Add \",sub_count,\" references from sars-cov-2-variants/lineage-proposals.\")\n",
    "\n",
    "w=open(\"lineage_notes.txt\",'w',newline='\\n',encoding='utf-8')\n",
    "w.writelines(data)\n",
    "w.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7756372f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace all \"From\" with \", from\" \n",
    "w=open(\"lineage_notes.txt\",'r')\n",
    "lines=w.readlines()\n",
    "for i in range(len(lines)):\n",
    "    lines[i]=lines[i].replace(\" From\",\", from\")\n",
    "w.close()\n",
    "w=open(\"lineage_notes.txt\",'w',newline='\\n')\n",
    "w.writelines(lines)\n",
    "w.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2e7c50c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:base] *",
   "language": "python",
   "name": "conda-base-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}