filter_metadata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "from itertools import islice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = pd.read_table('/nfs/turbo/dcmb-class/bioinf593/groups/group_05/output/trgt/repeatregion_10_parsedvcf.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chrom</th>\n",
       "      <th>pos</th>\n",
       "      <th>trid</th>\n",
       "      <th>end_pos</th>\n",
       "      <th>motif</th>\n",
       "      <th>len_motif</th>\n",
       "      <th>struc</th>\n",
       "      <th>MC</th>\n",
       "      <th>confidence_interval</th>\n",
       "      <th>flanking_reads</th>\n",
       "      <th>inrepeat_reads</th>\n",
       "      <th>spanning_reads</th>\n",
       "      <th>sample_name</th>\n",
       "      <th>len_repeat_region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>44836</td>\n",
       "      <td>chr1_44835_44867</td>\n",
       "      <td>44867</td>\n",
       "      <td>AAAT</td>\n",
       "      <td>4</td>\n",
       "      <td>(AAAT)n</td>\n",
       "      <td>8,9</td>\n",
       "      <td>8-8/9-9</td>\n",
       "      <td>(1, 5), (2, 4), (3, 2), (4, 1), (6, 4), (7, 1)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 24), (9, 15)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>167135</td>\n",
       "      <td>chr1_167134_167150</td>\n",
       "      <td>167150</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 2), (2, 1), (3, 3), (4, 4)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 28)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>370633</td>\n",
       "      <td>chr1_370632_370648</td>\n",
       "      <td>370648</td>\n",
       "      <td>TTTC</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTC)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 5), (2, 6), (3, 1), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 38)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>371009</td>\n",
       "      <td>chr1_371008_371033</td>\n",
       "      <td>371033</td>\n",
       "      <td>TTTTG</td>\n",
       "      <td>5</td>\n",
       "      <td>(TTTTG)n</td>\n",
       "      <td>5,5</td>\n",
       "      <td>4-4/5-5</td>\n",
       "      <td>(1, 3), (2, 4), (3, 2), (4, 3), (5, 2), (8, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 6), (5, 45)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>371275</td>\n",
       "      <td>chr1_371274_371292</td>\n",
       "      <td>371292</td>\n",
       "      <td>TG</td>\n",
       "      <td>2</td>\n",
       "      <td>(TG)n</td>\n",
       "      <td>9,8</td>\n",
       "      <td>9-9/10-10</td>\n",
       "      <td>(2, 1), (3, 1), (4, 3), (6, 3), (7, 3), (8, 2)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 3), (9, 36), (10, 21)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6519475</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155781112</td>\n",
       "      <td>chrX_155781111_155781127</td>\n",
       "      <td>155781127</td>\n",
       "      <td>AG</td>\n",
       "      <td>2</td>\n",
       "      <td>(AG)n</td>\n",
       "      <td>8,8</td>\n",
       "      <td>8-8/8-8</td>\n",
       "      <td>(1, 1), (2, 3), (3, 1), (5, 1), (6, 1), (7, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 25)</td>\n",
       "      <td>HG01071</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6519476</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155786790</td>\n",
       "      <td>chrX_155786789_155786809</td>\n",
       "      <td>155786809</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>5,5</td>\n",
       "      <td>5-5/5-5</td>\n",
       "      <td>(2, 2), (4, 2), (5, 2), (6, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(5, 22), (10, 1)</td>\n",
       "      <td>HG01071</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6519477</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155805374</td>\n",
       "      <td>chrX_155805373_155805397</td>\n",
       "      <td>155805397</td>\n",
       "      <td>AAC</td>\n",
       "      <td>3</td>\n",
       "      <td>(AAC)n</td>\n",
       "      <td>8,8</td>\n",
       "      <td>8-8/8-8</td>\n",
       "      <td>(1, 5), (2, 1), (3, 2), (4, 1), (5, 1), (7, 1)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(2, 1), (5, 1), (8, 39), (10, 1)</td>\n",
       "      <td>HG01071</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6519478</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155846315</td>\n",
       "      <td>chrX_155846314_155846329</td>\n",
       "      <td>155846329</td>\n",
       "      <td>ATTTT</td>\n",
       "      <td>5</td>\n",
       "      <td>(ATTTT)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 2), (2, 2), (3, 4), (4, 5)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 44)</td>\n",
       "      <td>HG01071</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6519479</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155890351</td>\n",
       "      <td>chrX_155890350_155890378</td>\n",
       "      <td>155890378</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>7,7</td>\n",
       "      <td>7-7/7-7</td>\n",
       "      <td>(1, 5), (2, 6), (3, 1), (4, 2), (6, 3), (7, 4)</td>\n",
       "      <td>()</td>\n",
       "      <td>(7, 25)</td>\n",
       "      <td>HG01071</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6519480 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        chrom        pos                      trid    end_pos  motif  \\\n",
       "0        chr1      44836          chr1_44835_44867      44867   AAAT   \n",
       "1        chr1     167135        chr1_167134_167150     167150   TTTA   \n",
       "2        chr1     370633        chr1_370632_370648     370648   TTTC   \n",
       "3        chr1     371009        chr1_371008_371033     371033  TTTTG   \n",
       "4        chr1     371275        chr1_371274_371292     371292     TG   \n",
       "...       ...        ...                       ...        ...    ...   \n",
       "6519475  chrX  155781112  chrX_155781111_155781127  155781127     AG   \n",
       "6519476  chrX  155786790  chrX_155786789_155786809  155786809   TTTA   \n",
       "6519477  chrX  155805374  chrX_155805373_155805397  155805397    AAC   \n",
       "6519478  chrX  155846315  chrX_155846314_155846329  155846329  ATTTT   \n",
       "6519479  chrX  155890351  chrX_155890350_155890378  155890378   TTTA   \n",
       "\n",
       "         len_motif     struc   MC confidence_interval  \\\n",
       "0                4   (AAAT)n  8,9             8-8/9-9   \n",
       "1                4   (TTTA)n  4,4             4-4/4-4   \n",
       "2                4   (TTTC)n  4,4             4-4/4-4   \n",
       "3                5  (TTTTG)n  5,5             4-4/5-5   \n",
       "4                2     (TG)n  9,8           9-9/10-10   \n",
       "...            ...       ...  ...                 ...   \n",
       "6519475          2     (AG)n  8,8             8-8/8-8   \n",
       "6519476          4   (TTTA)n  5,5             5-5/5-5   \n",
       "6519477          3    (AAC)n  8,8             8-8/8-8   \n",
       "6519478          5  (ATTTT)n  4,4             4-4/4-4   \n",
       "6519479          4   (TTTA)n  7,7             7-7/7-7   \n",
       "\n",
       "                                            flanking_reads inrepeat_reads  \\\n",
       "0        (1, 5), (2, 4), (3, 2), (4, 1), (6, 4), (7, 1)...             ()   \n",
       "1                           (1, 2), (2, 1), (3, 3), (4, 4)             ()   \n",
       "2                           (1, 5), (2, 6), (3, 1), (4, 3)             ()   \n",
       "3           (1, 3), (2, 4), (3, 2), (4, 3), (5, 2), (8, 1)             ()   \n",
       "4        (2, 1), (3, 1), (4, 3), (6, 3), (7, 3), (8, 2)...             ()   \n",
       "...                                                    ...            ...   \n",
       "6519475     (1, 1), (2, 3), (3, 1), (5, 1), (6, 1), (7, 1)             ()   \n",
       "6519476                     (2, 2), (4, 2), (5, 2), (6, 1)             ()   \n",
       "6519477  (1, 5), (2, 1), (3, 2), (4, 1), (5, 1), (7, 1)...             ()   \n",
       "6519478                     (1, 2), (2, 2), (3, 4), (4, 5)             ()   \n",
       "6519479     (1, 5), (2, 6), (3, 1), (4, 2), (6, 3), (7, 4)             ()   \n",
       "\n",
       "                           spanning_reads sample_name  len_repeat_region  \n",
       "0                        (8, 24), (9, 15)     HG01891                 32  \n",
       "1                                 (4, 28)     HG01891                 16  \n",
       "2                                 (4, 38)     HG01891                 16  \n",
       "3                         (4, 6), (5, 45)     HG01891                 25  \n",
       "4               (8, 3), (9, 36), (10, 21)     HG01891                 18  \n",
       "...                                   ...         ...                ...  \n",
       "6519475                           (8, 25)     HG01071                 16  \n",
       "6519476                  (5, 22), (10, 1)     HG01071                 20  \n",
       "6519477  (2, 1), (5, 1), (8, 39), (10, 1)     HG01071                 24  \n",
       "6519478                           (4, 44)     HG01071                 15  \n",
       "6519479                           (7, 25)     HG01071                 28  \n",
       "\n",
       "[6519480 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['HG01891', 'HG03492', 'HG02630', 'HG02257', 'NA19240', 'HG03098',\n",
       "       'HG01243', 'HG01358', 'NA20129', 'HG01175', 'HG01106', 'HG01258',\n",
       "       'HG01361', 'HG00621', 'HG03516', 'HG01952', 'HG00735', 'NA18906',\n",
       "       'HG00438', 'HG03453', 'HG00733', 'HG00741', 'HG02080', 'HG01928',\n",
       "       'HG03579', 'HG03540', 'HG02055', 'HG01109', 'HG03486', 'HG02572',\n",
       "       'HG02622', 'HG02723', 'HG01978', 'HG00673', 'HG02886', 'HG02145',\n",
       "       'HG02717', 'HG02818', 'HG02148', 'HG01071'], dtype=object)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata['sample_name'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "select_samples = ['HG01891', 'HG03492', 'HG02630', 'HG02257', 'NA19240', 'HG03098', 'HG01243']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get train/test split chromosomes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "chrom_counts = metadata[metadata['sample_name'].isin(select_samples)]['chrom'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1140909"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chrom_counts.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "912727.2000000001"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "0.8 * chrom_counts.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "chrom\n",
       "chr2     94934\n",
       "chr1     92631\n",
       "chr3     78554\n",
       "chr4     75866\n",
       "chr5     71197\n",
       "chr6     69517\n",
       "chr7     63875\n",
       "chrX     58030\n",
       "chr8     57827\n",
       "chr12    54978\n",
       "chr11    51450\n",
       "chr9     47236\n",
       "chr13    40999\n",
       "chr14    35658\n",
       "chr17    34741\n",
       "chr10    33943\n",
       "chr16    33894\n",
       "chr18    32340\n",
       "chr15    31535\n",
       "chr19    27811\n",
       "chr20    24906\n",
       "chr21    14973\n",
       "chr22    14014\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chrom_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum = 0 \n",
    "train_chroms = []\n",
    "test_chroms = []\n",
    "for count in chrom_counts.items():\n",
    "    if sum < 912727:\n",
    "        sum += count[1]\n",
    "        train_chroms.append(count[0])\n",
    "    else:\n",
    "        test_chroms.append(count[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['chr2',\n",
       " 'chr1',\n",
       " 'chr3',\n",
       " 'chr4',\n",
       " 'chr5',\n",
       " 'chr6',\n",
       " 'chr7',\n",
       " 'chrX',\n",
       " 'chr8',\n",
       " 'chr12',\n",
       " 'chr11',\n",
       " 'chr9',\n",
       " 'chr13',\n",
       " 'chr14',\n",
       " 'chr17']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_chroms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['chr10', 'chr16', 'chr18', 'chr15', 'chr19', 'chr20', 'chr21', 'chr22']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_chroms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter metadata table for only loci that made it through OHE processing and label them with train/test in new column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "ohe_dir = '/nfs/turbo/dcmb-class/bioinf593/groups/group_05/STRonvoli/data/ohe'\n",
    "trids = []\n",
    "filenames = defaultdict(lambda: defaultdict(list))\n",
    "for sample in select_samples:\n",
    "    ohe_path = os.path.join(ohe_dir, sample)\n",
    "    for file in os.listdir(ohe_path):\n",
    "        trid_split = re.split('[_.]', file)\n",
    "        trid = f'{trid_split[1]}_{trid_split[2]}_{trid_split[3]}'\n",
    "        trids.append(trid)\n",
    "        filenames[trid_split[0]][trid_split[1]].append(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1140880"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_metadata = metadata[(metadata['trid'].isin(trids)) & (metadata['sample_name'].isin(select_samples))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chrom</th>\n",
       "      <th>pos</th>\n",
       "      <th>trid</th>\n",
       "      <th>end_pos</th>\n",
       "      <th>motif</th>\n",
       "      <th>len_motif</th>\n",
       "      <th>struc</th>\n",
       "      <th>MC</th>\n",
       "      <th>confidence_interval</th>\n",
       "      <th>flanking_reads</th>\n",
       "      <th>inrepeat_reads</th>\n",
       "      <th>spanning_reads</th>\n",
       "      <th>sample_name</th>\n",
       "      <th>len_repeat_region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>44836</td>\n",
       "      <td>chr1_44835_44867</td>\n",
       "      <td>44867</td>\n",
       "      <td>AAAT</td>\n",
       "      <td>4</td>\n",
       "      <td>(AAAT)n</td>\n",
       "      <td>8,9</td>\n",
       "      <td>8-8/9-9</td>\n",
       "      <td>(1, 5), (2, 4), (3, 2), (4, 1), (6, 4), (7, 1)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 24), (9, 15)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>167135</td>\n",
       "      <td>chr1_167134_167150</td>\n",
       "      <td>167150</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 2), (2, 1), (3, 3), (4, 4)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 28)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>370633</td>\n",
       "      <td>chr1_370632_370648</td>\n",
       "      <td>370648</td>\n",
       "      <td>TTTC</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTC)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 5), (2, 6), (3, 1), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 38)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>371009</td>\n",
       "      <td>chr1_371008_371033</td>\n",
       "      <td>371033</td>\n",
       "      <td>TTTTG</td>\n",
       "      <td>5</td>\n",
       "      <td>(TTTTG)n</td>\n",
       "      <td>5,5</td>\n",
       "      <td>4-4/5-5</td>\n",
       "      <td>(1, 3), (2, 4), (3, 2), (4, 3), (5, 2), (8, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 6), (5, 45)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>371275</td>\n",
       "      <td>chr1_371274_371292</td>\n",
       "      <td>371292</td>\n",
       "      <td>TG</td>\n",
       "      <td>2</td>\n",
       "      <td>(TG)n</td>\n",
       "      <td>9,8</td>\n",
       "      <td>9-9/10-10</td>\n",
       "      <td>(2, 1), (3, 1), (4, 3), (6, 3), (7, 3), (8, 2)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 3), (9, 36), (10, 21)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140904</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155781112</td>\n",
       "      <td>chrX_155781111_155781127</td>\n",
       "      <td>155781127</td>\n",
       "      <td>AG</td>\n",
       "      <td>2</td>\n",
       "      <td>(AG)n</td>\n",
       "      <td>8,8</td>\n",
       "      <td>8-8/8-8</td>\n",
       "      <td>(4, 3), (6, 1), (8, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 32)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140905</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155786790</td>\n",
       "      <td>chrX_155786789_155786809</td>\n",
       "      <td>155786809</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>5,5</td>\n",
       "      <td>5-5/5-5</td>\n",
       "      <td>(3, 1), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(5, 23)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140906</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155805374</td>\n",
       "      <td>chrX_155805373_155805397</td>\n",
       "      <td>155805397</td>\n",
       "      <td>AAC</td>\n",
       "      <td>3</td>\n",
       "      <td>(AAC)n</td>\n",
       "      <td>8,7</td>\n",
       "      <td>7-7/8-8</td>\n",
       "      <td>(1, 3), (2, 1), (5, 1), (6, 3), (7, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(7, 16), (8, 17)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140907</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155846315</td>\n",
       "      <td>chrX_155846314_155846329</td>\n",
       "      <td>155846329</td>\n",
       "      <td>ATTTT</td>\n",
       "      <td>5</td>\n",
       "      <td>(ATTTT)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 1), (2, 4), (3, 2), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 22)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140908</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155890351</td>\n",
       "      <td>chrX_155890350_155890378</td>\n",
       "      <td>155890378</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>7,7</td>\n",
       "      <td>7-7/7-7</td>\n",
       "      <td>(1, 3), (2, 6), (3, 2), (4, 2), (5, 2), (6, 4)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(7, 27)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1140902 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        chrom        pos                      trid    end_pos  motif  \\\n",
       "0        chr1      44836          chr1_44835_44867      44867   AAAT   \n",
       "1        chr1     167135        chr1_167134_167150     167150   TTTA   \n",
       "2        chr1     370633        chr1_370632_370648     370648   TTTC   \n",
       "3        chr1     371009        chr1_371008_371033     371033  TTTTG   \n",
       "4        chr1     371275        chr1_371274_371292     371292     TG   \n",
       "...       ...        ...                       ...        ...    ...   \n",
       "1140904  chrX  155781112  chrX_155781111_155781127  155781127     AG   \n",
       "1140905  chrX  155786790  chrX_155786789_155786809  155786809   TTTA   \n",
       "1140906  chrX  155805374  chrX_155805373_155805397  155805397    AAC   \n",
       "1140907  chrX  155846315  chrX_155846314_155846329  155846329  ATTTT   \n",
       "1140908  chrX  155890351  chrX_155890350_155890378  155890378   TTTA   \n",
       "\n",
       "         len_motif     struc   MC confidence_interval  \\\n",
       "0                4   (AAAT)n  8,9             8-8/9-9   \n",
       "1                4   (TTTA)n  4,4             4-4/4-4   \n",
       "2                4   (TTTC)n  4,4             4-4/4-4   \n",
       "3                5  (TTTTG)n  5,5             4-4/5-5   \n",
       "4                2     (TG)n  9,8           9-9/10-10   \n",
       "...            ...       ...  ...                 ...   \n",
       "1140904          2     (AG)n  8,8             8-8/8-8   \n",
       "1140905          4   (TTTA)n  5,5             5-5/5-5   \n",
       "1140906          3    (AAC)n  8,7             7-7/8-8   \n",
       "1140907          5  (ATTTT)n  4,4             4-4/4-4   \n",
       "1140908          4   (TTTA)n  7,7             7-7/7-7   \n",
       "\n",
       "                                            flanking_reads inrepeat_reads  \\\n",
       "0        (1, 5), (2, 4), (3, 2), (4, 1), (6, 4), (7, 1)...             ()   \n",
       "1                           (1, 2), (2, 1), (3, 3), (4, 4)             ()   \n",
       "2                           (1, 5), (2, 6), (3, 1), (4, 3)             ()   \n",
       "3           (1, 3), (2, 4), (3, 2), (4, 3), (5, 2), (8, 1)             ()   \n",
       "4        (2, 1), (3, 1), (4, 3), (6, 3), (7, 3), (8, 2)...             ()   \n",
       "...                                                    ...            ...   \n",
       "1140904                             (4, 3), (6, 1), (8, 1)             ()   \n",
       "1140905                                     (3, 1), (4, 3)             ()   \n",
       "1140906             (1, 3), (2, 1), (5, 1), (6, 3), (7, 1)             ()   \n",
       "1140907                     (1, 1), (2, 4), (3, 2), (4, 3)             ()   \n",
       "1140908  (1, 3), (2, 6), (3, 2), (4, 2), (5, 2), (6, 4)...             ()   \n",
       "\n",
       "                    spanning_reads sample_name  len_repeat_region  \n",
       "0                 (8, 24), (9, 15)     HG01891                 32  \n",
       "1                          (4, 28)     HG01891                 16  \n",
       "2                          (4, 38)     HG01891                 16  \n",
       "3                  (4, 6), (5, 45)     HG01891                 25  \n",
       "4        (8, 3), (9, 36), (10, 21)     HG01891                 18  \n",
       "...                            ...         ...                ...  \n",
       "1140904                    (8, 32)     HG01243                 16  \n",
       "1140905                    (5, 23)     HG01243                 20  \n",
       "1140906           (7, 16), (8, 17)     HG01243                 24  \n",
       "1140907                    (4, 22)     HG01243                 15  \n",
       "1140908                    (7, 27)     HG01243                 28  \n",
       "\n",
       "[1140902 rows x 14 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "missing_files = []\n",
    "missing_indices = []\n",
    "for row in filtered_metadata[['trid','sample_name']].itertuples():\n",
    "    filename = f'{row[2]}_{row[1]}.npy'\n",
    "    chr = row[1].split('_')[0]\n",
    "    if filename not in filenames[row[2]][chr]:\n",
    "        missing_files.append(filename)\n",
    "        missing_indices.append(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['HG03492_chr6_31356477_31356492.npy',\n",
       " 'HG03492_chr7_8803792_8803837.npy',\n",
       " 'HG03492_chr9_113018278_113018298.npy',\n",
       " 'HG03492_chrX_89785621_89785661.npy',\n",
       " 'HG02257_chrX_52740629_52740647.npy',\n",
       " 'NA19240_chr5_71157098_71157126.npy',\n",
       " 'HG03098_chr9_42226505_42226515.npy',\n",
       " 'HG03098_chr9_42273338_42273362.npy',\n",
       " 'HG03098_chr9_42277908_42277928.npy',\n",
       " 'HG03098_chr9_42290064_42290084.npy',\n",
       " 'HG03098_chr9_42421130_42421140.npy',\n",
       " 'HG03098_chr9_42423455_42423473.npy',\n",
       " 'HG03098_chr9_42438921_42438941.npy',\n",
       " 'HG03098_chr9_42484937_42484953.npy',\n",
       " 'HG03098_chr9_62037601_62037625.npy',\n",
       " 'HG03098_chrX_18776209_18776245.npy',\n",
       " 'HG03098_chrX_90971849_90971865.npy',\n",
       " 'HG03098_chrX_112753270_112753288.npy',\n",
       " 'HG01243_chrX_52740629_52740647.npy',\n",
       " 'HG01243_chrX_68650481_68650514.npy',\n",
       " 'HG01243_chrX_112753270_112753288.npy',\n",
       " 'HG01243_chrX_118845429_118845485.npy']"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_filtered_metadata = filtered_metadata.drop(index=missing_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chrom</th>\n",
       "      <th>pos</th>\n",
       "      <th>trid</th>\n",
       "      <th>end_pos</th>\n",
       "      <th>motif</th>\n",
       "      <th>len_motif</th>\n",
       "      <th>struc</th>\n",
       "      <th>MC</th>\n",
       "      <th>confidence_interval</th>\n",
       "      <th>flanking_reads</th>\n",
       "      <th>inrepeat_reads</th>\n",
       "      <th>spanning_reads</th>\n",
       "      <th>sample_name</th>\n",
       "      <th>len_repeat_region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>44836</td>\n",
       "      <td>chr1_44835_44867</td>\n",
       "      <td>44867</td>\n",
       "      <td>AAAT</td>\n",
       "      <td>4</td>\n",
       "      <td>(AAAT)n</td>\n",
       "      <td>8,9</td>\n",
       "      <td>8-8/9-9</td>\n",
       "      <td>(1, 5), (2, 4), (3, 2), (4, 1), (6, 4), (7, 1)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 24), (9, 15)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>167135</td>\n",
       "      <td>chr1_167134_167150</td>\n",
       "      <td>167150</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 2), (2, 1), (3, 3), (4, 4)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 28)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>370633</td>\n",
       "      <td>chr1_370632_370648</td>\n",
       "      <td>370648</td>\n",
       "      <td>TTTC</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTC)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 5), (2, 6), (3, 1), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 38)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>371009</td>\n",
       "      <td>chr1_371008_371033</td>\n",
       "      <td>371033</td>\n",
       "      <td>TTTTG</td>\n",
       "      <td>5</td>\n",
       "      <td>(TTTTG)n</td>\n",
       "      <td>5,5</td>\n",
       "      <td>4-4/5-5</td>\n",
       "      <td>(1, 3), (2, 4), (3, 2), (4, 3), (5, 2), (8, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 6), (5, 45)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>371275</td>\n",
       "      <td>chr1_371274_371292</td>\n",
       "      <td>371292</td>\n",
       "      <td>TG</td>\n",
       "      <td>2</td>\n",
       "      <td>(TG)n</td>\n",
       "      <td>9,8</td>\n",
       "      <td>9-9/10-10</td>\n",
       "      <td>(2, 1), (3, 1), (4, 3), (6, 3), (7, 3), (8, 2)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 3), (9, 36), (10, 21)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140904</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155781112</td>\n",
       "      <td>chrX_155781111_155781127</td>\n",
       "      <td>155781127</td>\n",
       "      <td>AG</td>\n",
       "      <td>2</td>\n",
       "      <td>(AG)n</td>\n",
       "      <td>8,8</td>\n",
       "      <td>8-8/8-8</td>\n",
       "      <td>(4, 3), (6, 1), (8, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 32)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140905</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155786790</td>\n",
       "      <td>chrX_155786789_155786809</td>\n",
       "      <td>155786809</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>5,5</td>\n",
       "      <td>5-5/5-5</td>\n",
       "      <td>(3, 1), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(5, 23)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140906</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155805374</td>\n",
       "      <td>chrX_155805373_155805397</td>\n",
       "      <td>155805397</td>\n",
       "      <td>AAC</td>\n",
       "      <td>3</td>\n",
       "      <td>(AAC)n</td>\n",
       "      <td>8,7</td>\n",
       "      <td>7-7/8-8</td>\n",
       "      <td>(1, 3), (2, 1), (5, 1), (6, 3), (7, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(7, 16), (8, 17)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140907</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155846315</td>\n",
       "      <td>chrX_155846314_155846329</td>\n",
       "      <td>155846329</td>\n",
       "      <td>ATTTT</td>\n",
       "      <td>5</td>\n",
       "      <td>(ATTTT)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 1), (2, 4), (3, 2), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 22)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140908</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155890351</td>\n",
       "      <td>chrX_155890350_155890378</td>\n",
       "      <td>155890378</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>7,7</td>\n",
       "      <td>7-7/7-7</td>\n",
       "      <td>(1, 3), (2, 6), (3, 2), (4, 2), (5, 2), (6, 4)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(7, 27)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1140880 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        chrom        pos                      trid    end_pos  motif  \\\n",
       "0        chr1      44836          chr1_44835_44867      44867   AAAT   \n",
       "1        chr1     167135        chr1_167134_167150     167150   TTTA   \n",
       "2        chr1     370633        chr1_370632_370648     370648   TTTC   \n",
       "3        chr1     371009        chr1_371008_371033     371033  TTTTG   \n",
       "4        chr1     371275        chr1_371274_371292     371292     TG   \n",
       "...       ...        ...                       ...        ...    ...   \n",
       "1140904  chrX  155781112  chrX_155781111_155781127  155781127     AG   \n",
       "1140905  chrX  155786790  chrX_155786789_155786809  155786809   TTTA   \n",
       "1140906  chrX  155805374  chrX_155805373_155805397  155805397    AAC   \n",
       "1140907  chrX  155846315  chrX_155846314_155846329  155846329  ATTTT   \n",
       "1140908  chrX  155890351  chrX_155890350_155890378  155890378   TTTA   \n",
       "\n",
       "         len_motif     struc   MC confidence_interval  \\\n",
       "0                4   (AAAT)n  8,9             8-8/9-9   \n",
       "1                4   (TTTA)n  4,4             4-4/4-4   \n",
       "2                4   (TTTC)n  4,4             4-4/4-4   \n",
       "3                5  (TTTTG)n  5,5             4-4/5-5   \n",
       "4                2     (TG)n  9,8           9-9/10-10   \n",
       "...            ...       ...  ...                 ...   \n",
       "1140904          2     (AG)n  8,8             8-8/8-8   \n",
       "1140905          4   (TTTA)n  5,5             5-5/5-5   \n",
       "1140906          3    (AAC)n  8,7             7-7/8-8   \n",
       "1140907          5  (ATTTT)n  4,4             4-4/4-4   \n",
       "1140908          4   (TTTA)n  7,7             7-7/7-7   \n",
       "\n",
       "                                            flanking_reads inrepeat_reads  \\\n",
       "0        (1, 5), (2, 4), (3, 2), (4, 1), (6, 4), (7, 1)...             ()   \n",
       "1                           (1, 2), (2, 1), (3, 3), (4, 4)             ()   \n",
       "2                           (1, 5), (2, 6), (3, 1), (4, 3)             ()   \n",
       "3           (1, 3), (2, 4), (3, 2), (4, 3), (5, 2), (8, 1)             ()   \n",
       "4        (2, 1), (3, 1), (4, 3), (6, 3), (7, 3), (8, 2)...             ()   \n",
       "...                                                    ...            ...   \n",
       "1140904                             (4, 3), (6, 1), (8, 1)             ()   \n",
       "1140905                                     (3, 1), (4, 3)             ()   \n",
       "1140906             (1, 3), (2, 1), (5, 1), (6, 3), (7, 1)             ()   \n",
       "1140907                     (1, 1), (2, 4), (3, 2), (4, 3)             ()   \n",
       "1140908  (1, 3), (2, 6), (3, 2), (4, 2), (5, 2), (6, 4)...             ()   \n",
       "\n",
       "                    spanning_reads sample_name  len_repeat_region  \n",
       "0                 (8, 24), (9, 15)     HG01891                 32  \n",
       "1                          (4, 28)     HG01891                 16  \n",
       "2                          (4, 38)     HG01891                 16  \n",
       "3                  (4, 6), (5, 45)     HG01891                 25  \n",
       "4        (8, 3), (9, 36), (10, 21)     HG01891                 18  \n",
       "...                            ...         ...                ...  \n",
       "1140904                    (8, 32)     HG01243                 16  \n",
       "1140905                    (5, 23)     HG01243                 20  \n",
       "1140906           (7, 16), (8, 17)     HG01243                 24  \n",
       "1140907                    (4, 22)     HG01243                 15  \n",
       "1140908                    (7, 27)     HG01243                 28  \n",
       "\n",
       "[1140880 rows x 14 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_filtered_metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.select([final_filtered_metadata['chrom'].isin(train_chroms)], [0], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chrom</th>\n",
       "      <th>pos</th>\n",
       "      <th>trid</th>\n",
       "      <th>end_pos</th>\n",
       "      <th>motif</th>\n",
       "      <th>len_motif</th>\n",
       "      <th>struc</th>\n",
       "      <th>MC</th>\n",
       "      <th>confidence_interval</th>\n",
       "      <th>flanking_reads</th>\n",
       "      <th>inrepeat_reads</th>\n",
       "      <th>spanning_reads</th>\n",
       "      <th>sample_name</th>\n",
       "      <th>len_repeat_region</th>\n",
       "      <th>split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>44836</td>\n",
       "      <td>chr1_44835_44867</td>\n",
       "      <td>44867</td>\n",
       "      <td>AAAT</td>\n",
       "      <td>4</td>\n",
       "      <td>(AAAT)n</td>\n",
       "      <td>8,9</td>\n",
       "      <td>8-8/9-9</td>\n",
       "      <td>(1, 5), (2, 4), (3, 2), (4, 1), (6, 4), (7, 1)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 24), (9, 15)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>167135</td>\n",
       "      <td>chr1_167134_167150</td>\n",
       "      <td>167150</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 2), (2, 1), (3, 3), (4, 4)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 28)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>370633</td>\n",
       "      <td>chr1_370632_370648</td>\n",
       "      <td>370648</td>\n",
       "      <td>TTTC</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTC)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 5), (2, 6), (3, 1), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 38)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>371009</td>\n",
       "      <td>chr1_371008_371033</td>\n",
       "      <td>371033</td>\n",
       "      <td>TTTTG</td>\n",
       "      <td>5</td>\n",
       "      <td>(TTTTG)n</td>\n",
       "      <td>5,5</td>\n",
       "      <td>4-4/5-5</td>\n",
       "      <td>(1, 3), (2, 4), (3, 2), (4, 3), (5, 2), (8, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 6), (5, 45)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>371275</td>\n",
       "      <td>chr1_371274_371292</td>\n",
       "      <td>371292</td>\n",
       "      <td>TG</td>\n",
       "      <td>2</td>\n",
       "      <td>(TG)n</td>\n",
       "      <td>9,8</td>\n",
       "      <td>9-9/10-10</td>\n",
       "      <td>(2, 1), (3, 1), (4, 3), (6, 3), (7, 3), (8, 2)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 3), (9, 36), (10, 21)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140904</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155781112</td>\n",
       "      <td>chrX_155781111_155781127</td>\n",
       "      <td>155781127</td>\n",
       "      <td>AG</td>\n",
       "      <td>2</td>\n",
       "      <td>(AG)n</td>\n",
       "      <td>8,8</td>\n",
       "      <td>8-8/8-8</td>\n",
       "      <td>(4, 3), (6, 1), (8, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 32)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140905</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155786790</td>\n",
       "      <td>chrX_155786789_155786809</td>\n",
       "      <td>155786809</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>5,5</td>\n",
       "      <td>5-5/5-5</td>\n",
       "      <td>(3, 1), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(5, 23)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140906</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155805374</td>\n",
       "      <td>chrX_155805373_155805397</td>\n",
       "      <td>155805397</td>\n",
       "      <td>AAC</td>\n",
       "      <td>3</td>\n",
       "      <td>(AAC)n</td>\n",
       "      <td>8,7</td>\n",
       "      <td>7-7/8-8</td>\n",
       "      <td>(1, 3), (2, 1), (5, 1), (6, 3), (7, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(7, 16), (8, 17)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140907</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155846315</td>\n",
       "      <td>chrX_155846314_155846329</td>\n",
       "      <td>155846329</td>\n",
       "      <td>ATTTT</td>\n",
       "      <td>5</td>\n",
       "      <td>(ATTTT)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 1), (2, 4), (3, 2), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 22)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140908</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155890351</td>\n",
       "      <td>chrX_155890350_155890378</td>\n",
       "      <td>155890378</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>7,7</td>\n",
       "      <td>7-7/7-7</td>\n",
       "      <td>(1, 3), (2, 6), (3, 2), (4, 2), (5, 2), (6, 4)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(7, 27)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1140880 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        chrom        pos                      trid    end_pos  motif  \\\n",
       "0        chr1      44836          chr1_44835_44867      44867   AAAT   \n",
       "1        chr1     167135        chr1_167134_167150     167150   TTTA   \n",
       "2        chr1     370633        chr1_370632_370648     370648   TTTC   \n",
       "3        chr1     371009        chr1_371008_371033     371033  TTTTG   \n",
       "4        chr1     371275        chr1_371274_371292     371292     TG   \n",
       "...       ...        ...                       ...        ...    ...   \n",
       "1140904  chrX  155781112  chrX_155781111_155781127  155781127     AG   \n",
       "1140905  chrX  155786790  chrX_155786789_155786809  155786809   TTTA   \n",
       "1140906  chrX  155805374  chrX_155805373_155805397  155805397    AAC   \n",
       "1140907  chrX  155846315  chrX_155846314_155846329  155846329  ATTTT   \n",
       "1140908  chrX  155890351  chrX_155890350_155890378  155890378   TTTA   \n",
       "\n",
       "         len_motif     struc   MC confidence_interval  \\\n",
       "0                4   (AAAT)n  8,9             8-8/9-9   \n",
       "1                4   (TTTA)n  4,4             4-4/4-4   \n",
       "2                4   (TTTC)n  4,4             4-4/4-4   \n",
       "3                5  (TTTTG)n  5,5             4-4/5-5   \n",
       "4                2     (TG)n  9,8           9-9/10-10   \n",
       "...            ...       ...  ...                 ...   \n",
       "1140904          2     (AG)n  8,8             8-8/8-8   \n",
       "1140905          4   (TTTA)n  5,5             5-5/5-5   \n",
       "1140906          3    (AAC)n  8,7             7-7/8-8   \n",
       "1140907          5  (ATTTT)n  4,4             4-4/4-4   \n",
       "1140908          4   (TTTA)n  7,7             7-7/7-7   \n",
       "\n",
       "                                            flanking_reads inrepeat_reads  \\\n",
       "0        (1, 5), (2, 4), (3, 2), (4, 1), (6, 4), (7, 1)...             ()   \n",
       "1                           (1, 2), (2, 1), (3, 3), (4, 4)             ()   \n",
       "2                           (1, 5), (2, 6), (3, 1), (4, 3)             ()   \n",
       "3           (1, 3), (2, 4), (3, 2), (4, 3), (5, 2), (8, 1)             ()   \n",
       "4        (2, 1), (3, 1), (4, 3), (6, 3), (7, 3), (8, 2)...             ()   \n",
       "...                                                    ...            ...   \n",
       "1140904                             (4, 3), (6, 1), (8, 1)             ()   \n",
       "1140905                                     (3, 1), (4, 3)             ()   \n",
       "1140906             (1, 3), (2, 1), (5, 1), (6, 3), (7, 1)             ()   \n",
       "1140907                     (1, 1), (2, 4), (3, 2), (4, 3)             ()   \n",
       "1140908  (1, 3), (2, 6), (3, 2), (4, 2), (5, 2), (6, 4)...             ()   \n",
       "\n",
       "                    spanning_reads sample_name  len_repeat_region  split  \n",
       "0                 (8, 24), (9, 15)     HG01891                 32      0  \n",
       "1                          (4, 28)     HG01891                 16      0  \n",
       "2                          (4, 38)     HG01891                 16      0  \n",
       "3                  (4, 6), (5, 45)     HG01891                 25      0  \n",
       "4        (8, 3), (9, 36), (10, 21)     HG01891                 18      0  \n",
       "...                            ...         ...                ...    ...  \n",
       "1140904                    (8, 32)     HG01243                 16      0  \n",
       "1140905                    (5, 23)     HG01243                 20      0  \n",
       "1140906           (7, 16), (8, 17)     HG01243                 24      0  \n",
       "1140907                    (4, 22)     HG01243                 15      0  \n",
       "1140908                    (7, 27)     HG01243                 28      0  \n",
       "\n",
       "[1140880 rows x 15 columns]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_filtered_metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_filtered_metadata.to_csv('/nfs/turbo/dcmb-class/bioinf593/groups/group_05/STRonvoli/data/metadata.tsv', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_filtered_metadata = pd.read_table('/nfs/turbo/dcmb-class/bioinf593/groups/group_05/STRonvoli/data/metadata.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_filtered_metadata_no_periods = final_filtered_metadata[~final_filtered_metadata['MC'].str.contains('\\.')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chrom</th>\n",
       "      <th>pos</th>\n",
       "      <th>trid</th>\n",
       "      <th>end_pos</th>\n",
       "      <th>motif</th>\n",
       "      <th>len_motif</th>\n",
       "      <th>struc</th>\n",
       "      <th>MC</th>\n",
       "      <th>confidence_interval</th>\n",
       "      <th>flanking_reads</th>\n",
       "      <th>inrepeat_reads</th>\n",
       "      <th>spanning_reads</th>\n",
       "      <th>sample_name</th>\n",
       "      <th>len_repeat_region</th>\n",
       "      <th>split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>44836</td>\n",
       "      <td>chr1_44835_44867</td>\n",
       "      <td>44867</td>\n",
       "      <td>AAAT</td>\n",
       "      <td>4</td>\n",
       "      <td>(AAAT)n</td>\n",
       "      <td>8,9</td>\n",
       "      <td>8-8/9-9</td>\n",
       "      <td>(1, 5), (2, 4), (3, 2), (4, 1), (6, 4), (7, 1)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 24), (9, 15)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>167135</td>\n",
       "      <td>chr1_167134_167150</td>\n",
       "      <td>167150</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 2), (2, 1), (3, 3), (4, 4)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 28)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>370633</td>\n",
       "      <td>chr1_370632_370648</td>\n",
       "      <td>370648</td>\n",
       "      <td>TTTC</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTC)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 5), (2, 6), (3, 1), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 38)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>371009</td>\n",
       "      <td>chr1_371008_371033</td>\n",
       "      <td>371033</td>\n",
       "      <td>TTTTG</td>\n",
       "      <td>5</td>\n",
       "      <td>(TTTTG)n</td>\n",
       "      <td>5,5</td>\n",
       "      <td>4-4/5-5</td>\n",
       "      <td>(1, 3), (2, 4), (3, 2), (4, 3), (5, 2), (8, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 6), (5, 45)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>371275</td>\n",
       "      <td>chr1_371274_371292</td>\n",
       "      <td>371292</td>\n",
       "      <td>TG</td>\n",
       "      <td>2</td>\n",
       "      <td>(TG)n</td>\n",
       "      <td>9,8</td>\n",
       "      <td>9-9/10-10</td>\n",
       "      <td>(2, 1), (3, 1), (4, 3), (6, 3), (7, 3), (8, 2)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 3), (9, 36), (10, 21)</td>\n",
       "      <td>HG01891</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140875</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155781112</td>\n",
       "      <td>chrX_155781111_155781127</td>\n",
       "      <td>155781127</td>\n",
       "      <td>AG</td>\n",
       "      <td>2</td>\n",
       "      <td>(AG)n</td>\n",
       "      <td>8,8</td>\n",
       "      <td>8-8/8-8</td>\n",
       "      <td>(4, 3), (6, 1), (8, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(8, 32)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140876</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155786790</td>\n",
       "      <td>chrX_155786789_155786809</td>\n",
       "      <td>155786809</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>5,5</td>\n",
       "      <td>5-5/5-5</td>\n",
       "      <td>(3, 1), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(5, 23)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140877</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155805374</td>\n",
       "      <td>chrX_155805373_155805397</td>\n",
       "      <td>155805397</td>\n",
       "      <td>AAC</td>\n",
       "      <td>3</td>\n",
       "      <td>(AAC)n</td>\n",
       "      <td>8,7</td>\n",
       "      <td>7-7/8-8</td>\n",
       "      <td>(1, 3), (2, 1), (5, 1), (6, 3), (7, 1)</td>\n",
       "      <td>()</td>\n",
       "      <td>(7, 16), (8, 17)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140878</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155846315</td>\n",
       "      <td>chrX_155846314_155846329</td>\n",
       "      <td>155846329</td>\n",
       "      <td>ATTTT</td>\n",
       "      <td>5</td>\n",
       "      <td>(ATTTT)n</td>\n",
       "      <td>4,4</td>\n",
       "      <td>4-4/4-4</td>\n",
       "      <td>(1, 1), (2, 4), (3, 2), (4, 3)</td>\n",
       "      <td>()</td>\n",
       "      <td>(4, 22)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140879</th>\n",
       "      <td>chrX</td>\n",
       "      <td>155890351</td>\n",
       "      <td>chrX_155890350_155890378</td>\n",
       "      <td>155890378</td>\n",
       "      <td>TTTA</td>\n",
       "      <td>4</td>\n",
       "      <td>(TTTA)n</td>\n",
       "      <td>7,7</td>\n",
       "      <td>7-7/7-7</td>\n",
       "      <td>(1, 3), (2, 6), (3, 2), (4, 2), (5, 2), (6, 4)...</td>\n",
       "      <td>()</td>\n",
       "      <td>(7, 27)</td>\n",
       "      <td>HG01243</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1139625 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        chrom        pos                      trid    end_pos  motif  \\\n",
       "0        chr1      44836          chr1_44835_44867      44867   AAAT   \n",
       "1        chr1     167135        chr1_167134_167150     167150   TTTA   \n",
       "2        chr1     370633        chr1_370632_370648     370648   TTTC   \n",
       "3        chr1     371009        chr1_371008_371033     371033  TTTTG   \n",
       "4        chr1     371275        chr1_371274_371292     371292     TG   \n",
       "...       ...        ...                       ...        ...    ...   \n",
       "1140875  chrX  155781112  chrX_155781111_155781127  155781127     AG   \n",
       "1140876  chrX  155786790  chrX_155786789_155786809  155786809   TTTA   \n",
       "1140877  chrX  155805374  chrX_155805373_155805397  155805397    AAC   \n",
       "1140878  chrX  155846315  chrX_155846314_155846329  155846329  ATTTT   \n",
       "1140879  chrX  155890351  chrX_155890350_155890378  155890378   TTTA   \n",
       "\n",
       "         len_motif     struc   MC confidence_interval  \\\n",
       "0                4   (AAAT)n  8,9             8-8/9-9   \n",
       "1                4   (TTTA)n  4,4             4-4/4-4   \n",
       "2                4   (TTTC)n  4,4             4-4/4-4   \n",
       "3                5  (TTTTG)n  5,5             4-4/5-5   \n",
       "4                2     (TG)n  9,8           9-9/10-10   \n",
       "...            ...       ...  ...                 ...   \n",
       "1140875          2     (AG)n  8,8             8-8/8-8   \n",
       "1140876          4   (TTTA)n  5,5             5-5/5-5   \n",
       "1140877          3    (AAC)n  8,7             7-7/8-8   \n",
       "1140878          5  (ATTTT)n  4,4             4-4/4-4   \n",
       "1140879          4   (TTTA)n  7,7             7-7/7-7   \n",
       "\n",
       "                                            flanking_reads inrepeat_reads  \\\n",
       "0        (1, 5), (2, 4), (3, 2), (4, 1), (6, 4), (7, 1)...             ()   \n",
       "1                           (1, 2), (2, 1), (3, 3), (4, 4)             ()   \n",
       "2                           (1, 5), (2, 6), (3, 1), (4, 3)             ()   \n",
       "3           (1, 3), (2, 4), (3, 2), (4, 3), (5, 2), (8, 1)             ()   \n",
       "4        (2, 1), (3, 1), (4, 3), (6, 3), (7, 3), (8, 2)...             ()   \n",
       "...                                                    ...            ...   \n",
       "1140875                             (4, 3), (6, 1), (8, 1)             ()   \n",
       "1140876                                     (3, 1), (4, 3)             ()   \n",
       "1140877             (1, 3), (2, 1), (5, 1), (6, 3), (7, 1)             ()   \n",
       "1140878                     (1, 1), (2, 4), (3, 2), (4, 3)             ()   \n",
       "1140879  (1, 3), (2, 6), (3, 2), (4, 2), (5, 2), (6, 4)...             ()   \n",
       "\n",
       "                    spanning_reads sample_name  len_repeat_region  split  \n",
       "0                 (8, 24), (9, 15)     HG01891                 32      0  \n",
       "1                          (4, 28)     HG01891                 16      0  \n",
       "2                          (4, 38)     HG01891                 16      0  \n",
       "3                  (4, 6), (5, 45)     HG01891                 25      0  \n",
       "4        (8, 3), (9, 36), (10, 21)     HG01891                 18      0  \n",
       "...                            ...         ...                ...    ...  \n",
       "1140875                    (8, 32)     HG01243                 16      0  \n",
       "1140876                    (5, 23)     HG01243                 20      0  \n",
       "1140877           (7, 16), (8, 17)     HG01243                 24      0  \n",
       "1140878                    (4, 22)     HG01243                 15      0  \n",
       "1140879                    (7, 27)     HG01243                 28      0  \n",
       "\n",
       "[1139625 rows x 15 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_filtered_metadata_no_periods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# FINAL METADATA\n",
    "final_filtered_metadata_no_periods.to_csv('/nfs/turbo/dcmb-class/bioinf593/groups/group_05/STRonvoli/data/metadata.tsv', sep='\\t', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mybase",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}